From ebd9a163bc6b8f664e870a0e2eeeec9582e93059 Mon Sep 17 00:00:00 2001 From: m Date: Wed, 29 Jul 2026 16:32:11 +1000 Subject: [PATCH 001/132] fix(desktop): auto-enable master notif switch when OS permission already granted Follow-up to e9475dd079. That commit fixed the case where the user clicked the "Enable OS notifications" button on a fresh install (permission NotRequested \u2192 Granted) but the master toggle stayed off. It missed the two closely-related cases the user was still hitting on v1.13.1: 1. Permission was already granted from a previous session or install (e.g. an earlier v1.13.0 build, or the user allowed it via System Settings \u2192 Notifications directly). In this state, permissionState == Granted, so the "Enable OS notifications" button never renders \u2014 the button label promised the whole handshake but the code path that flipped the master switch only ran under NotRequested. 2. On Windows/Linux `permissionState` defaults to `NotApplicable` from the moment the app starts. The master switch is off by default (first-launch UX choice) and nothing ever flips it, so the auto-dispatcher stayed muted forever unless the user found the switch manually. Fix: - Add `NotificationSettings.wasExplicitlyDisabled()` so the Settings screen can distinguish "master switch is off because it defaults off on first launch" (auto-enable is fine) from "master switch is off because the user turned it off" (leave alone). Backed by a new java.util.prefs key `explicitly_disabled` that flips true on `setEnabled(false)` and gets cleared on `setEnabled(true)`. - In `NotificationSettingsScreen`, a `LaunchedEffect(permissionState, enabled)` observes when the OS permission is Granted OR NotApplicable and the master switch is off. If the user has never explicitly turned it off, it auto-flips on \u2014 matching the "Enable OS notifications" contract for the paths the previous fix missed. - Also render a "Turn on desktop notifications" button in the Granted branch when the user has explicitly turned notifications off. That's the recovery path for users who deliberately opted out and later want to opt back in without hunting for the master switch two rows away. Behaviour on the fresh-install macOS path (permission NotRequested) is unchanged \u2014 that path still runs the `requestPermission()` flow inside the button's onClick, and the auto-enable happens via the same LaunchedEffect once permissionState flips to Granted. Tests (jvmTest, hermetic \u2014 UUID-scoped prefs nodes so tests never share state or pollute real user prefs): PreferencesNotificationSettingsExplicitDisableTest: - fresh install defaults to not-explicitly-disabled - turning off marks explicitly disabled - turning on clears the explicit-disable flag - flag persists across new instances on the same prefs node \ud83e\udd16 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../notifications/NotificationTypes.kt | 10 +++ .../PreferencesNotificationSettings.kt | 13 +++ ...NotificationSettingsExplicitDisableTest.kt | 86 +++++++++++++++++++ .../ui/settings/NotificationSettingsScreen.kt | 32 +++++++ 4 files changed, 141 insertions(+) create mode 100644 commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/moderation/notifications/PreferencesNotificationSettingsExplicitDisableTest.kt diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/moderation/notifications/NotificationTypes.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/moderation/notifications/NotificationTypes.kt index bc3d5177d6..02f06b4147 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/moderation/notifications/NotificationTypes.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/moderation/notifications/NotificationTypes.kt @@ -81,6 +81,16 @@ interface NotificationSettings { fun setEnabled(v: Boolean) + /** + * True iff the user has taken an explicit action to disable + * notifications (i.e. flipped the master switch OFF at some point). + * Used by the Settings screen to distinguish "master switch is off + * because it defaults to off on first launch" from "master switch + * is off because the user asked for it to be off". Only the former + * gets auto-enabled when the OS permission check passes. + */ + fun wasExplicitlyDisabled(): Boolean + fun setKindToggle( kind: NotifKind, v: Boolean, diff --git a/commons/src/jvmMain/kotlin/com/vitorpamplona/amethyst/commons/moderation/notifications/PreferencesNotificationSettings.kt b/commons/src/jvmMain/kotlin/com/vitorpamplona/amethyst/commons/moderation/notifications/PreferencesNotificationSettings.kt index 349fab12ea..225b85d649 100644 --- a/commons/src/jvmMain/kotlin/com/vitorpamplona/amethyst/commons/moderation/notifications/PreferencesNotificationSettings.kt +++ b/commons/src/jvmMain/kotlin/com/vitorpamplona/amethyst/commons/moderation/notifications/PreferencesNotificationSettings.kt @@ -53,8 +53,20 @@ class PreferencesNotificationSettings( override fun setEnabled(v: Boolean) { _enabled.value = v prefs.putBoolean(KEY_ENABLED, v) + // Track explicit user intent so the Settings screen can auto-enable + // on next visit for users who never touched the switch, while + // respecting users who deliberately turned it off. Only false + // → "explicit disable"; going from off to on clears the flag so + // subsequent auto-enable heuristics work normally. + if (v) { + prefs.remove(KEY_EXPLICITLY_DISABLED) + } else { + prefs.putBoolean(KEY_EXPLICITLY_DISABLED, true) + } } + override fun wasExplicitlyDisabled(): Boolean = prefs.getBoolean(KEY_EXPLICITLY_DISABLED, false) + override fun setKindToggle( kind: NotifKind, v: Boolean, @@ -92,6 +104,7 @@ class PreferencesNotificationSettings( companion object { const val NODE = "com/vitorpamplona/amethyst/notifications" private const val KEY_ENABLED = "enabled" + private const val KEY_EXPLICITLY_DISABLED = "explicitly_disabled" private const val KEY_DND_UNTIL = "dnd_until" private const val KEY_PREVIEW = "preview_in_toast" diff --git a/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/moderation/notifications/PreferencesNotificationSettingsExplicitDisableTest.kt b/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/moderation/notifications/PreferencesNotificationSettingsExplicitDisableTest.kt new file mode 100644 index 0000000000..4678d92b97 --- /dev/null +++ b/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/moderation/notifications/PreferencesNotificationSettingsExplicitDisableTest.kt @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.moderation.notifications + +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import java.util.UUID +import java.util.prefs.Preferences + +/** + * Pins the semantics of the new [NotificationSettings.wasExplicitlyDisabled] + * flag added to unblock the "Enable OS notifications button doesn't work on + * desktop" bug's second failure mode. + * + * The Settings screen auto-enables the master notifications switch when it + * detects that the OS permission is fine and the switch is off. That's the + * common path for users who click the "Enable OS notifications" button and + * expect it to fully take effect (button label promise). But it MUST NOT + * override users who deliberately turned notifications off. The + * [wasExplicitlyDisabled] flag is how we distinguish the two. + */ +class PreferencesNotificationSettingsExplicitDisableTest { + private fun freshNode(): Preferences { + // Use a UUID-scoped node so tests never share state and never + // pollute the real user prefs on the machine running CI/dev builds. + return Preferences.userRoot().node("amethyst-test-" + UUID.randomUUID()) + } + + @Test + fun `fresh install defaults to not-explicitly-disabled`() { + val settings = PreferencesNotificationSettings(freshNode()) + assertFalse( + "First launch must not look like a deliberate opt-out; otherwise auto-enable stays off forever", + settings.wasExplicitlyDisabled(), + ) + } + + @Test + fun `turning off marks explicitly disabled`() { + val prefs = freshNode() + val settings = PreferencesNotificationSettings(prefs) + settings.setEnabled(false) + assertTrue(settings.wasExplicitlyDisabled()) + } + + @Test + fun `turning on clears the explicit-disable flag`() { + val prefs = freshNode() + val settings = PreferencesNotificationSettings(prefs) + settings.setEnabled(false) + assertTrue(settings.wasExplicitlyDisabled()) + settings.setEnabled(true) + assertFalse( + "Toggling back on must clear the flag so subsequent OFF->auto-enable cycles work", + settings.wasExplicitlyDisabled(), + ) + } + + @Test + fun `flag persists across new instances on the same prefs node`() { + val prefs = freshNode() + PreferencesNotificationSettings(prefs).setEnabled(false) + // Second instance opens the same node \u2014 flag must survive process restart. + val reopened = PreferencesNotificationSettings(prefs) + assertTrue(reopened.wasExplicitlyDisabled()) + } +} diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/settings/NotificationSettingsScreen.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/settings/NotificationSettingsScreen.kt index ef1c546c1d..161b929b68 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/settings/NotificationSettingsScreen.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/settings/NotificationSettingsScreen.kt @@ -132,6 +132,27 @@ fun NotificationSettingsScreen(onBack: (() -> Unit)? = null) { } } + // Handle the still-broken case that the previous fix missed: + // the user granted OS permission in a prior session (either via + // the older "Enable OS notifications" button whose auto-enable + // guard I initially forgot, via System Settings directly, or on + // Windows/Linux where permissionState defaults to NotApplicable). + // When they come back to Settings, permissionState == Granted so + // the "Enable OS notifications" button doesn't render, the master + // switch is still OFF from first-launch defaults, and there is no + // affordance that both tells them what's wrong and fixes it in + // one click. Auto-enable once per screen entry when we detect + // "permission is fine, but master switch is off and the user + // has never explicitly disabled it". PreferencesNotificationSettings + // exposes [wasExplicitlyDisabled] so we don't overrule a deliberate + // opt-out. + androidx.compose.runtime.LaunchedEffect(permissionState, enabled) { + val allowed = permissionState == PermissionState.Granted || permissionState == PermissionState.NotApplicable + if (allowed && !enabled && !settings.wasExplicitlyDisabled()) { + settings.setEnabled(true) + } + } + PlatformStatusCard( host = host, nativeAvailable = nativeAvailable, @@ -219,6 +240,17 @@ fun NotificationSettingsScreen(onBack: (() -> Unit)? = null) { } } PermissionState.Granted, PermissionState.NotApplicable -> { + // Turn-on button: renders only when master switch is + // off *and* the user explicitly disabled it before. + // The LaunchedEffect above auto-enables the switch + // for the common "never touched it" path; this button + // is the recovery for the deliberate-opt-out path. + if (!enabled) { + OutlinedButton( + onClick = { settings.setEnabled(true) }, + enabled = true, + ) { Text("Turn on desktop notifications") } + } OutlinedButton( onClick = { if (sendingTest) return@OutlinedButton From 5896ae5eff843e3ebd9b9c31bc55d37845d58c5a Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Mon, 3 Aug 2026 19:19:56 -0400 Subject: [PATCH 002/132] fix(relay): stripe the subscription-state lock per relay to end an ANR convoy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A production ANR on a Pixel 8 (Amethyst 1.13.1, anr_2026-08-03-12-55-26-256) showed the app burning 596% CPU — 6 of 9 cores — with the main thread stuck in WaitingForGcToComplete. 37 of 52 runnable DefaultDispatcher workers sat at ONE program point inside PoolRequests.onIncomingMessage and 12 more at one point in syncState, all state=R, while the single thread actually holding the lock was itself parked in GC. Root cause: RequestSubscriptionState.withLock was a raw busy-wait (`while (lock.exchange(true)) { while (lock.load()) {} }`) with no yield or backoff, and being `inline` it disappeared into its callers' frames. The lock is per subId, but one subId spans every relay it runs on — 191 live sockets on that device — so dozens of relay-dispatch threads piled onto a single AtomicBoolean. Spinning is only correct when the holder cannot be descheduled; on Android it always can. The fix stripes the lock per (subId, relay) rather than making waiting cheaper. All 11 withLock bodies in PoolRequests are already scoped to exactly one relay, and every field of RequestSubscriptionState is keyed by relay, so the sharing was purely an artifact of mutableMapOf not being thread-safe. State moves into a ConcurrentMap; locks live in a fixed 32-entry stripe array that is never mutated, so lock identity stays stable — if locks lived inside the map values, a thread holding one while another dropped and re-created that entry would leave both inside the critical section excluding nothing. A suspending Mutex was measured and rejected: it needs 262 method overrides and 110 call sites to become suspend, and ran at 0.35-0.63x the current throughput. Measured (LockDesignComparisonBenchmark, 191 relays / 64 dispatcher threads): striped vs per-sub lock 1.5-2.8x throughput, bystander p50 halved On device (SM-T220, playBenchmark, same account, n=3 per design): DefaultDispatcher CPU -35% mean / -30% median vs the spin lock, with non-overlapping ranges; GC -18% Plus 10 min of driven UI stress (feed, profiles, chat, notifications, communities): no ANRs, no crashes, thread pools stable. Also here: - PlatformLock: new expect/actual parking lock (ReentrantLock on jvmAndroid, NSRecursiveLock on Apple, spin only on linuxX64 which is a CI target). quartz cannot use commons' equivalent KmpLock because commons depends on quartz. - LiveNegentropyIndex had the identical busy-wait with a full list SORT inside the critical section; switched to PlatformLock. - ConcurrentMap.remove (+ tests), with a caution that a removable value must not own a lock callers acquire. - SpinLockConvoyBenchmark: regression guard asserting contended waiters PARK rather than spin (fails-before / passes-after). Pure benchmarks are gated behind -PprodRelayBench=1, so CI cost is 0.3s rather than 51.5s. Analysis and measurements: quartz/plans/2026-08-03-poolrequests-lock-contention.md Co-Authored-By: Claude Opus 5 (1M context) --- ...2026-08-03-poolrequests-lock-contention.md | 221 ++++++++++++ .../utils/concurrent/PlatformLock.apple.kt | 36 ++ .../relay/client/pool/PoolRequests.kt | 55 +-- .../client/reqs/RequestSubscriptionState.kt | 228 ++++++++----- .../nip77Negentropy/LiveNegentropyIndex.kt | 23 +- .../quartz/utils/concurrent/ConcurrentMap.kt | 11 + .../quartz/utils/concurrent/PlatformLock.kt | 75 +++++ .../concurrent/ConcurrentCollectionsTest.kt | 34 ++ .../concurrent/ConcurrentMap.jvmAndroid.kt | 2 + .../concurrent/PlatformLock.jvmAndroid.kt | 35 ++ .../LockDesignComparisonBenchmark.kt | 316 ++++++++++++++++++ .../prodbench/SpinLockConvoyBenchmark.kt | 242 ++++++++++++++ .../utils/concurrent/PlatformLock.linux.kt | 44 +++ .../utils/concurrent/ConcurrentMap.native.kt | 10 + 14 files changed, 1205 insertions(+), 127 deletions(-) create mode 100644 quartz/plans/2026-08-03-poolrequests-lock-contention.md create mode 100644 quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/PlatformLock.apple.kt create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/PlatformLock.kt create mode 100644 quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/PlatformLock.jvmAndroid.kt create mode 100644 quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/LockDesignComparisonBenchmark.kt create mode 100644 quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/SpinLockConvoyBenchmark.kt create mode 100644 quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/PlatformLock.linux.kt diff --git a/quartz/plans/2026-08-03-poolrequests-lock-contention.md b/quartz/plans/2026-08-03-poolrequests-lock-contention.md new file mode 100644 index 0000000000..5c14313e09 --- /dev/null +++ b/quartz/plans/2026-08-03-poolrequests-lock-contention.md @@ -0,0 +1,221 @@ +# PoolRequests subscription-state lock: what a suspending Mutex would cost + +**Date:** 2026-08-03 +**Status:** analysis + measurements; recommendation is NOT to use `Mutex` +**Harness:** `quartz/src/jvmTest/.../prodbench/LockDesignComparisonBenchmark.kt` +(`./gradlew :quartz:jvmTest --tests "*.LockDesignComparisonBenchmark"`) +**Context:** follows the Pixel 8 ANR (`anr_2026-08-03-12-55-26-256`) that replaced +`RequestSubscriptionState`'s busy-wait with a parking `PlatformLock`. + +## The question + +The parking lock removed the CPU burn, but it still **blocks** rather than suspends. +Relay consumers run as coroutines on `Dispatchers.IO` (limitedParallelism 64) and +production has ~191 live relays, so a blocked waiter occupies one of 64 dispatcher +threads. kotlinx's scheduler treats IO tasks as blocking and grows the pool when they +block — which is why the on-device fix moved CPU but left thread count flat (+7%). + +Would `kotlinx.coroutines.sync.Mutex` (a waiter *suspends*, freeing its thread) be +better? + +## What Mutex actually costs + +The lock sits at the bottom of a call chain that is **entirely non-suspend**: + +``` +OkHttpWebSocket: scope.launch { for (m in incomingMessages) out.onMessage(m) } <- coroutine + WebSocketListener.onMessage (non-suspend) + BasicRelayClient.MyWebsocketListener.onMessage + RelayPool.onIncomingMessage + NostrClient.onIncomingMessage (RelayConnectionListener) + PoolRequests.onIncomingMessage + RequestSubscriptionState.withLock <- the lock + SubscriptionListener.onEvent (non-suspend, fans out to the app) +``` + +`Mutex.lock()` is `suspend`, so every frame above it must become `suspend`. Measured +blast radius: + +| interface / API | count | +|---|---| +| `override fun onEvent(` | 59 | +| `override fun onEose(` | 52 | +| `override fun onIncomingMessage(` | 35 | +| `override fun onCannotConnect(` | 32 | +| `override fun onClosed(` | 25 | +| `override fun onDisconnected(` | 17 | +| `override fun onConnected(` | 16 | +| `override fun onSent(` | 14 | +| `override fun onConnecting(` | 11 | +| others (`onSubscriptionStarted`, …) | 1 | +| **total overrides to convert** | **262** | +| `.subscribe(` / `.unsubscribe(` call sites | **110** | + +Worse than the count: the *entry points* are not all coroutines. `INostrClient` is +non-suspend by design — `subscribe`, `unsubscribe`, `publish`, `syncFilters`, +`connect` — and is called from ViewModels, filter assemblers and Compose effects. +`PoolRequests.addOrUpdate` / `remove` / `sendToRelayIfChanged` reach the lock from +those paths. Making them suspend pushes coroutine scoping into every call site that +today just calls `subscribe(...)` synchronously. + +## The cheaper alternative: stripe the lock per relay + +**Enabling invariant (verified by reading all 11 `withLock` bodies):** *every* +critical section in `PoolRequests` is scoped to exactly one relay. Each one takes +`url` / `relay` / `relay.url` as its key: + +| line | body | key | +|---|---|---| +| 204 | `state.connecting(url)` | url | +| 218 | `state.onOpenReq(relay, cmd.filters)` | relay | +| 228 | `state.onSubscriptionClosed(relay)` | relay | +| 249 | `onNewEvent` / `currentState` / `lastKnownFilterStates` | relay.url | +| 267 | `onEose` + `decideCommandLocked` | relay.url | +| 296 | `onClosed` + `recordRefusalIfStructural` + `decideCommandLocked` | relay.url | +| 325 | `state.disconnected(url)` | url | +| 354 | `isStructurallyRefused` + `onOpenReq` | relay | +| 382 | `lastKnownFilterStates(url)` | url | +| 403 | `decideCommandLocked(state, subId, relay)` | relay | + +Every field in `RequestSubscriptionState` is a `Map` keyed by relay +(`subStates`, `filterStates`, `lastKnownFilterStates`, `refusedFilters`, +`refusalCounts`). The single cross-relay accessor, `currentFilters()` (no-arg, +returns the whole map), has **zero usages** — dead code, delete it. + +So the lock is only shared across relays as an artifact of `mutableMapOf` not being +thread-safe. One lock per `(subId, relay)` is semantically equivalent and drops +contention from ~191 threads to ~1–2 (that relay's consumer, plus the occasional app +thread in `sendToRelayIfChanged`). + +Blast radius: `RequestSubscriptionState` + `PoolRequests` only. Used by 4 production +files (`PoolRequests`, `RelayActiveRequestStates`, `RelayReqRefusals`) and 2 tests. +**No public API change.** + +## Measurements + +191 relay coroutines on a 64-thread limited-parallelism dispatcher, 3s windows. Each +iteration yields, modelling the real per-message suspension point of +`for (message in incomingMessages)` — without it the tight loops monopolise the +dispatcher and the harness measures itself, not the lock. + +`bystander` = a task that never touches the lock, on the same dispatcher. Its latency +answers "is the lock stealing dispatcher threads from unrelated work?" + +| subs | design | ops/s | bystander p50 | p99 | +|---|---|---|---|---| +| 1 | PER_SUB_BLOCKING (today) | 548,608 | 196.5µs | 770.9µs | +| 1 | PER_SUB_MUTEX | 190,672 | **1.9µs** | 30.4µs | +| 1 | **STRIPED** | **1,543,037** | 88.8µs | 237.6µs | +| 4 | PER_SUB_BLOCKING | 833,090 | 136.1µs | 507.6µs | +| 4 | PER_SUB_MUTEX | 523,440 | **3.1µs** | 26.0µs | +| 4 | **STRIPED** | **1,499,757** | 92.3µs | 337.5µs | +| 16 | PER_SUB_BLOCKING | 1,198,581 | 94.3µs | 444.3µs | +| 16 | PER_SUB_MUTEX | 749,487 | **5.0µs** | 16.5µs | +| 16 | **STRIPED** | **1,789,509** | 85.7µs | 219.8µs | + +Reading: + +- **STRIPED gives the most throughput** — 2.8× / 1.8× / 1.5× over today — and roughly + halves bystander p50. It removes the contention rather than tolerating it. +- **MUTEX is the slowest of the three** (0.35× / 0.63× / 0.63× of today): per-acquisition + suspend/resume is not free at these rates. +- **MUTEX does win bystander latency decisively** (1.9–5.0µs vs 85–196µs, and it collected + 626k vs 24k samples). But read that honestly — part of the win is that its coroutines + spend their time *suspended waiting for the mutex instead of doing work*. Better + thread-yielding is partly a symptom of lower throughput, not purely a win. + +## On-device result (SM-T220, playBenchmark, same account, n=3 per design) + +Cold-start burst, 75s window, exact per-thread CPU accounting from `/proc//stat`. +Warmups matched (~220-255 established sockets, ~200 relay reader threads each time). + +| design | DefaultDispatcher CPU (ms/75s) mean / median / range | GC ms | threads | +|---|---|---|---| +| spin lock (pre-fix) | 136,743 / 117,750 / 112,230–180,250 | 33,777 | 475 | +| parking, one lock per sub | 103,780 / 99,160 / 86,120–130,680 | 30,235 | 510 | +| **striped, per (sub, relay)** | **88,953 / 82,160 / 72,860–111,840** | **27,687** | 489 | + +- **Striped vs spin: −35% mean, −30% median CPU, −18% GC — and the ranges do not + overlap** (striped max 111,840 < spin min 112,230). That separation is what the + parking-only change could not show; its range overlapped the baseline heavily. +- **Striped vs parking: −14% mean / −17% median**, ranges still overlap — directional, + not conclusive at n=3. +- **Thread count is unchanged across all three** (475 / 510 / 489). Expected: the lock + is blocking, and kotlinx marks IO tasks blocking and grows the pool. Striping reduces + how *often* threads block, not the fact that they can. +- Not a false win from broken subscriptions: the feed rendered live notes 3-15 min old + with avatars and reaction counts, on 223 sockets / 200 relay reader threads. + +## Recommendation + +**Do the striping; do not convert to `Mutex`.** + +Striping attacks the cause — the lock is contended only because it is shared across +relays that touch disjoint keys. Once contention is ~0, the blocking-vs-suspending +question is moot: *a lock that is never contended never blocks a thread*. It also +happens to be the fastest option and is contained to two files, versus 262 overrides +and 110 call sites for a design that measures slower. + +`Mutex` only becomes the right answer if a future critical section must genuinely span +relays (or do I/O), which none does today. + +## The lock must not live inside a removable entry + +The obvious shape — `ConcurrentMap` where `PerRelayState` owns both +the fields *and* its lock — is **wrong as soon as entries can be removed**. +`connecting()` / `disconnected()` genuinely mean "forget this relay's wire state", so +they want removal, and then: + +``` +T1: getOrPut(R) -> stateA ; stateA.lock.lock() // in a critical section +T2: disconnected(R) -> remove(R) // stateA is now orphaned +T3: getOrPut(R) -> stateB (NEW lock) ; stateB.lock.lock() // acquires immediately + -> T1 and T3 are both "in" relay R's critical section, excluding nothing. +``` + +Two ways out: + +- **(A) never remove; null the fields.** Lock identity is stable, but entries + accumulate for every relay a sub has *ever* seen. Bounded but monotonic — and slow + monotonic growth in a long-lived process is precisely the class of bug this whole + investigation was about. +- **(B) stable stripe locks + removable state.** A fixed `Array(N) { PlatformLock() }` + indexed by `relay.hashCode()`, never mutated, so lock identity can't change; the + `ConcurrentMap` entries are then free to be added and removed with + exact `connecting`/`disconnected` semantics and no growth. + +**(B) is the recommendation.** With N = 32 and ~191 relays, ~6 relays share a stripe — +still a ~32x contention reduction versus today's single lock per sub, with none of the +lock-lifetime hazard. `ConcurrentMap.remove` (added 2026-08-03, with tests in +`ConcurrentCollectionsTest`) is what makes (B) possible. + +## Implementation sketch + +1. Delete the dead `currentFilters()` (no-arg). +2. In `RequestSubscriptionState`, replace the five relay-keyed maps with a single + `ConcurrentMap` holding the five fields — **no lock inside**. +3. Add a fixed `private val stripes = Array(32) { PlatformLock() }` and + `withLock(reference)` = `stripes[reference.hashCode().absoluteValue % 32].withLock { }`. + The array is never mutated, so lock identity is stable for the object's life. +4. `connecting()` / `disconnected()` become `map.remove(reference)` — exact current + semantics, no residue. +5. Update the 11 `withLock` call sites in `PoolRequests` to pass the relay. + `decideCommandLocked`'s "MUST hold the lock" contract becomes "MUST hold *that + relay's stripe*" — tighten the kdoc. +6. Extend `PoolRequestsRefusalTest` with concurrent multi-relay access on one subId, + and add a striped variant to `LockDesignComparisonBenchmark` to confirm the + measured win survives stripe collisions. + +## Risks + +- **Stripe collisions serialize unrelated relays.** With 32 stripes and 191 relays this + is ~6-way sharing; it is a contention *reduction*, not elimination. If a future + profile shows it mattering, raise N — it is a one-line change with no semantic effect. +- **Two relays on one stripe must never be locked simultaneously by one thread** — that + would self-deadlock (`PlatformLock` is reentrant on JVM/Apple, so same-thread + re-entry is survivable, but the invariant should be stated). `PoolRequests` already + locks one relay at a time. +- The state machine's atomicity comments are load-bearing; the per-relay scoping must + be re-verified against any new `withLock` body added between now and the change. +- Striping is NOT a fix for a critical section that does I/O or spans relays. If one is + ever added, this analysis must be redone. diff --git a/quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/PlatformLock.apple.kt b/quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/PlatformLock.apple.kt new file mode 100644 index 0000000000..c59d0ccecc --- /dev/null +++ b/quartz/src/appleMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/PlatformLock.apple.kt @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.utils.concurrent + +import platform.Foundation.NSRecursiveLock + +// NSRecursiveLock parks contended waiters in the kernel, matching the +// ReentrantLock semantics of the jvmAndroid actual (and the same choice +// commons' KmpLock made for iOS). This must NOT be a spin lock: quartz's +// relay client runs here too, and spinning is what produced the Android +// ANR documented on the expect declaration. +actual class PlatformLock { + private val delegate = NSRecursiveLock() + + actual fun lock() = delegate.lock() + + actual fun unlock() = delegate.unlock() +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolRequests.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolRequests.kt index eb8c220c7d..7d4b08315b 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolRequests.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolRequests.kt @@ -84,25 +84,26 @@ class PoolRequests( /* * Locking model: every compound access to a subscription's state machine * ([RequestSubscriptionState]) — including the check-then-send decision in - * [decideCommandLocked] — runs inside THAT subscription's own lock - * ([RequestSubscriptionState.withLock]). + * [decideCommandLocked] — runs inside the stripe for THAT (subscription, relay) + * pair ([RequestSubscriptionState.withLock], which takes the relay). * - * A single subscription can span many relays, and each relay's - * socket-reader thread delivers messages into this class concurrently - * while the app thread adds/removes subscriptions — so the plain maps - * inside [RequestSubscriptionState] are written from several threads at - * once. That is both a memory hazard (concurrent map mutation) and a - * logic hazard: two threads must never both observe "no REQ in flight" - * and both send a REQ for the same sub id. + * A single subscription can span many relays, and each relay's socket-reader + * thread delivers messages into this class concurrently while the app thread + * adds/removes subscriptions. Two threads must never both observe "no REQ in + * flight" and both send a REQ for the same (sub, relay). * - * The lock is per subscription, not global, because different subIds - * share no wire state: EVENT frames for different subs coming from - * different relay consumer threads must not serialize on each other (a - * global lock here measured negative scaling under 4 concurrent relay - * feeders). Listener callbacks and the actual socket sends are ALWAYS - * performed outside the lock — they re-enter this class through - * [onSent], so holding the lock across them would self-deadlock, and the - * lock is non-reentrant. Never hold two subscriptions' locks at once. + * The lock is striped PER RELAY rather than per subscription: every critical + * section below touches only one relay's slice of the state, so EVENT frames + * arriving for the same subId from different relays no longer serialize on each + * other. With ~191 relays that previously made a single per-sub lock contended + * ~191 threads deep — the production ANR in + * `quartz/plans/2026-08-03-poolrequests-lock-contention.md`. + * + * Two rules this file must keep: + * - Never hold two relays' stripes (or two subscriptions') at once. Every loop + * below locks exactly one relay at a time. + * - Listener callbacks and socket sends are ALWAYS performed outside the lock — + * they re-enter this class through [onSent]. */ /** @@ -201,7 +202,7 @@ class PoolRequests( fun onConnecting(url: NormalizedRelayUrl) { // Change states to connecting. One sub's lock at a time. relayState.forEach { subId, state -> - state.withLock { state.connecting(url) } + state.withLock(url) { state.connecting(url) } } } @@ -215,7 +216,7 @@ class PoolRequests( when (cmd) { is ReqCmd -> { subState(cmd.subId).let { state -> - state.withLock { state.onOpenReq(relay, cmd.filters) } + state.withLock(relay) { state.onOpenReq(relay, cmd.filters) } } desiredSubListeners.get(cmd.subId)?.onSubscriptionStarted( relay = relay.url, @@ -225,7 +226,7 @@ class PoolRequests( is CloseCmd -> { subState(cmd.subId).let { state -> - state.withLock { state.onSubscriptionClosed(relay) } + state.withLock(relay) { state.onSubscriptionClosed(relay) } } desiredSubListeners.get(cmd.subId)?.onSubscriptionClosed( relay = relay.url, @@ -246,7 +247,7 @@ class PoolRequests( var isLive = false var forFilters: List? = null relayState.get(msg.subId)?.let { state -> - state.withLock { + state.withLock(relay.url) { state.onNewEvent(relay.url) isLive = state.currentState(relay.url) == ReqSubStatus.LIVE forFilters = state.lastKnownFilterStates(relay.url) @@ -264,7 +265,7 @@ class PoolRequests( var forFilters: List? = null val cmd = relayState.get(msg.subId)?.let { state -> - state.withLock { + state.withLock(relay.url) { state.onEose(relay.url) forFilters = state.lastKnownFilterStates(relay.url) // Decide (and pre-mark) the resend while still holding the @@ -293,7 +294,7 @@ class PoolRequests( var forFilters: List? = null val cmd = relayState.get(msg.subId)?.let { state -> - state.withLock { + state.withLock(relay.url) { state.onClosed(relay.url) forFilters = state.lastKnownFilterStates(relay.url) recordRefusalIfStructural(state, relay.url, msg.message, forFilters) @@ -322,7 +323,7 @@ class PoolRequests( */ fun onDisconnected(url: NormalizedRelayUrl) { relayState.forEach { subId, state -> - state.withLock { state.disconnected(url) } + state.withLock(url) { state.disconnected(url) } } } @@ -351,7 +352,7 @@ class PoolRequests( if (!filters.isNullOrEmpty()) { val send = subState(subId).let { state -> - state.withLock { + state.withLock(relay) { if (isStructurallyRefused(state, relay, filters)) { false } else { @@ -379,7 +380,7 @@ class PoolRequests( // These are all my subs.. need to figure out which relays have them val subs = desiredSubs.get(subId) if (subs != null && url in subs.keys) { - toNotify.add(subId to state.withLock { state.lastKnownFilterStates(url) }) + toNotify.add(subId to state.withLock(url) { state.lastKnownFilterStates(url) }) } } @@ -400,7 +401,7 @@ class PoolRequests( val state = subState(subId) relaysToUpdate.forEach { relay -> // Decide + pre-mark atomically under the sub's lock, then send outside it. - val cmd = state.withLock { decideCommandLocked(state, subId, relay) } + val cmd = state.withLock(relay) { decideCommandLocked(state, subId, relay) } if (cmd != null) { sync(relay, cmd) } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/RequestSubscriptionState.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/RequestSubscriptionState.kt index 4bfdd6aa2a..47b417241a 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/RequestSubscriptionState.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/RequestSubscriptionState.kt @@ -21,84 +21,116 @@ package com.vitorpamplona.quartz.nip01Core.relay.client.reqs import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter -import kotlin.concurrent.atomics.AtomicBoolean -import kotlin.concurrent.atomics.ExperimentalAtomicApi +import com.vitorpamplona.quartz.utils.concurrent.ConcurrentMap +import com.vitorpamplona.quartz.utils.concurrent.PlatformLock /** * Manages the State of Subscriptions by logging states as the * subscription progresses. * - * Thread-safety: the plain maps below are only touched inside [withLock]. - * The lock lives HERE — one per subscription — instead of a single global - * lock in PoolRequests, because every mutation is scoped to one subId: - * relays delivering EVENTs for *different* subscriptions have no shared - * state and must not serialize on each other. (A single global spin lock - * measured *negative* scaling: 4 relay consumer threads pushed less - * aggregate throughput through it than 1 — see - * quartz/plans/2026-07-02-nostrclient-receiver-perf.md.) + * **Thread-safety: the lock is striped per reference (per relay), not per + * subscription.** Every operation below is scoped to a single [reference] — all state + * lives in [RelayState] objects held in a [ConcurrentMap] keyed by it — so two relays + * delivering EVENTs for the SAME subscription touch disjoint state and no longer + * serialize on each other. + * + * That mattered in production: one subId spans every relay it is subscribed on, so a + * single per-subscription lock was contended ~191 threads deep on the Pixel 8 that + * produced `anr_2026-08-03-12-55-26-256` (it was a *spin* lock then, which burned 6 of + * 9 cores — see [PlatformLock]). Striping removes the contention instead of making + * waiting cheaper: measured 1.5-2.8x the throughput of one lock per sub in + * `quartz/src/jvmTest/.../prodbench/LockDesignComparisonBenchmark.kt`. Full analysis, + * including why a suspending `Mutex` was rejected, is in + * `quartz/plans/2026-08-03-poolrequests-lock-contention.md`. + * + * The stripe array is allocated once and NEVER mutated, so a stripe's identity is + * stable for this object's whole life. That is load-bearing: if locks lived inside the + * per-relay values, a thread holding one while another thread dropped and re-created + * that entry would leave both "inside" the critical section excluding nothing. + * + * Callers MUST NOT hold two references' stripes at once ([PoolRequests] locks one relay + * at a time, including inside its all-subs iterations), and MUST keep socket sends and + * listener callbacks outside the critical section — they re-enter this class through + * `onSent`, and the point of a lock is to be held briefly. */ -@OptIn(ExperimentalAtomicApi::class) -class RequestSubscriptionState { +class RequestSubscriptionState { /** - * Tiny non-reentrant spin lock (same primitive as BasicRelayClient's - * connecting mutex). Critical sections are a handful of map operations, - * never I/O — callers MUST NOT re-enter and MUST NOT hold two - * subscriptions' locks at once (PoolRequests locks one sub at a time, - * including inside its all-subs iterations). + * One reference's (relay's) slice of this subscription's state. Plain `var`s: every + * field is written and read under that reference's stripe by [PoolRequests]. * - * `@PublishedApi internal` only because [withLock] is inline (this sits - * on the per-EVENT hot path; inlining avoids a closure allocation per - * message) — treat it as private. + * Note `RelayActiveRequestStates` uses this class WITHOUT locking. There the fields + * may be read stale — but the backing [ConcurrentMap] can no longer be structurally + * corrupted the way the plain `HashMap`s this replaced could. + */ + private class RelayState { + /** Null == no REQ state on this relay (fresh, or wiped by connecting/disconnected). */ + var status: ReqSubStatus? = null + + /** Filters of the REQ currently believed to be in flight. */ + var filters: List? = null + + /** + * Survives connect/disconnect so that if new events still arrive we can link + * them with the filters the relay was processing. + */ + var lastKnownFilters: List? = null + + /** + * Refused-filter memory. Unlike [status]/[filters] — per-connection wire state + * wiped by [connecting]/[disconnected] — this SURVIVES reconnects on purpose: a + * relay that structurally refuses a filter (a search-only relay CLOSING a plain + * kinds REQ, a relay that "does not accept REQs", "too many filters", …) refuses + * it again on every new socket, so [PoolRequests.syncState] replaying it each + * reconnect is pure waste. [refusalCount] accumulates repeated refusals of the + * same shape so a one-off (transient) close isn't mistaken for a structural one. + * Cleared on a successful REQ ([onEose]/[onNewEvent]) or when the caller observes + * the desired filter meaningfully changed. + */ + var refusedFilters: List? = null + + var refusalCount: Int = 0 + } + + private val states = ConcurrentMap() + + /** + * Fixed stripe array — allocated once, never mutated, so lock identity is stable. + * [STRIPE_COUNT] stripes over ~191 relays is roughly 6-way sharing: a ~32x + * contention reduction versus one lock per subscription. References colliding on a + * stripe merely serialize; correctness never depends on N. */ @PublishedApi - internal val lock = AtomicBoolean(false) + internal val stripes = Array(STRIPE_COUNT) { PlatformLock() } - inline fun withLock(block: () -> R): R { - while (lock.exchange(true)) { - // Test-and-test-and-set: spin-read until it looks free (cheaper - // on the cache line than hammering exchange), then retry above. - while (lock.load()) { } - } + @PublishedApi + internal fun stripeFor(reference: T): PlatformLock = stripes[(reference.hashCode() and 0x7FFFFFFF) % STRIPE_COUNT] + + /** + * Runs [block] holding [reference]'s stripe. Inline so the per-EVENT hot path + * allocates no closure. + */ + inline fun withLock( + reference: T, + block: () -> R, + ): R { + val lock = stripeFor(reference) + lock.lock() try { return block() } finally { - lock.store(false) + lock.unlock() } } - // Logs the state of each channel to: - // 1. inform when an event is received as live - // 2. to block REQs being sent before finished (receiving an EOSE or Closed) - // - // If 2 happens, the relay might send multiple EOSEs in sequence - // for the same sub and we won't know which REQ was it for. - private val subStates = mutableMapOf() - private val filterStates = mutableMapOf>() + /** Read-only lookup — never creates an entry. */ + private fun peek(reference: T): RelayState? = states[reference] - /** - * This cache is used to make sure we know what the relay was processing - * before a close or disconnect so that if new events still arrive - * we can link them with the appropriate filters. - */ - private val lastKnownFilterStates = mutableMapOf>() + /** Write lookup — creates the entry on first use. */ + private fun mutable(reference: T): RelayState = states.getOrPut(reference) { RelayState() } - /** - * Refused-filter memory. Unlike [subStates]/[filterStates] above — per-connection - * wire state wiped by [connecting]/[disconnected] — this SURVIVES reconnects on - * purpose: a relay that structurally refuses a filter (a search-only relay CLOSING - * a plain kinds REQ, a relay that "does not accept REQs", "too many filters", …) - * refuses it again on every new socket, so [PoolRequests.syncState] replaying it - * each reconnect is pure waste. [refusalCounts] accumulates repeated refusals of - * the same shape so a one-off (transient) close isn't mistaken for a structural - * one. Cleared on a successful REQ ([onEose]/[onNewEvent]) or when the caller - * observes the desired filter meaningfully changed. - */ - private val refusedFilters = mutableMapOf>() - private val refusalCounts = mutableMapOf() + fun refusedFilters(reference: T) = peek(reference)?.refusedFilters - fun refusedFilters(reference: T) = refusedFilters[reference] - - fun refusalCount(reference: T) = refusalCounts[reference] ?: 0 + fun refusalCount(reference: T) = peek(reference)?.refusalCount ?: 0 /** * Records that [reference] refused [filters]. [sameAsLastRefusal] must be true when @@ -111,73 +143,91 @@ class RequestSubscriptionState { filters: List, sameAsLastRefusal: Boolean, ) { + val state = mutable(reference) if (sameAsLastRefusal) { - refusalCounts[reference] = refusalCount(reference) + 1 + state.refusalCount += 1 } else { - refusedFilters[reference] = filters - refusalCounts[reference] = 1 + state.refusedFilters = filters + state.refusalCount = 1 } } fun clearRefusal(reference: T) { - refusedFilters.remove(reference) - refusalCounts.remove(reference) + peek(reference)?.let { + it.refusedFilters = null + it.refusalCount = 0 + } } - fun currentFilters() = filterStates + fun currentFilters(reference: T) = peek(reference)?.filters - fun currentFilters(reference: T) = filterStates[reference] + fun lastKnownFilterStates(reference: T) = peek(reference)?.lastKnownFilters - fun lastKnownFilterStates(reference: T) = lastKnownFilterStates[reference] - - fun currentState(reference: T) = subStates[reference] + fun currentState(reference: T) = peek(reference)?.status fun onNewEvent(reference: T) { + val state = mutable(reference) // The relay is serving this REQ (it matched an event), so any past refusal // no longer applies — let it be tried freely again. - clearRefusal(reference) - if (subStates[reference] == ReqSubStatus.SENT) { - subStates[reference] = ReqSubStatus.QUERYING_PAST + state.refusedFilters = null + state.refusalCount = 0 + if (state.status == ReqSubStatus.SENT) { + state.status = ReqSubStatus.QUERYING_PAST } } fun onEose(reference: T) { + val state = mutable(reference) // Reaching EOSE means the relay accepted and finished the REQ; clear any refusal. - clearRefusal(reference) - subStates[reference] = ReqSubStatus.LIVE + state.refusedFilters = null + state.refusalCount = 0 + state.status = ReqSubStatus.LIVE } fun onClosed(reference: T) { - subStates[reference] = ReqSubStatus.CLOSED - // Closed messages are usually relays refusing to process a REQ - // This message keeps the state of filterStates intact to - // avoid sending the same filter, and getting immediately closed, - // over and over again. - - // filterStates.remove(reference) + // Closed messages are usually relays refusing to process a REQ. This keeps + // [RelayState.filters] intact to avoid sending the same filter, and getting + // immediately closed, over and over again. + mutable(reference).status = ReqSubStatus.CLOSED } fun onOpenReq( reference: T, filters: List, ) { - subStates[reference] = ReqSubStatus.SENT - filterStates[reference] = filters - lastKnownFilterStates[reference] = filters + val state = mutable(reference) + state.status = ReqSubStatus.SENT + state.filters = filters + state.lastKnownFilters = filters } fun onSubscriptionClosed(reference: T) { - subStates[reference] = ReqSubStatus.CLOSED - filterStates.remove(reference) + val state = mutable(reference) + state.status = ReqSubStatus.CLOSED + state.filters = null } fun connecting(reference: T) { - subStates.remove(reference) - filterStates.remove(reference) + // Wipes per-connection wire state only; lastKnownFilters and the refusal memory + // deliberately survive (see [RelayState]). + peek(reference)?.let { + it.status = null + it.filters = null + } } fun disconnected(reference: T) { - subStates.remove(reference) - filterStates.remove(reference) + peek(reference)?.let { + it.status = null + it.filters = null + } + } + + companion object { + /** + * Comfortably above the IO dispatcher's thread count (64 / 2) so collisions stay + * rare even when many workers are inside this class at once. + */ + const val STRIPE_COUNT = 32 } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/LiveNegentropyIndex.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/LiveNegentropyIndex.kt index 0e0b134d63..9b39c70776 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/LiveNegentropyIndex.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/LiveNegentropyIndex.kt @@ -22,8 +22,7 @@ package com.vitorpamplona.quartz.nip77Negentropy import com.vitorpamplona.negentropy.storage.IStorage import com.vitorpamplona.quartz.nip01Core.store.IdAndTime -import kotlin.concurrent.atomics.AtomicBoolean -import kotlin.concurrent.atomics.ExperimentalAtomicApi +import com.vitorpamplona.quartz.utils.concurrent.PlatformLock /** * Always-current `(created_at, id)` index for NIP-77 negentropy — the @@ -54,13 +53,12 @@ import kotlin.concurrent.atomics.ExperimentalAtomicApi * sealed storage, so one snapshot backs any number of concurrent * sessions and stays valid even if the live index mutates after. * - * Thread-safety: all operations take a short spin lock (same pattern as - * `LiveEventStore`'s replay dedup). Mutations arrive from the store's - * single writer; snapshots from any REQ coroutine. + * Thread-safety: all operations take a short parking lock ([PlatformLock]). + * Mutations arrive from the store's single writer; snapshots from any REQ + * coroutine. */ -@OptIn(ExperimentalAtomicApi::class) class LiveNegentropyIndex { - private val lock = AtomicBoolean(false) + private val lock = PlatformLock() /** Sorted by (createdAt, id). Only touched under [locked]. */ private var entries = ArrayList() @@ -74,14 +72,17 @@ class LiveNegentropyIndex { private var cachedSnapshot: IStorage? = null private var cachedGeneration = -1L + // Parks rather than busy-waits. This lock's critical sections are FAR longer than a + // handful of map ops — [rebuild] sorts the whole entry list and snapshotting builds a + // storage — so a spinning waiter would burn a core for the duration of a sort while + // concurrent ingest threads pile up. Same defect that produced the client-side ANR + // documented on PlatformLock; see quartz/.../prodbench/SpinLockConvoyBenchmark.kt. private inline fun locked(block: () -> R): R { - while (lock.exchange(true)) { - while (lock.load()) { } - } + lock.lock() try { return block() } finally { - lock.store(false) + lock.unlock() } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.kt index 47d3233c9b..f06d914583 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.kt @@ -61,6 +61,17 @@ expect class ConcurrentMap() { remap: (old: V, new: V) -> V, ): V + /** + * Removes [key] and returns the value it held, or null when absent. + * + * CAUTION: if the removed value owns a lock that callers acquire, removal + * breaks mutual exclusion — a thread holding the old value's lock and a + * thread that re-created the entry are no longer excluding each other. Keep + * such locks in a structure whose identity is stable (see + * `quartz/plans/2026-08-03-poolrequests-lock-contention.md`). + */ + fun remove(key: K): V? + fun size(): Int /** A point-in-time copy of the entries — safe to iterate without holding a lock. */ diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/PlatformLock.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/PlatformLock.kt new file mode 100644 index 0000000000..b43f69e5ec --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/PlatformLock.kt @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.utils.concurrent + +/** + * A blocking mutual-exclusion lock whose waiters **park** (yield the core to the + * scheduler) instead of busy-waiting. + * + * Why this exists: `commonMain` has no `java.util.concurrent.locks.Lock`, so the + * relay client previously hand-rolled a spin lock over an `AtomicBoolean`. A + * busy-wait is only ever correct when the holder cannot be descheduled while + * holding the lock — and on Android that assumption is false. A production ANR on + * a Pixel 8 (`anr_2026-08-03-12-55-26-256`, Amethyst 1.13.1) caught the exact + * failure: the thread holding the lock was parked in `WaitingForGcToComplete` + * while **51 of 52** runnable relay-dispatch threads sat in the inlined spin loop, + * burning 596% CPU (6 of the phone's 9 cores) waiting for a holder that could not + * be scheduled to release it. The UI thread, needing to allocate, then waited on + * the same GC for over 5s and Android killed the frame with + * "Input dispatching timed out". + * + * Measured on `SpinLockConvoyBenchmark` (12-core dev machine — a phone is worse): + * the spin lock delivered 138M critical sections/s uncontended but only 1.2M/s + * with 52 contenders (0.86%, a 116x collapse), and an *unrelated* allocating + * thread's p90 latency went from 22µs to 10ms. Parking removes the CPU burn: a + * waiter costs one context switch instead of a whole core. + * + * Splits the same way [ConcurrentMap] does: + * - JVM / Android → `ReentrantLock` (real parking via `AbstractQueuedSynchronizer`). + * - Apple → `NSRecursiveLock`, which also parks. The relay client genuinely runs + * on iOS, so this must not spin — same choice commons' `KmpLock` already made. + * - Linux → a spin, since Kotlin/Native ships no parking lock and there is no + * Foundation; linuxX64 is a build/CI target, not a host for the many-relay + * workload. Swap in a pthread mutex if that changes. + * + * (`commons` has an equivalent `KmpLock`, but `commons` depends on `quartz` and not + * the reverse, so quartz cannot use it — keep the two in sync by hand.) + * + * Unlike the primitive it replaces, the JVM/Android actual is **reentrant**, so an + * accidental re-entry degrades into a no-op rather than a self-deadlock. Callers + * should still keep I/O and listener callbacks outside the critical section — that + * discipline is about holding the lock briefly, not about avoiding a hang. + */ +expect class PlatformLock() { + fun lock() + + fun unlock() +} + +/** Runs [block] holding [this]. Inline so hot paths allocate no closure. */ +inline fun PlatformLock.withLock(block: () -> R): R { + lock() + try { + return block() + } finally { + unlock() + } +} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentCollectionsTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentCollectionsTest.kt index 208054f818..5f78d107f1 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentCollectionsTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentCollectionsTest.kt @@ -71,6 +71,40 @@ class ConcurrentCollectionsTest { assertEquals(4, m["k"]) } + @Test + fun mapRemoveReturnsOldValueAndDeletes() { + val map = ConcurrentMap() + map["a"] = 1 + map["b"] = 2 + + assertEquals(1, map.remove("a")) + assertNull(map["a"]) + assertEquals(1, map.size()) + assertEquals(2, map["b"]) + } + + @Test + fun mapRemoveAbsentKeyIsNullAndNoOp() { + val map = ConcurrentMap() + map["b"] = 2 + + assertNull(map.remove("missing")) + assertEquals(1, map.size()) + assertEquals(2, map["b"]) + } + + @Test + fun mapRemoveThenGetOrPutRecreates() { + // The lifecycle PoolRequests needs: connecting()/disconnected() drop a relay's + // wire state, and the next REQ re-creates it. A stale value must never survive. + val map = ConcurrentMap() + map.getOrPut("relay") { 1 } + map.remove("relay") + + assertEquals(9, map.getOrPut("relay") { 9 }) + assertEquals(9, map["relay"]) + } + @Test fun mapSnapshotIsDetached() { val m = ConcurrentMap() diff --git a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.jvmAndroid.kt b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.jvmAndroid.kt index aaf40fdcb7..c68eb8da8a 100644 --- a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.jvmAndroid.kt +++ b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.jvmAndroid.kt @@ -49,6 +49,8 @@ actual class ConcurrentMap { remap: (old: V, new: V) -> V, ): V = map.merge(key, value) { old, new -> remap(old, new) }!! + actual fun remove(key: K): V? = map.remove(key) + actual fun size(): Int = map.size actual fun snapshot(): Map = HashMap(map) diff --git a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/PlatformLock.jvmAndroid.kt b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/PlatformLock.jvmAndroid.kt new file mode 100644 index 0000000000..58488489b6 --- /dev/null +++ b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/utils/concurrent/PlatformLock.jvmAndroid.kt @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.utils.concurrent + +import java.util.concurrent.locks.ReentrantLock + +// ReentrantLock parks contended waiters through AbstractQueuedSynchronizer, so a +// thread waiting on a holder that lost its core (GC, preemption) costs one context +// switch rather than a spinning core. Non-fair on purpose: fairness would add a +// handoff per acquisition and the critical sections here are microseconds long. +actual class PlatformLock { + private val lock = ReentrantLock() + + actual fun lock() = lock.lock() + + actual fun unlock() = lock.unlock() +} diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/LockDesignComparisonBenchmark.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/LockDesignComparisonBenchmark.kt new file mode 100644 index 0000000000..48e75ec97d --- /dev/null +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/LockDesignComparisonBenchmark.kt @@ -0,0 +1,316 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay.prodbench + +import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.RequestSubscriptionState +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.junit.Test +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.locks.ReentrantLock + +/** + * Compares the three candidate designs for `PoolRequests`' subscription-state lock, + * under the real production topology: R relay-consumer coroutines on a + * limited-parallelism dispatcher (`Dispatchers.IO` = 64) all delivering EVENTs for a + * small number of subscription ids. + * + * - **PER_SUB_BLOCKING** — today: one blocking lock per subId, shared by every relay + * that sub runs on. Waiters park, which fixed the CPU burn, but a parked waiter + * still OCCUPIES its dispatcher thread. + * - **PER_SUB_MUTEX** — kotlinx `Mutex`: a waiter *suspends* and releases its thread. + * Requires making the whole listener chain `suspend` (262 overrides, 110 call sites). + * - **STRIPED** — one lock per (subId, relay). Every critical section in PoolRequests + * is already scoped to a single relay, so this removes the contention outright and + * needs no API change. + * + * The metric that matters is NOT lock throughput — it is whether unrelated work can + * still get a thread while the lock is contended. `bystanderLatency` models that: a + * task that never touches the lock, submitted to the same dispatcher. + */ +class LockDesignComparisonBenchmark { + private val relays = 191 + private val dispatcherThreads = 64 + private val durationMs = 3000L + + /** One relay's slice of a subscription's state — all real fields are relay-keyed. */ + private class PerRelay { + val lock = ReentrantLock() + var status: Int = 0 + var filters: List? = null + var lastKnown: List? = null + } + + private class Striped { + val perRelay = ConcurrentHashMap() + + inline fun withLock( + relay: Int, + block: (PerRelay) -> R, + ): R { + val s = perRelay.computeIfAbsent(relay) { PerRelay() } + s.lock.lock() + try { + return block(s) + } finally { + s.lock.unlock() + } + } + } + + private class PerSubBlocking { + val lock = ReentrantLock() + val status = HashMap() + val filters = HashMap>() + } + + private class PerSubMutex { + val mutex = Mutex() + val status = HashMap() + val filters = HashMap>() + } + + private fun report( + name: String, + subs: Int, + ops: Long, + bystanderSamples: List, + ) { + val sorted = bystanderSamples.sorted() + val p50 = sorted[sorted.size / 2] / 1000.0 + val p99 = sorted[(sorted.size * 99) / 100] / 1000.0 + val max = sorted.last() / 1000.0 + println( + "%-18s subs=%-3d ops/s=%,10d bystander p50=%8.1fus p99=%9.1fus max=%9.1fus (n=%d)".format( + name, + subs, + ops * 1000 / durationMs, + p50, + p99, + max, + sorted.size, + ), + ) + } + + @Test + fun compareDesigns() { + if (System.getenv("PROD_RELAY_BENCH") == null && System.getProperty("prodRelayBench") == null) { + println("compareDesigns skipped. Run with -PprodRelayBench=1 to enable.") + return + } + println("relays=$relays dispatcherThreads=$dispatcherThreads window=${durationMs}ms") + println("bystander = a task that NEVER touches the lock, on the same dispatcher.") + println("Lower bystander latency = the lock is not stealing dispatcher threads.\n") + for (subs in listOf(1, 4, 16)) { + runPerSubBlocking(subs) + runPerSubMutex(subs) + runStriped(subs) + runRealStriped(subs) + println() + } + } + + private fun runPerSubBlocking(subs: Int) = + runBlocking { + @Suppress("DEPRECATION") + val dispatcher = Dispatchers.IO.limitedParallelism(dispatcherThreads) + val states = Array(subs) { PerSubBlocking() } + val stop = AtomicBoolean(false) + val ops = AtomicLong(0) + val bystander = ArrayList() + val jobs = + (0 until relays).map { relay -> + launch(dispatcher) { + var n = 0L + while (!stop.get()) { + val s = states[relay % subs] + s.lock.lock() + try { + s.status[relay] = 1 + s.filters[relay] = SAMPLE + s.status[relay] + } finally { + s.lock.unlock() + } + n++ + // Models `for (message in incomingMessages)`: every real + // iteration suspends, letting the dispatcher multiplex. + kotlinx.coroutines.yield() + } + ops.addAndGet(n) + } + } + val by = + launch(dispatcher) { + while (!stop.get()) { + val t = System.nanoTime() + kotlinx.coroutines.yield() + bystander.add(System.nanoTime() - t) + } + } + Thread.sleep(durationMs) + stop.set(true) + jobs.forEach { it.join() } + by.join() + report("PER_SUB_BLOCKING", subs, ops.get(), bystander.ifEmpty { listOf(0L) }) + } + + private fun runPerSubMutex(subs: Int) = + runBlocking { + @Suppress("DEPRECATION") + val dispatcher = Dispatchers.IO.limitedParallelism(dispatcherThreads) + val states = Array(subs) { PerSubMutex() } + val stop = AtomicBoolean(false) + val ops = AtomicLong(0) + val bystander = ArrayList() + val jobs = + (0 until relays).map { relay -> + launch(dispatcher) { + var n = 0L + while (!stop.get()) { + val s = states[relay % subs] + s.mutex.withLock { + s.status[relay] = 1 + s.filters[relay] = SAMPLE + s.status[relay] + } + n++ + // Models `for (message in incomingMessages)`: every real + // iteration suspends, letting the dispatcher multiplex. + kotlinx.coroutines.yield() + } + ops.addAndGet(n) + } + } + val by = + launch(dispatcher) { + while (!stop.get()) { + val t = System.nanoTime() + kotlinx.coroutines.yield() + bystander.add(System.nanoTime() - t) + } + } + Thread.sleep(durationMs) + stop.set(true) + jobs.forEach { it.join() } + by.join() + report("PER_SUB_MUTEX", subs, ops.get(), bystander.ifEmpty { listOf(0L) }) + } + + private fun runStriped(subs: Int) = + runBlocking { + @Suppress("DEPRECATION") + val dispatcher = Dispatchers.IO.limitedParallelism(dispatcherThreads) + val states = Array(subs) { Striped() } + val stop = AtomicBoolean(false) + val ops = AtomicLong(0) + val bystander = ArrayList() + val jobs = + (0 until relays).map { relay -> + launch(dispatcher) { + var n = 0L + while (!stop.get()) { + states[relay % subs].withLock(relay) { s -> + s.status = 1 + s.filters = SAMPLE + s.lastKnown = SAMPLE + } + n++ + // Models `for (message in incomingMessages)`: every real + // iteration suspends, letting the dispatcher multiplex. + kotlinx.coroutines.yield() + } + ops.addAndGet(n) + } + } + val by = + launch(dispatcher) { + while (!stop.get()) { + val t = System.nanoTime() + kotlinx.coroutines.yield() + bystander.add(System.nanoTime() - t) + } + } + Thread.sleep(durationMs) + stop.set(true) + jobs.forEach { it.join() } + by.join() + report("STRIPED", subs, ops.get(), bystander.ifEmpty { listOf(0L) }) + } + + /** + * Same topology, but driving the REAL shipped [RequestSubscriptionState] (striped per + * relay) instead of a prototype — so the measured win is a property of the code that + * ships, not of this file. + */ + private fun runRealStriped(subs: Int) = + runBlocking { + @Suppress("DEPRECATION") + val dispatcher = Dispatchers.IO.limitedParallelism(dispatcherThreads) + val states = Array(subs) { RequestSubscriptionState() } + val stop = AtomicBoolean(false) + val ops = AtomicLong(0) + val bystander = ArrayList() + val filters = listOf(Filter(kinds = listOf(1))) + val jobs = + (0 until relays).map { relay -> + launch(dispatcher) { + var n = 0L + val state = states[relay % subs] + while (!stop.get()) { + state.withLock(relay) { + state.onNewEvent(relay) + state.currentState(relay) + state.onOpenReq(relay, filters) + state.lastKnownFilterStates(relay) + } + n++ + kotlinx.coroutines.yield() + } + ops.addAndGet(n) + } + } + val by = + launch(dispatcher) { + while (!stop.get()) { + val t = System.nanoTime() + kotlinx.coroutines.yield() + bystander.add(System.nanoTime() - t) + } + } + Thread.sleep(durationMs) + stop.set(true) + jobs.forEach { it.join() } + by.join() + report("REAL_STRIPED", subs, ops.get(), bystander.ifEmpty { listOf(0L) }) + } + + companion object { + private val SAMPLE = listOf("kinds:1", "authors:abc") + } +} diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/SpinLockConvoyBenchmark.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/SpinLockConvoyBenchmark.kt new file mode 100644 index 0000000000..c8b17e6d85 --- /dev/null +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/SpinLockConvoyBenchmark.kt @@ -0,0 +1,242 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay.prodbench + +import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.RequestSubscriptionState +import org.junit.Assert.assertTrue +import org.junit.Test +import java.util.concurrent.CountDownLatch +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicLong +import kotlin.concurrent.thread + +/** + * Reproduces the production ANR seen on a Pixel 8 (2026-08-03, anr_2026-08-03-12-55-26-256): + * 191 live relay sockets feed EVENT frames for the same handful of subscription ids, so + * dozens of DefaultDispatcher workers pile onto ONE [RequestSubscriptionState] busy-wait + * lock. In that trace 51 of 52 runnable workers sat in the inlined spin loop while the + * single lock holder was parked in `WaitingForGcToComplete`. + * + * Two things are measured: + * - [contendedThroughput]: aggregate critical sections/s as the contender count grows past + * the core count (the "negative scaling" the 2026-07-02 plan measured with 4 feeders, + * re-run at production fan-out). + * - [victimLatencyUnderSpin]: what an UNRELATED thread (stand-in for the UI thread) sees + * while the spinners run — this is the ANR mechanism, not the lock throughput. + */ +class SpinLockConvoyBenchmark { + private val cores = Runtime.getRuntime().availableProcessors() + + /** + * Every waiter targets ONE reference on purpose. The lock is striped per relay, so + * spreading threads over distinct relays would put them on different stripes and + * this benchmark would measure nothing — the point here is the primitive's behaviour + * when contention DOES land on a single stripe. + */ + private val hotRelay = 0 + + /** Mirrors the real critical section: a handful of map reads/writes, no I/O. */ + private fun criticalSection( + state: RequestSubscriptionState, + relay: Int, + ) { + state.onNewEvent(relay) + state.currentState(relay) + state.lastKnownFilterStates(relay) + } + + @Test + fun contendedThroughput() { + if (System.getenv("PROD_RELAY_BENCH") == null && System.getProperty("prodRelayBench") == null) { + println("contendedThroughput skipped. Run with -PprodRelayBench=1 to enable.") + return + } + println("cores = $cores") + println("threads | ops/s | vs 1 thread") + var baseline = 0.0 + for (threads in listOf(1, 2, 4, 8, 16, 32, 52)) { + val state = RequestSubscriptionState() + val stop = AtomicBoolean(false) + val ops = AtomicLong(0) + val start = CountDownLatch(1) + val workers = + (0 until threads).map { id -> + thread { + start.await() + var local = 0L + while (!stop.get()) { + state.withLock(hotRelay) { criticalSection(state, hotRelay) } + local++ + } + ops.addAndGet(local) + } + } + val t0 = System.nanoTime() + start.countDown() + Thread.sleep(2000) + stop.set(true) + workers.forEach { it.join() } + val secs = (System.nanoTime() - t0) / 1e9 + val rate = ops.get() / secs + if (threads == 1) baseline = rate + println("%7d | %9.0f | %.2fx".format(threads, rate, rate / baseline)) + } + } + + /** + * REGRESSION GUARD: contended waiters on [RequestSubscriptionState.withLock] must PARK, + * never busy-wait. Fails if the lock is ever turned back into a spin lock. + * + * This is the signature that identified the production ANR: in + * `anr_2026-08-03-12-55-26-256`, 37 of 52 runnable workers sat at one obfuscated line of + * `PoolRequests.onIncomingMessage` and 12 more at one line of `syncState$lambda$0` — all + * `state=R`, `sCount=0`, burning 596% CPU while the lock holder was stuck in + * `WaitingForGcToComplete`. With a spin lock this test observes ~40/40 waiters RUNNABLE; + * with a parking lock it observes 0. + */ + @Test + fun contendedWaitersParkInsteadOfSpinning() { + val spinners = 40 + val state = RequestSubscriptionState() + val stop = AtomicBoolean(false) + val start = CountDownLatch(1) + + val holder = + thread(name = "holder") { + start.await() + while (!stop.get()) { + state.withLock(hotRelay) { + val t = System.nanoTime() + while (System.nanoTime() - t < 5_000_000) { /* hold 5ms */ } + } + } + } + val threads = + (0 until spinners).map { id -> + thread(name = "spinner-$id") { + start.await() + while (!stop.get()) { + state.withLock(hotRelay) { criticalSection(state, hotRelay) } + } + } + } + start.countDown() + Thread.sleep(300) + + // Sample every spinner's stack, exactly like an ANR dump would. + val points = HashMap() + var runnable = 0 + threads.forEach { t -> + if (t.state == Thread.State.RUNNABLE) runnable++ + val top = t.stackTrace.firstOrNull { it.className.contains("SpinLockConvoyBenchmark") || it.className.contains("RequestSubscriptionState") } + if (top != null) { + val key = "${top.className.substringAfterLast('.')}.${top.methodName}:${top.lineNumber}" + points[key] = (points[key] ?: 0) + 1 + } + } + stop.set(true) + threads.forEach { it.join() } + holder.join() + + println("sampled $spinners waiters: RUNNABLE=$runnable, distinct program points=${points.size}") + points.entries.sortedByDescending { it.value }.forEach { println(" ${it.value}x ${it.key}") } + + // A parked waiter reports WAITING, so the parking lock measures ~0 here while a + // spin lock measures ~100%. The line sits at 50% deliberately: it separates the + // two cases by a mile and leaves headroom on a loaded CI box, where a few waiters + // can legitimately be mid-acquire when we sample. + assertTrue( + "Expected contended waiters to park, but $runnable/$spinners were RUNNABLE — " + + "RequestSubscriptionState.withLock looks like it is busy-waiting again. " + + "See PlatformLock's kdoc: this is what caused anr_2026-08-03-12-55-26-256.", + runnable <= spinners / 2, + ) + } + + @Test + fun victimLatencyUnderSpin() { + if (System.getenv("PROD_RELAY_BENCH") == null && System.getProperty("prodRelayBench") == null) { + println("victimLatencyUnderSpin skipped. Run with -PprodRelayBench=1 to enable.") + return + } + // One holder that briefly stalls inside the critical section (in production: a GC + // pause, or simply being descheduled). Everyone else spins. + val spinners = 52 + val state = RequestSubscriptionState() + val stop = AtomicBoolean(false) + + fun measureVictim(label: String) { + // The "UI thread": allocates and measures its own scheduling latency. + val samples = ArrayList() + repeat(200) { + val t = System.nanoTime() + // trivial allocation work, like a Compose semantics traversal step + val junk = ArrayList(64) + repeat(64) { i -> junk.add("node$i") } + Thread.yield() + samples.add(System.nanoTime() - t) + } + samples.sort() + println( + "%-22s p50=%6.0fus p90=%7.0fus p99=%8.0fus max=%8.0fus".format( + label, + samples[samples.size / 2] / 1000.0, + samples[(samples.size * 90) / 100] / 1000.0, + samples[(samples.size * 99) / 100] / 1000.0, + samples.last() / 1000.0, + ), + ) + } + + measureVictim("idle (no spinners)") + + val start = CountDownLatch(1) + val holder = + thread { + start.await() + while (!stop.get()) { + state.withLock(hotRelay) { + // Simulate the holder losing its core / waiting on GC mid-section. + val t = System.nanoTime() + while (System.nanoTime() - t < 2_000_000) { /* 2ms stall */ } + } + Thread.sleep(1) + } + } + val threads = + (0 until spinners).map { id -> + thread { + start.await() + while (!stop.get()) { + state.withLock(hotRelay) { criticalSection(state, hotRelay) } + } + } + } + start.countDown() + Thread.sleep(500) + measureVictim("$spinners spinners") + stop.set(true) + threads.forEach { it.join() } + holder.join() + + measureVictim("after (no spinners)") + } +} diff --git a/quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/PlatformLock.linux.kt b/quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/PlatformLock.linux.kt new file mode 100644 index 0000000000..5f3c4fdf46 --- /dev/null +++ b/quartz/src/linuxMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/PlatformLock.linux.kt @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.utils.concurrent + +import kotlin.concurrent.atomics.AtomicBoolean +import kotlin.concurrent.atomics.ExperimentalAtomicApi + +// Linux/JVM-less target: Kotlin/Native's stdlib ships no parking lock and there is +// no Foundation here, so this keeps a test-and-test-and-set spin. Correct but not +// scalable. Acceptable ONLY because linuxX64 is a build/CI target for quartz, not a +// host for the many-relay client workload whose contention motivated the parking +// actuals on jvmAndroid and Apple. If that ever changes, swap in a pthread mutex. +@OptIn(ExperimentalAtomicApi::class) +actual class PlatformLock { + private val held = AtomicBoolean(false) + + actual fun lock() { + while (held.exchange(true)) { + while (held.load()) { } + } + } + + actual fun unlock() { + held.store(false) + } +} diff --git a/quartz/src/nativeMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.native.kt b/quartz/src/nativeMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.native.kt index de4ee540d8..8805e8d5a7 100644 --- a/quartz/src/nativeMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.native.kt +++ b/quartz/src/nativeMain/kotlin/com/vitorpamplona/quartz/utils/concurrent/ConcurrentMap.native.kt @@ -74,6 +74,16 @@ actual class ConcurrentMap { } } + actual fun remove(key: K): V? { + while (true) { + val cur = ref.load() + val old = cur[key] ?: return null + val copy = HashMap(cur) + copy.remove(key) + if (ref.compareAndSet(cur, copy)) return old + } + } + actual fun size(): Int = ref.load().size actual fun snapshot(): Map = HashMap(ref.load()) From a110ce0a30e797145fb100de2ff81c630ff4e708 Mon Sep 17 00:00:00 2001 From: mstrofnone Date: Tue, 4 Aug 2026 10:51:16 +1000 Subject: [PATCH 003/132] ci: publish linux-arm64 desktop, amy, and geode release assets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Amethyst v1.13.1 (and every prior release) shipped only linux-x64 desktop binaries — .deb, .rpm, .AppImage, .flatpak, .tar.gz. Same for the amy CLI and geode relay. Users on aarch64 hardware (Pinebook, Ampere Altra, Raspberry Pi 4/5, AWS Graviton, arm64 servers, arm64 Chromebooks running crostini, etc.) can't install any of them. This teaches the release matrix about arm64: - Add `ubuntu-24.04-arm` legs to build-desktop, build-cli, and build-geode. This is a standard, free public-repo GitHub-hosted runner (4 CPU / 16 GB / 14 GB SSD / arm64) since early 2025. No cross-compilation: jpackage / jlink / Compose Multiplatform 1.11 all produce host-native artifacts. - Fetch the matching `appimagetool-.AppImage` from the same 1.9.0 release with an arch-specific SHA256 pin. `APPIMAGETOOL_URL` becomes `APPIMAGETOOL_VERSION` + per-arch SHA256 env vars. - Parametrize the portable tarball/zip filename by `matrix.arch` (`amethyst-desktop--linux-arm64.tar.gz` is now produced). - Parametrize the Flatpak bundle filename and rewrite the manifest's `GST_PLUGIN_SYSTEM_PATH` from `x86_64-linux-gnu` to `aarch64-linux-gnu` on the arm64 leg. The Flathub-submission manifest (`desktopApp/packaging/flatpak/flathub/`) still gates on `only-arches: x86_64` — flipping that to include aarch64 is a follow-up once a Flathub aarch64 build has been validated end-to-end. - Make the `createReleaseAppImage` gradle task pick its host arch from `System.getProperty("os.arch")` (amd64/x86_64 → `x86_64`, aarch64/ arm64 → `aarch64`). Same task, same command, drives both legs. - Fix `desktopApp/packaging/appimage/AppRun` to compute the multiarch library path from `uname -m` at launch time instead of hard-coding `x86_64-linux-gnu`. One script works in both AppImages on the target machine. - Extend the desktop smoke test to run the release .deb build + launch probe on `ubuntu-24.04-arm` too, so arch-specific ProGuard/jlink breakage (missing native lib, arch-specific reflection root) is caught at PR time. - Update BUILDING.md and scripts/asset-name.sh docs with the new arm64 asset names. Follow-up assets published for the next tag push (v1.13.2+): - amethyst-desktop--linux-arm64.{deb,rpm,AppImage,flatpak,tar.gz} - amy--linux-arm64.{deb,rpm,tar.gz} - geode--linux-arm64.{deb,rpm,tar.gz} Verification (local, before submitting): - `python3 -c 'import yaml; yaml.safe_load(open(".github/workflows/create-release.yml"))'` — parses clean - `bash -n scripts/asset-name.sh desktopApp/packaging/appimage/AppRun` — parses clean - `actionlint` — reports only pre-existing shellcheck style hints; no new errors - Confirmed `linuxdeploy-aarch64.AppImage` and `appimagetool-aarch64.AppImage` exist under the same pinned release tags used for x86_64; SHA256 recorded from a fresh download. Not addressed (out of scope for this PR): - Homebrew / winget bump workflows (`bump-homebrew*.yml`, `bump-winget.yml`) — those consume the assets by name; the new arm64 filenames don't change any x86_64 name they already reference. - Android arm64 continues to ship as before (already had it). --- .github/workflows/create-release.yml | 68 +++++++++++++++++------- .github/workflows/smoke-test-desktop.yml | 13 ++++- BUILDING.md | 13 +++-- desktopApp/build.gradle.kts | 20 +++++-- desktopApp/packaging/appimage/AppRun | 5 +- desktopApp/packaging/flatpak/README.md | 8 ++- scripts/asset-name.sh | 11 ++++ 7 files changed, 106 insertions(+), 32 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index ce5d002955..f87c0556f4 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -26,8 +26,12 @@ env: # bundle deps — that fights jpackage's self-contained JRE (libjvm.so has # $ORIGIN RPATH so ldd can't resolve it standalone). appimagetool only # embeds the AppDir as-is, which is what we actually want. - APPIMAGETOOL_URL: https://github.com/AppImage/appimagetool/releases/download/1.9.0/appimagetool-x86_64.AppImage - APPIMAGETOOL_SHA256: 46fdd785094c7f6e545b61afcfb0f3d98d8eab243f644b4b17698c01d06083d1 + # + # Both arch binaries come from the same appimagetool release so their SHA256 + # values move in lockstep on version bumps. + APPIMAGETOOL_VERSION: '1.9.0' + APPIMAGETOOL_SHA256_X86_64: 46fdd785094c7f6e545b61afcfb0f3d98d8eab243f644b4b17698c01d06083d1 + APPIMAGETOOL_SHA256_AARCH64: 04f45ea45b5aa07bb2b071aed9dbf7a5185d3953b11b47358c1311f11ea94a96 jobs: # --------------------------------------------------------------------------- @@ -38,11 +42,17 @@ jobs: strategy: fail-fast: false matrix: + # Linux legs run on x64 and arm64 GitHub-hosted runners (the + # ubuntu-24.04-arm label is a standard free public-repo runner as of + # early 2025). jpackage / jlink / Compose Multiplatform 1.11 all + # produce host-native artifacts — no cross-compilation needed. include: - - { os: macos-14, arch: arm64, family: macos, tasks: "packageReleaseDmg" } - - { os: windows-latest, arch: x64, family: windows, tasks: "packageReleaseMsi createReleaseDistributable" } - - { os: ubuntu-latest, arch: x64, family: linux, tasks: "packageReleaseDeb packageReleaseRpm" } - - { os: ubuntu-latest, arch: x64, family: linux-portable, tasks: "createReleaseAppImage createReleaseDistributable" } + - { os: macos-14, arch: arm64, family: macos, tasks: "packageReleaseDmg" } + - { os: windows-latest, arch: x64, family: windows, tasks: "packageReleaseMsi createReleaseDistributable" } + - { os: ubuntu-latest, arch: x64, family: linux, tasks: "packageReleaseDeb packageReleaseRpm" } + - { os: ubuntu-24.04-arm, arch: arm64, family: linux, tasks: "packageReleaseDeb packageReleaseRpm" } + - { os: ubuntu-latest, arch: x64, family: linux-portable, tasks: "createReleaseAppImage createReleaseDistributable" } + - { os: ubuntu-24.04-arm, arch: arm64, family: linux-portable, tasks: "createReleaseAppImage createReleaseDistributable" } runs-on: ${{ matrix.os }} timeout-minutes: 60 # linux-portable leg also downloads the freedesktop runtime + builds the Flatpak bundle defaults: @@ -93,13 +103,21 @@ jobs: set -euo pipefail # appimagetool 1.9.0 validates the .desktop file via desktop-file-validate. sudo apt-get update && sudo apt-get install -y desktop-file-utils - curl -fsSL --retry 3 "$APPIMAGETOOL_URL" -o desktopApp/packaging/appimage/appimagetool-x86_64.AppImage - actual=$(sha256sum desktopApp/packaging/appimage/appimagetool-x86_64.AppImage | awk '{print $1}') - if [[ "$actual" != "$APPIMAGETOOL_SHA256" ]]; then - echo "::error::appimagetool SHA256 mismatch. Expected $APPIMAGETOOL_SHA256, got $actual" + # Map runner arch → upstream AppImage suffix (x86_64 / aarch64). + case "${{ matrix.arch }}" in + x64) TOOL_ARCH=x86_64 ; EXPECTED_SHA="$APPIMAGETOOL_SHA256_X86_64" ;; + arm64) TOOL_ARCH=aarch64; EXPECTED_SHA="$APPIMAGETOOL_SHA256_AARCH64" ;; + *) echo "::error::unsupported arch for AppImage: ${{ matrix.arch }}"; exit 1 ;; + esac + URL="https://github.com/AppImage/appimagetool/releases/download/${APPIMAGETOOL_VERSION}/appimagetool-${TOOL_ARCH}.AppImage" + DEST="desktopApp/packaging/appimage/appimagetool-${TOOL_ARCH}.AppImage" + curl -fsSL --retry 3 "$URL" -o "$DEST" + actual=$(sha256sum "$DEST" | awk '{print $1}') + if [[ "$actual" != "$EXPECTED_SHA" ]]; then + echo "::error::appimagetool SHA256 mismatch for $TOOL_ARCH. Expected $EXPECTED_SHA, got $actual" exit 1 fi - chmod +x desktopApp/packaging/appimage/appimagetool-x86_64.AppImage + chmod +x "$DEST" # Flatpak tooling + the freedesktop runtime/sdk the manifest pins # (runtime-version is greped from the manifest so this never drifts). @@ -208,12 +226,13 @@ jobs: run: | set -euo pipefail VER="${{ steps.ver.outputs.version }}" + ARCH="${{ matrix.arch }}" APP="desktopApp/build/compose/binaries/main-release/app" mkdir -p desktopApp/build/portable if [[ "${{ matrix.family }}" == "windows" ]]; then - ( cd "$APP" && 7z a -tzip "../../../../portable/amethyst-desktop-${VER}-windows-x64.zip" Amethyst/ ) + ( cd "$APP" && 7z a -tzip "../../../../portable/amethyst-desktop-${VER}-windows-${ARCH}.zip" Amethyst/ ) else - ( cd "$APP" && tar czf "../../../../portable/amethyst-desktop-${VER}-linux-x64.tar.gz" Amethyst/ ) + ( cd "$APP" && tar czf "../../../../portable/amethyst-desktop-${VER}-linux-${ARCH}.tar.gz" Amethyst/ ) fi # Flatpak bundle: wraps the same createReleaseDistributable tree the @@ -230,6 +249,17 @@ jobs: PKG="desktopApp/packaging/flatpak" APP_ID="com.vitorpamplona.amethyst.Desktop" OUT="desktopApp/build/flatpak" + # AppImage-style arch names for the bundle filename. + case "${{ matrix.arch }}" in + x64) BUNDLE_ARCH=x86_64 ; GST_TRIPLET=x86_64-linux-gnu ;; + arm64) BUNDLE_ARCH=aarch64 ; GST_TRIPLET=aarch64-linux-gnu ;; + *) echo "::error::unsupported arch for Flatpak: ${{ matrix.arch }}"; exit 1 ;; + esac + # Rewrite the arch-specific GStreamer plugin path in the manifest + # (checked-in default is x86_64-linux-gnu). Idempotent — the sed only + # matches the original triplet. + sed -i "s|/usr/lib/x86_64-linux-gnu/gstreamer-1.0|/usr/lib/${GST_TRIPLET}/gstreamer-1.0|g" \ + "${PKG}/${APP_ID}.yml" # Inject the AppStream entry for this build (the checked-in # metainfo deliberately carries none — CI is the source of truth). sed -i "s||\n |" \ @@ -241,7 +271,7 @@ jobs: "${OUT}/build-dir" \ "${PKG}/${APP_ID}.yml" flatpak build-bundle "${OUT}/repo" \ - "${OUT}/Amethyst-${VER}-x86_64.flatpak" \ + "${OUT}/Amethyst-${VER}-${BUNDLE_ARCH}.flatpak" \ "$APP_ID" \ --runtime-repo=https://dl.flathub.org/repo/flathub.flatpakrepo ls -la "$OUT" @@ -325,8 +355,9 @@ jobs: fail-fast: false matrix: include: - - { os: macos-14, arch: arm64, family: macos, tasks: "amyImage" } - - { os: ubuntu-latest, arch: x64, family: linux, tasks: "amyImage jpackageDeb jpackageRpm" } + - { os: macos-14, arch: arm64, family: macos, tasks: "amyImage" } + - { os: ubuntu-latest, arch: x64, family: linux, tasks: "amyImage jpackageDeb jpackageRpm" } + - { os: ubuntu-24.04-arm, arch: arm64, family: linux, tasks: "amyImage jpackageDeb jpackageRpm" } runs-on: ${{ matrix.os }} timeout-minutes: 45 # macOS leg also codesigns + notarizes the jlink image defaults: @@ -574,8 +605,9 @@ jobs: fail-fast: false matrix: include: - - { os: macos-14, arch: arm64, family: macos, tasks: "geodeImage" } - - { os: ubuntu-latest, arch: x64, family: linux, tasks: "geodeImage jpackageDeb jpackageRpm" } + - { os: macos-14, arch: arm64, family: macos, tasks: "geodeImage" } + - { os: ubuntu-latest, arch: x64, family: linux, tasks: "geodeImage jpackageDeb jpackageRpm" } + - { os: ubuntu-24.04-arm, arch: arm64, family: linux, tasks: "geodeImage jpackageDeb jpackageRpm" } runs-on: ${{ matrix.os }} timeout-minutes: 45 # macOS leg also codesigns + notarizes the jlink image defaults: diff --git a/.github/workflows/smoke-test-desktop.yml b/.github/workflows/smoke-test-desktop.yml index 3dafc575dd..6d8159ab0e 100644 --- a/.github/workflows/smoke-test-desktop.yml +++ b/.github/workflows/smoke-test-desktop.yml @@ -49,9 +49,17 @@ jobs: # package, installs it, and verifies the process stays alive for 10s. # Catches ProGuard stripping (JNI, reflection), missing jlink modules # (java.management, java.prefs), and native lib bundling issues. + # + # Runs on both x64 and arm64 hosted runners so release-time arm64 breakage + # (e.g. ProGuard rules missing an arch-specific reflection root) is caught + # at PR time instead of on the tag build. # ------------------------------------------------------------------------- release-deb-launch: - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm] + runs-on: ${{ matrix.os }} timeout-minutes: 45 steps: - name: Checkout code @@ -139,5 +147,6 @@ jobs: if: always() uses: actions/upload-artifact@v7 with: - name: Release DEB (smoke-tested) + # Artifact names must be unique across a run — disambiguate per arch. + name: Release DEB (smoke-tested, ${{ matrix.os }}) path: desktopApp/build/compose/binaries/main-release/deb/*.deb diff --git a/BUILDING.md b/BUILDING.md index 5a04173603..b931926c00 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -57,9 +57,12 @@ Install appimagetool locally (CI fetches its own — SHA-verified): # Debian/Ubuntu — appimagetool calls desktop-file-validate on the .desktop entry sudo apt-get install -y desktop-file-utils -curl -fsSL -o desktopApp/packaging/appimage/appimagetool-x86_64.AppImage \ - https://github.com/AppImage/appimagetool/releases/download/1.9.0/appimagetool-x86_64.AppImage -chmod +x desktopApp/packaging/appimage/appimagetool-x86_64.AppImage +# createReleaseAppImage picks appimagetool-.AppImage matching the JVM's +# os.arch — fetch the one for your host (x86_64 on Intel/AMD, aarch64 on ARM). +ARCH="$(uname -m)" +curl -fsSL -o "desktopApp/packaging/appimage/appimagetool-${ARCH}.AppImage" \ + "https://github.com/AppImage/appimagetool/releases/download/1.9.0/appimagetool-${ARCH}.AppImage" +chmod +x "desktopApp/packaging/appimage/appimagetool-${ARCH}.AppImage" ``` --- @@ -110,8 +113,8 @@ are **not** required to build Amethyst from the committed sources. | Windows MSI | `./gradlew :desktopApp:packageReleaseMsi` | `desktopApp/build/compose/binaries/main-release/msi/Amethyst-*.msi` | | Linux `.deb` | `./gradlew :desktopApp:packageReleaseDeb` | `desktopApp/build/compose/binaries/main-release/deb/amethyst_*.deb` | | Linux `.rpm` | `./gradlew :desktopApp:packageReleaseRpm` | `desktopApp/build/compose/binaries/main-release/rpm/amethyst-*.rpm` | -| Linux AppImage | `./gradlew :desktopApp:createReleaseAppImage` | `desktopApp/build/appimage/Amethyst-*-x86_64.AppImage` | -| Linux Flatpak | `flatpak-builder` over `createReleaseDistributable` output — see [`desktopApp/packaging/flatpak/README.md`](desktopApp/packaging/flatpak/README.md) | `desktopApp/build/flatpak/Amethyst-*-x86_64.flatpak` (CI) | +| Linux AppImage | `./gradlew :desktopApp:createReleaseAppImage` | `desktopApp/build/appimage/Amethyst-*-.AppImage` (x86_64 or aarch64, from host) | +| Linux Flatpak | `flatpak-builder` over `createReleaseDistributable` output — see [`desktopApp/packaging/flatpak/README.md`](desktopApp/packaging/flatpak/README.md) | `desktopApp/build/flatpak/Amethyst-*-.flatpak` (CI; x86_64 or aarch64) | | Windows `.zip` portable | See below (inline `7z`) | — | | Linux `.tar.gz` portable | See below (inline `tar`) | — | diff --git a/desktopApp/build.gradle.kts b/desktopApp/build.gradle.kts index 10ca2b3d48..fa41a3e276 100644 --- a/desktopApp/build.gradle.kts +++ b/desktopApp/build.gradle.kts @@ -246,18 +246,30 @@ compose.desktop { // - amethyst.png 512x512 icon // // appimagetool binary is fetched by CI (SHA-verified) into -// desktopApp/packaging/appimage/ as appimagetool-x86_64.AppImage. +// desktopApp/packaging/appimage/ as appimagetool-.AppImage. +// The arch is selected at task-execution time from the host JVM's os.arch, so +// the same task builds the correct AppImage on both x86_64 and aarch64 hosts. // BUILDING.md documents local-dev fetch. val createReleaseAppImage by tasks.registering(Exec::class) { group = "compose desktop" description = "Package createReleaseDistributable output into a Linux AppImage via appimagetool." dependsOn("createReleaseDistributable") + // AppImage's ARCH env accepts the Linux kernel arch names: x86_64 / aarch64 + // / armhf / i686. jpackage produces host-native binaries, so mirror the + // host JVM arch. Do not read the property inside doFirst — it needs to be + // resolved at configuration time so outputs.file() below is stable. + val hostArch = when (val a = System.getProperty("os.arch").lowercase()) { + "amd64", "x86_64" -> "x86_64" + "aarch64", "arm64" -> "aarch64" + else -> a + } + val distDir = layout.buildDirectory.dir("compose/binaries/main-release/app/Amethyst") val appDir = layout.buildDirectory.dir("appimage/Amethyst.AppDir") - val outFile = layout.buildDirectory.file("appimage/Amethyst-$appVersion-x86_64.AppImage") + val outFile = layout.buildDirectory.file("appimage/Amethyst-$appVersion-$hostArch.AppImage") val toolRoot = layout.projectDirectory.dir("packaging/appimage") - val appimagetool = toolRoot.file("appimagetool-x86_64.AppImage") + val appimagetool = toolRoot.file("appimagetool-$hostArch.AppImage") inputs.dir(distDir) inputs.dir(toolRoot) @@ -292,7 +304,7 @@ val createReleaseAppImage by tasks.registering(Exec::class) { appDir.get().asFile.absolutePath, outFile.get().asFile.absolutePath, ) - environment("ARCH", "x86_64") + environment("ARCH", hostArch) // Bypass FUSE requirement on CI runners (ubuntu-latest lacks libfuse.so.2). // AppImage standard env var: extracts + runs without mounting. environment("APPIMAGE_EXTRACT_AND_RUN", "1") diff --git a/desktopApp/packaging/appimage/AppRun b/desktopApp/packaging/appimage/AppRun index c42b59c001..7e208a8d54 100755 --- a/desktopApp/packaging/appimage/AppRun +++ b/desktopApp/packaging/appimage/AppRun @@ -7,7 +7,10 @@ # (Equivalent packages on Fedora/Arch.) set -eu HERE="$(dirname "$(readlink -f "${0}")")" -export LD_LIBRARY_PATH="${HERE}/usr/lib:${HERE}/usr/lib/x86_64-linux-gnu:${LD_LIBRARY_PATH:-}" +# Multiarch lib path is set by the host, not baked at build time — same +# AppRun works in both x86_64 and aarch64 AppImages. +GNU_TRIPLET="$(uname -m)-linux-gnu" +export LD_LIBRARY_PATH="${HERE}/usr/lib:${HERE}/usr/lib/${GNU_TRIPLET}:${LD_LIBRARY_PATH:-}" export PATH="${HERE}/usr/bin:${PATH}" export APPDIR="${HERE}" exec "${HERE}/usr/bin/Amethyst" "$@" diff --git a/desktopApp/packaging/flatpak/README.md b/desktopApp/packaging/flatpak/README.md index 552fba7e42..c6c1f7cd84 100644 --- a/desktopApp/packaging/flatpak/README.md +++ b/desktopApp/packaging/flatpak/README.md @@ -28,8 +28,12 @@ used two ways: manifest (archive source pinned to the release tarball URL + sha256, with `x-checker-data` so Flathub's update bot bumps it), its own metainfo (carries the permanent `` history Flathub requires), desktop - entry, icon, and `flathub.json` (`only-arches: x86_64` — we publish no - aarch64 tarball, and jpackage can't cross-compile one) + entry, icon, and `flathub.json` — currently gated to `only-arches: + x86_64` so the Flathub build machinery never tries the aarch64 tarball + before we've validated it end-to-end on Flathub's aarch64 builders. GitHub + releases already ship aarch64 flatpak bundles (built from the same source + tree on `ubuntu-24.04-arm`); flipping `only-arches` to include `aarch64` + is the follow-up once we've smoke-tested a Flathub aarch64 build. ## Local build diff --git a/scripts/asset-name.sh b/scripts/asset-name.sh index d419b10c0d..ee7b2bda98 100755 --- a/scripts/asset-name.sh +++ b/scripts/asset-name.sh @@ -29,15 +29,26 @@ # amethyst-desktop-1.08.0-linux-x64.AppImage # amethyst-desktop-1.08.0-linux-x64.flatpak # amethyst-desktop-1.08.0-linux-x64.tar.gz +# amethyst-desktop-1.08.0-linux-arm64.deb +# amethyst-desktop-1.08.0-linux-arm64.rpm +# amethyst-desktop-1.08.0-linux-arm64.AppImage +# amethyst-desktop-1.08.0-linux-arm64.flatpak +# amethyst-desktop-1.08.0-linux-arm64.tar.gz # amy-1.08.0-macos-arm64.tar.gz # amy-1.08.0-macos-x64.tar.gz # amy-1.08.0-linux-x64.tar.gz # amy-1.08.0-linux-x64.deb # amy-1.08.0-linux-x64.rpm +# amy-1.08.0-linux-arm64.tar.gz +# amy-1.08.0-linux-arm64.deb +# amy-1.08.0-linux-arm64.rpm # geode-1.08.0-macos-arm64.tar.gz # geode-1.08.0-linux-x64.tar.gz # geode-1.08.0-linux-x64.deb # geode-1.08.0-linux-x64.rpm +# geode-1.08.0-linux-arm64.tar.gz +# geode-1.08.0-linux-arm64.deb +# geode-1.08.0-linux-arm64.rpm # # Two assets break the family/arch shape on purpose: the no-JRE jar bundles for # Homebrew-core are pure JVM bytecode (no bundled runtime), so a single From 7b29f57526fba1529c66a18a44af78fcdf2cd52f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 02:55:14 +0000 Subject: [PATCH 004/132] docs: add event-store-semantics skill (IEventStore/SQLite store contract) Documents the store's observable behavior as named rules (STORE-F/W/D/C/S/N) so external IEventStore implementations can review pin bumps and annotate divergences against a stated contract instead of reverse-engineering QueryBuilder. Requested by the vespa-eventstore consumer. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ADBfjWhsXyHZb7ea2eeBhJ --- .claude/skills/event-store-semantics/SKILL.md | 325 ++++++++++++++++++ 1 file changed, 325 insertions(+) create mode 100644 .claude/skills/event-store-semantics/SKILL.md diff --git a/.claude/skills/event-store-semantics/SKILL.md b/.claude/skills/event-store-semantics/SKILL.md new file mode 100644 index 0000000000..1c99e335a9 --- /dev/null +++ b/.claude/skills/event-store-semantics/SKILL.md @@ -0,0 +1,325 @@ +--- +name: event-store-semantics +description: The authoritative behavioral contract of Quartz's event stores — `IEventStore` and its reference SQLite implementation (`nip01Core/store/sqlite/`). Use when implementing or asserting parity with a Quartz event store (external engines like Vespa, the filesystem store, geode), answering filter-semantics questions (since/until inclusivity, tag OR/AND, multi-filter limits, ordering tiebreaks), or working on the write-path rules for replaceable/addressable supersession, NIP-09 deletions, NIP-40 expiration, NIP-62 vanish, NIP-45 counts, or NIP-50 search inside the store. Every behavior has a named rule id (STORE-Fxx/Wxx/Dxx/Sxx/Cxx) so downstream implementations can annotate divergences precisely. +--- + +# Event Store Semantics — the `IEventStore` / SQLite-store contract + +The SQLite `EventStore` (`quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/`) +is the de-facto **reference implementation** of what a Quartz event store must do. Other +implementations — the in-repo filesystem store (`nip01Core/store/fs/`, held to parity by +`quartz/src/jvmTest/.../store/fs/FsParityTest.kt`) and external engines (e.g. a Vespa-backed +store) — reimplement its *observable behavior* and assert parity in CI. This skill states that +behavior as **named, numbered decisions** so a parity divergence becomes a lookup, not an +archaeology session through `QueryBuilder`/`MergeQueryExecutor`. + +Every rule below was verified against the code as of this skill's last update. When you change +store behavior, **update the rule here in the same PR** and add a line to the +[Semantics changelog](#semantics-changelog) — downstream implementations pin Quartz by commit and +review pin bumps against this file. + +## Key files + +| Concern | File | +|---|---| +| Public contract (KDoc is normative) | `nip01Core/store/IEventStore.kt` | +| High-level store (owns pool + planner) | `sqlite/EventStore.kt`, `sqlite/SQLiteEventStore.kt` | +| Filter → SQL, ordering, limits, counts | `sqlite/QueryBuilder.kt` | +| k-way merge fast path (feed shapes) | `sqlite/MergeQueryExecutor.kt` | +| Schema, tag hashing, immutability | `sqlite/EventIndexesModule.kt`, `sqlite/TagNameValueHasher.kt`, `sqlite/SeedModule.kt` | +| Replaceable / addressable supersession | `sqlite/ReplaceableModule.kt`, `sqlite/AddressableModule.kt` | +| NIP-09 / NIP-40 / NIP-62 / ephemeral | `sqlite/DeletionRequestModule.kt`, `sqlite/ExpirationModule.kt`, `sqlite/RightToVanishModule.kt`, `sqlite/EphemeralModule.kt` | +| NIP-50 FTS | `sqlite/FullTextSearchModule.kt` (see also the `searchable-events` skill) | +| Index/feature toggles | `sqlite/IndexingStrategy.kt` (client default) and geode's `RelayIndexingStrategy.kt` (relay preset) | +| Operational README | `sqlite/README.md` (concurrency, pragmas, maintenance) | + +Executable spec: the test suites in +`quartz/src/commonTest/.../store/sqlite/` (`BasicTest`, `ReplaceableTest`, `AddressableTest`, +`DeletionTest`, `ExpirationTest`, `RightToVanishTest`, `SearchTest`, `SearchRelevanceOrderTest`, +`MergeQueryCorrectnessTest`, `TagMergeCorrectnessTest`, `QueryAssemblerTest`, +`SnapshotIdsForNegentropyTest`, `FilterMatcherTest`, …). If a rule here ever contradicts a test, +the test wins — and this file has a bug to fix. + +## Kind classes (used throughout) + +- **Replaceable**: kind `0`, kind `3`, and `10000 ≤ kind < 20000`. +- **Ephemeral**: `20000 ≤ kind < 30000`. +- **Addressable**: `30000 ≤ kind < 40000`. +- Everything else is a regular event. + +--- + +## Filter matching (STORE-F) + +**STORE-F01 — `since`/`until` are both inclusive.** `since` compiles to +`created_at >= ?`, `until` to `created_at <= ?` (`QueryBuilder` uses +`greaterThanOrEquals`/`lessThanOrEquals` everywhere). An event with +`created_at == since == until` matches. + +**STORE-F02 — `ids` and `authors` are exact-match only.** They compile to `=`/`IN` against the +full 64-char hex columns. **NIP-01 prefix matching is NOT supported** anywhere in the store. +(`Filter`'s constructor logs an error for non-64-char ids/authors but still sends them; they +simply never match.) + +**STORE-F03 — tag filter combination.** Within one tag name, values are **OR** +(`tag_hash IN (…)`). Across different tag names in the same filter, conditions are **AND** +(each extra name becomes another `event_tags` self-join). `tagsAll` (NIP-91 `&x` syntax) demands +**every listed value** be present on the event — one join + equality per value — and composes by +AND with any plain `tags` in the same filter. + +**STORE-F04 — only single-letter tag names are indexed (by default).** +`DefaultIndexingStrategy.shouldIndex` indexes a tag iff `tag.size >= 2 && tag[0].length == 1`. +A filter on a multi-letter tag name (`#title`, `#alt`) matches **nothing** in the SQLite store. +Deployments can widen `shouldIndex`, but the stock contract is single-letter-only. + +**STORE-F05 — `d` is special-cased out of the tag index.** `#d` values are matched against the +`event_headers.d_tag` column, not `event_tags` (`Filter.toFilterWithDTags()`). Consequences: +`#d` works on addressable events (which populate `d_tag`); when all `kinds` are addressable the +query adds `kind >= 30000 AND kind < 40000` to pin the addressable index. **Only use `#d` via +plain `tags`.** A `#d` under `tagsAll` is handled inconsistently: on the simple (no other +tags/search) path it degrades to OR semantics (`toFilterWithDTags` folds it into `dTags`), and +when `tags["d"]` is also present it is dropped entirely; on the tag-join path it is ignored. +(An event has one d-tag, so AND-across-values could never match anyway.) + +**STORE-F06 — tag and author matching in the tag path is hash-based.** `event_tags` stores a +64-bit MurmurHash3 of `(tag name, value)` keyed by a per-database random seed (`SeedModule`, +`TagNameValueHasher`); the p/e/a-owner columns are hashes too. There is **no post-verification** +of hash matches, so a hash collision would return a false positive. Probability is negligible in +practice but nonzero — a parity harness comparing against an exact-match engine should know this +is the one place the reference can (theoretically) over-match. + +**STORE-F07 — multiple filters are a union with dedup; `limit` is per-filter.** Each filter +becomes its own row-id subquery with its **own** `ORDER BY … LIMIT`; branches are combined with +SQL `UNION` (dedup by row). There is **no global limit** — a 3-filter query with limits +10/20/30 can return up to 60 events, presented in one merged `created_at DESC` ordering. NIP-45 +counts and negentropy snapshots dedup the same way (`SELECT DISTINCT` / `UNION`). + +**STORE-F08 — result ordering.** Non-search queries order `created_at DESC`. The `id ASC` +tiebreak on equal `created_at` is applied **only when +`IndexingStrategy.useAndIndexIdOnOrderBy = true`** — which is `false` in the client default +**and** in geode's relay preset. So by default, same-second ordering is unspecified (SQLite +returns them in storage order). Any newest-N is valid; a parity suite must not assert +same-`created_at` order unless it configures the flag. One extra caveat with the flag ON: the +`MergeQueryExecutor` tag-stream path still yields same-second ties in rowid order (its cursors +run off `event_tags`, which has no id column) — a valid newest-N that may differ byte-for-byte +from the single-SQL ordering. + +**STORE-F09 — the merge fast path returns the same *set*.** Single-filter queries of the shape +"authors (+kinds) + limit" or "one `#x` IN-list (+kinds) + limit" (≤2048 streams) route through +`MergeQueryExecutor`, a k-way newest-first merge over per-(kind,author) / per-(tag-value,kind) +index cursors with dedup by id on the tag shape. This is an optimization, not a semantics +change — `MergeQueryCorrectnessTest`/`TagMergeCorrectnessTest` assert set-equality with the +single-SQL plan (ordering caveat per STORE-F08). + +**STORE-F10 — empty filter.** `query(Filter())` / `count(Filter())` match **everything** +(`Filter.isEmpty()` → the "everything" query). `delete(Filter())` is deliberately asymmetric: +it deletes **nothing** and returns 0, so a stray empty filter can't wipe the store (documented +on `QueryBuilder.delete`). + +**STORE-F11 — empty lists (`kinds = emptyList()` etc.) are a client error with inconsistent +handling; don't rely on either outcome.** On the single-filter simple path an empty list +renders as `1 = 0` → matches nothing. But `Filter.isEmpty()` treats empty lists the same as +`null`, so on the multi-filter union path such a filter contributes no subquery — and a list of +*only* empty-list filters degrades to the match-everything query. Known quirk; treat +empty-list filters as invalid input rather than replicating this shape. + +**STORE-F12 — `limit` edge cases.** `limit = 0` compiles to `LIMIT 0` → zero rows. +`limit = null` means unbounded. Negative limits are not defended against (don't send them). + +**STORE-F13 — the in-memory matcher is a separate (simpler) implementation.** +`Filter.match(event)` (`FilterMatcher`) is used for live-stream matching, not storage queries; +it checks ids/authors/kinds/tags/tagsAll/since/until but not `search` or `limit`. Parity work +targets the SQL semantics above, not `FilterMatcher`. + +--- + +## Write path (STORE-W) + +Inserts run every module in one transaction: header+tags → NIP-09 side effects → expiration +row → FTS row → vanish side effects. A trigger `RAISE(ABORT, …)` rejects the whole row with the +messages quoted below (they surface as the NIP-01 `OK false` reason). + +**STORE-W01 — replaceable supersession.** Unique index on `(kind, pubkey)` for replaceable +kinds. A `BEFORE INSERT` trigger deletes any stored version that is *older* — meaning +`created_at` smaller, **or equal `created_at` with lexicographically larger id** (NIP-01 +lowest-id-wins). Inserting a version that is *not* newer under that ordering leaves the stored +row in place and fails the unique index → rejected (`UNIQUE constraint failed`). Net contract: +exactly one version stored; newest wins; ties broken by lowest id; older re-inserts blocked. + +**STORE-W02 — addressable supersession.** Same as W01 with unique index +`(kind, pubkey, d_tag)` over `30000 ≤ kind < 40000`. Nuance: `d_tag` is populated from the +*parsed* event class (`AddressableEvent.dTag()`); an addressable-range kind whose class doesn't +parse as `AddressableEvent` stores `d_tag NULL`, and SQLite treats NULLs as distinct in unique +indexes — such events don't supersede each other. An event with no `d` tag parses as `dTag() = ""` +(empty string), which *does* dedupe normally. + +**STORE-W03 — ephemeral events are never stored but are acked as accepted.** +`insert()` returns silently and `batchInsert` reports `Accepted` for `20000 ≤ kind < 30000` +without writing (the live relay stream still broadcasts them). A DB-level backstop trigger +(`blocked: cannot store ephemeral events`) rejects any that sneak past the app-level check. + +**STORE-W04 — expired events are rejected at insert.** App-level check +(`event.isExpired()`) plus a trigger on the expiration-row insert +(`blocked: this event is expired` when `expiration <= unixepoch()`). Single-event `insert` +**throws**; `batchInsert` returns `Rejected`. + +**STORE-W05 — expiry is enforced at insert and by sweep, NOT at query time.** Events with a +future `expiration` store a row in `event_expirations`. Nothing filters them out of queries +after the timestamp passes: **a query between expiry and the next `deleteExpiredEvents()` sweep +returns the expired event.** Operators run the sweep periodically (README recommends ~15 min). +Re-inserting an already-expired event after the sweep is rejected per W04. + +**STORE-W06 — GiftWrap ownership is the recipient.** For kind 1059 the store computes +`pubkey_owner_hash` from the `p`-tag recipient (falling back to the random signer key if +absent). All owner-scoped machinery — NIP-09 re-insert blocking, NIP-62 vanish deletion and +blocking — operates on that owner hash, so **a user's deletions/vanish remove giftwraps +addressed to them**, even though the wrap's `pubkey` is a one-time key. (Consequently GiftWraps +are also excluded from `authorsMissingOutbox()`.) + +**STORE-W07 — immutability.** `event_headers`/`event_tags` rows are never updated +(`BEFORE UPDATE` triggers abort). All supersession is delete + insert; `event_tags`, +`event_expirations`, `event_vanish`, and the FTS row follow the header by +`ON DELETE CASCADE` / trigger. + +**STORE-W08 — batch insert.** One outer transaction, one SAVEPOINT per row: a bad row rolls +back alone and reports `Rejected(reason)`; the rest commit. If the **outer commit** fails, every +entry is treated as `Rejected` (the `IEventStore.batchInsert` contract). Outcomes are returned +in input order; OK frames pair by event id, not order. + +--- + +## Deletion lifecycle — NIP-09 / NIP-62 (STORE-D) + +**STORE-D01 — delete by id.** A kind-5's `e` tags delete stored events with those ids **whose +owner is the kind-5's author** (`pubkey_owner_hash` match — recipient for giftwraps per W06). +The id path has **no timestamp condition**: it deletes the target regardless of the relative +`created_at` values. + +**STORE-D02 — delete by address.** A kind-5's `a` tags delete events at that +`(kind, pubkey, d_tag)` coordinate with `created_at <= deletion.created_at` — **inclusive**; a +version newer than the deletion survives. Only coordinates whose pubkey equals the kind-5's +author are honored. Replaceable coordinates (`kind:pubkey:` with no d-tag) get the same +`created_at <=` treatment against `(kind, pubkey)`. + +**STORE-D03 — cross-author kind-5s are stored but inert.** A deletion naming someone else's +events is inserted like any regular event (it may be useful to other relays/clients) but its +delete pass removes zero rows and creates no blocking. + +**STORE-D04 — re-insert blocking.** A `BEFORE INSERT` trigger rejects +(`blocked: a deletion event exists`) any event whose id (`e`-hash) **or** address (`a`-hash) is +named by a stored kind-5 from the same owner with `deletion.created_at >= event.created_at`. +Note the asymmetry with D01: a *backdated* id-deletion (older `created_at` than its target) +still deletes on arrival, but would not block a later re-insert. + +**STORE-D05 — a kind-5 CAN delete another kind-5, and doing so un-blocks its targets.** +Nothing excludes kind 5 from the id path (D01). Deleting a deletion removes its tombstone rows +from `event_tags`, so events it had deleted become re-insertable. **Status: known quirk, not a +considered decision.** NIP-09 leaves it open; at least one external implementation +(vespa-eventstore) deliberately diverges by treating deletion-of-a-deletion as a no-op, which is +the safer reading (tombstones shouldn't be revocable). If you change this, update this rule and +the changelog — parity suites key off it. + +**STORE-D06 — NIP-62 vanish is relay-scoped.** A kind-62 only cascades when +`shouldVanishFrom(relay)` — its `relay` tags name this store's `relay` URL or `ALL_RELAYS`. +(A store constructed with `relay = null` matches only `ALL_RELAYS` requests.) Out-of-scope +vanish events are stored as regular events with no side effects. + +**STORE-D07 — vanish scope and horizon.** An in-scope vanish deletes every event whose +**owner** (W06) is the vanishing pubkey with `created_at < vanish.created_at` (strict — the +vanish event itself survives), and blocks inserts of owned events with +`created_at <= vanish.created_at` (`blocked: a request to vanish event exists`; note blocking is +inclusive where deletion is strict). Newer vanish requests supersede older ones per pubkey +(unique on `pubkey_hash`). + +**STORE-D08 — manual deletes.** `delete(id)` removes one row unconditionally (no blocking +created). `delete(filter)` deletes matching rows honoring per-filter limits, with the F10 +empty-filter no-op guard. Neither creates re-insert blocking — only stored kind-5/kind-62 +events do that. + +--- + +## NIP-45 count (STORE-C) + +**STORE-C01 — count = size of the deduped match set, honoring per-filter limits.** Single +filter: `COUNT(*)` over that filter's row-id subquery (including its `LIMIT`, so +`count(Filter(kinds=…, limit=10))` is at most 10). Multiple filters: branches are `UNION`ed +(dedup) **before** counting — an event matching several filters counts once. FTS-off + search +term → 0 (F-series search rules apply). + +--- + +## NIP-50 search inside the store (STORE-S) + +The indexing surface (which kinds are searchable, what text they contribute) is the +`searchable-events` skill; these rules are the store's query-side contract. + +**STORE-S01 — extension stripping at the store boundary.** Every filter-accepting method runs +`strippingSearchExtensions()`: NIP-50 `key:value` tokens (`include:spam`, `domain:…`, …) are +removed before FTS. Unsupported extensions are **ignored, never matched as literal text and +never match-nothing** — an extensions-only search collapses to an unconstrained query. Stores +that *do* implement extensions receive the raw string through the relay layer and parse it with +`nip50Search.SearchQuery.parse` (see the `IEventStore` KDoc). + +**STORE-S02 — relevance ordering.** Search results order by FTS5 `bm25` rank (best match +first), with `created_at DESC` only as tiebreak; the `LIMIT` keeps the most *relevant* N, not +the newest N. A multi-filter REQ is relevance-ordered only when **every** filter carries a +search term (best/min rank per event across branches); mixing search and non-search filters +falls back to `created_at DESC`. + +**STORE-S03 — search combines by AND with the structural parts** (ids/authors/kinds/tags/ +since/until) of the same filter — an FTS `MATCH` join on top of the normal conditions. + +**STORE-S04 — search grammar is SQLite FTS5 `MATCH`.** The raw (post-strip) string is passed to +FTS5, so implicit-AND terms, `"phrase queries"`, `OR`, and `prefix*` follow FTS5 semantics. +Tokenization details live in `FullTextSearchModule` (see `searchable-events`). + +**STORE-S05 — FTS off.** With `IndexingStrategy.indexFullTextSearch = false`: a filter with a +non-empty search term matches **nothing** (query/count/delete alike); an empty-string search +imposes no constraint. Everything else is unchanged. + +**STORE-S06 — deferred FTS.** Relays may set `deferFullTextSearchIndexing = true` (geode does): +tokenization moves off the insert path to a watermark-driven catch-up +(`needsFtsCatchUp`/`ftsCatchUp`), and search queries drain the backlog first — so NIP-50 +results are exactly as fresh as the synchronous path. + +--- + +## Negentropy / NIP-77 (STORE-N) + +**STORE-N01 —** `snapshotIdsForNegentropy(filters)` returns `(created_at, id)` pairs under the +**same filter semantics as `query`** (per-filter limits included, multi-filter dedup), order +unspecified (negentropy re-sorts). `maxEntries` returns up to `maxEntries + 1` as an overflow +sentinel. `liveNegentropySnapshot` serves full-corpus NEG-OPENs from an in-memory index when +`maintainLiveNegentropyIndex` is on; the delta plumbing in `SQLiteEventStore` keeps it exact +across replaceable displacement, kind-5s, and vanish (invalidate-and-rebuild for the +non-itemizable cases). + +--- + +## Configuration presets + +- **Client default** (`DefaultIndexingStrategy()`): FTS on (synchronous), optional indexes off, + `useAndIndexIdOnOrderBy` off, no live negentropy index. +- **Relay preset** (geode's `relayIndexingStrategy()`): adds created_at-alone, pubkey-alone and + tag+kind+pubkey indexes, defers FTS, maintains the live negentropy index — still leaves + `useAndIndexIdOnOrderBy` off. +- Flag-gated indexes are runtime config, not schema: flipping one on an existing DB builds the + index on next open (`ensureOptionalIndexes`), no migration. + +## For parity implementers + +- Treat the rule ids above as the vocabulary for divergence notes + (e.g. "diverges from STORE-D05: we no-op deletion-of-a-deletion"). +- The commonTest suites are the executable spec; `FsParityTest` shows the in-repo pattern for + holding a second engine to it. +- Remember F06 (hash-based tag matching) and F08 (unordered same-second ties by default) when + diffing results byte-for-byte — both are places where a "divergence" may be the reference's + own slack, not your bug. + +## Semantics changelog + +Add one line per behavior change, newest first: `YYYY-MM-DD — what changed`. + +- 2026-08-04 (baseline) — rules F01–F13, W01–W08, D01–D08, C01, S01–S06, N01 written from the + code at the time this skill was introduced. Changes before this date are not itemized; + archaeology starts at `git log` on `nip01Core/store/`. From 2b2e47256b0760ad21719e8cfe1a592521f94d9e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 03:04:17 +0000 Subject: [PATCH 005/132] docs: add nip85-trusted-assertions and searchable-events skills Completes the store-implementer skill set requested by the vespa-eventstore consumer: the NIP-85 trust-assertion model (kind map, tag vocabulary, value semantics, authorization conventions) and the NIP-50 indexing surface (the SearchableEvent contract plus an exhaustive kind -> indexableContent table external search engines can diff at version bumps). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01ADBfjWhsXyHZb7ea2eeBhJ --- .claude/core-skills-plan.md | 27 ++ .../skills/nip85-trusted-assertions/SKILL.md | 232 ++++++++++++++++++ .claude/skills/searchable-events/SKILL.md | 117 +++++++++ .../references/searchable-kinds.md | 166 +++++++++++++ 4 files changed, 542 insertions(+) create mode 100644 .claude/skills/nip85-trusted-assertions/SKILL.md create mode 100644 .claude/skills/searchable-events/SKILL.md create mode 100644 .claude/skills/searchable-events/references/searchable-kinds.md diff --git a/.claude/core-skills-plan.md b/.claude/core-skills-plan.md index 1af746f6dd..35eafb646f 100644 --- a/.claude/core-skills-plan.md +++ b/.claude/core-skills-plan.md @@ -85,3 +85,30 @@ skills verified clean): `ParseReturn.entity` (the `Nip19Parser.Return.*` sealed class never existed); Event Store section corrected from "Android only" to commonMain/all platforms with the real `store.sqlite.EventStore` import and suspend generic `query`. + +## Phase 4 (2026-08): Store-implementer skills (external consumer request) + +Three skills added at the request of an external Quartz consumer +(vespa-eventstore — a server-side `IEventStore` on Vespa that asserts result +parity against the SQLite store in CI). All three document the +**store/relay-implementer's perspective**, which `quartz-integration` and +`nostr-expert` (client-side) did not cover. Requirements doc: the skill-requests +file reviewed 2026-08-04; the requester's items #4 (storage-lifecycle-nips) was +folded into `event-store-semantics` per their own recommendation, and #5 +(relay-server/geode policies) was declined as not currently needed. + +- **`event-store-semantics/`** — the `IEventStore`/SQLite-store behavioral + contract as named rules (STORE-Fxx/Wxx/Dxx/Cxx/Sxx/Nxx) with a semantics + changelog for pin-bump review. Written from `QueryBuilder`, + `MergeQueryExecutor`, the seven `*Module.kt` files, and `IEventStore` KDoc. +- **`nip85-trusted-assertions/`** — the NIP-85 model (10040/30382/30383/30384/ + 30385), full tag vocabulary with value semantics, authorization conventions, + worked JSON examples, stability notes. +- **`searchable-events/`** — the `SearchableEvent` contract + maintenance + mandate, with `references/searchable-kinds.md` holding the exhaustive + kind → class → `indexableContent()` table (126 classes / 129 kinds) that + external search engines diff at version bumps. + +Follow-ups suggested but not implemented: a shared JSON test-vector corpus for +filter semantics (testFixtures both the SQLite tests and external parity suites +could run), and a snapshot test pinning the searchable-kind set. diff --git a/.claude/skills/nip85-trusted-assertions/SKILL.md b/.claude/skills/nip85-trusted-assertions/SKILL.md new file mode 100644 index 0000000000..b0b17f22ea --- /dev/null +++ b/.claude/skills/nip85-trusted-assertions/SKILL.md @@ -0,0 +1,232 @@ +--- +name: nip85-trusted-assertions +description: The NIP-85 trusted-assertions model in Quartz (`nip85TrustedAssertions/`) — kind 10040 trust-provider lists, kind 30382 contact cards / user assertions, 30383 event assertions, 30384 addressable assertions, 30385 external-id assertions. Use when building or parsing these events, working with the typed tags (RankTag, HopsTag, FollowerCountTag, ServiceProviderTag/ServiceType, …), wiring a consumer that resolves a 10040 provider entry to the 30382s it signs, ranking on assertion values, or touching the GrapeRank publisher, contact-card nicknames, or the trust projection of an external store. +--- + +# NIP-85 Trusted Assertions — the Quartz model + +Package: `quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip85TrustedAssertions/`. +NIP-85 is still an evolving spec; **this package is the operative definition** of what +Amethyst-family software writes and reads. This skill states the model (who signs what about +whom), the exact kind/d-tag/tag vocabulary, and what consumers may — and may not — assume. + +## The model in one paragraph + +An **assertion is signed by the asserting party** (a trust provider service, or the user +themself) **about a subject named in the d-tag**. All assertion kinds are addressable, so +"latest card by provider P about subject S" is just the addressable coordinate +`(kind, P, S)` and supersession is standard NIP-01 latest-wins. Discovery is the observer's +**kind 10040 list**: each entry says *"for metric M on kind K, I trust provider P — fetch +their assertions at relay R"*. Quartz enforces none of this cryptographically beyond normal +event signatures; the 10040→assertion link is **consumer-side convention** (see +"Authorization" below). + +## Kind map + +| Kind | Class | Kind class | d-tag = the subject | Content | +|---|---|---|---|---| +| 10040 | `list/TrustProviderListEvent` | replaceable | *(none — always `""`)* | NIP-44 private provider entries (optional) | +| 30382 | `users/ContactCardEvent` | addressable | **target user's pubkey** (hex) | NIP-44 private tags (petname/summary/emoji) | +| 30383 | `events/EventAssertionEvent` | addressable | **target event id** (hex) | `""` | +| 30384 | `addressables/AddressableAssertionEvent` | addressable | **target coordinate** `kind:pubkey:dtag` | `""` | +| 30385 | `externalIds/ExternalIdAssertionEvent` | addressable | **external identifier** (e.g. `isbn:978-0-13-468599-1`) | `""` | + +Addresses: `ContactCardEvent.createAddress(owner, target)` → `Address(30382, owner, target)` +(owner = signer, target = subject). `TrustProviderListEvent.createAddress(pubKey)` uses +`FIXED_D_TAG = ""`. `AssertionEventTest.eventKindsAreCorrect` pins all five numbers. + +`ContactCardEvent` is also a `SearchableEvent` — it indexes only the **public** petname/summary +tags plus topics; the encrypted card content is intentionally never indexed. + +## The 10040 provider entry (`ServiceProviderTag` / `ServiceType`) + +There is **no fixed tag name**: `tag[0]` *is* the service string. + +```json +["30382:rank", "", "wss://nip85.brainstorm.world"] +``` + +- `ServiceType(kind, type)` parses/renders `":"` — kind must be an int, the first + `:` splits, colons in the remainder stay in `type`. `ServiceType.isOfKind` is the + allocation-free prefix check. +- `ServiceProviderTag.parse` requires ≥3 elements, non-empty service, 64-char pubkey + (length-only check), and a **normalizable relay URL** (`RelayUrlNormalizer.normalizeOrNull`) — + entries failing any check are silently dropped, which is what keeps foreign tags like + `["client","nostria"]` out (regression-tested in `ServiceTypeParserTest`). +- Entries may be **public** (tag array) or **private** (NIP-44 content); `create`/`add` take + `isPrivate`. `remove` always needs decryption and strips from both sides by parsed-value + equality. +- `object ProviderTypes` (`list/tags/ServiceType.kt`) enumerates the *known* service types — + `30382:rank`, `30382:followers`, `30382:first_created_at`, per-metric `30383:*`/`30384:*`/ + `30385:*`, etc. It is an **open vocabulary**: real 10040s in the wild (see the fiatjaf → + brainstorm fixture in `commonTest/.../nip85TrustedAssertions/ServiceParser.kt`) carry types + Quartz doesn't enumerate (`30382:personalizedGrapeRank_influence`, `30382:hops`, + `30382:verifiedFollowersCount`, …). Parse any `kind:type`; special-case only what you rank on. + +## Authorization — what a consumer may assume + +- **A 30382 (or 30383/…) is meaningful to an observer only if its author is listed in the + observer's 10040 for a matching service type.** Quartz does not enforce this; the consuming + code does. The in-repo pattern is `commons/.../model/nip85TrustedAssertions/UserCardsCache.kt`: + `rankFlow(trustProviderList)` picks the received card whose **author pubkey equals the + provider entry's pubkey** and reads `rank()` from it. Assertions from unlisted signers are + simply ignored for trust purposes (they may still be stored; dropping them — as an external + store's orphan sweep does — is a legitimate storage policy, not a protocol rule). +- What an entry authorizes is scoped by its `ServiceType`: `30382:rank` authorizes that + provider's user-rank cards, nothing else. Amethyst models this as one provider slot per + metric (`liveUserRankProvider`, `liveUserFollowerCount` in + `amethyst/.../model/trustedAssertions/TrustProviderListState.kt`). +- **Multi-provider combination is unprescribed.** When two listed providers assert different + ranks, there is no spec'd merge; Amethyst avoids the question by selecting one provider per + metric slot. Consumers choose their own policy — document it. +- The relay URL in the entry is a **fetch hint, and it is honored**: + `amethyst/.../UserCardsSubAssembler.kt` subscribes for cards at the provider's declared relay + (`kinds=[30382], authors=[provider], #d=[targets]`). + +### The dual use of kind 30382 + +The same kind serves two roles, distinguished **by author**: + +1. **Provider WoT cards** — signed by a trust provider; public metric tags (`rank`, + `followers`, `hops`, …); this is what 10040 discovery points at. +2. **The account's own contact cards (nicknames, NIP-81-style)** — signed by the account, + one per target user. The petname, summary, and their NIP-30 emoji mappings **always live in + the NIP-44 encrypted content, never in public tags** (`ContactCardEvent.build`/ + `updatePetNameAndSummary` strip stray public copies; asserted by `ContactCardPetNameTest`). + `commons/.../ContactCardsState.kt` keys everything on `author == account` and ignores + provider cards. + +## Tag vocabulary and value semantics + +All tag classes share one shape: `TAG_NAME` + `parse(tag)` (null on wrong name/empty/non-numeric +value — a bad tag is *dropped*, never an error) + `assemble(value)` → `[name, value.toString()]`. +**A missing tag means "unknown" (`null` accessor), never zero.** There is deliberately no range +validation (rank isn't clamped, hours aren't checked against 0–23, counts may be negative) — +consumers must defend. + +**On 30382** (`users/tags/`, accessors on `ContactCardEvent` and as `TagArray` extensions in +`users/TagArrayExt.kt` so they also work on decrypted private arrays): + +| Tag name | Accessor | Type | Semantics | +|---|---|---|---| +| `rank` | `rank()` | Int | Provider-relative score; higher is better. GrapeRank publishes `round(score × 100)` (so 0–100 in practice), but nothing enforces a scale — treat it as comparable only *within one provider*. | +| `followers` | `followerCount()` | Int | Follower count as the provider computes it (cumulative, provider-defined). | +| `hops` | `hops()` | Int | Shortest follow-path length **from the observer the provider computed for** to the subject (1 = directly followed). Mirrors Brainstorm GrapeRank's `hops`. The only tag with KDoc. | +| `first_created_at` | `firstCreatedAt()` | Long | Unix seconds of subject's earliest known event. | +| `post_cnt` / `reply_cnt` / `reactions_cnt` | `postCount()` etc. | Int | Activity counts. | +| `zap_amt_recd` / `zap_amt_sent` | `zapAmountReceived()`/`…Sent()` | Long | Sats. | +| `zap_cnt_recd` / `zap_cnt_sent` | `zapCountReceived()`/`…Sent()` | Int | Counts. | +| `zap_avg_amt_day_recd` / `zap_avg_amt_day_sent` | `zapAvgAmountDay…()` | Long | Sats/day averages. | +| `reports_cnt_recd` / `reports_cnt_sent` | `reportsCount…()` | Int | NIP-56 report counts. | +| `t` (repeatable) | `topics()` | List\ | Subject's topics/interests. | +| `active_hours_start` / `active_hours_end` | `activeHours…()` | Int | Hour-of-day; **no timezone is specified in code** — treat as provider-defined (UTC in practice) and unclamped. | +| `petname` / `summary` | `petName()`/`summary()` | String | Nickname fields — conventionally private (see dual use above). | + +**On 30383/30384** (`tags/`, shared): `rank`, `comment_cnt`, `quote_cnt`, `repost_cnt`, +`reaction_cnt`, `zap_cnt` (Int) and `zap_amount` (Long, sats). +**On 30385**: only `rank`, `comment_cnt`, `reaction_cnt`. + +## Building and parsing (use the typed helpers, not raw `arrayOf`) + +```kotlin +// Provider list: declare a rank provider (this is what `amy graperank register` does) +val tag = ServiceProviderTag(ProviderTypes.rank, providerPubkeyHex, relayUrl) +val list = TrustProviderListEvent.create(tag, isPrivate = false, signer) +// or append to an existing one: +val updated = TrustProviderListEvent.add(existing, tag, isPrivate = false, signer) +val providers: List = updated.serviceProviders() // public +val private = updated.privateTags(signer)?.serviceProviders() // private side + +// Provider-style contact card (public metrics) — the GrapeRankPublisher pattern: +val card = ContactCardEvent.create( + targetUser = subjectPubkey, + signer = providerSigner, + publicInitializer = { + rank(87) + followers(1234) + hops(2) + }, +) +card.aboutUser() // d-tag → subject pubkey +card.rank() // 87 + +// Event assertion: unsigned template only (30383/84/85 have build(), no create()) +val template = EventAssertionEvent.build(targetEventId) { + rank(12) + reactionCount(40) + zapAmount(2100) +} +val signed = signer.sign(template) +``` + +## Worked end-to-end example + +Observer `O` trusts provider `P` for user ranks (kind 10040, replaceable, by `O`): + +```json +{ "kind": 10040, "pubkey": "", + "tags": [ + ["30382:rank", "

", "wss://nip85.brainstorm.world"], + ["30382:followers", "

", "wss://nip85.brainstorm.world"] + ], + "content": "" } +``` + +Provider `P` asserts about subject `S` (kind 30382, addressable at `30382:

:`): + +```json +{ "kind": 30382, "pubkey": "

", + "tags": [ + ["d", ""], + ["rank", "87"], ["followers", "1234"], ["hops", "2"] + ], + "content": "" } +``` + +`P` asserts about an event `E` (kind 30383, addressable at `30383:

:`): + +```json +{ "kind": 30383, "pubkey": "

", + "tags": [["d", ""], ["rank", "12"], ["reaction_cnt", "40"], ["zap_amount", "2100"]], + "content": "" } +``` + +Consumption chain: read `O`'s 10040 → entry matching `ServiceType(30382, "rank")` → subscribe +`{kinds:[30382], authors:["

"], "#d":["", …]}` at the hinted relay → newest card per +address wins → `rank()`. + +Literal fixtures: `quartz/src/commonTest/.../nip85TrustedAssertions/ServiceParser.kt` (a real +10040 — fiatjaf's, pointing at the Brainstorm provider) and `AssertionEventTest.kt` (all four +assertion kinds with every tag populated). + +## Freshness / supersession + +Assertions are addressable: **latest per `(kind, author, d-tag)` wins**; there is no expiry tag +convention and **no prescribed refresh cadence** — staleness policy is the consumer's. +Writers should avoid churn: `GrapeRankPublisher` re-signs a card only when +`(rank, followers, hops)` actually changed, and retracts with a NIP-09 kind-5 carrying the +card's `a`-tag (`30382::`). + +## Stability notes (as of 2026-08) + +- **Settled** (shipped consumers on both ends): the kind map; `ServiceProviderTag` entry shape; + `rank`/`followers`/`hops` on 30382; petname/summary-in-encrypted-content; 10040 relay-hint + consumption. +- **Written but lightly consumed** (parse, but gate ranking features carefully): the activity/ + zap/report count tags, `active_hours_*` (no timezone semantics), 30383/30384/30385 (builders + + tests exist; no in-repo publisher yet). +- **Known warts**: `ServiceProviderTag.assemble(id: ServiceProviderTag)` infers `Array` — + dead code, don't use it; `SummaryTag.assemble(ip:)`/`ActiveHours*Tag.assemble(count:)` params + are misnamed; the tests live under `commonTest/.../experimental/nip85TrustedAssertions/` + (stale path); `TrustProviderListEvent` extends the addressable base, so a stray on-wire `d` + tag is reflected by `dTag()` even though the convention is `""`. + +## Where it's consumed (reading list) + +- **Publisher**: `quartz/.../experimental/graperank/GrapeRankPublisher.kt` (canonical 30382 + writer), `cli/.../graperank/` (`amy graperank register|unregister|providers|publish`). +- **Client model**: `commons/.../model/nip85TrustedAssertions/` (`ContactCardsState`, + `UserCardsCache`, `ContactCardDecryptionCache`, `TrustProviderListDecryptionCache`), + `amethyst/.../model/trustedAssertions/TrustProviderListState.kt`. +- **Relay plumbing**: `commons/.../relayClient/assemblers/ContactCardFilters.kt`, + `amethyst/.../reqCommand/user/watchers/UserCardsSubAssembler.kt`. diff --git a/.claude/skills/searchable-events/SKILL.md b/.claude/skills/searchable-events/SKILL.md new file mode 100644 index 0000000000..aad4744f68 --- /dev/null +++ b/.claude/skills/searchable-events/SKILL.md @@ -0,0 +1,117 @@ +--- +name: searchable-events +description: The NIP-50 indexing surface of Quartz — the `SearchableEvent` interface, which event kinds are searchable, exactly what text each kind's `indexableContent()` contributes, how the SQLite/filesystem stores consume it, and the NIP-50 `SearchQuery` extension grammar plus `SearchRelayListEvent` (kind 10007). Use when making a kind searchable, changing what a kind indexes, diffing the searchable set at a Quartz version bump (external search engines mirror this table), debugging why an event is or isn't found by search, or working with search extensions (`include:spam`, `domain:`, …). +--- + +# Searchable Events — the NIP-50 indexing surface + +## The contract + +`quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchableEvent.kt`: + +```kotlin +interface SearchableEvent { + fun indexableContent(): String +} +``` + +One method; marker and extractor in one. An event kind is searchable **iff** its event class +implements this interface **and** the class is wired into `EventFactory` (the stores probe +searchability by kind through `EventFactory.create` — an unwired implementor is invisible). + +Rules every implementation follows (keep them when adding one): + +- **Plain text out.** Return the human-meaningful fields joined with `"\n"` (a handful of + metadata-ish kinds use `" "`); no markup stripping is performed — markdown/asciidoc content + goes in raw, JSON-content kinds (kind 0 metadata, marketplace stalls, channel info) **parse + first and join the extracted fields**, never the raw JSON. +- **Never throw, never null.** There is no defensive wrapper at any call site; a throw aborts + the insert transaction. Parsed-JSON implementations use `?.let { … } ?: ""`. +- **Only public data.** Encrypted content stays out (e.g. kind 30382 contact cards index only + the public petname/summary/topics, never the NIP-44 payload). +- Typical shapes: `content` alone (~33 kinds); `listOfNotNull(title(), content)`; + `listOfNotNull(title(), summary(), content)`; lists index `title() + description()`. + +## The full kind table + +**`references/searchable-kinds.md`** in this skill holds the authoritative table — every +implementor with its kind number, class, and the exact `indexableContent()` expression +(126 concrete classes / 129 kind values as of 2026-08). Diff that file at a version bump to +answer "did the searchable set or any kind's indexed text change?". + +Notables that surprise people: + +- **Kind 9735 (zap receipt) indexes the embedded zap request's content** + (`zapRequest?.content.orEmpty()`) — receipts are searchable by the zapper's comment. +- **Kind 0 / 31990** index many profile fields space-joined (name, about, nip05, lud16, + website, picture URL, …). +- **Kind 30063 is claimed twice** (`ReleaseArtifactSetEvent` in nip51Lists and the experimental + `SoftwareReleaseEvent`); `EventFactory` resolves 30063 to `ReleaseArtifactSetEvent`, so + `title()\ndescription()` is what actually gets indexed — `SoftwareReleaseEvent.indexableContent()` + is dead on the store path. +- Poll kinds (1068, 6969) append each option label on its own line. + +## MANDATORY maintenance when you touch this surface + +Adding `SearchableEvent` to a kind, removing it, or changing any `indexableContent()` body: + +1. **Update `references/searchable-kinds.md`** in the same PR (external search engines — e.g. + the Vespa-backed store's `SearchExtractors` — mirror this table at pin bumps; a silent + change ships them stale search results). +2. **Remember existing databases don't reindex themselves.** Old rows keep their old (or + missing) FTS text until `IEventStore.reindexFullTextSearch()` runs — the KDoc on that method + is the contract. App-side, schedule the resumable overload after shipping such a change. +3. New implementors must be **registered in `EventFactory`** or the reindex scan and kind + pre-filter (`FullTextSearchModule.isSearchableKind`) will never see them. + +Eligibility policy: a kind becomes searchable when it carries human-authored, human-meaningful +text (titles, bodies, names, descriptions). Pure-machine kinds (reactions, follow lists, zaps +minus their comment, relay lists) stay out to keep the index small. + +## How the stores consume it + +**SQLite** (`nip01Core/store/sqlite/FullTextSearchModule.kt`): +`CREATE VIRTUAL TABLE event_fts USING fts5(content, content='', contentless_delete=1)` — +contentless, `rowid` = `event_headers.row_id`, an `AFTER DELETE` trigger keeps it in sync. On +insert (when FTS is on and not deferred): `if (event is SearchableEvent)` → bind +`event.indexableContent()` — the only method ever called. Tokenization is entirely SQLite's +default FTS5 `unicode61`; queries are always a bound `event_fts MATCH ?` (never concatenated), +ordered by bm25 `rank` then `created_at DESC`. Query-side semantics (relevance ordering, +extension stripping, FTS-off behavior, deferred catch-up) are rules STORE-S01…S06 in the +`event-store-semantics` skill. + +**Filesystem store** (`jvmMain/.../store/fs/FsIndexer.kt` + `FsSearchTokenizer.kt`): tokenizes +`indexableContent()` itself, approximating `unicode61` (split on non-letter/digit, lowercase); +the same tokenizer runs on queries so drift cancels. + +## NIP-50 client side + +**`SearchQuery`** (`nip50Search/SearchQuery.kt`) — typed parse of the `search` filter string +into `terms` + `extensions`. A whitespace token is an extension iff it looks like +`lowercasekey:value` (the value not starting with `//`, so URLs stay free text); duplicate keys +keep the last; unknown extensions are preserved (`extension(key)`). Typed accessors: +`includeSpam`, `domain`, `language`, `sentiment`, `nsfw`. `stripExtensions()` / +`Filter.strippingSearchExtensions()` is the bridge the built-in stores use — unsupported +extensions are **ignored** (NIP-50), so an extensions-only search collapses to an unconstrained +query, never match-nothing. A server-side store that implements its own extensions +(`observer:`, `sort:rank`, …) receives the raw string (see the `IEventStore` KDoc) and should +parse with `SearchQuery.parse` so its syntax stays compatible with what clients send. + +**`SearchRelayListEvent`** — **kind 10007**, the user's search-relay list (NIP-51-style, public +tags + NIP-44 private tags; *not* a `SearchableEvent` itself). Client consumption: +`commons/.../actions/SearchActions.kt`, bootstrap defaults in +`commons/.../account/AccountBootstrapEvents.kt`. + +Don't confuse it with `commons/.../commons/search/SearchQuery.kt` — an app-level local-feed +query model (authors/kinds/hashtags/or-terms), unrelated to the NIP-50 wire string. + +## Tests (executable spec) + +- `commonTest/.../nip50Search/SearchQueryTest.kt` — the extension grammar, token by token. +- `commonTest/.../store/sqlite/SearchTest.kt` — per-kind indexing (kind 0 profile fields, + 40/41 channel JSON, 31924/30617), extension-token ignoring, reindex/resumable-reindex, + FTS cleanup on replaceable rotation. +- `commonTest/.../store/sqlite/SearchRelevanceOrderTest.kt` — bm25-before-recency ordering, + limit-after-score, multi-filter rank union. +- `commonTest/.../store/sqlite/NoFullTextSearchTest.kt` — FTS-off contract. +- `jvmTest/.../store/fs/FsSearchTest.kt` — tokenizer parity for the filesystem store. diff --git a/.claude/skills/searchable-events/references/searchable-kinds.md b/.claude/skills/searchable-events/references/searchable-kinds.md new file mode 100644 index 0000000000..ccae95ebfb --- /dev/null +++ b/.claude/skills/searchable-events/references/searchable-kinds.md @@ -0,0 +1,166 @@ +# Searchable kinds — the authoritative implementor table + +Every concrete `SearchableEvent` implementor in Quartz, with the exact `indexableContent()` +expression. **Update this file in the same PR as any change to the searchable set or to an +`indexableContent()` body** (see SKILL.md). Verified against the code 2026-08-04. + +Counts: 126 concrete classes covering 129 kind values (`GitStatusEvent` spans 4 kinds; +kind 30063 has a collision — see the footnote). File paths are under +`quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/`. + +Separator legend: **NL** = `joinToString("\n")`, **SP** = `joinToString(" ")`. + +| Kind | Class | Package | `indexableContent()` | +|---|---|---|---| +| 0 | MetadataEvent | nip01Core/metadata | `contactMetaData()?.let { listOfNotNull(it.name, it.displayName, it.about, it.nip05, it.lud06, it.lud16, it.website, it.picture, it.banner).joinToString(" ") } ?: ""` (SP) | +| 1 | TextNoteEvent | nip10Notes | `listOfNotNull(subject(), content)` NL | +| 9 | ChatEvent | nipC7Chats | `content` | +| 11 | ThreadEvent | nip7DThreads | `listOfNotNull(title(), content)` NL | +| 14 | ChatMessageEvent | nip17Dm/messages | `content` | +| 20 | PictureEvent | nip68Picture | `listOfNotNull(title(), content)` NL | +| 21 | VideoNormalEvent | nip71Video | inherited `RegularVideoEvent`: `listOfNotNull(title(), content)` NL | +| 22 | VideoShortEvent | nip71Video | inherited `RegularVideoEvent`: `listOfNotNull(title(), content)` NL | +| 24 | PublicMessageEvent | nipA4PublicMessages | `content` | +| 40 | ChannelCreateEvent | nip28PublicChat/admin | `channelInfo().let { listOfNotNull(it.name, it.about, it.picture).joinToString(" ") }` (SP) | +| 41 | ChannelMetadataEvent | nip28PublicChat/admin | same as kind 40 (SP) | +| 42 | ChannelMessageEvent | nip28PublicChat/message | `content` | +| 54 | PodcastEpisodeEvent | nipF4Podcasts/episode | `listOfNotNull(title(), description(), content)` NL | +| 1010 | TextNoteModificationEvent | experimental/edits | `listOfNotNull(content, summary())` NL (content first) | +| 1063 | FileHeaderEvent | nip94FileMetadata | `listOfNotNull(summary(), content)` NL | +| 1065 | FileStorageHeaderEvent | experimental/nip95/header | `listOfNotNull(summary())` NL | +| 1068 | PollEvent | nip88Polls/poll | `buildString { append(content); options().forEach { append('\n').append(it.label) } }` | +| 1111 | CommentEvent | nip22Comments | `(listOf(content) + tags.hashtags())` NL | +| 1163 | ProfileGalleryEntryEvent | experimental/profileGallery | `listOfNotNull(summary())` NL | +| 1301 | WorkoutRecordEvent | experimental/fitness/workout | `listOfNotNull(title(), content)` NL | +| 1311 | LiveActivitiesChatMessageEvent | nip53LiveActivities/chat | `(listOf(content) + tags.hashtags())` NL | +| 1312 | LiveActivitiesRaidEvent | nip53LiveActivities/raid | `content` | +| 1313 | LiveActivitiesClipEvent | nip53LiveActivities/clip | `listOfNotNull(title(), content)` NL | +| 1315 | RoadEventReportEvent | experimental/roadstr/report | `content` | +| 1337 | CodeSnippetEvent | nipC0CodeSnippets | `listOfNotNull(snippetName(), snippetDescription(), content)` NL | +| 1617 | GitPatchEvent | nip34Git/patch | `content` | +| 1618 | GitPullRequestEvent | nip34Git/pr | `listOfNotNull(subject(), content)` NL | +| 1621 | GitIssueEvent | nip34Git/issue | `listOfNotNull(subject(), content)` NL | +| 1622 | GitReplyEvent | nip34Git/reply | `content` | +| 1630–1633 | GitStatusEvent | nip34Git/status | `content` (open/applied/closed/draft) | +| 1808 | AudioHeaderEvent | experimental/audio/header | `content` | +| 1985 | LabelEvent | nip32Labeling | `(listOf(content) + labels().map { it.label }).filter { it.isNotEmpty() }` NL | +| 2003 | TorrentEvent | nip35Torrents | `listOfNotNull(title(), content)` NL | +| 2004 | TorrentCommentEvent | nip35Torrents | `content` | +| 2473 | BirdDetectionEvent | experimental/birdstar | `listOfNotNull(summary(), speciesName())` NL | +| 3302 | ConcordChatEditEvent | concord/cord03Channels | `content` | +| 5050 | NIP90TextGenerationRequestEvent | nip90Dvms/textGeneration | `inputs().filter { it.type == "prompt" \|\| it.type == "text" }.joinToString(" ") { it.value }` (SP) | +| 5100 | NIP90ImageGenerationRequestEvent | nip90Dvms/imageGeneration | `listOfNotNull(prompt(), negativePrompt()).joinToString(" ")` (SP) | +| 5129 | NappletSnapshotEvent | nip5dNapplets | `listOfNotNull(title(), description())` NL | +| 5250 | NIP90TextToSpeechRequestEvent | nip90Dvms/textToSpeech | `text() ?: ""` | +| 5302 | NIP90ContentSearchRequestEvent | nip90Dvms/contentSearch | `searchQuery() ?: ""` | +| 5303 | NIP90PeopleSearchRequestEvent | nip90Dvms/peopleSearch | `searchQuery() ?: ""` | +| 6969 | ZapPollEvent | experimental/zapPolls | `buildString { append(content); pollOptionsArray().forEach { append('\n').append(it.descriptor) } }` | +| 8333 | OnchainZapEvent | nipBCOnchainZaps/zap | `content` | +| 9002 | EditMetadataEvent | nip29RelayGroups/moderation | `(listOfNotNull(name(), about()) + hashtags())` NL | +| 9041 | GoalEvent | nip75ZapGoals | `listOfNotNull(summary(), content)` NL | +| 9321 | NutzapEvent | nip61Nutzaps/nutzap | `content` | +| 9734 | LnZapRequestEvent | nip57Zaps | `content` | +| 9735 | LnZapEvent | nip57Zaps | `zapRequest?.content.orEmpty()` — indexes the **embedded 9734's** content | +| 9736 | Bolt12ZapEvent | nipB1Bolt12Zaps/zap | `content` | +| 9737 | Bolt12ZapIntentEvent | nipB1Bolt12Zaps/intent | `content` | +| 9802 | HighlightEvent | nip84Highlights | `listOfNotNull(comment(), context(), content)` NL | +| 10003 | BookmarkListEvent | nip51Lists/bookmarkList | `listOfNotNull(title())` NL | +| 10100 | AgentProfileEvent | buzz/agentProfiles | `profileOrNull()?.let { listOfNotNull(it.name, it.displayName).joinToString("\n") } ?: ""` | +| 10154 | PodcastMetadataEvent | nipF4Podcasts/metadata | `listOfNotNull(title(), description())` NL | +| 11871 | AttestorProficiencyEvent | experimental/attestations/proficiency | `listOfNotNull(description())` NL | +| 12473 | BirdexEvent | experimental/birdstar | `(listOfNotNull(summary()) + speciesNames())` NL | +| 15128 | RootSiteEvent | nip5aStaticWebsites | `listOfNotNull(title(), description())` NL | +| 15129 | RootNappletEvent | nip5dNapplets | `listOfNotNull(title(), description())` NL | +| 30000 | PeopleListEvent | nip51Lists/peopleList | `listOfNotNull(titleOrName(), description())` NL | +| 30001 | OldBookmarkListEvent | nip51Lists/bookmarkList | `listOfNotNull(title())` NL | +| 30002 | RelaySetEvent | nip51Lists/relaySets | `listOfNotNull(title(), description())` NL | +| 30003 | LabeledBookmarkListEvent | nip51Lists/labeledBookmarkList | `listOfNotNull(titleOrName(), description())` NL | +| 30004 | ArticleCurationSetEvent | nip51Lists/articleCurationSet | `listOfNotNull(title(), description())` NL | +| 30005 | VideoCurationSetEvent | nip51Lists/videoCurationSet | `listOfNotNull(title(), description())` NL | +| 30006 | PictureCurationSetEvent | nip51Lists/pictureCurationSet | `listOfNotNull(title(), description())` NL | +| 30009 | BadgeDefinitionEvent | nip58Badges/definition | `listOfNotNull(name(), description(), content)` NL | +| 30015 | InterestSetEvent | nip51Lists/interestSet | `(listOfNotNull(title(), description()) + publicHashtags())` NL | +| 30017 | StallEvent | nip15Marketplace/stall | `stallData()?.let { listOfNotNull(it.name, it.description).joinToString("\n") } ?: ""` | +| 30018 | ProductEvent | nip15Marketplace/product | `productData()?.let { (listOfNotNull(it.name, it.description) + categories()).joinToString("\n") } ?: ""` | +| 30019 | MarketplaceEvent | nip15Marketplace/marketplace | `marketplaceData()?.let { listOfNotNull(it.name, it.about).joinToString("\n") } ?: ""` | +| 30020 | AuctionEvent | nip15Marketplace/auction | `auctionData()?.let { (listOfNotNull(it.name, it.description) + tags.hashtags()).joinToString("\n") } ?: ""` | +| 30023 | LongTextNoteEvent | nip23LongContent | `listOfNotNull(title(), summary(), content)` NL | +| 30030 | EmojiPackEvent | nip30CustomEmoji/pack | `listOfNotNull(titleOrName(), description(), content)` NL | +| 30054 | Podcasting20EpisodeEvent | nipXXPodcasting20/episode | `(listOfNotNull(title(), description(), content) + topics())` NL | +| 30055 | Podcasting20TrailerEvent | nipXXPodcasting20/trailer | `listOfNotNull(title(), content)` NL | +| 30063 | ReleaseArtifactSetEvent † | nip51Lists/releaseArtifactSet | `listOfNotNull(title(), description())` NL | +| 30175 | PersonaEvent | buzz/apPersonas | `personaOrNull()?.let { listOfNotNull(it.displayName, it.systemPrompt).joinToString("\n") } ?: ""` | +| 30176 | TeamEvent | buzz/teams | `teamOrNull()?.let { listOfNotNull(it.name, it.description, it.instructions).joinToString("\n") } ?: ""` | +| 30177 | ManagedAgentEvent | buzz/managedAgents | `agentOrNull()?.let { listOfNotNull(it.name, it.systemPrompt).joinToString("\n") } ?: ""` | +| 30267 | AppCurationSetEvent | nip51Lists/appCurationSet | `listOfNotNull(title(), description())` NL | +| 30296 | InteractiveStoryPrologueEvent | experimental/interactiveStories | inherited base: `listOfNotNull(title(), summary(), content)` NL | +| 30297 | InteractiveStorySceneEvent | experimental/interactiveStories | inherited base: `listOfNotNull(title(), summary(), content)` NL | +| 30311 | LiveActivitiesEvent | nip53LiveActivities/streaming | `listOfNotNull(title(), summary(), content)` NL | +| 30312 | MeetingSpaceEvent | nip53LiveActivities/meetingSpaces | `listOfNotNull(room(), summary(), content)` NL | +| 30313 | MeetingRoomEvent | nip53LiveActivities/meetingSpaces | `listOfNotNull(title(), summary())` NL | +| 30315 | StatusEvent | nip38UserStatus | `content` | +| 30382 | ContactCardEvent | nip85TrustedAssertions/users | `(listOfNotNull(petName(), summary()) + topics())` NL — public tags only, never the NIP-44 content | +| 30402 | ClassifiedsEvent | nip99Classifieds | `listOfNotNull(title(), summary(), content)` NL | +| 30617 | GitRepositoryEvent | nip34Git/repository | `listOfNotNull(name(), description(), content)` NL | +| 30620 | WorkflowDefEvent | buzz/workflow | `listOfNotNull(name(), content)` NL | +| 30817 | NipTextEvent | experimental/nipsOnNostr | `listOfNotNull(title(), content)` NL | +| 30818 | WikiNoteEvent | nip54Wiki | `listOfNotNull(title(), summary(), content)` NL | +| 31337 | AudioTrackEvent | experimental/audio/track | `listOfNotNull(subject())` NL | +| 31871 | AttestationEvent | experimental/attestations/attestation | `content` | +| 31872 | AttestationRequestEvent | experimental/attestations/request | `content` | +| 31873 | AttestorRecommendationEvent | experimental/attestations/recommendation | `listOfNotNull(description())` NL | +| 31890 | FeedDefinitionEvent | feedDefinition | `title().orEmpty()` | +| 31922 | CalendarDateSlotEvent | nip52Calendar/appt/day | `listOfNotNull(title(), summary(), content)` NL | +| 31923 | CalendarTimeSlotEvent | nip52Calendar/appt/time | `listOfNotNull(title(), summary(), content)` NL | +| 31924 | CalendarEvent | nip52Calendar/calendar | `listOfNotNull(title(), content)` NL | +| 31925 | CalendarRSVPEvent | nip52Calendar/rsvp | `content` | +| 31990 | AppDefinitionEvent | nip89AppHandlers/definition | `appMetaData()?.let { listOfNotNull(it.name, it.username, it.displayName, it.about, it.nip05, it.lud06, it.lud16, it.website, it.picture, it.banner, it.image).joinToString(" ") } ?: ""` (SP) | +| 32267 | SoftwareApplicationEvent | experimental/nip82SoftwareApps/application | `listOfNotNull(name(), summary(), content)` NL | +| 33401 | ExerciseTemplateEvent | experimental/fitness/workout | `listOfNotNull(title(), content)` NL | +| 33863 | FundraiserEvent | experimental/agora | `listOfNotNull(title(), content)` NL | +| 34139 | MusicPlaylistEvent | experimental/music/playlist | `listOfNotNull(title(), description(), content)` NL | +| 34235 | VideoHorizontalEvent | nip71Video | inherited `AddressableVideoEvent`: `listOfNotNull(title(), content)` NL | +| 34236 | VideoVerticalEvent | nip71Video | inherited `AddressableVideoEvent`: `listOfNotNull(title(), content)` NL | +| 34550 | CommunityDefinitionEvent | nip72ModCommunities/definition | `listOfNotNull(name(), description(), rules(), content)` NL | +| 35128 | NamedSiteEvent | nip5aStaticWebsites | `listOfNotNull(title(), description())` NL | +| 35129 | NamedNappletEvent | nip5dNapplets | `listOfNotNull(title(), description())` NL | +| 36787 | MusicTrackEvent | experimental/music/track | `listOfNotNull(title(), artist(), album(), content)` NL | +| 38000 | MintRecommendationEvent | nip87Ecash/recommendation | `content` | +| 38192 | Ps1SaveEvent | experimental/ps1saves | `listOfNotNull(summary(), saveTitle(), region(), filename())` NL | +| 38383 | P2POrderEvent | nip69P2pOrderEvents | `(listOfNotNull(makerName(), currency()) + paymentMethods().orEmpty()).joinToString(" ")` (SP) | +| 39000 | GroupMetadataEvent | nip29RelayGroups/metadata | `listOfNotNull(name(), about())` NL | +| 39089 | FollowListEvent | nip51Lists/followList | `listOfNotNull(title(), description())` NL | +| 39092 | MediaStarterPackEvent | nip51Lists/mediaStarterPack | `listOfNotNull(title(), description())` NL | +| 39701 | WebBookmarkEvent | nipB0WebBookmarks | `listOfNotNull(title(), description())` NL | +| 40002 | StreamMessageV2Event | buzz/stream | `content` | +| 40100 | CanvasEvent | buzz/stream | `content` | +| 45001 | ForumPostEvent | buzz/forum | `content` | +| 45003 | ForumCommentEvent | buzz/forum | `content` | +| 48106 | HuddleGuidelinesEvent | buzz/huddles | `content` | + +† **Kind 30063 collision:** `experimental/nip82SoftwareApps/release/SoftwareReleaseEvent` also +declares `KIND = 30063` and implements `SearchableEvent` (`content`), but `EventFactory` maps +30063 to `ReleaseArtifactSetEvent`, so on every store path kind 30063 indexes +`title()\ndescription()`. If the factory mapping ever changes, this table changes with it. + +## Abstract bases (no kind of their own) + +| Base class | Body | Concrete kinds | +|---|---|---| +| `InteractiveStoryBaseEvent` | `listOfNotNull(title(), summary(), content)` NL | 30296, 30297 | +| `AddressableVideoEvent` | `listOfNotNull(title(), content)` NL | 34235, 34236 | +| `RegularVideoEvent` | `listOfNotNull(title(), content)` NL | 21, 22 | + +## How to regenerate / verify this table + +```bash +# All implementor files: +grep -rln "override fun indexableContent" quartz/src/commonMain +# For each, pair the KIND constant with the indexableContent() body. +# Searchability on the store path additionally requires EventFactory registration: +grep -n "" quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/EventFactory.kt +``` + +A CI-diffable snapshot test (assert the set of kinds whose `EventFactory` product implements +`SearchableEvent` against a checked-in list) would make this table impossible to go stale — +suggested follow-up, not yet implemented. From d54e54b48ad3c5a49b2e74013fa6e29bc4df656a Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 03:33:48 +0000 Subject: [PATCH 006/132] Add battle-tested sync accessories from vespa-relay's mirror Four pieces extracted from a production Nostr mirror (NosFabrica/vespa-relay), generalized to quartz's multiplatform primitives, with their test suites: - nip01Core.store.insertBisecting: batchInsert fails as a unit, so one bad event costs its whole batch (999 good events per bad one at a 1000-event batch). Bisecting isolates the offender in ~2*log2(n) extra writes, and a fixed write budget keeps a store-wide failure (full disk, dead engine) from turning one failed write into ~2n. - accessories.SyncBands: resume memory for fetchAllPages. Remembers the created_at band covered per (relay, filter) and asks only for the legs outside it, with inclusive edges so a page boundary cannot strand a run of same-second events. A finished negentropy reconcile records completeness through its start instant; a periodic full re-walk keeps stale claims from narrowing forever. Persistence is the caller's, via export/restore and an onChange hook. - accessories.PagingProgress: progress for paged walks measured on the time axis, the only axis whose end is known in advance - count-based percentages degenerate to downloaded/downloaded = 100%. Needs no COUNT support. - nip66RelayMonitor.reachability.HostStrikes: per-authority strike counting for outbox-scale fan-outs, where a filtering relay mints one url per user and per-url counters never converge. Ever-delivered overrides eviction in both race orders, and eviction surfaces exactly once for publishing. reachability.Unreachability (jvmAndroid): which failures may be published as a signed NIP-66 unreachable record - connection-level only, so a relay that answered the handshake and hung up mid-page is never libelled, and a caller's own bug is never the relay's fault. 57 tests pass on the JVM target; the common code uses quartz's ConcurrentMap, ConcurrentSet and TimeUtils only. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y4Pi9YYMhdzTFxRiV2jF9R --- .../client/accessories/PagingProgress.kt | 124 ++++++ .../relay/client/accessories/SyncBands.kt | 275 ++++++++++++++ .../quartz/nip01Core/store/BisectingInsert.kt | 89 +++++ .../reachability/HostStrikes.kt | 133 +++++++ .../client/accessories/PagingProgressTest.kt | 140 +++++++ .../relay/client/accessories/SyncBandsTest.kt | 355 ++++++++++++++++++ .../nip01Core/store/BisectingInsertTest.kt | 200 ++++++++++ .../reachability/HostStrikesTest.kt | 176 +++++++++ .../reachability/Unreachability.kt | 48 +++ .../reachability/UnreachabilityTest.kt | 70 ++++ 10 files changed, 1610 insertions(+) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/PagingProgress.kt create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncBands.kt create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/BisectingInsert.kt create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/HostStrikes.kt create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/PagingProgressTest.kt create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncBandsTest.kt create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/BisectingInsertTest.kt create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/HostStrikesTest.kt create mode 100644 quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/Unreachability.kt create mode 100644 quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/UnreachabilityTest.kt diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/PagingProgress.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/PagingProgress.kt new file mode 100644 index 0000000000..028fa34979 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/PagingProgress.kt @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay.client.accessories + +import com.vitorpamplona.quartz.utils.TimeUtils +import com.vitorpamplona.quartz.utils.concurrent.ConcurrentMap +import kotlin.concurrent.Volatile + +/** + * How far a paged walk has got, measured on the time axis — the only axis + * whose end is known in advance. + * + * A paged fetch ([fetchAllPages]) has no event denominator: how many events + * exist is exactly what it is finding out, so every count-based percentage + * degenerates to `downloaded/downloaded = 100%`. The time axis has both ends + * before the first request — the filter's `until` (or now) down to its + * `since` (or [SyncBands.PLAUSIBLE_FLOOR]) — with each page's new `until` + * reporting the exact position between them. It needs no COUNT support. + * + * The estimate assumes events are spread evenly over time, which they are + * not — so it errs pessimistic on the tail, and is a bound, not a promise. + * + * One instance can serve many concurrent walks: keys are `"group|walk"`, and + * the group prefix scopes [fraction], [reached] and [etaMs] so two groups + * never report each other's numbers. + */ +class PagingProgress( + private val nowMillis: () -> Long = { TimeUtils.nowMillis() }, +) { + private class Walk( + val top: Long, + val bottom: Long, + val startedMs: Long, + @Volatile var current: Long, + ) + + private val walks = ConcurrentMap() + + /** Begin a walk over `[bottom, top]` seconds. An inverted window is not a walk. */ + fun begin( + key: String, + top: Long, + bottom: Long, + ) { + if (top > bottom) walks[key] = Walk(top, bottom, nowMillis(), top) + } + + /** The walk reached [until]; monotonic, so a page that jumps back cannot un-advance it. */ + fun mark( + key: String, + until: Long, + ) { + walks[key]?.let { + // Clamped to the walk's own floor: relays serve events stamped 0, + // and one of those would drag the position to the epoch. Below the + // floor means the walk is done, not time travel. + val reached = until.coerceAtLeast(it.bottom) + if (reached < it.current) it.current = reached + } + } + + fun finish(key: String) { + walks.remove(key) + } + + /** + * Fraction of the walk complete, averaged over every walk still going in + * [group] (or all of them when null) — averaged rather than summed + * because each covers its own span, so "half the walks done and half at + * zero" is 50%. + */ + fun fraction(group: String? = null): Double? { + val live = live(group) + if (live.isEmpty()) return null + return live.sumOf { w -> + val span = (w.top - w.bottom).coerceAtLeast(1) + ((w.top - w.current).toDouble() / span).coerceIn(0.0, 1.0) + } / live.size + } + + private fun live(group: String?): List = + if (group == null) { + walks.snapshot().values.toList() + } else { + walks + .snapshot() + .entries + .filter { it.key.startsWith("$group|") } + .map { it.value } + } + + /** The oldest second [group] has reached, or null when it is not walking. */ + fun reached(group: String? = null): Long? = live(group).minOfOrNull { it.current } + + /** Milliseconds left at the rate achieved so far, or null before it means anything. */ + fun etaMs(group: String? = null): Long? { + val f = fraction(group) ?: return null + // Under a few percent the extrapolation is dominated by connect time + // and produces numbers worse than saying nothing. + if (f < 0.02) return null + val oldestStart = live(group).minOfOrNull { it.startedMs } ?: return null + val elapsed = nowMillis() - oldestStart + if (elapsed < 5_000) return null + return ((elapsed / f) - elapsed).toLong() + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncBands.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncBands.kt new file mode 100644 index 0000000000..68bff2ca85 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncBands.kt @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay.client.accessories + +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.utils.TimeUtils +import com.vitorpamplona.quartz.utils.concurrent.ConcurrentMap + +/** + * How much of a filter's history has already been pulled from one relay, so a + * restart does not pull it again. + * + * A negentropy relay needs none of this — reconciliation downloads only the + * diff. Most relays lack NIP-77, and a paged fetch ([fetchAllPages]) has no + * memory: it re-downloads everything it walked last time, every restart, + * forever. So for those, remember the band of `created_at` covered per + * (relay, filter), and the next run asks only for the two legs outside it: + * + * stored band: |<-------- covered -------->| + * next fetch: <------| |------> + * + * Keyed by the WHOLE filter deliberately: any edit to a filter is a new key + * with no band, so the next run starts over — the safe direction to be wrong + * in, and the intended way to force a re-walk. + * + * A band does not guarantee completeness (a truncating relay, an event + * back-dated into a walked span). The trade is deliberate: re-reading a + * corpus on every restart is a certain daily cost, while both holes are + * occasional and self-heal on the next filter change or full re-walk. + * + * Persistence is the caller's: [export] the map on a schedule and [restore] + * it at startup. [onChange] fires whenever a band changes, so a persistence + * layer can mark itself dirty without polling. + */ +class SyncBands( + // How long a band may narrow work before the whole filter is walked + // again. Everything a band claims is a claim about the past; this is how + // long to trust it without re-testing. + private val fullResyncSeconds: Long = DEFAULT_FULL_RESYNC_SECONDS, + private val now: () -> Long = { TimeUtils.now() }, + private val onChange: () -> Unit = {}, +) { + /** + * What is already covered for one (relay, filter) pair. + * + * [complete] is the difference between "we walked this span" (a paged + * fetch) and "we are in sync below this point" (a finished negentropy + * reconcile, which compared the whole range). Only a complete band may + * skip its older leg. + * + * [fullAt] is when the last pass that started from nothing finished — the + * clock for the periodic re-walk. + */ + data class Band( + val minCreatedAt: Long, + val maxCreatedAt: Long, + val complete: Boolean = false, + val fullAt: Long = 0, + ) + + private val bands = ConcurrentMap() + + // filter -> its canonical json. Filter.toJson() runs to tens of thousands + // of characters for author-scoped filters, and a fan-out keys once per + // relay per cycle over the SAME handful of filter instances. Filter + // compares by identity, so this map is an identity cache; an + // equal-but-distinct filter still keys correctly, just without the cache. + private val fingerprints = ConcurrentMap() + + /** + * The filters to actually run now, given what is already covered: the + * whole filter when nothing is recorded (or the band went stale), + * otherwise the legs outside the band, clamped to the filter's own + * `since`/`until`. + * + * The legs are INCLUSIVE of the band's edges (`until = min`, not + * `min - 1`): a page boundary can split a run of events sharing one + * `created_at`, and excluding the edge would strand the rest of that + * second in no leg at all. The cost is re-reading one second's worth of + * events per leg, which a store rejects as duplicates. + */ + fun legs( + url: NormalizedRelayUrl, + filter: Filter, + ): List { + val band = bands[key(url, filter)] ?: return listOf(filter) + // Time for another full pass: relays gain old events, and without + // this the band's claim is never re-tested. + if (isStale(band)) return listOf(filter) + val legs = mutableListOf() + + // Older: up to and including the band's floor, but not past the + // filter's. A complete band has no older leg at all — the reconcile + // already compared the whole range. + if (!band.complete && (filter.since == null || band.minCreatedAt >= filter.since)) { + legs.add(filter.copy(until = minOf(band.minCreatedAt, filter.until ?: Long.MAX_VALUE))) + } + + // Newer: from the band's ceiling on, but not past the filter's. + if (filter.until == null || band.maxCreatedAt <= filter.until) { + legs.add(filter.copy(since = maxOf(band.maxCreatedAt, filter.since ?: Long.MIN_VALUE))) + } + return legs + } + + /** + * Widen the band for (url, filter) to include what a completed fetch saw. + * + * [paged] gates the mechanism: a negentropy sync needs no band, and + * recording one would only risk narrowing a future reconciliation. + * Nothing is recorded for a fetch that saw no events — an empty result + * says nothing about what the relay holds. + * + * [reconciledThrough] is the strong case: a FINISHED reconcile compared + * the filter's whole range, so the caller is in sync up to the instant + * the sync STARTED — recorded against that instant rather than the newest + * event seen, because "the relay had nothing newer" and "we never asked" + * must not look alike. + */ + fun record( + url: NormalizedRelayUrl, + filter: Filter, + observedMin: Long?, + observedMax: Long?, + paged: Boolean, + reconciledThrough: Long? = null, + ) { + if (reconciledThrough != null) { + put(url, filter, observedMin ?: reconciledThrough, reconciledThrough, complete = true) + return + } + if (!paged) return + // Guarded even though callers should filter with [isPlausible] per + // event: a 1970 floor or a far-future ceiling would make the band + // claim the whole timeline, and the leg outside it would ask for a + // range nothing can be in, forever. + if (observedMin == null || observedMax == null) return + if (!isPlausible(observedMin, now()) || !isPlausible(observedMax, now())) return + put(url, filter, observedMin, observedMax, complete = false) + } + + /** + * Widen (or reset) the band. A pass that ran because the previous band + * had gone stale REPLACES it: it re-walked the whole filter, so its own + * span is the complete picture and [Band.fullAt] restarts from here. + */ + private fun put( + url: NormalizedRelayUrl, + filter: Filter, + min: Long, + max: Long, + complete: Boolean, + ) { + val fresh = Band(min, max, complete, now()) + bands.merge(key(url, filter), fresh) { old, new -> + if (isStale(old)) { + new + } else { + Band( + minOf(old.minCreatedAt, new.minCreatedAt), + maxOf(old.maxCreatedAt, new.maxCreatedAt), + old.complete || new.complete, + old.fullAt, + ) + } + } + onChange() + } + + private fun isStale(band: Band): Boolean = now() - band.fullAt >= fullResyncSeconds + + /** + * The narrowest single filter that still covers what every one of [urls] + * needs — the window a shared negentropy snapshot has to be taken over. + * + * In steady state every relay carries a complete band and this collapses + * to `since = the oldest of their ceilings` — the difference between + * snapshotting an id set of millions and one of a few thousand. One relay + * that has never synced puts it back to the full filter, correctly: that + * relay genuinely needs everything. + */ + fun coveringWindow( + urls: List, + filter: Filter, + ): Filter { + if (urls.isEmpty()) return filter + var since = Long.MAX_VALUE + for (url in urls) { + val legs = legs(url, filter) + // More than one leg means an older gap this relay still wants, so + // the snapshot cannot start above the filter's own floor. + val only = legs.singleOrNull() ?: return filter + val legSince = only.since ?: return filter + since = minOf(since, legSince) + } + return if (since == Long.MAX_VALUE) filter else filter.copy(since = since) + } + + /** What is currently covered, for logging and tests. */ + fun band( + url: NormalizedRelayUrl, + filter: Filter, + ): Band? = bands[key(url, filter)] + + fun size(): Int = bands.size() + + /** A point-in-time copy of every band, for a persistence layer to write out. */ + fun export(): Map = bands.snapshot() + + /** Load previously [export]ed bands, e.g. at startup. */ + fun restore(entries: Map) { + for ((key, band) in entries) bands[key] = band + } + + /** + * The identity of one (relay, filter) pair. [Filter.toJson] is the + * protocol's own canonical form, so two filters that mean the same thing + * key the same way and any edit keys differently — exactly the "config + * changed, start over" rule. + */ + private fun key( + url: NormalizedRelayUrl, + filter: Filter, + ): String = "${url.url} ${fingerprints.getOrPut(filter) { filter.toJson() }}" + + companion object { + /** + * A week. Long enough that the narrow path is the normal one, short + * enough that anything a band is wrong about is wrong for days, not + * forever. + */ + const val DEFAULT_FULL_RESYNC_SECONDS = 7L * 24 * 60 * 60 + + /** + * 2020-01-01. Below this a `created_at` is a bug, not a date — the + * protocol did not exist. Also the natural floor for measuring a + * paged walk's progress when a filter names no `since`. + */ + const val PLAUSIBLE_FLOOR = 1_577_836_800L + + // Clock skew a relay may legitimately be ahead by. Past this, a + // created_at is the author's fiction rather than a time. + private const val FUTURE_SKEW_SECONDS = 86_400L + + /** + * Whether a `created_at` can be believed as evidence of coverage. + * Filter with this per EVENT, not over a leg's aggregate: one + * misdated event among hundreds of thousands would otherwise discard + * the whole relay's band. + */ + fun isPlausible( + createdAt: Long, + now: Long = TimeUtils.now(), + ): Boolean = createdAt in PLAUSIBLE_FLOOR..(now + FUTURE_SKEW_SECONDS) + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/BisectingInsert.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/BisectingInsert.kt new file mode 100644 index 0000000000..deae869a8d --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/BisectingInsert.kt @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store + +import com.vitorpamplona.quartz.nip01Core.core.Event +import kotlinx.coroutines.CancellationException + +/** + * Write [events] through [write]; if that throws, split the batch and write + * the halves, down to the single event the writer cannot take. + * + * A bulk write like [IEventStore.batchInsert] fails as a unit, so one bad + * event would otherwise cost the whole batch — 999 good events lost per bad + * one at a 1000-event batch, with no retry. Bisecting costs ~2·log2(n) extra + * writes on a failing batch, nothing on a healthy one, and ends holding the + * offender by itself for [onPoison] to report. Re-writing the good halves is + * safe: re-inserting an already-applied event is a duplicate the store + * rejects. + * + * Splitting assumes ONE event is at fault. When the store itself is refusing + * (a full disk, a dead engine) every half fails all the way down and + * isolation would turn one failed write into ~2n — precisely the wrong moment + * to multiply the load. So isolation spends a fixed [budget] of writes and + * hands the remainder to [onGaveUp]: "we could not say which" is a different + * fact from "this event is bad", and a caller should count them apart. + */ +suspend fun insertBisecting( + events: List, + write: suspend (List) -> List, + onOutcomes: (List) -> Unit, + onPoison: (Event, Throwable) -> Unit, + onGaveUp: (List, Throwable) -> Unit = { _, _ -> }, + budget: Int = ISOLATION_WRITE_BUDGET, +) = bisect(events, write, onOutcomes, onPoison, onGaveUp, intArrayOf(budget)) + +private suspend fun bisect( + events: List, + write: suspend (List) -> List, + onOutcomes: (List) -> Unit, + onPoison: (Event, Throwable) -> Unit, + onGaveUp: (List, Throwable) -> Unit, + budget: IntArray, +) { + if (events.isEmpty()) return + try { + onOutcomes(write(events)) + } catch (e: CancellationException) { + throw e + } catch (e: Throwable) { + if (events.size == 1) { + onPoison(events.single(), e) + return + } + if (budget[0] <= 0) { + onGaveUp(events, e) + return + } + budget[0] -= 2 + val mid = events.size / 2 + bisect(events.subList(0, mid), write, onOutcomes, onPoison, onGaveUp, budget) + bisect(events.subList(mid, events.size), write, onOutcomes, onPoison, onGaveUp, budget) + } +} + +/** + * Writes one batch may spend isolating its bad events before giving up. + * Isolating k bad events out of n costs about `2·k·log2(n)` writes, so 64 + * covers three in a 1000-event batch — past the rate seen in practice. What + * it really bounds is the store-wide case, where every write fails. + */ +const val ISOLATION_WRITE_BUDGET = 64 diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/HostStrikes.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/HostStrikes.kt new file mode 100644 index 0000000000..11a2c2d51d --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/HostStrikes.kt @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip66RelayMonitor.reachability + +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.utils.concurrent.ConcurrentMap +import com.vitorpamplona.quartz.utils.concurrent.ConcurrentSet + +/** + * Which relays are worth dialling, within one fan-out cycle. A discovered + * relay list is five figures of urls and most of them are corpses; without + * this, every cycle re-dials all of them and the working relays queue behind + * hosts that stopped existing years ago. + * + * Failures are counted per AUTHORITY (`host[:port]`), not per url: the outbox + * model mints one url per user for a filtering relay, so a per-url counter + * never reaches a threshold on any single one. The authority is host-only and + * does NOT fold a subdomain into its parent — those are different servers. + * + * [produced] overrides a strike race: a host that has ever delivered is never + * treated as dead for the rest of the cycle, whichever order the two events + * land in. Cycle-local; nothing persists — see [RelayReachabilityStore] for + * the part that survives a restart. + */ +class HostStrikes( + private val strikeLimit: Int = DEFAULT_STRIKE_LIMIT, + // Relays a previous run proved unreachable, and still within their TTL. + private val knownDead: Set = emptySet(), +) { + private val strikes = ConcurrentMap() + private val deadHosts = ConcurrentSet() + private val producedHosts = ConcurrentSet() + + private val delivered = ConcurrentSet() + private val failed = ConcurrentSet() + + /** Relays this cycle actually got something from — worth remembering as live. */ + val reachable: Set get() = delivered.snapshot() + + /** Relays this cycle could not reach at all. A relay that later delivered is not in it. */ + val unreachable: Set get() = failed.snapshot() - delivered.snapshot() + + /** + * Skip this relay? True when a previous run proved it dead (and no + * [produced] since), or when its whole authority has been struck out here. + */ + fun isDead(url: NormalizedRelayUrl): Boolean { + val authority = authorityOf(url.url) + if (authority in producedHosts) return false + return url in knownDead || authority in deadHosts + } + + /** + * This relay connected but delivered nothing before giving up. Count it + * against its authority and, at [strikeLimit], stop dialling the host. + * Returns the eviction — for the caller to publish — exactly when this + * strike is the one that took the host down: that is the only point where + * the evidence exists, because every sibling url is skipped without being + * dialled from here on. + */ + fun strike(url: NormalizedRelayUrl): Evicted? { + failed.add(url) + if (strikeLimit <= 0) return null + val authority = authorityOf(url.url) + if (authority in producedHosts || authority in deadHosts) return null + if (strikes.merge(authority, 1) { old, new -> old + new } < strikeLimit) return null + deadHosts.add(authority) + return Evicted(authority, strikeLimit) + } + + /** An authority struck out, and the evidence for it. */ + class Evicted( + val authority: String, + val strikes: Int, + ) + + /** This relay delivered. Its authority is alive, whatever else happened. */ + fun produced(url: NormalizedRelayUrl) { + delivered.add(url) + producedHosts.add(authorityOf(url.url)) + } + + /** For a cycle's closing line: how many hosts were dropped. */ + fun evictedHosts(): Int = deadHosts.size() + + fun summary(total: Int): String = + "${reachable.size} live, ${unreachable.size} unreachable, " + + "${deadHosts.size()} host(s) struck out, ${knownDead.size} skipped as known-dead of $total" + + companion object { + /** + * Three, because a single timeout is ordinary — a busy relay that + * never answered one REQ is not a dead one — while three separate + * urls on the same host all going silent is a server, not a + * coincidence. + */ + const val DEFAULT_STRIKE_LIMIT = 3 + + /** + * `host[:port]` — everything between the scheme and the first path + * slash. The port is part of it: two ports on one machine are two + * relays. + */ + fun authorityOf(url: String): String { + val afterScheme = + when { + url.startsWith("wss://") -> url.substring(6) + url.startsWith("ws://") -> url.substring(5) + else -> url + } + val slash = afterScheme.indexOf('/') + return if (slash >= 0) afterScheme.substring(0, slash) else afterScheme + } + } +} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/PagingProgressTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/PagingProgressTest.kt new file mode 100644 index 0000000000..bf8a550c63 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/PagingProgressTest.kt @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay.client.accessories + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class PagingProgressTest { + private fun assertClose( + expected: Double, + actual: Double?, + message: String? = null, + ) { + assertTrue(actual != null && kotlin.math.abs(expected - actual) < 0.001, "${message ?: ""} expected $expected got $actual") + } + + @Test + fun `progress is the walked share of the time window`() { + val p = PagingProgress() + p.begin("a", top = 1_000L, bottom = 0L) + + assertClose(0.0, p.fraction(), "nothing walked yet") + + p.mark("a", 750L) + assertClose(0.25, p.fraction()) + + p.mark("a", 100L) + assertClose(0.90, p.fraction()) + } + + @Test + fun `a page that jumps backwards cannot un-advance the walk`() { + // Pages arrive from one relay in order, but nothing in the protocol + // guarantees it, and a percentage that goes DOWN is worse than one that + // is slightly wrong — it reads as the sync having lost ground. + val p = PagingProgress() + p.begin("a", top = 1_000L, bottom = 0L) + + p.mark("a", 200L) + p.mark("a", 900L) + + assertClose(0.80, p.fraction(), "the later higher until is ignored") + } + + @Test + fun `walks average rather than sum`() { + // Two relays each walking their own window: one done and one untouched + // is half way — not 100% as summing would give. + val p = PagingProgress() + p.begin("a", top = 1_000L, bottom = 0L) + p.begin("b", top = 500L, bottom = 0L) + + p.mark("a", 0L) + + assertClose(0.5, p.fraction()) + } + + @Test + fun `a finished walk leaves the average`() { + val p = PagingProgress() + p.begin("a", top = 1_000L, bottom = 0L) + p.begin("b", top = 1_000L, bottom = 0L) + p.mark("b", 500L) + + p.finish("a") + + assertClose(0.5, p.fraction(), "only b is still walking") + p.finish("b") + assertNull(p.fraction(), "nothing walking means no number to report") + } + + @Test + fun `a group prefix scopes the numbers to its own walks`() { + // One instance serves many concurrent walks; without the scope two + // streams would print each other's percentages. + val p = PagingProgress() + p.begin("streamA|wss://r1", top = 1_000L, bottom = 0L) + p.begin("streamB|wss://r2", top = 1_000L, bottom = 0L) + p.mark("streamA|wss://r1", 0L) + + assertClose(1.0, p.fraction("streamA")) + assertClose(0.0, p.fraction("streamB")) + assertClose(0.5, p.fraction()) + } + + @Test + fun `an inverted or empty window is not a walk`() { + // A leg whose since is above its until asks for a range nothing can be + // in. Dividing by that span would produce infinities on the status line. + val p = PagingProgress() + + p.begin("a", top = 100L, bottom = 900L) + + assertNull(p.fraction()) + } + + @Test + fun `no ETA before the estimate means anything`() { + val p = PagingProgress() + p.begin("a", top = 1_000_000L, bottom = 0L) + + p.mark("a", 999_000L) + + // 0.1% in: extrapolating here yields days-long ETAs from connect + // latency alone, which is worse than printing nothing. + assertNull(p.etaMs(), "too early to extrapolate") + } + + @Test + fun `ETA extrapolates from the rate achieved so far`() { + var clock = 1_000_000L + val p = PagingProgress(nowMillis = { clock }) + p.begin("a", top = 1_000L, bottom = 0L) + p.mark("a", 500L) + + // Half way after six seconds: whatever has elapsed is also what remains. + clock += 6_000 + assertEquals(6_000L, p.etaMs()) + } +} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncBandsTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncBandsTest.kt new file mode 100644 index 0000000000..d0ffe5290a --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncBandsTest.kt @@ -0,0 +1,355 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay.client.accessories + +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer +import com.vitorpamplona.quartz.utils.TimeUtils +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertSame +import kotlin.test.assertTrue + +/** + * A paged relay has no memory of what it already sent, so without a band every + * restart re-downloads its whole corpus. These pin the band arithmetic and, more + * importantly, the cases where a band must NOT be used — a stale band silently + * skips events, which is a worse failure than re-reading them. + */ +class SyncBandsTest { + private val relay = RelayUrlNormalizer.normalize("wss://relay.example") + private val other = RelayUrlNormalizer.normalize("wss://other.example") + private val profiles = Filter(kinds = listOf(0)) + + private fun now(): Long = TimeUtils.now() + + // ---- the band arithmetic ---------------------------------------------- + + @Test + fun `with nothing recorded the whole filter is fetched`() { + val c = SyncBands() + assertEquals(listOf(profiles), c.legs(relay, profiles)) + } + + @Test + fun `a recorded band is fetched around rather than through`() { + val c = SyncBands() + c.record(relay, profiles, observedMin = 1_700_001_000L, observedMax = 1_700_002_000L, paged = true) + + val legs = c.legs(relay, profiles) + assertEquals(2, legs.size, "one leg older than the band and one newer") + assertEquals(1_700_001_000L, legs[0].until, "older leg stops AT the band floor") + assertNull(legs[0].since, "and reaches as far back as the filter allows") + assertEquals(1_700_002_000L, legs[1].since, "newer leg starts AT its ceiling") + assertNull(legs[1].until) + } + + @Test + fun `an event sharing the band boundary second is still reachable`() { + // A paged relay cuts pages by count, so a boundary can fall inside a run + // of events sharing one created_at. Excluding the edge would strand the + // rest of that second in no leg at all, while the band called it covered. + val c = SyncBands() + c.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = true) + + val legs = c.legs(relay, profiles) + + fun reachable(t: Long) = legs.any { (it.since ?: Long.MIN_VALUE) <= t && t <= (it.until ?: Long.MAX_VALUE) } + + assertTrue(reachable(1_700_001_000L), "the band floor second must be re-read") + assertTrue(reachable(1_700_002_000L), "and its ceiling second") + assertTrue(reachable(1_700_000_999L), "below the band") + assertTrue(reachable(1_700_002_001L), "above it") + // Only the interior is skipped, which is the entire point. + assertTrue(!reachable(1_700_001_500L), "the covered interior is not re-read") + } + + @Test + fun `successive runs widen the band rather than replacing it`() { + val c = SyncBands() + c.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = true) + // A later run reaches further back and picks up newer events. + c.record(relay, profiles, 1_700_000_500L, 1_700_002_500L, paged = true) + + val band = c.band(relay, profiles)!! + assertEquals(1_700_000_500L, band.minCreatedAt) + assertEquals(1_700_002_500L, band.maxCreatedAt) + } + + @Test + fun `a capped relay walks further back on each run`() { + // The case that makes this worth having: a relay that only ever answers + // with its newest N events. Each run starts below the last one's floor. + val c = SyncBands() + c.record(relay, profiles, 1_700_009_000L, 1_700_010_000L, paged = true) + assertEquals(1_700_009_000L, c.legs(relay, profiles)[0].until) + + c.record(relay, profiles, 1_700_008_000L, 1_700_008_999L, paged = true) + assertEquals(1_700_008_000L, c.legs(relay, profiles)[0].until) + } + + // ---- when a band must not be used -------------------------------------- + + @Test + fun `a negentropy sync that reported no outcome records nothing`() { + // Only a sync that says how far it reconciled earns a band; a bare + // paged=false call carries no claim to record. + val c = SyncBands() + c.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = false) + assertNull(c.band(relay, profiles)) + assertEquals(listOf(profiles), c.legs(relay, profiles)) + } + + // ---- coverage: what a finished reconcile earns ------------------------- + + @Test + fun `a finished reconcile is in sync through the instant it started`() { + // Not through the newest event it happened to see: "the relay had nothing + // newer" and "we never asked" must not record the same thing. + val c = SyncBands() + val startedAt = now() - 60 + c.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = false, reconciledThrough = startedAt) + + val band = c.band(relay, profiles)!! + assertTrue(band.complete) + assertEquals(startedAt, band.maxCreatedAt) + } + + @Test + fun `a reconcile that downloaded nothing still records coverage`() { + // The empty case is the WHOLE point: nothing came back because we already + // have it, and that is exactly when the next run should ask for a sliver. + val c = SyncBands() + val startedAt = now() - 60 + c.record(relay, profiles, null, null, paged = false, reconciledThrough = startedAt) + + val leg = c.legs(relay, profiles).single() + assertEquals(startedAt, leg.since) + assertNull(leg.until) + } + + @Test + fun `a complete band drops its older leg while a paged one keeps it`() { + val reconciled = SyncBands() + reconciled.record(relay, profiles, null, null, paged = false, reconciledThrough = 1_700_002_000L) + val only = reconciled.legs(relay, profiles).single() + assertEquals(1_700_002_000L, only.since) + + val walked = SyncBands() + walked.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = true) + assertEquals(2, walked.legs(relay, profiles).size, "a paged walk says nothing about what it never asked for") + } + + // ---- the periodic full re-walk ----------------------------------------- + + @Test + fun `a band stops narrowing once it is older than the resync period`() { + val c = SyncBands(fullResyncSeconds = 60) + c.record(relay, profiles, null, null, paged = false, reconciledThrough = now() - 3600) + // Recorded 'now' whatever the created_at claim, so age it by rewriting. + c.record(relay, profiles, null, null, paged = false, reconciledThrough = now()) + assertEquals(1, c.legs(relay, profiles).size, "fresh band still narrows") + + val stale = SyncBands(fullResyncSeconds = 0) + stale.record(relay, profiles, null, null, paged = false, reconciledThrough = now()) + assertSame(profiles, stale.legs(relay, profiles).single(), "a band past its period re-walks everything") + } + + @Test + fun `the re-walk replaces the old claim instead of widening it`() { + // Widening would carry the stale band's floor forward forever and the + // periodic pass would never actually reset anything. + val c = SyncBands(fullResyncSeconds = 0) + c.record(relay, profiles, 1_700_000_000L, 1_700_001_000L, paged = true) + c.record(relay, profiles, 1_700_005_000L, 1_700_006_000L, paged = true) + + val band = c.band(relay, profiles)!! + assertEquals(1_700_005_000L, band.minCreatedAt, "the second pass walked everything; its span is the whole picture") + } + + // ---- the shared snapshot window ---------------------------------------- + + @Test + fun `covering window collapses to the oldest ceiling once everyone is caught up`() { + val c = SyncBands() + c.record(relay, profiles, null, null, paged = false, reconciledThrough = 1_700_009_000L) + c.record(other, profiles, null, null, paged = false, reconciledThrough = 1_700_003_000L) + + assertEquals(1_700_003_000L, c.coveringWindow(listOf(relay, other), profiles).since) + } + + @Test + fun `one relay that has never synced puts the window back to the whole filter`() { + // It genuinely needs everything — narrowing the shared snapshot would + // reconcile it against ids we never looked up. + val c = SyncBands() + c.record(relay, profiles, null, null, paged = false, reconciledThrough = 1_700_009_000L) + + // The filter itself, unnarrowed — identity, since Filter has no equals. + assertSame(profiles, c.coveringWindow(listOf(relay, other), profiles)) + assertSame(profiles, c.coveringWindow(emptyList(), profiles)) + } + + @Test + fun `one shared window serves a whole stream of relays`() { + // Every url in a stream shares that stream's filter, so a backfill can + // take ONE snapshot for all of them instead of walking the identical + // range once per relay for byte-identical answers. + val c = SyncBands() + val third = RelayUrlNormalizer.normalize("wss://third.example") + c.record(relay, profiles, null, null, paged = false, reconciledThrough = 1_700_009_000L) + c.record(other, profiles, null, null, paged = false, reconciledThrough = 1_700_003_000L) + c.record(third, profiles, null, null, paged = false, reconciledThrough = 1_700_007_000L) + + // The hungriest of them sets the floor; the other two re-read a little. + assertEquals(1_700_003_000L, c.coveringWindow(listOf(relay, other, third), profiles).since) + } + + @Test + fun `a relay with an older gap also widens the shared window`() { + val c = SyncBands() + c.record(relay, profiles, null, null, paged = false, reconciledThrough = 1_700_009_000L) + c.record(other, profiles, 1_700_001_000L, 1_700_002_000L, paged = true) + + assertSame(profiles, c.coveringWindow(listOf(relay, other), profiles)) + } + + @Test + fun `an empty fetch records nothing`() { + // No events says nothing about what the relay holds, only that this + // window was empty — recording it would fabricate coverage. + val c = SyncBands() + c.record(relay, profiles, null, null, paged = true) + assertNull(c.band(relay, profiles)) + } + + @Test + fun `one misdated event does not cost a relay its whole band`() { + // A single future-dated stamp among hundreds of thousands must not fail + // a check applied to the aggregate. Screening per event keeps the rest. + val c = SyncBands() + val far = now() + 400L * 86_400 + val observed = listOf(1_700_001_000L, far, 1_700_002_000L, 0L) + + val plausible = observed.filter { SyncBands.isPlausible(it) } + c.record(relay, profiles, plausible.min(), plausible.max(), paged = true) + + val band = c.band(relay, profiles)!! + assertEquals(1_700_001_000L, band.minCreatedAt) + assertEquals(1_700_002_000L, band.maxCreatedAt) + } + + @Test + fun `changing the filter starts over`() { + val c = SyncBands() + c.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = true) + + // Widening the kinds means the old band skipped events it never fetched. + val wider = Filter(kinds = listOf(0, 10002)) + assertEquals(listOf(wider), c.legs(relay, wider), "a new filter has no band") + assertNull(c.band(relay, wider)) + // ...and the original is untouched, so reverting resumes where it was. + assertEquals(1_700_001_000L, c.band(relay, profiles)!!.minCreatedAt) + } + + @Test + fun `each relay keeps its own band`() { + val c = SyncBands() + c.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = true) + assertEquals(listOf(profiles), c.legs(other, profiles)) + } + + // ---- the filter's own bounds still win --------------------------------- + + @Test + fun `a bounded filter never widens past its own since and until`() { + val bounded = Filter(kinds = listOf(0), since = 1_700_001_000L, until = 1_700_005_000L) + val c = SyncBands() + c.record(relay, bounded, 1_700_002_000L, 1_700_003_000L, paged = true) + + val legs = c.legs(relay, bounded) + assertEquals(2, legs.size) + assertEquals(1_700_001_000L, legs[0].since, "the older leg keeps the configured floor") + assertEquals(1_700_002_000L, legs[0].until) + assertEquals(1_700_003_000L, legs[1].since) + assertEquals(1_700_005_000L, legs[1].until, "the newer leg keeps the configured ceiling") + } + + @Test + fun `a fully covered bounded filter re-reads only its two edge seconds`() { + // Inclusive edges mean "covered" can never quite mean "ask for nothing": + // the two boundary seconds are always re-read, because that is the only + // way to catch a run of same-second events a page boundary cut in half. + val bounded = Filter(kinds = listOf(0), since = 1_700_001_000L, until = 1_700_005_000L) + val c = SyncBands() + c.record(relay, bounded, 1_700_001_000L, 1_700_005_000L, paged = true) + + val legs = c.legs(relay, bounded) + assertEquals(2, legs.size) + assertEquals(1_700_001_000L to 1_700_001_000L, legs[0].since to legs[0].until, "the floor second only") + assertEquals(1_700_005_000L to 1_700_005_000L, legs[1].since to legs[1].until, "the ceiling second only") + } + + // ---- persistence hooks -------------------------------------------------- + + @Test + fun `export and restore round-trip the bands`() { + val c = SyncBands() + c.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = true) + + val reopened = SyncBands() + reopened.restore(c.export()) + + val band = reopened.band(relay, profiles)!! + assertEquals(1_700_001_000L, band.minCreatedAt) + assertEquals(1_700_002_000L, band.maxCreatedAt) + } + + @Test + fun `onChange fires when a band changes so persistence can mark dirty`() { + var changes = 0 + val c = SyncBands(onChange = { changes++ }) + + c.record(relay, profiles, null, null, paged = true) + assertEquals(0, changes, "an empty fetch records nothing and must not dirty the store") + + c.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = true) + assertEquals(1, changes) + } + + @Test + fun `the same filter instance is fingerprinted once`() { + // Filter.toJson() runs to tens of thousands of characters for an + // author-scoped filter, and a fan-out keys once per relay per cycle. + val big = Filter(kinds = listOf(30382), authors = (1..500).map { it.toString(16).padStart(64, '0') }) + val c = SyncBands() + c.record(relay, big, 1_700_001_000L, 1_700_002_000L, paged = true) + + // Same instance, many lookups: still one band, and cheap. + repeat(50) { c.legs(relay, big) } + assertEquals(1_700_001_000L, c.band(relay, big)!!.minCreatedAt) + + // An equal-but-distinct instance keys the same way; it just misses the cache. + val copy = Filter(kinds = listOf(30382), authors = (1..500).map { it.toString(16).padStart(64, '0') }) + assertEquals(1_700_001_000L, c.band(relay, copy)?.minCreatedAt, "identity caching must not change the key") + } +} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/BisectingInsertTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/BisectingInsertTest.kt new file mode 100644 index 0000000000..2b813995c9 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/BisectingInsertTest.kt @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store + +import com.vitorpamplona.quartz.nip01Core.core.Event +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.test.runTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertTrue + +/** + * A bulk write fails as a unit, so without isolation one event the store cannot + * take costs its whole batch — 999 good events per bad one at the default size, + * dropped silently and counted as a multiple of the batch rather than as a number + * of bad events. These pin the isolation that stops that. + */ +class BisectingInsertTest { + private fun event(n: Int) = + Event( + id = n.toString().padStart(64, '0'), + pubKey = "a1".repeat(32), + createdAt = 1_700_000_000L + n, + kind = 1, + tags = emptyArray(), + content = "e$n", + sig = "b2".repeat(32), + ) + + /** Accepts everything except the named ids, which make the whole write throw. */ + private class Writer( + private val poison: Set, + ) { + val calls = mutableListOf() + var eventsWritten = 0 + + suspend fun write(batch: List): List { + calls.add(batch.size) + batch.firstOrNull { it.id in poison }?.let { + throw IndexOutOfBoundsException("Index: 1 Size: 1") + } + eventsWritten += batch.size + return batch.map { IEventStore.InsertOutcome.Accepted } + } + } + + private val gaveUp = mutableListOf() + + private suspend fun run( + events: List, + poison: Set, + ): Triple>> { + val writer = Writer(poison) + var accepted = 0 + val poisoned = mutableListOf>() + gaveUp.clear() + insertBisecting( + events = events, + write = { writer.write(it) }, + onOutcomes = { accepted += it.size }, + onPoison = { e, t -> poisoned.add(e to t) }, + onGaveUp = { batch, _ -> gaveUp.add(batch.size) }, + ) + return Triple(writer, accepted, poisoned) + } + + @Test + fun `a healthy batch is written once and costs nothing extra`() = + runTest { + val events = (1..64).map(::event) + val (writer, accepted, poisoned) = run(events, emptySet()) + + assertEquals(listOf(64), writer.calls, "no bisection on a batch that works") + assertEquals(64, accepted) + assertTrue(poisoned.isEmpty()) + } + + @Test + fun `one poison event costs only itself and not the batch`() = + runTest { + val events = (1..64).map(::event) + val bad = events[37].id + val (_, accepted, poisoned) = run(events, setOf(bad)) + + // This is the whole point: 63 of 64 still land. + assertEquals(63, accepted, "every event except the poison one must still be written") + assertEquals(1, poisoned.size) + assertEquals(bad, poisoned.single().first.id, "the isolated event is the one that throws") + assertTrue(poisoned.single().second is IndexOutOfBoundsException) + } + + @Test + fun `isolating stays logarithmic instead of falling back to one-by-one`() = + runTest { + val events = (1..1024).map(::event) + val (writer, accepted, _) = run(events, setOf(events[500].id)) + + assertEquals(1023, accepted) + // ~2*log2(n) writes, nowhere near the 1024 a per-event fallback would cost. + assertTrue(writer.calls.size < 32, "expected a logarithmic split, got ${writer.calls.size} writes") + } + + @Test + fun `several poison events are each isolated`() = + runTest { + val events = (1..64).map(::event) + val bad = setOf(events[0].id, events[31].id, events[63].id) + val (_, accepted, poisoned) = run(events, bad) + + assertEquals(61, accepted) + assertEquals(bad, poisoned.map { it.first.id }.toSet()) + } + + @Test + fun `a store-wide failure gives up instead of splitting all the way down`() = + runTest { + // Everything fails — a full disk, a dead engine. Splitting to singletons + // would cost ~2n writes at the worst possible moment. + val events = (1..1024).map(::event) + val (writer, accepted, poisoned) = run(events, events.map { it.id }.toSet()) + + assertEquals(0, accepted) + assertTrue( + writer.calls.size < 100, + "a store-wide failure must not cost ~2n writes; spent ${writer.calls.size}", + ) + // Nothing is silently lost: whatever isolation could not name is still + // handed back, so the caller can count it. + assertEquals(1024, poisoned.size + gaveUp.sum(), "every event must be accounted for") + assertTrue(gaveUp.isNotEmpty(), "the remainder should be reported as unisolated") + } + + @Test + fun `the budget is spent isolating rather than hoarded`() = + runTest { + // One bad event in 1024 must still be found — the guard bounds the + // pathological case without breaking the case it was built for. + val events = (1..1024).map(::event) + val (_, accepted, poisoned) = run(events, setOf(events[900].id)) + + assertEquals(1023, accepted) + assertEquals(events[900].id, poisoned.single().first.id) + assertTrue(gaveUp.isEmpty(), "a single bad event fits well inside the budget") + } + + @Test + fun `a batch of nothing but poison loses nothing else`() = + runTest { + val events = (1..4).map(::event) + val (_, accepted, poisoned) = run(events, events.map { it.id }.toSet()) + + assertEquals(0, accepted) + assertEquals(4, poisoned.size, "each one named, rather than one count of four") + } + + @Test + fun `cancellation propagates instead of being mistaken for a poison event`() = + runTest { + // Shutdown cancels the ingest scope mid-write. Treating that as "this + // event is bad" would drop good events and keep bisecting while the + // caller is trying to stop. + val events = (1..16).map(::event) + assertFailsWith { + insertBisecting( + events = events, + write = { throw CancellationException("shutting down") }, + onOutcomes = { }, + onPoison = { _, _ -> error("cancellation must not be reported as poison") }, + ) + } + } + + @Test + fun `an empty batch is a no-op`() = + runTest { + val (writer, accepted, poisoned) = run(emptyList(), emptySet()) + assertTrue(writer.calls.isEmpty()) + assertEquals(0, accepted) + assertTrue(poisoned.isEmpty()) + } +} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/HostStrikesTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/HostStrikesTest.kt new file mode 100644 index 0000000000..d86141992a --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/HostStrikesTest.kt @@ -0,0 +1,176 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip66RelayMonitor.reachability + +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * An outbox list is five figures of urls and mostly corpses. These pin the two + * rules that make the difference: failures count per HOST (or a filtering + * relay's hundreds of per-user urls never add up to anything), and a host that + * has ever delivered is never dropped (or a fan-out this wide sheds working + * relays on a race). + */ +class HostStrikesTest { + private fun url(u: String) = RelayUrlNormalizer.normalize(u) + + // ---- the authority key -------------------------------------------------- + + @Test + fun `per-user path urls on one host share an authority`() { + val a = HostStrikes.authorityOf("wss://filter.nostr.wine/npub1aaaa?broadcast=true") + val b = HostStrikes.authorityOf("wss://filter.nostr.wine/npub1bbbb") + assertEquals("filter.nostr.wine", a) + assertEquals(a, b) + assertEquals(a, HostStrikes.authorityOf("wss://filter.nostr.wine")) + } + + @Test + fun `a subdomain is not folded into its parent`() { + // Different servers. Shedding the filtering one must never take out the + // bare host, which may be perfectly open. + assertTrue( + HostStrikes.authorityOf("wss://filter.nostr.wine/npub1x") != + HostStrikes.authorityOf("wss://nostr.wine"), + ) + } + + @Test + fun `the port is part of the authority`() { + assertEquals("relay.example.com:443", HostStrikes.authorityOf("wss://relay.example.com:443/npub1z")) + assertTrue( + HostStrikes.authorityOf("wss://example.com:443") != HostStrikes.authorityOf("wss://example.com:8080"), + ) + } + + // ---- striking ----------------------------------------------------------- + + @Test + fun `one host is struck out by failures spread across its many urls`() { + // The whole point: no single url would ever reach the threshold alone. + val h = HostStrikes() + h.strike(url("wss://filter.example/npub1aaa")) + h.strike(url("wss://filter.example/npub1bbb")) + assertFalse(h.isDead(url("wss://filter.example/npub1ccc")), "two strikes is not yet a verdict") + + h.strike(url("wss://filter.example/npub1ccc")) + assertTrue(h.isDead(url("wss://filter.example/npub1ddd")), "the host is out — including urls never tried") + } + + @Test + fun `eviction returns a verdict exactly once for publishing`() { + // The eviction is the only finding that will ever exist about the sibling + // urls under this host: from here they are skipped without being dialled, + // so nothing observes them again. It must surface exactly once — silent + // would publish nothing, repeated would rewrite the record every strike. + val h = HostStrikes() + assertNull(h.strike(url("wss://filter.example/npub1")), "one strike is not a verdict") + assertNull(h.strike(url("wss://filter.example/npub2")), "two is not either") + + val evicted = h.strike(url("wss://filter.example/npub3")) + assertNotNull(evicted, "the third strike is the finding") + assertEquals("filter.example", evicted.authority) + assertEquals(3, evicted.strikes) + + assertNull(h.strike(url("wss://filter.example/npub4")), "already evicted — do not report it again") + } + + @Test + fun `a host that has delivered is never evicted so nothing is published`() { + val h = HostStrikes() + h.produced(url("wss://busy.example/npubY")) + repeat(5) { assertNull(h.strike(url("wss://busy.example/npub$it")), "ever-produced outranks any strike") } + } + + @Test + fun `striking one host leaves every other alone`() { + val h = HostStrikes() + repeat(5) { h.strike(url("wss://filter.example/npub$it")) } + assertTrue(h.isDead(url("wss://filter.example/npub1"))) + assertFalse(h.isDead(url("wss://example.com"))) + assertFalse(h.isDead(url("wss://other.example"))) + } + + @Test + fun `a host that ever delivered is never dead whichever way the race lands`() { + // At a hundred relays in flight one worker can strike an authority out at + // the same instant another is receiving from it. Ever-produced must win in + // both orders, which is why it overrides rather than clearing strikes. + val strikeFirst = HostStrikes() + repeat(3) { strikeFirst.strike(url("wss://busy.example/npub$it")) } + assertTrue(strikeFirst.isDead(url("wss://busy.example/npubX"))) + strikeFirst.produced(url("wss://busy.example/npubY")) + assertFalse(strikeFirst.isDead(url("wss://busy.example/npubX")), "a delivery revives the whole host") + + val produceFirst = HostStrikes() + produceFirst.produced(url("wss://busy.example/npubY")) + repeat(5) { produceFirst.strike(url("wss://busy.example/npub$it")) } + assertFalse(produceFirst.isDead(url("wss://busy.example/npubX")), "later strikes cannot bury it") + } + + @Test + fun `a zero strike limit disables eviction entirely`() { + val h = HostStrikes(strikeLimit = 0) + repeat(50) { h.strike(url("wss://filter.example/npub$it")) } + assertFalse(h.isDead(url("wss://filter.example/npub1"))) + } + + // ---- what a previous run already learned --------------------------------- + + @Test + fun `a relay a previous run proved dead is skipped without dialling`() { + val gone = url("wss://gone.example") + val h = HostStrikes(knownDead = setOf(gone)) + assertTrue(h.isDead(gone)) + assertFalse(h.isDead(url("wss://alive.example"))) + } + + @Test + fun `a known-dead relay that answers anyway is believed over the record`() { + // Relays come back. A TTL'd record is "not now" and never "never again": + // a delivery this cycle must beat what an earlier one wrote down. + val back = url("wss://back.example") + val h = HostStrikes(knownDead = setOf(back)) + h.produced(back) + assertFalse(h.isDead(back)) + } + + // ---- what gets written back --------------------------------------------- + + @Test + fun `only relays actually dialled are reported and delivery clears a failure`() { + val h = HostStrikes() + val good = url("wss://good.example") + val bad = url("wss://bad.example") + h.strike(bad) + h.strike(good) + h.produced(good) + + assertEquals(setOf(good), h.reachable) + assertEquals(setOf(bad), h.unreachable, "a relay that later delivered is not reported dead") + } +} diff --git a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/Unreachability.kt b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/Unreachability.kt new file mode 100644 index 0000000000..a9b1cf634e --- /dev/null +++ b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/Unreachability.kt @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip66RelayMonitor.reachability + +/** + * Whether a failure may be published as "this relay is unreachable". + * + * The distinction matters because the answer is PUBLISHED: a negative NIP-66 + * record is a signed, public statement about someone else's server. A relay + * that completes a handshake and then hangs up mid-page is emphatically + * reachable, and an exception thrown by the caller's own code says nothing + * about the relay at all. So this asks only about the connection itself — + * name resolution, routing, refusal, TLS. + * + * Unknown failures stay quiet: the cost of silence is one retry next cycle, + * the cost of being wrong is a false record carrying the monitor's signature. + */ +object Unreachability { + fun proves(e: Exception): Boolean = + when (e) { + is java.net.UnknownHostException, + is java.net.ConnectException, + is java.net.NoRouteToHostException, + is java.net.PortUnreachableException, + is javax.net.ssl.SSLHandshakeException, + -> true + + else -> false + } +} diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/UnreachabilityTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/UnreachabilityTest.kt new file mode 100644 index 0000000000..066f4b4b67 --- /dev/null +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/UnreachabilityTest.kt @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip66RelayMonitor.reachability + +import java.io.EOFException +import java.net.ConnectException +import java.net.SocketTimeoutException +import java.net.UnknownHostException +import javax.net.ssl.SSLHandshakeException +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +/** + * What a monitor is willing to SAY about someone else's relay. A negative + * NIP-66 record is signed and public, so only failures of the connection + * itself may be published as "unreachable". + */ +class UnreachabilityTest { + private fun proves(e: Exception) = Unreachability.proves(e) + + @Test + fun `a connection that never opened is unreachable`() { + assertTrue(proves(UnknownHostException("no such host"))) + assertTrue(proves(ConnectException("connection refused"))) + assertTrue(proves(SSLHandshakeException("cert expired"))) + } + + @Test + fun `a relay that hung up mid-transfer is not unreachable`() { + // A relay that answers the handshake in 50ms and then sends EOFException + // part-way through a large page is reachable; it declined to finish a + // query. Publishing "unreachable" would be a false statement about a + // working server. + assertFalse(proves(EOFException("stream closed"))) + } + + @Test + fun `our own bug is never the relay's fault`() { + assertFalse(proves(ConcurrentModificationException())) + assertFalse(proves(NullPointerException())) + assertFalse(proves(ClassCastException("HashMap\$Node cannot be cast"))) + } + + @Test + fun `an unrecognised failure stays quiet`() { + // Conservative on purpose: staying quiet costs one retry next cycle, + // being wrong costs a false record carrying the monitor's signature. + assertFalse(proves(SocketTimeoutException("read timed out"))) + assertFalse(proves(RuntimeException("something new"))) + } +} From b46063713e81a494dcf6867698075eca04ae72bb Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 03:48:04 +0000 Subject: [PATCH 007/132] Centralize the insert-rejection vocabulary in RejectionReason MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every store spelled its rejection reasons inline — the expired-event string was duplicated four times across ObservableEventStore and SQLiteEventStore. RejectionReason now carries the NIP-01 OK machine-readable prefixes plus the standard store reasons, so InsertOutcome.Rejected tallies and OK-frame building see one vocabulary no matter which store produced the outcome. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MfwV3xgMSmfxy16ujGPxGW --- .../quartz/nip01Core/store/IEventStore.kt | 2 +- .../nip01Core/store/ObservableEventStore.kt | 2 +- .../quartz/nip01Core/store/RejectionReason.kt | 58 +++++++++++++++++++ .../store/sqlite/SQLiteEventStore.kt | 9 +-- 4 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/RejectionReason.kt diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt index 8f77817522..68e1b18fb9 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt @@ -122,7 +122,7 @@ interface IEventStore : AutoCloseable { insert(event) InsertOutcome.Accepted } catch (e: Throwable) { - InsertOutcome.Rejected(e.message ?: e::class.simpleName ?: "insert failed") + InsertOutcome.Rejected(e.message ?: e::class.simpleName ?: RejectionReason.INSERT_FAILED) } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt index c314c4e3b2..6111f2106e 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt @@ -114,7 +114,7 @@ class ObservableEventStore( if (event.kind.isEphemeral()) { outcomes[i] = if (event.isExpired()) { - IEventStore.InsertOutcome.Rejected("blocked: Cannot insert an expired event") + IEventStore.InsertOutcome.Rejected(RejectionReason.EXPIRED) } else { IEventStore.InsertOutcome.Accepted } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/RejectionReason.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/RejectionReason.kt new file mode 100644 index 0000000000..a9262e6482 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/RejectionReason.kt @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store + +/** + * The machine-readable insert-rejection vocabulary, shared by every + * [IEventStore] implementation so the same condition always rejects with the + * same words — a caller tallying [IEventStore.InsertOutcome.Rejected] reasons, + * or a relay building `OK false` frames, must never see two stores spell + * "duplicate" differently. + * + * NIP-01: an `OK false` message SHOULD begin with a single-word + * machine-readable prefix followed by `:`. The `PREFIX_*` constants are that + * vocabulary; the full-sentence constants are the standard reasons the + * built-in stores emit. `replaced:` is not in NIP-01 but is the de-facto + * prefix (strfry and others) for a replaceable event that lost to a stored + * newer version — distinct from `duplicate:` (the exact event is already + * held). + */ +object RejectionReason { + // The NIP-01 machine-readable prefixes. + const val PREFIX_DUPLICATE = "duplicate:" + const val PREFIX_POW = "pow:" + const val PREFIX_BLOCKED = "blocked:" + const val PREFIX_RATE_LIMITED = "rate-limited:" + const val PREFIX_INVALID = "invalid:" + const val PREFIX_RESTRICTED = "restricted:" + const val PREFIX_ERROR = "error:" + + /** De-facto prefix (not in NIP-01) for a stale version of a replaceable/addressable event. */ + const val PREFIX_REPLACED = "replaced:" + + // The standard store reasons. + const val DUPLICATE = "duplicate: already have this event" + const val EXPIRED = "blocked: Cannot insert an expired event" + const val DELETED = "blocked: a deletion event exists" + const val VANISHED = "blocked: a request to vanish event exists" + const val REPLACED = "replaced: a newer version exists" + const val INSERT_FAILED = "error: insert failed" +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index 515ca0b916..80e31f58d6 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -37,6 +37,7 @@ import com.vitorpamplona.quartz.nip01Core.store.FtsReindexProgress import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip01Core.store.IdAndTime import com.vitorpamplona.quartz.nip01Core.store.RawEvent +import com.vitorpamplona.quartz.nip01Core.store.RejectionReason import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip40Expiration.isExpired import com.vitorpamplona.quartz.nip50Search.strippingSearchExtensions @@ -433,7 +434,7 @@ class SQLiteEventStore( } suspend fun insertEvent(event: Event) { - if (event.isExpired()) throw SQLiteException("blocked: Cannot insert an expired event") + if (event.isExpired()) throw SQLiteException(RejectionReason.EXPIRED) if (event.kind.isEphemeral()) return pool.useWriter { db -> @@ -489,7 +490,7 @@ class SQLiteEventStore( delta: LiveIndexDelta?, ): IEventStore.InsertOutcome { if (event.isExpired()) { - return IEventStore.InsertOutcome.Rejected("blocked: Cannot insert an expired event") + return IEventStore.InsertOutcome.Rejected(RejectionReason.EXPIRED) } if (event.kind.isEphemeral()) return IEventStore.InsertOutcome.Accepted @@ -509,7 +510,7 @@ class SQLiteEventStore( // ROLLBACK shouldn't mask the original cause. runCatching { db.execSQL("ROLLBACK TRANSACTION TO SAVEPOINT $sp") } runCatching { db.execSQL("RELEASE SAVEPOINT $sp") } - IEventStore.InsertOutcome.Rejected(e.message ?: e::class.simpleName ?: "insert failed") + IEventStore.InsertOutcome.Rejected(e.message ?: e::class.simpleName ?: RejectionReason.INSERT_FAILED) } } @@ -518,7 +519,7 @@ class SQLiteEventStore( private val delta: LiveIndexDelta?, ) : IEventStore.ITransaction { override fun insert(event: Event) { - if (event.isExpired()) throw SQLiteException("blocked: Cannot insert an expired event") + if (event.isExpired()) throw SQLiteException(RejectionReason.EXPIRED) if (event.kind.isEphemeral()) return innerInsertEvent(event, db, delta) From e54a546d107b44a88b879537277e6f7c9671c7ab Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 03:48:04 +0000 Subject: [PATCH 008/132] SearchQuery: parse quoted phrases and -word exclusions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NIP-50 search strings in the wild carry Google-style syntax the extension tokenizer could not see: "exact phrase" requirements, -"phrase" and -word exclusions. SearchQuery now lifts quoted spans BEFORE the extension pass — the order is load-bearing: the extension pass is quote-blind, so a span ending in an extension-shaped token would lose its closing quote, and lifting first also lets quotes protect extension-shaped tokens ("include:spam" is a phrase, not an extension). toSearchString() and stripExtensions() reassemble the full grammar; existing parses are unchanged. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MfwV3xgMSmfxy16ujGPxGW --- .../quartz/nip50Search/SearchQuery.kt | 155 ++++++++++++++---- .../quartz/nip50Search/SearchQueryTest.kt | 117 +++++++++++++ 2 files changed, 242 insertions(+), 30 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQuery.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQuery.kt index f96b620d5c..fbcb4c35d1 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQuery.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQuery.kt @@ -28,43 +28,70 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter * NIP-50 defines the [com.vitorpamplona.quartz.nip01Core.relay.filters.Filter.search] * field as "a string describing a query in a human-readable form", optionally * carrying `key:value` extension tokens such as `domain:example.com` or - * `language:en`. This class splits that raw string into the free-text [terms] - * and the recognized [extensions], giving relays (and search redirectors) a - * typed view of the query instead of forcing each one to re-parse the string. + * `language:en`. This class splits that raw string into the free-text [terms], + * the Google-style term syntax ([phrases], [notPhrases], [notTerms]), and the + * recognized [extensions], giving relays (and search redirectors) a typed view + * of the query instead of forcing each one to re-parse the string. * * Example: * ``` - * val q = SearchQuery.parse("best nostr apps domain:example.com language:en") - * q.terms // "best nostr apps" + * val q = SearchQuery.parse("best \"nostr apps\" -spam domain:example.com") + * q.terms // "best" + * q.phrases // ["nostr apps"] + * q.notTerms // ["spam"] * q.domain // "example.com" - * q.language // "en" * ``` * - * ## Tokenization + * ## Parse order (load-bearing) * - * The string is split on whitespace. A token is treated as an extension when: - * - it contains a `:`, - * - the part before the `:` is a non-empty run of lowercase ASCII letters - * (`a`–`z`), and - * - the part after the `:` is non-empty and does not start with `//` (so URLs - * like `https://example.com` stay in [terms]). + * Quoted spans are lifted off the RAW string FIRST, then the residual is + * tokenized for extensions, then `-word` exclusions split off. The order + * matters twice over: the extension pass is quote-blind, so a span ending in + * an extension-shaped token (`"pizza sort:rank" -spam`) would otherwise lose + * its closing quote and the unclosed quote would swallow the rest of the + * query; and lifting first lets quotes protect extension-shaped tokens — + * `"include:spam"` is the phrase [include, spam], not an extension. * - * Everything else is free text. Per NIP-50 unknown extensions are kept (so they - * can be forwarded to a backend) — relays "SHOULD ignore extensions they don't - * support", which this models by simply not having a typed accessor for them; - * they remain readable through [extensions] / [extension]. + * ## Term syntax * - * Extension keys are matched case-sensitively against the lowercase forms - * documented by NIP-50. Duplicate keys keep the last occurrence. + * - A `"quoted span"` is an exact-phrase requirement ([phrases]); `-"…"` is a + * phrase exclusion ([notPhrases]). A quote opens a span only at a token + * boundary — mid-token quotes stay ordinary characters. An unclosed span + * runs to the end of the string. Empty spans are dropped, but a positive + * phrase keeps content a text index may not hold ("⚡"): it is an + * unsatisfiable requirement the backend turns into provably-no-match — + * dropping it here would silently flip that into match-all. + * - A leading `-` on a 2+ character token makes it an exclusion ([notTerms], + * all leading dashes stripped); a lone `-` stays an ordinary term. There is + * no `-extension` syntax: extension keys are strictly `a`–`z`, so + * `-include:spam` fails the key test and becomes the excluded literal. + * + * ## Extension tokenization + * + * The residual is split on whitespace. A token is treated as an extension when + * it contains a `:`, the part before the `:` is a non-empty run of lowercase + * ASCII letters (`a`–`z`), and the part after is non-empty and does not start + * with `//` (so URLs like `https://example.com` stay in [terms]). Per NIP-50 + * unknown extensions are kept (readable through [extensions] / [extension]) so + * they can be forwarded to a backend; relays "SHOULD ignore extensions they + * don't support". Extension keys are matched case-sensitively against the + * lowercase forms documented by NIP-50. Duplicate keys keep the last + * occurrence. */ class SearchQuery( - /** The human-readable search terms with all extension tokens removed. */ + /** The loose human-readable search terms: extensions, phrases, and exclusions all removed. */ val terms: String, /** * All recognized `key:value` extension tokens, in the order they appeared. * Known keys: [INCLUDE], [DOMAIN], [LANGUAGE], [SENTIMENT], [NSFW]. */ val extensions: Map, + /** Exact-phrase requirements (`"nostr apps"`), quotes removed, in order. */ + val phrases: List = emptyList(), + /** Exact-phrase exclusions (`-"nostr apps"`), quotes removed, in order. */ + val notPhrases: List = emptyList(), + /** Single-word exclusions (`-spam`), dashes removed, in order. */ + val notTerms: List = emptyList(), ) { /** `true` when the query carries the `include:spam` token (NIP-50: disable spam filtering). */ val includeSpam: Boolean @@ -93,20 +120,41 @@ class SearchQuery( val nsfwIncluded: Boolean get() = nsfw ?: true + /** + * Whether the query REQUIRES any text — loose terms or phrases. Exclusions + * alone don't count: an exclusions-only query is plain recall minus the + * excluded words, not a ranked text search. + */ + val hasText: Boolean + get() = terms.isNotEmpty() || phrases.isNotEmpty() + /** Returns the raw value of an arbitrary extension key (including unknown ones), or null. */ fun extension(key: String): String? = extensions[key] - /** Returns true when there are no free-text terms (the query is extensions-only or empty). */ + /** Returns true when there are no loose free-text terms. Phrases don't count — see [hasText] for "any required text". */ fun isTermsEmpty(): Boolean = terms.isEmpty() /** - * Re-assembles a canonical NIP-50 search string: the free-text [terms] - * followed by each `key:value` extension. Useful for a redirector that - * normalizes the incoming query before forwarding it to a backend. + * Re-assembles a canonical NIP-50 search string: the free-text [terms], + * then each `"phrase"`, `-exclusion`, `-"phrase exclusion"`, and + * `key:value` extension. Canonical, not order-preserving. Useful for a + * redirector that normalizes the incoming query before forwarding it. */ fun toSearchString(): String = buildString { append(terms) + for (phrase in phrases) { + if (isNotEmpty()) append(' ') + append('"').append(phrase).append('"') + } + for (word in notTerms) { + if (isNotEmpty()) append(' ') + append('-').append(word) + } + for (phrase in notPhrases) { + if (isNotEmpty()) append(' ') + append("-\"").append(phrase).append('"') + } for ((key, value) in extensions) { if (isNotEmpty()) append(' ') append(key).append(':').append(value) @@ -134,20 +182,24 @@ class SearchQuery( private val WHITESPACE = Regex("\\s+") - /** Empty query — no terms and no extensions. */ + /** Empty query — no terms, no syntax, no extensions. */ val EMPTY = SearchQuery("", emptyMap()) /** * Parses a raw NIP-50 [search] string into a [SearchQuery]. A null or - * blank input yields [EMPTY]. + * blank input yields [EMPTY]. See the class KDoc for the grammar and + * why the quote pass runs before the extension pass. */ fun parse(search: String?): SearchQuery { if (search.isNullOrBlank()) return EMPTY + val quoted = liftQuotedSpans(search) val extensions = LinkedHashMap() val terms = StringBuilder() + val notTerms = ArrayList() - for (token in search.trim().split(WHITESPACE)) { + for (token in quoted.residual.trim().split(WHITESPACE)) { + if (token.isEmpty()) continue val colon = token.indexOf(':') if (colon > 0 && colon < token.length - 1) { val key = token.substring(0, colon) @@ -157,18 +209,60 @@ class SearchQuery( continue } } + if (token.length > 1 && token[0] == '-') { + notTerms += token.trimStart('-') + continue + } if (terms.isNotEmpty()) terms.append(' ') terms.append(token) } - return SearchQuery(terms.toString(), extensions) + return SearchQuery(terms.toString(), extensions, quoted.phrases, quoted.notPhrases, notTerms) } private fun isExtensionKey(key: String): Boolean = key.isNotEmpty() && key.all { it in 'a'..'z' } + /** The quoted spans lifted off the raw text, plus the residual for the extension and `-word` passes. */ + private class QuotedSpans( + val phrases: List, + val notPhrases: List, + val residual: String, + ) + + /** Stage one, over the RAW string: lift every `"…"` / `-"…"` span. See the class KDoc for the rules. */ + private fun liftQuotedSpans(text: String): QuotedSpans { + val phrases = ArrayList() + val notPhrases = ArrayList() + val residual = StringBuilder() + var i = 0 + var boundary = true + while (i < text.length) { + val c = text[i] + val neg = c == '-' && i + 1 < text.length && text[i + 1] == '"' + if (boundary && (c == '"' || neg)) { + val start = i + if (neg) 2 else 1 + val close = text.indexOf('"', start) + val end = if (close < 0) text.length else close + val span = text.substring(start, end).trim() + i = if (close < 0) text.length else close + 1 + if (span.isNotEmpty()) { + if (neg) notPhrases += span else phrases += span + } + // The lifted span's place stays a token boundary for what follows. + residual.append(' ') + } else { + residual.append(c) + boundary = c.isWhitespace() + i++ + } + } + return QuotedSpans(phrases, notPhrases, residual.toString()) + } + /** * Returns [search] with every `key:value` extension token removed, - * leaving only the free-text terms (tokenized as in [parse]). + * leaving the free-text query (terms, phrases, and exclusions, + * re-assembled as in [toSearchString]). * * Backends that hand the search string to an engine with its own * query syntax — e.g. SQLite FTS, where `:` is column-filter @@ -184,7 +278,8 @@ class SearchQuery( fun stripExtensions(search: String?): String? { if (search.isNullOrBlank()) return search val parsed = parse(search) - return if (parsed.extensions.isEmpty()) search else parsed.terms + if (parsed.extensions.isEmpty()) return search + return SearchQuery(parsed.terms, emptyMap(), parsed.phrases, parsed.notPhrases, parsed.notTerms).toSearchString() } } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQueryTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQueryTest.kt index 4864e04162..956c73fee6 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQueryTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQueryTest.kt @@ -199,4 +199,121 @@ class SearchQueryTest { assertSame(plain, mixed[0]) assertEquals("bitcoin", mixed[1].search) } + + // ---- quoted phrases and -word exclusions -------------------------------- + + @Test + fun quotedSpanBecomesPhrase() { + val q = SearchQuery.parse("best \"nostr apps\" today") + assertEquals("best today", q.terms) + assertEquals(listOf("nostr apps"), q.phrases) + assertTrue(q.notPhrases.isEmpty()) + assertTrue(q.hasText) + } + + @Test + fun negatedQuotedSpanBecomesPhraseExclusion() { + val q = SearchQuery.parse("pizza -\"pineapple pizza\"") + assertEquals("pizza", q.terms) + assertEquals(listOf("pineapple pizza"), q.notPhrases) + assertTrue(q.phrases.isEmpty()) + } + + @Test + fun minusWordBecomesExclusion() { + val q = SearchQuery.parse("pizza -pineapple") + assertEquals("pizza", q.terms) + assertEquals(listOf("pineapple"), q.notTerms) + // Exclusions alone are not required text. + assertFalse(SearchQuery.parse("-pineapple").hasText) + } + + @Test + fun loneMinusStaysATerm() { + val q = SearchQuery.parse("a - b") + assertEquals("a - b", q.terms) + assertTrue(q.notTerms.isEmpty()) + } + + @Test + fun allLeadingDashesAreStripped() { + assertEquals(listOf("word"), SearchQuery.parse("--word").notTerms) + } + + @Test + fun quotesProtectExtensionShapedTokens() { + // The quote pass runs BEFORE the extension pass, so a quoted + // extension-shaped token is a phrase, not an extension. + val q = SearchQuery.parse("\"include:spam\"") + assertEquals(listOf("include:spam"), q.phrases) + assertFalse(q.includeSpam) + assertTrue(q.extensions.isEmpty()) + } + + @Test + fun spanEndingInExtensionKeepsTrailingExclusion() { + // Quote-blind extension parsing would eat the closing quote of + // "pizza include:spam" and swallow the trailing -word; the quote-first + // order keeps the exclusion an exclusion. + val q = SearchQuery.parse("\"pizza include:spam\" -pineapple") + assertEquals(listOf("pizza include:spam"), q.phrases) + assertEquals(listOf("pineapple"), q.notTerms) + assertTrue(q.extensions.isEmpty()) + } + + @Test + fun minusOnExtensionShapedTokenExcludesTheLiteral() { + // There is no `-extension` syntax: keys are strictly a-z, so the `-` + // makes the whole token an excluded literal. + val q = SearchQuery.parse("pizza -include:spam") + assertEquals(listOf("include:spam"), q.notTerms) + assertFalse(q.includeSpam) + } + + @Test + fun unclosedQuoteRunsToEnd() { + val q = SearchQuery.parse("\"nostr apps today") + assertEquals(listOf("nostr apps today"), q.phrases) + assertEquals("", q.terms) + } + + @Test + fun midTokenQuoteStaysOrdinary() { + val q = SearchQuery.parse("don\"t panic") + assertEquals("don\"t panic", q.terms) + assertTrue(q.phrases.isEmpty()) + } + + @Test + fun emptySpansAreDropped() { + val q = SearchQuery.parse("a \"\" b -\"\"") + assertEquals("a b", q.terms) + assertTrue(q.phrases.isEmpty()) + assertTrue(q.notPhrases.isEmpty()) + } + + @Test + fun phrasesComposeWithExtensions() { + val q = SearchQuery.parse("\"nostr apps\" best domain:example.com -spam") + assertEquals("best", q.terms) + assertEquals(listOf("nostr apps"), q.phrases) + assertEquals(listOf("spam"), q.notTerms) + assertEquals("example.com", q.domain) + } + + @Test + fun toSearchStringRoundTripsFullGrammar() { + val q = SearchQuery.parse("best \"nostr apps\" -spam -\"bad phrase\" domain:example.com") + assertEquals("best \"nostr apps\" -spam -\"bad phrase\" domain:example.com", q.toSearchString()) + } + + @Test + fun stripExtensionsKeepsPhrasesAndExclusions() { + assertEquals( + "best \"nostr apps\" -spam", + SearchQuery.stripExtensions("best \"nostr apps\" -spam language:en"), + ) + // No extensions -> the original string comes back untouched. + assertEquals("best \"nostr apps\" -spam", SearchQuery.stripExtensions("best \"nostr apps\" -spam")) + } } From 564cafedc4d5b999b243a4df8b34326d97c91380 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 03:53:39 +0000 Subject: [PATCH 009/132] Replace insertBisecting with a Failed outcome on the batchInsert contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bisecting existed to reconstruct per-event attribution after a store threw a batch-wide exception. Better to never lose the attribution: batchInsert's contract now requires per-row isolation, with a third outcome telling whose fault a miss was. Rejected is the EVENT's fault (duplicate, expired, invalid, blocked) and is final; Failed is the STORE's fault (schema drift, a failed feed, a resource error) — the event was good, it is lost unless re-offered, and a rising Failed count means the store is broken rather than that upstreams send junk. Throwing is reserved for failures with no per-event answer (engine unreachable, transaction never started), readable as "nothing in this batch was written". Consumers updated: RelaySession maps Failed to OK false with NIP-01's "error:" prefix; IngestQueue converts a thrown batch and a missing outcome to Failed instead of Rejected; NdjsonImportExport counts failed apart from rejected; geode's MirrorWorker logs store failures at warn instead of folding them into debug-level rejections. BisectingInsert and its test are removed — with attribution guaranteed by the contract, retry-by-splitting has nothing left to do. Full :quartz:jvmTest passes. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y4Pi9YYMhdzTFxRiV2jF9R --- .../geode/mirror/MirrorWorker.kt | 11 + .../nip01Core/relay/server/RelaySession.kt | 7 + .../relay/server/backend/IngestQueue.kt | 8 +- .../relay/server/backend/LiveEventStore.kt | 4 + .../quartz/nip01Core/store/BisectingInsert.kt | 89 -------- .../quartz/nip01Core/store/IEventStore.kt | 41 +++- .../nip01Core/store/NdjsonImportExport.kt | 10 +- .../store/sqlite/SQLiteEventStore.kt | 5 +- .../nip01Core/store/BisectingInsertTest.kt | 200 ------------------ .../prodbench/ConcurrentIngestLossTest.kt | 1 + 10 files changed, 69 insertions(+), 307 deletions(-) delete mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/BisectingInsert.kt delete mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/BisectingInsertTest.kt diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt index 62936d17fe..942fe7769f 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt @@ -300,6 +300,13 @@ class MirrorWorker( rejected.incrementAndGet() Log.d("MirrorWorker") { "rejected ${msg.event.id}: ${outcome.reason}" } } + is IEventStore.InsertOutcome.Failed -> { + // The store's error, not the event's: the event + // was good and nothing will re-offer it. Louder + // than a rejection on purpose. + rejected.incrementAndGet() + Log.w("MirrorWorker") { "store failed ${msg.event.id}: ${outcome.reason}" } + } } } } catch (e: CancellationException) { @@ -431,6 +438,10 @@ class MirrorWorker( when (outcome) { IEventStore.InsertOutcome.Accepted -> accepted.incrementAndGet() is IEventStore.InsertOutcome.Rejected -> rejected.incrementAndGet() + is IEventStore.InsertOutcome.Failed -> { + rejected.incrementAndGet() + Log.w("MirrorWorker") { "store failed ${event.id}: ${outcome.reason}" } + } } } } catch (e: CancellationException) { diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/RelaySession.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/RelaySession.kt index 630ddec74a..9520dd9848 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/RelaySession.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/RelaySession.kt @@ -214,6 +214,13 @@ class RelaySession( is IEventStore.InsertOutcome.Rejected -> { send(OkMessage(cmd.event.id, false, outcome.reason)) } + + is IEventStore.InsertOutcome.Failed -> { + // The store's error, not the event's — NIP-01's + // machine-readable prefix for that is "error:". + val reason = outcome.reason + send(OkMessage(cmd.event.id, false, if (reason.startsWith("error:")) reason else "error: $reason")) + } } } } catch (_: ClosedSendChannelException) { diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/IngestQueue.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/IngestQueue.kt index 39f1191882..d955ed927c 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/IngestQueue.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/IngestQueue.kt @@ -268,8 +268,8 @@ class IngestQueue( * Run the SQLite transaction for the verified subset of [batch] * and stitch outcomes back to a per-batch-index array. Failed * verifies pre-mark `Rejected` and skip the insert. A whole-batch - * commit failure converts every persisted entry to `Rejected` - * with the throw message. + * commit failure converts every persisted entry to `Failed` with + * the throw message — the events were good; the store was not. */ private suspend fun runInsertStage( batch: List, @@ -293,7 +293,7 @@ class IngestQueue( } catch (e: Throwable) { Log.w("IngestQueue") { "batchInsert failed for ${toInsert.size} events: ${e.message}" } val reason = e.message ?: e::class.simpleName ?: "insert failed" - List(toInsert.size) { IEventStore.InsertOutcome.Rejected(reason) } + List(toInsert.size) { IEventStore.InsertOutcome.Failed(reason) } } } @@ -349,7 +349,7 @@ class IngestQueue( * inserts), so the message is informational, not user-facing. */ private val missingOutcome = - IEventStore.InsertOutcome.Rejected("internal error: missing outcome") + IEventStore.InsertOutcome.Failed("internal error: missing outcome") /** * Cap per batch. Sized to keep per-batch latency low (each diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/LiveEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/LiveEventStore.kt index 345a4337a5..2c37a5b904 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/LiveEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/LiveEventStore.kt @@ -180,6 +180,10 @@ class LiveEventStore( is IEventStore.InsertOutcome.Rejected -> { done.completeExceptionally(IllegalStateException(outcome.reason)) } + + is IEventStore.InsertOutcome.Failed -> { + done.completeExceptionally(IllegalStateException(outcome.reason)) + } } } done.await() diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/BisectingInsert.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/BisectingInsert.kt deleted file mode 100644 index deae869a8d..0000000000 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/BisectingInsert.kt +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2025 Vitor Pamplona - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the - * Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN - * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package com.vitorpamplona.quartz.nip01Core.store - -import com.vitorpamplona.quartz.nip01Core.core.Event -import kotlinx.coroutines.CancellationException - -/** - * Write [events] through [write]; if that throws, split the batch and write - * the halves, down to the single event the writer cannot take. - * - * A bulk write like [IEventStore.batchInsert] fails as a unit, so one bad - * event would otherwise cost the whole batch — 999 good events lost per bad - * one at a 1000-event batch, with no retry. Bisecting costs ~2·log2(n) extra - * writes on a failing batch, nothing on a healthy one, and ends holding the - * offender by itself for [onPoison] to report. Re-writing the good halves is - * safe: re-inserting an already-applied event is a duplicate the store - * rejects. - * - * Splitting assumes ONE event is at fault. When the store itself is refusing - * (a full disk, a dead engine) every half fails all the way down and - * isolation would turn one failed write into ~2n — precisely the wrong moment - * to multiply the load. So isolation spends a fixed [budget] of writes and - * hands the remainder to [onGaveUp]: "we could not say which" is a different - * fact from "this event is bad", and a caller should count them apart. - */ -suspend fun insertBisecting( - events: List, - write: suspend (List) -> List, - onOutcomes: (List) -> Unit, - onPoison: (Event, Throwable) -> Unit, - onGaveUp: (List, Throwable) -> Unit = { _, _ -> }, - budget: Int = ISOLATION_WRITE_BUDGET, -) = bisect(events, write, onOutcomes, onPoison, onGaveUp, intArrayOf(budget)) - -private suspend fun bisect( - events: List, - write: suspend (List) -> List, - onOutcomes: (List) -> Unit, - onPoison: (Event, Throwable) -> Unit, - onGaveUp: (List, Throwable) -> Unit, - budget: IntArray, -) { - if (events.isEmpty()) return - try { - onOutcomes(write(events)) - } catch (e: CancellationException) { - throw e - } catch (e: Throwable) { - if (events.size == 1) { - onPoison(events.single(), e) - return - } - if (budget[0] <= 0) { - onGaveUp(events, e) - return - } - budget[0] -= 2 - val mid = events.size / 2 - bisect(events.subList(0, mid), write, onOutcomes, onPoison, onGaveUp, budget) - bisect(events.subList(mid, events.size), write, onOutcomes, onPoison, onGaveUp, budget) - } -} - -/** - * Writes one batch may spend isolating its bad events before giving up. - * Isolating k bad events out of n costs about `2·k·log2(n)` writes, so 64 - * covers three in a 1000-event batch — past the rate seen in practice. What - * it really bounds is the store-wide case, where every write fails. - */ -const val ISOLATION_WRITE_BUDGET = 64 diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt index 8f77817522..16b82132e9 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt @@ -92,29 +92,52 @@ interface IEventStore : AutoCloseable { /** * Per-row outcome from [batchInsert]. The OK frame on the wire is - * built from this — `Accepted` becomes `OK true`, `Rejected.reason` - * becomes the false reason. NIP-01 says OK pairs to its EVENT by + * built from this — `Accepted` becomes `OK true`, the other two + * become the false reason. NIP-01 says OK pairs to its EVENT by * id, not by order, so callers may dispatch outcomes in any order. */ sealed class InsertOutcome { data object Accepted : InsertOutcome() + /** + * The EVENT's fault: policy said no — a duplicate, an expired + * or invalid event, a blocked author. Final: re-offering the + * same event yields the same answer, so dropping it is correct. + */ data class Rejected( val reason: String, ) : InsertOutcome() + + /** + * The STORE's fault: the event was acceptable but could not be + * written — schema drift, a failed feed, a resource error. The + * event is lost unless the caller re-offers it, and nothing + * else will. Callers should count these apart from [Rejected]: + * a rising [Rejected] is usually the protocol working + * (duplicates on a wide fan-out), while a rising [Failed] + * means the store is losing good events. + */ + data class Failed( + val reason: String, + ) : InsertOutcome() } /** - * Bulk insert in a single transaction with per-row error isolation. - * Returns one outcome per input event in the same order. + * Bulk insert with per-row attribution. Returns one outcome per + * input event in the same order. * - * Implementations must isolate per-row failures so one bad event - * doesn't roll back the others (SQLite uses SAVEPOINTs). If the - * outer commit itself fails, every entry in the returned list is - * `Rejected` with the commit-failure reason. + * Implementations must isolate per-row problems so one bad event + * never costs the batch: a policy refusal is [InsertOutcome.Rejected], + * a store-side write error is [InsertOutcome.Failed] (SQLite uses + * SAVEPOINTs for the isolation). Throwing is reserved for failures + * with no per-event answer — the engine unreachable, the transaction + * never started — and a caller may read a throw as "nothing in this + * batch was written". * * Default impl runs each insert in its own transaction — correct - * but loses the group-commit win. SQLite overrides this. + * but loses the group-commit win — and cannot classify a throw from + * [insert], so it reports `Rejected`. Implementations that can tell + * a refusal from a write error should override and say which. */ suspend fun batchInsert(events: List): List = events.map { event -> diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/NdjsonImportExport.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/NdjsonImportExport.kt index d7c85a742a..5c9125bd2e 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/NdjsonImportExport.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/NdjsonImportExport.kt @@ -52,15 +52,17 @@ object NdjsonImportExport { val imported: Long, /** Events the store rejected — overwhelmingly duplicates (unique-id). */ val rejected: Long, + /** Good events the store could not write — a store-side error, not the event's. */ + val failed: Long, /** Events dropped for a bad signature (only when verifying). */ val invalid: Long, /** Lines that didn't parse as a NIP-01 event. */ val malformed: Long, ) { - operator fun plus(o: ImportStats) = ImportStats(read + o.read, imported + o.imported, rejected + o.rejected, invalid + o.invalid, malformed + o.malformed) + operator fun plus(o: ImportStats) = ImportStats(read + o.read, imported + o.imported, rejected + o.rejected, failed + o.failed, invalid + o.invalid, malformed + o.malformed) companion object { - val ZERO = ImportStats(0, 0, 0, 0, 0) + val ZERO = ImportStats(0, 0, 0, 0, 0, 0) } } @@ -81,6 +83,7 @@ object NdjsonImportExport { var read = 0L var imported = 0L var rejected = 0L + var failed = 0L var invalid = 0L var malformed = 0L val batch = ArrayList(batchSize) @@ -91,6 +94,7 @@ object NdjsonImportExport { when (outcome) { IEventStore.InsertOutcome.Accepted -> imported++ is IEventStore.InsertOutcome.Rejected -> rejected++ + is IEventStore.InsertOutcome.Failed -> failed++ } } batch.clear() @@ -112,7 +116,7 @@ object NdjsonImportExport { if (batch.size >= batchSize) flush() } flush() - return ImportStats(read, imported, rejected, invalid, malformed) + return ImportStats(read, imported, rejected, failed, invalid, malformed) } /** diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index 515ca0b916..93f3bb5bb6 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -463,8 +463,9 @@ class SQLiteEventStore( * stream still surfaces them; persistence is intentionally a * no-op per NIP-01. * - * Outer-commit failure throws; the caller treats every entry as - * `Rejected` (this is what the IEventStore contract documents). + * Outer-commit failure throws; per the IEventStore contract a + * throw means "nothing in this batch was written", and the + * IngestQueue converts it to per-event `Failed`. */ suspend fun batchInsertEvents(events: List): List { if (events.isEmpty()) return emptyList() diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/BisectingInsertTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/BisectingInsertTest.kt deleted file mode 100644 index 2b813995c9..0000000000 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/store/BisectingInsertTest.kt +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright (c) 2025 Vitor Pamplona - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the - * Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN - * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package com.vitorpamplona.quartz.nip01Core.store - -import com.vitorpamplona.quartz.nip01Core.core.Event -import kotlinx.coroutines.CancellationException -import kotlinx.coroutines.test.runTest -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertFailsWith -import kotlin.test.assertTrue - -/** - * A bulk write fails as a unit, so without isolation one event the store cannot - * take costs its whole batch — 999 good events per bad one at the default size, - * dropped silently and counted as a multiple of the batch rather than as a number - * of bad events. These pin the isolation that stops that. - */ -class BisectingInsertTest { - private fun event(n: Int) = - Event( - id = n.toString().padStart(64, '0'), - pubKey = "a1".repeat(32), - createdAt = 1_700_000_000L + n, - kind = 1, - tags = emptyArray(), - content = "e$n", - sig = "b2".repeat(32), - ) - - /** Accepts everything except the named ids, which make the whole write throw. */ - private class Writer( - private val poison: Set, - ) { - val calls = mutableListOf() - var eventsWritten = 0 - - suspend fun write(batch: List): List { - calls.add(batch.size) - batch.firstOrNull { it.id in poison }?.let { - throw IndexOutOfBoundsException("Index: 1 Size: 1") - } - eventsWritten += batch.size - return batch.map { IEventStore.InsertOutcome.Accepted } - } - } - - private val gaveUp = mutableListOf() - - private suspend fun run( - events: List, - poison: Set, - ): Triple>> { - val writer = Writer(poison) - var accepted = 0 - val poisoned = mutableListOf>() - gaveUp.clear() - insertBisecting( - events = events, - write = { writer.write(it) }, - onOutcomes = { accepted += it.size }, - onPoison = { e, t -> poisoned.add(e to t) }, - onGaveUp = { batch, _ -> gaveUp.add(batch.size) }, - ) - return Triple(writer, accepted, poisoned) - } - - @Test - fun `a healthy batch is written once and costs nothing extra`() = - runTest { - val events = (1..64).map(::event) - val (writer, accepted, poisoned) = run(events, emptySet()) - - assertEquals(listOf(64), writer.calls, "no bisection on a batch that works") - assertEquals(64, accepted) - assertTrue(poisoned.isEmpty()) - } - - @Test - fun `one poison event costs only itself and not the batch`() = - runTest { - val events = (1..64).map(::event) - val bad = events[37].id - val (_, accepted, poisoned) = run(events, setOf(bad)) - - // This is the whole point: 63 of 64 still land. - assertEquals(63, accepted, "every event except the poison one must still be written") - assertEquals(1, poisoned.size) - assertEquals(bad, poisoned.single().first.id, "the isolated event is the one that throws") - assertTrue(poisoned.single().second is IndexOutOfBoundsException) - } - - @Test - fun `isolating stays logarithmic instead of falling back to one-by-one`() = - runTest { - val events = (1..1024).map(::event) - val (writer, accepted, _) = run(events, setOf(events[500].id)) - - assertEquals(1023, accepted) - // ~2*log2(n) writes, nowhere near the 1024 a per-event fallback would cost. - assertTrue(writer.calls.size < 32, "expected a logarithmic split, got ${writer.calls.size} writes") - } - - @Test - fun `several poison events are each isolated`() = - runTest { - val events = (1..64).map(::event) - val bad = setOf(events[0].id, events[31].id, events[63].id) - val (_, accepted, poisoned) = run(events, bad) - - assertEquals(61, accepted) - assertEquals(bad, poisoned.map { it.first.id }.toSet()) - } - - @Test - fun `a store-wide failure gives up instead of splitting all the way down`() = - runTest { - // Everything fails — a full disk, a dead engine. Splitting to singletons - // would cost ~2n writes at the worst possible moment. - val events = (1..1024).map(::event) - val (writer, accepted, poisoned) = run(events, events.map { it.id }.toSet()) - - assertEquals(0, accepted) - assertTrue( - writer.calls.size < 100, - "a store-wide failure must not cost ~2n writes; spent ${writer.calls.size}", - ) - // Nothing is silently lost: whatever isolation could not name is still - // handed back, so the caller can count it. - assertEquals(1024, poisoned.size + gaveUp.sum(), "every event must be accounted for") - assertTrue(gaveUp.isNotEmpty(), "the remainder should be reported as unisolated") - } - - @Test - fun `the budget is spent isolating rather than hoarded`() = - runTest { - // One bad event in 1024 must still be found — the guard bounds the - // pathological case without breaking the case it was built for. - val events = (1..1024).map(::event) - val (_, accepted, poisoned) = run(events, setOf(events[900].id)) - - assertEquals(1023, accepted) - assertEquals(events[900].id, poisoned.single().first.id) - assertTrue(gaveUp.isEmpty(), "a single bad event fits well inside the budget") - } - - @Test - fun `a batch of nothing but poison loses nothing else`() = - runTest { - val events = (1..4).map(::event) - val (_, accepted, poisoned) = run(events, events.map { it.id }.toSet()) - - assertEquals(0, accepted) - assertEquals(4, poisoned.size, "each one named, rather than one count of four") - } - - @Test - fun `cancellation propagates instead of being mistaken for a poison event`() = - runTest { - // Shutdown cancels the ingest scope mid-write. Treating that as "this - // event is bad" would drop good events and keep bisecting while the - // caller is trying to stop. - val events = (1..16).map(::event) - assertFailsWith { - insertBisecting( - events = events, - write = { throw CancellationException("shutting down") }, - onOutcomes = { }, - onPoison = { _, _ -> error("cancellation must not be reported as poison") }, - ) - } - } - - @Test - fun `an empty batch is a no-op`() = - runTest { - val (writer, accepted, poisoned) = run(emptyList(), emptySet()) - assertTrue(writer.calls.isEmpty()) - assertEquals(0, accepted) - assertTrue(poisoned.isEmpty()) - } -} diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/ConcurrentIngestLossTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/ConcurrentIngestLossTest.kt index cd1e7b21ba..335d587cc6 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/ConcurrentIngestLossTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/ConcurrentIngestLossTest.kt @@ -130,6 +130,7 @@ class ConcurrentIngestLossTest { when (outcome) { is IEventStore.InsertOutcome.Accepted -> accepted.add(e.id) is IEventStore.InsertOutcome.Rejected -> rejected.incrementAndGet() + is IEventStore.InsertOutcome.Failed -> rejected.incrementAndGet() } window.release() if (remaining.decrementAndGet() == 0) done.complete(Unit) From 19d8e3069de5ca5144307bb3ab1ba153d24a1811 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 04:24:19 +0000 Subject: [PATCH 010/132] Align the sync accessories with quartz vocabulary; geode catch-up resumes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Renames from review: SyncBands -> SyncCoverage ("sync" reads negentropy-ish in quartz, and coverage is the role — the bands are the records), and PagingProgress moves into relay.client.paging as PagingWindowProgress, beside RelayLoadingCursors and RelayPagingProgress, with its docs swept from "walk" dialect to quartz's pagination vocabulary and cross-references delineating the three: cursors are in-memory positions for demand-driven UI paging, the window progress is fraction/ETA for a bulk pagination over a known window, coverage is persistent intervals that license skipping work. geode adopts both halves of the new contract. MirrorWorker counts InsertOutcome.Failed in its own `failed` counter instead of folding it into `rejected`, and the down catch-up gains resume memory: SyncCoverageFile persists SyncCoverage next to the event database (admin state-file convention, temp-file + atomic move, daemon flush), and runCatchUpDown asks only for the legs outside the covered band. Bands are keyed on the stable scoped filter — never the boot window, whose since/until change every start — and clamped to the window, which only slides forward, so an old band can never license skipping a range an earlier boot could not ask about. A clean reconcile records completeness through its snapshot instant; a paged fallback earns only the span it saw. For an upstream without NIP-77 this turns the every-boot full re-download of the backfill window into a resumed walk. Off unless wired: MirrorWorker's coverage parameter defaults to null and in-memory stores keep no state file, so existing tests and setups are unchanged. Full :quartz:jvmTest and :geode:test pass. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Y4Pi9YYMhdzTFxRiV2jF9R --- .../kotlin/com/vitorpamplona/geode/Main.kt | 16 ++ .../geode/config/StaticConfig.kt | 10 ++ .../geode/mirror/MirrorWorker.kt | 124 ++++++++++++--- .../geode/mirror/SyncCoverageFile.kt | 147 ++++++++++++++++++ .../geode/mirror/SyncCoverageFileTest.kt | 130 ++++++++++++++++ .../{SyncBands.kt => SyncCoverage.kt} | 7 +- .../PagingWindowProgress.kt} | 60 +++---- .../{SyncBandsTest.kt => SyncCoverageTest.kt} | 58 +++---- .../PagingWindowProgressTest.kt} | 32 ++-- 9 files changed, 489 insertions(+), 95 deletions(-) create mode 100644 geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt create mode 100644 geode/src/test/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFileTest.kt rename quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/{SyncBands.kt => SyncCoverage.kt} (97%) rename quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/{accessories/PagingProgress.kt => paging/PagingWindowProgress.kt} (62%) rename quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/{SyncBandsTest.kt => SyncCoverageTest.kt} (93%) rename quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/{accessories/PagingProgressTest.kt => paging/PagingWindowProgressTest.kt} (84%) diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt index 729b95b3a3..2026a7f3ba 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt @@ -28,6 +28,7 @@ import com.vitorpamplona.geode.config.StaticConfig import com.vitorpamplona.geode.mirror.MirrorDirection import com.vitorpamplona.geode.mirror.MirrorUpstream import com.vitorpamplona.geode.mirror.MirrorWorker +import com.vitorpamplona.geode.mirror.SyncCoverageFile import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl @@ -411,6 +412,16 @@ private fun serve(args: Array) { require(upstreams.none { it.url.displayUrl() == advertisedIdentity }) { "[[mirror]] must not list this relay's own URL ($advertisedUrl)" } + // Resume state for the mirror catch-up, following the admin state-file + // convention: next to the event database unless configured. An in-memory + // store keeps none — bands only pay off across restarts. + val syncCoverage = + if (upstreams.isEmpty()) { + null + } else { + (config.options.mirror_sync_state_file ?: config.database.file?.let { "$it.sync-coverage.json" }) + ?.let { SyncCoverageFile(File(it)) } + } val mirror = if (upstreams.isEmpty()) { null @@ -425,6 +436,9 @@ private fun serve(args: Array) { // for the historical window, then live REQ tail. Auto-falls back // to paged REQ for upstreams without NIP-77. negentropyBackfill = true, + // Without NIP-77 the catch-up is a paged re-download; the + // coverage bands remember what previous boots already walked. + coverage = syncCoverage?.coverage, ).also { it.start() } } @@ -462,6 +476,8 @@ private fun serve(args: Array) { // queue and store beneath them shut down. runCatching { maintenanceScope.cancel() } runCatching { mirror?.close() } + // After the mirror, so the final flush carries the last bands. + runCatching { syncCoverage?.close() } runCatching { server.stop() } runCatching { relay.close() } }, diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/config/StaticConfig.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/config/StaticConfig.kt index dffe4e50e3..8cfecb4e41 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/config/StaticConfig.kt +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/config/StaticConfig.kt @@ -185,6 +185,16 @@ data class StaticConfig( * that don't offer NIP-50 at all (strfry, for example). */ val full_text_search: Boolean = true, + /** + * Where the mirror catch-up's resume state lives: the per-upstream + * `created_at` coverage bands (quartz's `SyncCoverage`). Without it + * every restart re-syncs each upstream's whole backfill window — + * a full re-download for an upstream without NIP-77. Defaults to + * `.sync-coverage.json` when the store is + * file-backed; an in-memory store keeps no resume state (bands + * only pay off across restarts). + */ + val mirror_sync_state_file: String? = null, ) /** diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt index 942fe7769f..5916a5f794 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt @@ -23,6 +23,7 @@ package com.vitorpamplona.geode.mirror import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper import com.vitorpamplona.quartz.nip01Core.relay.client.NostrClient +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.SyncCoverage import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.negentropyReconcile import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.negentropySyncOrFetch import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener @@ -159,6 +160,13 @@ class MirrorWorker( * transparent and needs no separate toggle. */ private val negentropyBackfill: Boolean = false, + /** + * Resume memory for the down catch-up, shared across upstreams and — via + * [SyncCoverageFile] — across restarts. Null keeps the old behavior: + * every boot re-syncs the whole backfill window, which for an upstream + * without NIP-77 is a full re-download. + */ + private val coverage: SyncCoverage? = null, ) : AutoCloseable { private val scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) @@ -201,6 +209,14 @@ class MirrorWorker( /** Events the store rejected — mostly duplicate replays after a reconnect. */ val rejected = AtomicLong(0) + /** + * Good events the store could not write ([IEventStore.InsertOutcome.Failed]). + * The store's fault, not the event's, and nothing re-offers them — counted + * apart from [rejected] so a schema drift reads as store damage rather + * than as upstreams sending junk. + */ + val failed = AtomicLong(0) + /** * Deliveries dropped before ever reaching the store: events outside * the [MirrorUpstream.filter] scope (an upstream answering outside @@ -304,7 +320,7 @@ class MirrorWorker( // The store's error, not the event's: the event // was good and nothing will re-offer it. Louder // than a rejection on purpose. - rejected.incrementAndGet() + failed.incrementAndGet() Log.w("MirrorWorker") { "store failed ${msg.event.id}: ${outcome.reason}" } } } @@ -419,11 +435,21 @@ class MirrorWorker( initialSince: Long, until: Long, ) { - val catchUpFilter = scopedBase.copy(since = initialSince, until = until) - // Reconcile against what we already hold in this window → download only - // the diff (like `strfry sync`). No store wired → empty local set → the - // whole window is downloaded and the store's unique-id constraint dedups. - val localEntries = store?.snapshotIdsForNegentropy(listOf(catchUpFilter)) ?: emptyList() + // Resume memory: coverage is keyed on the STABLE scoped filter, never + // on the boot window — whose since/until change every start and would + // never match a stored band. The window only slides forward + // (initialSince = boot − backfillSeconds), so a band recorded against + // an earlier boot's lower floor never licenses skipping a range this + // boot can ask about but the last one could not. + val legs = + coverage + ?.legs(up.url, scopedBase) + ?.mapNotNull { clampToWindow(it, initialSince, until) } + ?: listOf(scopedBase.copy(since = initialSince, until = until)) + if (legs.isEmpty()) { + Log.i("MirrorWorker") { "catch-up from ${up.url.url}: window already covered - nothing outside the synced band" } + return + } // Bounded hand-off → one ingest consumer. `onEvent` can't suspend, so it // blocks here when the sink falls behind; because negentropySyncOrFetch's @@ -439,7 +465,7 @@ class MirrorWorker( IEventStore.InsertOutcome.Accepted -> accepted.incrementAndGet() is IEventStore.InsertOutcome.Rejected -> rejected.incrementAndGet() is IEventStore.InsertOutcome.Failed -> { - rejected.incrementAndGet() + failed.incrementAndGet() Log.w("MirrorWorker") { "store failed ${event.id}: ${outcome.reason}" } } } @@ -454,24 +480,59 @@ class MirrorWorker( } try { - val result = - client.negentropySyncOrFetch( - relay = up.url, - filter = catchUpFilter, - localEntries = localEntries, - onEvent = { event -> - // Same containment as the live path: even a trusted - // upstream may only inject events inside the declared scope. - if (up.filter == null || up.filter.match(event)) { - handoff.trySendBlocking(event) - } else { - filtered.incrementAndGet() - } - }, + var downloaded = 0 + var paged = false + for (leg in legs) { + // Reconcile against what we already hold in this leg → download + // only the diff (like `strfry sync`). No store wired → empty + // local set → the whole leg is downloaded and the store's + // unique-id constraint dedups. + val localEntries = store?.snapshotIdsForNegentropy(listOf(leg)) ?: emptyList() + // Coverage is stamped from when the local ids were read — that + // is the state the relay is being compared against. + val syncStartedAt = TimeUtils.now() + var seenMin: Long? = null + var seenMax: Long? = null + val result = + client.negentropySyncOrFetch( + relay = up.url, + filter = leg, + localEntries = localEntries, + onEvent = { event -> + // Same containment as the live path: even a trusted + // upstream may only inject events inside the declared scope. + if (up.filter == null || up.filter.match(event)) { + // Only plausible stamps widen a band — one + // misdated event must not discard the rest. + if (SyncCoverage.isPlausible(event.createdAt)) { + seenMin = minOf(seenMin ?: event.createdAt, event.createdAt) + seenMax = maxOf(seenMax ?: event.createdAt, event.createdAt) + } + handoff.trySendBlocking(event) + } else { + filtered.incrementAndGet() + } + }, + ) + downloaded += result.downloaded + paged = paged || result.pagedFallback + // Recorded per leg, so a failure between legs keeps the ground + // the first one gained. A clean reconcile is complete through + // the instant its snapshot was read; a paged fallback earns + // only the span it actually saw. + coverage?.record( + up.url, + scopedBase, + seenMin, + seenMax, + paged = result.pagedFallback, + reconciledThrough = if (result.pagedFallback) null else syncStartedAt, ) + } Log.i("MirrorWorker") { - val how = if (result.pagedFallback) "paged REQ (upstream has no NIP-77)" else "negentropy" - "catch-up from ${up.url.url}: ${result.downloaded} events via $how" + val how = if (paged) "paged REQ (upstream has no NIP-77)" else "negentropy" + val resumed = if (coverage != null && legs.size > 1) " [resumed: ${legs.size} legs outside the synced band]" else "" + "catch-up from ${up.url.url}: $downloaded events via $how$resumed" } } catch (e: CancellationException) { throw e @@ -731,3 +792,20 @@ class MirrorWorker( const val UP_SYNC_SETTLE_MS = 1_500L } } + +/** + * [leg] intersected with the boot window `[since, until]`, or null when the + * band already covers everything this window could ask. Coverage legs come + * off the stable scoped filter and are unbounded on one side; the catch-up + * only ever asks inside its own window. + */ +internal fun clampToWindow( + leg: Filter, + since: Long, + until: Long, +): Filter? { + val newSince = maxOf(leg.since ?: since, since) + val newUntil = minOf(leg.until ?: until, until) + if (newSince > newUntil) return null + return leg.copy(since = newSince, until = newUntil) +} diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt new file mode 100644 index 0000000000..f02d8f56f5 --- /dev/null +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.geode.mirror + +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.SyncCoverage +import com.vitorpamplona.quartz.utils.Log +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.boolean +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.long +import kotlinx.serialization.json.put +import java.io.File +import java.nio.file.Files +import java.nio.file.StandardCopyOption + +/** + * File persistence for [SyncCoverage], so the mirror's catch-up resumes + * across restarts instead of re-syncing each upstream's whole backfill + * window — which, for an upstream without NIP-77, is a full re-download + * every boot. + * + * Same shape as the admin state file: JSON next to the event database, + * written via a temp file and an atomic move so a reader never sees a half + * map. A daemon timer flushes changed state so progress survives a hard + * kill; [close] flushes the rest. A corrupt file starts fresh — the cost of + * losing it is one re-sync, the cost of refusing to start is the relay. + */ +class SyncCoverageFile( + private val file: File, + flushSeconds: Long = DEFAULT_FLUSH_SECONDS, +) : AutoCloseable { + @Volatile private var dirty = false + + val coverage = SyncCoverage(onChange = { dirty = true }) + + private val flusher: Thread + + init { + load() + // Loading marks every restored band dirty; the file already has them. + dirty = false + flusher = + Thread { + while (!Thread.currentThread().isInterrupted) { + try { + Thread.sleep(flushSeconds * 1000) + } catch (_: InterruptedException) { + return@Thread + } + flush() + } + }.apply { + isDaemon = true + name = "mirror-sync-coverage-flush" + start() + } + } + + /** Write the map if anything changed since the last write. */ + @Synchronized + fun flush() { + if (!dirty) return + dirty = false + save() + } + + override fun close() { + flusher.interrupt() + flush() + } + + private fun load() { + if (!file.isFile) return + runCatching { + val root = Json.parseToJsonElement(file.readText()).jsonObject + coverage.restore( + root.mapValues { (_, v) -> + val o = v.jsonObject + SyncCoverage.Band( + o.getValue("min").jsonPrimitive.long, + o.getValue("max").jsonPrimitive.long, + o["complete"]?.jsonPrimitive?.boolean ?: false, + o["fullAt"]?.jsonPrimitive?.long ?: 0L, + ) + }, + ) + }.onFailure { + Log.w("SyncCoverageFile") { "could not read ${file.path} (${it.message}); starting fresh" } + } + } + + @Synchronized + private fun save() { + runCatching { + val doc = + buildJsonObject { + coverage.export().forEach { (key, band) -> + put( + key, + buildJsonObject { + put("min", band.minCreatedAt) + put("max", band.maxCreatedAt) + put("complete", band.complete) + put("fullAt", band.fullAt) + }, + ) + } + } + file.parentFile?.mkdirs() + val tmp = File(file.parentFile ?: File("."), "${file.name}.tmp") + tmp.writeText(json.encodeToString(JsonObject.serializer(), doc)) + Files.move(tmp.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING) + }.onFailure { + Log.w("SyncCoverageFile") { "could not write ${file.path}: ${it.message}" } + } + } + + companion object { + // Pretty-printed: this file is read by a human debugging why an + // upstream re-synced. + private val json = Json { prettyPrint = true } + + // Often enough that a kill costs little, rare enough to be free. + private const val DEFAULT_FLUSH_SECONDS = 30L + } +} diff --git a/geode/src/test/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFileTest.kt b/geode/src/test/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFileTest.kt new file mode 100644 index 0000000000..4a02507db8 --- /dev/null +++ b/geode/src/test/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFileTest.kt @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.geode.mirror + +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer +import java.io.File +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * The catch-up's resume memory across restarts. Without it every boot + * re-syncs each upstream's whole backfill window — a full re-download for an + * upstream without NIP-77. These pin the restart round-trip and the window + * clamping that keys bands on the stable filter rather than the sliding + * boot window. + */ +class SyncCoverageFileTest { + private val relay = RelayUrlNormalizer.normalize("wss://relay.example") + private val profiles = Filter(kinds = listOf(0)) + + private fun tempFile(): File { + val f = File.createTempFile("sync-coverage", ".json") + f.delete() + return f + } + + @Test + fun `bands survive a restart`() { + val f = tempFile() + SyncCoverageFile(f).use { + it.coverage.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = true) + } + + // A fresh instance, as a restart would build. + SyncCoverageFile(f).use { reopened -> + val band = reopened.coverage.band(relay, profiles)!! + assertEquals(1_700_001_000L, band.minCreatedAt) + assertEquals(1_700_002_000L, band.maxCreatedAt) + assertFalse(band.complete) + } + } + + @Test + fun `a complete band survives with its completeness`() { + val f = tempFile() + SyncCoverageFile(f).use { + it.coverage.record(relay, profiles, null, null, paged = false, reconciledThrough = 1_700_005_000L) + } + SyncCoverageFile(f).use { reopened -> + assertTrue(reopened.coverage.band(relay, profiles)!!.complete) + } + } + + @Test + fun `a corrupt file starts fresh instead of refusing to start`() { + val f = tempFile() + f.writeText("{ not json") + SyncCoverageFile(f).use { + assertNull(it.coverage.band(relay, profiles)) + } + } + + @Test + fun `recording does not write but closing does`() { + val f = tempFile() + val store = SyncCoverageFile(f) + store.coverage.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = true) + assertFalse(f.isFile, "a record marks dirty; only a flush writes") + store.close() + assertTrue(f.isFile, "close flushes") + } + + @Test + fun `reopening without new records does not rewrite the file`() { + val f = tempFile() + SyncCoverageFile(f).use { + it.coverage.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = true) + } + val written = f.lastModified() + SyncCoverageFile(f).close() + assertEquals(written, f.lastModified(), "restoring bands must not mark the store dirty") + } + + // ---- the window clamp -------------------------------------------------- + + @Test + fun `an unbanded filter clamps to exactly the boot window`() { + val leg = clampToWindow(profiles, since = 1_000L, until = 2_000L)!! + assertEquals(1_000L, leg.since) + assertEquals(2_000L, leg.until) + } + + @Test + fun `a leg outside the window is dropped rather than inverted`() { + // The band covers past the window's floor: the older leg would ask + // [since..band.min] with since above until — a range nothing can be in. + val olderLeg = profiles.copy(until = 500L) + assertNull(clampToWindow(olderLeg, since = 1_000L, until = 2_000L)) + } + + @Test + fun `a leg inside the window keeps its own tighter bound`() { + val newerLeg = profiles.copy(since = 1_500L) + val clamped = clampToWindow(newerLeg, since = 1_000L, until = 2_000L)!! + assertEquals(1_500L, clamped.since, "the band's ceiling wins over the window floor") + assertEquals(2_000L, clamped.until) + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncBands.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt similarity index 97% rename from quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncBands.kt rename to quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt index 68bff2ca85..c6ddff4dce 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncBands.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt @@ -50,8 +50,13 @@ import com.vitorpamplona.quartz.utils.concurrent.ConcurrentMap * Persistence is the caller's: [export] the map on a schedule and [restore] * it at startup. [onChange] fires whenever a band changes, so a persistence * layer can mark itself dirty without polling. + * + * Not to be confused with the `relay.client.paging` package: its + * `RelayLoadingCursors` are in-memory POSITIONS for demand-driven UI paging + * within one session, while these are persistent INTERVALS — a claim about + * coverage that outlives the process and licenses skipping work. */ -class SyncBands( +class SyncCoverage( // How long a band may narrow work before the whole filter is walked // again. Everything a band claims is a claim about the past; this is how // long to trust it without re-testing. diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/PagingProgress.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/paging/PagingWindowProgress.kt similarity index 62% rename from quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/PagingProgress.kt rename to quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/paging/PagingWindowProgress.kt index 028fa34979..0f515d0ca5 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/PagingProgress.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/paging/PagingWindowProgress.kt @@ -18,73 +18,81 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.quartz.nip01Core.relay.client.accessories +package com.vitorpamplona.quartz.nip01Core.relay.client.paging import com.vitorpamplona.quartz.utils.TimeUtils import com.vitorpamplona.quartz.utils.concurrent.ConcurrentMap import kotlin.concurrent.Volatile /** - * How far a paged walk has got, measured on the time axis — the only axis - * whose end is known in advance. + * How far a bulk pagination has got, measured on the time axis — the only + * axis whose end is known in advance. * - * A paged fetch ([fetchAllPages]) has no event denominator: how many events + * A paged fetch (`fetchAllPages`) has no event denominator: how many events * exist is exactly what it is finding out, so every count-based percentage * degenerates to `downloaded/downloaded = 100%`. The time axis has both ends * before the first request — the filter's `until` (or now) down to its - * `since` (or [SyncBands.PLAUSIBLE_FLOOR]) — with each page's new `until` - * reporting the exact position between them. It needs no COUNT support. + * `since` (or the accessories' `SyncCoverage.PLAUSIBLE_FLOOR`) — with each + * page's new `until` cursor reporting the exact position between them. It + * needs no COUNT support. * * The estimate assumes events are spread evenly over time, which they are * not — so it errs pessimistic on the tail, and is a bound, not a promise. * - * One instance can serve many concurrent walks: keys are `"group|walk"`, and - * the group prefix scopes [fraction], [reached] and [etaMs] so two groups - * never report each other's numbers. + * One instance can serve many concurrent paginations: keys are + * `"group|name"`, and the group prefix scopes [fraction], [reached] and + * [etaMs] so two groups never report each other's numbers. + * + * Its siblings in this package track different things: [RelayLoadingCursors] + * is demand-driven `until`+`limit` paging for one scope (a feed pulling + * older pages on demand, no window), and [RelayPagingProgress] is the + * per-relay display state derived from it. This class is for a BULK + * pagination over a known `[since, until]` window, where "how far through + * the window, and when will it finish" is the question. */ -class PagingProgress( +class PagingWindowProgress( private val nowMillis: () -> Long = { TimeUtils.nowMillis() }, ) { - private class Walk( + private class Window( val top: Long, val bottom: Long, val startedMs: Long, @Volatile var current: Long, ) - private val walks = ConcurrentMap() + private val windows = ConcurrentMap() - /** Begin a walk over `[bottom, top]` seconds. An inverted window is not a walk. */ + /** Begin a pagination over `[bottom, top]` seconds. An inverted window is not one. */ fun begin( key: String, top: Long, bottom: Long, ) { - if (top > bottom) walks[key] = Walk(top, bottom, nowMillis(), top) + if (top > bottom) windows[key] = Window(top, bottom, nowMillis(), top) } - /** The walk reached [until]; monotonic, so a page that jumps back cannot un-advance it. */ + /** The pagination reached [until]; monotonic, so a page that jumps back cannot un-advance it. */ fun mark( key: String, until: Long, ) { - walks[key]?.let { - // Clamped to the walk's own floor: relays serve events stamped 0, - // and one of those would drag the position to the epoch. Below the - // floor means the walk is done, not time travel. + windows[key]?.let { + // Clamped to the window's own floor: relays serve events stamped + // 0, and one of those would drag the position to the epoch. Below + // the floor means the pagination is done, not time travel. val reached = until.coerceAtLeast(it.bottom) if (reached < it.current) it.current = reached } } fun finish(key: String) { - walks.remove(key) + windows.remove(key) } /** - * Fraction of the walk complete, averaged over every walk still going in + * Fraction complete, averaged over every pagination still going in * [group] (or all of them when null) — averaged rather than summed - * because each covers its own span, so "half the walks done and half at + * because each covers its own span, so "half of them done and half at * zero" is 50%. */ fun fraction(group: String? = null): Double? { @@ -96,18 +104,18 @@ class PagingProgress( } / live.size } - private fun live(group: String?): List = + private fun live(group: String?): List = if (group == null) { - walks.snapshot().values.toList() + windows.snapshot().values.toList() } else { - walks + windows .snapshot() .entries .filter { it.key.startsWith("$group|") } .map { it.value } } - /** The oldest second [group] has reached, or null when it is not walking. */ + /** The oldest second [group] has reached, or null when nothing is paging. */ fun reached(group: String? = null): Long? = live(group).minOfOrNull { it.current } /** Milliseconds left at the rate achieved so far, or null before it means anything. */ diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncBandsTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt similarity index 93% rename from quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncBandsTest.kt rename to quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt index d0ffe5290a..78abe61682 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncBandsTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt @@ -35,7 +35,7 @@ import kotlin.test.assertTrue * importantly, the cases where a band must NOT be used — a stale band silently * skips events, which is a worse failure than re-reading them. */ -class SyncBandsTest { +class SyncCoverageTest { private val relay = RelayUrlNormalizer.normalize("wss://relay.example") private val other = RelayUrlNormalizer.normalize("wss://other.example") private val profiles = Filter(kinds = listOf(0)) @@ -46,13 +46,13 @@ class SyncBandsTest { @Test fun `with nothing recorded the whole filter is fetched`() { - val c = SyncBands() + val c = SyncCoverage() assertEquals(listOf(profiles), c.legs(relay, profiles)) } @Test fun `a recorded band is fetched around rather than through`() { - val c = SyncBands() + val c = SyncCoverage() c.record(relay, profiles, observedMin = 1_700_001_000L, observedMax = 1_700_002_000L, paged = true) val legs = c.legs(relay, profiles) @@ -68,7 +68,7 @@ class SyncBandsTest { // A paged relay cuts pages by count, so a boundary can fall inside a run // of events sharing one created_at. Excluding the edge would strand the // rest of that second in no leg at all, while the band called it covered. - val c = SyncBands() + val c = SyncCoverage() c.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = true) val legs = c.legs(relay, profiles) @@ -85,7 +85,7 @@ class SyncBandsTest { @Test fun `successive runs widen the band rather than replacing it`() { - val c = SyncBands() + val c = SyncCoverage() c.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = true) // A later run reaches further back and picks up newer events. c.record(relay, profiles, 1_700_000_500L, 1_700_002_500L, paged = true) @@ -99,7 +99,7 @@ class SyncBandsTest { fun `a capped relay walks further back on each run`() { // The case that makes this worth having: a relay that only ever answers // with its newest N events. Each run starts below the last one's floor. - val c = SyncBands() + val c = SyncCoverage() c.record(relay, profiles, 1_700_009_000L, 1_700_010_000L, paged = true) assertEquals(1_700_009_000L, c.legs(relay, profiles)[0].until) @@ -113,7 +113,7 @@ class SyncBandsTest { fun `a negentropy sync that reported no outcome records nothing`() { // Only a sync that says how far it reconciled earns a band; a bare // paged=false call carries no claim to record. - val c = SyncBands() + val c = SyncCoverage() c.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = false) assertNull(c.band(relay, profiles)) assertEquals(listOf(profiles), c.legs(relay, profiles)) @@ -125,7 +125,7 @@ class SyncBandsTest { fun `a finished reconcile is in sync through the instant it started`() { // Not through the newest event it happened to see: "the relay had nothing // newer" and "we never asked" must not record the same thing. - val c = SyncBands() + val c = SyncCoverage() val startedAt = now() - 60 c.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = false, reconciledThrough = startedAt) @@ -138,7 +138,7 @@ class SyncBandsTest { fun `a reconcile that downloaded nothing still records coverage`() { // The empty case is the WHOLE point: nothing came back because we already // have it, and that is exactly when the next run should ask for a sliver. - val c = SyncBands() + val c = SyncCoverage() val startedAt = now() - 60 c.record(relay, profiles, null, null, paged = false, reconciledThrough = startedAt) @@ -149,12 +149,12 @@ class SyncBandsTest { @Test fun `a complete band drops its older leg while a paged one keeps it`() { - val reconciled = SyncBands() + val reconciled = SyncCoverage() reconciled.record(relay, profiles, null, null, paged = false, reconciledThrough = 1_700_002_000L) val only = reconciled.legs(relay, profiles).single() assertEquals(1_700_002_000L, only.since) - val walked = SyncBands() + val walked = SyncCoverage() walked.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = true) assertEquals(2, walked.legs(relay, profiles).size, "a paged walk says nothing about what it never asked for") } @@ -163,13 +163,13 @@ class SyncBandsTest { @Test fun `a band stops narrowing once it is older than the resync period`() { - val c = SyncBands(fullResyncSeconds = 60) + val c = SyncCoverage(fullResyncSeconds = 60) c.record(relay, profiles, null, null, paged = false, reconciledThrough = now() - 3600) // Recorded 'now' whatever the created_at claim, so age it by rewriting. c.record(relay, profiles, null, null, paged = false, reconciledThrough = now()) assertEquals(1, c.legs(relay, profiles).size, "fresh band still narrows") - val stale = SyncBands(fullResyncSeconds = 0) + val stale = SyncCoverage(fullResyncSeconds = 0) stale.record(relay, profiles, null, null, paged = false, reconciledThrough = now()) assertSame(profiles, stale.legs(relay, profiles).single(), "a band past its period re-walks everything") } @@ -178,7 +178,7 @@ class SyncBandsTest { fun `the re-walk replaces the old claim instead of widening it`() { // Widening would carry the stale band's floor forward forever and the // periodic pass would never actually reset anything. - val c = SyncBands(fullResyncSeconds = 0) + val c = SyncCoverage(fullResyncSeconds = 0) c.record(relay, profiles, 1_700_000_000L, 1_700_001_000L, paged = true) c.record(relay, profiles, 1_700_005_000L, 1_700_006_000L, paged = true) @@ -190,7 +190,7 @@ class SyncBandsTest { @Test fun `covering window collapses to the oldest ceiling once everyone is caught up`() { - val c = SyncBands() + val c = SyncCoverage() c.record(relay, profiles, null, null, paged = false, reconciledThrough = 1_700_009_000L) c.record(other, profiles, null, null, paged = false, reconciledThrough = 1_700_003_000L) @@ -201,7 +201,7 @@ class SyncBandsTest { fun `one relay that has never synced puts the window back to the whole filter`() { // It genuinely needs everything — narrowing the shared snapshot would // reconcile it against ids we never looked up. - val c = SyncBands() + val c = SyncCoverage() c.record(relay, profiles, null, null, paged = false, reconciledThrough = 1_700_009_000L) // The filter itself, unnarrowed — identity, since Filter has no equals. @@ -214,7 +214,7 @@ class SyncBandsTest { // Every url in a stream shares that stream's filter, so a backfill can // take ONE snapshot for all of them instead of walking the identical // range once per relay for byte-identical answers. - val c = SyncBands() + val c = SyncCoverage() val third = RelayUrlNormalizer.normalize("wss://third.example") c.record(relay, profiles, null, null, paged = false, reconciledThrough = 1_700_009_000L) c.record(other, profiles, null, null, paged = false, reconciledThrough = 1_700_003_000L) @@ -226,7 +226,7 @@ class SyncBandsTest { @Test fun `a relay with an older gap also widens the shared window`() { - val c = SyncBands() + val c = SyncCoverage() c.record(relay, profiles, null, null, paged = false, reconciledThrough = 1_700_009_000L) c.record(other, profiles, 1_700_001_000L, 1_700_002_000L, paged = true) @@ -237,7 +237,7 @@ class SyncBandsTest { fun `an empty fetch records nothing`() { // No events says nothing about what the relay holds, only that this // window was empty — recording it would fabricate coverage. - val c = SyncBands() + val c = SyncCoverage() c.record(relay, profiles, null, null, paged = true) assertNull(c.band(relay, profiles)) } @@ -246,11 +246,11 @@ class SyncBandsTest { fun `one misdated event does not cost a relay its whole band`() { // A single future-dated stamp among hundreds of thousands must not fail // a check applied to the aggregate. Screening per event keeps the rest. - val c = SyncBands() + val c = SyncCoverage() val far = now() + 400L * 86_400 val observed = listOf(1_700_001_000L, far, 1_700_002_000L, 0L) - val plausible = observed.filter { SyncBands.isPlausible(it) } + val plausible = observed.filter { SyncCoverage.isPlausible(it) } c.record(relay, profiles, plausible.min(), plausible.max(), paged = true) val band = c.band(relay, profiles)!! @@ -260,7 +260,7 @@ class SyncBandsTest { @Test fun `changing the filter starts over`() { - val c = SyncBands() + val c = SyncCoverage() c.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = true) // Widening the kinds means the old band skipped events it never fetched. @@ -273,7 +273,7 @@ class SyncBandsTest { @Test fun `each relay keeps its own band`() { - val c = SyncBands() + val c = SyncCoverage() c.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = true) assertEquals(listOf(profiles), c.legs(other, profiles)) } @@ -283,7 +283,7 @@ class SyncBandsTest { @Test fun `a bounded filter never widens past its own since and until`() { val bounded = Filter(kinds = listOf(0), since = 1_700_001_000L, until = 1_700_005_000L) - val c = SyncBands() + val c = SyncCoverage() c.record(relay, bounded, 1_700_002_000L, 1_700_003_000L, paged = true) val legs = c.legs(relay, bounded) @@ -300,7 +300,7 @@ class SyncBandsTest { // the two boundary seconds are always re-read, because that is the only // way to catch a run of same-second events a page boundary cut in half. val bounded = Filter(kinds = listOf(0), since = 1_700_001_000L, until = 1_700_005_000L) - val c = SyncBands() + val c = SyncCoverage() c.record(relay, bounded, 1_700_001_000L, 1_700_005_000L, paged = true) val legs = c.legs(relay, bounded) @@ -313,10 +313,10 @@ class SyncBandsTest { @Test fun `export and restore round-trip the bands`() { - val c = SyncBands() + val c = SyncCoverage() c.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = true) - val reopened = SyncBands() + val reopened = SyncCoverage() reopened.restore(c.export()) val band = reopened.band(relay, profiles)!! @@ -327,7 +327,7 @@ class SyncBandsTest { @Test fun `onChange fires when a band changes so persistence can mark dirty`() { var changes = 0 - val c = SyncBands(onChange = { changes++ }) + val c = SyncCoverage(onChange = { changes++ }) c.record(relay, profiles, null, null, paged = true) assertEquals(0, changes, "an empty fetch records nothing and must not dirty the store") @@ -341,7 +341,7 @@ class SyncBandsTest { // Filter.toJson() runs to tens of thousands of characters for an // author-scoped filter, and a fan-out keys once per relay per cycle. val big = Filter(kinds = listOf(30382), authors = (1..500).map { it.toString(16).padStart(64, '0') }) - val c = SyncBands() + val c = SyncCoverage() c.record(relay, big, 1_700_001_000L, 1_700_002_000L, paged = true) // Same instance, many lookups: still one band, and cheap. diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/PagingProgressTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/paging/PagingWindowProgressTest.kt similarity index 84% rename from quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/PagingProgressTest.kt rename to quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/paging/PagingWindowProgressTest.kt index bf8a550c63..2c1c0ed433 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/PagingProgressTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/paging/PagingWindowProgressTest.kt @@ -18,14 +18,14 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.quartz.nip01Core.relay.client.accessories +package com.vitorpamplona.quartz.nip01Core.relay.client.paging import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertNull import kotlin.test.assertTrue -class PagingProgressTest { +class PagingWindowProgressTest { private fun assertClose( expected: Double, actual: Double?, @@ -35,8 +35,8 @@ class PagingProgressTest { } @Test - fun `progress is the walked share of the time window`() { - val p = PagingProgress() + fun `progress is the paged share of the time window`() { + val p = PagingWindowProgress() p.begin("a", top = 1_000L, bottom = 0L) assertClose(0.0, p.fraction(), "nothing walked yet") @@ -49,11 +49,11 @@ class PagingProgressTest { } @Test - fun `a page that jumps backwards cannot un-advance the walk`() { + fun `a page that jumps backwards cannot un-advance the pagination`() { // Pages arrive from one relay in order, but nothing in the protocol // guarantees it, and a percentage that goes DOWN is worse than one that // is slightly wrong — it reads as the sync having lost ground. - val p = PagingProgress() + val p = PagingWindowProgress() p.begin("a", top = 1_000L, bottom = 0L) p.mark("a", 200L) @@ -63,10 +63,10 @@ class PagingProgressTest { } @Test - fun `walks average rather than sum`() { + fun `windows average rather than sum`() { // Two relays each walking their own window: one done and one untouched // is half way — not 100% as summing would give. - val p = PagingProgress() + val p = PagingWindowProgress() p.begin("a", top = 1_000L, bottom = 0L) p.begin("b", top = 500L, bottom = 0L) @@ -76,8 +76,8 @@ class PagingProgressTest { } @Test - fun `a finished walk leaves the average`() { - val p = PagingProgress() + fun `a finished window leaves the average`() { + val p = PagingWindowProgress() p.begin("a", top = 1_000L, bottom = 0L) p.begin("b", top = 1_000L, bottom = 0L) p.mark("b", 500L) @@ -86,14 +86,14 @@ class PagingProgressTest { assertClose(0.5, p.fraction(), "only b is still walking") p.finish("b") - assertNull(p.fraction(), "nothing walking means no number to report") + assertNull(p.fraction(), "nothing paging means no number to report") } @Test fun `a group prefix scopes the numbers to its own walks`() { // One instance serves many concurrent walks; without the scope two // streams would print each other's percentages. - val p = PagingProgress() + val p = PagingWindowProgress() p.begin("streamA|wss://r1", top = 1_000L, bottom = 0L) p.begin("streamB|wss://r2", top = 1_000L, bottom = 0L) p.mark("streamA|wss://r1", 0L) @@ -104,10 +104,10 @@ class PagingProgressTest { } @Test - fun `an inverted or empty window is not a walk`() { + fun `an inverted or empty window is not a pagination`() { // A leg whose since is above its until asks for a range nothing can be // in. Dividing by that span would produce infinities on the status line. - val p = PagingProgress() + val p = PagingWindowProgress() p.begin("a", top = 100L, bottom = 900L) @@ -116,7 +116,7 @@ class PagingProgressTest { @Test fun `no ETA before the estimate means anything`() { - val p = PagingProgress() + val p = PagingWindowProgress() p.begin("a", top = 1_000_000L, bottom = 0L) p.mark("a", 999_000L) @@ -129,7 +129,7 @@ class PagingProgressTest { @Test fun `ETA extrapolates from the rate achieved so far`() { var clock = 1_000_000L - val p = PagingProgress(nowMillis = { clock }) + val p = PagingWindowProgress(nowMillis = { clock }) p.begin("a", top = 1_000L, bottom = 0L) p.mark("a", 500L) From 42fc73f6cb9949d1ad713b36d9f4bcc382afc0ba Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 03:48:05 +0000 Subject: [PATCH 011/132] nip50Search: per-kind weighted search-field extraction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SearchableEvent.indexableContent() flattens a kind's searchable text into one blob, so a weighted full-text backend (title above summary above body) had to re-derive the decomposition itself — and drift every time a kind's parsing changed here. SearchFieldExtractor now lives beside the kinds it decomposes: title-like accessors primary, summary/description secondary, body tertiary, kind-0-shaped metadata in profile roles, with an indexableContent() fallback so every searchable kind, current or future, is covered. IndexableFields is a sealed shape — Profile or Tiered — so a kind cannot mix identity fields with content tiers, and each shape declares its own website role (a profile's homepage; a content kind's affiliation URLs). Multi-valued roles are carried UNJOINED, as lists: hashtag and location tags ride raw beside the tiers (filled by the one tiers() funnel every content branch uses, so no branch can forget them and profile shapes never see them), and separator or weighting choices — hashtags at summary weight, "\n" vs " ", arrays vs joined columns — belong to the backend, not the library. Empty extractions always normalize to None. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MfwV3xgMSmfxy16ujGPxGW --- .../quartz/nip50Search/IndexableFields.kt | 87 ++++ .../nip50Search/SearchFieldExtractor.kt | 486 ++++++++++++++++++ .../nip50Search/SearchFieldExtractorTest.kt | 127 +++++ 3 files changed, 700 insertions(+) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/IndexableFields.kt create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractor.kt create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractorTest.kt diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/IndexableFields.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/IndexableFields.kt new file mode 100644 index 0000000000..6227d28a1d --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/IndexableFields.kt @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip50Search + +/** + * An event's searchable text decomposed by ROLE, for full-text backends that + * weight fields instead of indexing one concatenated blob + * ([SearchableEvent.indexableContent]'s flat form). Produced per kind by + * [SearchFieldExtractor]. + * + * A kind is either PROFILE-shaped ([Profile] — kind-0-style identity: kind 0 + * itself, app handlers) or CONTENT-shaped ([Tiered] — title above summary + * above body). The shape is part of the value, so a consumer discriminates on + * the type instead of keeping its own list of profile kinds. Both shapes + * carry a website role — a profile's homepage ([Profile.website]), or a + * content kind's affiliation URLs ([Tiered.websites]: repo, trackers, + * bookmarked page) — declared on each shape rather than the interface, so + * [None] answers no question that doesn't apply to it. + * + * Multi-valued roles are carried UNJOINED, as lists: which separator to use — + * or whether to index the values separately — is the backend's decision, and + * once values are pre-joined a backend can't unmix them. All strings are + * trimmed and non-empty. + */ +sealed interface IndexableFields { + fun isEmpty(): Boolean + + /** No searchable text — the extraction result for non-searchable kinds. */ + data object None : IndexableFields { + override fun isEmpty(): Boolean = true + } + + /** Kind-0-shaped identity, each field in its own role. An all-null value means "profile-shaped kind, nothing indexable". */ + data class Profile( + val name: String? = null, + val displayName: String? = null, + val about: String? = null, + val nip05: String? = null, + val lud16: String? = null, + val website: String? = null, + ) : IndexableFields { + override fun isEmpty(): Boolean = this == EMPTY + + private companion object { + val EMPTY = Profile() + } + } + + /** + * Content decomposed by priority tier: [primary] (title-like values), + * [secondary] (summary/description-like values), [text] (the body — the + * one inherently single-valued role), plus the raw [hashtags] and + * [locations] tag values. + */ + data class Tiered( + val primary: List = emptyList(), + val secondary: List = emptyList(), + val text: String? = null, + val hashtags: List = emptyList(), + val locations: List = emptyList(), + val websites: List = emptyList(), + ) : IndexableFields { + override fun isEmpty(): Boolean = this == EMPTY + + private companion object { + val EMPTY = Tiered() + } + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractor.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractor.kt new file mode 100644 index 0000000000..c88d966a88 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractor.kt @@ -0,0 +1,486 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip50Search + +import com.vitorpamplona.quartz.buzz.agentProfiles.AgentProfileEvent +import com.vitorpamplona.quartz.buzz.apPersonas.PersonaEvent +import com.vitorpamplona.quartz.buzz.managedAgents.ManagedAgentEvent +import com.vitorpamplona.quartz.buzz.teams.TeamEvent +import com.vitorpamplona.quartz.buzz.workflow.WorkflowDefEvent +import com.vitorpamplona.quartz.experimental.agora.FundraiserEvent +import com.vitorpamplona.quartz.experimental.audio.track.AudioTrackEvent +import com.vitorpamplona.quartz.experimental.fitness.workout.ExerciseTemplateEvent +import com.vitorpamplona.quartz.experimental.fitness.workout.WorkoutRecordEvent +import com.vitorpamplona.quartz.experimental.interactiveStories.InteractiveStoryBaseEvent +import com.vitorpamplona.quartz.experimental.music.playlist.MusicPlaylistEvent +import com.vitorpamplona.quartz.experimental.music.track.MusicTrackEvent +import com.vitorpamplona.quartz.experimental.nip82SoftwareApps.application.SoftwareApplicationEvent +import com.vitorpamplona.quartz.experimental.nipsOnNostr.NipTextEvent +import com.vitorpamplona.quartz.feedDefinition.FeedDefinitionEvent +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent +import com.vitorpamplona.quartz.nip01Core.tags.hashtags.hashtags +import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent +import com.vitorpamplona.quartz.nip14Subject.subject +import com.vitorpamplona.quartz.nip23LongContent.LongTextNoteEvent +import com.vitorpamplona.quartz.nip28PublicChat.admin.ChannelCreateEvent +import com.vitorpamplona.quartz.nip28PublicChat.admin.ChannelMetadataEvent +import com.vitorpamplona.quartz.nip29RelayGroups.metadata.GroupMetadataEvent +import com.vitorpamplona.quartz.nip30CustomEmoji.pack.EmojiPackEvent +import com.vitorpamplona.quartz.nip34Git.issue.GitIssueEvent +import com.vitorpamplona.quartz.nip34Git.pr.GitPullRequestEvent +import com.vitorpamplona.quartz.nip34Git.repository.GitRepositoryEvent +import com.vitorpamplona.quartz.nip35Torrents.TorrentEvent +import com.vitorpamplona.quartz.nip51Lists.appCurationSet.AppCurationSetEvent +import com.vitorpamplona.quartz.nip51Lists.articleCurationSet.ArticleCurationSetEvent +import com.vitorpamplona.quartz.nip51Lists.bookmarkList.BookmarkListEvent +import com.vitorpamplona.quartz.nip51Lists.bookmarkList.OldBookmarkListEvent +import com.vitorpamplona.quartz.nip51Lists.followList.FollowListEvent +import com.vitorpamplona.quartz.nip51Lists.interestSet.InterestSetEvent +import com.vitorpamplona.quartz.nip51Lists.labeledBookmarkList.LabeledBookmarkListEvent +import com.vitorpamplona.quartz.nip51Lists.mediaStarterPack.MediaStarterPackEvent +import com.vitorpamplona.quartz.nip51Lists.peopleList.PeopleListEvent +import com.vitorpamplona.quartz.nip51Lists.pictureCurationSet.PictureCurationSetEvent +import com.vitorpamplona.quartz.nip51Lists.relaySets.RelaySetEvent +import com.vitorpamplona.quartz.nip51Lists.releaseArtifactSet.ReleaseArtifactSetEvent +import com.vitorpamplona.quartz.nip51Lists.videoCurationSet.VideoCurationSetEvent +import com.vitorpamplona.quartz.nip52Calendar.appt.day.CalendarDateSlotEvent +import com.vitorpamplona.quartz.nip52Calendar.appt.time.CalendarTimeSlotEvent +import com.vitorpamplona.quartz.nip52Calendar.calendar.CalendarEvent +import com.vitorpamplona.quartz.nip53LiveActivities.clip.LiveActivitiesClipEvent +import com.vitorpamplona.quartz.nip53LiveActivities.meetingSpaces.MeetingRoomEvent +import com.vitorpamplona.quartz.nip53LiveActivities.meetingSpaces.MeetingSpaceEvent +import com.vitorpamplona.quartz.nip53LiveActivities.streaming.LiveActivitiesEvent +import com.vitorpamplona.quartz.nip54Wiki.WikiNoteEvent +import com.vitorpamplona.quartz.nip58Badges.definition.BadgeDefinitionEvent +import com.vitorpamplona.quartz.nip5aStaticWebsites.NamedSiteEvent +import com.vitorpamplona.quartz.nip5aStaticWebsites.RootSiteEvent +import com.vitorpamplona.quartz.nip5dNapplets.NamedNappletEvent +import com.vitorpamplona.quartz.nip5dNapplets.NappletSnapshotEvent +import com.vitorpamplona.quartz.nip5dNapplets.RootNappletEvent +import com.vitorpamplona.quartz.nip68Picture.PictureEvent +import com.vitorpamplona.quartz.nip71Video.AddressableVideoEvent +import com.vitorpamplona.quartz.nip71Video.RegularVideoEvent +import com.vitorpamplona.quartz.nip72ModCommunities.definition.CommunityDefinitionEvent +import com.vitorpamplona.quartz.nip75ZapGoals.GoalEvent +import com.vitorpamplona.quartz.nip7DThreads.ThreadEvent +import com.vitorpamplona.quartz.nip84Highlights.HighlightEvent +import com.vitorpamplona.quartz.nip89AppHandlers.definition.AppDefinitionEvent +import com.vitorpamplona.quartz.nip94FileMetadata.FileHeaderEvent +import com.vitorpamplona.quartz.nip99Classifieds.ClassifiedsEvent +import com.vitorpamplona.quartz.nipB0WebBookmarks.WebBookmarkEvent +import com.vitorpamplona.quartz.nipC0CodeSnippets.CodeSnippetEvent +import com.vitorpamplona.quartz.nipF4Podcasts.episode.PodcastEpisodeEvent +import com.vitorpamplona.quartz.nipF4Podcasts.metadata.PodcastMetadataEvent + +/** + * Decomposes every [SearchableEvent] into [IndexableFields] by priority tier: + * title-like accessors primary, summary/description secondary, body tertiary, + * with hashtag and location tags carried raw beside the tiers. Each explicit + * branch splits exactly the accessors that kind's `indexableContent()` + * concatenates — keep the two in sync when a kind's parsing changes. Kinds + * without an explicit branch fall back to the [SearchableEvent] branch (whole + * `indexableContent()` in the tertiary tier), so EVERY searchable kind, + * current or future, is extracted. + * + * A kind may also fill the profile roles when it carries that shape: kind + * 31990 goes through the kind-0 fields wholesale, and any kind with a + * homepage/site URL fills [IndexableFields.websites]. Hashtags and `location` + * tags are filled SYSTEMICALLY by the [tiers] funnel every content branch + * uses, so recall never depends on a branch remembering them. + * + * Non-searchable kinds return [IndexableFields.None]. The extraction is + * derived data baked into the build: stores should re-derive after upgrades + * (see [com.vitorpamplona.quartz.nip01Core.store.IEventStore.reindexFullTextSearch]). + */ +object SearchFieldExtractor { + /** Empty extractions always come back as [IndexableFields.None], whatever shape produced them. */ + fun extract(event: Event): IndexableFields = base(event).let { if (it.isEmpty()) IndexableFields.None else it } + + private fun base(event: Event): IndexableFields = + when (event) { + // kind 0 -> the profile fields, each in its own role. + is MetadataEvent -> { + val md = event.contactMetaData() + if (md == null) { + IndexableFields.Profile() + } else { + IndexableFields.Profile( + name = clean(md.name), + displayName = clean(md.displayName), + about = clean(md.about), + nip05 = clean(md.nip05), + lud16 = clean(md.lud16), + website = clean(md.website), + ) + } + } + + is LongTextNoteEvent -> { + tiers(event, event.title(), event.summary(), event.content) + } + + is WikiNoteEvent -> { + tiers(event, event.title(), event.summary(), event.content) + } + + is ClassifiedsEvent -> { + tiers(event, event.title(), event.summary(), event.content) + } + + is GitRepositoryEvent -> { + tiers(event, listOf(event.name()), listOf(event.description()), event.content, websites = event.webs()) + } + + is GitIssueEvent -> { + tiers(event, event.subject(), null, event.content) + } + + is GitPullRequestEvent -> { + tiers(event, event.subject(), null, event.content) + } + + is CommunityDefinitionEvent -> { + tiers(event, listOf(event.name()), listOf(event.description(), event.rules()), event.content) + } + + is EmojiPackEvent -> { + tiers(event, event.titleOrName(), event.description(), event.content) + } + + is ChannelCreateEvent -> { + event.channelInfo().let { tiers(event, it.name, it.about, null) } + } + + is ChannelMetadataEvent -> { + event.channelInfo().let { tiers(event, it.name, it.about, null) } + } + + is PictureEvent -> { + tiers(event, event.title(), null, event.content) + } + + is RegularVideoEvent -> { + tiers(event, event.title(), null, event.content) + } + + is AddressableVideoEvent -> { + tiers(event, event.title(), null, event.content) + } + + // Torrents are searched by FILE NAME above all — index the file + // list into the secondary tier, trackers as the affiliation URL. + is TorrentEvent -> { + tiers(event, listOf(event.title()), event.files().map { it.fileName }, event.content, websites = event.trackers()) + } + + is ThreadEvent -> { + tiers(event, event.title(), null, event.content) + } + + is FundraiserEvent -> { + tiers(event, event.title(), null, event.content) + } + + is NipTextEvent -> { + tiers(event, event.title(), null, event.content) + } + + is ExerciseTemplateEvent -> { + tiers(event, event.title(), null, event.content) + } + + is WorkoutRecordEvent -> { + tiers(event, event.title(), null, event.content) + } + + is CalendarEvent -> { + tiers(event, event.title(), null, event.content) + } + + is LiveActivitiesClipEvent -> { + tiers(event, event.title(), null, event.content) + } + + is CalendarDateSlotEvent -> { + tiers(event, event.title(), event.summary(), event.content) + } + + is CalendarTimeSlotEvent -> { + tiers(event, event.title(), event.summary(), event.content) + } + + is LiveActivitiesEvent -> { + tiers(event, event.title(), event.summary(), event.content, website = event.streaming()) + } + + is InteractiveStoryBaseEvent -> { + tiers(event, event.title(), event.summary(), event.content) + } + + is MeetingSpaceEvent -> { + tiers(event, event.room(), event.summary(), event.content) + } + + is MeetingRoomEvent -> { + tiers(event, event.title(), event.summary(), null) + } + + // Code snippets are searched by language/runtime as much as name — + // fold those keywords into the secondary tier, repo as affiliation. + is CodeSnippetEvent -> { + tiers( + event, + listOf(event.snippetName()), + listOf(event.snippetDescription(), event.language(), event.extension(), event.runtime()), + event.content, + websites = listOf(event.repo()), + ) + } + + is BadgeDefinitionEvent -> { + tiers(event, event.name(), event.description(), event.content) + } + + is MusicPlaylistEvent -> { + tiers(event, event.title(), event.description(), event.content) + } + + is MusicTrackEvent -> { + tiers(event, listOf(event.title()), listOf(event.artist(), event.album()), event.content) + } + + is SoftwareApplicationEvent -> { + tiers(event, listOf(event.name()), listOf(event.summary()), event.content, websites = listOf(event.url(), event.repository())) + } + + is PodcastEpisodeEvent -> { + tiers(event, event.title(), event.description(), event.content) + } + + is PodcastMetadataEvent -> { + tiers(event, listOf(event.title()), listOf(event.description()), null, websites = event.websites()) + } + + is GroupMetadataEvent -> { + tiers(event, event.name(), event.about(), null) + } + + is InterestSetEvent -> { + tiers(event, event.title(), event.description(), null) + } + + is FollowListEvent -> { + tiers(event, event.title(), event.description(), null) + } + + is MediaStarterPackEvent -> { + tiers(event, event.title(), event.description(), null) + } + + is PictureCurationSetEvent -> { + tiers(event, event.title(), event.description(), null) + } + + is ArticleCurationSetEvent -> { + tiers(event, event.title(), event.description(), null) + } + + is VideoCurationSetEvent -> { + tiers(event, event.title(), event.description(), null) + } + + is ReleaseArtifactSetEvent -> { + tiers(event, event.title(), event.description(), null) + } + + is AppCurationSetEvent -> { + tiers(event, event.title(), event.description(), null) + } + + is RelaySetEvent -> { + tiers(event, event.title(), event.description(), null) + } + + // A web bookmark IS its URL — route it to the affiliation website + // field so the bookmark is findable by its domain. + is WebBookmarkEvent -> { + tiers(event, event.title(), event.description(), null, website = event.url()) + } + + is NamedSiteEvent -> { + tiers(event, event.title(), event.description(), null) + } + + is RootSiteEvent -> { + tiers(event, event.title(), event.description(), null) + } + + is RootNappletEvent -> { + tiers(event, event.title(), event.description(), null) + } + + is NappletSnapshotEvent -> { + tiers(event, event.title(), event.description(), null) + } + + is NamedNappletEvent -> { + tiers(event, event.title(), event.description(), null) + } + + is FeedDefinitionEvent -> { + tiers(event, event.title(), null, null) + } + + is LabeledBookmarkListEvent -> { + tiers(event, event.titleOrName(), event.description(), null) + } + + is PeopleListEvent -> { + tiers(event, event.titleOrName(), event.description(), null) + } + + is BookmarkListEvent -> { + tiers(event, event.title(), null, null) + } + + is OldBookmarkListEvent -> { + tiers(event, event.title(), null, null) + } + + is GoalEvent -> { + tiers(event, null, event.summary(), event.content) + } + + is HighlightEvent -> { + tiers(event, emptyList(), listOf(event.comment(), event.context()), event.content) + } + + is FileHeaderEvent -> { + tiers(event, null, event.summary(), event.content) + } + + is AudioTrackEvent -> { + tiers(event, event.subject(), null, null) + } + + // Buzz agent/workspace kinds carry their metadata as JSON in `content`; + // split each decoded object the way its indexableContent() concatenates it. + is AgentProfileEvent -> { + event.profileOrNull()?.let { tiers(event, listOf(it.name, it.displayName), emptyList(), null) } ?: tiers(event, null, null, null) + } + + is PersonaEvent -> { + event.personaOrNull()?.let { tiers(event, it.displayName, null, it.systemPrompt) } ?: tiers(event, null, null, null) + } + + is ManagedAgentEvent -> { + event.agentOrNull()?.let { tiers(event, it.name, null, it.systemPrompt) } ?: tiers(event, null, null, null) + } + + is TeamEvent -> { + event.teamOrNull()?.let { tiers(event, it.name, it.description, it.instructions) } ?: tiers(event, null, null, null) + } + + is WorkflowDefEvent -> { + tiers(event, event.name(), null, event.content) + } + + // kind 31990 — the app handler's metadata IS a UserMetadata clone, + // so route it through the kind-0 profile fields: an app's + // @-handle and site get the same treatment a person's do. + is AppDefinitionEvent -> { + val md = event.appMetaData() + if (md == null) { + IndexableFields.Profile() + } else { + IndexableFields.Profile( + // Per NIP-24 the deprecated `username` folds into `name`. + name = clean(md.name ?: md.username), + displayName = clean(md.displayName), + about = clean(md.about), + nip05 = clean(md.nip05), + lud16 = clean(md.lud16), + website = clean(md.website), + ) + } + } + + // kind 1 LAST among the explicit branches: several kinds extend the + // text-note base, and their own branches above must win. + is TextNoteEvent -> { + tiers(event, event.subject(), null, event.content) + } + + // Everything else Quartz can search, current or future: the whole + // indexableContent lands in the tertiary tier. + is SearchableEvent -> { + tiers(event, null, null, event.indexableContent()) + } + + else -> { + IndexableFields.None + } + } + + /** Single-value convenience over the list funnel — most kinds carry one title, one summary, one body. */ + private fun tiers( + event: Event, + primary: String?, + secondary: String?, + text: String?, + website: String? = null, + ) = tiers(event, listOf(primary), listOf(secondary), text, listOf(website)) + + /** + * The one funnel every content branch uses — [IndexableFields.Tiered.hashtags] + * and [IndexableFields.Tiered.locations] are filled here, so no branch can + * forget them. Values stay UNJOINED: separator choices belong to the backend. + */ + private fun tiers( + event: Event, + primary: List, + secondary: List, + text: String?, + websites: List = emptyList(), + ) = IndexableFields.Tiered( + primary = cleanAll(primary), + secondary = cleanAll(secondary), + text = clean(text), + hashtags = cleanAll(event.tags.hashtags()), + locations = locationValues(event), + websites = cleanAll(websites), + ) + + /** Trim and drop empties at the single funnel every derived string passes through. */ + private fun clean(s: String?): String? = s?.trim()?.ifEmpty { null } + + private fun cleanAll(parts: List): List = parts.mapNotNull { clean(it) } + + /** + * Every `location` tag value, on ANY kind. Deliberately a raw scan, not a + * typed accessor: Quartz's LocationTag classes are per-NIP (calendar, + * picture, classifieds) and only those kinds expose locations(), while + * this funnel must also catch location tags on kinds whose class doesn't + * model them. + */ + private fun locationValues(event: Event): List = event.tags.mapNotNull { tag -> tag.getOrNull(1)?.trim()?.takeIf { tag.getOrNull(0) == "location" && it.isNotEmpty() } } +} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractorTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractorTest.kt new file mode 100644 index 0000000000..3c543a9442 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractorTest.kt @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip50Search + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent +import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent +import com.vitorpamplona.quartz.nip23LongContent.LongTextNoteEvent +import com.vitorpamplona.quartz.nip35Torrents.TorrentEvent +import com.vitorpamplona.quartz.nip89AppHandlers.definition.AppDefinitionEvent +import com.vitorpamplona.quartz.nipB0WebBookmarks.WebBookmarkEvent +import kotlin.test.Test +import kotlin.test.assertEquals + +class SearchFieldExtractorTest { + private val alice = "a1".repeat(32) + + @Test + fun kind0DecomposesIntoTheProfileRoles() { + val content = """{"name":"vitor","display_name":"Vitor P","about":"builds nostr","nip05":"vitor@vitorpamplona.com","lud16":"me@wallet.com","website":"https://vitorpamplona.com","picture":"https://x/y.jpg"}""" + val fields = SearchFieldExtractor.extract(MetadataEvent("1".repeat(64), alice, 1L, emptyArray(), content, "")) + assertEquals( + IndexableFields.Profile( + name = "vitor", + displayName = "Vitor P", + about = "builds nostr", + nip05 = "vitor@vitorpamplona.com", + lud16 = "me@wallet.com", + website = "https://vitorpamplona.com", + ), + fields, + ) + } + + @Test + fun longFormDecomposesIntoTitleSummaryHashtagsContent() { + val tags = arrayOf(arrayOf("d", "post"), arrayOf("title", "My Post"), arrayOf("summary", "tl;dr"), arrayOf("t", "nostr"), arrayOf("t", "search")) + val fields = SearchFieldExtractor.extract(LongTextNoteEvent("2".repeat(64), alice, 1L, tags, "the whole article", "")) + assertEquals(IndexableFields.Tiered(primary = listOf("My Post"), secondary = listOf("tl;dr"), text = "the whole article", hashtags = listOf("nostr", "search")), fields) + } + + @Test + fun notesUseTheSubjectAndHashtags() { + val tags = arrayOf(arrayOf("subject", "meetup"), arrayOf("t", "brazil")) + val fields = SearchFieldExtractor.extract(TextNoteEvent("3".repeat(64), alice, 1L, tags, "see you there", "")) + assertEquals(IndexableFields.Tiered(primary = listOf("meetup"), text = "see you there", hashtags = listOf("brazil")), fields) + } + + @Test + fun locationTagsAreCarriedRawLikeHashtags() { + val tags = arrayOf(arrayOf("location", "Rio de Janeiro")) + val fields = SearchFieldExtractor.extract(TextNoteEvent("4".repeat(64), alice, 1L, tags, "gm", "")) + assertEquals(IndexableFields.Tiered(text = "gm", locations = listOf("Rio de Janeiro")), fields) + } + + @Test + fun torrentsIndexFileNamesAndTrackers() { + val tags = + arrayOf( + arrayOf("title", "Great Torrent"), + arrayOf("file", "episode1.mkv"), + arrayOf("file", "episode2.mkv"), + arrayOf("tracker", "https://tracker.example.com"), + ) + val fields = SearchFieldExtractor.extract(TorrentEvent("5".repeat(64), alice, 1L, tags, "a series", "")) + assertEquals( + IndexableFields.Tiered( + primary = listOf("Great Torrent"), + // Unjoined: the backend decides how file names are indexed. + secondary = listOf("episode1.mkv", "episode2.mkv"), + text = "a series", + websites = listOf("https://tracker.example.com"), + ), + fields, + ) + } + + @Test + fun webBookmarksAreFindableByTheirUrl() { + // NIP-B0: the d tag carries the URL scheme-less; url() re-adds https://. + val tags = arrayOf(arrayOf("d", "vitorpamplona.com/post"), arrayOf("title", "A Post")) + val fields = SearchFieldExtractor.extract(WebBookmarkEvent("6".repeat(64), alice, 1L, tags, "", "")) + assertEquals(IndexableFields.Tiered(primary = listOf("A Post"), websites = listOf("https://vitorpamplona.com/post")), fields) + } + + @Test + fun appHandlerMetadataReusesTheProfileRoles() { + val content = """{"name":"CoolApp","about":"an app","website":"https://coolapp.example"}""" + val fields = SearchFieldExtractor.extract(AppDefinitionEvent("7".repeat(64), alice, 1L, arrayOf(arrayOf("d", "x")), content, "")) + assertEquals(IndexableFields.Profile(name = "CoolApp", about = "an app", website = "https://coolapp.example"), fields) + } + + @Test + fun nonSearchableKindsExtractNothing() { + // Kind 7 reactions are not SearchableEvent. + val reaction = Event("8".repeat(64), alice, 1L, 7, emptyArray(), "+", "") + assertEquals(IndexableFields.None, SearchFieldExtractor.extract(reaction)) + } + + @Test + fun profileShapesNeverGetHashtagFolding() { + // Hashtags/locations are filled only by the tiers() funnel, which + // profile branches never use: a kind-0 with t-tags stays a pure + // profile (and an empty one normalizes to None). + val tags = arrayOf(arrayOf("t", "nostr")) + val fields = SearchFieldExtractor.extract(MetadataEvent("9".repeat(64), alice, 1L, tags, "{}", "")) + assertEquals(IndexableFields.None, fields) + } +} From 804b850095f04707c48f568621807346389caa23 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 05:00:40 +0000 Subject: [PATCH 012/132] Audit fixes: empty exclusions, missed fallback, one vocabulary An all-dash search token ("--") stripped to an empty exclusion that toSearchString/stripExtensions round-tripped into a REQUIRED "-" term reaching SQLite FTS; it is now dropped at parse. IngestQueue's batchInsert fallback was the one site still hand-writing "insert failed" (unprefixed) while every other path emits RejectionReason.INSERT_FAILED. RejectionReason no longer duplicates the NIP-01 prefixes MachineReadablePrefix already owns, and the expiration trigger now rejects with the same words as the Kotlin pre-check instead of its own spelling. Tests pin the "--" drop, consecutive quoted spans, text after a closing quote, the extractor's fallback tier, blank-content normalization, and the unparseable-buzz-content hashtag seam; extractor KDoc now states where the trimmed/non-empty and never-empty-Profile guarantees actually live. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MfwV3xgMSmfxy16ujGPxGW --- .../relay/server/backend/IngestQueue.kt | 3 +- .../quartz/nip01Core/store/RejectionReason.kt | 28 ++++++++----------- .../store/sqlite/ExpirationModule.kt | 3 +- .../quartz/nip50Search/IndexableFields.kt | 13 +++++++-- .../nip50Search/SearchFieldExtractor.kt | 6 ++-- .../quartz/nip50Search/SearchQuery.kt | 5 +++- .../nip50Search/SearchFieldExtractorTest.kt | 23 +++++++++++++++ .../quartz/nip50Search/SearchQueryTest.kt | 27 ++++++++++++++++++ 8 files changed, 83 insertions(+), 25 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/IngestQueue.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/IngestQueue.kt index 39f1191882..0a578630f1 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/IngestQueue.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/IngestQueue.kt @@ -22,6 +22,7 @@ package com.vitorpamplona.quartz.nip01Core.relay.server.backend import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.store.IEventStore +import com.vitorpamplona.quartz.nip01Core.store.RejectionReason import com.vitorpamplona.quartz.utils.Log import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -292,7 +293,7 @@ class IngestQueue( store.batchInsert(toInsert) } catch (e: Throwable) { Log.w("IngestQueue") { "batchInsert failed for ${toInsert.size} events: ${e.message}" } - val reason = e.message ?: e::class.simpleName ?: "insert failed" + val reason = e.message ?: e::class.simpleName ?: RejectionReason.INSERT_FAILED List(toInsert.size) { IEventStore.InsertOutcome.Rejected(reason) } } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/RejectionReason.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/RejectionReason.kt index a9262e6482..4e14cb2550 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/RejectionReason.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/RejectionReason.kt @@ -28,24 +28,20 @@ package com.vitorpamplona.quartz.nip01Core.store * "duplicate" differently. * * NIP-01: an `OK false` message SHOULD begin with a single-word - * machine-readable prefix followed by `:`. The `PREFIX_*` constants are that - * vocabulary; the full-sentence constants are the standard reasons the - * built-in stores emit. `replaced:` is not in NIP-01 but is the de-facto - * prefix (strfry and others) for a replaceable event that lost to a stored - * newer version — distinct from `duplicate:` (the exact event is already - * held). + * machine-readable prefix followed by `:`. The full-sentence constants here + * are the standard reasons the built-in stores emit. `replaced:` is not in + * NIP-01 but is the de-facto prefix (strfry and others) for a replaceable + * event that lost to a stored newer version — distinct from `duplicate:` + * (the exact event is already held). */ object RejectionReason { - // The NIP-01 machine-readable prefixes. - const val PREFIX_DUPLICATE = "duplicate:" - const val PREFIX_POW = "pow:" - const val PREFIX_BLOCKED = "blocked:" - const val PREFIX_RATE_LIMITED = "rate-limited:" - const val PREFIX_INVALID = "invalid:" - const val PREFIX_RESTRICTED = "restricted:" - const val PREFIX_ERROR = "error:" - - /** De-facto prefix (not in NIP-01) for a stale version of a replaceable/addressable event. */ + /** + * The NIP-01 prefix vocabulary itself lives in + * [com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.MachineReadablePrefix] + * — use its `format`/`parse` instead of hand-writing prefixes. The one + * prefix that enum lacks is the de-facto (not in NIP-01) word for a stale + * version of a replaceable/addressable event: + */ const val PREFIX_REPLACED = "replaced:" // The standard store reasons. diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ExpirationModule.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ExpirationModule.kt index 99baf1cace..38c6703fb2 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ExpirationModule.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/ExpirationModule.kt @@ -22,6 +22,7 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite import androidx.sqlite.SQLiteConnection import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.store.RejectionReason import com.vitorpamplona.quartz.nip40Expiration.expiration class ExpirationModule : IModule { @@ -44,7 +45,7 @@ class ExpirationModule : IModule { FOR EACH ROW BEGIN -- Check for existing newer record - SELECT RAISE(ABORT, 'blocked: this event is expired') + SELECT RAISE(ABORT, '${RejectionReason.EXPIRED}') WHERE NEW.expiration <= unixepoch(); END; """.trimIndent(), diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/IndexableFields.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/IndexableFields.kt index 6227d28a1d..d0e8720804 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/IndexableFields.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/IndexableFields.kt @@ -37,8 +37,9 @@ package com.vitorpamplona.quartz.nip50Search * * Multi-valued roles are carried UNJOINED, as lists: which separator to use — * or whether to index the values separately — is the backend's decision, and - * once values are pre-joined a backend can't unmix them. All strings are - * trimmed and non-empty. + * once values are pre-joined a backend can't unmix them. Values produced by + * [SearchFieldExtractor] are trimmed and non-empty; the types themselves do + * not enforce it. */ sealed interface IndexableFields { fun isEmpty(): Boolean @@ -48,7 +49,13 @@ sealed interface IndexableFields { override fun isEmpty(): Boolean = true } - /** Kind-0-shaped identity, each field in its own role. An all-null value means "profile-shaped kind, nothing indexable". */ + /** + * Kind-0-shaped identity, each field in its own role. [SearchFieldExtractor] + * never RETURNS an empty Profile — extract() normalizes every empty shape + * to [None] — so an all-null value only exists mid-extraction or when + * hand-built. Note the equality trap for hand-built values: an empty + * shape isEmpty() but is not equal to [None]. + */ data class Profile( val name: String? = null, val displayName: String? = null, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractor.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractor.kt index c88d966a88..39cfd48ce3 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractor.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractor.kt @@ -424,8 +424,8 @@ object SearchFieldExtractor { } } - // kind 1 LAST among the explicit branches: several kinds extend the - // text-note base, and their own branches above must win. + // kind 1 LAST among the explicit branches, defensively: a future + // kind extending the text-note base must hit its own branch first. is TextNoteEvent -> { tiers(event, event.subject(), null, event.content) } @@ -482,5 +482,5 @@ object SearchFieldExtractor { * this funnel must also catch location tags on kinds whose class doesn't * model them. */ - private fun locationValues(event: Event): List = event.tags.mapNotNull { tag -> tag.getOrNull(1)?.trim()?.takeIf { tag.getOrNull(0) == "location" && it.isNotEmpty() } } + private fun locationValues(event: Event): List = event.tags.mapNotNull { tag -> if (tag.getOrNull(0) != "location") null else clean(tag.getOrNull(1)) } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQuery.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQuery.kt index fbcb4c35d1..9361d58e25 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQuery.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQuery.kt @@ -210,7 +210,10 @@ class SearchQuery( } } if (token.length > 1 && token[0] == '-') { - notTerms += token.trimStart('-') + // All-dash tokens ("--") strip to nothing: an + // exclude-nothing token is dropped, not surfaced — an + // empty exclusion would round-trip into a required "-". + token.trimStart('-').takeIf { it.isNotEmpty() }?.let { notTerms += it } continue } if (terms.isNotEmpty()) terms.append(' ') diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractorTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractorTest.kt index 3c543a9442..daa1ce232d 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractorTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchFieldExtractorTest.kt @@ -20,9 +20,11 @@ */ package com.vitorpamplona.quartz.nip50Search +import com.vitorpamplona.quartz.buzz.agentProfiles.AgentProfileEvent import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent +import com.vitorpamplona.quartz.nip17Dm.messages.ChatMessageEvent import com.vitorpamplona.quartz.nip23LongContent.LongTextNoteEvent import com.vitorpamplona.quartz.nip35Torrents.TorrentEvent import com.vitorpamplona.quartz.nip89AppHandlers.definition.AppDefinitionEvent @@ -115,6 +117,27 @@ class SearchFieldExtractorTest { assertEquals(IndexableFields.None, SearchFieldExtractor.extract(reaction)) } + @Test + fun unmappedSearchableKindsFallBackToTheTextTier() { + val fields = SearchFieldExtractor.extract(ChatMessageEvent("a".repeat(64), alice, 1L, emptyArray(), "hello group", "")) + assertEquals(IndexableFields.Tiered(text = "hello group"), fields) + } + + @Test + fun blankContentKindsNormalizeToNone() { + val fields = SearchFieldExtractor.extract(TextNoteEvent("b".repeat(64), alice, 1L, emptyArray(), " ", "")) + assertEquals(IndexableFields.None, fields) + } + + @Test + fun unparseableBuzzContentStillIndexesItsHashtags() { + // The branch finds no text, but the tiers() funnel still carries the + // event's raw tags — the one subtle reachability seam of the port. + val tags = arrayOf(arrayOf("t", "agents")) + val fields = SearchFieldExtractor.extract(AgentProfileEvent("c".repeat(64), alice, 1L, tags, "not json", "")) + assertEquals(IndexableFields.Tiered(hashtags = listOf("agents")), fields) + } + @Test fun profileShapesNeverGetHashtagFolding() { // Hashtags/locations are filled only by the tiers() funnel, which diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQueryTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQueryTest.kt index 956c73fee6..2e5169a0eb 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQueryTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip50Search/SearchQueryTest.kt @@ -307,6 +307,33 @@ class SearchQueryTest { assertEquals("best \"nostr apps\" -spam -\"bad phrase\" domain:example.com", q.toSearchString()) } + @Test + fun allDashTokensAreDroppedNotEmptied() { + // "--" strips to nothing; surfacing an empty exclusion would + // round-trip into a required "-" term. + val q = SearchQuery.parse("a --") + assertEquals("a", q.terms) + assertTrue(q.notTerms.isEmpty()) + assertEquals("a", SearchQuery.parse(q.toSearchString()).terms) + assertEquals("", SearchQuery.stripExtensions("-- include:spam")) + } + + @Test + fun consecutiveSpansEachLift() { + val q = SearchQuery.parse("\"a\"\"b\" -\"c\"") + assertEquals(listOf("a", "b"), q.phrases) + assertEquals(listOf("c"), q.notPhrases) + assertEquals("", q.terms) + } + + @Test + fun textAfterClosingQuoteIsItsOwnTerm() { + // The lifted span's place stays a token boundary for what follows. + val q = SearchQuery.parse("\"a b\"c") + assertEquals(listOf("a b"), q.phrases) + assertEquals("c", q.terms) + } + @Test fun stripExtensionsKeepsPhrasesAndExclusions() { assertEquals( From 4081ef16814ddbe7334c2e57414ceb0131d98a7e Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 05:00:40 +0000 Subject: [PATCH 013/132] nip01Core: one owner rule, one supersession rule, one tag-name rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three semantics rules each existed as multiple independent copies: - Event.owner() (gift-wrap recipient controls the wrap, else the author — NIP-09/62 authority) was derived inline in EventIndexesModule and again in EventStoreProjection.ownerOf. - The NIP-01 replaceable tiebreak (newest created_at, ties to the lexically smallest id) lived in EventStoreProjection.supersedes, in SQL, and downstream. - "Indexable tag name" was spelled `length == 1` in four places, which admits "5" and "#" — names the NIP-01 #x filter space (single a-zA-Z letters) cannot address, letting stores disagree about which tags filters reach. isIndexableTagName encodes the NIP-01 rule; converging FilterIndex and the SQLite IndexingStrategy on it deliberately tightens single-char non-letter tag names out of the index. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MfwV3xgMSmfxy16ujGPxGW --- .../cache/projection/EventStoreProjection.kt | 12 +++---- .../nip01Core/core/ReplaceableSupersession.kt | 35 +++++++++++++++++++ .../nip01Core/relay/filters/FilterIndex.kt | 7 ++-- .../quartz/nip01Core/store/EventOwner.kt | 35 +++++++++++++++++++ .../store/sqlite/EventIndexesModule.kt | 10 ++---- .../store/sqlite/IndexingStrategy.kt | 3 +- .../quartz/nip01Core/tags/IndexableTagName.kt | 30 ++++++++++++++++ 7 files changed, 113 insertions(+), 19 deletions(-) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/ReplaceableSupersession.kt create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/EventOwner.kt create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/tags/IndexableTagName.kt diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt index 3c45f8af17..58c3ecc6ff 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/cache/projection/EventStoreProjection.kt @@ -25,12 +25,13 @@ import com.vitorpamplona.quartz.nip01Core.core.AddressableEvent import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.isReplaceable +import com.vitorpamplona.quartz.nip01Core.core.supersedes import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.store.ObservableEventStore import com.vitorpamplona.quartz.nip01Core.store.ObservableEventStore.StoreChange +import com.vitorpamplona.quartz.nip01Core.store.owner import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent import com.vitorpamplona.quartz.nip40Expiration.isExpirationBefore -import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent import com.vitorpamplona.quartz.utils.SortedList import com.vitorpamplona.quartz.utils.TimeUtils @@ -342,19 +343,14 @@ class EventStoreProjection( private fun supersedes( new: Event, existing: Event, - ): Boolean = - when { - new.createdAt > existing.createdAt -> true - new.createdAt < existing.createdAt -> false - else -> new.id < existing.id - } + ): Boolean = new.supersedes(existing) /** * Owner pubkey for ownership checks (NIP-09 author match, * NIP-62 vanish target). For GiftWrap the owner is the p-tag * recipient; for everything else it's `event.pubKey`. */ - private fun ownerOf(event: Event): HexKey = (event as? GiftWrapEvent)?.recipientPubKey() ?: event.pubKey + private fun ownerOf(event: Event): HexKey = event.owner() /** * created_at DESC, id ASC. Sort keys are frozen at slot diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/ReplaceableSupersession.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/ReplaceableSupersession.kt new file mode 100644 index 0000000000..0719ea24f5 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/core/ReplaceableSupersession.kt @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.core + +/** + * The NIP-01 replaceable/addressable supersession rule: the winner of an + * address is the highest `created_at`, with ties broken by the LEXICALLY + * SMALLEST id. True when THIS event beats [existing] — equal events (same id) + * do not supersede themselves. One rule, shared by every store; the SQLite + * triggers encode the same comparison in SQL. + */ +fun Event.supersedes(existing: Event): Boolean = + when { + createdAt > existing.createdAt -> true + createdAt < existing.createdAt -> false + else -> id < existing.id + } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/filters/FilterIndex.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/filters/FilterIndex.kt index 4cd54f7594..da083e9dd7 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/filters/FilterIndex.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/filters/FilterIndex.kt @@ -22,6 +22,7 @@ package com.vitorpamplona.quartz.nip01Core.relay.filters import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.tags.isIndexableTagName import kotlinx.collections.immutable.PersistentMap import kotlinx.collections.immutable.PersistentSet import kotlinx.collections.immutable.persistentHashMapOf @@ -238,7 +239,7 @@ class FilterIndex { s.kinds[event.kind]?.let { result.addAll(it) } if (s.tags.isNotEmpty()) { for (tag in event.tags) { - if (tag.size >= 2 && tag[0].length == 1) { + if (tag.size >= 2 && isIndexableTagName(tag[0])) { s.tags[tag[0]]?.get(tag[1])?.let { result.addAll(it) } } } @@ -348,14 +349,14 @@ class FilterIndex { if (!filter.tags.isNullOrEmpty()) { val first = filter.tags.entries.firstOrNull { - it.key.length == 1 && it.value.isNotEmpty() + isIndexableTagName(it.key) && it.value.isNotEmpty() } if (first != null) return first.value.map { TagKey(first.key, it) } } if (!filter.tagsAll.isNullOrEmpty()) { val first = filter.tagsAll.entries.firstOrNull { - it.key.length == 1 && it.value.isNotEmpty() + isIndexableTagName(it.key) && it.value.isNotEmpty() } if (first != null) return first.value.map { TagKey(first.key, it) } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/EventOwner.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/EventOwner.kt new file mode 100644 index 0000000000..10126099bc --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/EventOwner.kt @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.store + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent + +/** + * The pubkey that CONTROLS this event for ownership checks — NIP-09 deletion + * authority and NIP-62 vanish targeting. A gift wrap (kind 1059) is signed by + * a random one-time key, so control belongs to its p-tag RECIPIENT (the + * recipient deletes the wraps addressed to them); every other event is + * controlled by its author. One rule, shared by every store — do not re-derive + * it inline. + */ +fun Event.owner(): HexKey = (this as? GiftWrapEvent)?.recipientPubKey() ?: pubKey diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt index dba6e2f9f3..f5347a4372 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/EventIndexesModule.kt @@ -25,7 +25,7 @@ import com.vitorpamplona.quartz.nip01Core.core.AddressSerializer import com.vitorpamplona.quartz.nip01Core.core.AddressableEvent import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper -import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent +import com.vitorpamplona.quartz.nip01Core.store.owner class EventIndexesModule( val hasher: (db: SQLiteConnection) -> TagNameValueHasher, @@ -206,12 +206,8 @@ class EventIndexesModule( val kindLong = event.kind.toLong() val pubkeyHash = hasher.hash(event.pubKey) - val eventOwnerHash = - if (event is GiftWrapEvent) { - event.recipientPubKey()?.let { hasher.hash(it) } ?: pubkeyHash - } else { - pubkeyHash - } + val ownerKey = event.owner() + val eventOwnerHash = if (ownerKey == event.pubKey) pubkeyHash else hasher.hash(ownerKey) val eTagHash = hasher.hashETag(event.id) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt index 01362e03fb..eb704e1781 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/IndexingStrategy.kt @@ -21,6 +21,7 @@ package com.vitorpamplona.quartz.nip01Core.store.sqlite import com.vitorpamplona.quartz.nip01Core.core.Tag +import com.vitorpamplona.quartz.nip01Core.tags.isIndexableTagName interface IndexingStrategy { /** @@ -165,5 +166,5 @@ class DefaultIndexingStrategy( override fun shouldIndex( kind: Int, tag: Tag, - ) = tag.size >= 2 && tag[0].length == 1 + ) = tag.size >= 2 && isIndexableTagName(tag[0]) } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/tags/IndexableTagName.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/tags/IndexableTagName.kt new file mode 100644 index 0000000000..e3a682efd5 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/tags/IndexableTagName.kt @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.tags + +/** + * Whether [name] is a tag name the NIP-01 `#x` filter space can address: a + * single ASCII LETTER (`a`-`z` / `A`-`Z`), per NIP-01's "single-letter + * (a-zA-Z) English-alphabet letters". Deliberately stricter than + * `length == 1`: a `"5"` or `"#"` tag name is not filterable and indexing + * it would let stores disagree about which tags `#x` filters reach. + */ +fun isIndexableTagName(name: String): Boolean = name.length == 1 && (name[0] in 'a'..'z' || name[0] in 'A'..'Z') From cdbd550405b5249b1af0a52bd2246338b492e72c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 05:22:40 +0000 Subject: [PATCH 014/132] Fix audit findings across the sync accessories and their consumers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit quartz: - SQLiteEventStore: classify per-row savepoint errors — policy refusals (blocked:/constraint/not allowed) stay Rejected, everything else is now Failed, so disk-full no longer masquerades as 2M duplicate rejections - IEventStore.batchInsert default: rethrow CancellationException and map unknown throws to Failed (re-offering a duplicate is idempotent; dropping a good event on a transient store error is not) - IngestQueue: rethrow CancellationException instead of stamping a cancelled batch Failed and continuing - HostStrikes: make the eviction verdict exactly-once under concurrency (deadHosts.add is the atomic gate) and re-check produced before publishing - SyncCoverage: bound the identity fingerprint cache (a caller minting fresh Filter instances per cycle could grow it forever); legs() gains a floor parameter so a complete band re-opens its older span when the caller's window deepens; coveringWindow no longer treats a fully covered relay as needing the whole filter - PagingWindowProgress: accept single-second windows (a band's re-read edge leg is exactly that shape) geode: - MirrorWorker: cap reconciledThrough at the leg's own ceiling — the older leg of a resumed catch-up no longer stamps the band complete through 'now' before the newer leg has run (silent event loss for up to fullResyncSeconds if that leg failed) - MirrorWorker: run negentropy and the paged fallback by hand instead of negentropySyncOrFetch: drops the O(delivered-ids) dedup set from the mirror path, and a fallback resets the observed span so a band never claims interior ranges only a half-finished reconcile scattered over - MirrorWorker: clamp a paged band's ceiling to the snapshot instant so one future-dated event cannot suppress the next boot's newer leg - MirrorWorker.close(): join the workers (bounded) so the final coverage flush carries the last records - Main: gate the coverage file on the store actually being persistent — database.file with in_memory=true (the default) persisted bands over a volatile store, and the next boot skipped the backfill over an empty database; honor --db overrides - SyncCoverageFile: request ATOMIC_MOVE explicitly; fix the restore/dirty comment - Import summary now prints the failed count; document mirror_sync_state_file in config.example.toml --- geode/config.example.toml | 9 + .../kotlin/com/vitorpamplona/geode/Main.kt | 25 ++- .../geode/mirror/MirrorWorker.kt | 155 ++++++++++++------ .../geode/mirror/SyncCoverageFile.kt | 15 +- .../relay/client/accessories/SyncCoverage.kt | 45 ++++- .../client/paging/PagingWindowProgress.kt | 15 +- .../relay/server/backend/IngestQueue.kt | 6 + .../quartz/nip01Core/store/IEventStore.kt | 11 +- .../nip01Core/store/ObservableEventStore.kt | 2 +- .../store/sqlite/SQLiteEventStore.kt | 24 ++- .../reachability/HostStrikes.kt | 8 +- .../client/accessories/SyncCoverageTest.kt | 30 ++++ .../client/paging/PagingWindowProgressTest.kt | 27 ++- .../prodbench/ConcurrentIngestLossTest.kt | 5 +- 14 files changed, 296 insertions(+), 81 deletions(-) diff --git a/geode/config.example.toml b/geode/config.example.toml index 2ba85e0ad4..4651105623 100644 --- a/geode/config.example.toml +++ b/geode/config.example.toml @@ -112,6 +112,15 @@ require_auth = false # the future. Enforced by RejectFutureEventsPolicy. # reject_future_seconds = 1800 +# Path for the JSON file that remembers what the [[mirror]] catch-up +# has already synced (per upstream, per scope), so a restart resumes +# instead of re-downloading each upstream's whole backfill window. +# Defaults to ".sync-coverage.json" next to the event +# store; only written when the store itself is file-backed (an +# in-memory store keeps no resume state — saved coverage would +# describe events that no longer exist). +# mirror_sync_state_file = "/var/lib/geode/events.db.sync-coverage.json" + [authorization] # Allow / deny lists. Allow is a permissive ceiling; deny still # removes specific entries inside it. Enforced by Pubkey/KindAllowDenyPolicy. diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt index 2026a7f3ba..1ed17bbcdb 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/Main.kt @@ -45,6 +45,7 @@ import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip01Core.store.NdjsonImportExport import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore import com.vitorpamplona.quartz.nip77Negentropy.NegentropySettings +import com.vitorpamplona.quartz.utils.Log import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -224,7 +225,8 @@ private fun runImport(args: Array) { } System.err.println( "geode import: read=${stats.read} imported=${stats.imported} " + - "rejected=${stats.rejected} invalid-sig=${stats.invalid} malformed=${stats.malformed} " + + "rejected=${stats.rejected} failed=${stats.failed} " + + "invalid-sig=${stats.invalid} malformed=${stats.malformed} " + "→ ${ctx.dbFile ?: "(in-memory — not persisted; pass --db)"}", ) } finally { @@ -413,14 +415,25 @@ private fun serve(args: Array) { "[[mirror]] must not list this relay's own URL ($advertisedUrl)" } // Resume state for the mirror catch-up, following the admin state-file - // convention: next to the event database unless configured. An in-memory - // store keeps none — bands only pay off across restarts. + // convention: next to the event database unless configured. Keyed to the + // store's ACTUAL persistence, not to `database.file` being set: a + // volatile store with a persistent coverage file would claim, on the + // next boot, that an empty database already holds the backfill window — + // and the mirror would never fetch it. + val sqliteBackend = + config.database.backend + .trim() + .lowercase() in StoreFactory.SQLITE_BACKEND_KEYWORDS + val persistentLocation = + a.opt("--db") ?: config.database.file?.takeUnless { sqliteBackend && config.database.in_memory } val syncCoverage = - if (upstreams.isEmpty()) { + if (upstreams.isEmpty() || persistentLocation == null) { + if (upstreams.isNotEmpty() && config.options.mirror_sync_state_file != null) { + Log.w("Main") { "mirror_sync_state_file ignored: the event store is in-memory, so saved coverage would outlive the events it describes" } + } null } else { - (config.options.mirror_sync_state_file ?: config.database.file?.let { "$it.sync-coverage.json" }) - ?.let { SyncCoverageFile(File(it)) } + SyncCoverageFile(File(config.options.mirror_sync_state_file ?: "$persistentLocation.sync-coverage.json")) } val mirror = if (upstreams.isEmpty()) { diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt index 5916a5f794..c98e050e37 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt @@ -23,9 +23,11 @@ package com.vitorpamplona.geode.mirror import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper import com.vitorpamplona.quartz.nip01Core.relay.client.NostrClient +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.NegentropySyncException import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.SyncCoverage +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.fetchAllPages import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.negentropyReconcile -import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.negentropySyncOrFetch +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.negentropySync import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.EventMessage import com.vitorpamplona.quartz.nip01Core.relay.commands.toRelay.ReqCmd @@ -41,12 +43,15 @@ import com.vitorpamplona.quartz.utils.TimeUtils import kotlinx.coroutines.CancellationException import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.channels.trySendBlocking import kotlinx.coroutines.delay import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeoutOrNull import okhttp3.OkHttpClient import java.time.Duration import java.util.concurrent.atomic.AtomicLong @@ -155,9 +160,9 @@ class MirrorWorker( * historical window before the live REQ tail takes over. The geode binary * turns this on (see `Main`); it defaults **off** so the many existing * MirrorWorker tests keep exercising the pure live-REQ path unchanged. - * When on, `negentropySyncOrFetch` automatically falls back to paged REQ - * against an upstream that doesn't speak NIP-77 — so "either mode" is - * transparent and needs no separate toggle. + * When on, the catch-up automatically falls back to paged REQ against an + * upstream that doesn't speak NIP-77 — so "either mode" is transparent + * and needs no separate toggle. */ private val negentropyBackfill: Boolean = false, /** @@ -422,9 +427,9 @@ class MirrorWorker( * **client-paced** so a fast upstream can't overrun the sink: a plain REQ * backfill of a large set dies here (strfry kills a slow REQ client once its * unsent-outbound buffer crosses `maxPendingOutboundBytes`), which is exactly - * why this uses negentropy. [INostrClient.negentropySyncOrFetch] falls back - * to paged REQ automatically when the upstream doesn't speak NIP-77, so the - * mirror is compatible with either kind of upstream with no config. + * why this uses negentropy. When the upstream doesn't speak NIP-77 the + * catch-up falls back to paged REQ, so the mirror is compatible with + * either kind of upstream with no config. * * A failure here is non-fatal: the live subscription keeps the mirror current * and the reconnect watermark narrows any residual gap. @@ -437,13 +442,12 @@ class MirrorWorker( ) { // Resume memory: coverage is keyed on the STABLE scoped filter, never // on the boot window — whose since/until change every start and would - // never match a stored band. The window only slides forward - // (initialSince = boot − backfillSeconds), so a band recorded against - // an earlier boot's lower floor never licenses skipping a range this - // boot can ask about but the last one could not. + // never match a stored band. The window itself rides along as the + // floor argument, so a band recorded against a shallower window + // re-opens the older span when the operator deepens the backfill. val legs = coverage - ?.legs(up.url, scopedBase) + ?.legs(up.url, scopedBase, initialSince) ?.mapNotNull { clampToWindow(it, initialSince, until) } ?: listOf(scopedBase.copy(since = initialSince, until = until)) if (legs.isEmpty()) { @@ -452,9 +456,10 @@ class MirrorWorker( } // Bounded hand-off → one ingest consumer. `onEvent` can't suspend, so it - // blocks here when the sink falls behind; because negentropySyncOrFetch's - // own delivery pipeline is bounded, that backpressure reaches all the way - // to the upstream — no unbounded buffering (unlike the live-tail path). + // blocks here when the sink falls behind; because negentropySync's own + // delivery pipeline is bounded (and a paged REQ is paced by its pages), + // that backpressure reaches all the way to the upstream — no unbounded + // buffering (unlike the live-tail path). val handoff = Channel(capacity = CATCHUP_HANDOFF) val consumer = scope.launch { @@ -493,41 +498,81 @@ class MirrorWorker( val syncStartedAt = TimeUtils.now() var seenMin: Long? = null var seenMax: Long? = null - val result = - client.negentropySyncOrFetch( - relay = up.url, - filter = leg, - localEntries = localEntries, - onEvent = { event -> - // Same containment as the live path: even a trusted - // upstream may only inject events inside the declared scope. - if (up.filter == null || up.filter.match(event)) { - // Only plausible stamps widen a band — one - // misdated event must not discard the rest. - if (SyncCoverage.isPlausible(event.createdAt)) { - seenMin = minOf(seenMin ?: event.createdAt, event.createdAt) - seenMax = maxOf(seenMax ?: event.createdAt, event.createdAt) - } - handoff.trySendBlocking(event) - } else { - filtered.incrementAndGet() - } - }, - ) - downloaded += result.downloaded - paged = paged || result.pagedFallback + + fun observe(event: Event) { + // Same containment as the live path: even a trusted + // upstream may only inject events inside the declared scope. + if (up.filter == null || up.filter.match(event)) { + // Only plausible stamps widen a band — one + // misdated event must not discard the rest. + if (SyncCoverage.isPlausible(event.createdAt)) { + seenMin = minOf(seenMin ?: event.createdAt, event.createdAt) + seenMax = maxOf(seenMax ?: event.createdAt, event.createdAt) + } + handoff.trySendBlocking(event) + } else { + filtered.incrementAndGet() + } + } + + // The two phases run by hand rather than through + // negentropySyncOrFetch, for two reasons. The combinator keeps + // every delivered id for cross-phase dedup — a multi-million- + // event catch-up cannot afford that heap, and the store's + // unique-id constraint dedups anyway. And the fallback must + // reset the observed span: a half-finished reconcile delivers + // events scattered across the whole leg, and a band built from + // that scatter would claim interior ranges nobody walked. + val legPaged = + try { + downloaded += + client + .negentropySync( + relay = up.url, + filter = leg, + localEntries = localEntries, + onEvent = ::observe, + ).downloaded + false + } catch (e: NegentropySyncException) { + seenMin = null + seenMax = null + // The watchdog matches negentropySync's default rather + // than fetchAllPages' shorter one: a paged catch-up + // sits behind the same slow upstreams. + downloaded += client.fetchAllPages(up.url, listOf(leg), idleTimeoutMs = 120_000L) { observe(it) } + true + } + paged = paged || legPaged // Recorded per leg, so a failure between legs keeps the ground - // the first one gained. A clean reconcile is complete through - // the instant its snapshot was read; a paged fallback earns - // only the span it actually saw. - coverage?.record( - up.url, - scopedBase, - seenMin, - seenMax, - paged = result.pagedFallback, - reconciledThrough = if (result.pagedFallback) null else syncStartedAt, - ) + // the first one gained — and no more: a reconcile compared only + // its own leg, so completeness reaches the leg's ceiling, never + // "now" while a later leg is still pending. A paged fallback + // earns only the span it actually saw, capped at the snapshot + // instant so one future-dated event cannot lift the band's + // ceiling past what was asked. + if (legPaged) { + coverage?.record( + up.url, + scopedBase, + seenMin, + seenMax?.coerceAtMost(syncStartedAt), + paged = true, + ) + } else { + val legFloor = leg.since ?: initialSince + coverage?.record( + up.url, + scopedBase, + // The compared range starts at the leg's floor whether + // or not anything was observed there — that is what a + // clean reconcile proves. + observedMin = minOf(seenMin ?: legFloor, legFloor), + observedMax = null, + paged = false, + reconciledThrough = minOf(leg.until ?: syncStartedAt, syncStartedAt), + ) + } } Log.i("MirrorWorker") { val how = if (paged) "paged REQ (upstream has no NIP-77)" else "negentropy" @@ -743,11 +788,21 @@ class MirrorWorker( runCatching { client.close() } inbound.close() scope.cancel() + // Wait (bounded) for the workers to land: a coverage.record racing + // past the state file's final flush would be recorded and lost. + // Bounded because a worker parked in a blocking hand-off does not + // feel the cancel, and shutdown must not hang on it. + runBlocking { + withTimeoutOrNull(CLOSE_JOIN_MS) { scope.coroutineContext[Job]?.join() } + } okhttp?.dispatcher?.executorService?.shutdown() okhttp?.connectionPool?.evictAll() } private companion object { + /** How long [close] waits for the worker coroutines to land. */ + const val CLOSE_JOIN_MS = 5_000L + /** Matches the Android app's relay-pool WebSocket ping interval. */ const val PING_INTERVAL_SECS = 120L @@ -773,7 +828,7 @@ class MirrorWorker( /** * Depth of the catch-up hand-off between the negentropy download and the - * ingest consumer. Small: negentropySyncOrFetch is already internally + * ingest consumer. Small: the negentropy download is already internally * backpressured, so this only smooths the seam — the bounded IngestQueue * behind `server.ingest` is the real limiter. */ diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt index f02d8f56f5..b31c51a60d 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt @@ -31,6 +31,7 @@ import kotlinx.serialization.json.jsonPrimitive import kotlinx.serialization.json.long import kotlinx.serialization.json.put import java.io.File +import java.nio.file.AtomicMoveNotSupportedException import java.nio.file.Files import java.nio.file.StandardCopyOption @@ -58,7 +59,8 @@ class SyncCoverageFile( init { load() - // Loading marks every restored band dirty; the file already has them. + // restore() bypasses onChange, but stay defensive: reopening a file + // must never count as a change, or every boot rewrites it. dirty = false flusher = Thread { @@ -130,7 +132,16 @@ class SyncCoverageFile( file.parentFile?.mkdirs() val tmp = File(file.parentFile ?: File("."), "${file.name}.tmp") tmp.writeText(json.encodeToString(JsonObject.serializer(), doc)) - Files.move(tmp.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING) + // ATOMIC_MOVE requested explicitly: without it the JVM may + // legally fall back to copy+delete, and a reader could see a + // half map. Same-directory rename, so support is the norm; a + // filesystem that truly can't gets the plain move (and the + // corrupt-file recovery absorbs the residual risk). + try { + Files.move(tmp.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE) + } catch (_: AtomicMoveNotSupportedException) { + Files.move(tmp.toPath(), file.toPath(), StandardCopyOption.REPLACE_EXISTING) + } }.onFailure { Log.w("SyncCoverageFile") { "could not write ${file.path}: ${it.message}" } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt index c6ddff4dce..35ba9b330a 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt @@ -87,8 +87,10 @@ class SyncCoverage( // filter -> its canonical json. Filter.toJson() runs to tens of thousands // of characters for author-scoped filters, and a fan-out keys once per // relay per cycle over the SAME handful of filter instances. Filter - // compares by identity, so this map is an identity cache; an - // equal-but-distinct filter still keys correctly, just without the cache. + // compares by identity, so this map is an identity cache — and an + // identity cache retains every distinct instance it is handed. A caller + // that rebuilds its filter each cycle would grow it forever, so past + // MAX_FINGERPRINTS new instances key correctly but are not cached. private val fingerprints = ConcurrentMap() /** @@ -106,6 +108,7 @@ class SyncCoverage( fun legs( url: NormalizedRelayUrl, filter: Filter, + floor: Long? = null, ): List { val band = bands[key(url, filter)] ?: return listOf(filter) // Time for another full pass: relays gain old events, and without @@ -114,9 +117,20 @@ class SyncCoverage( val legs = mutableListOf() // Older: up to and including the band's floor, but not past the - // filter's. A complete band has no older leg at all — the reconcile - // already compared the whole range. - if (!band.complete && (filter.since == null || band.minCreatedAt >= filter.since)) { + // filter's (or, when the filter has no `since`, the caller's + // [floor] — a sync window the filter itself must not carry, or it + // would change the band's key every run). A complete band compared + // its whole range already, but only down to the floor it ran + // against: a caller now reaching deeper — a raised backfill window + // — re-opens the span below the band. + val since = filter.since ?: floor + val wantsOlder = + if (band.complete) { + since != null && since < band.minCreatedAt + } else { + since == null || band.minCreatedAt >= since + } + if (wantsOlder) { legs.add(filter.copy(until = minOf(band.minCreatedAt, filter.until ?: Long.MAX_VALUE))) } @@ -211,12 +225,17 @@ class SyncCoverage( var since = Long.MAX_VALUE for (url in urls) { val legs = legs(url, filter) + // Nothing outside its band: this relay asks nothing of the + // snapshot at all — the best case must not widen the window. + if (legs.isEmpty()) continue // More than one leg means an older gap this relay still wants, so // the snapshot cannot start above the filter's own floor. val only = legs.singleOrNull() ?: return filter val legSince = only.since ?: return filter since = minOf(since, legSince) } + // Every relay fully covered: any window would do; the unnarrowed + // filter is merely safe, and callers usually skip the sync entirely. return if (since == Long.MAX_VALUE) filter else filter.copy(since = since) } @@ -245,9 +264,23 @@ class SyncCoverage( private fun key( url: NormalizedRelayUrl, filter: Filter, - ): String = "${url.url} ${fingerprints.getOrPut(filter) { filter.toJson() }}" + ): String { + val fingerprint = + fingerprints[filter] + ?: filter.toJson().also { + // Bounded: stable callers hit the cache at any size a real + // config produces; a caller minting fresh instances just + // pays the toJson each time instead of growing the heap. + if (fingerprints.size() < MAX_FINGERPRINTS) fingerprints[filter] = it + } + return "${url.url} $fingerprint" + } companion object { + // More filter instances than any deliberate configuration holds; only + // a caller rebuilding filters per cycle ever reaches it. + private const val MAX_FINGERPRINTS = 1_000 + /** * A week. Long enough that the narrow path is the normal one, short * enough that anything a band is wrong about is wrong for days, not diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/paging/PagingWindowProgress.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/paging/PagingWindowProgress.kt index 0f515d0ca5..348a27544d 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/paging/PagingWindowProgress.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/paging/PagingWindowProgress.kt @@ -62,16 +62,25 @@ class PagingWindowProgress( private val windows = ConcurrentMap() - /** Begin a pagination over `[bottom, top]` seconds. An inverted window is not one. */ + /** + * Begin a pagination over `[bottom, top]` seconds. An inverted window is + * not one; a single-second window (`top == bottom`) is — coverage legs + * that re-read a band's edge second are exactly that shape. + */ fun begin( key: String, top: Long, bottom: Long, ) { - if (top > bottom) windows[key] = Window(top, bottom, nowMillis(), top) + if (top >= bottom) windows[key] = Window(top, bottom, nowMillis(), top) } - /** The pagination reached [until]; monotonic, so a page that jumps back cannot un-advance it. */ + /** + * The pagination reached [until]; monotonic, so a page that jumps back + * cannot un-advance it. The check-then-set is unsynchronized on purpose: + * one pagination is one coroutine, and a display racing a mark can only + * ever read a value one page stale. + */ fun mark( key: String, until: Long, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/IngestQueue.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/IngestQueue.kt index d955ed927c..18c0dfa34e 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/IngestQueue.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/backend/IngestQueue.kt @@ -36,6 +36,7 @@ import kotlin.concurrent.atomics.AtomicBoolean import kotlin.concurrent.atomics.AtomicInt import kotlin.concurrent.atomics.ExperimentalAtomicApi import kotlin.coroutines.CoroutineContext +import kotlin.coroutines.cancellation.CancellationException /** * Group-commit writer for incoming EVENT publishes. @@ -290,6 +291,11 @@ class IngestQueue( } else { try { store.batchInsert(toInsert) + } catch (e: CancellationException) { + // Shutdown, not a store failure: rethrow so the loop + // stops instead of stamping the batch Failed and + // carrying on while cancelled. + throw e } catch (e: Throwable) { Log.w("IngestQueue") { "batchInsert failed for ${toInsert.size} events: ${e.message}" } val reason = e.message ?: e::class.simpleName ?: "insert failed" diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt index 16b82132e9..ae49e25749 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/IEventStore.kt @@ -27,6 +27,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent +import kotlin.coroutines.cancellation.CancellationException /** * Storage contract for Nostr events: insert, filter-query, count, delete, @@ -136,16 +137,20 @@ interface IEventStore : AutoCloseable { * * Default impl runs each insert in its own transaction — correct * but loses the group-commit win — and cannot classify a throw from - * [insert], so it reports `Rejected`. Implementations that can tell - * a refusal from a write error should override and say which. + * [insert], so it reports `Failed`: re-offering a duplicate is + * idempotent, while dropping a good event on a transient store + * error is not. Implementations that can tell a refusal from a + * write error should override and say which. */ suspend fun batchInsert(events: List): List = events.map { event -> try { insert(event) InsertOutcome.Accepted + } catch (e: CancellationException) { + throw e } catch (e: Throwable) { - InsertOutcome.Rejected(e.message ?: e::class.simpleName ?: "insert failed") + InsertOutcome.Failed(e.message ?: e::class.simpleName ?: "insert failed") } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt index c314c4e3b2..71a6e3c292 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/ObservableEventStore.kt @@ -103,7 +103,7 @@ class ObservableEventStore( // order. Already-expired ephemerals are dropped (matching // [insert]). Accepted events are emitted on [_changes] only // after the inner batch returns, so a commit failure that - // converts everything to Rejected suppresses the emits. + // converts everything to Failed suppresses the emits. if (events.isEmpty()) return emptyList() val outcomes = arrayOfNulls(events.size) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt index 93f3bb5bb6..3db879925d 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/store/sqlite/SQLiteEventStore.kt @@ -510,7 +510,29 @@ class SQLiteEventStore( // ROLLBACK shouldn't mask the original cause. runCatching { db.execSQL("ROLLBACK TRANSACTION TO SAVEPOINT $sp") } runCatching { db.execSQL("RELEASE SAVEPOINT $sp") } - IEventStore.InsertOutcome.Rejected(e.message ?: e::class.simpleName ?: "insert failed") + classifyRowError(e) + } + } + + /** + * Which side failed decides whether the caller may drop the event. + * Policy refusals are recognizable — every schema trigger RAISEs with + * a `blocked:` prefix, the immutability guards say "not allowed", and + * a duplicate id is a constraint violation. Anything else (disk full, + * I/O error, schema drift) is the store failing to write an acceptable + * event: `Failed`, so a rising count is loud instead of blending into + * the duplicate tally. + */ + private fun classifyRowError(e: Throwable): IEventStore.InsertOutcome { + val message = e.message ?: e::class.simpleName ?: "insert failed" + val refusal = + message.contains("blocked:") || + message.contains("not allowed") || + message.contains("constraint", ignoreCase = true) + return if (refusal) { + IEventStore.InsertOutcome.Rejected(message) + } else { + IEventStore.InsertOutcome.Failed(message) } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/HostStrikes.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/HostStrikes.kt index 11a2c2d51d..1b846eac93 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/HostStrikes.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/HostStrikes.kt @@ -82,7 +82,13 @@ class HostStrikes( val authority = authorityOf(url.url) if (authority in producedHosts || authority in deadHosts) return null if (strikes.merge(authority, 1) { old, new -> old + new } < strikeLimit) return null - deadHosts.add(authority) + // Concurrent strikers can cross the threshold together; add() is the + // atomic exactly-once gate on who publishes. Re-check produced after + // winning it: a delivery that landed while this strike was in flight + // outranks the verdict, and a verdict for a host that just answered + // would be a false public record. + if (!deadHosts.add(authority)) return null + if (authority in producedHosts) return null return Evicted(authority, strikeLimit) } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt index 78abe61682..53120624a4 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt @@ -159,6 +159,22 @@ class SyncCoverageTest { assertEquals(2, walked.legs(relay, profiles).size, "a paged walk says nothing about what it never asked for") } + @Test + fun `a deeper floor re-opens history below a complete band`() { + // A reconcile only compared down to the window it ran against. When + // the operator raises the backfill window, the span below the band's + // recorded floor is ground nobody ever asked for. + val c = SyncCoverage() + c.record(relay, profiles, 1_700_000_000L, null, paged = false, reconciledThrough = 1_700_002_000L) + + assertEquals(1, c.legs(relay, profiles, floor = 1_700_000_000L).size, "same floor: nothing older to ask") + + val legs = c.legs(relay, profiles, floor = 1_600_000_000L) + assertEquals(2, legs.size, "a deeper floor re-opens the older span") + assertEquals(1_700_000_000L, legs[0].until, "up to the floor the reconcile actually compared") + assertEquals(1_700_002_000L, legs[1].since) + } + // ---- the periodic full re-walk ----------------------------------------- @Test @@ -224,6 +240,20 @@ class SyncCoverageTest { assertEquals(1_700_003_000L, c.coveringWindow(listOf(relay, other, third), profiles).since) } + @Test + fun `a fully covered relay does not widen the shared window`() { + // A complete band past a bounded filter's ceiling needs no legs at + // all. The best case must not force the snapshot back to the whole + // filter — that would make full coverage cost the most. + val window = Filter(kinds = listOf(0), since = 1_700_000_000L, until = 1_700_005_000L) + val c = SyncCoverage() + c.record(relay, window, 1_700_000_000L, null, paged = false, reconciledThrough = 1_700_009_000L) + c.record(other, window, 1_700_000_000L, null, paged = false, reconciledThrough = 1_700_003_000L) + + assertEquals(0, c.legs(relay, window).size, "covered past the ceiling: nothing to ask") + assertEquals(1_700_003_000L, c.coveringWindow(listOf(relay, other), window).since) + } + @Test fun `a relay with an older gap also widens the shared window`() { val c = SyncCoverage() diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/paging/PagingWindowProgressTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/paging/PagingWindowProgressTest.kt index 2c1c0ed433..35c5ecfdab 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/paging/PagingWindowProgressTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/paging/PagingWindowProgressTest.kt @@ -39,7 +39,7 @@ class PagingWindowProgressTest { val p = PagingWindowProgress() p.begin("a", top = 1_000L, bottom = 0L) - assertClose(0.0, p.fraction(), "nothing walked yet") + assertClose(0.0, p.fraction(), "nothing paged yet") p.mark("a", 750L) assertClose(0.25, p.fraction()) @@ -64,7 +64,7 @@ class PagingWindowProgressTest { @Test fun `windows average rather than sum`() { - // Two relays each walking their own window: one done and one untouched + // Two relays each paging their own window: one done and one untouched // is half way — not 100% as summing would give. val p = PagingWindowProgress() p.begin("a", top = 1_000L, bottom = 0L) @@ -84,14 +84,14 @@ class PagingWindowProgressTest { p.finish("a") - assertClose(0.5, p.fraction(), "only b is still walking") + assertClose(0.5, p.fraction(), "only b is still paging") p.finish("b") assertNull(p.fraction(), "nothing paging means no number to report") } @Test - fun `a group prefix scopes the numbers to its own walks`() { - // One instance serves many concurrent walks; without the scope two + fun `a group prefix scopes the numbers to its own paginations`() { + // One instance serves many concurrent paginations; without the scope two // streams would print each other's percentages. val p = PagingWindowProgress() p.begin("streamA|wss://r1", top = 1_000L, bottom = 0L) @@ -104,7 +104,7 @@ class PagingWindowProgressTest { } @Test - fun `an inverted or empty window is not a pagination`() { + fun `an inverted window is not a pagination`() { // A leg whose since is above its until asks for a range nothing can be // in. Dividing by that span would produce infinities on the status line. val p = PagingWindowProgress() @@ -114,6 +114,21 @@ class PagingWindowProgressTest { assertNull(p.fraction()) } + @Test + fun `a single-second window is a pagination`() { + // Coverage legs re-read a band's edge second: since == until is a + // real, one-second range, not an inverted one. + val p = PagingWindowProgress() + + p.begin("a", top = 500L, bottom = 500L) + + assertClose(0.0, p.fraction(), "tracked from its start") + p.mark("a", 500L) + assertClose(0.0, p.fraction(), "still at its only second") + p.finish("a") + assertNull(p.fraction()) + } + @Test fun `no ETA before the estimate means anything`() { val p = PagingWindowProgress() diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/ConcurrentIngestLossTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/ConcurrentIngestLossTest.kt index 335d587cc6..3d87b2f457 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/ConcurrentIngestLossTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/ConcurrentIngestLossTest.kt @@ -120,6 +120,7 @@ class ConcurrentIngestLossTest { val accepted = ConcurrentHashMap.newKeySet() val rejected = AtomicInteger() + val failed = AtomicInteger() val window = Semaphore(200) val done = CompletableDeferred() val remaining = AtomicInteger(events.size) @@ -130,7 +131,7 @@ class ConcurrentIngestLossTest { when (outcome) { is IEventStore.InsertOutcome.Accepted -> accepted.add(e.id) is IEventStore.InsertOutcome.Rejected -> rejected.incrementAndGet() - is IEventStore.InsertOutcome.Failed -> rejected.incrementAndGet() + is IEventStore.InsertOutcome.Failed -> failed.incrementAndGet() } window.release() if (remaining.decrementAndGet() == 0) done.complete(Unit) @@ -155,7 +156,7 @@ class ConcurrentIngestLossTest { } } val stored = store.count(Filter()) - println(" submitted=${events.size} accepted=${accepted.size} rejected=${rejected.get()} stored=$stored lostAcceptedRegular=$lost") + println(" submitted=${events.size} accepted=${accepted.size} rejected=${rejected.get()} failed=${failed.get()} stored=$stored lostAcceptedRegular=$lost") ingest.close() queueJob.cancel() From 156176f5fe8c72df074beb6438f85d6722d0797f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 13:22:55 +0000 Subject: [PATCH 015/132] fix(quartz): stop RelayUrlNormalizer from accepting urls that can never be relays MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validated against a 45k-entry corpus of relay-url hints exported from real events (317k tag occurrences). The normalizer was converting ~30k distinct https:// urls with paths (Mastodon/bridge actor urls from proxy tags, web pages, images) into wss:// addresses that can never answer, wasting connection attempts and relay-pool slots. - http(s) → ws(s) scheme swap now only applies to bare hosts (host[:port] plus optional trailing slash); an http url with a path, query or fragment is a web resource, not a mistyped relay. - Authority validation for all schemes: rejects empty hosts, userinfo (@), percent-encoding and commas in the host, and paths that start with // (the signature of a second pasted url, e.g. wss://https//host). - Interior whitespace and backslashes reject the whole string (multiple urls or prose in one field). - Zero-width characters (U+200B..D, U+2060, BOM) are stripped instead of corrupting the parse (wss://\u200Bnos.lol previously normalized to the scheme-less //nos.lol/). - Schemeless candidates must look like host[:port] (single colon, numeric port), rejecting addressable pointers (31990:pubkey:dtag) and bare scheme leftovers (wss:) before the expensive RFC 3986 parse. - Protocol-relative //host/ inputs normalize as wss:// instead of resolving to https://. - normalizeOrNull now double-checks the parser output still starts with ws(s):// and rejects otherwise. Corpus impact: 30,014 garbage urls (30,333 events) now rejected, 0 real relays lost (all 15,162 kept urls normalize byte-identically), 6 broken outputs fixed. fix() itself stays allocation-free on the happy path (~357ns vs ~318ns per call on the garbage-heavy corpus). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01A9sSyh1QLJD3PZ18tVPPVK --- .../relay/normalizer/RelayUrlNormalizer.kt | 147 ++++++++++++++++-- .../nip01Core/relay/RelayUrlFormatterTest.kt | 81 ++++++++++ 2 files changed, 216 insertions(+), 12 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt index 04f8c9cc06..8949a0b82c 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt @@ -84,6 +84,97 @@ class RelayUrlNormalizer { private fun norm(url: String) = NormalizedRelayUrl(Rfc3986.normalize(url)) + private fun isInvisible(c: Char) = c == '\u200B' || c == '\u200C' || c == '\u200D' || c == '\u2060' || c == '\uFEFF' + + /** + * Scans the authority (host[:port]) that starts at [start] and ends at the first + * `/`, `?` or `#`. Returns the end index, or -1 when the authority is empty or + * contains characters that never appear in a real relay host (`@` userinfo, + * percent-encoding, commas). + */ + private fun authorityEnd( + url: String, + start: Int, + ): Int { + if (start >= url.length) return -1 + var i = start + if (url[i] == '[') { + // IPv6 literal: defer validation to the RFC 3986 parser + while (i < url.length && url[i] != '/' && url[i] != '?' && url[i] != '#') i++ + return i + } + while (i < url.length) { + val c = url[i] + if (c == '/' || c == '?' || c == '#') break + if (c == '@' || c == '%' || c == ',') return -1 + i++ + } + return if (i == start) -1 else i + } + + /** + * Accepts a ws/wss url whose host starts at [hostStart] if the authority is sane + * and the path does not start with `//` (the signature of a second URL or a broken + * `https//` pasted after the scheme, e.g. `wss://https//nostr.watch/relay/x`). + */ + private fun fixWs( + url: String, + hostStart: Int, + ): String? { + val end = authorityEnd(url, hostStart) + if (end < 0) return null + if (end + 1 < url.length && url[end] == '/' && url[end + 1] == '/') return null + return url + } + + /** + * Converts an http(s) url to ws(s) only when it is a bare host — nothing after + * `host[:port]` but an optional trailing `/`. An http url with a path, query or + * fragment (Mastodon actor urls from bridge `proxy` tags, web pages, images) is + * a web resource, not a relay: converting it creates a wss:// url that can never + * answer and only wastes connection attempts. + */ + private fun fixHttp( + url: String, + hostStart: Int, + newScheme: String, + ): String? { + val end = authorityEnd(url, hostStart) + if (end < 0) return null + val bareHost = end == url.length || (end == url.length - 1 && url[end] == '/') + if (!bareHost) return null + return "$newScheme${url.substring(hostStart)}" + } + + /** + * Validates a schemeless candidate: the part before the first `/` must look like + * `host` or `host:port` — letters, digits, `.`, `-`, `_`, plus at most one `:` + * followed by digits only. Rejects addressable-event pointers (`31990:hex:dtag`), + * bare scheme leftovers (`wss:`) and anything else that would otherwise be blindly + * prefixed with `wss://`. + */ + private fun isBareHostAndPath(url: String): Boolean { + if (url[0] == '[') return true // IPv6 literal: defer to the RFC 3986 parser + var i = 0 + var portStart = -1 + while (i < url.length) { + val c = url[i] + if (c == '/') break + if (c == ':') { + if (portStart >= 0) return false + portStart = i + 1 + } else if (portStart >= 0) { + if (c < '0' || c > '9') return false + } else if (!c.isLetterOrDigit() && c != '.' && c != '-' && c != '_') { + return false + } + i++ + } + if (i == 0) return false + if (portStart >= 0 && portStart == i) return false + return true + } + @OptIn(ExperimentalContracts::class) fun fix(rawUrl: String): String? { if (rawUrl.length < 4) return null @@ -109,17 +200,33 @@ class RelayUrlNormalizer { } } - val trimmed = + var trimmed = if (url[0].isWhitespace() || url[url.length - 1].isWhitespace()) { url.trim() } else { url } + // Single pass: interior whitespace means multiple urls or prose in one field, + // backslashes never appear in a real relay url; both are garbage. Invisible + // characters (zero-width spaces, BOM) are copy-paste artifacts — strip them. + var hasInvisible = false + for (c in trimmed) { + if (c == '\\') return null + if (c.isWhitespace()) return null + if (isInvisible(c)) hasInvisible = true + } + if (hasInvisible) { + trimmed = buildString(trimmed.length) { for (c in trimmed) if (!isInvisible(c)) append(c) } + if (trimmed.length < 4) return null + } + // fast for good wss:// urls if (isRelaySchemePrefix(trimmed)) { - if (isRelaySchemePrefixSecure(trimmed) || isRelaySchemePrefixInsecure(trimmed)) { - return trimmed + if (isRelaySchemePrefixSecure(trimmed)) { + return fixWs(trimmed, 6) + } else if (isRelaySchemePrefixInsecure(trimmed)) { + return fixWs(trimmed, 5) } } @@ -127,31 +234,31 @@ class RelayUrlNormalizer { if (isHttpPrefix(trimmed)) { if (isHttpSSuffix(trimmed)) { // https:// - return "wss://${trimmed.drop(8)}" + return fixHttp(trimmed, 8, "wss://") } else if (isHttpSuffix(trimmed)) { // http:// - return "ws://${trimmed.drop(7)}" + return fixHttp(trimmed, 7, "ws://") } } // fast for good ww:// urls if (trimmed.startsWith("ww://")) { - return "wss://${trimmed.drop(5)}" + return fixWs("wss://${trimmed.drop(5)}", 6) } // fast for good ww:// urls if (trimmed.startsWith("was://")) { - return "wss://${trimmed.drop(6)}" + return fixWs("wss://${trimmed.drop(6)}", 6) } // fast for good ww:// urls if (trimmed.startsWith("Wws://")) { - return "wss://${trimmed.drop(6)}" + return fixWs("wss://${trimmed.drop(6)}", 6) } // fast for good ww:// urls if (trimmed.startsWith("Wss://")) { - return "wss://${trimmed.drop(6)}" + return fixWs("wss://${trimmed.drop(6)}", 6) } if (trimmed.contains("://")) { @@ -160,10 +267,19 @@ class RelayUrlNormalizer { return null } - return if (isOnion(trimmed) || isLocalHost(trimmed)) { - "ws://$trimmed" + // protocol-relative urls (`//host/`) are just missing the scheme + val bare = if (trimmed.startsWith("//")) trimmed.drop(2) else trimmed + if (bare.length < 4) return null + + if (!isBareHostAndPath(bare)) { + Log.d("RelayUrlNormalizer") { "Rejected $url" } + return null + } + + return if (isOnion(bare) || isLocalHost(bare)) { + "ws://$bare" } else { - "wss://$trimmed" + "wss://$bare" } } @@ -186,6 +302,13 @@ class RelayUrlNormalizer { val fixed = fix(url) if (fixed != null) { val normalized = norm(fixed) + // the RFC 3986 parser can drop or replace the scheme on odd inputs; + // anything that is not ws(s):// at this point cannot be connected to. + if (!isRelayUrl(normalized.url)) { + Log.d("NormalizedRelayUrl") { "Rejected $url" } + normalizedUrls.put(url, NormalizationResult.Error) + return null + } normalizedUrls.put(url, NormalizationResult.Success(normalized)) normalized } else { diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/RelayUrlFormatterTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/RelayUrlFormatterTest.kt index 82287dcbad..fc708eddba 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/RelayUrlFormatterTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/RelayUrlFormatterTest.kt @@ -52,4 +52,85 @@ class RelayUrlFormatterTest { fun weirdRelay() { assertNull(RelayUrlNormalizer.normalizeOrNull("wss://relay%20list%20to%20discover%20the%20user's%20content")) } + + @Test + fun httpWithPathIsNotARelay() { + // Mastodon/bridge actor urls from `proxy` tags: web resources, not relays + assertNull(RelayUrlNormalizer.normalizeOrNull("https://mastodon.social/users/amanita_muscaria")) + assertNull(RelayUrlNormalizer.normalizeOrNull("https://fosstodon.org/ap/users/115532410310000993")) + assertNull(RelayUrlNormalizer.normalizeOrNull("http://example.com/relay")) + assertNull(RelayUrlNormalizer.normalizeOrNull("https://nostr.mom/?author=0")) + assertNull(RelayUrlNormalizer.normalizeOrNull("https://nostr.mom/#section")) + + // but bare hosts still convert, with or without port and trailing slash + assertEquals("wss://nostr.mom/", RelayUrlNormalizer.normalizeOrNull("https://nostr.mom")?.url) + assertEquals("wss://nostr.mom/", RelayUrlNormalizer.normalizeOrNull("https://nostr.mom/")?.url) + assertEquals("wss://nostr.mom:4443/", RelayUrlNormalizer.normalizeOrNull("https://nostr.mom:4443/")?.url) + assertEquals("ws://nostr.mom/", RelayUrlNormalizer.normalizeOrNull("http://nostr.mom")?.url) + } + + @Test + fun wsWithPathIsStillARelay() { + assertEquals("wss://relay.nostr.band/all", RelayUrlNormalizer.normalizeOrNull("wss://relay.nostr.band/all")?.url) + assertEquals( + "wss://bostr.lecturify.net/?accept=0,1", + RelayUrlNormalizer.normalizeOrNull("wss://bostr.lecturify.net/?accept=0,1")?.url, + ) + } + + @Test + fun brokenSchemeGarbage() { + assertNull(RelayUrlNormalizer.normalizeOrNull("wss:")) + assertNull(RelayUrlNormalizer.normalizeOrNull("wss://https//nostr.watch/relay/nostr.21crypto.ch")) + assertNull(RelayUrlNormalizer.normalizeOrNull("wss://https://lockbox.fiatjaf.com")) + assertNull(RelayUrlNormalizer.normalizeOrNull("ws://http//nos.lol")) + assertNull(RelayUrlNormalizer.normalizeOrNull("wss://://plebstr.com")) + } + + @Test + fun nostrUriIsNotARelay() { + assertNull(RelayUrlNormalizer.normalizeOrNull("nostr://nrelay1qqxhwumn8ghj77tpvf6jumt9e2ckgn/")) + assertNull(RelayUrlNormalizer.normalizeOrNull("nostr://npub1dwy079xmpz7mk02kvz6wan49h02635umk32aa4ufek8t8mjxv58qy2nr22/")) + assertNull(RelayUrlNormalizer.normalizeOrNull("nostr:nrelay1qq8k2cnfwejhyum99eek7cmfv9kqsm7sdm")) + } + + @Test + fun addressablePointerIsNotARelay() { + assertNull(RelayUrlNormalizer.normalizeOrNull("31990:6be38f8c63df7dbf84db7ec4a6e6fbbd8d19dca3b980efad18585c46f04b26f9:mostr")) + } + + @Test + fun authorityGarbage() { + // userinfo, percent-encoding and commas never appear in a real relay host + assertNull(RelayUrlNormalizer.normalizeOrNull("wss://catuaba@plebs.place/")) + assertNull(RelayUrlNormalizer.normalizeOrNull("wss://africa.nostr.joburg%0A/")) + assertNull(RelayUrlNormalizer.normalizeOrNull("wss://bitcoiner,social/")) + assertNull(RelayUrlNormalizer.normalizeOrNull("wss://#web3/")) + assertNull(RelayUrlNormalizer.normalizeOrNull("name@domain.com")) + } + + @Test + fun interiorWhitespaceAndBackslashes() { + assertNull(RelayUrlNormalizer.normalizeOrNull("wss://nos lol")) + assertNull(RelayUrlNormalizer.normalizeOrNull("wss://nos.lol/ wss:/nostr.land/ avatar wss:/nostr.wine/")) + assertNull(RelayUrlNormalizer.normalizeOrNull("wss://\\\\relay.damus.io/")) + } + + @Test + fun invisibleCharactersAreStripped() { + assertEquals("wss://nos.lol/", RelayUrlNormalizer.normalizeOrNull("wss://\u200Bnos.lol")?.url) + assertEquals("wss://nos.lol/", RelayUrlNormalizer.normalizeOrNull("\uFEFFwss://nos.lol")?.url) + } + + @Test + fun protocolRelativeUrls() { + assertEquals("wss://relay.most.pub/", RelayUrlNormalizer.normalizeOrNull("//relay.most.pub/")?.url) + assertEquals("wss://nos.lol/", RelayUrlNormalizer.normalizeOrNull("//nos.lol/")?.url) + } + + @Test + fun ipv6AndLanHostsStillWork() { + assertEquals("ws://[31b:6f20:c7f2:3ddf::3221]/", RelayUrlNormalizer.normalizeOrNull("ws://[31b:6f20:c7f2:3ddf::3221]/")?.url) + assertEquals("ws://geyser-relay:7777/", RelayUrlNormalizer.normalizeOrNull("ws://geyser-relay:7777/")?.url) + } } From ba7cc72dd28bdffee33fba3cc459e1e6be8b0381 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 13:22:55 +0000 Subject: [PATCH 016/132] feat(cli): amy relay probe --file to census external relay-url candidates Feeds a file of raw candidate urls (one per line) through the same RelayUrlNormalizer the app uses, then probes the surviving clearnet set alongside the store's known universe. Rejected and onion counts are reported (file_urls/file_normalized/file_rejected in the JSON output), and results land in the NIP-66 kind:30166 reachability cache keyed by the normalized url as d-tag, as usual. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01A9sSyh1QLJD3PZ18tVPPVK --- .../amethyst/cli/commands/RelayCommands.kt | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/RelayCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/RelayCommands.kt index 64084a9356..8734ba1b02 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/RelayCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/RelayCommands.kt @@ -28,6 +28,7 @@ import com.vitorpamplona.quartz.marmot.mip00KeyPackages.KeyPackageRelayListEvent import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrlOrNull import com.vitorpamplona.quartz.nip01Core.relay.normalizer.toHttp import com.vitorpamplona.quartz.nip11RelayInfo.Nip11RelayInformation @@ -50,6 +51,7 @@ import com.vitorpamplona.quartz.nip65RelayList.tags.AdvertisedRelayType import com.vitorpamplona.quartz.nip66RelayMonitor.reachability.RelayProber import okhttp3.OkHttpClient import okhttp3.Request +import java.io.File /** * `amy relay …` — manage every relay list this account maintains, mirroring @@ -112,10 +114,12 @@ object RelayCommands { | relay info URL fetch + print a relay's NIP-11 info document (stateless) | relay probe [--timeout SECS] relay census: mass-connect every relay the store | [--concurrency N] knows and record live/dead + measured rtt-open - | into the reachability cache (NIP-66 kind:30166), + | [--file PATH] into the reachability cache (NIP-66 kind:30166), | so reachability-aware commands (graperank crawl/ | refresh) skip dead relays and wait once - | (--timeout: per wave, default 15s) + | (--timeout: per wave, default 15s; --file: also + | probe candidate urls, one per line, each run + | through the relay url normalizer first) """.trimMargin() // ------------------------------------------------------------------ @@ -337,12 +341,36 @@ object RelayCommands { // Relays dialed at once; --relay-concurrency accepted as the alias the // graperank verbs spell it with. val waveSize = args.intFlag("concurrency", args.intFlag("relay-concurrency", Context.defaultPreconnectCap)) + // Optional external candidate list: one raw url per line, run through the + // same RelayUrlNormalizer the app uses, so a probe doubles as a census of + // how a corpus of relay hints normalizes (rejects are counted, not dialed). + val fromFile = args.flag("file") args.rejectUnknown() + var fileRaw = 0 + var fileRejected = 0 + val fileRelays = HashSet() + if (fromFile != null) { + File(fromFile).forEachLine { line -> + if (line.isBlank()) return@forEachLine + fileRaw++ + val normalized = line.normalizeRelayUrlOrNull() + if (normalized == null) { + fileRejected++ + } else if (!RelayUrlNormalizer.isOnion(normalized.url)) { + fileRelays.add(normalized) + } + } + System.err.println( + "[relay-probe] $fromFile: $fileRaw urls → ${fileRelays.size} unique clearnet relays " + + "($fileRejected rejected by the normalizer)", + ) + } + Context.openOrAnonymous(dataDir).use { ctx -> ctx.prepare() val cached = ctx.reachability.snapshot() - val universe = RelayProber.knownRelayUniverse(ctx.store) + cached.live + cached.dead + val universe = RelayProber.knownRelayUniverse(ctx.store) + cached.live + cached.dead + fileRelays if (universe.isEmpty()) { Output.emit( linkedMapOf( @@ -381,6 +409,9 @@ object RelayCommands { Output.emit( linkedMapOf( "probed" to result.verdicts.size, + "file_urls" to (if (fromFile != null) fileRaw else null), + "file_normalized" to (if (fromFile != null) fileRelays.size else null), + "file_rejected" to (if (fromFile != null) fileRejected else null), "reachable" to result.reachable.size, "dead" to result.dead.size, "closed_by_policy" to authWalled, From d9a58950ecc24a566d3626948e106de6675c1be8 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 13:57:00 +0000 Subject: [PATCH 017/132] feat(quartz): stream NIP-66 probe verdicts and expose them as signable 30166 templates RelayProber.probeFlow(urls) is a cold Flow that emits each relay's Verdict the moment the relay resolves (EOSE, CLOSED or connect failure) instead of at the end of the whole census; only silent relays wait for their wave's deadline. probeWave now resolves verdicts per-terminal, so the batch probe() shares the same path. Verdict.toDiscoveryEventTemplate() renders a verdict as an UNSIGNED kind:30166 template (d = normalized url, n network type, rtt-open when reachable, R auth when the probe hit a NIP-42 auth-required CLOSED) so an external consumer signs with its own monitor key: prober.probeFlow(urls).map { it.toDiscoveryEventTemplate() } .collect { publish(signer.sign(it)) } rtt-eose is deliberately never published as rtt-read: it is measured from the wave start (dial + TLS + queueing + read), and aggregators rank on rtt values. The RelayObserver/RelayMonitor path supplies honest rtt-read/rtt-write from real traffic. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01A9sSyh1QLJD3PZ18tVPPVK --- .../reachability/RelayProber.kt | 101 ++++++-- .../reachability/RelayProberFlowTest.kt | 219 ++++++++++++++++++ 2 files changed, 301 insertions(+), 19 deletions(-) create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt index a95da3f70a..364f4e1bd0 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt @@ -29,10 +29,19 @@ import com.vitorpamplona.quartz.nip01Core.relay.client.single.newSubId import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer +import com.vitorpamplona.quartz.nip01Core.signers.EventTemplate import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent +import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.RelayDiscoveryEvent +import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.networkType +import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.requirement +import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.rtt +import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.RttType +import com.vitorpamplona.quartz.utils.TimeUtils import com.vitorpamplona.quartz.utils.concurrent.ConcurrentMap import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.flow import kotlinx.coroutines.withTimeoutOrNull import kotlin.time.TimeSource @@ -102,7 +111,7 @@ class RelayProber( val distinct = relays.toSet() var done = 0 for (wave in distinct.chunked(waveSize.coerceAtLeast(1))) { - all += probeWave(wave, timeoutMs) + probeWave(wave, timeoutMs) { all += it } done += wave.size if (distinct.size > wave.size) { val liveSoFar = all.count { it.reachable } @@ -112,10 +121,33 @@ class RelayProber( return Result(all, mark.elapsedNow().inWholeMilliseconds) } + /** + * Streaming variant of [probe]: a cold [Flow] that emits each relay's [Verdict] + * the moment that relay resolves — an answering relay's verdict arrives as soon + * as its EOSE/CLOSED/connect-failure lands, not when the whole census ends. Only + * relays that stay silent wait for their wave's [timeoutMs] deadline. + * + * Probing starts when the flow is collected and pauses between waves while the + * collector is busy (emission is sequential). Pair each verdict with + * [toDiscoveryEventTemplate] to turn the stream into signable NIP-66 kind:30166 + * records for another process to sign and publish. + */ + fun probeFlow( + relays: Collection, + timeoutMs: Long = 15_000, + waveSize: Int = 1000, + ): Flow = + flow { + for (wave in relays.toSet().chunked(waveSize.coerceAtLeast(1))) { + probeWave(wave, timeoutMs) { emit(it) } + } + } + private suspend fun probeWave( wave: List, timeoutMs: Long, - ): List { + onVerdict: suspend (Verdict) -> Unit, + ) { val mark = TimeSource.Monotonic.markNow() val waveSet = wave.toHashSet() val openRtt = ConcurrentMap() @@ -178,22 +210,7 @@ class RelayProber( } } - client.addConnectionListener(connListener) - try { - client.subscribe(subId, wave.associateWith { PROBE_FILTERS }, subListener) - val remaining = wave.toMutableSet() - withTimeoutOrNull(timeoutMs) { - while (remaining.isNotEmpty()) { - remaining.remove(terminals.receive()) - } - } - } finally { - client.unsubscribe(subId) - client.removeConnectionListener(connListener) - terminals.close() - } - - return wave.map { relay -> + fun verdictOf(relay: NormalizedRelayUrl): Verdict { val opened = openRtt[relay] val answered = eoseMs[relay] val error = errors[relay] @@ -202,7 +219,7 @@ class RelayProber( // Only a connect failure, or silence with no socket, is dead. val cannot = error?.startsWith("cannot:") == true val reachable = !cannot && (opened != null || answered != null || error != null) - Verdict( + return Verdict( relay = relay, reachable = reachable, rttOpenMs = opened ?: -1, @@ -210,6 +227,27 @@ class RelayProber( error = error, ) } + + client.addConnectionListener(connListener) + try { + client.subscribe(subId, wave.associateWith { PROBE_FILTERS }, subListener) + val remaining = wave.toMutableSet() + while (remaining.isNotEmpty()) { + val left = timeoutMs - mark.elapsedNow().inWholeMilliseconds + if (left <= 0) break + val relay = withTimeoutOrNull(left) { terminals.receive() } ?: break + // The emission happens OUTSIDE the timeout window so a collector that + // suspends on a verdict can never be cancelled mid-emission and lose it. + if (remaining.remove(relay)) onVerdict(verdictOf(relay)) + } + // Whatever is left resolved nothing by the deadline: dead if the socket + // never opened, reachable-but-slow if it did. + for (relay in remaining) onVerdict(verdictOf(relay)) + } finally { + client.unsubscribe(subId) + client.removeConnectionListener(connListener) + terminals.close() + } } companion object { @@ -263,3 +301,28 @@ class RelayProber( } } } + +/** + * This verdict as an UNSIGNED NIP-66 kind:30166 Relay Discovery template — the d-tag + * is the normalized relay url; sign it with the consumer's own monitor key (per + * NIP-66 a monitor is its own identity, so the prober never signs on its own). + * + * Only facts this probe actually observed are tagged: + * - `n` network type inferred from the url (clearnet/tor/i2p); + * - `rtt-open` when the relay was reachable — the measured WS-upgrade round trip, + * or 0 for "reachable, latency not observed" (liveness is the tag's PRESENCE); + * - `R auth` when the relay answered the probe REQ with a NIP-42 `auth-required` + * CLOSED — an observed auth wall, not a NIP-11 claim. + * + * [Verdict.rttEoseMs] is deliberately NOT written as `rtt-read`: it is measured from + * the wave start, so it bundles dial, TLS and any handshake queueing with the read — + * publishing it as a read round trip would hand aggregators an inflated latency. + * The [RelayObserver]/[RelayMonitor] path supplies honest `rtt-read`/`rtt-write` + * from real traffic instead. + */ +fun RelayProber.Verdict.toDiscoveryEventTemplate(createdAt: Long = TimeUtils.now()): EventTemplate = + RelayDiscoveryEvent.build(relay, createdAt = createdAt) { + networkType(RelayReachabilityStore.networkTypeOf(relay)) + if (reachable) rtt(RttType.OPEN, rttOpenMs.coerceAtLeast(0)) + if (error?.startsWith("closed:auth-required") == true) requirement("auth") + } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt new file mode 100644 index 0000000000..34b1ce15fc --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt @@ -0,0 +1,219 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip66RelayMonitor.reachability + +import com.vitorpamplona.quartz.nip01Core.relay.client.EmptyNostrClient +import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient +import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer +import com.vitorpamplona.quartz.nip01Core.signers.EventTemplate +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.test.currentTime +import kotlinx.coroutines.test.runTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * Pins [RelayProber.probeFlow]'s streaming contract: an answering relay's verdict + * is emitted the moment its terminal arrives, while silent relays only resolve at + * the wave deadline — and pins the observed-facts-only tag set of + * [toDiscoveryEventTemplate]. + */ +@OptIn(ExperimentalCoroutinesApi::class) +class RelayProberFlowTest { + /** Captures the probe subscription so the test can play the relays. */ + private class ScriptedClient : INostrClient by EmptyNostrClient() { + var listener: SubscriptionListener? = null + + override fun subscribe( + subId: String, + filters: Map>, + listener: SubscriptionListener?, + ) { + this.listener = listener + } + } + + private val fast = RelayUrlNormalizer.normalize("wss://fast.example.com") + private val silent = RelayUrlNormalizer.normalize("wss://silent.example.com") + private val walled = RelayUrlNormalizer.normalize("wss://walled.example.com") + + @Test + fun answeringRelayStreamsBeforeTheWaveDeadline() = + runTest { + val client = ScriptedClient() + val arrivals = mutableListOf>() + + val collector = + launch { + RelayProber(client) + .probeFlow(listOf(fast, silent), timeoutMs = 10_000) + .collect { arrivals += it to currentTime } + } + launch { + delay(200) + client.listener!!.onEose(fast, null) + } + collector.join() + + assertEquals(listOf(fast, silent), arrivals.map { it.first.relay }) + // The EOSE'd relay resolved when it answered, not at the deadline … + val (fastVerdict, fastAt) = arrivals[0] + assertTrue(fastVerdict.reachable) + assertTrue(fastAt < 1_000, "verdict should stream at answer time, arrived at ${fastAt}ms") + // … while the silent one waited out the wave and is dead (socket never opened). + val (silentVerdict, silentAt) = arrivals[1] + assertFalse(silentVerdict.reachable) + assertTrue(silentAt >= 10_000, "silent relay must wait for the deadline, arrived at ${silentAt}ms") + } + + @Test + fun authWalledRelayIsReachableWithTheWallRecorded() = + runTest { + val client = ScriptedClient() + val arrivals = mutableListOf() + + val collector = + launch { + RelayProber(client) + .probeFlow(listOf(walled), timeoutMs = 10_000) + .collect { arrivals += it } + } + launch { + delay(100) + client.listener!!.onClosed("auth-required: sign in first", walled, null) + } + collector.join() + + assertEquals(1, arrivals.size) + assertTrue(arrivals[0].reachable, "an auth wall is an app-level answer: the relay works") + assertEquals("closed:auth-required: sign in first", arrivals[0].error) + } + + @Test + fun connectFailureStreamsImmediatelyAsDead() = + runTest { + val client = ScriptedClient() + val arrivals = mutableListOf>() + + val collector = + launch { + RelayProber(client) + .probeFlow(listOf(fast), timeoutMs = 10_000) + .collect { arrivals += it to currentTime } + } + launch { + delay(50) + client.listener!!.onCannotConnect(fast, "dns failure", null) + } + collector.join() + + val (verdict, at) = arrivals.single() + assertFalse(verdict.reachable) + assertEquals("cannot:dns failure", verdict.error) + assertTrue(at < 1_000, "a failed dial must not wait for the deadline, arrived at ${at}ms") + } + + // ------------------------------------------------------------------ + // toDiscoveryEventTemplate — only observed facts become tags + // ------------------------------------------------------------------ + + private fun tagsOf(template: EventTemplate<*>) = template.tags.map { it.toList() } + + @Test + fun reachableVerdictTemplateCarriesLivenessAndNetwork() { + val template = + RelayProber + .Verdict(fast, reachable = true, rttOpenMs = 150, rttEoseMs = 480, error = null) + .toDiscoveryEventTemplate(createdAt = 1000) + + val tags = tagsOf(template) + assertEquals(30166, template.kind) + assertEquals(1000, template.createdAt) + assertTrue(listOf("d", fast.url) in tags) + assertTrue(listOf("n", "clearnet") in tags) + assertTrue(listOf("rtt-open", "150") in tags) + // rtt-eose is wave-relative (dial + queue + read) — never published as rtt-read. + assertNull(tags.firstOrNull { it[0] == "rtt-read" }) + } + + @Test + fun deadVerdictTemplateHasNoRttOpen() { + val template = + RelayProber + .Verdict(silent, reachable = false, rttOpenMs = -1, rttEoseMs = -1, error = "cannot:timeout") + .toDiscoveryEventTemplate() + + val tags = tagsOf(template) + assertTrue(listOf("d", silent.url) in tags) + // Liveness is the PRESENCE of rtt-open; a dead record must not carry one. + assertNull(tags.firstOrNull { it[0] == "rtt-open" }) + } + + @Test + fun reachableWithoutMeasuredLatencyWritesZeroFlag() { + val template = + RelayProber + .Verdict(fast, reachable = true, rttOpenMs = -1, rttEoseMs = 300, error = null) + .toDiscoveryEventTemplate() + + // 0 = "reachable, latency not observed": the flag form, never an invented number. + assertTrue(listOf("rtt-open", "0") in tagsOf(template)) + } + + @Test + fun observedAuthWallBecomesARequirementTag() { + val template = + RelayProber + .Verdict(walled, reachable = true, rttOpenMs = 90, rttEoseMs = -1, error = "closed:auth-required: sign in") + .toDiscoveryEventTemplate() + + assertTrue(listOf("R", "auth") in tagsOf(template)) + } + + @Test + fun policyClosedIsNotAnAuthRequirement() { + val template = + RelayProber + .Verdict(walled, reachable = true, rttOpenMs = 90, rttEoseMs = -1, error = "closed:blocked: not welcome") + .toDiscoveryEventTemplate() + + assertNull(tagsOf(template).firstOrNull { it[0] == "R" }) + } + + @Test + fun onionRelayIsTaggedTor() { + val onion = RelayUrlNormalizer.normalize("ws://someonionaddressabcdefghijklmnop.onion") + val template = + RelayProber + .Verdict(onion, reachable = true, rttOpenMs = 900, rttEoseMs = -1, error = null) + .toDiscoveryEventTemplate() + + assertTrue(listOf("n", "tor") in tagsOf(template)) + } +} From a7eec1d605728ca931b6e596e6f9832091d4c358 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 14:18:05 +0000 Subject: [PATCH 018/132] =?UTF-8?q?feat(quartz):=20NIP-66=20check=20option?= =?UTF-8?q?s=20=E2=80=94=20read-test=20filters,=20write-test=20event,=20ca?= =?UTF-8?q?ched=20NIP-11=20fetcher?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RelayProber.probe()/probeFlow() take a filters option choosing the check: LIVENESS_FILTERS (default, impossible-id REQ — EOSE proves liveness with no payload) or readTestFilter(limit = 1) — a REQ the relay must actually work for, querying and streaming real events, making Verdict.rttEoseMs a genuine read test. RelayProbeWriteTest.build() creates the write-check event: ephemeral kind 20166 (never stored by compliant relays) carrying a NIP-40 expiration tag 60s out as belt-and-braces for relays that store unknown ephemeral kinds. Publish it under the monitor key, time the OK for rtt-write, map rejection prefixes to R requirement tags — an OK false still proves the write path. Nip11Fetcher is the missing fetch seam for relay information documents, mirroring Nip05Fetcher: the interface lives in commonMain, OkHttpNip11Fetcher (jvmAndroid) does the Accept: application/nostr+json GET, and CachedNip11Fetcher wraps any implementation with a TTL cache — successes trusted for a day, failures remembered for five minutes so a census doesn't hammer hosts that just refused. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01A9sSyh1QLJD3PZ18tVPPVK --- .../nip11RelayInfo/CachedNip11Fetcher.kt | 97 ++++++++++++++ .../quartz/nip11RelayInfo/Nip11Fetcher.kt | 44 ++++++ .../reachability/RelayProbeWriteTest.kt | 59 +++++++++ .../reachability/RelayProber.kt | 36 ++++- .../nip11RelayInfo/CachedNip11FetcherTest.kt | 125 ++++++++++++++++++ .../reachability/RelayProberFlowTest.kt | 51 +++++++ .../nip11RelayInfo/OkHttpNip11Fetcher.kt | 60 +++++++++ 7 files changed, 465 insertions(+), 7 deletions(-) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip11RelayInfo/CachedNip11Fetcher.kt create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip11RelayInfo/Nip11Fetcher.kt create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProbeWriteTest.kt create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip11RelayInfo/CachedNip11FetcherTest.kt create mode 100644 quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip11RelayInfo/OkHttpNip11Fetcher.kt diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip11RelayInfo/CachedNip11Fetcher.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip11RelayInfo/CachedNip11Fetcher.kt new file mode 100644 index 0000000000..0805118b6b --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip11RelayInfo/CachedNip11Fetcher.kt @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip11RelayInfo + +import androidx.collection.LruCache +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.utils.TimeUtils +import kotlinx.coroutines.CancellationException + +/** + * TTL cache around any [Nip11Fetcher]. NIP-11 documents change rarely, so a + * successful fetch is served from memory for [ttlSeconds]; a FAILED fetch is + * also remembered — for the shorter [errorTtlSeconds] — so a mass census does + * not hammer a host that just refused, while still retrying it soon. + * + * Concurrent first fetches of the same relay are not deduplicated: both hit the + * network and the second result wins the cache slot. That is harmless (the + * document is idempotent) and keeps this class lock-free. + */ +class CachedNip11Fetcher( + private val delegate: Nip11Fetcher, + private val ttlSeconds: Long = DEFAULT_TTL_SECONDS, + private val errorTtlSeconds: Long = DEFAULT_ERROR_TTL_SECONDS, + maxEntries: Int = 1000, + private val now: () -> Long = { TimeUtils.now() }, +) : Nip11Fetcher { + private sealed interface Cached { + val at: Long + } + + private class Hit( + val info: Nip11RelayInformation, + override val at: Long, + ) : Cached + + private class Miss( + val message: String?, + override val at: Long, + ) : Cached + + private val cache = LruCache(maxEntries) + + /** The cached document if present and fresh; null otherwise. Never touches the network. */ + fun cachedOrNull(relay: NormalizedRelayUrl): Nip11RelayInformation? { + val hit = cache[relay] as? Hit ?: return null + return if (now() - hit.at < ttlSeconds) hit.info else null + } + + /** Drops the cache entry (success or failure) so the next [fetch] is fresh. */ + fun invalidate(relay: NormalizedRelayUrl) { + cache.remove(relay) + } + + override suspend fun fetch(relay: NormalizedRelayUrl): Nip11RelayInformation { + when (val cached = cache[relay]) { + is Hit -> if (now() - cached.at < ttlSeconds) return cached.info + is Miss -> + if (now() - cached.at < errorTtlSeconds) { + throw Nip11FetchException(cached.message ?: "cached NIP-11 failure for ${relay.url}") + } + null -> {} + } + return try { + delegate.fetch(relay).also { cache.put(relay, Hit(it, now())) } + } catch (e: Exception) { + if (e is CancellationException) throw e + cache.put(relay, Miss(e.message, now())) + throw e + } + } + + companion object { + /** Documents are near-static: trust a success for a day. */ + const val DEFAULT_TTL_SECONDS = 24L * 60 * 60 + + /** Failures are often transient: retry after five minutes. */ + const val DEFAULT_ERROR_TTL_SECONDS = 5L * 60 + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip11RelayInfo/Nip11Fetcher.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip11RelayInfo/Nip11Fetcher.kt new file mode 100644 index 0000000000..cc97860576 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip11RelayInfo/Nip11Fetcher.kt @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip11RelayInfo + +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl + +/** + * Fetches and parses a relay's NIP-11 information document (the + * `application/nostr+json` answer on the relay's https url). Mirrors the + * [com.vitorpamplona.quartz.nip05DnsIdentifiers.Nip05Fetcher] seam: the HTTP + * transport lives in a platform implementation (OkHttpNip11Fetcher on + * JVM/Android), so common code — probes, monitors, the CLI — depends only on + * this interface. Wrap any implementation in [CachedNip11Fetcher] to add a TTL + * cache. + * + * Throws [Nip11FetchException] (or a transport exception) when the document is + * unavailable or unparseable. + */ +interface Nip11Fetcher { + suspend fun fetch(relay: NormalizedRelayUrl): Nip11RelayInformation +} + +/** The relay answered, but not with a usable NIP-11 document (bad status, not JSON). */ +class Nip11FetchException( + message: String, +) : Exception(message) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProbeWriteTest.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProbeWriteTest.kt new file mode 100644 index 0000000000..12bc086047 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProbeWriteTest.kt @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip66RelayMonitor.reachability + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.signers.EventTemplate +import com.vitorpamplona.quartz.nip01Core.signers.eventTemplate +import com.vitorpamplona.quartz.nip40Expiration.ExpirationTag +import com.vitorpamplona.quartz.utils.TimeUtils + +/** + * The event a NIP-66 monitor publishes to measure a relay's WRITE path: publish + * one of these (signed with the monitor key), time the `OK`, and read the + * rejection prefix when refused (`auth-required:`/`restricted:`/`pow:` map to + * the discovery record's `R` requirement tags; a timed acceptance is `rtt-write`). + * + * The kind is EPHEMERAL (20000–29999 per NIP-01), so a compliant relay serves it + * to current subscribers and never stores it — the probe leaves nothing behind. + * [KIND] 20166 is this library's convention (30166 discovery minus the + * addressable range), not something NIP-66 standardizes; any ephemeral kind + * works. Belt-and-braces, the template also carries a NIP-40 `expiration` tag + * [EXPIRATION_SECONDS] out, so a relay that stores unknown ephemeral kinds + * anyway purges it promptly. + * + * A rejection is still a MEASUREMENT: an `OK false` proves the write path works + * and documents the relay's policy. Only silence is a failed write test. + */ +object RelayProbeWriteTest { + const val KIND = 20166 + + /** Storage-window ceiling for non-compliant relays that store ephemeral events. */ + const val EXPIRATION_SECONDS = 60L + + fun build( + content: String = "NIP-66 write probe", + createdAt: Long = TimeUtils.now(), + ): EventTemplate = + eventTemplate(KIND, content, createdAt) { + add(ExpirationTag.assemble(createdAt + EXPIRATION_SECONDS)) + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt index 364f4e1bd0..58663c2728 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt @@ -100,18 +100,24 @@ class RelayProber( /** * Probe every relay in [relays], [waveSize] at a time, giving each wave up to * [timeoutMs] to reach terminals. Returns one [Verdict] per input relay. + * + * [filters] is the REQ each relay is asked to answer. The default + * [LIVENESS_FILTERS] matches nothing, so an EOSE proves liveness without + * streaming a payload; pass [readTestFilter] to make [Verdict.rttEoseMs] a + * real read test instead (the relay must query and stream an actual event). */ suspend fun probe( relays: Collection, timeoutMs: Long = 15_000, waveSize: Int = 1000, + filters: List = LIVENESS_FILTERS, ): Result { val mark = TimeSource.Monotonic.markNow() val all = ArrayList(relays.size) val distinct = relays.toSet() var done = 0 for (wave in distinct.chunked(waveSize.coerceAtLeast(1))) { - probeWave(wave, timeoutMs) { all += it } + probeWave(wave, timeoutMs, filters) { all += it } done += wave.size if (distinct.size > wave.size) { val liveSoFar = all.count { it.reachable } @@ -127,6 +133,9 @@ class RelayProber( * as its EOSE/CLOSED/connect-failure lands, not when the whole census ends. Only * relays that stay silent wait for their wave's [timeoutMs] deadline. * + * [filters] picks the check, as in [probe]: [LIVENESS_FILTERS] (default) or + * [readTestFilter]. + * * Probing starts when the flow is collected and pauses between waves while the * collector is busy (emission is sequential). Pair each verdict with * [toDiscoveryEventTemplate] to turn the stream into signable NIP-66 kind:30166 @@ -136,16 +145,18 @@ class RelayProber( relays: Collection, timeoutMs: Long = 15_000, waveSize: Int = 1000, + filters: List = LIVENESS_FILTERS, ): Flow = flow { for (wave in relays.toSet().chunked(waveSize.coerceAtLeast(1))) { - probeWave(wave, timeoutMs) { emit(it) } + probeWave(wave, timeoutMs, filters) { emit(it) } } } private suspend fun probeWave( wave: List, timeoutMs: Long, + filters: List, onVerdict: suspend (Verdict) -> Unit, ) { val mark = TimeSource.Monotonic.markNow() @@ -230,7 +241,7 @@ class RelayProber( client.addConnectionListener(connListener) try { - client.subscribe(subId, wave.associateWith { PROBE_FILTERS }, subListener) + client.subscribe(subId, wave.associateWith { filters }, subListener) val remaining = wave.toMutableSet() while (remaining.isNotEmpty()) { val left = timeoutMs - mark.elapsedNow().inWholeMilliseconds @@ -251,10 +262,21 @@ class RelayProber( } companion object { - // A filter no event can match (ids are 64-hex of a hash): the relay answers - // with an immediate EOSE and never streams a payload. Same trick as the - // crawler's warm pool. - private val PROBE_FILTERS = listOf(Filter(ids = listOf("0".repeat(64)))) + /** + * A filter no event can match (ids are 64-hex of a hash): the relay answers + * with an immediate EOSE and never streams a payload. Same trick as the + * crawler's warm pool. This is the default check — pure liveness. + */ + val LIVENESS_FILTERS = listOf(Filter(ids = listOf("0".repeat(64)))) + + /** + * A REQ the relay must actually WORK for: query its store and stream up to + * [limit] real events before the EOSE. Pass to [probe]/[probeFlow] as + * [filters] to turn [Verdict.rttEoseMs] into a genuine read test rather + * than a liveness ping — the time still counts from the wave start (dial + * included), so compare it against [Verdict.rttOpenMs], not across waves. + */ + fun readTestFilter(limit: Int = 1) = listOf(Filter(limit = limit)) /** * The relay universe the local store knows: every read/write relay advertised diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip11RelayInfo/CachedNip11FetcherTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip11RelayInfo/CachedNip11FetcherTest.kt new file mode 100644 index 0000000000..95bd844de0 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip11RelayInfo/CachedNip11FetcherTest.kt @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip11RelayInfo + +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertNull + +class CachedNip11FetcherTest { + private val relay = RelayUrlNormalizer.normalize("wss://nostr.example.com") + + /** Counts network hits; serves [doc] or throws when [failing]. */ + private class FakeFetcher : Nip11Fetcher { + var calls = 0 + var failing = false + var doc = Nip11RelayInformation(name = "v1") + + override suspend fun fetch(relay: NormalizedRelayUrl): Nip11RelayInformation { + calls++ + if (failing) throw Nip11FetchException("boom") + return doc + } + } + + private fun cached( + delegate: FakeFetcher, + clock: () -> Long, + ) = CachedNip11Fetcher(delegate, ttlSeconds = 100, errorTtlSeconds = 10, now = clock) + + @Test + fun freshSuccessIsServedFromCache() = + runBlocking { + val net = FakeFetcher() + var now = 0L + val fetcher = cached(net) { now } + + assertEquals("v1", fetcher.fetch(relay).name) + now = 99 + assertEquals("v1", fetcher.fetch(relay).name) + assertEquals(1, net.calls, "second fetch inside the TTL must not touch the network") + } + + @Test + fun successExpiresAfterTtl() = + runBlocking { + val net = FakeFetcher() + var now = 0L + val fetcher = cached(net) { now } + + fetcher.fetch(relay) + net.doc = Nip11RelayInformation(name = "v2") + now = 100 + assertEquals("v2", fetcher.fetch(relay).name) + assertEquals(2, net.calls) + } + + @Test + fun failureIsCachedForItsOwnShorterTtl() = + runBlocking { + val net = FakeFetcher().apply { failing = true } + var now = 0L + val fetcher = cached(net) { now } + + assertFailsWith { fetcher.fetch(relay) } + now = 9 + assertFailsWith { fetcher.fetch(relay) } + assertEquals(1, net.calls, "a fresh failure must be served from cache, not re-fetched") + + now = 10 + net.failing = false + assertEquals("v1", fetcher.fetch(relay).name) + assertEquals(2, net.calls, "an expired failure must be retried") + } + + @Test + fun invalidateForcesAFreshFetch() = + runBlocking { + val net = FakeFetcher() + val fetcher = cached(net) { 0 } + + fetcher.fetch(relay) + fetcher.invalidate(relay) + fetcher.fetch(relay) + assertEquals(2, net.calls) + } + + @Test + fun cachedOrNullNeverTouchesTheNetwork() = + runBlocking { + val net = FakeFetcher() + var now = 0L + val fetcher = cached(net) { now } + + assertNull(fetcher.cachedOrNull(relay)) + assertEquals(0, net.calls) + + fetcher.fetch(relay) + assertEquals("v1", fetcher.cachedOrNull(relay)?.name) + now = 100 + assertNull(fetcher.cachedOrNull(relay), "a stale hit must not be served") + assertEquals(1, net.calls) + } +} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt index 34b1ce15fc..2efc801324 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt @@ -49,6 +49,7 @@ class RelayProberFlowTest { /** Captures the probe subscription so the test can play the relays. */ private class ScriptedClient : INostrClient by EmptyNostrClient() { var listener: SubscriptionListener? = null + var sentFilters: Map>? = null override fun subscribe( subId: String, @@ -56,6 +57,7 @@ class RelayProberFlowTest { listener: SubscriptionListener?, ) { this.listener = listener + this.sentFilters = filters } } @@ -139,6 +141,55 @@ class RelayProberFlowTest { assertTrue(at < 1_000, "a failed dial must not wait for the deadline, arrived at ${at}ms") } + // ------------------------------------------------------------------ + // Check options — liveness default, read-test override, write-test event + // ------------------------------------------------------------------ + + @Test + fun livenessFilterIsTheDefaultCheck() = + runTest { + val client = ScriptedClient() + val collector = + launch { + RelayProber(client).probeFlow(listOf(fast), timeoutMs = 1_000).collect {} + } + launch { + delay(10) + assertEquals(RelayProber.LIVENESS_FILTERS, client.sentFilters!![fast]) + client.listener!!.onEose(fast, null) + } + collector.join() + } + + @Test + fun readTestFilterIsSentWhenChosen() = + runTest { + val client = ScriptedClient() + val collector = + launch { + RelayProber(client) + .probeFlow(listOf(fast), timeoutMs = 1_000, filters = RelayProber.readTestFilter()) + .collect {} + } + launch { + delay(10) + val sent = client.sentFilters!![fast]!!.single() + assertEquals(1, sent.limit, "read test defaults to limit 1") + assertNull(sent.ids, "read test must query real events, not the impossible id") + client.listener!!.onEose(fast, null) + } + collector.join() + } + + @Test + fun writeTestEventIsEphemeralAndSelfExpiring() { + val template = RelayProbeWriteTest.build(createdAt = 5000) + + assertEquals(20166, template.kind) + assertTrue(template.kind in 20000..29999, "the write probe must be an ephemeral kind") + assertTrue(listOf("expiration", "5060") in template.tags.map { it.toList() }) + } + // ------------------------------------------------------------------ // toDiscoveryEventTemplate — only observed facts become tags // ------------------------------------------------------------------ diff --git a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip11RelayInfo/OkHttpNip11Fetcher.kt b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip11RelayInfo/OkHttpNip11Fetcher.kt new file mode 100644 index 0000000000..a28033c0d1 --- /dev/null +++ b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip11RelayInfo/OkHttpNip11Fetcher.kt @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip11RelayInfo + +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.toHttp +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.coroutines.executeAsync + +/** + * OkHttp-backed [Nip11Fetcher]: GETs the relay's https url with + * `Accept: application/nostr+json` and parses the document. The client is + * resolved per relay so callers can route Tor/proxy relays through a different + * OkHttp instance (the same seam Amethyst's Nip11Retriever uses). + */ +class OkHttpNip11Fetcher( + private val okHttpClient: (NormalizedRelayUrl) -> OkHttpClient, +) : Nip11Fetcher { + override suspend fun fetch(relay: NormalizedRelayUrl): Nip11RelayInformation = + withContext(Dispatchers.IO) { + val request = + Request + .Builder() + .header("Accept", "application/nostr+json") + .url(relay.toHttp()) + .build() + + okHttpClient(relay).newCall(request).executeAsync().use { response -> + if (!response.isSuccessful) { + throw Nip11FetchException("HTTP ${response.code} fetching NIP-11 from ${relay.url}") + } + val body = response.body.string() + if (!body.startsWith("{")) { + throw Nip11FetchException("Not a NIP-11 document from ${relay.url}") + } + Nip11RelayInformation.fromJson(body) + } + } +} From fd5bd994a1df5e43408423c4979a24a12cc73d8f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 14:41:15 +0000 Subject: [PATCH 019/132] =?UTF-8?q?feat(quartz):=20read+write=20relay=20ch?= =?UTF-8?q?ecks=20=E2=80=94=20observed=20facts=20only,=20no=20NIP-11=20cla?= =?UTF-8?q?ims?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RelayProber.readWriteCheck(relays, signer) is the deeper check pair for relays already proven live (warm sockets from a probe that just ran): - READ: a real limit-1 REQ the relay must query its store for, timed REQ→first answer (honest rtt-read on an open socket). - WRITE: one ephemeral RelayProbeWriteTest event signed by the monitor key, timed publish→OK (honest rtt-write). An OK false is a measured policy answer, kept with its NIP-01 machine-readable reason; only silence leaves the write side unobserved (writeAccepted = null). publishAndCollectResults now stamps each OK with its elapsedMs (a rejection is still a round trip; -1 when the relay never answered), so any caller gets write latency for free. toDiscoveryEventTemplate(readWrite = ...) folds the pair into the 30166 template: rtt-read/rtt-write when measured, R auth / R pow when the write was refused with auth-required:/pow:. NIP-11-derived tags (N supported NIPs, k kinds, T type) are deliberately NOT emitted — those are relay self-claims, and publishing them under a monitor signature without per-NIP compliance tests would launder claims into measurements. Per-NIP/per-kind compliance suites can come later as opt-in checks; open/read/write is the default surface. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01A9sSyh1QLJD3PZ18tVPPVK --- .../accessories/NostrClientPublishExt.kt | 13 +- .../reachability/RelayProber.kt | 100 +++++++++-- .../reachability/RelayProberFlowTest.kt | 164 +++++++++++++++++- 3 files changed, 264 insertions(+), 13 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientPublishExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientPublishExt.kt index 9cbec380b4..112345d123 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientPublishExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientPublishExt.kt @@ -34,6 +34,7 @@ import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.channels.Channel.Factory.UNLIMITED import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.withTimeoutOrNull +import kotlin.time.TimeSource /** * One relay's verdict on a published event: [accepted] plus the reason the @@ -45,6 +46,12 @@ import kotlinx.coroutines.withTimeoutOrNull class PublishResult( val accepted: Boolean, val message: String, + /** + * Milliseconds from the publish to this relay's OK (true or false — a rejection + * is still a measured round trip), or -1 when the relay never answered with an + * OK. On an already-open socket this is an honest NIP-66 `rtt-write`. + */ + val elapsedMs: Long = -1, ) { /** * True when this failure came from the transport (never connected, @@ -102,6 +109,7 @@ suspend fun INostrClient.publishAndCollectResults( timeoutInSeconds: Long = 15, ): Map { val resultChannel = Channel(UNLIMITED) + val mark = TimeSource.Monotonic.markNow() Log.d("publishAndConfirm") { "Waiting for ${relayList.size} responses" } @@ -134,7 +142,7 @@ suspend fun INostrClient.publishAndCollectResults( when (msg) { is OkMessage -> { if (msg.eventId == event.id) { - resultChannel.trySend(DetailedResult(relay.url, msg.success, msg.message)) + resultChannel.trySend(DetailedResult(relay.url, msg.success, msg.message, mark.elapsedNow().inWholeMilliseconds)) Log.d("publishAndConfirm") { "onSendResponse Received response for ${msg.eventId} from relay ${relay.url} message ${msg.message} success ${msg.success}" } } } @@ -160,7 +168,7 @@ suspend fun INostrClient.publishAndCollectResults( val currentResult = receivedResults[result.relay] // do not override a successful result. if (currentResult == null || !currentResult.accepted) { - receivedResults[result.relay] = PublishResult(result.success, result.message) + receivedResults[result.relay] = PublishResult(result.success, result.message, result.elapsedMs) } } } @@ -191,4 +199,5 @@ private class DetailedResult( val relay: NormalizedRelayUrl, val success: Boolean, val message: String, + val elapsedMs: Long = -1, ) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt index 58663c2728..86e57c7e58 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt @@ -22,6 +22,8 @@ package com.vitorpamplona.quartz.nip66RelayMonitor.reachability import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.PublishResult +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.publishAndCollectResults import com.vitorpamplona.quartz.nip01Core.relay.client.listeners.RelayConnectionListener import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener import com.vitorpamplona.quartz.nip01Core.relay.client.single.IRelayClient @@ -30,6 +32,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer import com.vitorpamplona.quartz.nip01Core.signers.EventTemplate +import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner import com.vitorpamplona.quartz.nip01Core.store.IEventStore import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.RelayDiscoveryEvent @@ -83,6 +86,19 @@ class RelayProber( val error: String?, ) + /** + * One relay's read+write check outcome (see [readWriteCheck]). Latencies are + * -1 when unobserved; [writeAccepted] is null when the relay never answered + * the write with an OK (transport failure or silence). + */ + class ReadWriteVerdict( + val relay: NormalizedRelayUrl, + val rttReadMs: Long, + val rttWriteMs: Long, + val writeAccepted: Boolean?, + val writeMessage: String?, + ) + class Result( val verdicts: List, val elapsedMs: Long, @@ -153,6 +169,55 @@ class RelayProber( } } + /** + * The deeper, still-honest check pair: READ (a real limit-[readLimit] REQ the + * relay must query its store for) and WRITE (one ephemeral [RelayProbeWriteTest] + * event signed by [signer], the monitor key, timed to its OK). Everything is a + * direct observation — nothing is copied from the relay's NIP-11 self-claims. + * + * Run it against relays ALREADY PROVEN LIVE — typically [Result.reachable] of a + * [probe] that just ran, while the pool's sockets are still open. On a warm + * socket both numbers are honest NIP-66 rtts (`rtt-read`, `rtt-write`); against + * a cold relay they silently include the dial, so don't. + * + * A write REJECTION is still a measurement: `OK false` proves the write path + * works and documents policy ([ReadWriteVerdict.writeMessage] keeps the NIP-01 + * machine-readable reason; [toDiscoveryEventTemplate] maps `auth-required:` and + * `pow:` to `R` tags). Only silence leaves [ReadWriteVerdict.writeAccepted] null. + */ + suspend fun readWriteCheck( + relays: Collection, + signer: NostrSigner, + timeoutMs: Long = 15_000, + waveSize: Int = 1000, + readLimit: Int = 1, + ): Map { + val out = HashMap() + val distinct = relays.toSet() + for (wave in distinct.chunked(waveSize.coerceAtLeast(1))) { + val reads = HashMap() + probeWave(wave, timeoutMs, readTestFilter(readLimit)) { reads[it.relay] = it.rttEoseMs } + + val event = signer.sign(RelayProbeWriteTest.build()) + val writes = client.publishAndCollectResults(event, wave.toSet(), (timeoutMs / 1000).coerceAtLeast(1)) + + for (relay in wave) { + // Only a real OK (true or false) counts as an answer; transport + // failures and silence leave the write side unobserved. + val answered = writes[relay]?.takeUnless { it.isTransportFailure || it.message == PublishResult.NO_RESPONSE } + out[relay] = + ReadWriteVerdict( + relay = relay, + rttReadMs = reads[relay] ?: -1, + rttWriteMs = answered?.elapsedMs ?: -1, + writeAccepted = answered?.accepted, + writeMessage = answered?.message, + ) + } + } + return out + } + private suspend fun probeWave( wave: List, timeoutMs: Long, @@ -329,22 +394,37 @@ class RelayProber( * is the normalized relay url; sign it with the consumer's own monitor key (per * NIP-66 a monitor is its own identity, so the prober never signs on its own). * - * Only facts this probe actually observed are tagged: + * Only facts a probe actually observed are tagged: * - `n` network type inferred from the url (clearnet/tor/i2p); * - `rtt-open` when the relay was reachable — the measured WS-upgrade round trip, * or 0 for "reachable, latency not observed" (liveness is the tag's PRESENCE); - * - `R auth` when the relay answered the probe REQ with a NIP-42 `auth-required` - * CLOSED — an observed auth wall, not a NIP-11 claim. + * - `rtt-read`/`rtt-write` when a [RelayProber.readWriteCheck] result is passed + * as [readWrite] and actually measured that side; + * - `R auth` when the relay answered a probe with a NIP-42 `auth-required` + * (CLOSED on the REQ, or OK-false on the write) and `R pow` when the write + * was refused with a `pow:` reason — observed walls, not NIP-11 claims. * - * [Verdict.rttEoseMs] is deliberately NOT written as `rtt-read`: it is measured from - * the wave start, so it bundles dial, TLS and any handshake queueing with the read — - * publishing it as a read round trip would hand aggregators an inflated latency. - * The [RelayObserver]/[RelayMonitor] path supplies honest `rtt-read`/`rtt-write` - * from real traffic instead. + * NIP-11-derived tags (`N` supported NIPs, `k` kinds, `T` type) are deliberately + * absent: those are the relay's self-claims, and asserting them under a monitor + * signature without a per-NIP compliance test would launder claims into + * measurements. [Verdict.rttEoseMs] is likewise never written as `rtt-read` — it + * is wave-relative (dial + TLS + queueing), so the honest read number only comes + * from [readWrite] (or the [RelayObserver]/[RelayMonitor] real-traffic path). */ -fun RelayProber.Verdict.toDiscoveryEventTemplate(createdAt: Long = TimeUtils.now()): EventTemplate = +fun RelayProber.Verdict.toDiscoveryEventTemplate( + createdAt: Long = TimeUtils.now(), + readWrite: RelayProber.ReadWriteVerdict? = null, +): EventTemplate = RelayDiscoveryEvent.build(relay, createdAt = createdAt) { networkType(RelayReachabilityStore.networkTypeOf(relay)) if (reachable) rtt(RttType.OPEN, rttOpenMs.coerceAtLeast(0)) - if (error?.startsWith("closed:auth-required") == true) requirement("auth") + if (readWrite != null) { + if (readWrite.rttReadMs >= 0) rtt(RttType.READ, readWrite.rttReadMs) + if (readWrite.rttWriteMs >= 0) rtt(RttType.WRITE, readWrite.rttWriteMs) + } + val authWalled = + error?.startsWith("closed:auth-required") == true || + readWrite?.writeMessage?.startsWith("auth-required") == true + if (authWalled) requirement("auth") + if (readWrite?.writeMessage?.startsWith("pow:") == true) requirement("pow") } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt index 2efc801324..0195e52b53 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt @@ -20,13 +20,20 @@ */ package com.vitorpamplona.quartz.nip66RelayMonitor.reachability +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair import com.vitorpamplona.quartz.nip01Core.relay.client.EmptyNostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient +import com.vitorpamplona.quartz.nip01Core.relay.client.listeners.RelayConnectionListener import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener +import com.vitorpamplona.quartz.nip01Core.relay.client.single.IRelayClient +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.OkMessage +import com.vitorpamplona.quartz.nip01Core.relay.commands.toRelay.Command import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer import com.vitorpamplona.quartz.nip01Core.signers.EventTemplate +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.delay import kotlinx.coroutines.launch @@ -46,10 +53,12 @@ import kotlin.test.assertTrue */ @OptIn(ExperimentalCoroutinesApi::class) class RelayProberFlowTest { - /** Captures the probe subscription so the test can play the relays. */ + /** Captures the probe subscription and publish so the test can play the relays. */ private class ScriptedClient : INostrClient by EmptyNostrClient() { var listener: SubscriptionListener? = null var sentFilters: Map>? = null + var published: Event? = null + val connListeners = mutableListOf() override fun subscribe( subId: String, @@ -59,6 +68,49 @@ class RelayProberFlowTest { this.listener = listener this.sentFilters = filters } + + override fun publish( + event: Event, + relayList: Set, + ) { + published = event + } + + override fun addConnectionListener(listener: RelayConnectionListener) { + connListeners += listener + } + + override fun removeConnectionListener(listener: RelayConnectionListener) { + connListeners -= listener + } + + /** Plays a relay's OK answer for the published event to every armed listener. */ + fun answerOk( + relay: NormalizedRelayUrl, + success: Boolean, + message: String, + ) { + val ok = OkMessage(published!!.id, success, message) + connListeners.toList().forEach { it.onIncomingMessage(FakeRelayClient(relay), "", ok) } + } + } + + private class FakeRelayClient( + override val url: NormalizedRelayUrl, + ) : IRelayClient { + override fun connect() = Unit + + override fun needsToReconnect() = false + + override fun connectAndSyncFiltersIfDisconnected(ignoreRetryDelays: Boolean) = Unit + + override fun isConnected() = true + + override fun sendOrConnectAndSync(cmd: Command) = Unit + + override fun sendIfConnected(cmd: Command) = Unit + + override fun disconnect() = Unit } private val fast = RelayUrlNormalizer.normalize("wss://fast.example.com") @@ -190,6 +242,82 @@ class RelayProberFlowTest { assertTrue(listOf("expiration", "5060") in template.tags.map { it.toList() }) } + // ------------------------------------------------------------------ + // readWriteCheck — honest read + write measurements, nothing claimed + // ------------------------------------------------------------------ + + private suspend fun ScriptedClient.playReadThenWrite( + relay: NormalizedRelayUrl, + ok: Boolean?, + okMessage: String = "", + ) { + delay(50) + listener!!.onEose(relay, null) // read phase answers + while (published == null) delay(10) // write phase begins + if (ok != null) answerOk(relay, ok, okMessage) + } + + @Test + fun readWriteCheckMeasuresBothSides() = + runTest { + val client = ScriptedClient() + val signer = NostrSignerInternal(KeyPair()) + var result: Map? = null + + val check = + launch { + result = RelayProber(client).readWriteCheck(listOf(fast), signer, timeoutMs = 5_000) + } + launch { client.playReadThenWrite(fast, ok = true) } + check.join() + + val verdict = result!![fast]!! + assertTrue(verdict.rttReadMs >= 0, "an answered read must be measured") + assertTrue(verdict.rttWriteMs >= 0, "an answered write must be measured") + assertEquals(true, verdict.writeAccepted) + assertEquals(20166, client.published!!.kind, "the write test must use the ephemeral probe event") + } + + @Test + fun writeRejectionIsAnAnswerNotAFailure() = + runTest { + val client = ScriptedClient() + val signer = NostrSignerInternal(KeyPair()) + var result: Map? = null + + val check = + launch { + result = RelayProber(client).readWriteCheck(listOf(walled), signer, timeoutMs = 5_000) + } + launch { client.playReadThenWrite(walled, ok = false, okMessage = "pow: 28 bits needed") } + check.join() + + val verdict = result!![walled]!! + assertEquals(false, verdict.writeAccepted, "OK false is a measured policy answer") + assertEquals("pow: 28 bits needed", verdict.writeMessage) + assertTrue(verdict.rttWriteMs >= 0, "a rejection is still a round trip") + } + + @Test + fun silentWriteLeavesTheWriteSideUnobserved() = + runTest { + val client = ScriptedClient() + val signer = NostrSignerInternal(KeyPair()) + var result: Map? = null + + val check = + launch { + result = RelayProber(client).readWriteCheck(listOf(fast), signer, timeoutMs = 2_000) + } + launch { client.playReadThenWrite(fast, ok = null) } + check.join() + + val verdict = result!![fast]!! + assertTrue(verdict.rttReadMs >= 0) + assertNull(verdict.writeAccepted, "silence is not evidence about the write path") + assertEquals(-1, verdict.rttWriteMs) + } + // ------------------------------------------------------------------ // toDiscoveryEventTemplate — only observed facts become tags // ------------------------------------------------------------------ @@ -257,6 +385,40 @@ class RelayProberFlowTest { assertNull(tagsOf(template).firstOrNull { it[0] == "R" }) } + @Test + fun readWriteResultsBecomeRttTags() { + val verdict = RelayProber.Verdict(fast, reachable = true, rttOpenMs = 100, rttEoseMs = 300, error = null) + val readWrite = RelayProber.ReadWriteVerdict(fast, rttReadMs = 40, rttWriteMs = 55, writeAccepted = true, writeMessage = "") + + val tags = tagsOf(verdict.toDiscoveryEventTemplate(readWrite = readWrite)) + assertTrue(listOf("rtt-read", "40") in tags) + assertTrue(listOf("rtt-write", "55") in tags) + } + + @Test + fun unobservedReadWriteSidesStayUntagged() { + val verdict = RelayProber.Verdict(fast, reachable = true, rttOpenMs = 100, rttEoseMs = -1, error = null) + val readWrite = RelayProber.ReadWriteVerdict(fast, rttReadMs = -1, rttWriteMs = -1, writeAccepted = null, writeMessage = null) + + val tags = tagsOf(verdict.toDiscoveryEventTemplate(readWrite = readWrite)) + assertNull(tags.firstOrNull { it[0] == "rtt-read" }) + assertNull(tags.firstOrNull { it[0] == "rtt-write" }) + } + + @Test + fun writeRejectionReasonsBecomeRequirementTags() { + val verdict = RelayProber.Verdict(walled, reachable = true, rttOpenMs = 100, rttEoseMs = -1, error = null) + + val pow = RelayProber.ReadWriteVerdict(walled, -1, 30, writeAccepted = false, writeMessage = "pow: 28 bits needed") + assertTrue(listOf("R", "pow") in tagsOf(verdict.toDiscoveryEventTemplate(readWrite = pow))) + + val auth = RelayProber.ReadWriteVerdict(walled, -1, 30, writeAccepted = false, writeMessage = "auth-required: sign in") + assertTrue(listOf("R", "auth") in tagsOf(verdict.toDiscoveryEventTemplate(readWrite = auth))) + + val blocked = RelayProber.ReadWriteVerdict(walled, -1, 30, writeAccepted = false, writeMessage = "blocked: not welcome") + assertNull(tagsOf(verdict.toDiscoveryEventTemplate(readWrite = blocked)).firstOrNull { it[0] == "R" }) + } + @Test fun onionRelayIsTaggedTor() { val onion = RelayUrlNormalizer.normalize("ws://someonionaddressabcdefghijklmnop.onion") From 0d0117061a8c897b8bcfd50b8f837e757982d73d Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Tue, 4 Aug 2026 11:05:45 -0400 Subject: [PATCH 020/132] fix(relay): compare every filter in FiltersChanged, not just the first MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `needsToResendRequest(List, List)` used a non-local `return` inside `forEachIndexed`, so the loop always returned on iteration 0 and only `filters[0]` was ever compared. A subscription whose first filter happened to be unchanged reported "no resend needed" however much the rest had changed, leaving the relay serving a stale filter set and the app silently missing events. Only the size check offered any protection, so the bug was invisible whenever the filter count stayed constant. Replaces the loop with an indexed scan over all filters, which also drops the lambda allocation and matches the hot-path style in this package. Adds FiltersChangedTest. 3 of its 9 cases fail on the unfixed code — all of them changes beyond index 0 — while the other 6 pass both before and after, pinning the blast radius to exactly the buggy behaviour. Coverage includes the deliberate `since`-moves-forward exemption, which must not trigger a resend on any index. Note for reviewers: PoolRequests.kt:490 and :528 use this inverted as a "same as last" refusal check, so those become stricter — filter sets that differ only beyond index 0 were previously treated as identical and will now correctly be treated as changed. Co-Authored-By: Claude Opus 5 (1M context) --- .../relay/client/pool/FiltersChanged.kt | 12 ++- .../relay/client/pool/FiltersChangedTest.kt | 93 +++++++++++++++++++ 2 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/FiltersChangedTest.kt diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/FiltersChanged.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/FiltersChanged.kt index bd038c11f3..2d1638f48a 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/FiltersChanged.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/FiltersChanged.kt @@ -29,10 +29,14 @@ object FiltersChanged { ): Boolean { if (oldFilters.size != newFilters.size) return true - oldFilters.forEachIndexed { index, oldFilter -> - val newFilter = newFilters.getOrNull(index) ?: return true - - return needsToResendRequest(oldFilter, newFilter) + // Every filter must be compared. This used to be a forEachIndexed whose body + // `return`ed on the first iteration — a non-local return from this function — so + // only filters[0] was ever checked and a subscription whose first filter happened + // to be unchanged reported "no resend needed" however much the rest had changed, + // leaving the relay serving a stale filter set. + // Indexing is safe: the sizes were just proven equal. + for (i in oldFilters.indices) { + if (needsToResendRequest(oldFilters[i], newFilters[i])) return true } return false } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/FiltersChangedTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/FiltersChangedTest.kt new file mode 100644 index 0000000000..4d6af28c2b --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/FiltersChangedTest.kt @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay.client.pool + +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class FiltersChangedTest { + private fun authors(vararg a: String) = Filter(authors = a.toList()) + + @Test + fun emptyListsAreUnchanged() { + assertFalse(FiltersChanged.needsToResendRequest(emptyList(), emptyList())) + } + + @Test + fun differentSizesNeedResend() { + assertTrue(FiltersChanged.needsToResendRequest(listOf(authors("a")), listOf(authors("a"), authors("b")))) + } + + @Test + fun identicalSingleFilterDoesNotNeedResend() { + assertFalse(FiltersChanged.needsToResendRequest(listOf(authors("a")), listOf(authors("a")))) + } + + @Test + fun changedFirstFilterNeedsResend() { + assertTrue(FiltersChanged.needsToResendRequest(listOf(authors("a")), listOf(authors("b")))) + } + + /** + * Regression: the loop used a non-local `return` on the first iteration, so only + * filters[0] was ever compared. A subscription whose first filter was unchanged + * reported "no resend needed" no matter what happened to the rest, and silently + * went stale — the relay kept serving the old filter set. + */ + @Test + fun changedSecondFilterNeedsResend() { + val old = listOf(authors("a"), authors("b")) + val new = listOf(authors("a"), authors("CHANGED")) + assertTrue(FiltersChanged.needsToResendRequest(old, new)) + } + + @Test + fun changedLastOfManyNeedsResend() { + val old = listOf(authors("a"), authors("b"), authors("c"), authors("d")) + val new = listOf(authors("a"), authors("b"), authors("c"), authors("CHANGED")) + assertTrue(FiltersChanged.needsToResendRequest(old, new)) + } + + @Test + fun allIdenticalOfManyDoesNotNeedResend() { + val old = listOf(authors("a"), authors("b"), authors("c")) + val new = listOf(authors("a"), authors("b"), authors("c")) + assertFalse(FiltersChanged.needsToResendRequest(old, new)) + } + + /** `since` moving forward is deliberately NOT a resend trigger, on any index. */ + @Test + fun sinceMovingForwardOnLaterFilterDoesNotNeedResend() { + val old = listOf(Filter(authors = listOf("a"), since = 100), Filter(authors = listOf("b"), since = 100)) + val new = listOf(Filter(authors = listOf("a"), since = 100), Filter(authors = listOf("b"), since = 200)) + assertFalse(FiltersChanged.needsToResendRequest(old, new)) + } + + /** ...but moving backwards in time is, including on a later filter. */ + @Test + fun sinceMovingBackwardsOnLaterFilterNeedsResend() { + val old = listOf(Filter(authors = listOf("a"), since = 100), Filter(authors = listOf("b"), since = 200)) + val new = listOf(Filter(authors = listOf("a"), since = 100), Filter(authors = listOf("b"), since = 100)) + assertTrue(FiltersChanged.needsToResendRequest(old, new)) + } +} From c767298368a5b8495ca80a201218cbb9462eb39c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 15:14:17 +0000 Subject: [PATCH 021/132] =?UTF-8?q?fix(quartz):=20audit=20fixes=20?= =?UTF-8?q?=E2=80=94=20foreign-OK=20confirmation=20bug,=20normalizer=20hot?= =?UTF-8?q?-path=20allocation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audit findings across the branch, each verified with a failing test or measurement before the fix: - publishAndCollectResults counted an OK from a relay OUTSIDE relayList (same event id — a probe-wave straggler, or any republish of the same event to a different relay set) toward its confirmation window, ending the wait loop early and misreporting still-pending listed relays as NO_RESPONSE. The OK branch now carries the same relayList guard the onCannotConnect/onDisconnected branches always had. Regression test proves the failure without the guard. readWriteCheck additionally varies the probe event content per wave so wave N's confirmation window can never match wave N-1's event id at all. - RelayUrlNormalizer.fix() called trimEnd('%','2','0') unconditionally, allocating a full string copy for ANY url merely ending in '%', '2' or '0' — which includes every relay port ending in zero (wss://host:3030). Now gated on endsWith("%20"), keeping the hot path allocation-free; semantics unchanged (test pins both the trim and the untouched-port cases). - amy relay probe --file: unreadable file is now a clean bad_args error instead of a stack trace, and skipped onion urls are counted and reported (file_onion_skipped) instead of vanishing from the tally. - probeFlow KDoc now states that a slow collector eats into the current wave's absolute deadline (answers are still recorded; silent relays get less listening time), not just that it delays the next wave. Verified non-issues: androidx.collection LruCache is internally locked (safe for CachedNip11Fetcher/normalizer concurrency); probeWave's per-terminal emission cannot lose or double-emit verdicts (remaining-set guard, data maps read at emission time); existing publish callers all benefit from the OK guard rather than depending on the old behavior. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01A9sSyh1QLJD3PZ18tVPPVK --- .../amethyst/cli/commands/RelayCommands.kt | 12 +++++-- .../accessories/NostrClientPublishExt.kt | 7 +++- .../relay/normalizer/RelayUrlNormalizer.kt | 11 +++---- .../reachability/RelayProber.kt | 14 +++++--- .../nip01Core/relay/RelayUrlFormatterTest.kt | 8 +++++ .../reachability/RelayProberFlowTest.kt | 32 +++++++++++++++++++ 6 files changed, 70 insertions(+), 14 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/RelayCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/RelayCommands.kt index 8734ba1b02..292a19b6b0 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/RelayCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/RelayCommands.kt @@ -349,21 +349,26 @@ object RelayCommands { var fileRaw = 0 var fileRejected = 0 + var fileOnion = 0 val fileRelays = HashSet() if (fromFile != null) { - File(fromFile).forEachLine { line -> + val candidates = File(fromFile) + if (!candidates.canRead()) return Output.error("bad_args", "cannot read --file $fromFile") + candidates.forEachLine { line -> if (line.isBlank()) return@forEachLine fileRaw++ val normalized = line.normalizeRelayUrlOrNull() if (normalized == null) { fileRejected++ - } else if (!RelayUrlNormalizer.isOnion(normalized.url)) { + } else if (RelayUrlNormalizer.isOnion(normalized.url)) { + fileOnion++ + } else { fileRelays.add(normalized) } } System.err.println( "[relay-probe] $fromFile: $fileRaw urls → ${fileRelays.size} unique clearnet relays " + - "($fileRejected rejected by the normalizer)", + "($fileRejected rejected by the normalizer, $fileOnion onion skipped)", ) } @@ -412,6 +417,7 @@ object RelayCommands { "file_urls" to (if (fromFile != null) fileRaw else null), "file_normalized" to (if (fromFile != null) fileRelays.size else null), "file_rejected" to (if (fromFile != null) fileRejected else null), + "file_onion_skipped" to (if (fromFile != null) fileOnion else null), "reachable" to result.reachable.size, "dead" to result.dead.size, "closed_by_policy" to authWalled, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientPublishExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientPublishExt.kt index 112345d123..c1aa8c14a2 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientPublishExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientPublishExt.kt @@ -141,7 +141,12 @@ suspend fun INostrClient.publishAndCollectResults( when (msg) { is OkMessage -> { - if (msg.eventId == event.id) { + // The relayList guard matters, not just the id: the same event may + // have been published to OTHER relays by an earlier call (probe + // waves, republish), and counting their late OKs here would inflate + // receivedResults and end the wait loop before every listed relay + // answered — misreporting the missing ones as NO_RESPONSE. + if (msg.eventId == event.id && relay.url in relayList) { resultChannel.trySend(DetailedResult(relay.url, msg.success, msg.message, mark.elapsedNow().inWholeMilliseconds)) Log.d("publishAndConfirm") { "onSendResponse Received response for ${msg.eventId} from relay ${relay.url} message ${msg.message} success ${msg.success}" } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt index 8949a0b82c..bc5c524482 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt @@ -180,12 +180,11 @@ class RelayUrlNormalizer { if (rawUrl.length < 4) return null if (rawUrl.contains("%00")) return null - // Trim trailing %20 (percent-encoded spaces from malformed event data) - val url = - rawUrl.trimEnd('%', '2', '0').let { trimmed -> - // Only accept if we actually removed a trailing %20 pattern - if (trimmed.length < rawUrl.length && rawUrl.endsWith("%20")) trimmed else rawUrl - } + // Trim trailing %20 (percent-encoded spaces from malformed event data). + // The endsWith gate keeps the hot path allocation-free: trimEnd would + // copy the string for ANY url merely ending in '%', '2' or '0' — which + // includes every port ending in zero ("wss://host:3030"). + val url = if (rawUrl.endsWith("%20")) rawUrl.trimEnd('%', '2', '0') else rawUrl if (url.length < 4) return null // Reject URLs with %20 in the middle — these are garbage diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt index 86e57c7e58..5dcf878aad 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt @@ -152,8 +152,11 @@ class RelayProber( * [filters] picks the check, as in [probe]: [LIVENESS_FILTERS] (default) or * [readTestFilter]. * - * Probing starts when the flow is collected and pauses between waves while the - * collector is busy (emission is sequential). Pair each verdict with + * Probing starts when the flow is collected, and emission is sequential — a slow + * collector delays the next wave AND eats into the current wave's [timeoutMs] + * window (the deadline is absolute; answers keep being recorded while the + * collector runs, but silent relays get less listening time). Keep per-verdict + * work light, or buffer, when precise deadlines matter. Pair each verdict with * [toDiscoveryEventTemplate] to turn the stream into signable NIP-66 kind:30166 * records for another process to sign and publish. */ @@ -194,11 +197,14 @@ class RelayProber( ): Map { val out = HashMap() val distinct = relays.toSet() - for (wave in distinct.chunked(waveSize.coerceAtLeast(1))) { + for ((waveIndex, wave) in distinct.chunked(waveSize.coerceAtLeast(1)).withIndex()) { val reads = HashMap() probeWave(wave, timeoutMs, readTestFilter(readLimit)) { reads[it.relay] = it.rttEoseMs } - val event = signer.sign(RelayProbeWriteTest.build()) + // A distinct event id per wave (createdAt has second granularity, so the + // content must vary) keeps a straggler OK from an earlier wave's relays + // from ever matching this wave's confirmation window. + val event = signer.sign(RelayProbeWriteTest.build(content = "NIP-66 write probe $waveIndex")) val writes = client.publishAndCollectResults(event, wave.toSet(), (timeoutMs / 1000).coerceAtLeast(1)) for (relay in wave) { diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/RelayUrlFormatterTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/RelayUrlFormatterTest.kt index fc708eddba..6357c09591 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/RelayUrlFormatterTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/RelayUrlFormatterTest.kt @@ -53,6 +53,14 @@ class RelayUrlFormatterTest { assertNull(RelayUrlNormalizer.normalizeOrNull("wss://relay%20list%20to%20discover%20the%20user's%20content")) } + @Test + fun trailingPercentTwentyIsTrimmedButBareTrailingZeroIsNot() { + assertEquals("wss://nostr.mom/", RelayUrlNormalizer.normalizeOrNull("wss://nostr.mom%20")?.url) + // urls merely ending in '%', '2' or '0' (every port ending in zero) must pass untouched + assertEquals("wss://nostr.mom:3030/", RelayUrlNormalizer.normalizeOrNull("wss://nostr.mom:3030")?.url) + assertEquals("wss://nostr.mom:8020/", RelayUrlNormalizer.normalizeOrNull("wss://nostr.mom:8020")?.url) + } + @Test fun httpWithPathIsNotARelay() { // Mastodon/bridge actor urls from `proxy` tags: web resources, not relays diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt index 0195e52b53..153c6e5d3e 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt @@ -298,6 +298,38 @@ class RelayProberFlowTest { assertTrue(verdict.rttWriteMs >= 0, "a rejection is still a round trip") } + @Test + fun foreignRelayOkDoesNotEndTheWriteConfirmationEarly() = + runTest { + // A relay OUTSIDE the checked set answering with the same event id (a + // straggler from an earlier wave that got the same probe event) must not + // count toward the confirmation window — before the relayList guard in + // publishAndCollectResults, it ended the wait early and misreported the + // real relay as silent. + val client = ScriptedClient() + val signer = NostrSignerInternal(KeyPair()) + val foreign = RelayUrlNormalizer.normalize("wss://foreign.example.com") + var result: Map? = null + + val check = + launch { + result = RelayProber(client).readWriteCheck(listOf(fast), signer, timeoutMs = 5_000) + } + launch { + delay(50) + client.listener!!.onEose(fast, null) + while (client.published == null) delay(10) + client.answerOk(foreign, true, "") + delay(100) + client.answerOk(fast, true, "") + } + check.join() + + val verdict = result!![fast]!! + assertEquals(true, verdict.writeAccepted, "the listed relay's OK must still be awaited and recorded") + assertNull(result!![foreign], "the foreign relay must not appear in the result") + } + @Test fun silentWriteLeavesTheWriteSideUnobserved() = runTest { From 9a4f6c6cdd96500be8b35fe2e2490fe4e2562968 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 15:26:54 +0000 Subject: [PATCH 022/132] feat(quartz): optional kinds on the read-test filter (production finding) Verified the new probe surface end-to-end against production relays (probeFlow streaming, readWriteCheck, signed 30166 templates). One compatibility finding: purpose relays like purplepag.es reject any REQ that names no kind ('blocked: filters must specify at least one kind'), leaving their read side unobserved. readTestFilter/readWriteCheck now take an optional kinds list for those; the default stays kind-less because naming kinds also narrows the query on every other relay. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01A9sSyh1QLJD3PZ18tVPPVK --- .../nip66RelayMonitor/reachability/RelayProber.kt | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt index 5dcf878aad..d03fb01208 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt @@ -194,12 +194,13 @@ class RelayProber( timeoutMs: Long = 15_000, waveSize: Int = 1000, readLimit: Int = 1, + readKinds: List? = null, ): Map { val out = HashMap() val distinct = relays.toSet() for ((waveIndex, wave) in distinct.chunked(waveSize.coerceAtLeast(1)).withIndex()) { val reads = HashMap() - probeWave(wave, timeoutMs, readTestFilter(readLimit)) { reads[it.relay] = it.rttEoseMs } + probeWave(wave, timeoutMs, readTestFilter(readLimit, readKinds)) { reads[it.relay] = it.rttEoseMs } // A distinct event id per wave (createdAt has second granularity, so the // content must vary) keeps a straggler OK from an earlier wave's relays @@ -346,8 +347,17 @@ class RelayProber( * [filters] to turn [Verdict.rttEoseMs] into a genuine read test rather * than a liveness ping — the time still counts from the wave start (dial * included), so compare it against [Verdict.rttOpenMs], not across waves. + * + * [kinds] widens compatibility with purpose relays: some (purplepag.es) + * reject any REQ that names no kind with `blocked: filters must specify at + * least one kind`, which leaves their read side unobserved. Passing e.g. + * `listOf(0, 1)` satisfies them; the default stays kind-less because a + * kind list also narrows the query on every OTHER relay. */ - fun readTestFilter(limit: Int = 1) = listOf(Filter(limit = limit)) + fun readTestFilter( + limit: Int = 1, + kinds: List? = null, + ) = listOf(Filter(kinds = kinds, limit = limit)) /** * The relay universe the local store knows: every read/write relay advertised From 5ec35c77726c36cf9ee8581e7bc6e0f684ce4557 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 15:35:22 +0000 Subject: [PATCH 023/132] feat(quartz): default the read test to kind 0, limit 1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A kind-0, limit-1 REQ works everywhere: purpose relays (purplepag.es) reject kind-less filters outright, and practically every relay stores some profile. Verified against production — purplepag.es's read side now measures instead of going unobserved. Pass a different kinds list to probe a specific shelf, or null for a kind-less query on relays known to allow one. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01A9sSyh1QLJD3PZ18tVPPVK --- .../nip66RelayMonitor/reachability/RelayProber.kt | 14 +++++++------- .../reachability/RelayProberFlowTest.kt | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt index d03fb01208..48d33d9185 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt @@ -194,7 +194,7 @@ class RelayProber( timeoutMs: Long = 15_000, waveSize: Int = 1000, readLimit: Int = 1, - readKinds: List? = null, + readKinds: List? = listOf(0), ): Map { val out = HashMap() val distinct = relays.toSet() @@ -348,15 +348,15 @@ class RelayProber( * than a liveness ping — the time still counts from the wave start (dial * included), so compare it against [Verdict.rttOpenMs], not across waves. * - * [kinds] widens compatibility with purpose relays: some (purplepag.es) - * reject any REQ that names no kind with `blocked: filters must specify at - * least one kind`, which leaves their read side unobserved. Passing e.g. - * `listOf(0, 1)` satisfies them; the default stays kind-less because a - * kind list also narrows the query on every OTHER relay. + * [kinds] defaults to kind 0: purpose relays (purplepag.es) reject any REQ + * that names no kind with `blocked: filters must specify at least one kind`, + * and practically every relay stores SOME profile — so a kind-0, limit-1 + * query works everywhere. Pass a different list to probe a specific shelf, + * or null for a kind-less query on relays known to allow one. */ fun readTestFilter( limit: Int = 1, - kinds: List? = null, + kinds: List? = listOf(0), ) = listOf(Filter(kinds = kinds, limit = limit)) /** diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt index 153c6e5d3e..8593489ab2 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt @@ -227,6 +227,7 @@ class RelayProberFlowTest { delay(10) val sent = client.sentFilters!![fast]!!.single() assertEquals(1, sent.limit, "read test defaults to limit 1") + assertEquals(listOf(0), sent.kinds, "read test defaults to kind 0 — accepted by purpose relays too") assertNull(sent.ids, "read test must query real events, not the impossible id") client.listener!!.onEose(fast, null) } From a5d2d153c875904e5d40a6ff27a34d13df56eb4e Mon Sep 17 00:00:00 2001 From: vitorpamplona <532031+vitorpamplona@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:40:56 +0000 Subject: [PATCH 024/132] chore: sync Crowdin translations and seed translator npub placeholders --- amethyst/src/main/res/values-hi-rIN/strings.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/amethyst/src/main/res/values-hi-rIN/strings.xml b/amethyst/src/main/res/values-hi-rIN/strings.xml index 14c4d09d1d..59eea919c8 100644 --- a/amethyst/src/main/res/values-hi-rIN/strings.xml +++ b/amethyst/src/main/res/values-hi-rIN/strings.xml @@ -332,7 +332,7 @@ क्या समुदाय छोडें। क्या %1$s छोडें। इसे हटाया जाएगा इस लेखा की सूची से तथा आपके यन्त्रों पर समचरणीकरण रुक जाएगा। समुदाय को सूचित नहीं किया जाएगा। तथा आपको उसके सदस्य कार्यसूची से हटाया नहीं जाएगा। सन्देश जिनका आप अरहस्यीकरण नहीं कर सकेंगे सम्भाव्यतः पुनःप्राप्तव्य नहीं होंगे। तथा आप केवल नए आमन्त्रण के साथ लौट सकेंगे। आपने इस समुदाय को बनाया। छोड जाने से यह मिटेगा नहीं। किसी अन्य के हाथ सौंपा नहीं जाएगा। परन्तु स्वत्वधारी कुंचिका जो आपकी सूची में हैं वह हटाया जाएगा। आप आगे से इसका प्रबन्धन नहीं कर पाएँगे। - जहाँ इस समुदाय के रहस्यीकृत पत्रों का प्रकाशन तथा पठन किया जाता है। + जहाँ इस समुदाय के रहस्यीकृत समतलों का प्रकाशन तथा पठन किया जाता है। इस समुदाय को विघटित किया गया है तथा अब पठनेवशक्य है। आप इसका इतिहास पढ सकते हैं परन्तु कोई नए सन्देश नहीं भेज सकते। %1$s टंकण मध्य… %1$s तथा %2$s टंकण मध्य… @@ -1233,7 +1233,7 @@ मौन हटाएँ सम्भाव्यतः उन ग्राहकों द्वारा उपेक्षित जो आज्ञा का सम्मान नहीं करते। क्या शाला से निष्कासित करें। - %1$s को ध्वनि तल से हटाए जाएँगे तथा सहभागी सूची से भी। वे पुनः जुड सकते हैं यदि वे शाला योजक प्राप्त कर लें। + %1$s को ध्वनि समतल से हटाए जाएँगे तथा सहभागी सूची से भी। वे पुनः जुड सकते हैं यदि वे शाला योजक प्राप्त कर लें। पदप्रहार क्या वक्ता को मौन करें। %1$s के ग्राहक को अपना ध्वनिग्राहक मौन करने का अनुरोध करता है। कुछ ग्राहक इस आदेश की उपेक्षा कर सकते हैं। @@ -1991,7 +1991,7 @@ आपके आगतपेटिका पुनःप्रसारक तथा कुछ अल्पमात्रा परिभ्रमणवर्ती दृष्टान्त पुनःप्रसारक जिनपर आपके अनुचरित पत्र प्रकाशित करते हैं। यदि कोई उल्लेख अन्यत्र भेजा गया। आपके सीधासन्देश पुनःप्रसारक। जहाँ उपहारकोषयुक्त सन्देश भेजे जाते हैं। मुख्य पुनःप्रसारक प्रत्येक चर्चा का जिन्हें आप खोल रखे हैं अथवा जिनसे आप जुड चुके हैं। - पुनःप्रसारक जिनपर प्रत्येक समुदाय अपने पत्र प्रकाशित करते हैं। + पुनःप्रसारक जिनपर प्रत्येक समुदाय अपने समतलों को प्रकाशित करते हैं। समूह सन्देश तथा कुंचिकापेटलियाँ। प्रत्येक समूह के पुनःप्रसारकों पर। शाला के पुनःप्रसारक। जब वह खुला हो। आपके अपने परिचय तथा स्थापना विकल्प तथा पाण्डुलिपियाँ। आपके मुख्य पुनःप्रसारकों पर। From e56e2269df3fbb965a1040c633f8d28a73cc2db5 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 18:39:39 +0000 Subject: [PATCH 025/132] feat: offer Copy Original / Copy Translated when copying translated notes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every Copy Text menu (note quick-action popup, the shared note-action sections behind the 3-dot menu and chat long-press sheet, and bookmark group item options) now checks whether the rendered note was translated and, if so, pops a chooser offering Copy Original / Copy Translated instead of silently copying the original. Rather than plumbing the translated string from TranslatableRichTextViewer down to the menus, the shared copyNoteTextAction flow re-derives it from the process-wide TranslationsCache keyed by (content, language settings) — rendering the note is what populated that cache, so a hit means the user is looking at a translation. The cache is play-flavor-only, so the lookup goes through a new cachedTranslation() flavor pair (fdroid always null, keeping its Copy Text single-option). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01J7wEshKZC5HSgQykQiMQnt --- .../ui/components/CachedTranslation.kt | 32 +++++ .../amethyst/ui/note/CopyNoteText.kt | 133 ++++++++++++++++++ .../amethyst/ui/note/NoteQuickActionMenu.kt | 22 +-- .../ui/note/elements/NoteActionSections.kt | 16 ++- .../display/BookmarkGroupItemOptions.kt | 18 ++- amethyst/src/main/res/values/strings.xml | 2 + .../ui/components/CachedTranslation.kt | 45 ++++++ 7 files changed, 250 insertions(+), 18 deletions(-) create mode 100644 amethyst/src/fdroid/java/com/vitorpamplona/amethyst/ui/components/CachedTranslation.kt create mode 100644 amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/CopyNoteText.kt create mode 100644 amethyst/src/play/java/com/vitorpamplona/amethyst/ui/components/CachedTranslation.kt diff --git a/amethyst/src/fdroid/java/com/vitorpamplona/amethyst/ui/components/CachedTranslation.kt b/amethyst/src/fdroid/java/com/vitorpamplona/amethyst/ui/components/CachedTranslation.kt new file mode 100644 index 0000000000..1388e9a5bc --- /dev/null +++ b/amethyst/src/fdroid/java/com/vitorpamplona/amethyst/ui/components/CachedTranslation.kt @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.ui.components + +import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel + +/** + * No translation service in this flavor, so no note is ever translated and the copy-text + * menus never need to offer a "Copy Translated" option. + */ +fun cachedTranslation( + content: String, + accountViewModel: AccountViewModel, +): String? = null diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/CopyNoteText.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/CopyNoteText.kt new file mode 100644 index 0000000000..74e23ee3f6 --- /dev/null +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/CopyNoteText.kt @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.ui.note + +import androidx.compose.runtime.Composable +import androidx.compose.runtime.Immutable +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.ui.platform.LocalClipboard +import com.vitorpamplona.amethyst.R +import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols +import com.vitorpamplona.amethyst.model.Note +import com.vitorpamplona.amethyst.ui.components.M3ActionDialog +import com.vitorpamplona.amethyst.ui.components.M3ActionRow +import com.vitorpamplona.amethyst.ui.components.M3ActionSection +import com.vitorpamplona.amethyst.ui.components.cachedTranslation +import com.vitorpamplona.amethyst.ui.components.util.setText +import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel +import com.vitorpamplona.amethyst.ui.stringRes +import kotlinx.coroutines.launch + +/** Both texts of a translated note, held while the user picks which one to copy. */ +@Immutable +data class CopyTextChoice( + val original: String, + val translated: String, +) + +/** + * The "Copy Text" flow shared by every menu that copies an event's text. + * + * The copy menus sit far from the `TranslatableRichTextViewer` that rendered (and possibly + * translated) the note, so instead of plumbing the translated string down the hierarchy this + * flow re-derives it from [cachedTranslation]: the process-wide translation cache keyed by + * (content, language settings). By the time any copy menu is reachable the note has been + * rendered, which is what populated that cache — so a hit means the user is looking at a + * translation and gets a chooser (Copy Original / Copy Translated); a miss copies directly. + * + * Returns the click handler for the menu entry, taking the note whose text to copy (callers + * pass the latest version of a versioned post). [onCopied] runs after the text lands on the + * clipboard, [onDismiss] when the chooser is cancelled without copying; callers must keep + * their menu in composition until one of the two runs, because the chooser dialog is emitted + * from this composable. + */ +@Composable +fun copyNoteTextAction( + accountViewModel: AccountViewModel, + onCopied: () -> Unit, + onDismiss: () -> Unit, +): (Note) -> Unit { + val clipboardManager = LocalClipboard.current + val scope = rememberCoroutineScope() + val choice = remember { mutableStateOf(null) } + + val copy: (String) -> Unit = { text -> + scope.launch { + clipboardManager.setText(text) + onCopied() + } + } + + choice.value?.let { options -> + CopyTextChooserDialog( + onCopyOriginal = { + choice.value = null + copy(options.original) + }, + onCopyTranslated = { + choice.value = null + copy(options.translated) + }, + onDismiss = { + choice.value = null + onDismiss() + }, + ) + } + + return { note -> + accountViewModel.decrypt(note) { original -> + val translated = cachedTranslation(original, accountViewModel) + if (translated == null) { + copy(original) + } else { + choice.value = CopyTextChoice(original, translated) + } + } + } +} + +@Composable +fun CopyTextChooserDialog( + onCopyOriginal: () -> Unit, + onCopyTranslated: () -> Unit, + onDismiss: () -> Unit, +) { + M3ActionDialog( + title = stringRes(R.string.copy_text), + onDismiss = onDismiss, + ) { + M3ActionSection { + M3ActionRow( + icon = MaterialSymbols.ContentCopy, + text = stringRes(R.string.copy_text_original), + onClick = onCopyOriginal, + ) + M3ActionRow( + icon = MaterialSymbols.Translate, + text = stringRes(R.string.copy_text_translated), + onClick = onCopyTranslated, + ) + } + } +} diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/NoteQuickActionMenu.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/NoteQuickActionMenu.kt index a1161ab2b5..fc98c537af 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/NoteQuickActionMenu.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/NoteQuickActionMenu.kt @@ -299,20 +299,26 @@ fun CardBody( ) } + // When the rendered note was translated, tapping Copy Text opens a chooser + // (Copy Original / Copy Translated) on top of this popup; the popup stays up + // until the flow resolves so the chooser survives in composition. + val copyNoteText = + copyNoteTextAction( + accountViewModel = accountViewModel, + onCopied = { + showToast(R.string.copied_note_text_to_clipboard) + onDismiss() + }, + onDismiss = onDismiss, + ) + Column(modifier = Modifier.width(IntrinsicSize.Min)) { Row(modifier = Modifier.height(IntrinsicSize.Min)) { NoteQuickActionItem( icon = MaterialSymbols.ContentCopy, label = stringRes(R.string.quick_action_copy_text), ) { - accountViewModel.decrypt(note) { - scope.launch { - clipboardManager.setText(it) - showToast(R.string.copied_note_text_to_clipboard) - } - } - - onDismiss() + copyNoteText(note) } VerticalDivider(color = primaryLight) NoteQuickActionItem( diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/elements/NoteActionSections.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/elements/NoteActionSections.kt index df3f8c5572..9cb33f5778 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/elements/NoteActionSections.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/elements/NoteActionSections.kt @@ -35,6 +35,7 @@ import com.vitorpamplona.amethyst.ui.components.util.setText import com.vitorpamplona.amethyst.ui.navigation.navs.INav import com.vitorpamplona.amethyst.ui.navigation.routes.Route import com.vitorpamplona.amethyst.ui.note.QuickActionAlertDialogOneButton +import com.vitorpamplona.amethyst.ui.note.copyNoteTextAction import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel import com.vitorpamplona.amethyst.ui.stringRes import com.vitorpamplona.amethyst.ui.theme.LightRedColor @@ -129,14 +130,21 @@ fun noteActionSections( ) } + // When the rendered note was translated, Copy Text opens a chooser (Copy + // Original / Copy Translated) on top of the menu; the menu dismisses only + // after the flow resolves so the chooser survives in composition. + val copyNoteText = + copyNoteTextAction( + accountViewModel = accountViewModel, + onCopied = handlers.onDismiss, + onDismiss = handlers.onDismiss, + ) + val copyAndShare = buildList { add( NoteAction(MaterialSymbols.ContentCopy, stringRes(R.string.copy_text)) { - accountViewModel.decrypt(noteVersionToCopy) { - scope.launch { clipboardManager.setText(it) } - } - handlers.onDismiss() + copyNoteText(noteVersionToCopy) }, ) add( diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/bookmarkgroups/display/BookmarkGroupItemOptions.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/bookmarkgroups/display/BookmarkGroupItemOptions.kt index 35f9774204..229f21c690 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/bookmarkgroups/display/BookmarkGroupItemOptions.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/bookmarkgroups/display/BookmarkGroupItemOptions.kt @@ -45,6 +45,7 @@ import com.vitorpamplona.amethyst.ui.navigation.navs.INav import com.vitorpamplona.amethyst.ui.navigation.routes.Route import com.vitorpamplona.amethyst.ui.navigation.routes.routeEditDraftTo import com.vitorpamplona.amethyst.ui.note.VerticalDotsIcon +import com.vitorpamplona.amethyst.ui.note.copyNoteTextAction import com.vitorpamplona.amethyst.ui.note.elements.DropDownParams import com.vitorpamplona.amethyst.ui.note.elements.observeBookmarksFollowsAndAccount import com.vitorpamplona.amethyst.ui.note.externalLinkForNote @@ -186,16 +187,21 @@ fun BookmarkGroupItemOptionsMenu( } } + // When the rendered note was translated, Copy Text opens a chooser (Copy + // Original / Copy Translated) on top of the menu; the menu dismisses only + // after the flow resolves so the chooser survives in composition. + val copyNoteText = + copyNoteTextAction( + accountViewModel = accountViewModel, + onCopied = onDismiss, + onDismiss = onDismiss, + ) + // Copy & Share section M3ActionSection { M3ActionRow(icon = MaterialSymbols.ContentCopy, text = stringRes(R.string.copy_text)) { val lastNoteVersion = (editState?.value as? GenericLoadable.Loaded)?.loaded?.modificationToShow?.value ?: note - accountViewModel.decrypt(lastNoteVersion) { - scope.launch { - clipboardManager.setText(it) - } - } - onDismiss() + copyNoteText(lastNoteVersion) } M3ActionRow(icon = MaterialSymbols.ContentCopy, text = stringRes(R.string.copy_user_pubkey)) { note.author?.let { diff --git a/amethyst/src/main/res/values/strings.xml b/amethyst/src/main/res/values/strings.xml index 1843dbfdd4..bdc64269b3 100644 --- a/amethyst/src/main/res/values/strings.xml +++ b/amethyst/src/main/res/values/strings.xml @@ -46,6 +46,8 @@ Violence Unknown Author Copy Text + Copy Original + Copy Translated Copy Author ID Copy Note ID Copy raw JSON diff --git a/amethyst/src/play/java/com/vitorpamplona/amethyst/ui/components/CachedTranslation.kt b/amethyst/src/play/java/com/vitorpamplona/amethyst/ui/components/CachedTranslation.kt new file mode 100644 index 0000000000..cc7229491f --- /dev/null +++ b/amethyst/src/play/java/com/vitorpamplona/amethyst/ui/components/CachedTranslation.kt @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.ui.components + +import com.vitorpamplona.amethyst.service.lang.TranslationsCache +import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel + +/** + * The already-computed translation of [content] under the current language settings, or null + * when no translation occurred (same language, undetected source, blocklisted) or none is + * cached. Cache-only on purpose: this backs the "Copy Translated" option of the copy-text + * menus, which only applies to text the user is looking at — and rendering it through + * [TranslatableRichTextViewer] is what populated the cache. + */ +fun cachedTranslation( + content: String, + accountViewModel: AccountViewModel, +): String? { + val languages = accountViewModel.account.settings.syncedSettings.languages + val config = + TranslationsCache.get(content, languages.translateTo.value, languages.dontTranslateFrom.value) + ?: return null + val source = config.sourceLang ?: return null + val target = config.targetLang ?: return null + if (source == target || config.result == content) return null + return config.result +} From 4f6d6acefd2d340014d7b38be0a2a6274fd5cd04 Mon Sep 17 00:00:00 2001 From: sandwich <299465+dskvr@users.noreply.github.com> Date: Tue, 4 Aug 2026 19:59:22 +0100 Subject: [PATCH 026/132] Align napplet host with current NIP-5D and NAPs --- .../amethyst/favorites/FavoriteAppLauncher.kt | 26 +- .../napplet/DataStoreNappletStorage.kt | 27 +- .../amethyst/napplet/NappletBrokerService.kt | 77 +++-- .../napplet/NappletCapabilityLabels.kt | 2 - .../amethyst/napplet/NappletConsentSummary.kt | 5 +- .../amethyst/napplet/NappletIdentityWatch.kt | 17 +- .../amethyst/napplet/NappletLaunchRegistry.kt | 2 +- .../amethyst/napplet/NappletLauncher.kt | 107 +++++-- .../napplet/NappletLiveSubscriptions.kt | 54 +++- .../amethyst/napplet/NappletRelayCleartext.kt | 75 +++++ .../gateways/AccountNappletGateways.kt | 4 +- .../gateways/NappletResourceFetcher.kt | 302 ++++++++++++++---- .../loggedIn/napplets/NappletCapabilityExt.kt | 1 - .../napplet/NappletProtocolJsonTest.kt | 106 +++++- .../napplet/NappletRelayCleartextTest.kt | 108 +++++++ .../napplet/NappletSdkConformanceTest.kt | 53 +-- .../NappletResourceFetcherPolicyTest.kt | 70 ++++ .../composeResources/files/napplet/shell.html | 26 +- .../composeResources/files/napplet/shim.js | 77 +++-- .../commons/napplet/NappletArtifactPolicy.kt | 42 +++ .../amethyst/commons/napplet/NappletBroker.kt | 74 +++-- .../napplet/NappletBrokerCollaborators.kt | 16 +- .../commons/napplet/NappletCapability.kt | 34 +- .../commons/napplet/NappletIdentity.kt | 8 + .../commons/napplet/NappletWebContract.kt | 102 ++++-- .../napplet/protocol/NappletRequest.kt | 32 +- .../napplet/protocol/NappletResponse.kt | 27 +- .../napplet/NappletArtifactPolicyTest.kt | 54 ++++ .../commons/napplet/NappletBrokerTest.kt | 86 ++++- .../commons/napplet/NappletCapabilityTest.kt | 24 +- .../commons/napplet/NappletWebContractTest.kt | 79 +++++ .../commons/napplet/NappletRequestRouter.kt | 30 +- .../napplet/protocol/NappletProtocolJson.kt | 100 ++++-- .../napplet/NappletRequestRouterTest.kt | 43 +-- .../napplethost/NappletContentServer.kt | 76 +++-- .../napplethost/NappletHostActivity.kt | 18 +- .../napplethost/NappletHostService.kt | 26 +- 37 files changed, 1537 insertions(+), 473 deletions(-) create mode 100644 amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletRelayCleartext.kt create mode 100644 amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/NappletRelayCleartextTest.kt create mode 100644 amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/gateways/NappletResourceFetcherPolicyTest.kt create mode 100644 commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletArtifactPolicy.kt create mode 100644 commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletArtifactPolicyTest.kt create mode 100644 commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletWebContractTest.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/favorites/FavoriteAppLauncher.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/favorites/FavoriteAppLauncher.kt index bfdc4466ed..f46e2e7f24 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/favorites/FavoriteAppLauncher.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/favorites/FavoriteAppLauncher.kt @@ -165,27 +165,17 @@ object FavoriteAppLauncher { return when (event) { is RootNappletEvent -> NappletLauncher.buildLaunchParams( - context, - event.paths(), - event.servers(), - event.pubKey, - "", - event.declaredAggregateHash() ?: event.computeAggregateHash(), - event.title() ?: "Napplet", - event.requires(), - HostProfile.NAPPLET, + context = context, + manifest = event, + authorPubKey = event.pubKey, + identifier = "", ) is NamedNappletEvent -> NappletLauncher.buildLaunchParams( - context, - event.paths(), - event.servers(), - event.pubKey, - event.identifier(), - event.declaredAggregateHash() ?: event.computeAggregateHash(), - event.title() ?: event.identifier(), - event.requires(), - HostProfile.NAPPLET, + context = context, + manifest = event, + authorPubKey = event.pubKey, + identifier = event.identifier(), ) is RootSiteEvent -> NappletLauncher.buildLaunchParams( diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/DataStoreNappletStorage.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/DataStoreNappletStorage.kt index 893f02ec87..14d3499371 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/DataStoreNappletStorage.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/DataStoreNappletStorage.kt @@ -57,7 +57,27 @@ class DataStoreNappletStorage( key: String, value: String, ) { - dataStore.edit { it[keyOf(coordinate, key)] = value } + dataStore.edit { preferences -> + val prefix = prefixOf(coordinate) + val target = keyOf(coordinate, key) + val currentBytes = + preferences + .asMap() + .entries + .asSequence() + .filter { it.key.name.startsWith(prefix) } + .sumOf { (storedKey, storedValue) -> + storedKey.name + .removePrefix(prefix) + .encodeToByteArray() + .size + + ((storedValue as? String)?.encodeToByteArray()?.size ?: 0) + } + val replacedBytes = key.encodeToByteArray().size + (preferences[target]?.encodeToByteArray()?.size ?: 0) + val proposedBytes = currentBytes - replacedBytes + key.encodeToByteArray().size + value.encodeToByteArray().size + require(proposedBytes <= MAX_STORAGE_BYTES) { "Napplet storage quota exceeded." } + preferences[target] = value + } } override suspend fun remove( @@ -87,4 +107,9 @@ class DataStoreNappletStorage( coordinate: String, key: String, ) = stringPreferencesKey(prefixOf(coordinate) + key) + + companion object { + /** NAP-STORAGE's recommended per-napplet UTF-8 quota. */ + const val MAX_STORAGE_BYTES = 512 * 1024 + } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletBrokerService.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletBrokerService.kt index 37a124c67e..58453f479b 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletBrokerService.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletBrokerService.kt @@ -51,6 +51,7 @@ import com.vitorpamplona.amethyst.napplethost.NappletIpc import com.vitorpamplona.amethyst.ui.MainActivity import com.vitorpamplona.quartz.nip01Core.core.HexKey import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.CoroutineStart import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job import kotlinx.coroutines.SupervisorJob @@ -58,6 +59,7 @@ import kotlinx.coroutines.cancel import kotlinx.coroutines.delay import kotlinx.coroutines.flow.map import kotlinx.coroutines.launch +import java.util.concurrent.ConcurrentHashMap /** * The trust boundary's main-process endpoint. The untrusted `:napplet` process binds this @@ -93,7 +95,11 @@ class NappletBrokerService : Service() { // Live relay subscriptions, keyed by the applet's subId. The account comes per-open from the // requesting surface's launch token, so a surface's REQs always target the account it acts as. - private val liveSubscriptions = NappletLiveSubscriptions() + private val liveSubscriptions = NappletLiveSubscriptions(scope) + + // NAP-RESOURCE cancellation is keyed by the trusted launch token plus the caller's request id. + // Cancelling removes the job before it can emit a late terminal envelope to the sandbox. + private val resourceRequests = ConcurrentHashMap() // The app-wide inc pub/sub bus: routes inc.emit between live napplet sessions as inc.event pushes. private val incBus = NappletIncBus { replyTo, payload -> push(replyTo, payload) } @@ -117,7 +123,7 @@ class NappletBrokerService : Service() { override fun onDestroy() { liveSubscriptions.closeAll() - identityWatch.stop() + identityWatch.stopAll() // Every applet/browser surface has unbound, so the "session" the user granted for is over. // The ledger and the broker cache are now app-wide singletons that outlive this service, so // their in-memory session grants have to be dropped explicitly here — that keeps the lifetime @@ -280,38 +286,57 @@ class NappletBrokerService : Service() { // Resolve the launch token to the trusted identity + declared set. The sandbox never states // its own coordinate, so a compromised :napplet process can only ever act as the napplet it // was launched as (it holds only its own token). An unknown token = no session; refuse. - val session = NappletLaunchRegistry.resolve(data.getString(NappletIpc.KEY_LAUNCH_TOKEN)) + val launchToken = data.getString(NappletIpc.KEY_LAUNCH_TOKEN) + val session = NappletLaunchRegistry.resolve(launchToken) if (session == null) { reply(replyTo, requestId, NappletProtocolJson.encodeResponse(requestType, NappletResponse.Failed("Unknown napplet session."))) return true } val identity = session.identity val declared = session.declared + val resourceRequestKey = "$launchToken\u0000$requestId" + if (requestType == "resource.cancel") { + resourceRequests.remove(resourceRequestKey)?.cancel() + return true + } + val tracksResourceRequest = requestType == "resource.bytes" || requestType == "resource.bytesMany" - scope.launch { - // The shared, host-agnostic router owns decode → broker → encode and the subscribe-vs-reply - // decision (it stays wire-identical with the future desktop host). This service only supplies - // the broker, the Messenger transport, and the live relay subscription each Outcome implies. - // The launch token decides whose key signs — not the active account. A surface opened by - // one account can never be handed another's signer, even while it stays open across a switch. - val broker = brokerFor(session.accountPubKey) - if (broker == null) { - reply(replyTo, requestId, NappletProtocolJson.encodeResponse(requestType, NappletResponse.Failed("That account is no longer signed in."))) - return@launch - } - when (val outcome = NappletRequestRouter.route(broker, identity, declared, payload)) { - is NappletRequestRouter.Outcome.Ignore -> {} - is NappletRequestRouter.Outcome.Reply -> reply(replyTo, requestId, outcome.payload) - is NappletRequestRouter.Outcome.OpenSubscription -> - liveSubscriptions.open(outcome.subId, outcome.filters, accountFor(session.accountPubKey)) { push(replyTo, it) } - is NappletRequestRouter.Outcome.CloseSubscription -> liveSubscriptions.close(outcome.subId) - is NappletRequestRouter.Outcome.WatchIdentity -> identityWatch.start(session.accountPubKey) { push(replyTo, it) } - is NappletRequestRouter.Outcome.UnwatchIdentity -> identityWatch.stop() - is NappletRequestRouter.Outcome.Push -> outcome.payloads.forEach { push(replyTo, it) } - is NappletRequestRouter.Outcome.SubscribeInc -> incBus.subscribe(replyTo, outcome.topic) - is NappletRequestRouter.Outcome.UnsubscribeInc -> incBus.unsubscribe(replyTo, outcome.topic) - is NappletRequestRouter.Outcome.EmitInc -> incBus.emit(replyTo, identity.coordinate, outcome.topic, outcome.payloadRaw) + val requestJob = + scope.launch(start = if (tracksResourceRequest) CoroutineStart.LAZY else CoroutineStart.DEFAULT) { + // The shared, host-agnostic router owns decode → broker → encode and the subscribe-vs-reply + // decision (it stays wire-identical with the future desktop host). This service only supplies + // the broker, the Messenger transport, and the live relay subscription each Outcome implies. + // The launch token decides whose key signs — not the active account. A surface opened by + // one account can never be handed another's signer, even while it stays open across a switch. + val broker = brokerFor(session.accountPubKey) + if (broker == null) { + reply(replyTo, requestId, NappletProtocolJson.encodeResponse(requestType, NappletResponse.Failed("That account is no longer signed in."))) + return@launch + } + when (val outcome = NappletRequestRouter.route(broker, identity, declared, payload)) { + is NappletRequestRouter.Outcome.Ignore -> {} + is NappletRequestRouter.Outcome.Reply -> { + reply(replyTo, requestId, outcome.payload) + // NAP-IDENTITY has no watch/unwatch request. Once the consent-gated startup + // snapshot succeeds, the runtime owns identity.changed delivery for this + // trusted launch token until the broker service closes. + if (requestType == "identity.getPublicKey" && outcome.payload.contains("\"ok\":true") && launchToken != null) { + identityWatch.start(launchToken, session.accountPubKey) { push(replyTo, it) } + } + } + is NappletRequestRouter.Outcome.OpenSubscription -> + liveSubscriptions.open(outcome.subId, outcome.filters, accountFor(session.accountPubKey)) { push(replyTo, it) } + is NappletRequestRouter.Outcome.CloseSubscription -> liveSubscriptions.close(outcome.subId) + is NappletRequestRouter.Outcome.Push -> outcome.payloads.forEach { push(replyTo, it) } + is NappletRequestRouter.Outcome.SubscribeInc -> incBus.subscribe(replyTo, outcome.topic) + is NappletRequestRouter.Outcome.UnsubscribeInc -> incBus.unsubscribe(replyTo, outcome.topic) + is NappletRequestRouter.Outcome.EmitInc -> incBus.emit(replyTo, identity.coordinate, outcome.topic, outcome.payloadRaw) + } } + if (tracksResourceRequest) { + resourceRequests.put(resourceRequestKey, requestJob)?.cancel() + requestJob.invokeOnCompletion { resourceRequests.remove(resourceRequestKey, requestJob) } + requestJob.start() } return true } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletCapabilityLabels.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletCapabilityLabels.kt index adff3b46f6..15ab1abd71 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletCapabilityLabels.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletCapabilityLabels.kt @@ -28,7 +28,6 @@ import com.vitorpamplona.amethyst.commons.napplet.NappletCapability @StringRes fun NappletCapability.labelRes(): Int = when (this) { - NappletCapability.SHELL -> R.string.napplet_cap_shell NappletCapability.IDENTITY -> R.string.napplet_cap_identity NappletCapability.KEYS -> R.string.napplet_cap_keys NappletCapability.RELAY -> R.string.napplet_cap_relay @@ -45,7 +44,6 @@ fun NappletCapability.labelRes(): Int = @StringRes fun NappletCapability.descriptionRes(): Int = when (this) { - NappletCapability.SHELL -> R.string.napplet_cap_shell_desc NappletCapability.IDENTITY -> R.string.napplet_cap_identity_desc NappletCapability.KEYS -> R.string.napplet_cap_keys_desc NappletCapability.RELAY -> R.string.napplet_cap_relay_desc diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletConsentSummary.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletConsentSummary.kt index 42f4ec93f5..583cda40b4 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletConsentSummary.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletConsentSummary.kt @@ -294,9 +294,10 @@ class NappletConsentSummary( context.getString(R.string.napplet_consent_pay_no_amount) } } - is NappletRequest.ResourceBytes -> context.getString(R.string.napplet_consent_resource) + NappletRequest.ResourceInfo, is NappletRequest.ResourceBytes, is NappletRequest.ResourceBytesMany -> + context.getString(R.string.napplet_consent_resource) is NappletRequest.UploadBlob -> context.getString(R.string.napplet_consent_upload) // Resolved in the broker before consent (negotiation / shell-mediated / cosmetic); never shown. - is NappletRequest.ShellSupports, is NappletRequest.RegisterAction, is NappletRequest.UnregisterAction, is NappletRequest.ThemeGet -> "" + is NappletRequest.RegisterAction, is NappletRequest.UnregisterAction, is NappletRequest.ThemeGet -> "" } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletIdentityWatch.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletIdentityWatch.kt index f94f7bc2cf..bbb6a3aaa2 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletIdentityWatch.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletIdentityWatch.kt @@ -34,21 +34,22 @@ import kotlinx.coroutines.launch * value is dropped — the applet already has it via `getPublicKey`), encodes and pushes the new key * (or `""` when no account is signed in) to the caller-supplied sink. * - * One watch at a time per host binding; [start] replaces any prior one. Reached only after the - * router confirmed the applet declared the IDENTITY capability. + * Watches are keyed by the trusted launch token so concurrent surfaces cannot replace each other's + * streams. A watch starts only after that surface successfully obtains its public-key snapshot. */ class NappletIdentityWatch( private val scope: CoroutineScope, private val pubKey: (boundPubKey: String) -> Flow, ) { - private var job: Job? = null + private val jobs = mutableMapOf() fun start( + watchId: String, boundPubKey: String, push: (String) -> Unit, ) { - stop() - job = + if (jobs.containsKey(watchId)) return + jobs[watchId] = scope.launch { pubKey(boundPubKey) .distinctUntilChanged() @@ -57,8 +58,8 @@ class NappletIdentityWatch( } } - fun stop() { - job?.cancel() - job = null + fun stopAll() { + jobs.values.forEach { it.cancel() } + jobs.clear() } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLaunchRegistry.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLaunchRegistry.kt index b91aabad36..8f6741e564 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLaunchRegistry.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLaunchRegistry.kt @@ -76,7 +76,7 @@ object NappletLaunchRegistry { accountPubKey: HexKey, ): String { val token = ByteArray(32).also(secureRandom::nextBytes).toHexKey() - sessions[token] = Session(identity, declared, accountPubKey) + sessions[token] = Session(identity.copy(instanceId = token), declared, accountPubKey) return token } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLauncher.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLauncher.kt index 2c98ffb8fe..29c0a2f091 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLauncher.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLauncher.kt @@ -24,14 +24,18 @@ import android.content.Context import android.content.Intent import android.content.res.Configuration import android.os.Bundle +import android.util.Log import com.vitorpamplona.amethyst.Amethyst +import com.vitorpamplona.amethyst.commons.napplet.NappletArtifactPolicy import com.vitorpamplona.amethyst.commons.napplet.NappletIdentity import com.vitorpamplona.amethyst.model.LocalCache import com.vitorpamplona.amethyst.model.ThemeType import com.vitorpamplona.amethyst.napplethost.HostProfile import com.vitorpamplona.amethyst.napplethost.NappletHostActivity import com.vitorpamplona.amethyst.napplethost.NappletHostContract +import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.crypto.verify import com.vitorpamplona.quartz.nip5aStaticWebsites.tags.PathTag import com.vitorpamplona.quartz.nip5dNapplets.NappletManifest import com.vitorpamplona.quartz.nipB7Blossom.BlossomServersEvent @@ -49,20 +53,18 @@ object NappletLauncher { manifest: NappletManifest, authorPubKey: HexKey, identifier: String, - ) = launch( - context = context, - paths = manifest.paths(), - servers = manifest.servers(), - authorPubKey = authorPubKey, - identifier = identifier, - aggregateHash = manifest.declaredAggregateHash() ?: manifest.computeAggregateHash(), - title = manifest.title() ?: identifier.ifBlank { "Napplet" }, - requires = manifest.requires(), - ) + ) { + val event = manifest as? Event + if (event?.verify() != true || event.pubKey != authorPubKey) { + Log.w(TAG, "Refusing NIP-5D manifest that failed signature/author verification") + return + } + buildLaunchParams(context, manifest, authorPubKey, identifier)?.let { openHost(context, it) } + } /** - * Opens any NIP-5A static site (nsite or napplet). [requires] is empty for a plain nsite — - * the broker then refuses every capability, so the site renders as inert static content. + * Opens a NIP-5A website from its already-resolved path data. NIP-5D napplets use the verified + * manifest overload so raw callers cannot bypass signature/author validation. */ fun launch( context: Context, @@ -73,12 +75,26 @@ object NappletLauncher { aggregateHash: HexKey?, title: String, requires: List, - // nSites open as [HostProfile.WEBSITE]: a NIP-07 window.nostr provider + normal network. The - // broker then grants the IDENTITY + RELAY capabilities NIP-07 needs (consent-gated), regardless - // of the (empty) manifest `requires`. Napplets keep the default locked [HostProfile.NAPPLET]. - profile: HostProfile = HostProfile.NAPPLET, + // Raw path data is accepted only for the legacy NIP-5A website profile. NIP-5D callers must + // use the signature-checking manifest overload above. + profile: HostProfile, + ) { + if (profile != HostProfile.WEBSITE) { + Log.w(TAG, "Refusing raw NIP-5D launch without a verified manifest") + return + } + val params = + runCatching { buildLaunchParams(context, paths, servers, authorPubKey, identifier, aggregateHash, title, requires, profile) } + .onFailure { Log.w(TAG, "Refusing invalid ${profile.name.lowercase()} launch", it) } + .getOrNull() + ?: return + openHost(context, params) + } + + private fun openHost( + context: Context, + params: Bundle, ) { - val params = buildLaunchParams(context, paths, servers, authorPubKey, identifier, aggregateHash, title, requires, profile) val intent = Intent(context, NappletHostActivity::class.java).apply { putExtras(params) @@ -105,6 +121,29 @@ object NappletLauncher { requires: List, profile: HostProfile, ): Bundle { + require(profile == HostProfile.WEBSITE) { "NIP-5D launch parameters require a verified manifest." } + return buildLaunchParamsTrusted(context, paths, servers, authorPubKey, identifier, aggregateHash, title, requires, profile) + } + + private fun buildLaunchParamsTrusted( + context: Context, + paths: List, + servers: List, + authorPubKey: HexKey, + identifier: String, + aggregateHash: HexKey?, + title: String, + requires: List, + profile: HostProfile, + ): Bundle { + val effectiveAggregateHash = + if (profile == HostProfile.NAPPLET) { + requireNotNull(NappletArtifactPolicy.verifiedAggregateHash(paths, aggregateHash)) { + "NIP-5D requires one self-contained /index.html with a valid blob hash and matching aggregate." + } + } else { + aggregateHash + } val proxyPort = Amethyst.instance.torManager.activePortOrNull.value ?: -1 // Augment the manifest's servers with the author's published Blossom list (kind:10063), if @@ -118,7 +157,7 @@ object NappletLauncher { // Mint the launch token in the (trusted) main process: the broker resolves the sandbox's // requests back to THIS identity + declared set, regardless of anything the sandbox sends. - val identity = NappletIdentity(authorPubKey = authorPubKey, identifier = identifier, aggregateHash = aggregateHash) + val identity = NappletIdentity(authorPubKey = authorPubKey, identifier = identifier, aggregateHash = effectiveAggregateHash) val declared = profile.declaredCapabilities(requires) // Bound to the account launching it, so the surface keeps signing as that account even if the // user switches while it is open (an embedded surface is rebuilt on a switch and re-mints). @@ -156,7 +195,7 @@ object NappletLauncher { putStringArrayList(NappletHostContract.EXTRA_SERVERS, ArrayList(allServers)) putString(NappletHostContract.EXTRA_AUTHOR, authorPubKey) putString(NappletHostContract.EXTRA_IDENTIFIER, identifier) - putString(NappletHostContract.EXTRA_AGGREGATE_HASH, aggregateHash) + putString(NappletHostContract.EXTRA_AGGREGATE_HASH, effectiveAggregateHash) putString(NappletHostContract.EXTRA_TITLE, title) putStringArrayList(NappletHostContract.EXTRA_REQUIRES, ArrayList(requires)) putStringArrayList(NappletHostContract.EXTRA_CAP_LABELS, ArrayList(capLabels)) @@ -170,4 +209,34 @@ object NappletLauncher { putString(NappletHostContract.EXTRA_WEBVIEW_PROFILE, NappletWebViewProfiles.current()) } } + + /** Signature-checking entry point for embedded NIP-5D surfaces. */ + fun buildLaunchParams( + context: Context, + manifest: NappletManifest, + authorPubKey: HexKey, + identifier: String, + ): Bundle? { + val event = manifest as? Event + if (event?.verify() != true || event.pubKey != authorPubKey) { + Log.w(TAG, "Refusing embedded NIP-5D manifest that failed signature/author verification") + return null + } + return runCatching { + buildLaunchParamsTrusted( + context = context, + paths = manifest.paths(), + servers = manifest.servers(), + authorPubKey = authorPubKey, + identifier = identifier, + aggregateHash = manifest.declaredAggregateHash() ?: manifest.computeAggregateHash(), + title = manifest.title() ?: identifier.ifBlank { "Napplet" }, + requires = manifest.requires(), + profile = HostProfile.NAPPLET, + ) + }.onFailure { Log.w(TAG, "Refusing invalid embedded NIP-5D launch", it) } + .getOrNull() + } + + private const val TAG = "NappletLauncher" } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLiveSubscriptions.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLiveSubscriptions.kt index 8429a0b721..12d3500cd1 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLiveSubscriptions.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLiveSubscriptions.kt @@ -27,6 +27,10 @@ import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Job +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.launch import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicInteger @@ -44,7 +48,9 @@ import java.util.concurrent.atomic.AtomicInteger * signatures still came from the old one. [open] is reached only after the broker authorized the * subscription (RELAY consent). */ -class NappletLiveSubscriptions { +class NappletLiveSubscriptions( + private val scope: CoroutineScope, +) { private val liveSubs = ConcurrentHashMap() private val liveSeq = AtomicInteger(0) @@ -53,6 +59,20 @@ class NappletLiveSubscriptions { val client: INostrClient, ) { val eoseSent = AtomicBoolean(false) + val deliveries = Channel(Channel.UNLIMITED) + var deliveryJob: Job? = null + } + + private sealed interface Delivery { + data class RelayEvent( + val event: Event, + ) : Delivery + + data object Eose : Delivery + + data class Closed( + val reason: String, + ) : Delivery } /** @@ -77,6 +97,20 @@ class NappletLiveSubscriptions { // can't collide with the subscription it's replacing. val sub = LiveSub("napplet-$nappletSubId-${liveSeq.incrementAndGet()}", account.client) liveSubs[nappletSubId] = sub + sub.deliveryJob = + scope.launch { + for (delivery in sub.deliveries) { + if (liveSubs[nappletSubId] !== sub) break + when (delivery) { + is Delivery.RelayEvent -> + NappletRelayCleartext.forDelivery(delivery.event, account.signer)?.let { + push(NappletProtocolJson.encodeRelayEvent(nappletSubId, it)) + } + Delivery.Eose -> push(NappletProtocolJson.encodeRelayEose(nappletSubId)) + is Delivery.Closed -> push(NappletProtocolJson.encodeRelayClosed(nappletSubId, delivery.reason)) + } + } + } val listener = object : SubscriptionListener { @@ -85,7 +119,9 @@ class NappletLiveSubscriptions { isLive: Boolean, relay: NormalizedRelayUrl, forFilters: List?, - ) = push(NappletProtocolJson.encodeRelayEvent(nappletSubId, event)) + ) { + sub.deliveries.trySend(Delivery.RelayEvent(event)) + } // A subscription fans out to several relays; collapse their EOSEs into the single // relay.eose the SDK expects (fired when the first relay finishes its stored events). @@ -93,14 +129,16 @@ class NappletLiveSubscriptions { relay: NormalizedRelayUrl, forFilters: List?, ) { - if (sub.eoseSent.compareAndSet(false, true)) push(NappletProtocolJson.encodeRelayEose(nappletSubId)) + if (sub.eoseSent.compareAndSet(false, true)) sub.deliveries.trySend(Delivery.Eose) } override fun onClosed( message: String, relay: NormalizedRelayUrl, forFilters: List?, - ) = push(NappletProtocolJson.encodeRelayClosed(nappletSubId, message)) + ) { + sub.deliveries.trySend(Delivery.Closed(message)) + } } runCatching { sub.client.subscribe(sub.clientSubId, relays.associateWith { filters }, listener) } @@ -109,12 +147,18 @@ class NappletLiveSubscriptions { /** Stops the live subscription for [nappletSubId], unsubscribing from the client that opened it. */ fun close(nappletSubId: String) { val sub = liveSubs.remove(nappletSubId) ?: return + sub.deliveries.close() + sub.deliveryJob?.cancel() runCatching { sub.client.unsubscribe(sub.clientSubId) } } /** Tears down every open subscription (service teardown). */ fun closeAll() { - liveSubs.values.forEach { sub -> runCatching { sub.client.unsubscribe(sub.clientSubId) } } + liveSubs.values.forEach { sub -> + sub.deliveries.close() + sub.deliveryJob?.cancel() + runCatching { sub.client.unsubscribe(sub.clientSubId) } + } liveSubs.clear() } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletRelayCleartext.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletRelayCleartext.kt new file mode 100644 index 0000000000..3b55dad5cd --- /dev/null +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletRelayCleartext.kt @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.napplet + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner +import com.vitorpamplona.quartz.nip04Dm.crypto.EncryptedInfo +import com.vitorpamplona.quartz.nip04Dm.messages.PrivateDmEvent +import com.vitorpamplona.quartz.nip44Encryption.Nip44v2 + +/** NAP-RELAY read boundary: encrypted event content is decrypted or withheld, never exposed. */ +internal object NappletRelayCleartext { + suspend fun forDelivery( + event: Event, + signer: NostrSigner, + ): Event? = forDelivery(event, signer.pubKey, signer::decrypt) + + internal suspend fun forDelivery( + event: Event, + userPubKey: HexKey, + decrypt: suspend (String, HexKey) -> String, + ): Event? { + if (!isEncrypted(event)) return event + + val peer = + when { + event.pubKey == userPubKey -> event.recipientPubKey() + event.isAddressedTo(userPubKey) -> event.pubKey + else -> null + } ?: return null + val cleartext = runCatching { decrypt(event.content, peer) }.getOrNull() ?: return null + + // NAP-RELAY defines a decrypted read projection. Retain the relay event's identity and + // signature fields so callers can still correlate it, while making clear that this object + // must never be republished as a signed event after its content projection has changed. + return Event(event.id, event.pubKey, event.createdAt, event.kind, event.tags, cleartext, event.sig) + } + + internal fun isEncrypted(event: Event): Boolean = + event is PrivateDmEvent || + EncryptedInfo.isNIP04(event.content) || + isNip44V2(event.content) + + private fun isNip44V2(content: String): Boolean = + content.length >= MIN_NIP44_V2_LENGTH && + runCatching { Nip44v2.EncryptedInfo.decodePayload(content) }.isSuccess + + private fun Event.recipientPubKey(): HexKey? = + tags.firstNotNullOfOrNull { tag -> + tag.getOrNull(1)?.takeIf { tag.getOrNull(0) == "p" } + } + + private fun Event.isAddressedTo(pubKey: HexKey): Boolean = tags.any { tag -> tag.getOrNull(0) == "p" && tag.getOrNull(1) == pubKey } + + private const val MIN_NIP44_V2_LENGTH = 132 +} diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/gateways/AccountNappletGateways.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/gateways/AccountNappletGateways.kt index 139a935d5f..3222e810c8 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/gateways/AccountNappletGateways.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/gateways/AccountNappletGateways.kt @@ -50,6 +50,7 @@ import com.vitorpamplona.amethyst.model.Account import com.vitorpamplona.amethyst.napplet.NappletConsentCoordinator import com.vitorpamplona.amethyst.napplet.NappletConsentSummary import com.vitorpamplona.amethyst.napplet.NappletNotificationStore +import com.vitorpamplona.amethyst.napplet.NappletRelayCleartext import com.vitorpamplona.amethyst.napplet.buildConnectInfo import com.vitorpamplona.amethyst.napplet.buildSignerConsentInfo import com.vitorpamplona.amethyst.service.uploads.blossom.BlossomUploader @@ -265,7 +266,8 @@ class AccountNappletGateways( .distinctBy { it.id } .sortedByDescending { it.createdAt } val limit = filters.mapNotNull { it.limit }.maxOrNull() - return limit?.let { merged.take(it) } ?: merged + val limited = limit?.let { merged.take(it) } ?: merged + return limited.mapNotNull { NappletRelayCleartext.forDelivery(it, account.signer) } } /** diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/gateways/NappletResourceFetcher.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/gateways/NappletResourceFetcher.kt index c64fb12eca..9acdf7b690 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/gateways/NappletResourceFetcher.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/gateways/NappletResourceFetcher.kt @@ -22,6 +22,7 @@ package com.vitorpamplona.amethyst.napplet.gateways import android.util.Base64 import com.vitorpamplona.amethyst.commons.napplet.NappletResource +import com.vitorpamplona.amethyst.commons.napplet.NappletResourceResult import com.vitorpamplona.amethyst.model.Account import com.vitorpamplona.amethyst.napplet.NappletNetworkRegistry import com.vitorpamplona.quartz.nip01Core.core.Address @@ -39,9 +40,21 @@ import com.vitorpamplona.quartz.nip5aStaticWebsites.resolver.StaticSiteResolver import com.vitorpamplona.quartz.nip5aStaticWebsites.resolver.sniffContentType import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext +import kotlinx.serialization.json.Json +import okhttp3.Authenticator +import okhttp3.CookieJar +import okhttp3.Dns +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import okhttp3.OkHttpClient import okhttp3.Request +import java.io.ByteArrayOutputStream +import java.io.InterruptedIOException +import java.net.InetAddress import java.net.URLDecoder +import java.nio.ByteBuffer +import java.nio.charset.CodingErrorAction +import java.util.concurrent.TimeUnit /** * Fetches a resource URL on an applet's behalf — the applet has no direct network @@ -63,41 +76,100 @@ class NappletResourceFetcher( private val account: Account, private val httpClient: (useProxy: Boolean) -> OkHttpClient, ) { - /** Fetches an https/data/blossom resource for the applet at [coordinate], or null if unsupported/unavailable. */ + /** Fetches an https/data/blossom/nostr resource and preserves the NAP-RESOURCE error category. */ suspend fun fetch( url: String, coordinate: String, - ): NappletResource? = + ): NappletResourceResult = withContext(Dispatchers.IO) { - // Route like the applet's own page: Tor when its network mode is Tor, clearnet otherwise. - NappletNetworkRegistry.awaitReady() - val client = httpClient(NappletNetworkRegistry.useTor(coordinate)) when { url.startsWith("data:") -> decodeDataUrl(url) - url.startsWith("https://") -> { - runCatching { - client - .newCall( - Request - .Builder() - .url(url) - .get() - .build(), - ).execute() - .use { r -> - if (!r.isSuccessful) return@withContext null - val body = r.body.bytes() - val type = r.header("Content-Type") ?: "application/octet-stream" - NappletResource(body, type) - } - }.getOrNull() + url.startsWith("nostr:") -> + resolveNostr(url)?.let(::success) ?: failure(ERROR_NOT_FOUND, "Nostr resource not found.") + url.startsWith("https://") || url.startsWith("blossom:") -> { + // Route like the applet's own page: locked napplets stay on Tor. The derived + // client removes ambient cookies/auth and validates DNS before every hop. + NappletNetworkRegistry.awaitReady() + val client = hardenedClient(httpClient(NappletNetworkRegistry.useTor(coordinate))) + if (url.startsWith("https://")) fetchHttps(url, client) else fetchBlossom(url, client) } - url.startsWith("blossom:") -> fetchBlossom(url, client) - url.startsWith("nostr:") -> resolveNostr(url) - else -> null + else -> failure(ERROR_UNSUPPORTED_SCHEME, "Unsupported resource URL scheme.") } } + private fun hardenedClient(baseClient: OkHttpClient): OkHttpClient = + baseClient + .newBuilder() + .followRedirects(false) + .followSslRedirects(false) + .cache(null) + .cookieJar(CookieJar.NO_COOKIES) + .authenticator(Authenticator.NONE) + .proxyAuthenticator(Authenticator.NONE) + .callTimeout(FETCH_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .dns( + Dns { hostname -> + baseClient.dns.lookup(hostname).also { addresses -> + if (addresses.isEmpty() || !addresses.all(::isPublicAddress)) { + throw BlockedResourceException("Resolved address is not public.") + } + } + }, + ).addNetworkInterceptor { chain -> + chain.proceed( + chain + .request() + .newBuilder() + .removeHeader("Authorization") + .removeHeader("Cookie") + .removeHeader("Proxy-Authorization") + .build(), + ) + }.build() + + private fun fetchHttps( + url: String, + client: OkHttpClient, + ): NappletResourceResult { + var current = safeHttpsUrl(url) ?: return failure(ERROR_BLOCKED, "Only credential-free HTTPS URLs are allowed.") + repeat(MAX_REDIRECTS + 1) { hop -> + try { + client + .newCall( + Request + .Builder() + .url(current) + .get() + .build(), + ).execute() + .use { response -> + if (response.isRedirect) { + if (hop >= MAX_REDIRECTS) return failure(ERROR_BLOCKED, "Redirect limit exceeded.") + val location = response.header("Location") ?: return failure(ERROR_NETWORK, "Redirect has no location.") + current = safeHttpsUrl(current.resolve(location)) ?: return failure(ERROR_BLOCKED, "Redirect left credential-free HTTPS.") + return@repeat + } + if (response.code == 404) return failure(ERROR_NOT_FOUND) + if (!response.isSuccessful) return failure(ERROR_NETWORK, "Upstream returned HTTP ${response.code}.") + if (response.body.contentLength() > MAX_RESOURCE_BYTES) return failure(ERROR_TOO_LARGE) + val body = readBounded(response.body.byteStream()) ?: return failure(ERROR_TOO_LARGE) + return classify(body) + } + } catch (e: BlockedResourceException) { + return failure(ERROR_BLOCKED, e.message) + } catch (_: InterruptedIOException) { + return failure(ERROR_TIMEOUT) + } catch (_: Exception) { + return failure(ERROR_NETWORK) + } + } + return failure(ERROR_BLOCKED, "Redirect limit exceeded.") + } + + private fun safeHttpsUrl(url: String): HttpUrl? = url.toHttpUrlOrNull()?.takeIf { isSafeHttpsResourceUrl(url) } + + private fun safeHttpsUrl(url: HttpUrl?): HttpUrl? = url?.takeIf { it.scheme == "https" && it.username.isEmpty() && it.password.isEmpty() } + /** * Resolves a `nostr:` URI (NIP-19) to the referenced event and returns its JSON. An `nembed` * carries the event inline; `note`/`nevent`/`naddr` resolve from the local cache, falling back to @@ -158,62 +230,176 @@ class NappletResourceFetcher( private fun fetchBlossom( url: String, client: OkHttpClient, - ): NappletResource? { - val hash = - url - .removePrefix("blossom://") - .removePrefix("blossom:") - .substringBefore('/') - .substringBefore('?') - .trim() - .lowercase() - if (!hash.matches(Regex("^[0-9a-f]{64}$"))) return null + ): NappletResourceResult { + if (!url.startsWith(BLOSSOM_SHA256_PREFIX)) return failure(ERROR_INVALID_REQUEST, "Malformed Blossom SHA-256 URL.") + val hash = url.removePrefix(BLOSSOM_SHA256_PREFIX).lowercase() + if (!hash.matches(SHA256)) return failure(ERROR_INVALID_REQUEST, "Malformed Blossom SHA-256 URL.") val servers = account.blossomServers .getBlossomServersList() ?.servers() .orEmpty() + var sawHashMismatch = false for (candidate in StaticSiteResolver.candidateUrls(servers, hash)) { - val bytes = - runCatching { - client - .newCall( - Request - .Builder() - .url(candidate) - .get() - .build(), - ).execute() - .use { r -> - if (r.isSuccessful) r.body.bytes() else null - } - }.getOrNull() ?: continue - if (StaticSiteResolver.verify(bytes, hash)) { - return NappletResource(bytes, sniffContentType(bytes) ?: "application/octet-stream") + when (val fetched = fetchHttps(candidate, client)) { + is NappletResourceResult.Success -> { + if (!StaticSiteResolver.verify(fetched.resource.bytes, hash)) { + sawHashMismatch = true + continue + } + return fetched + } + is NappletResourceResult.Failure -> if (fetched.error == ERROR_BLOCKED) return fetched } } - return null + if (sawHashMismatch) return failure(ERROR_DECODE_FAILED, "Blossom SHA-256 verification failed.") + return failure(ERROR_NOT_FOUND, "No Blossom server returned the verified blob.") } /** Parses a `data:[][;base64],` URL into bytes + content type. */ - private fun decodeDataUrl(url: String): NappletResource? { + private fun decodeDataUrl(url: String): NappletResourceResult { val comma = url.indexOf(',') - if (comma < 0) return null + if (comma < 0) return failure(ERROR_INVALID_REQUEST, "Malformed data URL.") val meta = url.substring("data:".length, comma) val data = url.substring(comma + 1) + if (data.length > MAX_DATA_URL_CHARS) return failure(ERROR_TOO_LARGE) val isBase64 = meta.endsWith(";base64") - val contentType = meta.removeSuffix(";base64").ifEmpty { "text/plain" } + val declaredType = + meta + .removeSuffix(";base64") + .substringBefore(';') + .ifEmpty { "text/plain" } + .lowercase() val bytes = if (isBase64) { - runCatching { Base64.decode(data, Base64.DEFAULT) }.getOrNull() ?: return null + runCatching { Base64.decode(data, Base64.DEFAULT) }.getOrNull() + ?: return failure(ERROR_DECODE_FAILED, "Invalid base64 data URL.") } else { - URLDecoder.decode(data, "UTF-8").encodeToByteArray() + runCatching { URLDecoder.decode(data, "UTF-8").encodeToByteArray() }.getOrNull() + ?: return failure(ERROR_DECODE_FAILED, "Invalid escaped data URL.") } - return NappletResource(bytes, contentType) + if (bytes.size > MAX_RESOURCE_BYTES) return failure(ERROR_TOO_LARGE) + return classify(bytes, declaredType) + } + + private fun classify( + bytes: ByteArray, + declaredType: String? = null, + ): NappletResourceResult { + if (looksLikeSvg(bytes)) return failure(ERROR_BLOCKED, "Raw SVG is not delivered by this runtime.") + val sniffed = sniffContentType(bytes) + val type = + when { + sniffed in ALLOWED_SNIFFED_TYPES -> sniffed + declaredType == "application/json" && isJson(bytes) -> "application/json" + declaredType == "text/plain" && isPlainText(bytes) -> "text/plain" + else -> null + } ?: return failure(ERROR_DECODE_FAILED, "Resource MIME is not in the runtime allowlist.") + return success(NappletResource(bytes, type)) + } + + private fun looksLikeSvg(bytes: ByteArray): Boolean { + val prefix = bytes.copyOfRange(0, minOf(bytes.size, MIME_PREFIX_BYTES)).decodeToString().lowercase() + return prefix.contains(" + val output = ByteArrayOutputStream() + val buffer = ByteArray(8 * 1024) + var total = 0 + while (true) { + val read = source.read(buffer) + if (read < 0) break + total += read + if (total > MAX_RESOURCE_BYTES) return null + output.write(buffer, 0, read) + } + return output.toByteArray() + } } companion object { + internal fun isSafeHttpsResourceUrl(url: String): Boolean = url.toHttpUrlOrNull()?.let { it.scheme == "https" && it.username.isEmpty() && it.password.isEmpty() } == true + + internal fun isPublicAddress(address: InetAddress): Boolean { + if (address.isAnyLocalAddress || address.isLoopbackAddress || address.isLinkLocalAddress || address.isSiteLocalAddress || address.isMulticastAddress) { + return false + } + val bytes = address.address + if (bytes.size == 4) { + val first = bytes[0].toInt() and 0xff + val second = bytes[1].toInt() and 0xff + // Shared address space (100.64/10) and reserved/non-routed ranges Java does not classify. + if (first == 0 || first >= 224) return false + if (first == 100 && second in 64..127) return false + if (first == 192 && second == 0) return false + if (first == 198 && second in 18..19) return false + if (first == 198 && second == 51 && (bytes[2].toInt() and 0xff) == 100) return false + if (first == 203 && second == 0 && (bytes[2].toInt() and 0xff) == 113) return false + } else if (bytes.size == 16) { + val first = bytes[0].toInt() and 0xff + if (first and 0xfe == 0xfc) return false // fc00::/7 unique-local + if ( + first == 0x20 && + (bytes[1].toInt() and 0xff) == 0x01 && + (bytes[2].toInt() and 0xff) == 0x0d && + (bytes[3].toInt() and 0xff) == 0xb8 + ) { + return false // 2001:db8::/32 documentation range + } + } + return true + } + private const val NOSTR_FETCH_TIMEOUT_MS = 8_000L + private const val FETCH_TIMEOUT_SECONDS = 30L + private const val MAX_REDIRECTS = 5 + private const val MIME_PREFIX_BYTES = 8 * 1024 + private const val MAX_DATA_URL_CHARS = 24 * 1024 * 1024 + private const val BLOSSOM_SHA256_PREFIX = "blossom:sha256:" + const val MAX_RESOURCE_BYTES = 10 * 1024 * 1024 + private const val ERROR_INVALID_REQUEST = "invalid-request" + private const val ERROR_NOT_FOUND = "not-found" + private const val ERROR_BLOCKED = "blocked-by-policy" + private const val ERROR_TIMEOUT = "timeout" + private const val ERROR_TOO_LARGE = "too-large" + private const val ERROR_UNSUPPORTED_SCHEME = "unsupported-scheme" + private const val ERROR_DECODE_FAILED = "decode-failed" + private const val ERROR_NETWORK = "network-error" + private val SHA256 = Regex("^[0-9a-f]{64}$") + private val ALLOWED_SNIFFED_TYPES = + setOf( + "image/png", + "image/jpeg", + "image/gif", + "image/webp", + "image/bmp", + "audio/ogg", + "video/mp4", + ) } + + private class BlockedResourceException( + message: String, + ) : java.io.IOException(message) } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/napplets/NappletCapabilityExt.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/napplets/NappletCapabilityExt.kt index 66bce77ba8..201c818aa3 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/napplets/NappletCapabilityExt.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/napplets/NappletCapabilityExt.kt @@ -26,7 +26,6 @@ import com.vitorpamplona.amethyst.commons.napplet.NappletCapability internal fun NappletCapability.symbol(): MaterialSymbol = when (this) { - NappletCapability.SHELL -> MaterialSymbols.Tune NappletCapability.IDENTITY -> MaterialSymbols.AccountCircle NappletCapability.KEYS -> MaterialSymbols.Key NappletCapability.RELAY -> MaterialSymbols.Public diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/NappletProtocolJsonTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/NappletProtocolJsonTest.kt index 9fc041b421..211ea1a7be 100644 --- a/amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/NappletProtocolJsonTest.kt +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/NappletProtocolJsonTest.kt @@ -24,6 +24,7 @@ import com.vitorpamplona.amethyst.commons.napplet.NappletCapability import com.vitorpamplona.amethyst.commons.napplet.protocol.NappletProtocolJson import com.vitorpamplona.amethyst.commons.napplet.protocol.NappletRequest import com.vitorpamplona.amethyst.commons.napplet.protocol.NappletResponse +import com.vitorpamplona.amethyst.commons.napplet.protocol.NappletStorageScope import com.vitorpamplona.quartz.nip01Core.core.Event import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonNull @@ -73,11 +74,6 @@ class NappletProtocolJsonTest { assertEquals(NappletRequest.GetPublicKey, NappletProtocolJson.decodeRequest("""{"type":"identity.getPublicKey","id":"1"}""")) } - @Test - fun decodesShellSupports() { - assertEquals(NappletRequest.ShellSupports("relay"), NappletProtocolJson.decodeRequest("""{"type":"shell.supports","id":"1","domain":"relay"}""")) - } - @Test fun decodesPublishFromAnUnsignedTemplateInTheEventField() { // @napplet/shim carries the unsigned template in the `event` field. The shell signs it. @@ -165,13 +161,22 @@ class NappletProtocolJsonTest { assertEquals(NappletRequest.StorageGet("k"), NappletProtocolJson.decodeRequest("""{"type":"storage.get","key":"k"}""")) assertEquals(NappletRequest.StorageSet("k", "v"), NappletProtocolJson.decodeRequest("""{"type":"storage.set","key":"k","value":"v"}""")) assertEquals(NappletRequest.StorageRemove("k"), NappletProtocolJson.decodeRequest("""{"type":"storage.remove","key":"k"}""")) - assertEquals(NappletRequest.StorageKeys, NappletProtocolJson.decodeRequest("""{"type":"storage.keys"}""")) + assertEquals(NappletRequest.StorageKeys(), NappletProtocolJson.decodeRequest("""{"type":"storage.keys"}""")) + assertEquals( + NappletRequest.StorageGet("k", NappletStorageScope.INSTANCE), + NappletProtocolJson.decodeRequest("""{"type":"storage.get","key":"k","scope":"instance"}"""), + ) } @Test fun decodesValueResourceUpload() { assertEquals(NappletRequest.PayInvoice("lnbc1"), NappletProtocolJson.decodeRequest("""{"type":"value.payInvoice","invoice":"lnbc1"}""")) + assertEquals(NappletRequest.ResourceInfo, NappletProtocolJson.decodeRequest("""{"type":"resource.info"}""")) assertEquals(NappletRequest.ResourceBytes("https://x"), NappletProtocolJson.decodeRequest("""{"type":"resource.bytes","url":"https://x"}""")) + assertEquals( + NappletRequest.ResourceBytesMany(listOf("https://x", "data:text/plain,hi")), + NappletProtocolJson.decodeRequest("""{"type":"resource.bytesMany","urls":["https://x","data:text/plain,hi"]}"""), + ) // "SGk=" is base64 for "Hi"; shell.html inlines the request Blob as request.dataBase64. val up = NappletProtocolJson.decodeRequest("""{"type":"upload.upload","request":{"dataBase64":"SGk=","mimeType":"text/plain","filename":"a.txt"}}""") as NappletRequest.UploadBlob assertEquals("text/plain", up.contentType) @@ -184,6 +189,7 @@ class NappletProtocolJsonTest { assertNull(NappletProtocolJson.decodeRequest("""{"type":"inc.emit","id":"1"}""")) // keys.signEvent is not a real domain method (keys = keyboard actions, not signing). assertNull(NappletProtocolJson.decodeRequest("""{"type":"keys.signEvent","id":"1"}""")) + assertNull(NappletProtocolJson.decodeRequest("""{"type":"identity.futureMethod","id":"1"}""")) assertNull(NappletProtocolJson.decodeRequest("""{"foo":"bar"}""")) } @@ -211,7 +217,9 @@ class NappletProtocolJsonTest { assertEquals("s1", ev["subId"]?.jsonPrimitive?.content) assertEquals( "a".repeat(64), - ev["event"] + ev["result"] + ?.jsonObject + ?.get("event") ?.jsonObject ?.get("id") ?.jsonPrimitive @@ -233,13 +241,6 @@ class NappletProtocolJsonTest { assertEquals("pk", o["pubkey"]?.jsonPrimitive?.content) } - @Test - fun encodesSupported() { - val o = json.parseToJsonElement(NappletProtocolJson.encodeResponse("shell.supports", NappletResponse.Supported(true))).jsonObject - assertEquals("shell.supports.result", o["type"]?.jsonPrimitive?.content) - assertTrue(o["supported"]!!.jsonPrimitive.boolean) - } - @Test fun encodesPublishedEventAndEvents() { // relay.publish resolves to the signed event (matching upstream NostrEvent return). @@ -257,6 +258,18 @@ class NappletProtocolJsonTest { val events = json.parseToJsonElement(NappletProtocolJson.encodeResponse("relay.query", NappletResponse.Events(listOf(sampleEvent())))).jsonObject assertEquals(1, events["events"]?.jsonArray?.size) + assertEquals( + "a".repeat(64), + events["events"] + ?.jsonArray + ?.first() + ?.jsonObject + ?.get("event") + ?.jsonObject + ?.get("id") + ?.jsonPrimitive + ?.content, + ) } @Test @@ -269,6 +282,71 @@ class NappletProtocolJsonTest { assertEquals(2, keys["keys"]?.jsonArray?.size) } + @Test + fun encodesResourceInfoBulkItemsAndTypedErrors() { + val info = + json + .parseToJsonElement( + NappletProtocolJson.encodeResponse( + "resource.info", + NappletResponse.ResourceInfo(listOf("https"), 10L * 1024L * 1024L, 16), + ), + ).jsonObject + assertEquals( + "https", + info["info"] + ?.jsonObject + ?.get("schemes") + ?.jsonArray + ?.first() + ?.jsonObject + ?.get("scheme") + ?.jsonPrimitive + ?.content, + ) + + val items = + json + .parseToJsonElement( + NappletProtocolJson.encodeResponse( + "resource.bytesMany", + NappletResponse.ResourceItems( + listOf( + NappletResponse.ResourceItem("https://x", NappletResponse.Bytes("Hi".encodeToByteArray(), "text/plain")), + NappletResponse.ResourceItem("https://y", error = "not-found"), + ), + ), + ), + ).jsonObject["items"] + ?.jsonArray + assertEquals( + "SGk=", + items + ?.first() + ?.jsonObject + ?.get("bytes") + ?.jsonPrimitive + ?.content, + ) + assertEquals( + "not-found", + items + ?.get(1) + ?.jsonObject + ?.get("error") + ?.jsonPrimitive + ?.content, + ) + + val failure = + json + .parseToJsonElement( + NappletProtocolJson.encodeResponse("resource.bytes", NappletResponse.ResourceFailure("blocked-by-policy", "private target")), + ).jsonObject + assertEquals("resource.bytes.error", failure["type"]?.jsonPrimitive?.content) + assertEquals("blocked-by-policy", failure["error"]?.jsonPrimitive?.content) + } + @Test fun encodesBytesAsBase64WithMime() { val o = json.parseToJsonElement(NappletProtocolJson.encodeResponse("resource.bytes", NappletResponse.Bytes("Hi".encodeToByteArray(), "text/plain"))).jsonObject diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/NappletRelayCleartextTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/NappletRelayCleartextTest.kt new file mode 100644 index 0000000000..d2de5606d6 --- /dev/null +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/NappletRelayCleartextTest.kt @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.napplet + +import com.vitorpamplona.quartz.nip01Core.core.Event +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertSame +import org.junit.Test + +class NappletRelayCleartextTest { + @Test + fun plaintextPassesThroughWithoutDecrypting() = + runTest { + val event = event(content = "hello") + val result = + NappletRelayCleartext.forDelivery(event, USER) { _, _ -> + error("plaintext must not be decrypted") + } + + assertSame(event, result) + } + + @Test + fun inboundNip04IsProjectedAsCleartext() = + runTest { + val event = event(content = NIP04, tags = arrayOf(arrayOf("p", USER))) + val result = + NappletRelayCleartext.forDelivery(event, USER) { ciphertext, peer -> + assertEquals(NIP04, ciphertext) + assertEquals(AUTHOR, peer) + "secret" + } + + assertEquals("secret", result?.content) + assertEquals(event.id, result?.id) + assertEquals(event.sig, result?.sig) + } + + @Test + fun outboundEncryptedEventUsesItsRecipientAsPeer() = + runTest { + val event = event(pubKey = USER, content = NIP04, tags = arrayOf(arrayOf("p", RECIPIENT))) + val result = + NappletRelayCleartext.forDelivery(event, USER) { _, peer -> + assertEquals(RECIPIENT, peer) + "sent secret" + } + + assertEquals("sent secret", result?.content) + } + + @Test + fun encryptedEventForAnotherUserIsWithheld() = + runTest { + val event = event(content = NIP04, tags = arrayOf(arrayOf("p", RECIPIENT))) + val result = + NappletRelayCleartext.forDelivery(event, USER) { _, _ -> + error("unrelated ciphertext must not be offered to the signer") + } + + assertNull(result) + } + + @Test + fun decryptionFailureWithholdsCiphertext() = + runTest { + val event = event(content = NIP04, tags = arrayOf(arrayOf("p", USER))) + val result = + NappletRelayCleartext.forDelivery(event, USER) { _, _ -> + error("signer refused") + } + + assertNull(result) + } + + private fun event( + pubKey: String = AUTHOR, + content: String, + tags: Array> = emptyArray(), + ) = Event("id", pubKey, 1L, 4, tags, content, "sig") + + companion object { + private const val USER = "user" + private const val AUTHOR = "author" + private const val RECIPIENT = "recipient" + private const val NIP04 = "ciphertext-that-is-long-enough?iv=123456789012345678901234" + } +} diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/NappletSdkConformanceTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/NappletSdkConformanceTest.kt index fe65a51e6c..82f99a1a98 100644 --- a/amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/NappletSdkConformanceTest.kt +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/NappletSdkConformanceTest.kt @@ -24,6 +24,7 @@ import com.vitorpamplona.amethyst.commons.napplet.NappletCapability import com.vitorpamplona.amethyst.commons.napplet.protocol.NappletProtocolJson import com.vitorpamplona.amethyst.commons.napplet.protocol.NappletRequest import com.vitorpamplona.amethyst.commons.napplet.protocol.NappletResponse +import com.vitorpamplona.amethyst.commons.napplet.protocol.NappletStorageScope import com.vitorpamplona.quartz.nip01Core.core.Event import kotlinx.serialization.json.Json import kotlinx.serialization.json.jsonArray @@ -114,17 +115,31 @@ class NappletSdkConformanceTest { // RelayQueryResultMessage: { type:'relay.query.result', id, events, error? } val o = result("relay.query", NappletResponse.Events(listOf(sampleEvent()))) assertEquals(1, o["events"]?.jsonArray?.size) + assertEquals( + "a".repeat(64), + o["events"] + ?.jsonArray + ?.first() + ?.jsonObject + ?.get("event") + ?.jsonObject + ?.get("id") + ?.jsonPrimitive + ?.content, + ) } @Test fun relayEventAndEosePushesMatchTheSdk() { - // RelayEventMessage (PUSH): { type:'relay.event', subId, event } + // RelayEventMessage (PUSH): { type:'relay.event', subId, result:{event, sidecar?} } val ev = json.parseToJsonElement(NappletProtocolJson.encodeRelayEvent("s1", sampleEvent())).jsonObject assertEquals("relay.event", ev["type"]?.jsonPrimitive?.content) assertEquals("s1", ev["subId"]?.jsonPrimitive?.content) assertEquals( "a".repeat(64), - ev["event"] + ev["result"] + ?.jsonObject + ?.get("event") ?.jsonObject ?.get("id") ?.jsonPrimitive @@ -175,7 +190,11 @@ class NappletSdkConformanceTest { assertEquals(NappletRequest.StorageGet("k"), NappletProtocolJson.decodeRequest("""{"type":"storage.get","id":"1","key":"k"}""")) assertEquals(NappletRequest.StorageSet("k", "v"), NappletProtocolJson.decodeRequest("""{"type":"storage.set","id":"1","key":"k","value":"v"}""")) assertEquals(NappletRequest.StorageRemove("k"), NappletProtocolJson.decodeRequest("""{"type":"storage.remove","id":"1","key":"k"}""")) - assertEquals(NappletRequest.StorageKeys, NappletProtocolJson.decodeRequest("""{"type":"storage.keys","id":"1"}""")) + assertEquals(NappletRequest.StorageKeys(), NappletProtocolJson.decodeRequest("""{"type":"storage.keys","id":"1"}""")) + assertEquals( + NappletRequest.StorageGet("k", NappletStorageScope.INSTANCE), + NappletProtocolJson.decodeRequest("""{"type":"storage.get","id":"1","key":"k","scope":"instance"}"""), + ) // StorageGetResultMessage.value, StorageKeysResultMessage.keys assertTrue(result("storage.get", NappletResponse.StorageValue("v")).containsKey("value")) @@ -186,7 +205,12 @@ class NappletSdkConformanceTest { @Test fun resourceBytesRequestAndResultMatch() { + assertEquals(NappletRequest.ResourceInfo, NappletProtocolJson.decodeRequest("""{"type":"resource.info","id":"0"}""")) assertEquals(NappletRequest.ResourceBytes("https://x"), NappletProtocolJson.decodeRequest("""{"type":"resource.bytes","id":"1","url":"https://x"}""")) + assertEquals( + NappletRequest.ResourceBytesMany(listOf("https://x", "data:text/plain,hi")), + NappletProtocolJson.decodeRequest("""{"type":"resource.bytesMany","id":"2","urls":["https://x","data:text/plain,hi"]}"""), + ) // The host emits base64 bytes + mime; shell.html rebuilds the Blob the SDK expects. val o = result("resource.bytes", NappletResponse.Bytes("Hi".encodeToByteArray(), "text/plain")) assertEquals("SGk=", o["bytes"]?.jsonPrimitive?.content) @@ -203,29 +227,6 @@ class NappletSdkConformanceTest { assertTrue(result("relay.query", NappletResponse.Failed("boom")).containsKey("error")) } - // ---------- shell handshake (ShellReadyMessage / ShellInitMessage) ---------- - - @Test - fun shellInitAdvertisesTheCapabilityEnvironment() { - // shell.ready is answered by the host (not the codec) with this shell.init env, which the - // SDK caches and answers shell.supports() from locally. ShellInitMessage: - // { type:'shell.init', capabilities:{ domains, protocols }, services }. - val o = json.parseToJsonElement(NappletProtocolJson.encodeShellInit(listOf("shell", "relay"), listOf("shell", "relay"))).jsonObject - assertEquals("shell.init", o["type"]?.jsonPrimitive?.content) - assertEquals( - 2, - o["capabilities"] - ?.jsonObject - ?.get("domains") - ?.jsonArray - ?.size, - ) - assertTrue(o["capabilities"]?.jsonObject?.containsKey("protocols") == true) - assertEquals(2, o["services"]?.jsonArray?.size) - // shell.ready stays a host-layer message — the codec doesn't treat it as a broker request. - assertNull(NappletProtocolJson.decodeRequest("""{"type":"shell.ready"}""")) - } - // ---------- keys (keyboard/command actions) ---------- @Test diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/gateways/NappletResourceFetcherPolicyTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/gateways/NappletResourceFetcherPolicyTest.kt new file mode 100644 index 0000000000..5f8643317d --- /dev/null +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/gateways/NappletResourceFetcherPolicyTest.kt @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.napplet.gateways + +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import java.net.InetAddress + +class NappletResourceFetcherPolicyTest { + @Test + fun blocksPrivateSpecialAndLocalAddresses() { + val blocked = + listOf( + "0.0.0.0", + "10.0.0.1", + "100.64.0.1", + "127.0.0.1", + "169.254.169.254", + "172.16.0.1", + "192.0.0.1", + "192.0.2.1", + "192.168.1.1", + "198.18.0.1", + "198.51.100.1", + "203.0.113.1", + "224.0.0.1", + "::1", + "2001:db8::1", + "fc00::1", + "fe80::1", + ) + + blocked.forEach { + assertFalse(it, NappletResourceFetcher.isPublicAddress(InetAddress.getByName(it))) + } + } + + @Test + fun permitsPublicAddresses() { + assertTrue(NappletResourceFetcher.isPublicAddress(InetAddress.getByName("1.1.1.1"))) + assertTrue(NappletResourceFetcher.isPublicAddress(InetAddress.getByName("2606:4700:4700::1111"))) + } + + @Test + fun acceptsOnlyCredentialFreeHttpsUrls() { + assertTrue(NappletResourceFetcher.isSafeHttpsResourceUrl("https://example.com/a")) + assertFalse(NappletResourceFetcher.isSafeHttpsResourceUrl("http://example.com/a")) + assertFalse(NappletResourceFetcher.isSafeHttpsResourceUrl("https://user:secret@example.com/a")) + assertFalse(NappletResourceFetcher.isSafeHttpsResourceUrl("file:///etc/passwd")) + } +} diff --git a/commons/src/commonMain/composeResources/files/napplet/shell.html b/commons/src/commonMain/composeResources/files/napplet/shell.html index 78e42c56c3..e7658bad3e 100644 --- a/commons/src/commonMain/composeResources/files/napplet/shell.html +++ b/commons/src/commonMain/composeResources/files/napplet/shell.html @@ -1,13 +1,10 @@ @@ -19,7 +16,7 @@ - + diff --git a/commons/src/commonMain/composeResources/files/napplet/shim.js b/commons/src/commonMain/composeResources/files/napplet/shim.js index 89e256a2f4..6ccd099cac 100644 --- a/commons/src/commonMain/composeResources/files/napplet/shim.js +++ b/commons/src/commonMain/composeResources/files/napplet/shim.js @@ -79,7 +79,7 @@ // Subscription pushes are keyed by subId, not a request id. if (msg.type === 'relay.event' || msg.type === 'relay.eose' || msg.type === 'relay.closed') { var sub = subs[msg.subId]; if (!sub) return; - if (msg.type === 'relay.event') { if (sub.onEvent) sub.onEvent(msg.event); } + if (msg.type === 'relay.event') { if (sub.onEvent) sub.onEvent(msg.result); } else if (msg.type === 'relay.eose') { if (sub.onEose) sub.onEose(); } else { delete subs[msg.subId]; if (sub.onClosed) sub.onClosed(msg.reason); } return; @@ -91,7 +91,7 @@ // identity.changed push: the active user's key changed (account switch / connect / disconnect). if (msg.type === 'identity.changed') { identityHandlers.slice().forEach(function(h){ try { h(msg.pubkey); } catch (_) {} }); return; } if (!msg.id) return; - var p = pending[msg.id]; if (!p) return; delete pending[msg.id]; + var p = pending[msg.id]; if (!p) return; delete pending[msg.id]; if (p.cleanup) p.cleanup(); if (msg.ok) p.resolve(msg); else { var err = new Error(msg.reason || msg.operation || msg.error || 'napplet error'); err.napplet = msg; p.reject(err); } } @@ -101,16 +101,31 @@ window.addEventListener('message', function(e){ if (e.source !== parent) return; onIncoming(e.data); }); } function field(promise, name){ return promise.then(function(m){ return m[name]; }); } - function normFilters(filters){ return Array.isArray(filters) ? { filters: filters } : { filter: filters || {} }; } + function resourceCall(type, fields, opts){ + var signal = opts && opts.signal; + if (signal && signal.aborted) return Promise.reject(new DOMException('Aborted', 'AbortError')); + return new Promise(function(resolve, reject){ + var env = { type: type }; for (var k in fields) env[k] = fields[k]; + var id = send(env), onAbort; + var cleanup = function(){ if (signal && onAbort) signal.removeEventListener('abort', onAbort); }; + pending[id] = { resolve: resolve, reject: reject, cleanup: cleanup }; + if (signal) { + onAbort = function(){ + if (!pending[id]) return; + delete pending[id]; cleanup(); + post('resource.cancel', { id: id }); + reject(new DOMException('Aborted', 'AbortError')); + }; + signal.addEventListener('abort', onAbort, { once: true }); + } + }); + } + function normFilters(filters){ return { filters: Array.isArray(filters) ? filters : [filters || {}] }; } function bytesToB64(bytes){ var u = bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes); var s=''; for (var i=0;i= 0) identityHandlers.splice(i, 1); - if (identityHandlers.length === 0) post('identity.unwatch'); } }; } }, @@ -152,6 +164,7 @@ var subId = 's' + (seq++); subs[subId] = { onEvent: onEvent, onEose: onEose }; var env = normFilters(filters); env.subId = subId; + if (options && options.relay) env.relay = options.relay; post('relay.subscribe', env); return { close: function(){ delete subs[subId]; post('relay.close', { subId: subId }); } }; } @@ -161,23 +174,49 @@ getItem: function(key){ return field(call('storage.get', { key: key }), 'value'); }, setItem: function(key, value){ return call('storage.set', { key: key, value: value }).then(function(){}); }, removeItem: function(key){ return call('storage.remove', { key: key }).then(function(){}); }, - keys: function(){ return field(call('storage.keys'), 'keys'); } + keys: function(){ return field(call('storage.keys'), 'keys'); }, + instance: { + getItem: function(key){ return field(call('storage.get', { key: key, scope: 'instance' }), 'value'); }, + setItem: function(key, value){ return call('storage.set', { key: key, value: value, scope: 'instance' }).then(function(){}); }, + removeItem: function(key){ return call('storage.remove', { key: key, scope: 'instance' }).then(function(){}); }, + keys: function(){ return field(call('storage.keys', { scope: 'instance' }), 'keys'); } + } }, // value.payInvoice is an Amethyst-specific extension (not part of @napplet/shim). value: { payInvoice: function(invoice){ return field(call('value.payInvoice', { invoice: invoice }), 'preimage'); } }, resource: { - // The shell rebuilds the Blob from the host's base64 before this resolves. - bytes: function(url){ return field(call('resource.bytes', { url: url }), 'blob'); }, - bytesAsObjectURL: function(url){ return field(call('resource.bytes', { url: url }), 'blob').then(function(blob){ return URL.createObjectURL(blob); }); } + info: function(){ return field(call('resource.info'), 'info'); }, + // The shell rebuilds Blobs from the host's base64 before these resolve. + bytes: function(url, opts){ return field(resourceCall('resource.bytes', { url: url }, opts), 'blob'); }, + bytesMany: function(urls, opts){ return field(resourceCall('resource.bytesMany', { urls: Array.from(urls || []) }, opts), 'items'); }, + bytesAsObjectURL: function(url){ + var objectUrl = '', revoked = false; + var handle = { url: '', revoke: function(){ if (revoked) return; revoked = true; if (objectUrl) URL.revokeObjectURL(objectUrl); } }; + var ready = available.resource.bytes(url).then(function(blob){ + if (revoked) return; + objectUrl = URL.createObjectURL(blob); handle.url = objectUrl; return objectUrl; + }); + Object.defineProperty(handle, 'ready', { value: ready, enumerable: false }); + return handle; + } }, upload: { // Sends the SDK's upload.upload; we inline the bytes as base64 (shell.html does the same for // a Blob from a stock napplet). Resolves to the uploaded URL. blob: function(bytes, contentType){ return field(call('upload.upload', { request: { dataBase64: bytesToB64(bytes), mimeType: contentType } }), 'url'); } + }, + theme: { + get: function(){ return field(call('theme.get'), 'theme'); } } }; + var requested = []; + try { if (Array.isArray(window.__nappletDomains)) requested = window.__nappletDomains; } catch (_) {} + var napplet = {}; + requested.forEach(function(domain){ + if (typeof domain === 'string' && Object.prototype.hasOwnProperty.call(available, domain)) napplet[domain] = available[domain]; + }); window.napplet = Object.freeze(napplet); // ---- IME agent (in-app browser only) ------------------------------------------------------- diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletArtifactPolicy.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletArtifactPolicy.kt new file mode 100644 index 0000000000..8cc23cdada --- /dev/null +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletArtifactPolicy.kt @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.napplet + +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip5aStaticWebsites.SiteAggregateHash +import com.vitorpamplona.quartz.nip5aStaticWebsites.tags.PathTag + +/** Pure NIP-5D artifact checks shared by every launch surface. */ +object NappletArtifactPolicy { + /** Returns the runtime-computed artifact identity, or null when the manifest must not execute. */ + fun verifiedAggregateHash( + paths: List, + declaredAggregateHash: HexKey?, + ): HexKey? { + val entry = paths.singleOrNull() ?: return null + if (entry.path != "/index.html" || !SHA256.matches(entry.hash)) return null + val computed = SiteAggregateHash.compute(paths) + if (declaredAggregateHash != null && !declaredAggregateHash.equals(computed, ignoreCase = true)) return null + return computed + } + + private val SHA256 = Regex("^[0-9a-fA-F]{64}$") +} diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletBroker.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletBroker.kt index 48800d865e..0a9a923848 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletBroker.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletBroker.kt @@ -122,12 +122,6 @@ class NappletBroker( ): NappletResponse { val capability = request.capability - // shell.supports is capability negotiation: always answerable, no declaration/consent. - if (request is NappletRequest.ShellSupports) { - val cap = NappletCapability.fromNapDomain(request.domain) - return NappletResponse.Supported(cap != null && cap in declared) - } - if (capability !in declared) { return NappletResponse.Denied(capability, "This napplet did not declare the '${capability.name.lowercase()}' capability.") } @@ -149,7 +143,7 @@ class NappletBroker( // Keyboard/command action registration is a shell-mediated UI affordance, not key // access — declared is enough; it never prompts. request is NappletRequest.RegisterAction || request is NappletRequest.UnregisterAction -> true - // Cosmetic/negotiation capabilities (theme) never prompt. + // Cosmetic capabilities (theme) never prompt. !capability.requiresConsent -> true // A standing allow short-circuits, except for per-use capabilities (e.g. payments). ledger.decide(identity, capability) == PermissionDecision.ALLOW && !capability.requiresPerUseConsent -> true @@ -219,9 +213,6 @@ class NappletBroker( request: NappletRequest, ): NappletResponse = when (request) { - // Negotiation is resolved in handle(); execute() is never reached for it. - is NappletRequest.ShellSupports -> NappletResponse.Supported(true) - is NappletRequest.GetPublicKey -> NappletResponse.PublicKey(signer.pubKey) is NappletRequest.ThemeGet -> { @@ -267,24 +258,24 @@ class NappletBroker( is NappletRequest.StorageGet -> { val store = storage ?: return NappletResponse.Unsupported("storage.getItem") - NappletResponse.StorageValue(store.get(identity.coordinate, request.key)) + NappletResponse.StorageValue(store.get(storageCoordinate(identity, request.scope), request.key)) } is NappletRequest.StorageSet -> { val store = storage ?: return NappletResponse.Unsupported("storage.setItem") - store.set(identity.coordinate, request.key, request.value) + store.set(storageCoordinate(identity, request.scope), request.key, request.value) NappletResponse.Done } is NappletRequest.StorageRemove -> { val store = storage ?: return NappletResponse.Unsupported("storage.removeItem") - store.remove(identity.coordinate, request.key) + store.remove(storageCoordinate(identity, request.scope), request.key) NappletResponse.Done } is NappletRequest.StorageKeys -> { val store = storage ?: return NappletResponse.Unsupported("storage.keys") - NappletResponse.Strings(store.keys(identity.coordinate)) + NappletResponse.Strings(store.keys(storageCoordinate(identity, request.scope))) } is NappletRequest.NotifyCreate -> { @@ -316,8 +307,38 @@ class NappletBroker( is NappletRequest.ResourceBytes -> { val gateway = resource ?: return NappletResponse.Unsupported("resource.bytes") - val fetched = gateway.fetch(request.url, identity.coordinate) ?: return NappletResponse.Failed("Could not fetch the resource.") - NappletResponse.Bytes(fetched.bytes, fetched.contentType) + when (val fetched = gateway.fetch(request.url, identity.coordinate)) { + is NappletResourceResult.Success -> NappletResponse.Bytes(fetched.resource.bytes, fetched.resource.contentType) + is NappletResourceResult.Failure -> NappletResponse.ResourceFailure(fetched.error, fetched.message) + } + } + + is NappletRequest.ResourceInfo -> { + resource ?: return NappletResponse.Unsupported("resource.info") + NappletResponse.ResourceInfo( + schemes = listOf("data", "https", "blossom", "nostr"), + maxBytes = RESOURCE_MAX_BYTES, + maxUrls = RESOURCE_MAX_URLS, + ) + } + + is NappletRequest.ResourceBytesMany -> { + val gateway = resource ?: return NappletResponse.Unsupported("resource.bytesMany") + if (request.urls.isEmpty()) return NappletResponse.ResourceFailure("invalid-request", "Resource URL list is empty.") + if (request.urls.size > RESOURCE_MAX_URLS) return NappletResponse.ResourceFailure("too-large", "Resource URL limit exceeded.") + NappletResponse.ResourceItems( + request.urls.map { url -> + when (val fetched = gateway.fetch(url, identity.coordinate)) { + is NappletResourceResult.Success -> + NappletResponse.ResourceItem( + url = url, + resource = NappletResponse.Bytes(fetched.resource.bytes, fetched.resource.contentType), + ) + is NappletResourceResult.Failure -> + NappletResponse.ResourceItem(url = url, error = fetched.error, message = fetched.message) + } + }, + ) } is NappletRequest.UploadBlob -> { @@ -353,6 +374,15 @@ class NappletBroker( tags + arrayOf(arrayOf("p", recipient)) } + private fun storageCoordinate( + identity: NappletIdentity, + scope: com.vitorpamplona.amethyst.commons.napplet.protocol.NappletStorageScope, + ): String = + when (scope) { + com.vitorpamplona.amethyst.commons.napplet.protocol.NappletStorageScope.SHARED -> identity.storageCoordinate + com.vitorpamplona.amethyst.commons.napplet.protocol.NappletStorageScope.INSTANCE -> identity.instanceStorageCoordinate + } + /** * Shows the first-connect "Connect to Nostr" dialog if no signer policy exists yet. * On success, stores the chosen policy and bulk-grants all declared non-payment capabilities. @@ -505,16 +535,6 @@ class NappletBroker( } } - /** - * True when [capability] carries a standing denial for [identity]. Push-subscription edge ops - * (identity.watch and friends) short-circuit before [handle], so they have to apply the same - * "a standing denial always wins" rule themselves rather than trusting the declaration alone. - */ - suspend fun isDenied( - identity: NappletIdentity, - capability: NappletCapability, - ): Boolean = ledger.decide(identity, capability) == PermissionDecision.DENY - companion object { /** * How long (ms) a Cancel on the first-connect dialog suppresses re-prompting for the same app. @@ -522,5 +542,7 @@ class NappletBroker( * the dialog per request; short enough that a deliberate user retry seconds later prompts again. */ private const val CANCEL_REPROMPT_COOLDOWN_MS = 3_000L + private const val RESOURCE_MAX_BYTES = 10L * 1024L * 1024L + private const val RESOURCE_MAX_URLS = 16 } } diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletBrokerCollaborators.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletBrokerCollaborators.kt index e229884068..27b84f70de 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletBrokerCollaborators.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletBrokerCollaborators.kt @@ -156,6 +156,17 @@ class NappletResource( val contentType: String, ) +sealed interface NappletResourceResult { + data class Success( + val resource: NappletResource, + ) : NappletResourceResult + + data class Failure( + val error: String, + val message: String? = null, + ) : NappletResourceResult +} + /** * Bridges the broker to sandboxed resource fetching for [NappletCapability.RESOURCE] * (`resource.bytes`). The host fetches https/blossom/nostr/data URLs on the applet's behalf — @@ -164,13 +175,14 @@ class NappletResource( * * [coordinate] is the calling applet's identity coordinate (`author:identifier`), so the host can * route the fetch the same way the applet's own page loads — through Tor or the open web — per that - * applet's/site's network mode. + * applet's/site's network mode. Failures carry the stable NAP-RESOURCE error code rather than + * collapsing policy rejections and network failures into one nullable result. */ fun interface NappletResourceGateway { suspend fun fetch( url: String, coordinate: String, - ): NappletResource? + ): NappletResourceResult } /** A completed upload: where the blob lives plus NIP-94-ish metadata. */ diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletCapability.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletCapability.kt index a781dc7bc1..33ee94cf7c 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletCapability.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletCapability.kt @@ -25,14 +25,11 @@ package com.vitorpamplona.amethyst.commons.napplet * (`napplet/naps`, `@napplet/web`). A napplet declares the domains it needs via `requires` tags; * [fromNapDomain] maps each bare domain string to the capability the broker enforces. * - * The mapping is **default-deny**: an unrecognized NAP domain maps to `null` and the shell must - * surface it as unknown rather than silently granting it. Domains we don't yet broker - * (`intent`, `media`, `config`, `outbox`, `ifc`, `cvm`) therefore resolve to `null` for now. + * The mapping is **default-deny**: an unrecognized or only partially implemented NAP domain maps + * to `null`. Keeping a broker implementation below does not advertise conformance; only domains + * whose current NAP contract is implemented are injected into `window.napplet`. */ enum class NappletCapability { - /** `shell` — capability negotiation (`shell.supports`). Always available; needs no consent. */ - SHELL, - /** `identity` — read-only identity queries (`getPublicKey`, `onChanged`). */ IDENTITY, @@ -70,13 +67,13 @@ enum class NappletCapability { ; /** - * Whether using this capability requires user consent. Negotiation ([SHELL]) and the cosmetic, - * read-only theme read ([THEME]) never prompt; everything else does (subject to the broker's + * Whether using this capability requires user consent. The cosmetic, read-only theme read + * ([THEME]) never prompts; everything else does (subject to the broker's * signer-self-gating and standing-grant rules). [INC] is authorized at the router edge on its * declaration alone, so it never reaches the consent path regardless of this flag. */ val requiresConsent: Boolean - get() = this != SHELL && this != THEME + get() = this != THEME /** * Whether the user must confirm **every single use** — no standing auto-approval. True for @@ -96,25 +93,22 @@ enum class NappletCapability { companion object { /** - * Maps a bare NAP domain to the capability the broker enforces, case-insensitively. - * Returns `null` for any domain the shell does not recognize — callers MUST treat that as - * "unknown, do not grant". + * Maps a bare, currently supported NAP domain to the capability the broker enforces. + * Returns `null` for unknown and partial/legacy domains — callers MUST treat that as + * "unavailable, do not inject or grant". NIP-5D domain names are exact lowercase strings. */ fun fromNapDomain(domain: String): NappletCapability? = - when (domain.trim().lowercase()) { - "shell" -> SHELL + when (domain) { "identity" -> IDENTITY - "keys" -> KEYS - "relay", "relays" -> RELAY + "relay" -> RELAY "storage" -> STORAGE - "value" -> VALUE "resource" -> RESOURCE - "upload" -> UPLOAD "theme" -> THEME - "notify" -> NOTIFY - "inc" -> INC else -> null } + + /** Exact NIP-5D domain names Amethyst currently exposes through its injection prelude. */ + val supportedNapDomains: Set = setOf("identity", "relay", "storage", "resource", "theme") } } diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletIdentity.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletIdentity.kt index a543bf505f..9d2a759ab6 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletIdentity.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletIdentity.kt @@ -41,7 +41,15 @@ data class NappletIdentity( val authorPubKey: HexKey, val identifier: String, val aggregateHash: HexKey? = null, + /** Opaque host-assigned lifetime id used only for NAP-STORAGE's instance scope. */ + val instanceId: String? = null, ) { /** The ledger key: coordinate only, never the [aggregateHash], so grants survive updates. */ val coordinate: String = "$authorPubKey:$identifier" + + /** NAP-STORAGE shared namespace: exact publisher + dTag + verified artifact identity. */ + val storageCoordinate: String = "$coordinate:${aggregateHash.orEmpty()}" + + /** NAP-STORAGE instance namespace, stable for this host launch and isolated from sibling launches. */ + val instanceStorageCoordinate: String = "$storageCoordinate:instance:${instanceId.orEmpty()}" } diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletWebContract.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletWebContract.kt index 74f960c331..43d780edf3 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletWebContract.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletWebContract.kt @@ -41,33 +41,19 @@ object NappletWebContract { const val SHELL_URL = "$ORIGIN/__shell__" /** - * The applet runs on its **own per-applet origin** — a unique subdomain of [HOST] — served at the - * origin root, NOT on the shell [ORIGIN]. Two reasons, both load-bearing: - * - * 1. **A real (non-opaque) origin is what gives the applet working, persistent storage.** An - * `allow-scripts`-only opaque-origin iframe has no `localStorage`/`IndexedDB`/service worker - * (reads throw `SecurityError`), which crash-loops essentially every SPA. A real origin with - * `allow-same-origin` has them, scoped and isolated per applet (subdomains don't share - * storage), so applets can't read each other's data. - * 2. **Keeping it on a DISTINCT origin from the shell is what preserves the trust boundary.** The - * native bridge is origin-restricted to the shell [ORIGIN]; the applet, being cross-origin, - * still can't reach it (nor read the shell DOM) — it talks only via `postMessage`, which the - * shell relays. `allow-same-origin` is therefore safe here precisely because the applet is - * same-origin only with *itself*, never with the shell. - * - * The applet is served at its origin root because SPA bundlers (Vite, CRA, webpack, nsyte, …) emit - * **absolute** asset URLs (`/assets/app.js`, `/fonts/x.woff2`) that resolve against the origin root. - * - * [appId] must be a stable, unique, DNS-label-safe token per applet (the host derives it from the - * applet's author + identifier), so the same applet keeps its storage across launches. + * Per-site origin retained for Amethyst's NIP-5A WEBSITE profile. NIP-5D napplets never navigate + * here: their verified, self-contained `/index.html` is assigned through `srcdoc` and therefore + * executes with an opaque origin in an `allow-scripts`-only sandbox. */ fun appOrigin(appId: String): String = "https://$appId.$HOST" /** True for the shell host and any per-applet subdomain — i.e. everything we serve internally. */ fun isInternalHost(host: String?): Boolean = host == HOST || (host != null && host.endsWith(".$HOST")) - /** Placeholder in [SHELL_HTML_PATH] the host replaces with the per-applet [appOrigin] before serving. */ + /** Placeholders in [SHELL_HTML_PATH] replaced by the host before serving the trusted shell. */ const val APP_ORIGIN_PLACEHOLDER = "__APP_ORIGIN__" + const val APP_SANDBOX_PLACEHOLDER = "__APP_SANDBOX__" + const val APP_BOOTSTRAP_PLACEHOLDER = "__APP_BOOTSTRAP__" /** Name of the origin-restricted native bridge the shell (and only the shell) can reach. */ const val BRIDGE_NAME = "__nappletBridge" @@ -76,24 +62,72 @@ object NappletWebContract { * CSP for the shell document: it may inline its own bridge script/style and frame **only this * applet's** origin, but has no network and cannot navigate or submit anywhere. */ - fun shellCsp(appOrigin: String): String = + fun shellCsp(frameSource: String): String = "default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; " + - "frame-src $appOrigin; base-uri 'none'; form-action 'none'" + "frame-src $frameSource; base-uri 'none'; form-action 'none'; frame-ancestors 'none'" /** - * CSP for the applet document. The applet has a real origin now, so `'self'` resolves to its own - * per-applet origin and the shell origin is deliberately NOT granted. The key lever is - * `connect-src 'none'`: the applet gets **no** direct network — every fetch goes through the - * brokered, consent-gated `resource.bytes`. + * Conservative NIP-5D CSP injected as the first element of the verified napplet's `head` before + * the runtime prelude. A `srcdoc` napplet has an opaque origin, so self-hosted subresources are + * intentionally unavailable; a conforming napplet is one self-contained `/index.html`. */ const val APP_CSP: String = - "default-src 'self'; " + - "script-src 'self' 'unsafe-inline'; " + - "style-src 'self' 'unsafe-inline'; " + - "img-src 'self' data: blob:; " + - "font-src 'self' data:; " + - "media-src 'self' blob: data:; " + - "connect-src 'none'; frame-src 'none'; object-src 'none'; base-uri 'self'; form-action 'none'" + "default-src 'none'; script-src 'unsafe-inline'; style-src 'unsafe-inline'; " + + "img-src data: blob:; font-src data:; connect-src 'none'; worker-src 'none'; " + + "child-src 'none'; frame-src 'none'; media-src 'none'; object-src 'none'; " + + "manifest-src 'none'; base-uri 'none'; form-action 'none'" + + /** + * Injects host-owned policy and the runtime prelude before any authored element in `head`. + * [locked] is the NIP-5D posture; WEBSITE callers deliberately retain Amethyst's NIP-07 and + * normal-origin behavior. Only syntactically valid, explicitly authorized NAP domains are + * projected onto `window.napplet` by the trusted [shimJs]. + */ + fun injectPrelude( + html: ByteArray, + shimJs: String, + declaredDomains: List, + locked: Boolean, + injectNip07: Boolean = false, + imeProxy: Boolean = false, + ): ByteArray { + val text = html.decodeToString() + val policy = + if (locked) { + "" + } else { + "" + } + val style = "" + val flags = + "" + val safeDomains = + declaredDomains + .filter { it.matches(NAP_DOMAIN) && it in NappletCapability.supportedNapDomains } + .distinct() + val domainsJson = safeDomains.joinToString(prefix = "[", postfix = "]") { "\"$it\"" } + val prelude = "$policy$style$flags" + val headIdx = text.indexOf("= 0 -> { + val close = text.indexOf('>', headIdx) + if (close >= 0) text.substring(0, close + 1) + prelude + text.substring(close + 1) else prelude + text + } + else -> { + val htmlIdx = text.indexOf("= 0) text.indexOf('>', htmlIdx) else -1 + if (htmlClose >= 0) { + text.substring(0, htmlClose + 1) + "$prelude" + text.substring(htmlClose + 1) + } else { + "$prelude$text" + } + } + } + return injected.encodeToByteArray() + } const val SHELL_HTML_PATH = "files/napplet/shell.html" const val SHIM_JS_PATH = "files/napplet/shim.js" @@ -114,4 +148,6 @@ object NappletWebContract { /** The `window.napplet` client shim a host injects into the applet document. */ @OptIn(ExperimentalResourceApi::class) suspend fun shimJs(): ByteArray = Res.readBytes(SHIM_JS_PATH) + + private val NAP_DOMAIN = Regex("^[a-z][a-z0-9-]*$") } diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/protocol/NappletRequest.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/protocol/NappletRequest.kt index d85925650f..25a58bca43 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/protocol/NappletRequest.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/protocol/NappletRequest.kt @@ -24,6 +24,11 @@ import com.vitorpamplona.amethyst.commons.napplet.NappletCapability import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +enum class NappletStorageScope { + SHARED, + INSTANCE, +} + /** * A capability request from a napplet, after it has crossed the postMessage + IPC edge * and been deserialized into a typed object. Each variant declares the [capability] the @@ -68,14 +73,6 @@ sealed interface NappletRequest { override val capability get() = NappletCapability.THEME } - /** `shell.supports(domain, protocol?)` — capability negotiation; always answerable, no consent. */ - data class ShellSupports( - val domain: String, - val protocol: String? = null, - ) : NappletRequest { - override val capability get() = NappletCapability.SHELL - } - /** * Publish an event built from an **unsigned template**. The napplet supplies only `kind`, * `tags`, and `content`; the shell sets `pubkey` from the real signer, stamps `created_at`, @@ -198,6 +195,7 @@ sealed interface NappletRequest { /** Read a value from this napplet's sandboxed key-value store (`storage.getItem`). */ data class StorageGet( val key: String, + val scope: NappletStorageScope = NappletStorageScope.SHARED, ) : NappletRequest { override val capability get() = NappletCapability.STORAGE } @@ -206,6 +204,7 @@ sealed interface NappletRequest { data class StorageSet( val key: String, val value: String, + val scope: NappletStorageScope = NappletStorageScope.SHARED, ) : NappletRequest { override val capability get() = NappletCapability.STORAGE } @@ -213,12 +212,15 @@ sealed interface NappletRequest { /** Remove a value from this napplet's sandboxed key-value store (`storage.removeItem`). */ data class StorageRemove( val key: String, + val scope: NappletStorageScope = NappletStorageScope.SHARED, ) : NappletRequest { override val capability get() = NappletCapability.STORAGE } /** List the keys this napplet has stored (`storage.keys`). */ - data object StorageKeys : NappletRequest { + data class StorageKeys( + val scope: NappletStorageScope = NappletStorageScope.SHARED, + ) : NappletRequest { override val capability get() = NappletCapability.STORAGE } @@ -282,6 +284,18 @@ sealed interface NappletRequest { override val capability get() = NappletCapability.RESOURCE } + /** Describe the bounded schemes and limits of this shell's existing resource broker. */ + data object ResourceInfo : NappletRequest { + override val capability get() = NappletCapability.RESOURCE + } + + /** Fetch several resources in input order, returning a per-URL success/error record. */ + data class ResourceBytesMany( + val urls: List, + ) : NappletRequest { + override val capability get() = NappletCapability.RESOURCE + } + /** Upload a blob to the user's Blossom server (`upload.upload`). */ data class UploadBlob( val bytes: ByteArray, diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/protocol/NappletResponse.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/protocol/NappletResponse.kt index 63ad63afff..07f0394b0c 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/protocol/NappletResponse.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/protocol/NappletResponse.kt @@ -51,11 +51,6 @@ sealed interface NappletResponse { val events: List, ) : NappletResponse - /** Result of `shell.supports(domain)`. */ - data class Supported( - val supported: Boolean, - ) : NappletResponse - /** Result of `keys.registerAction`: the shell-assigned [actionId] and the [binding] it honored (e.g. `"Ctrl+S"`). */ data class ActionRegistered( val actionId: String, @@ -97,6 +92,28 @@ sealed interface NappletResponse { override fun hashCode(): Int = 31 * contentType.hashCode() + bytes.contentHashCode() } + data class ResourceInfo( + val schemes: List, + val maxBytes: Long, + val maxUrls: Int, + ) : NappletResponse + + data class ResourceItem( + val url: String, + val resource: Bytes? = null, + val error: String? = null, + val message: String? = null, + ) + + data class ResourceItems( + val items: List, + ) : NappletResponse + + data class ResourceFailure( + val error: String, + val message: String? = null, + ) : NappletResponse + /** Result of an `upload.upload`; [url] is where the blob can be fetched, plus NIP-94-ish metadata. */ data class Uploaded( val url: String, diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletArtifactPolicyTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletArtifactPolicyTest.kt new file mode 100644 index 0000000000..a079eeab6d --- /dev/null +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletArtifactPolicyTest.kt @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.napplet + +import com.vitorpamplona.quartz.nip5aStaticWebsites.SiteAggregateHash +import com.vitorpamplona.quartz.nip5aStaticWebsites.tags.PathTag +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull + +class NappletArtifactPolicyTest { + private val htmlHash = "11".repeat(32) + private val index = PathTag("/index.html", htmlHash) + + @Test + fun computesIdentityFromTheSignedSingleIndexPath() { + assertEquals( + SiteAggregateHash.compute(listOf(index)), + NappletArtifactPolicy.verifiedAggregateHash(listOf(index), null), + ) + } + + @Test + fun acceptsMatchingDeclaredIdentityCaseInsensitively() { + val computed = SiteAggregateHash.compute(listOf(index)) + assertEquals(computed, NappletArtifactPolicy.verifiedAggregateHash(listOf(index), computed.uppercase())) + } + + @Test + fun rejectsDriftedOrNonSelfContainedArtifacts() { + assertNull(NappletArtifactPolicy.verifiedAggregateHash(listOf(index), "22".repeat(32))) + assertNull(NappletArtifactPolicy.verifiedAggregateHash(listOf(PathTag("index.html", htmlHash)), null)) + assertNull(NappletArtifactPolicy.verifiedAggregateHash(listOf(index, PathTag("/app.js", "22".repeat(32))), null)) + assertNull(NappletArtifactPolicy.verifiedAggregateHash(listOf(PathTag("/index.html", "not-a-sha256")), null)) + } +} diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletBrokerTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletBrokerTest.kt index 0b5bcca09f..fc59bb7f87 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletBrokerTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletBrokerTest.kt @@ -182,9 +182,18 @@ class NappletBrokerTest { relay: NappletRelayGateway? = null, storage: NappletStorage? = null, wallet: NappletWalletGateway? = null, + resource: NappletResourceGateway? = null, signer: NostrSigner = this.signer, ledger: NappletPermissionLedger = NappletPermissionLedger(InMemoryNappletPermissionStore()), - ) = NappletBroker(signer, ledger, prompt, relay, storage, wallet) + ) = NappletBroker( + signer = signer, + ledger = ledger, + consentPrompt = prompt, + relay = relay, + storage = storage, + wallet = wallet, + resource = resource, + ) @Test fun getPublicKeyReturnsTheUsersKeyWhenAllowed() = @@ -545,8 +554,8 @@ class NappletBrokerTest { assertEquals(NappletResponse.Done, broker.handle(applet, NappletRequest.StorageSet("k", "v"), allDeclared)) assertEquals(NappletResponse.StorageValue("v"), broker.handle(applet, NappletRequest.StorageGet("k"), allDeclared)) - // Stored under the applet coordinate, never a shared namespace. - assertEquals("v", storage.data["${applet.coordinate}::k"]) + // Shared storage is scoped to the verified artifact identity, not just the d-tag. + assertEquals("v", storage.data["${applet.storageCoordinate}::k"]) assertEquals(NappletResponse.Done, broker.handle(applet, NappletRequest.StorageRemove("k"), allDeclared)) assertEquals(NappletResponse.StorageValue(null), broker.handle(applet, NappletRequest.StorageGet("k"), allDeclared)) @@ -563,7 +572,7 @@ class NappletBrokerTest { broker.handle(applet, NappletRequest.StorageSet("b", "2"), allDeclared) broker.handle(other, NappletRequest.StorageSet("c", "3"), allDeclared) - val response = broker.handle(applet, NappletRequest.StorageKeys, allDeclared) + val response = broker.handle(applet, NappletRequest.StorageKeys(), allDeclared) assertIs(response) assertEquals(setOf("a", "b"), response.values.toSet()) // never sees the other applet's "c" } @@ -643,19 +652,6 @@ class NappletBrokerTest { assertIs(response) } - @Test - fun shellSupportsReflectsDeclaredCapabilitiesWithoutConsent() = - runTest { - // The DENY prompt would block anything that reached consent; supports must not. - val broker = broker(ScriptedPrompt(GrantState.DENY)) - val declared = setOf(NappletCapability.RELAY) - - assertEquals(NappletResponse.Supported(true), broker.handle(applet, NappletRequest.ShellSupports("relay"), declared)) - assertEquals(NappletResponse.Supported(false), broker.handle(applet, NappletRequest.ShellSupports("storage"), declared)) - // Unknown/unbrokered domain. - assertEquals(NappletResponse.Supported(false), broker.handle(applet, NappletRequest.ShellSupports("cvm"), declared)) - } - @Test fun identityReadReturnsGatewayJsonOrUnsupported() = runTest { @@ -693,4 +689,60 @@ class NappletBrokerTest { assertIs(broker.handle(applet, NappletRequest.ResourceBytes("https://x"), allDeclared)) assertIs(broker.handle(applet, NappletRequest.UploadBlob(ByteArray(0), "image/png"), allDeclared)) } + + @Test + fun resourceInfoAndTypedFailuresFollowTheCurrentNapContract() = + runTest { + val resource = + NappletResourceGateway { _, _ -> + NappletResourceResult.Failure("blocked-by-policy", "private target") + } + val broker = broker(ScriptedPrompt(GrantState.ALLOW_ALWAYS), resource = resource) + + val info = broker.handle(applet, NappletRequest.ResourceInfo, allDeclared) + assertIs(info) + assertEquals(listOf("data", "https", "blossom", "nostr"), info.schemes) + assertEquals(10L * 1024L * 1024L, info.maxBytes) + + assertEquals( + NappletResponse.ResourceFailure("blocked-by-policy", "private target"), + broker.handle(applet, NappletRequest.ResourceBytes("https://internal.example"), allDeclared), + ) + } + + @Test + fun resourceBytesManyPreservesOrderAndSiblingResults() = + runTest { + val resource = + NappletResourceGateway { url, _ -> + if (url.endsWith("ok")) { + NappletResourceResult.Success(NappletResource("ok".encodeToByteArray(), "text/plain")) + } else { + NappletResourceResult.Failure("not-found") + } + } + val response = + broker(ScriptedPrompt(GrantState.ALLOW_ALWAYS), resource = resource) + .handle(applet, NappletRequest.ResourceBytesMany(listOf("https://x/ok", "https://x/missing")), allDeclared) + + assertIs(response) + assertEquals(listOf("https://x/ok", "https://x/missing"), response.items.map { it.url }) + assertIs(response.items[0].resource) + assertEquals("not-found", response.items[1].error) + } + + @Test + fun resourceBytesManyRejectsInvalidBulkSizesWithNapErrorCodes() = + runTest { + val resource = NappletResourceGateway { _, _ -> error("invalid bulk must not fetch") } + val broker = broker(ScriptedPrompt(GrantState.ALLOW_ALWAYS), resource = resource) + + val empty = broker.handle(applet, NappletRequest.ResourceBytesMany(emptyList()), allDeclared) + val tooLarge = broker.handle(applet, NappletRequest.ResourceBytesMany(List(17) { "data:,x" }), allDeclared) + + assertIs(empty) + assertEquals("invalid-request", empty.error) + assertIs(tooLarge) + assertEquals("too-large", tooLarge.error) + } } diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletCapabilityTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletCapabilityTest.kt index 4894ab6951..52cf819efd 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletCapabilityTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletCapabilityTest.kt @@ -28,18 +28,12 @@ import kotlin.test.assertTrue class NappletCapabilityTest { @Test - fun mapsKnownDomainsCaseInsensitively() { - assertEquals(NappletCapability.SHELL, NappletCapability.fromNapDomain("shell")) + fun mapsOnlyConformingDomainsExactly() { assertEquals(NappletCapability.IDENTITY, NappletCapability.fromNapDomain("identity")) - assertEquals(NappletCapability.KEYS, NappletCapability.fromNapDomain("keys")) - assertEquals(NappletCapability.RELAY, NappletCapability.fromNapDomain("Relay")) - assertEquals(NappletCapability.VALUE, NappletCapability.fromNapDomain("value")) - assertEquals(NappletCapability.STORAGE, NappletCapability.fromNapDomain(" STORAGE ")) + assertEquals(NappletCapability.RELAY, NappletCapability.fromNapDomain("relay")) + assertEquals(NappletCapability.STORAGE, NappletCapability.fromNapDomain("storage")) assertEquals(NappletCapability.RESOURCE, NappletCapability.fromNapDomain("resource")) - assertEquals(NappletCapability.UPLOAD, NappletCapability.fromNapDomain("upload")) assertEquals(NappletCapability.THEME, NappletCapability.fromNapDomain("theme")) - assertEquals(NappletCapability.NOTIFY, NappletCapability.fromNapDomain("notify")) - assertEquals(NappletCapability.INC, NappletCapability.fromNapDomain("inc")) } @Test @@ -48,6 +42,14 @@ class NappletCapabilityTest { assertNull(NappletCapability.fromNapDomain("intent")) assertNull(NappletCapability.fromNapDomain("cvm")) assertNull(NappletCapability.fromNapDomain("filesystem")) + assertNull(NappletCapability.fromNapDomain("shell")) + assertNull(NappletCapability.fromNapDomain("keys")) + assertNull(NappletCapability.fromNapDomain("value")) + assertNull(NappletCapability.fromNapDomain("upload")) + assertNull(NappletCapability.fromNapDomain("notify")) + assertNull(NappletCapability.fromNapDomain("inc")) + assertNull(NappletCapability.fromNapDomain("Relay")) + assertNull(NappletCapability.fromNapDomain(" storage ")) assertNull(NappletCapability.fromNapDomain("")) } @@ -56,10 +58,10 @@ class NappletCapabilityTest { val resolved = resolveRequiredCapabilities(listOf("identity", "relay", "intent", "value")) assertEquals( - setOf(NappletCapability.IDENTITY, NappletCapability.RELAY, NappletCapability.VALUE), + setOf(NappletCapability.IDENTITY, NappletCapability.RELAY), resolved.capabilities, ) - assertEquals(listOf(UnknownNapDomain("intent")), resolved.unknown) + assertEquals(listOf(UnknownNapDomain("intent"), UnknownNapDomain("value")), resolved.unknown) assertTrue(resolved.hasUnknown) } diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletWebContractTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletWebContractTest.kt new file mode 100644 index 0000000000..656ebcc1a6 --- /dev/null +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletWebContractTest.kt @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.napplet + +import kotlinx.coroutines.test.runTest +import kotlin.test.Test +import kotlin.test.assertContains +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +class NappletWebContractTest { + @Test + fun lockedPreludeIsTheFirstHeadContentAndRunsBeforeAuthoredCode() { + val authored = "" + val injected = + NappletWebContract + .injectPrelude( + html = authored.encodeToByteArray(), + shimJs = "window.__shimRan=true;", + declaredDomains = listOf("identity", "relay", "shell", "Relay", "bad\"domain", "relay"), + locked = true, + ).decodeToString() + + val head = injected.indexOf("") + "".length + val csp = injected.indexOf(", @@ -89,7 +83,7 @@ object NappletRequestRouter { declared: Set, payload: String, ): Outcome { - val requestType = runCatching { NappletProtocolJson.readType(payload) }.getOrNull() ?: "napplet" + val requestType = runCatching { NappletProtocolJson.readType(payload) }.getOrNull() ?: return Outcome.Ignore // Fire-and-forget edge ops that never reach the broker. when (requestType) { @@ -97,25 +91,9 @@ object NappletRequestRouter { val subId = runCatching { NappletProtocolJson.readSubId(payload) }.getOrNull() return if (subId != null) Outcome.CloseSubscription(subId) else Outcome.Ignore } - "resource.cancel" -> - return Outcome.Reply(NappletProtocolJson.encodeResponse(requestType, NappletResponse.Done)) - // identity.watch/unwatch are a push subscription (like relay.subscribe), gated on the - // IDENTITY declaration directly — the actual pubkey stream is the host's job. The - // ledger still gets a say: this bypasses NappletBroker.handle, so without an explicit - // check a standing IDENTITY denial would not stop the pushes (and would keep leaking - // account-switch timing and every npub the user rotates between). - "identity.watch" -> - return if (NappletCapability.IDENTITY in declared && - !broker.isDenied(identity, NappletCapability.IDENTITY) - ) { - Outcome.WatchIdentity - } else { - Outcome.Ignore - } - "identity.unwatch" -> - return Outcome.UnwatchIdentity + "resource.cancel" -> return Outcome.Ignore // inc bus: a topic pub/sub between napplets/services, authorized on the INC declaration - // alone (like identity.watch) — no per-call consent. The host owns the cross-session fan-out. + // alone. The host owns the cross-session fan-out. "inc.subscribe" -> { val topic = runCatching { NappletProtocolJson.readTopic(payload) }.getOrNull() return if (topic != null && NappletCapability.INC in declared) Outcome.SubscribeInc(topic) else Outcome.Ignore @@ -136,7 +114,7 @@ object NappletRequestRouter { val request = runCatching { NappletProtocolJson.decodeRequest(payload) }.getOrNull() - ?: return Outcome.Reply(NappletProtocolJson.encodeResponse(requestType, NappletResponse.Failed("Malformed or unsupported request."))) + ?: return Outcome.Ignore val response = broker.handle(identity, request, declared) diff --git a/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/napplet/protocol/NappletProtocolJson.kt b/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/napplet/protocol/NappletProtocolJson.kt index 4a1c52243b..1df933a9e7 100644 --- a/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/napplet/protocol/NappletProtocolJson.kt +++ b/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/napplet/protocol/NappletProtocolJson.kt @@ -26,6 +26,7 @@ import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonNull import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.add +import kotlinx.serialization.json.addJsonObject import kotlinx.serialization.json.buildJsonArray import kotlinx.serialization.json.buildJsonObject import kotlinx.serialization.json.int @@ -34,6 +35,7 @@ import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive import kotlinx.serialization.json.long import kotlinx.serialization.json.put +import kotlinx.serialization.json.putJsonArray import kotlinx.serialization.json.putJsonObject import java.util.Base64 @@ -62,7 +64,7 @@ object NappletProtocolJson { /** The `subId` of a subscription request, used to key the `relay.event`/`relay.eose` pushes back to it. */ fun readSubId(envelopeJson: String): String? = json.parseToJsonElement(envelopeJson).jsonObject.str("subId") - /** A `relay.event` push: delivers one matching [event] to the subscription [subId] (no request id). */ + /** A `relay.event` push carrying the NAP-RELAY `RelayEventResult` wrapper. */ fun encodeRelayEvent( subId: String, event: Event, @@ -70,7 +72,9 @@ object NappletProtocolJson { buildJsonObject { put("type", "relay.event") put("subId", subId) - put("event", json.parseToJsonElement(event.toJson())) + putJsonObject("result") { + put("event", json.parseToJsonElement(event.toJson())) + } }.toString() /** A `relay.eose` push: signals end-of-stored-events for the subscription [subId]. */ @@ -130,29 +134,10 @@ object NappletProtocolJson { put("reason", reason) }.toString() - /** - * The `shell.init` handshake reply (`@napplet/core`): the capability environment the napplet - * caches and answers `shell.supports()` from. [domains] is the set of NAP domains this shell - * will broker for the applet; [services] mirrors them. We don't advertise numbered protocols. - */ - fun encodeShellInit( - domains: List, - services: List, - ): String = - buildJsonObject { - put("type", "shell.init") - putJsonObject("capabilities") { - put("domains", buildJsonArray { domains.forEach { add(it) } }) - putJsonObject("protocols") {} - } - put("services", buildJsonArray { services.forEach { add(it) } }) - }.toString() - /** Parses a request envelope. Returns `null` for an unrecognized `type` so the broker can deny it. */ fun decodeRequest(envelopeJson: String): NappletRequest? { val o = json.parseToJsonElement(envelopeJson).jsonObject return when (o.str("type")) { - "shell.supports" -> NappletRequest.ShellSupports(o.req("domain"), o.str("protocol")) "theme.get" -> NappletRequest.ThemeGet "identity.getPublicKey" -> NappletRequest.GetPublicKey "relay.publish" -> { @@ -181,10 +166,10 @@ object NappletProtocolJson { createdAt = t["created_at"]?.jsonPrimitive?.long ?: (System.currentTimeMillis() / 1000), ) } - "storage.get" -> NappletRequest.StorageGet(o.req("key")) - "storage.set" -> NappletRequest.StorageSet(o.req("key"), o.req("value")) - "storage.remove" -> NappletRequest.StorageRemove(o.req("key")) - "storage.keys" -> NappletRequest.StorageKeys + "storage.get" -> NappletRequest.StorageGet(o.req("key"), o.storageScope()) + "storage.set" -> NappletRequest.StorageSet(o.req("key"), o.req("value"), o.storageScope()) + "storage.remove" -> NappletRequest.StorageRemove(o.req("key"), o.storageScope()) + "storage.keys" -> NappletRequest.StorageKeys(o.storageScope()) "notify.create" -> NappletRequest.NotifyCreate(o.str("title") ?: "", o.str("body") ?: "") "notify.list" -> NappletRequest.NotifyList "notify.dismiss" -> NappletRequest.NotifyDismiss(o.str("notificationId") ?: o.str("id") ?: "") @@ -194,7 +179,9 @@ object NappletProtocolJson { } "keys.unregisterAction" -> NappletRequest.UnregisterAction(o.req("actionId")) "value.payInvoice" -> NappletRequest.PayInvoice(o.req("invoice")) + "resource.info" -> NappletRequest.ResourceInfo "resource.bytes" -> NappletRequest.ResourceBytes(o.req("url")) + "resource.bytesMany" -> NappletRequest.ResourceBytesMany(o.getValue("urls").jsonArray.map { it.jsonPrimitive.content }) "upload.upload" -> { // UploadUploadMessage: { type, id, request: { data, mimeType?, filename?, ... } }. // The Blob in `request.data` is inlined as base64 `request.dataBase64` by shell.html. @@ -209,7 +196,7 @@ object NappletProtocolJson { // Any other identity.* read (getProfile/getRelays/getFollows/getList/...) routes through // a generic IdentityRead; the broker/gateway decides which are implemented. val type = o.str("type") - if (type != null && type.startsWith("identity.")) { + if (type != null && type in IDENTITY_READ_TYPES) { NappletRequest.IdentityRead(type.removePrefix("identity."), o.str("listType") ?: o.str("argument")) } else { null @@ -230,6 +217,7 @@ object NappletProtocolJson { when (response) { is NappletResponse.NotifyCreated -> "notify.created" is NappletResponse.NotifyListed -> "notify.listed" + is NappletResponse.ResourceFailure -> "$requestType.error" else -> "$requestType.result" } put("type", responseType) @@ -247,11 +235,11 @@ object NappletProtocolJson { } is NappletResponse.Events -> { put("ok", true) - put("events", buildJsonArray { response.events.forEach { add(json.parseToJsonElement(it.toJson())) } }) - } - is NappletResponse.Supported -> { - put("ok", true) - put("supported", response.supported) + putJsonArray("events") { + response.events.forEach { event -> + addJsonObject { put("event", json.parseToJsonElement(event.toJson())) } + } + } } is NappletResponse.ActionRegistered -> { put("ok", true) @@ -279,6 +267,42 @@ object NappletProtocolJson { put("bytes", Base64.getEncoder().encodeToString(response.bytes)) put("mime", response.contentType) } + is NappletResponse.ResourceInfo -> { + put("ok", true) + putJsonObject("info") { + putJsonArray("schemes") { + response.schemes.forEach { scheme -> + addJsonObject { + put("scheme", scheme) + put("enabled", true) + } + } + } + put("maxBytes", response.maxBytes) + put("maxUrls", response.maxUrls) + } + } + is NappletResponse.ResourceItems -> { + put("ok", true) + putJsonArray("items") { + response.items.forEach { item -> + addJsonObject { + put("url", item.url) + put("ok", item.resource != null) + item.resource?.let { + put("bytes", Base64.getEncoder().encodeToString(it.bytes)) + put("mime", it.contentType) + } + item.error?.let { put("error", it) } + item.message?.let { put("message", it) } + } + } + } + } + is NappletResponse.ResourceFailure -> { + put("error", response.error) + response.message?.let { put("message", it) } + } is NappletResponse.Uploaded -> { // UploadResult: { ok, uploadId, status, url?, sha256?, size?, mimeType?, ... }. put("ok", true) @@ -397,6 +421,8 @@ object NappletProtocolJson { private fun JsonObject.kindOf(): Int = getValue("kind").jsonPrimitive.int + private fun JsonObject.storageScope(): NappletStorageScope = if (str("scope") == "instance") NappletStorageScope.INSTANCE else NappletStorageScope.SHARED + /** The result field a given identity read returns, matching `@napplet/nap` identity message types. */ private fun identityResultField(requestType: String): String = when (requestType) { @@ -408,4 +434,16 @@ object NappletProtocolJson { "identity.getBadges" -> "badges" else -> "result" } + + private val IDENTITY_READ_TYPES = + setOf( + "identity.getRelays", + "identity.getProfile", + "identity.getFollows", + "identity.getList", + "identity.getZaps", + "identity.getMutes", + "identity.getBlocked", + "identity.getBadges", + ) } diff --git a/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletRequestRouterTest.kt b/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletRequestRouterTest.kt index 94e3e4c763..eca9000437 100644 --- a/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletRequestRouterTest.kt +++ b/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletRequestRouterTest.kt @@ -85,11 +85,12 @@ class NappletRequestRouterTest { } @Test - fun resourceCancelRepliesDone() = + fun resourceCancelIsSilentlyHandled() = runTest { - val outcome = route("""{"type":"resource.cancel"}""") - assertIs(outcome) - assertTrue(outcome.payload.contains("resource.cancel.result")) + assertEquals( + NappletRequestRouter.Outcome.Ignore, + route("""{"type":"resource.cancel"}"""), + ) } @Test @@ -117,11 +118,11 @@ class NappletRequestRouterTest { } @Test - fun malformedRequestRepliesFailed() = + fun unknownAndMalformedRequestsAreSilentlyIgnored() = runTest { - val outcome = route("""{"type":"totally.unknown"}""") - assertIs(outcome) - assertTrue(outcome.payload.contains("failed")) + assertEquals(NappletRequestRouter.Outcome.Ignore, route("""{"type":"totally.unknown"}""")) + assertEquals(NappletRequestRouter.Outcome.Ignore, route("not json")) + assertEquals(NappletRequestRouter.Outcome.Ignore, route("""{"type":"relay.publish"}""")) } @Test @@ -153,29 +154,9 @@ class NappletRequestRouterTest { } @Test - fun identityWatchWhenDeclaredBecomesWatchIdentity() = + fun removedIdentityWatchMessagesAreIgnored() = runTest { - assertEquals( - NappletRequestRouter.Outcome.WatchIdentity, - NappletRequestRouter.route(broker(), applet, allDeclared, """{"type":"identity.watch"}"""), - ) - } - - @Test - fun identityWatchWithoutDeclarationIsIgnored() = - runTest { - assertEquals( - NappletRequestRouter.Outcome.Ignore, - NappletRequestRouter.route(broker(), applet, emptySet(), """{"type":"identity.watch"}"""), - ) - } - - @Test - fun identityUnwatchBecomesUnwatchIdentity() = - runTest { - assertEquals( - NappletRequestRouter.Outcome.UnwatchIdentity, - NappletRequestRouter.route(broker(), applet, emptySet(), """{"type":"identity.unwatch"}"""), - ) + assertEquals(NappletRequestRouter.Outcome.Ignore, route("""{"type":"identity.watch"}""")) + assertEquals(NappletRequestRouter.Outcome.Ignore, route("""{"type":"identity.unwatch"}""")) } } diff --git a/nappletHost/src/main/kotlin/com/vitorpamplona/amethyst/napplethost/NappletContentServer.kt b/nappletHost/src/main/kotlin/com/vitorpamplona/amethyst/napplethost/NappletContentServer.kt index 7f32a39474..b7186df770 100644 --- a/nappletHost/src/main/kotlin/com/vitorpamplona/amethyst/napplethost/NappletContentServer.kt +++ b/nappletHost/src/main/kotlin/com/vitorpamplona/amethyst/napplethost/NappletContentServer.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.amethyst.napplethost +import android.util.Base64 import android.webkit.WebResourceRequest import android.webkit.WebResourceResponse import com.vitorpamplona.amethyst.commons.napplet.NappletWebContract @@ -60,6 +61,9 @@ class NappletContentServer( // The host posture: a WEBSITE nSite is a normal web app — NIP-07 window.nostr provider, normal // network (no app CSP; off-origin requests defer to the WebView), unlike a locked NAPPLET. private val profile: HostProfile = HostProfile.NAPPLET, + // Exact NAP domains the trusted main process authorized for this launch. The injected prelude + // projects only these objects; absence is the NIP-5D capability-availability signal. + private val declaredDomains: List = emptyList(), // Embedded surfaces (a windowless Service) can't host the soft keyboard, so the shim installs the // IME proxy agent that relays the focused field to the host's keyboard. The full-screen Activity // host has a native keyboard and leaves this false. @@ -130,15 +134,44 @@ class NappletContentServer( } private fun serveShell(): WebResourceResponse { - // The shell HTML carries an APP_ORIGIN_PLACEHOLDER for the iframe src; bind it to this applet's - // origin so the shell frames exactly this applet (and the CSP frame-src is pinned to it too). - val html = shellHtmlBytes.decodeToString().replace(NappletWebContract.APP_ORIGIN_PLACEHOLDER, appOrigin).encodeToByteArray() + val sandbox: String + val frameSource: String + val bootstrap: String + + if (profile == HostProfile.NAPPLET) { + // NIP-5D identity is bound to the exact verified bytes that execute. Resolve the sole + // /index.html blob, inject host-owned policy/prelude outside its aggregate hash, then + // hand those bytes to the opaque-origin child via srcdoc (never a navigated src). + val resolution = resolveCacheFirst("/index.html") + if (resolution !is StaticSiteResolution.Resolved) return notFound() + val encoded = Base64.encodeToString(injectShim(resolution.bytes), Base64.NO_WRAP) + sandbox = "allow-scripts" + frameSource = "'self'" + bootstrap = + "var b=atob('$encoded'),u=new Uint8Array(b.length);" + + "for(var j=0;jwindow.__nappletNip07=true;" else "" - // Embedded surface: turn on the IME proxy agent (set before the shim runs). - val imeFlag = if (imeProxy) "" else "" - val script = "$style$nip07Flag$imeFlag" - val headIdx = text.indexOf("= 0 -> { - val close = text.indexOf('>', headIdx) - if (close >= 0) text.substring(0, close + 1) + script + text.substring(close + 1) else script + text - } - else -> script + text - } - return injected.encodeToByteArray() - } + /** Inserts host policy and the explicit-domain `window.napplet` prelude before authored code. */ + private fun injectShim(html: ByteArray): ByteArray = + NappletWebContract.injectPrelude( + html = html, + shimJs = shimJs, + declaredDomains = declaredDomains, + locked = profile == HostProfile.NAPPLET, + injectNip07 = profile.injectsNip07, + imeProxy = imeProxy, + ) private fun notFound(): WebResourceResponse = WebResourceResponse("text/plain", "utf-8", 404, "Not Found", emptyMap(), ByteArrayInputStream(ByteArray(0))) diff --git a/nappletHost/src/main/kotlin/com/vitorpamplona/amethyst/napplethost/NappletHostActivity.kt b/nappletHost/src/main/kotlin/com/vitorpamplona/amethyst/napplethost/NappletHostActivity.kt index d061653f72..74c3c6aaac 100644 --- a/nappletHost/src/main/kotlin/com/vitorpamplona/amethyst/napplethost/NappletHostActivity.kt +++ b/nappletHost/src/main/kotlin/com/vitorpamplona/amethyst/napplethost/NappletHostActivity.kt @@ -67,7 +67,6 @@ import androidx.webkit.WebViewCompat import androidx.webkit.WebViewFeature import com.vitorpamplona.amethyst.commons.napplet.NappletWebContract import com.vitorpamplona.amethyst.commons.napplet.protocol.NappletProtocolJson -import com.vitorpamplona.amethyst.commons.napplet.resolveRequiredCapabilities import com.vitorpamplona.amethyst.napplethost.R import com.vitorpamplona.quartz.nip01Core.core.toHexKey import com.vitorpamplona.quartz.nip5aStaticWebsites.resolver.StaticSiteResolution @@ -243,7 +242,7 @@ class NappletHostActivity : ComponentActivity() { // "Open web" for a site makes everything direct — both its blob fetches (here) and its live // web traffic (the WebView proxy, below). Tor (the default) routes both through the SOCKS port. val effectiveProxy = if (useTor) proxyPort else -1 - contentServer = NappletContentServer(paths, servers, effectiveProxy, cacheDir, shellHtml, shim, appOrigin, profile) + contentServer = NappletContentServer(paths, servers, effectiveProxy, cacheDir, shellHtml, shim, appOrigin, profile, declaredDomains) // Create + warm the WebView NOW so its (slow, first-in-process) Chromium init runs on the main // thread concurrently with the index probe below (which runs on IO) — instead of serially after @@ -473,10 +472,10 @@ class NappletHostActivity : ComponentActivity() { webViewProfile = intent.getStringExtra(NappletHostContract.EXTRA_WEBVIEW_PROFILE) val requires = intent.getStringArrayListExtra(NappletHostContract.EXTRA_REQUIRES) ?: emptyList() - val resolved = resolveRequiredCapabilities(requires) - // shell is always available; the rest are the declared domains advertised to the applet in the - // handshake. (The broker enforces the authoritative set from the launch token, not this list.) - declaredDomains = (listOf("shell") + resolved.capabilities.map { it.name.lowercase() }).distinct() + val resolved = profile.declaredCapabilities(requires) + // Domain-object presence is the NIP-5D availability signal. The broker still enforces the + // authoritative set minted into the launch token in the main process. + declaredDomains = resolved.map { it.name.lowercase() }.distinct() return author.isNotEmpty() && launchToken.isNotEmpty() } @@ -677,13 +676,6 @@ class NappletHostActivity : ComponentActivity() { // correlate on its id. The broker reads `type` to decode and to build the .result reply. val envelope = runCatching { JSONObject(raw) }.getOrNull() ?: return - // Shell handshake: the SDK posts `shell.ready` (no id) and answers shell.supports() locally - // from the `shell.init` environment we send back here. - if (envelope.optString("type") == "shell.ready") { - runCatching { replyProxy.postMessage(NappletProtocolJson.encodeShellInit(declaredDomains, declaredDomains)) } - return - } - // Unbind a keyboard action as soon as the applet drops it (the broker's Done reply carries no // actionId, so the binding is removed here from the envelope itself). if (envelope.optString("type") == "keys.unregisterAction") { diff --git a/nappletHost/src/main/kotlin/com/vitorpamplona/amethyst/napplethost/NappletHostService.kt b/nappletHost/src/main/kotlin/com/vitorpamplona/amethyst/napplethost/NappletHostService.kt index ceaf53b6e2..37dd512352 100644 --- a/nappletHost/src/main/kotlin/com/vitorpamplona/amethyst/napplethost/NappletHostService.kt +++ b/nappletHost/src/main/kotlin/com/vitorpamplona/amethyst/napplethost/NappletHostService.kt @@ -54,8 +54,6 @@ import androidx.webkit.WebMessageCompat import androidx.webkit.WebViewCompat import androidx.webkit.WebViewFeature import com.vitorpamplona.amethyst.commons.napplet.NappletWebContract -import com.vitorpamplona.amethyst.commons.napplet.protocol.NappletProtocolJson -import com.vitorpamplona.amethyst.commons.napplet.resolveRequiredCapabilities import com.vitorpamplona.quartz.nip01Core.core.toHexKey import com.vitorpamplona.quartz.nip5aStaticWebsites.tags.PathTag import com.vitorpamplona.quartz.utils.sha256.sha256 @@ -201,7 +199,8 @@ class NappletHostService : Service() { if (author.isEmpty() || launchToken.isEmpty()) return null val requires = data.getStringArrayList(NappletHostContract.EXTRA_REQUIRES) ?: emptyList() - val declaredDomains = (listOf("shell") + resolveRequiredCapabilities(requires).capabilities.map { it.name.lowercase() }).distinct() + val profile = HostProfile.fromName(data.getString(NappletHostContract.EXTRA_HOST_PROFILE)) + val declaredDomains = profile.declaredCapabilities(requires).map { it.name.lowercase() }.distinct() val tab = NappletTab( @@ -212,7 +211,7 @@ class NappletHostService : Service() { author = author, identifier = data.getString(NappletHostContract.EXTRA_IDENTIFIER).orEmpty(), launchToken = launchToken, - profile = HostProfile.fromName(data.getString(NappletHostContract.EXTRA_HOST_PROFILE)), + profile = profile, useTor = data.getBoolean(NappletHostContract.EXTRA_USE_TOR, true), proxyPort = data.getInt(NappletHostContract.EXTRA_PROXY_PORT, -1), bgColor = data.getInt(NappletHostContract.EXTRA_BG_COLOR, android.graphics.Color.WHITE), @@ -300,7 +299,19 @@ class NappletHostService : Service() { NappletWebViewProfile.apply(context, wv, tab.webViewProfile) val appOrigin = NappletWebContract.appOrigin(deriveAppId(tab.author, tab.identifier)) val effectiveProxy = if (tab.useTor) tab.proxyPort else -1 - tab.contentServer = NappletContentServer(tab.paths, tab.servers, effectiveProxy, cacheDir, shellHtml, shimJs, appOrigin, tab.profile, imeProxy = true) + tab.contentServer = + NappletContentServer( + tab.paths, + tab.servers, + effectiveProxy, + cacheDir, + shellHtml, + shimJs, + appOrigin, + tab.profile, + tab.declaredDomains, + imeProxy = true, + ) hardenWebView(wv, tab) // Theme the pre-load background so the shell/app loading shows Amethyst's background, not white. @@ -468,11 +479,6 @@ class NappletHostService : Service() { val raw = message.data ?: return val envelope = runCatching { JSONObject(raw) }.getOrNull() ?: return - if (envelope.optString("type") == "shell.ready") { - runCatching { replyProxy.postMessage(NappletProtocolJson.encodeShellInit(tab.declaredDomains, tab.declaredDomains)) } - return - } - // IME events aren't brokered — the main app hosts the keyboard. Relay the envelope to the client. if (envelope.optString("type").startsWith("ime.")) { val reply = From 129401bdaf5f51025e3b89409199fd0fd043be7f Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 21:36:22 +0000 Subject: [PATCH 027/132] test(relay): characterize Yggdrasil/IPv6 relay handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assesses how the app fares when relays live on an Yggdrasil overlay, where every relay is a bracketed IPv6 literal in 0200::/7 served over plain ws:// (no DNS, no CA-issuable certificate). The happy path works: a hand-typed ws://[...]:port normalizes, survives the RFC 3986 pass and is dialed by OkHttp; nothing in the stack is IPv4-only and cleartext is already permitted globally. Four gaps are pinned by the new characterization tests: 1. RelayUrlNormalizer folds hex case but not zero-compression, so two legal spellings of one address yield two NormalizedRelayUrl values while OkHttp collapses them to one host — duplicate sockets, REQs and stat entries. 2. isLocalHost() does not know 0200::/7, so a schemeless literal defaults to wss:// and can only fail its TLS handshake. 3. An unbracketed literal (what yggdrasilctl getSelf prints) is rejected, and RelayUrlEditField.submitRelay has no else branch — the Add button silently does nothing. 4. TorRelayEvaluation classifies mesh relays as "new", so with Tor on they are dialed through the SOCKS proxy, which cannot route 0200::/7. No behavior is changed. quartz/plans/2026-08-04-yggdrasil-ipv6-relays.md records the full assessment, the NIP-65/outbox propagation consequences of publishing a key-derived mesh address, and what could not be verified here (the analysis container has no IPv6 stack, so nothing below the socket was exercised). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DQr8CDsznzCRUeB5tS8VYk --- .../commons/tor/YggdrasilTorRoutingTest.kt | 65 ++++++++++ .../plans/2026-08-04-yggdrasil-ipv6-relays.md | 103 ++++++++++++++++ .../YggdrasilCompatCharacterizationTest.kt | 116 ++++++++++++++++++ 3 files changed, 284 insertions(+) create mode 100644 commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/tor/YggdrasilTorRoutingTest.kt create mode 100644 quartz/plans/2026-08-04-yggdrasil-ipv6-relays.md create mode 100644 quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/YggdrasilCompatCharacterizationTest.kt diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/tor/YggdrasilTorRoutingTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/tor/YggdrasilTorRoutingTest.kt new file mode 100644 index 0000000000..68a8941821 --- /dev/null +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/tor/YggdrasilTorRoutingTest.kt @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.tor + +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +/** + * GAP 4 — an Yggdrasil relay is classified as a plain clearnet relay, so with Tor on it is + * dialed through the SOCKS proxy. Tor cannot route `0200::/7`: the connection can only fail. + * + * Compare `ws://192.168.1.100:8080/`, which [TorRelayEvaluation] correctly keeps off Tor + * because `isLocalHost()` recognizes the LAN prefix. Yggdrasil has no such recognition. + */ +class YggdrasilTorRoutingTest { + private val yggdrasilRelay = NormalizedRelayUrl("ws://[201:d0e:9ba5:8bbc::1]:8080/") + private val lanRelay = NormalizedRelayUrl("ws://192.168.1.100:8080/") + + private fun evaluation(newViaTor: Boolean) = + TorRelayEvaluation( + torSettings = + TorRelaySettings( + torType = TorType.INTERNAL, + onionRelaysViaTor = true, + dmRelaysViaTor = true, + newRelaysViaTor = newViaTor, + trustedRelaysViaTor = false, + moneyOperationsViaTor = false, + ), + trustedRelayList = emptySet(), + dmRelayList = emptySet(), + ) + + @Test + fun yggdrasilRelayIsSentThroughTorWhileLanRelayIsNot() { + val eval = evaluation(newViaTor = true) + assertTrue(eval.useTor(yggdrasilRelay), "Yggdrasil relay is routed via Tor, which cannot reach 0200::/7") + assertFalse(eval.useTor(lanRelay), "LAN relay is correctly kept off Tor") + } + + @Test + fun yggdrasilRelayWorksOnlyWhenNewRelaysViaTorIsOff() { + assertFalse(evaluation(newViaTor = false).useTor(yggdrasilRelay)) + } +} diff --git a/quartz/plans/2026-08-04-yggdrasil-ipv6-relays.md b/quartz/plans/2026-08-04-yggdrasil-ipv6-relays.md new file mode 100644 index 0000000000..f854bffaab --- /dev/null +++ b/quartz/plans/2026-08-04-yggdrasil-ipv6-relays.md @@ -0,0 +1,103 @@ +# Amethyst over Yggdrasil (IPv6 overlay) — compatibility assessment + +Status: **analysis only** — no behavior changed. Characterization tests landed alongside +this doc pin the current behavior so a fix has a baseline to diff against. + +## What Yggdrasil looks like to the app + +Yggdrasil is an encrypted end-to-end mesh. Every node gets an IPv6 address derived from its +public key inside `0200::/7`, and hands out `0300::/8` subnets. Consequences that matter here: + +- **No DNS.** A relay on the mesh is addressed as a bracketed IPv6 literal, always. +- **No certificates.** No CA issues for `0200::/7` literals, so relays run plain `ws://`. + This is not a downgrade — the overlay already provides end-to-end encryption and + authenticates the peer by its address. +- **On Android it is a `VpnService`**, so the app's default network becomes the VPN network. +- The address is a **stable node identifier**, so publishing it is equivalent to publishing + a long-lived pseudonymous handle for the device. + +## Verdict + +A hand-typed `ws://[…]:port` relay works end to end: it normalizes, survives the RFC 3986 +pass, and OkHttp parses and dials it. Nothing in the stack is IPv4-only, `TcpNoDelaySocketFactory` +is family-agnostic, and `network_security_config.xml` permits cleartext globally, so the +`ws://` requirement is already satisfied. + +Everything around that happy path is where it degrades. Four gaps, in severity order. + +### GAP 1 — one relay, two identities (correctness) + +`RelayUrlNormalizer` folds hex case but does **not** canonicalize zero-compression or +leading zeros: + +| input | `NormalizedRelayUrl` | OkHttp host | +|---|---|---| +| `ws://[201:d0e:9ba5:8bbc::1]:8080` | `ws://[201:d0e:9ba5:8bbc::1]:8080/` | `201:d0e:9ba5:8bbc::1` | +| `ws://[201:0d0e:9ba5:8bbc:0000:0000:0000:0001]:8080` | `ws://[201:0d0e:…:0001]:8080/` | `201:d0e:9ba5:8bbc::1` | + +OkHttp collapses both to one host; the app does not. `NormalizedRelayUrl` is the key of the +connection pool (`PoolRequests`, `RelayPool`), every relay-list set, the NIP-11 cache and the +per-relay stat maps — so the same relay written two ways gets **two sockets, two REQ sets and +doubled traffic**, and appears twice in the relay UI. This affects all IPv6 literals, but it +only bites Yggdrasil users in practice, because on the mesh a literal is the *only* way to +name a relay. Fix: canonicalize the bracketed literal to RFC 5952 inside `fix()`. + +### GAP 2 — schemeless entry defaults to `wss://` (dead end) + +`RelayUrlNormalizer.isLocalHost()` recognizes `127.0.0.1`, `localhost`, `//umbrel:`, +`192.168.`, `.local:` / `.local/`. An Yggdrasil address matches none of them, so a schemeless +`[201:…]:8080` falls through to the clearnet default and becomes `wss://[201:…]:8080/` — a +URL whose TLS handshake can never succeed. The user must know to type `ws://` themselves. +Fix: teach `isLocalHost()` (or a sibling `isOverlayNetwork()`) the `0200::/7` prefix. + +### GAP 3 — unbracketed literal is silently rejected (UX) + +`yggdrasilctl getSelf` prints the address **unbracketed**, which is exactly what a user +copies into the "add a relay" box. `isBareHostAndPath()` rejects it (correctly — it is +ambiguous with a scheme), so `normalizeOrNull` returns null. But +`RelayUrlEditField.submitRelay()` has no else branch: the Add button just does nothing, with +no error. Fix: either auto-bracket a candidate that parses as an IPv6 address, or surface a +validation message instead of a silent no-op. + +### GAP 4 — Tor routing sends mesh traffic into the SOCKS proxy (breaks the relay) + +`TorRelayEvaluation` classifies relays as localhost / onion / dm / trusted / new. Yggdrasil +lands in **new**, so with Tor on and the default "new relays via Tor", the relay is dialed +through the Tor SOCKS proxy — which cannot route `0200::/7`. The connection can only fail. +Compare `ws://192.168.1.100:8080/`, which is correctly kept off Tor because `isLocalHost()` +knows the LAN prefix. Today the only workaround is turning "new relays via Tor" off, which +weakens the setting for every genuine clearnet relay. The same gap exists on desktop +(`DesktopHttpClient`) and for non-relay HTTP (`RoleBasedHttpClientBuilder`). Fixing GAP 2's +prefix check fixes this one too, since both read the same predicate. + +## Propagation / privacy note (not a bug, a decision) + +An Yggdrasil relay is not filtered out of NIP-65 publishing (`AdvertisedRelayInfoTag` only +rejects localhost) nor out of the outbox model +(`RelayListRecommendationProcessor.filterValidRelays`). So a mesh relay in your relay list is +**published to public relays and recommended to other users**. Two effects: + +- Peers not on the mesh dial `[201:…]` and burn reconnect attempts on an unreachable host. +- Your Yggdrasil address — a stable, key-derived node identifier — becomes public. + +Onion relays get special handling here (`hasOnionConnection` gates whether they are even +considered). An overlay-network classification would let Yggdrasil be treated the same way. + +## Not covered + +- **No live socket test.** The analysis container has no IPv6 stack at all + (`AF_INET6` → `EAFNOSUPPORT`), so everything above is verified below the socket: URL + normalization, OkHttp URL/host parsing, and the Tor routing decision. An on-device run + against a real mesh relay is still needed to confirm the happy path end to end. +- **Android VPN interaction untested.** `ConnectivityFlow` uses + `registerDefaultNetworkCallback`, so it follows the app's default network into the VPN. + Whether `isMeteredOrMobileData()` reads correctly through Yggdrasil's `VpnService` depends + on whether that app declares underlying networks; worth checking on device before assuming + data-saving mode behaves. +- Media loading (Coil) and NIP-05 resolution against mesh hosts were not exercised. + +## Tests + +- `quartz/src/jvmAndroidTest/…/relay/YggdrasilCompatCharacterizationTest.kt` — GAPs 1–3 plus + the working happy path. +- `commons/src/commonTest/…/tor/YggdrasilTorRoutingTest.kt` — GAP 4. diff --git a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/YggdrasilCompatCharacterizationTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/YggdrasilCompatCharacterizationTest.kt new file mode 100644 index 0000000000..638858dbd8 --- /dev/null +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/YggdrasilCompatCharacterizationTest.kt @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay + +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.isLocalHost +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.toHttp +import okhttp3.Request +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * Characterization of how relay URLs on an Yggdrasil overlay behave today. + * + * Yggdrasil gives every node an IPv6 address inside `0200::/7` (nodes) and hands out + * `0300::/8` subnets, with no DNS and no CA-issuable certificate. A relay on the mesh is + * therefore always reached as a **bracketed IPv6 literal over plain `ws://`** — a shape + * the relay stack only partially handles. + * + * These tests document the CURRENT behavior (including the gaps) so a later fix has a + * baseline to diff against. Each gap is marked GAP with what a user sees. + */ +class YggdrasilCompatCharacterizationTest { + // Same node, three legal RFC 4291 spellings of one address. + private val canonical = "ws://[201:d0e:9ba5:8bbc::1]:8080" + private val expanded = "ws://[201:0d0e:9ba5:8bbc:0000:0000:0000:0001]:8080" + private val uppercase = "ws://[201:D0E:9BA5:8BBC::1]:8080" + + private fun host(url: String) = + Request + .Builder() + .url(url) + .build() + .url.host + + @Test + fun bracketedLiteralsSurviveNormalizationAndReachOkHttp() { + val n = canonical.normalizeRelayUrl() + assertEquals("ws://[201:d0e:9ba5:8bbc::1]:8080/", n.url) + assertEquals("201:d0e:9ba5:8bbc::1", host(n.url)) + // NIP-11 / relay-icon fetches derive their http url from the same string. + assertEquals("http://[201:d0e:9ba5:8bbc::1]:8080/", n.toHttp()) + } + + @Test + fun yggdrasilSubnetAddressesAndUppercaseHexWork() { + assertEquals("ws://[300:1b5d:d0e9:ba58::1]:4848/", "ws://[300:1b5d:d0e9:ba58::1]:4848".normalizeRelayUrl().url) + // Hex case IS folded, so the uppercase spelling collapses onto the canonical one. + assertEquals(canonical.normalizeRelayUrl(), uppercase.normalizeRelayUrl()) + } + + /** + * GAP 1 — zero-compression is NOT canonicalized, so one relay gets two identities. + * + * `NormalizedRelayUrl` is the key of the connection pool, the relay-list sets, the NIP-11 + * cache and every per-relay stat map. OkHttp collapses both spellings to one host (below), + * so the app opens two sockets to the same relay and counts it twice everywhere. + */ + @Test + fun gapZeroCompressionSplitsOneRelayIntoTwoIdentities() { + assertNotEquals(canonical.normalizeRelayUrl(), expanded.normalizeRelayUrl()) + // ...even though they are literally the same host on the wire: + assertEquals(host(canonical), host(expanded)) + } + + /** + * GAP 2 — a schemeless IPv6 literal defaults to `wss://`. + * + * `isLocalHost()` only knows 127.0.0.1 / localhost / umbrel / 192.168. / .local, so an + * Yggdrasil address falls through to the clearnet default. No CA issues certificates for + * `0200::/7` literals, so the resulting wss:// url can only ever fail its TLS handshake. + */ + @Test + fun gapSchemelessYggdrasilAddressDefaultsToWss() { + assertEquals("wss://[201:d0e:9ba5:8bbc::1]:8080/", "[201:d0e:9ba5:8bbc::1]:8080".normalizeRelayUrl().url) + assertFalse("ws://[201:d0e:9ba5:8bbc::1]:8080/".normalizeRelayUrl().isLocalHost()) + } + + /** + * GAP 3 — an unbracketed IPv6 literal is rejected outright. + * + * `yggdrasilctl getSelf` prints the address unbracketed, which is what a user copies into + * the "add a relay" field. Normalization returns null and `RelayUrlEditField.submitRelay` + * has no else branch, so the Add button silently does nothing. + */ + @Test + fun gapUnbracketedYggdrasilAddressIsRejected() { + assertNull(RelayUrlNormalizer.normalizeOrNull("201:d0e:9ba5:8bbc:f4a1:d34:1c2:eae5")) + assertNull(RelayUrlNormalizer.normalizeOrNull("201:d0e:9ba5:8bbc:f4a1:d34:1c2:eae5:8080")) + // Bracketing it by hand is the only accepted form. + assertTrue(RelayUrlNormalizer.normalizeOrNull("[201:d0e:9ba5:8bbc:f4a1:d34:1c2:eae5]:8080") != null) + } +} From 067d68b89cced66b7ea0caede630004310bb7c04 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 4 Aug 2026 22:28:07 +0000 Subject: [PATCH 028/132] feat(relay): canonicalize IPv6 relay urls and support overlay meshes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the four gaps the previous commit characterized for relays on an Yggdrasil overlay, where every relay is an IPv6 literal in 0200::/7 served over plain ws:// (no DNS, no CA-issuable certificate). New quartz/utils/Ipv6.kt: pure-Kotlin literal parsing, RFC 5952 canonical formatting and range classification. No java.net, so it works on every KMP target. - Canonicalize the bracketed host in RelayUrlNormalizer.norm(). RFC 4291 lets one address be spelled many ways and the RFC 3986 pass only folded hex case, so two spellings survived as two NormalizedRelayUrl values for one host — and that value keys the connection pool, the relay-list sets, the NIP-11 cache and the per-relay stats, so the app dialed one relay twice. The canonical form matches what OkHttp renders when it dials; the tests assert that agreement differentially. Relay lists rehydrate through normalizeOrNull, so stored entries fold on load and no migration is needed. - Add isOverlayNetwork() for 0200::/7 and default those relays to ws://: nothing can issue a certificate for the range, so wss:// could only fail its handshake, and the overlay already encrypts end to end. - Teach isLocalHost() the IPv6 twins of the literals it already knew — ::1, fc00::/7 and fe80::/10 — so a relay on one skips TLS and Tor and stays out of published relay lists, as its IPv4 equivalent already did. - Never route an overlay relay through Tor: the range is unroutable there, so proxying guaranteed failure rather than privacy. TorRelayEvaluation covers both the Android and desktop relay paths; RoleBasedHttpClientBuilder covers non-relay HTTP. - Bracket a bare IPv6 literal automatically (what yggdrasilctl getSelf prints), but only when the whole string parses as an address, so host:port and addressable pointers still fall through. RelayUrlEditField now shows an error instead of no-opping, fixing the silent Add button for all invalid input. Mesh relays are still published in NIP-65 and offered by the outbox model; the plan doc explains why that is left as a maintainer's call, and records that no live socket test was possible here (the container has no IPv6 stack). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DQr8CDsznzCRUeB5tS8VYk --- .../RoleBasedHttpClientBuilder.kt | 6 +- .../relays/common/RelayUrlEditField.kt | 17 ++ amethyst/src/main/res/values/strings.xml | 1 + .../commons/tor/TorRelayEvaluation.kt | 6 + .../commons/tor/YggdrasilTorRoutingTest.kt | 25 +- .../plans/2026-08-04-yggdrasil-ipv6-relays.md | 149 +++++----- .../relay/normalizer/NormalizedRelayUrl.kt | 3 + .../relay/normalizer/RelayUrlNormalizer.kt | 93 +++++- .../com/vitorpamplona/quartz/utils/Ipv6.kt | 264 ++++++++++++++++++ .../vitorpamplona/quartz/utils/Ipv6Test.kt | 139 +++++++++ .../YggdrasilCompatCharacterizationTest.kt | 118 +++++--- 11 files changed, 696 insertions(+), 125 deletions(-) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/Ipv6.kt create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/Ipv6Test.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/privacyOptions/RoleBasedHttpClientBuilder.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/privacyOptions/RoleBasedHttpClientBuilder.kt index b2ca4dcafc..1f2a0722e1 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/privacyOptions/RoleBasedHttpClientBuilder.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/privacyOptions/RoleBasedHttpClientBuilder.kt @@ -67,7 +67,9 @@ class RoleBasedHttpClientBuilder( normalizedUrl: String, final: Boolean, ): Boolean = - if (RelayUrlNormalizer.isLocalHost(normalizedUrl)) { + if (RelayUrlNormalizer.isLocalHost(normalizedUrl) || RelayUrlNormalizer.isOverlayNetwork(normalizedUrl)) { + // Overlay-mesh hosts (0200::/7) are reachable only through the local mesh + // interface — Tor cannot route the range, so proxying only breaks the fetch. false } else if (RelayUrlNormalizer.isOnion(normalizedUrl)) { true @@ -113,7 +115,7 @@ class RoleBasedHttpClientBuilder( isOnionRelaysActive: Boolean, final: Boolean, ): Boolean = - if (RelayUrlNormalizer.isLocalHost(normalizedUrl)) { + if (RelayUrlNormalizer.isLocalHost(normalizedUrl) || RelayUrlNormalizer.isOverlayNetwork(normalizedUrl)) { false } else if (RelayUrlNormalizer.isOnion(normalizedUrl)) { isOnionRelaysActive diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/relays/common/RelayUrlEditField.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/relays/common/RelayUrlEditField.kt index e14ca5c66f..d523058cfb 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/relays/common/RelayUrlEditField.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/relays/common/RelayUrlEditField.kt @@ -170,6 +170,7 @@ fun RelayUrlEditField( nav: INav, ) { var url by remember { mutableStateOf("") } + var isInvalid by remember { mutableStateOf(false) } fun submitRelay() { if (url.isNotBlank()) { @@ -177,7 +178,13 @@ fun RelayUrlEditField( if (relay != null) { onNewRelay(relay) url = "" + isInvalid = false relaySuggestions.reset() + } else { + // Without this the Add button is a silent no-op, which reads as a broken button. + // Bare IPv6 literals are the common way to land here: an overlay-mesh address + // pasted straight out of `yggdrasilctl getSelf` needs brackets to carry a port. + isInvalid = true } } } @@ -189,8 +196,18 @@ fun RelayUrlEditField( value = url, onValueChange = { url = it + isInvalid = false relaySuggestions.processInput(it) }, + isError = isInvalid, + supportingText = { + if (isInvalid) { + Text( + text = stringRes(R.string.relay_url_not_valid), + color = MaterialTheme.colorScheme.error, + ) + } + }, placeholder = { Text( text = "server.com", diff --git a/amethyst/src/main/res/values/strings.xml b/amethyst/src/main/res/values/strings.xml index 1843dbfdd4..2221700aea 100644 --- a/amethyst/src/main/res/values/strings.xml +++ b/amethyst/src/main/res/values/strings.xml @@ -214,6 +214,7 @@ Percentage of successful connections to the relay Search and add user Add a Relay + Not a valid relay address. Use a host name, or an IP address in brackets (for example [201:d0e:9ba5:8bbc::1]:8080). My @tag name Display Name My display name diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/tor/TorRelayEvaluation.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/tor/TorRelayEvaluation.kt index 42987dfb56..40695348b8 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/tor/TorRelayEvaluation.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/tor/TorRelayEvaluation.kt @@ -23,6 +23,7 @@ package com.vitorpamplona.amethyst.commons.tor import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.isLocalHost import com.vitorpamplona.quartz.nip01Core.relay.normalizer.isOnion +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.isOverlayNetwork class TorRelayEvaluation( val torSettings: TorRelaySettings, @@ -36,6 +37,11 @@ class TorRelayEvaluation( } else { if (relay.isLocalHost()) { false + } else if (relay.isOverlayNetwork()) { + // An overlay-mesh relay (0200::/7, e.g. Yggdrasil) is reachable only through the + // local mesh interface: Tor cannot route the range at all, so proxying it would + // guarantee failure rather than privacy. The overlay already encrypts end to end. + false } else if (relay.isOnion()) { // .onion is only reachable over Tor regardless of any other classification. torSettings.onionRelaysViaTor diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/tor/YggdrasilTorRoutingTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/tor/YggdrasilTorRoutingTest.kt index 68a8941821..9a2dbb577b 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/tor/YggdrasilTorRoutingTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/tor/YggdrasilTorRoutingTest.kt @@ -26,15 +26,16 @@ import kotlin.test.assertFalse import kotlin.test.assertTrue /** - * GAP 4 — an Yggdrasil relay is classified as a plain clearnet relay, so with Tor on it is - * dialed through the SOCKS proxy. Tor cannot route `0200::/7`: the connection can only fail. - * - * Compare `ws://192.168.1.100:8080/`, which [TorRelayEvaluation] correctly keeps off Tor - * because `isLocalHost()` recognizes the LAN prefix. Yggdrasil has no such recognition. + * An overlay-mesh relay (`0200::/7`, e.g. Yggdrasil) must never be dialed through the Tor SOCKS + * proxy: Tor cannot route the range, so proxying guarantees failure rather than privacy. The + * overlay already encrypts end to end and authenticates the peer by its key-derived address. */ class YggdrasilTorRoutingTest { private val yggdrasilRelay = NormalizedRelayUrl("ws://[201:d0e:9ba5:8bbc::1]:8080/") + private val yggdrasilSubnetRelay = NormalizedRelayUrl("ws://[300:1b5d:d0e9:ba58::1]:4848/") private val lanRelay = NormalizedRelayUrl("ws://192.168.1.100:8080/") + private val ulaRelay = NormalizedRelayUrl("ws://[fd12:3456::1]:8080/") + private val clearnetIpv6Relay = NormalizedRelayUrl("wss://[2001:db8::1]:8080/") private fun evaluation(newViaTor: Boolean) = TorRelayEvaluation( @@ -52,14 +53,18 @@ class YggdrasilTorRoutingTest { ) @Test - fun yggdrasilRelayIsSentThroughTorWhileLanRelayIsNot() { + fun overlayRelaysAreNeverTorifiedEvenWhenNewRelaysViaTorIsOn() { val eval = evaluation(newViaTor = true) - assertTrue(eval.useTor(yggdrasilRelay), "Yggdrasil relay is routed via Tor, which cannot reach 0200::/7") - assertFalse(eval.useTor(lanRelay), "LAN relay is correctly kept off Tor") + assertFalse(eval.useTor(yggdrasilRelay), "0200::/8 node address must not be proxied") + assertFalse(eval.useTor(yggdrasilSubnetRelay), "0300::/8 subnet address must not be proxied") + assertFalse(eval.useTor(lanRelay), "LAN relay stays off Tor") + assertFalse(eval.useTor(ulaRelay), "IPv6 unique local address stays off Tor") } @Test - fun yggdrasilRelayWorksOnlyWhenNewRelaysViaTorIsOff() { - assertFalse(evaluation(newViaTor = false).useTor(yggdrasilRelay)) + fun clearnetIpv6RelaysStillFollowTheTorSetting() { + // The overlay exemption must not leak into ordinary IPv6 relays. + assertTrue(evaluation(newViaTor = true).useTor(clearnetIpv6Relay)) + assertFalse(evaluation(newViaTor = false).useTor(clearnetIpv6Relay)) } } diff --git a/quartz/plans/2026-08-04-yggdrasil-ipv6-relays.md b/quartz/plans/2026-08-04-yggdrasil-ipv6-relays.md index f854bffaab..33d8aaef31 100644 --- a/quartz/plans/2026-08-04-yggdrasil-ipv6-relays.md +++ b/quartz/plans/2026-08-04-yggdrasil-ipv6-relays.md @@ -1,103 +1,114 @@ -# Amethyst over Yggdrasil (IPv6 overlay) — compatibility assessment +# Amethyst over Yggdrasil (IPv6 overlay) -Status: **analysis only** — no behavior changed. Characterization tests landed alongside -this doc pin the current behavior so a fix has a baseline to diff against. +Status: **fixed** — the four gaps found in the original assessment are closed. The last +section records what was deliberately left alone. ## What Yggdrasil looks like to the app Yggdrasil is an encrypted end-to-end mesh. Every node gets an IPv6 address derived from its -public key inside `0200::/7`, and hands out `0300::/8` subnets. Consequences that matter here: +public key inside `0200::/7` (nodes in `0200::/8`, subnets in `0300::/8`). Consequences: -- **No DNS.** A relay on the mesh is addressed as a bracketed IPv6 literal, always. -- **No certificates.** No CA issues for `0200::/7` literals, so relays run plain `ws://`. - This is not a downgrade — the overlay already provides end-to-end encryption and - authenticates the peer by its address. +- **No DNS.** A relay on the mesh is addressed as an IPv6 literal, always. +- **No certificates.** No CA issues for `0200::/7`, so relays run plain `ws://`. Not a + downgrade — the overlay already encrypts end to end and authenticates the peer by an + address derived from its public key. - **On Android it is a `VpnService`**, so the app's default network becomes the VPN network. -- The address is a **stable node identifier**, so publishing it is equivalent to publishing - a long-lived pseudonymous handle for the device. +- `0200::/7` is deprecated NSAP space, so nothing else routes there. An address in the range + is reachable *only* through a running mesh interface — which is what makes it safe to key + behavior off the prefix. -## Verdict +## What was wrong, and what fixed it -A hand-typed `ws://[…]:port` relay works end to end: it normalizes, survives the RFC 3986 -pass, and OkHttp parses and dials it. Nothing in the stack is IPv4-only, `TcpNoDelaySocketFactory` -is family-agnostic, and `network_security_config.xml` permits cleartext globally, so the -`ws://` requirement is already satisfied. +### 1. One relay, two identities -Everything around that happy path is where it degrades. Four gaps, in severity order. +`RelayUrlNormalizer` folded hex case but not zero-compression, so +`[201:0d0e:9ba5:8bbc:0000:0000:0000:0001]` and `[201:d0e:9ba5:8bbc::1]` stayed two distinct +`NormalizedRelayUrl`s for one host — while OkHttp collapsed both to the same host when +dialing. Since that value keys the connection pool, the relay-list sets, the NIP-11 cache and +the per-relay stat maps, the app opened two sockets to one relay and counted it twice. -### GAP 1 — one relay, two identities (correctness) +**Fix:** new `Ipv6` util (`quartz/utils/Ipv6.kt`) — pure-Kotlin parse, RFC 5952 canonical +format and range classification, no `java.net`, so it works on every KMP target. +`RelayUrlNormalizer.norm()` now canonicalizes the bracketed host. The canonical form is +byte-for-byte what OkHttp renders, so the key the app stores is the host it actually dials — +asserted differentially against OkHttp in `YggdrasilCompatCharacterizationTest`. -`RelayUrlNormalizer` folds hex case but does **not** canonicalize zero-compression or -leading zeros: +Affects every IPv6 relay, not just mesh ones; it only bit Yggdrasil users because on the mesh +a literal is the *only* way to name a relay. No migration needed: relay lists are rehydrated +from event tags through `normalizeOrNull`, so stored entries fold on load. -| input | `NormalizedRelayUrl` | OkHttp host | -|---|---|---| -| `ws://[201:d0e:9ba5:8bbc::1]:8080` | `ws://[201:d0e:9ba5:8bbc::1]:8080/` | `201:d0e:9ba5:8bbc::1` | -| `ws://[201:0d0e:9ba5:8bbc:0000:0000:0000:0001]:8080` | `ws://[201:0d0e:…:0001]:8080/` | `201:d0e:9ba5:8bbc::1` | +### 2. Schemeless entry defaulted to `wss://` -OkHttp collapses both to one host; the app does not. `NormalizedRelayUrl` is the key of the -connection pool (`PoolRequests`, `RelayPool`), every relay-list set, the NIP-11 cache and the -per-relay stat maps — so the same relay written two ways gets **two sockets, two REQ sets and -doubled traffic**, and appears twice in the relay UI. This affects all IPv6 literals, but it -only bites Yggdrasil users in practice, because on the mesh a literal is the *only* way to -name a relay. Fix: canonicalize the bracketed literal to RFC 5952 inside `fix()`. +`isLocalHost()` knew `127.0.0.1` / `localhost` / `//umbrel:` / `192.168.` / `.local`, so a +mesh address fell through to the clearnet default and produced a `wss://` url whose TLS +handshake could never succeed. -### GAP 2 — schemeless entry defaults to `wss://` (dead end) +**Fix:** new `RelayUrlNormalizer.isOverlayNetwork()` recognizes `0200::/7` and joins +`isOnion` / `isLocalHost` in choosing `ws://`. Clearnet IPv6 (`2001:db8::1`) still gets +`wss://`. -`RelayUrlNormalizer.isLocalHost()` recognizes `127.0.0.1`, `localhost`, `//umbrel:`, -`192.168.`, `.local:` / `.local/`. An Yggdrasil address matches none of them, so a schemeless -`[201:…]:8080` falls through to the clearnet default and becomes `wss://[201:…]:8080/` — a -URL whose TLS handshake can never succeed. The user must know to type `ws://` themselves. -Fix: teach `isLocalHost()` (or a sibling `isOverlayNetwork()`) the `0200::/7` prefix. +`isLocalHost()` separately grew the IPv6 twins of the literals it already knew — `::1` +(loopback), `fc00::/7` (unique local, the 192.168. analogue) and `fe80::/10` (link-local). +Those are the same question every caller is asking, so a relay on one now correctly skips TLS +and Tor and stays out of published relay lists. -### GAP 3 — unbracketed literal is silently rejected (UX) +### 3. Unbracketed literal silently rejected -`yggdrasilctl getSelf` prints the address **unbracketed**, which is exactly what a user -copies into the "add a relay" box. `isBareHostAndPath()` rejects it (correctly — it is -ambiguous with a scheme), so `normalizeOrNull` returns null. But -`RelayUrlEditField.submitRelay()` has no else branch: the Add button just does nothing, with -no error. Fix: either auto-bracket a candidate that parses as an IPv6 address, or surface a -validation message instead of a silent no-op. +`yggdrasilctl getSelf` prints the address unbracketed — exactly what gets pasted into "add a +relay". Normalization returned null (correctly: it is ambiguous with a scheme) and +`RelayUrlEditField.submitRelay` had no else branch, so the Add button did nothing at all. -### GAP 4 — Tor routing sends mesh traffic into the SOCKS proxy (breaks the relay) +**Fix, two halves:** +- `fix()` brackets a bare literal automatically, but only when the whole string parses as an + IPv6 address — so `31990:hex:dtag` (addressable pointer), `abcd:1234` (host:port) and + `relay.example.com:8080` still fall through untouched. +- The edit field now sets `isError` and shows `relay_url_not_valid` instead of no-opping. + That fixes the dead button for *all* invalid input, not just IPv6. -`TorRelayEvaluation` classifies relays as localhost / onion / dm / trusted / new. Yggdrasil -lands in **new**, so with Tor on and the default "new relays via Tor", the relay is dialed -through the Tor SOCKS proxy — which cannot route `0200::/7`. The connection can only fail. -Compare `ws://192.168.1.100:8080/`, which is correctly kept off Tor because `isLocalHost()` -knows the LAN prefix. Today the only workaround is turning "new relays via Tor" off, which -weakens the setting for every genuine clearnet relay. The same gap exists on desktop -(`DesktopHttpClient`) and for non-relay HTTP (`RoleBasedHttpClientBuilder`). Fixing GAP 2's -prefix check fixes this one too, since both read the same predicate. +### 4. Tor routing broke mesh relays -## Propagation / privacy note (not a bug, a decision) +`TorRelayEvaluation` classified mesh relays as "new", so with Tor on and the default "new +relays via Tor" they were dialed through the SOCKS proxy — which cannot route `0200::/7`. +Guaranteed failure, not privacy. -An Yggdrasil relay is not filtered out of NIP-65 publishing (`AdvertisedRelayInfoTag` only -rejects localhost) nor out of the outbox model -(`RelayListRecommendationProcessor.filterValidRelays`). So a mesh relay in your relay list is -**published to public relays and recommended to other users**. Two effects: +**Fix:** `useTor()` returns false for `isOverlayNetwork()`, checked right after the localhost +branch. Both the Android and desktop relay paths delegate here (`TorRelayState`, +`DesktopHttpClient`), so one change covers both. `RoleBasedHttpClientBuilder` got the same +treatment for non-relay HTTP (images, previews, NIP-05, money ops). Clearnet IPv6 relays keep +following the Tor setting — asserted in `YggdrasilTorRoutingTest`. + +## Deliberately not changed + +**Mesh relays are still published and recommended.** `AdvertisedRelayInfoTag` (NIP-65) and +`RelayListRecommendationProcessor.filterValidRelays` only exclude localhost, so a mesh relay +in your relay list is still published to public relays and offered to other users via the +outbox model. Two consequences worth a maintainer's decision: - Peers not on the mesh dial `[201:…]` and burn reconnect attempts on an unreachable host. - Your Yggdrasil address — a stable, key-derived node identifier — becomes public. -Onion relays get special handling here (`hasOnionConnection` gates whether they are even -considered). An overlay-network classification would let Yggdrasil be treated the same way. +Onion relays already have precedent for both readings: they *are* published, but +`filterValidRelays` gates them behind `hasOnionConnection`. The equivalent for overlay relays +would be a `hasMeshConnection` gate. That is a product call about whether mesh relays are +meant to be discoverable, so it is flagged rather than decided here. -## Not covered +## Not verified here -- **No live socket test.** The analysis container has no IPv6 stack at all - (`AF_INET6` → `EAFNOSUPPORT`), so everything above is verified below the socket: URL - normalization, OkHttp URL/host parsing, and the Tor routing decision. An on-device run - against a real mesh relay is still needed to confirm the happy path end to end. +- **No live socket test.** The analysis container has no IPv6 stack at all (`AF_INET6` → + `EAFNOSUPPORT`), so everything is verified below the socket: normalization, OkHttp URL/host + agreement, and the Tor routing decision. An on-device run against a real mesh relay is + still needed to confirm the happy path end to end. - **Android VPN interaction untested.** `ConnectivityFlow` uses - `registerDefaultNetworkCallback`, so it follows the app's default network into the VPN. - Whether `isMeteredOrMobileData()` reads correctly through Yggdrasil's `VpnService` depends - on whether that app declares underlying networks; worth checking on device before assuming + `registerDefaultNetworkCallback`, so it follows the app into the VPN network. Whether + `isMeteredOrMobileData()` reads correctly through Yggdrasil's `VpnService` depends on + whether that app declares underlying networks — worth checking on device before assuming data-saving mode behaves. - Media loading (Coil) and NIP-05 resolution against mesh hosts were not exercised. ## Tests -- `quartz/src/jvmAndroidTest/…/relay/YggdrasilCompatCharacterizationTest.kt` — GAPs 1–3 plus - the working happy path. -- `commons/src/commonTest/…/tor/YggdrasilTorRoutingTest.kt` — GAP 4. +- `quartz/…/utils/Ipv6Test.kt` — parser, RFC 5952 formatting, range classification. +- `quartz/…/relay/YggdrasilCompatCharacterizationTest.kt` — normalization end to end, plus + the differential assertions that our identity matches the host OkHttp dials. +- `commons/…/tor/YggdrasilTorRoutingTest.kt` — overlay relays never Torified, clearnet IPv6 + still follows the setting. diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/NormalizedRelayUrl.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/NormalizedRelayUrl.kt index ac3f69a1f3..0196ac0bbf 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/NormalizedRelayUrl.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/NormalizedRelayUrl.kt @@ -47,3 +47,6 @@ fun NormalizedRelayUrl.toHttp() = fun NormalizedRelayUrl.isOnion() = url.contains(".onion/") fun NormalizedRelayUrl.isLocalHost() = RelayUrlNormalizer.isLocalHost(this.url) + +/** True for a relay inside an encrypted IPv6 overlay mesh. See [RelayUrlNormalizer.isOverlayNetwork]. */ +fun NormalizedRelayUrl.isOverlayNetwork() = RelayUrlNormalizer.isOverlayNetwork(this.url) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt index bc5c524482..18d83c66af 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt @@ -21,6 +21,7 @@ package com.vitorpamplona.quartz.nip01Core.relay.normalizer import androidx.collection.LruCache +import com.vitorpamplona.quartz.utils.Ipv6 import com.vitorpamplona.quartz.utils.Log import com.vitorpamplona.quartz.utils.Rfc3986 import kotlinx.coroutines.CancellationException @@ -44,7 +45,51 @@ class RelayUrlNormalizer { url.contains("//umbrel:") || url.contains("192.168.") || url.contains(".local:") || - url.contains(".local/") + url.contains(".local/") || + isPrivateIpv6(url) + + /** + * The IPv6 twins of the literals above: `::1` (127.0.0.1), `fc00::/7` unique local + * addresses (192.168.0.0/16) and `fe80::/10` link-local. All three name a host that + * only exists on this machine or this LAN, which is what every caller of [isLocalHost] + * means by the question — so a relay on one must not be Torified, must not need TLS, + * and must not be advertised to the network. + */ + private fun isPrivateIpv6(url: String): Boolean { + val bytes = ipv6HostOf(url) ?: return false + return Ipv6.isLoopback(bytes) || Ipv6.isUniqueLocal(bytes) || Ipv6.isLinkLocal(bytes) + } + + /** + * True for a relay inside an encrypted IPv6 overlay mesh — today `0200::/7`, the range + * Yggdrasil derives node addresses and subnets from. + * + * Unlike [isLocalHost] this is not a private address: it is reachable from anywhere on + * the mesh. But it is unreachable *off* the mesh, which has two consequences the relay + * stack has to honour — it can never be dialed through a SOCKS/Tor proxy, and it can + * never present a CA-issued certificate, so it speaks plain `ws://`. Both are safe: + * the overlay already encrypts end to end and authenticates the peer by its address, + * which is derived from the peer's public key. + */ + fun isOverlayNetwork(url: String): Boolean { + val bytes = ipv6HostOf(url) ?: return false + return Ipv6.isOverlayMesh(bytes) + } + + /** + * Extracts the bracketed IPv6 host of [url] as raw bytes, dropping any `%zone` suffix. + * Returns null — cheaply, on a single `indexOf` — for the overwhelmingly common case of + * a url with a DNS host. + */ + private fun ipv6HostOf(url: String): ByteArray? { + val open = url.indexOf('[') + if (open < 0) return null + val close = url.indexOf(']', open + 1) + if (close <= open + 1) return null + val zone = url.indexOf('%', open + 1) + val end = if (zone in (open + 1) until close) zone else close + return Ipv6.parse(url.substring(open + 1, end)) + } fun isOnion(url: String) = url.endsWith(".onion") || url.contains(".onion/") @@ -82,7 +127,31 @@ class RelayUrlNormalizer { return false } - private fun norm(url: String) = NormalizedRelayUrl(Rfc3986.normalize(url)) + private fun norm(url: String) = NormalizedRelayUrl(canonicalizeIpv6Host(Rfc3986.normalize(url))) + + /** + * Rewrites a bracketed IPv6 host into its RFC 5952 canonical form. + * + * RFC 4291 lets one address be spelled many ways, and the RFC 3986 pass only folds hex + * case — so `[201:0d0e:9ba5:8bbc:0000:0000:0000:0001]` and `[201:d0e:9ba5:8bbc::1]` + * survive as two different [NormalizedRelayUrl]s for one host. That value keys the + * connection pool, the relay-list sets, the NIP-11 cache and the per-relay stats, so the + * app would dial the same relay twice and count it twice. OkHttp canonicalizes to this + * exact form when it dials, so folding here makes the stored key the host on the wire. + * + * Returns [url] itself — no allocation — when there is no literal or it is already + * canonical, which is every url with a DNS host. + */ + private fun canonicalizeIpv6Host(url: String): String { + val open = url.indexOf('[') + if (open < 0) return url + val close = url.indexOf(']', open + 1) + if (close <= open + 1) return url + val inner = url.substring(open + 1, close) + val canonical = Ipv6.canonicalizeOrNull(inner) ?: return url + if (canonical == inner) return url + return url.substring(0, open + 1) + canonical + url.substring(close) + } private fun isInvisible(c: Char) = c == '\u200B' || c == '\u200C' || c == '\u200D' || c == '\u2060' || c == '\uFEFF' @@ -267,15 +336,29 @@ class RelayUrlNormalizer { } // protocol-relative urls (`//host/`) are just missing the scheme - val bare = if (trimmed.startsWith("//")) trimmed.drop(2) else trimmed - if (bare.length < 4) return null + val protocolRelative = if (trimmed.startsWith("//")) trimmed.drop(2) else trimmed + if (protocolRelative.length < 4) return null + + // A bare IPv6 literal is missing its brackets, not malformed. This is the shape a + // user actually has in hand — `yggdrasilctl getSelf` prints the address unbracketed + // — and without the brackets `isBareHostAndPath` rejects it below as a host with too + // many colons. Only a string that parses as a whole address is bracketed, so an + // addressable-event pointer (`31990:hex:dtag`) or a `host:port` still falls through. + val bare = + if (protocolRelative[0] != '[' && Ipv6.isLiteral(protocolRelative)) { + "[$protocolRelative]" + } else { + protocolRelative + } if (!isBareHostAndPath(bare)) { Log.d("RelayUrlNormalizer") { "Rejected $url" } return null } - return if (isOnion(bare) || isLocalHost(bare)) { + // Overlay and localhost relays cannot hold a certificate, so wss:// could only ever + // fail its handshake. Both carry their own encryption, so ws:// is not a downgrade. + return if (isOnion(bare) || isLocalHost(bare) || isOverlayNetwork(bare)) { "ws://$bare" } else { "wss://$bare" diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/Ipv6.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/Ipv6.kt new file mode 100644 index 0000000000..27d0d724ed --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/Ipv6.kt @@ -0,0 +1,264 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.utils + +/** + * Pure-Kotlin IPv6 literal parsing, RFC 5952 canonical formatting and address + * classification. No `java.net`, so it works on every KMP target. + * + * Exists because relay identity is a *string*: [com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl] + * is the key of the connection pool, the relay-list sets, the NIP-11 cache and every + * per-relay stat map. RFC 4291 lets one address be written many ways + * (`[201:0d0e:9ba5:8bbc:0000:0000:0000:0001]` and `[201:d0e:9ba5:8bbc::1]` are the same + * host), and without folding them the app treats one relay as two — two sockets, two REQ + * sets, two rows in the UI. The canonical form here matches what OkHttp renders, so the + * key the app stores is the host it actually dials. + */ +object Ipv6 { + /** Longest legal literal is 45 chars (`::ffff:` + dotted quad is shorter than 8 full groups). */ + private const val MAX_LITERAL = 45 + + /** + * Parses a bracket-less, zone-less IPv6 literal into its 16 bytes, or null when [address] + * is not a valid literal. Accepts `::` compression and a trailing dotted quad + * (`::ffff:192.168.1.1`). + */ + fun parse(address: String): ByteArray? { + val len = address.length + if (len < 2 || len > MAX_LITERAL) return null + + val out = ByteArray(16) + // Bytes written so far, counting from the left. When a `::` is present the bytes after + // it are written contiguously here and shifted to the right end at the very end. + var fill = 0 + var gapAt = -1 + var i = 0 + + if (address[0] == ':') { + if (address[1] != ':') return null + gapAt = 0 + i = 2 + if (i == len) return out + } + + while (true) { + val groupStart = i + var value = 0 + var digits = 0 + while (i < len) { + val digit = hexDigit(address[i]) + if (digit < 0) break + if (digits == 4) return null + value = (value shl 4) or digit + digits++ + i++ + } + + if (i < len && address[i] == '.') { + // Trailing dotted quad: occupies the last four bytes, so nothing may follow it. + if (fill > 12) return null + if (!parseIpv4Into(address, groupStart, len, out, fill)) return null + fill += 4 + i = len + break + } + + if (digits == 0) return null + if (fill + 2 > 16) return null + out[fill++] = (value ushr 8).toByte() + out[fill++] = value.toByte() + + if (i == len) break + if (address[i] != ':') return null + i++ + if (i == len) return null // a single trailing ':' is not a valid literal + if (address[i] == ':') { + if (gapAt >= 0) return null // only one `::` allowed + gapAt = fill + i++ + if (i == len) break + } + } + + if (gapAt < 0) { + if (fill != 16) return null + } else { + // `::` must stand for at least one omitted group. + if (fill == 16) return null + val tail = fill - gapAt + for (k in tail - 1 downTo 0) { + out[16 - tail + k] = out[gapAt + k] + out[gapAt + k] = 0 + } + } + return out + } + + /** + * RFC 5952 text form: lowercase hex, no leading zeros, and the longest run of two or more + * zero groups replaced by `::` (leftmost run wins a tie). IPv4-mapped addresses keep their + * dotted tail. This is byte-for-byte what OkHttp prints for the same address. + */ + fun format(bytes: ByteArray): String { + require(bytes.size == 16) { "An IPv6 address is 16 bytes, got ${bytes.size}" } + + var bestStart = -1 + var bestLen = 0 + var i = 0 + while (i < 16) { + if (bytes[i] == ZERO && bytes[i + 1] == ZERO) { + val runStart = i + var j = i + while (j < 16 && bytes[j] == ZERO && bytes[j + 1] == ZERO) j += 2 + if (j - runStart > bestLen) { + bestLen = j - runStart + bestStart = runStart + } + i = j + } else { + i += 2 + } + } + // A single zero group is written as `0`, never as `::`. + if (bestLen < 4) { + bestStart = -1 + bestLen = 0 + } + + val out = StringBuilder(39) + // ::ffff:a.b.c.d — IPv4-mapped addresses read as IPv4 everywhere else, so keep them that way. + if (bestStart == 0 && bestLen == 10 && bytes[10] == ALL_ONES && bytes[11] == ALL_ONES) { + out.append("::ffff:") + appendIpv4(out, bytes, 12) + return out.toString() + } + + i = 0 + while (i < 16) { + if (i == bestStart) { + out.append(':') + i += bestLen + if (i == 16) out.append(':') + } else { + if (i > 0) out.append(':') + out.append(group(bytes, i).toString(16)) + i += 2 + } + } + return out.toString() + } + + /** + * Canonicalizes a bracket-less literal, preserving any `%zone` suffix verbatim (in URLs the + * zone arrives percent-encoded, e.g. `fe80::1%25wlan0`). Returns null when [address] is not + * a valid literal. + */ + fun canonicalizeOrNull(address: String): String? { + val zoneAt = address.indexOf('%') + if (zoneAt < 0) return parse(address)?.let(::format) + val bytes = parse(address.substring(0, zoneAt)) ?: return null + return format(bytes) + address.substring(zoneAt) + } + + /** True when [address] is a valid bracket-less literal that names more than one group. */ + fun isLiteral(address: String): Boolean = address.indexOf(':') >= 0 && parse(address) != null + + /** `::1` — the IPv6 loopback, twin of 127.0.0.1. */ + fun isLoopback(bytes: ByteArray): Boolean { + for (i in 0 until 15) if (bytes[i] != ZERO) return false + return bytes[15] == ONE + } + + /** `fe80::/10` — link-local, only meaningful on the interface it came from. */ + fun isLinkLocal(bytes: ByteArray): Boolean = bytes[0] == FE.toByte() && (bytes[1].toInt() and 0xC0) == 0x80 + + /** `fc00::/7` — unique local addresses, the IPv6 twin of 192.168.0.0/16. */ + fun isUniqueLocal(bytes: ByteArray): Boolean = (bytes[0].toInt() and 0xFE) == 0xFC + + /** + * `0200::/7` — the range Yggdrasil derives node addresses (`0200::/8`) and subnets + * (`0300::/8`) from. Formally deprecated NSAP space, so nothing else routes here: an + * address in this range is reachable only through a running mesh interface, is already + * end-to-end encrypted by the overlay, and can never hold a CA-issued certificate. + */ + fun isOverlayMesh(bytes: ByteArray): Boolean = (bytes[0].toInt() and 0xFE) == 0x02 + + private fun group( + bytes: ByteArray, + at: Int, + ) = ((bytes[at].toInt() and 0xFF) shl 8) or (bytes[at + 1].toInt() and 0xFF) + + private fun appendIpv4( + out: StringBuilder, + bytes: ByteArray, + from: Int, + ) { + for (k in 0 until 4) { + if (k > 0) out.append('.') + out.append(bytes[from + k].toInt() and 0xFF) + } + } + + /** + * Parses `a.b.c.d` in `[from, to)` into four bytes at [at]. Leading zeros are rejected — + * they invite the octal reading that makes `010.1.1.1` ambiguous across resolvers. + */ + private fun parseIpv4Into( + text: String, + from: Int, + to: Int, + out: ByteArray, + at: Int, + ): Boolean { + var i = from + for (octet in 0 until 4) { + if (octet > 0) { + if (i >= to || text[i] != '.') return false + i++ + } + var value = 0 + var digits = 0 + while (i < to && text[i] in '0'..'9') { + if (digits == 3) return false + if (digits == 1 && value == 0) return false // leading zero + value = value * 10 + (text[i] - '0') + digits++ + i++ + } + if (digits == 0 || value > 255) return false + out[at + octet] = value.toByte() + } + return i == to + } + + private fun hexDigit(c: Char): Int = + when (c) { + in '0'..'9' -> c - '0' + in 'a'..'f' -> c - 'a' + 10 + in 'A'..'F' -> c - 'A' + 10 + else -> -1 + } + + private const val FE = 0xFE + private const val ZERO = 0.toByte() + private const val ONE = 1.toByte() + private const val ALL_ONES = 0xFF.toByte() +} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/Ipv6Test.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/Ipv6Test.kt new file mode 100644 index 0000000000..66d4059de4 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/Ipv6Test.kt @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.utils + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class Ipv6Test { + private fun canonical(address: String) = Ipv6.canonicalizeOrNull(address) + + @Test + fun rfc5952CanonicalForm() { + // leading zeros suppressed, hex lowercased + assertEquals("201:d0e:9ba5:8bbc::1", canonical("201:0d0e:9ba5:8bbc:0000:0000:0000:0001")) + assertEquals("201:d0e:9ba5:8bbc::1", canonical("201:D0E:9BA5:8BBC::1")) + assertEquals("2001:db8::1", canonical("2001:0DB8:0000:0000:0000:0000:0000:0001")) + // already canonical stays put + assertEquals("201:d0e:9ba5:8bbc:f4a1:d34:1c2:eae5", canonical("201:d0e:9ba5:8bbc:f4a1:d34:1c2:eae5")) + assertEquals("::", canonical("::")) + assertEquals("::1", canonical("0:0:0:0:0:0:0:1")) + } + + @Test + fun singleZeroGroupIsNotCompressed() { + // RFC 5952 §4.2.2: `::` must not stand for a single group. + assertEquals("2001:db8:0:1:1:1:1:1", canonical("2001:db8:0:1:1:1:1:1")) + } + + @Test + fun longestZeroRunWinsAndTiesGoLeft() { + assertEquals("2001:0:0:1::1", canonical("2001:0:0:1:0:0:0:1")) + // equal runs of two groups: the leftmost is the one compressed + assertEquals("2001::1:1:0:0:1", canonical("2001:0:0:1:1:0:0:1")) + } + + @Test + fun ipv4MappedKeepsDottedTail() { + assertEquals("::ffff:192.168.1.1", canonical("::ffff:192.168.1.1")) + assertEquals("::ffff:127.0.0.1", canonical("::FFFF:127.0.0.1")) + // an embedded quad that is not ipv4-mapped collapses to plain hex + assertEquals("::c0a8:101", canonical("::192.168.1.1")) + } + + @Test + fun zoneIdIsPreservedVerbatim() { + // In URLs the zone arrives percent-encoded. + assertEquals("fe80::1%25wlan0", canonical("fe80:0000:0000:0000:0000:0000:0000:0001%25wlan0")) + } + + @Test + fun rejectsMalformedLiterals() { + assertNull(canonical("201:d0e:9ba5:8bbc:f4a1:d34:1c2")) // too few groups + assertNull(canonical("201:d0e:9ba5:8bbc:f4a1:d34:1c2:eae5:1234")) // too many + assertNull(canonical("201::9ba5::1")) // two `::` + assertNull(canonical("201:d0e:9ba5:8bbc:f4a1:d34:1c2:eae5:")) // trailing colon + assertNull(canonical("201:d0e:9ba5:8bbc:f4a1:d34:1c2:gggg")) // non-hex + assertNull(canonical("201:00d0e:9ba5:8bbc::1")) // five-digit group + assertNull(canonical("192.168.1.1")) // ipv4 + assertNull(canonical("localhost")) + assertNull(canonical("::ffff:192.168.1")) // short quad + assertNull(canonical("::ffff:010.1.1.1")) // leading zero in quad + assertNull(canonical("0:0:0:0:0:0:0:0:0")) + } + + @Test + fun compressionMustCoverAtLeastOneGroup() { + // A `::` that stands for nothing is not a legal literal. + assertNull(canonical("1:2:3:4:5:6:7::8")) + } + + @Test + fun classifiesYggdrasilAndPrivateRanges() { + assertTrue(Ipv6.isOverlayMesh(Ipv6.parse("201:d0e:9ba5:8bbc::1")!!), "0200::/8 node address") + assertTrue(Ipv6.isOverlayMesh(Ipv6.parse("300:1b5d:d0e9:ba58::1")!!), "0300::/8 subnet address") + assertTrue(Ipv6.isOverlayMesh(Ipv6.parse("2ff::1")!!)) + assertFalse(Ipv6.isOverlayMesh(Ipv6.parse("2001:db8::1")!!), "documentation range is clearnet") + assertFalse(Ipv6.isOverlayMesh(Ipv6.parse("400::1")!!), "just past 0200::/7") + assertFalse(Ipv6.isOverlayMesh(Ipv6.parse("::1")!!)) + + assertTrue(Ipv6.isLoopback(Ipv6.parse("::1")!!)) + assertFalse(Ipv6.isLoopback(Ipv6.parse("::2")!!)) + assertFalse(Ipv6.isLoopback(Ipv6.parse("::")!!)) + + assertTrue(Ipv6.isLinkLocal(Ipv6.parse("fe80::1")!!)) + assertTrue(Ipv6.isLinkLocal(Ipv6.parse("febf::1")!!)) + assertFalse(Ipv6.isLinkLocal(Ipv6.parse("fec0::1")!!)) + + assertTrue(Ipv6.isUniqueLocal(Ipv6.parse("fd00::1")!!)) + assertTrue(Ipv6.isUniqueLocal(Ipv6.parse("fc00::1")!!)) + assertFalse(Ipv6.isUniqueLocal(Ipv6.parse("fe00::1")!!)) + } + + @Test + fun isLiteralDiscriminatesAgainstNonAddresses() { + assertTrue(Ipv6.isLiteral("201:d0e:9ba5:8bbc:f4a1:d34:1c2:eae5")) + assertTrue(Ipv6.isLiteral("201:d0e:9ba5:8bbc::1")) + // Things a relay-url field realistically receives, none of which may pass as an address. + assertFalse(Ipv6.isLiteral("relay.example.com:8080")) + assertFalse(Ipv6.isLiteral("wss:")) + assertFalse(Ipv6.isLiteral("localhost:4869")) + assertFalse(Ipv6.isLiteral("31990:abcdef:mydtag"), "addressable event pointer") + assertFalse(Ipv6.isLiteral("abcd:1234")) + assertFalse(Ipv6.isLiteral("nos.lol")) + } + + @Test + fun roundTripsEveryFormOfTheSameAddress() { + val forms = + listOf( + "201:d0e:9ba5:8bbc:0:0:0:1", + "201:0d0e:9ba5:8bbc:0000:0000:0000:0001", + "201:d0e:9ba5:8bbc::1", + "201:D0E:9BA5:8BBC::0001", + ) + val canonicalForms = forms.map { canonical(it) }.toSet() + assertEquals(setOf("201:d0e:9ba5:8bbc::1"), canonicalForms) + } +} diff --git a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/YggdrasilCompatCharacterizationTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/YggdrasilCompatCharacterizationTest.kt index 638858dbd8..160ca7c9e2 100644 --- a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/YggdrasilCompatCharacterizationTest.kt +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/YggdrasilCompatCharacterizationTest.kt @@ -22,29 +22,30 @@ package com.vitorpamplona.quartz.nip01Core.relay import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer import com.vitorpamplona.quartz.nip01Core.relay.normalizer.isLocalHost +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.isOverlayNetwork import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.toHttp import okhttp3.Request import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFalse -import kotlin.test.assertNotEquals -import kotlin.test.assertNull +import kotlin.test.assertNotNull import kotlin.test.assertTrue /** - * Characterization of how relay URLs on an Yggdrasil overlay behave today. + * Relay URLs on an Yggdrasil overlay, end to end through the normalizer. * - * Yggdrasil gives every node an IPv6 address inside `0200::/7` (nodes) and hands out - * `0300::/8` subnets, with no DNS and no CA-issuable certificate. A relay on the mesh is - * therefore always reached as a **bracketed IPv6 literal over plain `ws://`** — a shape - * the relay stack only partially handles. + * Yggdrasil gives every node an IPv6 address inside `0200::/7` (nodes in `0200::/8`, subnets in + * `0300::/8`), with no DNS and no CA-issuable certificate. A relay on the mesh is therefore + * always a **bracketed IPv6 literal over plain `ws://`**. * - * These tests document the CURRENT behavior (including the gaps) so a later fix has a - * baseline to diff against. Each gap is marked GAP with what a user sees. + * The differential assertions against OkHttp are the point of this file living in + * `jvmAndroidTest`: OkHttp is what actually dials the socket, so a normalized url that + * disagrees with OkHttp's own canonical host is a relay the app tracks under a name it does + * not connect to. */ class YggdrasilCompatCharacterizationTest { - // Same node, three legal RFC 4291 spellings of one address. + // Same node, several legal RFC 4291 spellings of one address. private val canonical = "ws://[201:d0e:9ba5:8bbc::1]:8080" private val expanded = "ws://[201:0d0e:9ba5:8bbc:0000:0000:0000:0001]:8080" private val uppercase = "ws://[201:D0E:9BA5:8BBC::1]:8080" @@ -66,51 +67,90 @@ class YggdrasilCompatCharacterizationTest { } @Test - fun yggdrasilSubnetAddressesAndUppercaseHexWork() { + fun yggdrasilSubnetAddressesWork() { assertEquals("ws://[300:1b5d:d0e9:ba58::1]:4848/", "ws://[300:1b5d:d0e9:ba58::1]:4848".normalizeRelayUrl().url) - // Hex case IS folded, so the uppercase spelling collapses onto the canonical one. - assertEquals(canonical.normalizeRelayUrl(), uppercase.normalizeRelayUrl()) } /** - * GAP 1 — zero-compression is NOT canonicalized, so one relay gets two identities. - * - * `NormalizedRelayUrl` is the key of the connection pool, the relay-list sets, the NIP-11 - * cache and every per-relay stat map. OkHttp collapses both spellings to one host (below), - * so the app opens two sockets to the same relay and counts it twice everywhere. + * Every legal spelling of one address collapses to one [NormalizedRelayUrl] — the key of the + * connection pool, the relay-list sets, the NIP-11 cache and the per-relay stat maps. Without + * this the app dials one relay twice and shows it twice. */ @Test - fun gapZeroCompressionSplitsOneRelayIntoTwoIdentities() { - assertNotEquals(canonical.normalizeRelayUrl(), expanded.normalizeRelayUrl()) - // ...even though they are literally the same host on the wire: - assertEquals(host(canonical), host(expanded)) + fun everySpellingOfOneAddressIsOneRelay() { + val identities = listOf(canonical, expanded, uppercase).map { it.normalizeRelayUrl() }.toSet() + assertEquals(setOf("ws://[201:d0e:9ba5:8bbc::1]:8080/"), identities.map { it.url }.toSet()) + // ...and that one identity is the host OkHttp dials for all of them. + assertEquals(setOf("201:d0e:9ba5:8bbc::1"), listOf(canonical, expanded, uppercase).map { host(it) }.toSet()) + } + + @Test + fun normalizedIdentityAlwaysMatchesTheHostOkHttpDials() { + listOf( + "ws://[201:0d0e:9ba5:8bbc:0000:0000:0000:0001]:8080", + "ws://[300:1b5d:d0e9:ba58:0:0:0:1]:4848", + "ws://[2001:0DB8:0000:0000:0000:0000:0000:0001]:7777", + "ws://[::1]:4869", + ).forEach { raw -> + val normalized = raw.normalizeRelayUrl().url + assertEquals(host(normalized), host(raw), "identity for $raw disagrees with the dialed host") + } } /** - * GAP 2 — a schemeless IPv6 literal defaults to `wss://`. - * - * `isLocalHost()` only knows 127.0.0.1 / localhost / umbrel / 192.168. / .local, so an - * Yggdrasil address falls through to the clearnet default. No CA issues certificates for - * `0200::/7` literals, so the resulting wss:// url can only ever fail its TLS handshake. + * A schemeless overlay address defaults to `ws://`: no CA issues certificates for + * `0200::/7`, so `wss://` could only ever fail its handshake. The mesh already encrypts + * end to end, so this is not a downgrade. */ @Test - fun gapSchemelessYggdrasilAddressDefaultsToWss() { - assertEquals("wss://[201:d0e:9ba5:8bbc::1]:8080/", "[201:d0e:9ba5:8bbc::1]:8080".normalizeRelayUrl().url) + fun schemelessOverlayAddressDefaultsToWs() { + assertEquals("ws://[201:d0e:9ba5:8bbc::1]:8080/", "[201:d0e:9ba5:8bbc::1]:8080".normalizeRelayUrl().url) + assertTrue("ws://[201:d0e:9ba5:8bbc::1]:8080/".normalizeRelayUrl().isOverlayNetwork()) + // A clearnet IPv6 relay keeps requiring TLS. + assertEquals("wss://[2001:db8::1]:8080/", "[2001:db8::1]:8080".normalizeRelayUrl().url) + assertFalse("wss://[2001:db8::1]:8080/".normalizeRelayUrl().isOverlayNetwork()) + } + + /** + * `::1`, `fc00::/7` and `fe80::/10` are the IPv6 twins of 127.0.0.1 and 192.168., so they + * answer [isLocalHost] the same way — no TLS, no Tor, never advertised to the network. + */ + @Test + fun ipv6LoopbackAndPrivateRangesCountAsLocalHost() { + assertEquals("ws://[::1]:4869/", "[::1]:4869".normalizeRelayUrl().url) + assertTrue("ws://[::1]:4869/".normalizeRelayUrl().isLocalHost()) + assertTrue("ws://[fd12:3456::1]:8080/".normalizeRelayUrl().isLocalHost(), "unique local address") + assertTrue("ws://[fe80::1]:8080/".normalizeRelayUrl().isLocalHost(), "link local address") + assertFalse("wss://[2001:db8::1]:8080/".normalizeRelayUrl().isLocalHost(), "clearnet ipv6") + // An overlay relay is reachable across the mesh, so it is NOT localhost. assertFalse("ws://[201:d0e:9ba5:8bbc::1]:8080/".normalizeRelayUrl().isLocalHost()) } /** - * GAP 3 — an unbracketed IPv6 literal is rejected outright. - * - * `yggdrasilctl getSelf` prints the address unbracketed, which is what a user copies into - * the "add a relay" field. Normalization returns null and `RelayUrlEditField.submitRelay` - * has no else branch, so the Add button silently does nothing. + * `yggdrasilctl getSelf` prints the address unbracketed, which is what a user pastes into + * the "add a relay" field. It is bracketed automatically rather than rejected. */ @Test - fun gapUnbracketedYggdrasilAddressIsRejected() { - assertNull(RelayUrlNormalizer.normalizeOrNull("201:d0e:9ba5:8bbc:f4a1:d34:1c2:eae5")) - assertNull(RelayUrlNormalizer.normalizeOrNull("201:d0e:9ba5:8bbc:f4a1:d34:1c2:eae5:8080")) - // Bracketing it by hand is the only accepted form. - assertTrue(RelayUrlNormalizer.normalizeOrNull("[201:d0e:9ba5:8bbc:f4a1:d34:1c2:eae5]:8080") != null) + fun bareUnbracketedLiteralIsBracketedAutomatically() { + assertEquals( + "ws://[201:d0e:9ba5:8bbc:f4a1:d34:1c2:eae5]/", + "201:d0e:9ba5:8bbc:f4a1:d34:1c2:eae5".normalizeRelayUrl().url, + ) + assertEquals("ws://[201:d0e:9ba5:8bbc::1]/", "201:d0e:9ba5:8bbc::1".normalizeRelayUrl().url) + assertNotNull(RelayUrlNormalizer.normalizeOrNull("[201:d0e:9ba5:8bbc:f4a1:d34:1c2:eae5]:8080")) + } + + /** + * Auto-bracketing must not swallow the other colon-bearing strings that reach the + * normalizer. Only a string that parses as a whole IPv6 address is bracketed. + */ + @Test + fun autoBracketingDoesNotCaptureNonAddresses() { + assertEquals("wss://relay.example.com:8080/", "relay.example.com:8080".normalizeRelayUrl().url) + assertEquals("ws://localhost:4869/", "localhost:4869".normalizeRelayUrl().url) + // addressable-event pointer, not a relay + assertEquals(null, RelayUrlNormalizer.normalizeOrNull("31990:abcdef:mydtag")) + // two hex-looking groups are a host and a port, not an address + assertEquals("wss://abcd:1234/", "abcd:1234".normalizeRelayUrl().url) } } From 70d51cc98b8bea7e6ed3ad2d90573ab6c212b986 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 04:22:59 +0000 Subject: [PATCH 029/132] fix(relay): parse the authority instead of substring-matching the url MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audit of the IPv6 work found a family of bugs in isLocalHost/isOnion, most predating this branch, all with one root cause: the predicates ran `contains` over the whole url rather than parsing the authority. These decide whether a relay is exempt from Tor, and relay urls arrive from other people (NIP-65 lists, relay hints, r tags), so they are attacker-controlled input. - A path could impersonate the host. `wss://evil.example.com/127.0.0.1` answered isLocalHost() == true, so any relay list could hand the app a url that silently dropped its own Tor routing. The IPv6 lookup added earlier on this branch had the same flaw via `/[fd00::1]`, and IPv6 canonicalization could rewrite a path outright, corrupting the url. - `.onion:8080` never matched the `.onion/` test, so an onion relay on an explicit port was not treated as onion at all: never forced onto Tor, and its hostname went to the clearnet DNS resolver. The fully-qualified `.onion.` spelling missed the same way. - Host tests were case-sensitive, but fix() asks them before the RFC 3986 pass folds case, so LOCALHOST:8080 and ABC.ONION:8080 were handed a wss:// scheme neither host can serve. - Private IPv4 was substring-matched, which missed 10.0.0.5, 172.16.3.4 and 127.1.2.3 — a LAN relay got wss:// and was dialed through Tor — while matching 192.168.evil.com and 127.0.0.1.evil.com, registrable domains that could therefore exempt themselves from Tor. Same for notlocalhost.example.com against `contains("localhost")`. - A `://` inside a path was read as a scheme separator, so `relay.com/x://127.0.0.1` read its path as the authority. Fixes: a shared hostStart/hostEnd/hostEndWithoutPort trio bounds every test to the authority, strips :port and trailing dots and validates the scheme; private ranges are parsed via a new Ipv4 util rather than substring-matched; comparisons are case-insensitive per RFC 4343; NormalizedRelayUrl.isOnion() delegates instead of keeping a second, weaker copy of the test. No performance regression: the old form ran six full-string scans, the new one bounds its work to the authority and rejects a DNS host from an IP parse on one character. Ipv6.isLiteral gained a two-colon gate so the schemeless host:port case answers without allocating the parser's buffer. Ipv6 is now pinned by a differential test: 4000 random addresses round-trip against java.net.InetAddress in both directions, and the canonical form is asserted equal to OkHttp's host for the same address, so the relay identity the app stores provably matches the host it dials. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DQr8CDsznzCRUeB5tS8VYk --- .../relays/common/RelayUrlEditField.kt | 19 +- .../plans/2026-08-04-yggdrasil-ipv6-relays.md | 40 ++++ .../relay/normalizer/NormalizedRelayUrl.kt | 4 +- .../relay/normalizer/RelayUrlNormalizer.kt | 191 +++++++++++++++--- .../com/vitorpamplona/quartz/utils/Ipv4.kt | 99 +++++++++ .../com/vitorpamplona/quartz/utils/Ipv6.kt | 51 ++--- .../relay/RelayUrlAuthorityAnchoringTest.kt | 186 +++++++++++++++++ .../quartz/utils/Ipv6DifferentialTest.kt | 108 ++++++++++ 8 files changed, 623 insertions(+), 75 deletions(-) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/Ipv4.kt create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/RelayUrlAuthorityAnchoringTest.kt create mode 100644 quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/utils/Ipv6DifferentialTest.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/relays/common/RelayUrlEditField.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/relays/common/RelayUrlEditField.kt index d523058cfb..963ddb3724 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/relays/common/RelayUrlEditField.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/relays/common/RelayUrlEditField.kt @@ -200,14 +200,19 @@ fun RelayUrlEditField( relaySuggestions.processInput(it) }, isError = isInvalid, - supportingText = { + // Null, not an empty lambda: a non-null slot reserves its line height even when it + // draws nothing, which would pad the field permanently for every user. + supportingText = if (isInvalid) { - Text( - text = stringRes(R.string.relay_url_not_valid), - color = MaterialTheme.colorScheme.error, - ) - } - }, + { + Text( + text = stringRes(R.string.relay_url_not_valid), + color = MaterialTheme.colorScheme.error, + ) + } + } else { + null + }, placeholder = { Text( text = "server.com", diff --git a/quartz/plans/2026-08-04-yggdrasil-ipv6-relays.md b/quartz/plans/2026-08-04-yggdrasil-ipv6-relays.md index 33d8aaef31..412210a81b 100644 --- a/quartz/plans/2026-08-04-yggdrasil-ipv6-relays.md +++ b/quartz/plans/2026-08-04-yggdrasil-ipv6-relays.md @@ -77,6 +77,46 @@ branch. Both the Android and desktop relay paths delegate here (`TorRelayState`, treatment for non-relay HTTP (images, previews, NIP-05, money ops). Clearnet IPv6 relays keep following the Tor setting — asserted in `YggdrasilTorRoutingTest`. +## Audit round: bugs found in the host predicates + +Auditing the change above turned up a family of bugs in `isLocalHost` / `isOnion` that predate +it. All shared one root cause — the predicates ran `contains` over the **whole url** instead of +parsing the authority — and all are now anchored, parsed and covered by +`RelayUrlAuthorityAnchoringTest`. + +These predicates decide whether a relay is exempt from Tor, and relay urls arrive from other +people (NIP-65 lists, relay hints, `r` tags), so they are attacker-controlled input. + +| # | Bug | Effect | +|---|---|---| +| 1 | A path could impersonate the host: `wss://evil.example.com/127.0.0.1` answered `isLocalHost() == true` | Any relay list could hand the app a url that silently dropped its own Tor routing | +| 2 | `.onion:8080` never matched the `.onion/` test | An onion relay on an explicit port was not treated as onion — never forced onto Tor, hostname sent to the clearnet DNS resolver | +| 3 | `.onion.` / `localhost.` (RFC 1034 fully-qualified form) matched nothing | Same leak as #2, via a different spelling | +| 4 | Host tests were case-sensitive, but `fix()` runs *before* the RFC 3986 pass folds case | `LOCALHOST:8080` and `ABC.ONION:8080` were given a `wss://` scheme neither host can serve | +| 5 | Private IPv4 was substring-matched | `10.0.0.5`, `172.16.3.4`, `127.1.2.3` were not local (LAN relay got `wss://` and Tor), while `192.168.evil.com` — a registrable domain — was | +| 6 | `contains("localhost")` matched `notlocalhost.example.com` | Same Tor exemption as #1, via a registrable domain | +| 7 | A `://` inside a path was read as a scheme separator | `relay.com/x://127.0.0.1` read its path as the authority | + +Two bugs were introduced by this branch and caught in the same pass: the IPv6 host lookup had +the #1 flaw (`wss://evil.example.com/[fd00::1]` read as localhost), and IPv6 canonicalization +could rewrite a **path** (`/x[0:0:0:0:0:0:0:1]y` → `/x[::1]y`), corrupting the url. + +Fixes: a shared `hostStart` / `hostEnd` / `hostEndWithoutPort` trio bounds every test to the +authority, strips `:port` and trailing dots, and validates the scheme; private ranges are +parsed via the new `Ipv4` util and `Ipv6` rather than substring-matched; comparisons are +case-insensitive per RFC 4343; `NormalizedRelayUrl.isOnion()` now delegates instead of keeping +a second, weaker copy of the test. + +Performance: no regression, likely a small win. The old form ran six full-string `contains` +scans; the new one bounds its work to the authority and rejects a DNS host from an IP parse on +a single character. `Ipv6.isLiteral` gained a two-colon gate so the schemeless `host:port` case +answers without allocating the parser's 16-byte buffer. + +`Ipv6` itself is pinned by `Ipv6DifferentialTest`: 4000 random addresses round-trip against +`java.net.InetAddress` in both directions, and the canonical form is asserted equal to +OkHttp's host for the same address — so the relay identity the app stores provably matches the +host it dials. + ## Deliberately not changed **Mesh relays are still published and recommended.** `AdvertisedRelayInfoTag` (NIP-65) and diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/NormalizedRelayUrl.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/NormalizedRelayUrl.kt index 0196ac0bbf..5215e1f473 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/NormalizedRelayUrl.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/NormalizedRelayUrl.kt @@ -44,7 +44,9 @@ fun NormalizedRelayUrl.toHttp() = "https://$url" } -fun NormalizedRelayUrl.isOnion() = url.contains(".onion/") +// Delegates rather than re-implementing `contains(".onion/")`: that copy missed +// `wss://host.onion:8080/`, so an onion relay on an explicit port was never forced onto Tor. +fun NormalizedRelayUrl.isOnion() = RelayUrlNormalizer.isOnion(this.url) fun NormalizedRelayUrl.isLocalHost() = RelayUrlNormalizer.isLocalHost(this.url) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt index 18d83c66af..5d68b066f6 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/normalizer/RelayUrlNormalizer.kt @@ -21,6 +21,7 @@ package com.vitorpamplona.quartz.nip01Core.relay.normalizer import androidx.collection.LruCache +import com.vitorpamplona.quartz.utils.Ipv4 import com.vitorpamplona.quartz.utils.Ipv6 import com.vitorpamplona.quartz.utils.Log import com.vitorpamplona.quartz.utils.Rfc3986 @@ -39,25 +40,60 @@ val normalizedUrls = LruCache(5000) class RelayUrlNormalizer { companion object { - fun isLocalHost(url: String) = - url.contains("127.0.0.1") || - url.contains("localhost") || - url.contains("//umbrel:") || - url.contains("192.168.") || - url.contains(".local:") || - url.contains(".local/") || - isPrivateIpv6(url) + /** + * Every host test below is anchored to the **authority** (`host[:port]`), never to the + * whole url. A plain `contains` reads the path and query too, so + * `wss://evil.example.com/127.0.0.1` used to answer true here — and since [isLocalHost] + * is what exempts a relay from Tor, any relay list could hand the app a url that quietly + * dropped its own Tor routing. Relay urls arrive from other people (NIP-65 lists, relay + * hints, `r` tags), so they are attacker-controlled input and have to be parsed as such. + * + * Anchoring is also what makes `host:port` work: `.onion:8080` never matched the old + * `.onion/` test, so an onion relay on an explicit port was not recognized as onion at + * all and its hostname went to the clearnet DNS resolver. + */ + fun isLocalHost(url: String): Boolean { + val start = hostStart(url) + val end = hostEnd(url, start) + if (end <= start) return false + val hostEnd = hostEndWithoutPort(url, start, end) + return isPrivateIpv4(url, start, hostEnd) || + // RFC 6761: `localhost` and anything under it — the same rule SurgeDns applies + // when deciding whether a loopback answer is legitimate. A substring test would + // also match `notlocalhost.example.com`, which is registrable. + regionEquals(url, "localhost", start, hostEnd) || + regionEndsWith(url, ".localhost", start, hostEnd) || + regionEquals(url, "umbrel", start, hostEnd) || + regionEndsWith(url, ".local", start, hostEnd) || + isPrivateIpv6(url, start, end) + } /** - * The IPv6 twins of the literals above: `::1` (127.0.0.1), `fc00::/7` unique local - * addresses (192.168.0.0/16) and `fe80::/10` link-local. All three name a host that - * only exists on this machine or this LAN, which is what every caller of [isLocalHost] - * means by the question — so a relay on one must not be Torified, must not need TLS, - * and must not be advertised to the network. + * The IPv4 ranges that are never a public relay: `127.0.0.0/8` loopback, the RFC 1918 + * private blocks, `169.254.0.0/16` link-local and `0.0.0.0/8`. + * + * Parsed rather than substring-matched, which was wrong both ways: `contains("192.168.")` + * missed `10.0.0.5` and `172.16.3.4` — so a LAN relay was given `wss://` and dialed + * through Tor — while matching `192.168.evil.com`, a registrable domain that could + * therefore exempt itself from Tor. */ - private fun isPrivateIpv6(url: String): Boolean { - val bytes = ipv6HostOf(url) ?: return false - return Ipv6.isLoopback(bytes) || Ipv6.isUniqueLocal(bytes) || Ipv6.isLinkLocal(bytes) + private fun isPrivateIpv4( + url: String, + start: Int, + end: Int, + ): Boolean { + val bytes = Ipv4.parse(url, start, end) ?: return false + return Ipv4.isLoopback(bytes) || + Ipv4.isPrivate(bytes) || + Ipv4.isLinkLocal(bytes) || + Ipv4.isUnspecified(bytes) + } + + fun isOnion(url: String): Boolean { + val start = hostStart(url) + val end = hostEnd(url, start) + if (end <= start) return false + return regionEndsWith(url, ".onion", start, hostEndWithoutPort(url, start, end)) } /** @@ -72,26 +108,115 @@ class RelayUrlNormalizer { * which is derived from the peer's public key. */ fun isOverlayNetwork(url: String): Boolean { - val bytes = ipv6HostOf(url) ?: return false + val start = hostStart(url) + val bytes = ipv6HostOf(url, start, hostEnd(url, start)) ?: return false return Ipv6.isOverlayMesh(bytes) } /** - * Extracts the bracketed IPv6 host of [url] as raw bytes, dropping any `%zone` suffix. - * Returns null — cheaply, on a single `indexOf` — for the overwhelmingly common case of - * a url with a DNS host. + * The IPv6 twins of the literals in [isLocalHost]: `::1` (127.0.0.1), `fc00::/7` unique + * local addresses (192.168.0.0/16) and `fe80::/10` link-local. All three name a host that + * only exists on this machine or this LAN, which is what every caller of [isLocalHost] + * means by the question — so a relay on one must not be Torified, must not need TLS, + * and must not be advertised to the network. */ - private fun ipv6HostOf(url: String): ByteArray? { - val open = url.indexOf('[') - if (open < 0) return null - val close = url.indexOf(']', open + 1) - if (close <= open + 1) return null - val zone = url.indexOf('%', open + 1) - val end = if (zone in (open + 1) until close) zone else close - return Ipv6.parse(url.substring(open + 1, end)) + private fun isPrivateIpv6( + url: String, + start: Int, + end: Int, + ): Boolean { + val bytes = ipv6HostOf(url, start, end) ?: return false + return Ipv6.isLoopback(bytes) || Ipv6.isUniqueLocal(bytes) || Ipv6.isLinkLocal(bytes) } - fun isOnion(url: String) = url.endsWith(".onion") || url.contains(".onion/") + /** + * Parses the authority of [url] as a bracketed IPv6 literal, dropping any `%zone` suffix. + * Returns null — on a single char comparison — for the overwhelmingly common case of a + * url with a DNS host. + */ + private fun ipv6HostOf( + url: String, + start: Int, + end: Int, + ): ByteArray? { + if (start >= end || url[start] != '[') return null + val close = url.indexOf(']', start + 1) + if (close < 0 || close >= end || close <= start + 1) return null + val zone = url.indexOf('%', start + 1) + val addressEnd = if (zone in (start + 1) until close) zone else close + return Ipv6.parse(url.substring(start + 1, addressEnd)) + } + + /** + * Index of the first char of the authority: past `://`, or 0 for a schemeless host. + * + * The `://` only counts when what precedes it is a real RFC 3986 scheme + * (`ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )`). Otherwise `relay.com/x://127.0.0.1` + * would have its *path* read as the authority and answer true to [isLocalHost]. + */ + private fun hostStart(url: String): Int { + val scheme = url.indexOf("://") + if (scheme <= 0 || !url[0].isLetter()) return 0 + for (i in 1 until scheme) { + val c = url[i] + if (!c.isLetterOrDigit() && c != '+' && c != '-' && c != '.') return 0 + } + return scheme + 3 + } + + /** Index just past the authority — the first `/`, `?` or `#`, or the end of [url]. */ + private fun hostEnd( + url: String, + start: Int, + ): Int { + var i = start + while (i < url.length) { + val c = url[i] + if (c == '/' || c == '?' || c == '#') return i + i++ + } + return i + } + + /** + * [end] trimmed back past a `:port` and any trailing dots, so the host tests see the + * name alone. RFC 1034's fully-qualified form ends in a dot (`abc.onion.`), and missing + * that spelling on [isOnion] would send a `.onion` name to the clearnet DNS resolver. + */ + private fun hostEndWithoutPort( + url: String, + start: Int, + end: Int, + ): Int { + var stop = + if (url[start] == '[') { + // In an IPv6 authority only the `:` after the `]` can start a port. + val close = url.indexOf(']', start + 1) + if (close in start until end) close + 1 else end + } else { + val colon = url.lastIndexOf(':', end - 1) + if (colon >= start) colon else end + } + while (stop > start && url[stop - 1] == '.') stop-- + return stop + } + + // Host names are case-insensitive (RFC 4343), and `fix()` asks these questions *before* + // the RFC 3986 pass folds the case — so a case-sensitive test gave `LOCALHOST:8080` and + // `ABC.ONION:8080` a `wss://` scheme neither host can ever serve. + private fun regionEndsWith( + url: String, + suffix: String, + start: Int, + end: Int, + ): Boolean = end - start >= suffix.length && url.regionMatches(end - suffix.length, suffix, 0, suffix.length, ignoreCase = true) + + private fun regionEquals( + url: String, + name: String, + start: Int, + end: Int, + ): Boolean = end - start == name.length && url.regionMatches(start, name, 0, name.length, ignoreCase = true) fun isRelaySchemePrefix(url: String) = url.length > 6 && url[0] == 'w' && url[1] == 's' @@ -143,10 +268,12 @@ class RelayUrlNormalizer { * canonical, which is every url with a DNS host. */ private fun canonicalizeIpv6Host(url: String): String { - val open = url.indexOf('[') - if (open < 0) return url + // Anchored to the authority: a `[...]` in a path or query is data, and rewriting it + // would silently corrupt the url. + val open = hostStart(url) + if (open >= url.length || url[open] != '[') return url val close = url.indexOf(']', open + 1) - if (close <= open + 1) return url + if (close <= open + 1 || close >= hostEnd(url, open)) return url val inner = url.substring(open + 1, close) val canonical = Ipv6.canonicalizeOrNull(inner) ?: return url if (canonical == inner) return url diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/Ipv4.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/Ipv4.kt new file mode 100644 index 0000000000..41bc1c34d1 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/Ipv4.kt @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.utils + +/** + * Pure-Kotlin IPv4 literal parsing and range classification, the companion to [Ipv6]. + * + * Exists because asking "is this host private?" with `url.contains("192.168.")` is wrong in + * both directions: it misses `10.0.0.5` and `172.16.3.4` (so a LAN relay gets `wss://` and is + * dialed through Tor) and it matches `192.168.evil.com`, a perfectly registrable domain (so a + * hostile relay url can exempt itself from Tor). Parsing the host and testing the range is the + * only form of the question that has a right answer. + */ +object Ipv4 { + /** Parses a dotted quad in `[from, to)`, or null when the region is not one. */ + fun parse( + text: String, + from: Int, + to: Int, + ): ByteArray? { + // Cheapest possible rejection of a DNS host: a literal always starts with a digit. + if (from >= to || text[from] !in '0'..'9') return null + val out = ByteArray(4) + return if (parseInto(text, from, to, out, 0)) out else null + } + + /** + * Parses `a.b.c.d` in `[from, to)` into four bytes at [at]. Leading zeros are rejected — + * they invite the octal reading that makes `010.1.1.1` ambiguous across resolvers. + */ + fun parseInto( + text: String, + from: Int, + to: Int, + out: ByteArray, + at: Int, + ): Boolean { + var i = from + for (octet in 0 until 4) { + if (octet > 0) { + if (i >= to || text[i] != '.') return false + i++ + } + var value = 0 + var digits = 0 + while (i < to && text[i] in '0'..'9') { + if (digits == 3) return false + if (digits == 1 && value == 0) return false // leading zero + value = value * 10 + (text[i] - '0') + digits++ + i++ + } + if (digits == 0 || value > 255) return false + out[at + octet] = value.toByte() + } + return i == to + } + + /** `127.0.0.0/8` — the whole loopback block, not just 127.0.0.1. */ + fun isLoopback(bytes: ByteArray): Boolean = octet(bytes, 0) == 127 + + /** RFC 1918: `10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`. */ + fun isPrivate(bytes: ByteArray): Boolean { + val first = octet(bytes, 0) + val second = octet(bytes, 1) + return first == 10 || + (first == 172 && second in 16..31) || + (first == 192 && second == 168) + } + + /** `169.254.0.0/16` — link-local / APIPA, reachable only on the local segment. */ + fun isLinkLocal(bytes: ByteArray): Boolean = octet(bytes, 0) == 169 && octet(bytes, 1) == 254 + + /** `0.0.0.0/8` — "this network"; never a routable relay. */ + fun isUnspecified(bytes: ByteArray): Boolean = octet(bytes, 0) == 0 + + private fun octet( + bytes: ByteArray, + at: Int, + ) = bytes[at].toInt() and 0xFF +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/Ipv6.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/Ipv6.kt index 27d0d724ed..ec00f67a46 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/Ipv6.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/Ipv6.kt @@ -75,7 +75,7 @@ object Ipv6 { if (i < len && address[i] == '.') { // Trailing dotted quad: occupies the last four bytes, so nothing may follow it. if (fill > 12) return null - if (!parseIpv4Into(address, groupStart, len, out, fill)) return null + if (!Ipv4.parseInto(address, groupStart, len, out, fill)) return null fill += 4 i = len break @@ -178,8 +178,21 @@ object Ipv6 { return format(bytes) + address.substring(zoneAt) } - /** True when [address] is a valid bracket-less literal that names more than one group. */ - fun isLiteral(address: String): Boolean = address.indexOf(':') >= 0 && parse(address) != null + /** + * True when [address] is a valid bracket-less literal. + * + * The two-colon gate is what keeps this off the normalizer's hot path: every schemeless + * `host:port` reaching [com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer] + * has exactly one colon, and a literal needs at least two, so the common case answers + * without entering [parse] and allocating its 16-byte buffer. + */ + fun isLiteral(address: String): Boolean { + val firstColon = address.indexOf(':') + if (firstColon < 0 || address.indexOf(':', firstColon + 1) < 0) return false + // A zone id is part of the literal (`fe80::1%25eth0`), not a reason to reject it. + val zoneAt = address.indexOf('%') + return parse(if (zoneAt < 0) address else address.substring(0, zoneAt)) != null + } /** `::1` — the IPv6 loopback, twin of 127.0.0.1. */ fun isLoopback(bytes: ByteArray): Boolean { @@ -217,38 +230,6 @@ object Ipv6 { } } - /** - * Parses `a.b.c.d` in `[from, to)` into four bytes at [at]. Leading zeros are rejected — - * they invite the octal reading that makes `010.1.1.1` ambiguous across resolvers. - */ - private fun parseIpv4Into( - text: String, - from: Int, - to: Int, - out: ByteArray, - at: Int, - ): Boolean { - var i = from - for (octet in 0 until 4) { - if (octet > 0) { - if (i >= to || text[i] != '.') return false - i++ - } - var value = 0 - var digits = 0 - while (i < to && text[i] in '0'..'9') { - if (digits == 3) return false - if (digits == 1 && value == 0) return false // leading zero - value = value * 10 + (text[i] - '0') - digits++ - i++ - } - if (digits == 0 || value > 255) return false - out[at + octet] = value.toByte() - } - return i == to - } - private fun hexDigit(c: Char): Int = when (c) { in '0'..'9' -> c - '0' diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/RelayUrlAuthorityAnchoringTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/RelayUrlAuthorityAnchoringTest.kt new file mode 100644 index 0000000000..7d91ab2542 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/RelayUrlAuthorityAnchoringTest.kt @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay + +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.normalizeRelayUrl +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +/** + * [RelayUrlNormalizer.isLocalHost] and [RelayUrlNormalizer.isOnion] decide whether a relay is + * exempt from Tor, so they must read the authority and nothing else. Relay urls arrive from + * other people — NIP-65 lists, relay hints, `r` tags — so a url whose *path* can flip those + * answers is a url that can drop a Tor user's protection. + */ +class RelayUrlAuthorityAnchoringTest { + @Test + fun aPathCannotMakeAForeignHostLookLocal() { + listOf( + "wss://evil.example.com/127.0.0.1", + "wss://evil.example.com/localhost", + "wss://evil.example.com/192.168.1.1", + "wss://evil.example.com/umbrel", + "wss://evil.example.com/x.local/y", + "wss://evil.example.com/[fd00::1]", + "wss://evil.example.com/[::1]", + "wss://evil.example.com/?q=127.0.0.1", + "wss://evil.example.com/?q=[::1]", + ).forEach { + assertFalse(RelayUrlNormalizer.isLocalHost(it), "$it must not read as localhost") + } + } + + @Test + fun aPathCannotMakeAForeignHostLookLikeAnOverlayOrOnion() { + assertFalse(RelayUrlNormalizer.isOverlayNetwork("wss://evil.example.com/[201:d0e:9ba5:8bbc::1]")) + assertFalse(RelayUrlNormalizer.isOnion("wss://evil.example.com/?u=http://nos.lol/.onion/")) + assertFalse(RelayUrlNormalizer.isOnion("wss://evil.example.com/abc.onion")) + } + + @Test + fun realLocalAndOnionHostsStillMatch() { + listOf( + "ws://127.0.0.1:8080/", + "ws://localhost:4869/", + "ws://umbrel:4848/", + "ws://192.168.1.100:8080/", + "ws://myrelay.local:8080/", + "ws://myrelay.local/", + "ws://foo.localhost:8080/", + "ws://[::1]:4869/", + "ws://[fd12:3456::1]:8080/", + "ws://[fe80::1]/", + ).forEach { + assertTrue(RelayUrlNormalizer.isLocalHost(it), "$it must read as localhost") + } + // schemeless, as fix() sees it before choosing ws:// vs wss:// + assertTrue(RelayUrlNormalizer.isLocalHost("127.0.0.1:8080")) + assertTrue(RelayUrlNormalizer.isLocalHost("umbrel:4848")) + } + + /** + * `.onion:8080` never matched the old `.onion/` test, so an onion relay on an explicit port + * was not recognized as onion — it skipped the forced-Tor branch and its hostname went to + * the clearnet DNS resolver. + */ + @Test + fun onionRelaysOnAnExplicitPortAreRecognized() { + assertTrue(RelayUrlNormalizer.isOnion("wss://abc123.onion:8080/")) + assertTrue(RelayUrlNormalizer.isOnion("wss://abc123.onion/")) + assertTrue(RelayUrlNormalizer.isOnion("abc123.onion:8080")) + assertTrue(RelayUrlNormalizer.isOnion("abc123.onion")) + // and it now gets ws:// like any other onion relay + assertEquals("ws://abc123.onion:8080/", "abc123.onion:8080".normalizeRelayUrl().url) + assertFalse(RelayUrlNormalizer.isOnion("wss://notonion.example.com/")) + } + + /** Canonicalization must never rewrite anything outside the authority. */ + @Test + fun canonicalizationLeavesPathsAndQueriesAlone() { + assertEquals( + "wss://evil.example.com/x[0:0:0:0:0:0:0:1]y", + "wss://evil.example.com/x[0:0:0:0:0:0:0:1]y".normalizeRelayUrl().url, + ) + // ...while still folding a real IPv6 authority + assertEquals( + "wss://[::1]/x[0:0:0:0:0:0:0:1]y", + "wss://[0:0:0:0:0:0:0:1]/x[0:0:0:0:0:0:0:1]y".normalizeRelayUrl().url, + ) + } + + /** + * Private IPv4 was substring-matched, which was wrong in both directions. + */ + @Test + fun allPrivateIpv4RangesCountAsLocal() { + listOf( + "ws://127.0.0.1:8080/", + "ws://127.1.2.3:8080/", + "ws://10.0.0.5:4869/", + "ws://172.16.3.4:4869/", + "ws://172.31.255.1/", + "ws://192.168.1.5:4869/", + "ws://169.254.1.1:4869/", + "ws://0.0.0.0:4869/", + ).forEach { + assertTrue(RelayUrlNormalizer.isLocalHost(it), "$it must read as localhost") + } + // a LAN relay therefore gets ws://, not a wss:// that can never hold a certificate + assertEquals("ws://10.0.0.5:4869/", "10.0.0.5:4869".normalizeRelayUrl().url) + } + + @Test + fun publicIpv4AndPrivateLookalikeDomainsAreNotLocal() { + listOf( + "wss://127.0.0.1.evil.com/", + "wss://192.168.evil.com/", + "wss://10.0.0.5.evil.com/", + "wss://8.8.8.8:4869/", + "wss://172.32.0.1/", + "wss://193.168.1.5/", + "wss://relay.damus.io/", + "wss://notlocalhost.example.com/", + "wss://mylocalhost.io/", + ).forEach { + assertFalse(RelayUrlNormalizer.isLocalHost(it), "$it must not read as localhost") + } + } + + /** + * Host names are case-insensitive (RFC 4343), and `fix()` asks these questions before the + * RFC 3986 pass folds the case — so a case-sensitive test handed `LOCALHOST:8080` and + * `ABC.ONION:8080` a `wss://` scheme neither host can serve. + */ + @Test + fun hostTestsAreCaseInsensitive() { + assertTrue(RelayUrlNormalizer.isLocalHost("wss://LocalHost:8080/")) + assertTrue(RelayUrlNormalizer.isLocalHost("LOCALHOST:8080")) + assertTrue(RelayUrlNormalizer.isLocalHost("wss://MyRelay.LOCAL/")) + assertTrue(RelayUrlNormalizer.isOnion("wss://ABC123.ONION/")) + assertTrue(RelayUrlNormalizer.isOnion("ABC.ONION:8080")) + assertEquals("ws://localhost:8080/", "LOCALHOST:8080".normalizeRelayUrl().url) + assertEquals("ws://abc.onion:8080/", "ABC.ONION:8080".normalizeRelayUrl().url) + } + + /** RFC 1034's fully-qualified form ends in a dot; it names the same host. */ + @Test + fun trailingDotFqdnIsTheSameHost() { + assertTrue(RelayUrlNormalizer.isLocalHost("wss://localhost./")) + assertTrue(RelayUrlNormalizer.isLocalHost("wss://myrelay.local./")) + assertTrue(RelayUrlNormalizer.isOnion("wss://abc123.onion./")) + assertTrue(RelayUrlNormalizer.isOnion("wss://abc123.onion.:8080/")) + } + + /** + * A `://` inside a path is not a scheme separator; only a real RFC 3986 scheme starts the + * authority. Otherwise the path gets read as the host. + */ + @Test + fun aColonSlashSlashInThePathIsNotASchemeSeparator() { + assertFalse(RelayUrlNormalizer.isLocalHost("relay.example.com/x://127.0.0.1")) + assertFalse(RelayUrlNormalizer.isOnion("nos.lol/?u=x://abc.onion")) + // a real scheme still starts the authority + assertTrue(RelayUrlNormalizer.isLocalHost("wss://127.0.0.1/x://evil.com")) + } +} diff --git a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/utils/Ipv6DifferentialTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/utils/Ipv6DifferentialTest.kt new file mode 100644 index 0000000000..43b9934f21 --- /dev/null +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/utils/Ipv6DifferentialTest.kt @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.utils + +import okhttp3.HttpUrl.Companion.toHttpUrl +import java.net.InetAddress +import kotlin.random.Random +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * Differential tests for [Ipv6] against the two parsers that actually matter at runtime: the + * JDK's (what `InetAddress` will do with the host) and OkHttp's (what dials the socket). + * + * A hand-written address parser is exactly the kind of code that passes its own examples and + * then disagrees with the real world on the hundredth input, so this pins it against + * references over a deterministic random corpus rather than against more of my own examples. + */ +class Ipv6DifferentialTest { + /** + * Parses through the JDK. IPv4-mapped literals come back as an `Inet4Address` of 4 bytes, + * so they are widened back to the 16-byte mapped form — [Ipv6] keeps them at 16 bytes, + * which is also what OkHttp does. + */ + private fun jdkBytes(literal: String): ByteArray { + val raw = InetAddress.getByName("[$literal]").address + if (raw.size == 16) return raw + return ByteArray(16).also { + it[10] = 0xFF.toByte() + it[11] = 0xFF.toByte() + raw.copyInto(it, 12) + } + } + + private fun randomAddresses(count: Int): List { + val rnd = Random(20260805) + return List(count) { + ByteArray(16) { rnd.nextInt(256).toByte() }.also { bytes -> + // Sprinkle zero runs so every `::` compression path gets exercised. + val runStart = rnd.nextInt(8) * 2 + val runLen = rnd.nextInt(1, 5) * 2 + for (k in runStart until minOf(16, runStart + runLen)) bytes[k] = 0 + } + } + } + + @Test + fun ourTextParsesToTheSameBytesInTheJdk() { + randomAddresses(4000).forEach { bytes -> + val text = Ipv6.format(bytes) + assertTrue(Ipv6.parse(text)!!.contentEquals(bytes), "our own round trip failed for $text") + assertTrue(jdkBytes(text).contentEquals(bytes), "the JDK reads $text as a different address") + } + } + + @Test + fun theJdksTextParsesBackThroughUs() { + randomAddresses(2000).forEach { bytes -> + val jdkText = InetAddress.getByAddress(bytes).hostAddress!! + assertTrue(Ipv6.parse(jdkText)?.contentEquals(bytes) == true, "we cannot read the JDK's own rendering: $jdkText") + } + } + + /** + * The canonical form is the app's relay identity, so it has to equal the host OkHttp shows + * for the same address — otherwise the app keys a relay under a name it does not dial. + */ + @Test + fun ourCanonicalFormMatchesOkHttp() { + randomAddresses(2000).forEach { bytes -> + val text = Ipv6.format(bytes) + assertEquals("http://[$text]/".toHttpUrl().host, text) + } + } + + @Test + fun expandedSpellingsCollapseOntoOkHttpsHost() { + randomAddresses(500).forEach { bytes -> + // The fully expanded, zero-padded, uppercase spelling of the same address. + val expanded = + (0 until 8).joinToString(":") { g -> + val value = ((bytes[g * 2].toInt() and 0xFF) shl 8) or (bytes[g * 2 + 1].toInt() and 0xFF) + value.toString(16).padStart(4, '0').uppercase() + } + assertEquals(Ipv6.format(bytes), Ipv6.canonicalizeOrNull(expanded)) + assertEquals("http://[$expanded]/".toHttpUrl().host, Ipv6.canonicalizeOrNull(expanded)) + } + } +} From f9bef87160c2769dd1382841f9119c1fe58cc982 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Wed, 5 Aug 2026 01:47:53 -0400 Subject: [PATCH 030/132] style(desktop): import Compose symbols in NotificationSettingsScreen Follow-up to 8f8713d8 (nostr proposal 259a0bb1). CLAUDE.md forbids fully-qualified class names inline in function bodies; the merged proposal introduced one (androidx.compose.runtime.LaunchedEffect) and the file already carried four more that predate it. Import them all and reference them by simple name: LaunchedEffect, snapshotFlow, rememberCoroutineScope, LocalWindowInfo. Also rewrites two comments the proposal added: - the auto-enable comment was written in the first person and described the author's own earlier mistake; restate it as what the code does and which two paths it covers. - the "Turn on desktop notifications" comment claimed the button renders only when the user explicitly disabled notifications, but the guard is `!enabled` alone. Describe the actual condition and why it is enough. Drops a redundant `enabled = true` on that OutlinedButton (the default). No behaviour change. :desktopApp:compileKotlin and :commons:jvmTest green. Co-Authored-By: Claude Opus 5 (1M context) --- .../ui/settings/NotificationSettingsScreen.kt | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/settings/NotificationSettingsScreen.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/settings/NotificationSettingsScreen.kt index 161b929b68..3e559f4f92 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/settings/NotificationSettingsScreen.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/settings/NotificationSettingsScreen.kt @@ -39,13 +39,17 @@ import androidx.compose.material3.Switch import androidx.compose.material3.Text import androidx.compose.material3.TextButton import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope import androidx.compose.runtime.setValue +import androidx.compose.runtime.snapshotFlow import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalWindowInfo import androidx.compose.ui.unit.dp import com.vitorpamplona.amethyst.commons.moderation.notifications.HostOs import com.vitorpamplona.amethyst.commons.moderation.notifications.NotifKind @@ -112,7 +116,7 @@ fun NotificationSettingsScreen(onBack: (() -> Unit)? = null) { dispatcher?.nativeAvailable?.collectAsState() ?: remember { mutableStateOf(false) } ) - val coroutineScope = androidx.compose.runtime.rememberCoroutineScope() + val coroutineScope = rememberCoroutineScope() var testStatus by remember { mutableStateOf(null) } var requestingPermission by remember { mutableStateOf(false) } var sendingTest by remember { mutableStateOf(false) } @@ -120,33 +124,30 @@ fun NotificationSettingsScreen(onBack: (() -> Unit)? = null) { // Re-sync permission state whenever this screen enters composition // and whenever the window regains focus — user may have toggled // Amethyst in System Settings → Notifications while we were open. - val windowInfo = androidx.compose.ui.platform.LocalWindowInfo.current - androidx.compose.runtime.LaunchedEffect(dispatcher) { + val windowInfo = LocalWindowInfo.current + LaunchedEffect(dispatcher) { dispatcher?.refreshPermission() } - androidx.compose.runtime.LaunchedEffect(dispatcher, windowInfo) { - androidx.compose.runtime - .snapshotFlow { windowInfo.isWindowFocused } + LaunchedEffect(dispatcher, windowInfo) { + snapshotFlow { windowInfo.isWindowFocused } .collect { focused -> if (focused) dispatcher?.refreshPermission() } } - // Handle the still-broken case that the previous fix missed: - // the user granted OS permission in a prior session (either via - // the older "Enable OS notifications" button whose auto-enable - // guard I initially forgot, via System Settings directly, or on - // Windows/Linux where permissionState defaults to NotApplicable). - // When they come back to Settings, permissionState == Granted so - // the "Enable OS notifications" button doesn't render, the master - // switch is still OFF from first-launch defaults, and there is no - // affordance that both tells them what's wrong and fixes it in - // one click. Auto-enable once per screen entry when we detect - // "permission is fine, but master switch is off and the user - // has never explicitly disabled it". PreferencesNotificationSettings - // exposes [wasExplicitlyDisabled] so we don't overrule a deliberate - // opt-out. - androidx.compose.runtime.LaunchedEffect(permissionState, enabled) { + // Covers the two paths the earlier fix (e9475dd0) missed: the OS + // permission was already granted in a prior session (an older + // build asked, or the user allowed Amethyst in System Settings + // directly), and Windows/Linux, where permissionState is + // NotApplicable from startup. On both, permissionState is not + // NotRequested, so the "Enable OS notifications" button never + // renders, yet the master switch is still OFF from first-launch + // defaults — leaving no affordance that both explains the problem + // and fixes it. Auto-enable when the permission is fine, the + // master switch is off, and the user has never explicitly turned + // it off; [NotificationSettings.wasExplicitlyDisabled] is what + // keeps a deliberate opt-out from being overruled. + LaunchedEffect(permissionState, enabled) { val allowed = permissionState == PermissionState.Granted || permissionState == PermissionState.NotApplicable if (allowed && !enabled && !settings.wasExplicitlyDisabled()) { settings.setEnabled(true) @@ -240,15 +241,18 @@ fun NotificationSettingsScreen(onBack: (() -> Unit)? = null) { } } PermissionState.Granted, PermissionState.NotApplicable -> { - // Turn-on button: renders only when master switch is - // off *and* the user explicitly disabled it before. - // The LaunchedEffect above auto-enables the switch - // for the common "never touched it" path; this button - // is the recovery for the deliberate-opt-out path. + // Recovery affordance for the deliberate-opt-out path. + // The LaunchedEffect above already re-enables the + // switch for anyone who never touched it, so in + // practice the only state that still reaches here with + // `enabled == false` is an explicit opt-out. Guarding + // on `!enabled` alone (rather than also calling + // wasExplicitlyDisabled) keeps this a pure Compose + // state read and leaves the button visible for the one + // frame before the effect runs. if (!enabled) { OutlinedButton( onClick = { settings.setEnabled(true) }, - enabled = true, ) { Text("Turn on desktop notifications") } } OutlinedButton( From f1d2bdee49a3130ae2502615a5f6290a8ab71822 Mon Sep 17 00:00:00 2001 From: mstrofnone Date: Wed, 5 Aug 2026 15:56:00 +1000 Subject: [PATCH 031/132] ci(release): add libegl1 to arm64 .deb Depends MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compose-desktop skiko native shipped under ${app}/lib/app/libskiko-linux-arm64.so declares libEGL.so.1 in DT_NEEDED — the aarch64 skiko uses EGL alongside GLX, unlike the x86_64 skiko which only links libGL.so.1. jpackage --type deb only auto-generates Depends from dpkg-shlibdeps against the bundled JRE under lib/runtime/, NOT the app payload under lib/app/. As a result the arm64 .deb produced by the newly-added linux-arm64 CI leg lists libgl1/libglvnd0/libglx0 in Depends but not libegl1. On minimal aarch64 installs — Armbian Server + a lightweight WM, Raspberry Pi OS Lite + LXDE, or any distro base image without an EGL implementation pulled in transitively — Amethyst desktop crashes at startup with: Exception in thread "main" org.jetbrains.skiko.LibraryLoadException: Failed to loade library …/libskiko-linux-arm64.so Caused by: java.lang.UnsatisfiedLinkError: libEGL.so.1: cannot open shared object file: No such file or directory Neither jpackage nor the Compose Multiplatform 1.11 DSL exposes a way to add extra deb Depends, so we rewrite the .deb after the fact — same approach as scripts/relax-deb-libicu.sh (which handles the libicu SONAME divergence across Debian/Ubuntu releases). scripts/add-deb-libegl-dep.sh only touches .debs whose payload actually contains libskiko-linux-arm64.so, and is idempotent (skips if libegl1 is already listed). The workflow step is gated on matrix.arch == 'arm64' so the x64 .deb is untouched (its skiko does NOT NEED libEGL and its GLX-only path stays as-is). Local validation on Apple Silicon (native linux/arm64 in Docker): 1. Rebuilt v1.13.1 arm64 .deb from a110ce0a30's CI leg. 2. readelf -d libskiko-linux-arm64.so | grep NEEDED → confirms libEGL.so.1 3. Ran scripts/add-deb-libegl-dep.sh over the .deb; Depends line now ends `..., zlib1g, libegl1`. Idempotent on re-run. 4. `apt-get install -y -f ./amethyst_*.deb` in a base eclipse-temurin:21-jdk-noble aarch64 container (which lacks libegl1 by default) now pulls libegl1 as a dep. 5. Amethyst launches under Xvfb, `xwininfo -root -tree` shows the 1200×800 "Amethyst" window + Content window + sun-awt-X11-XCanvasPeer Skia canvas. No UnsatisfiedLinkError. --- .github/workflows/create-release.yml | 14 ++++++ scripts/add-deb-libegl-dep.sh | 68 ++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100755 scripts/add-deb-libegl-dep.sh diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index f87c0556f4..0dfea9bfcd 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -221,6 +221,20 @@ jobs: chmod +x scripts/relax-deb-libicu.sh scripts/relax-deb-libicu.sh desktopApp/build/compose/binaries/main-release/deb/*.deb + # jpackage --type deb only auto-generates Depends from dpkg-shlibdeps + # against the bundled JRE under lib/runtime/, NOT the app payload under + # lib/app/. libskiko-linux-arm64.so has libEGL.so.1 in DT_NEEDED (unlike + # the x64 skiko which only links libGL.so.1), so a minimal aarch64 + # install without EGL crashes at startup with: + # UnsatisfiedLinkError: libEGL.so.1: cannot open shared object file + # Rewrite the arm64 .deb to add libegl1 to Depends. x64 .deb is untouched. + - name: Add libegl1 dep to arm64 .deb + if: matrix.family == 'linux' && matrix.arch == 'arm64' + run: | + set -euo pipefail + chmod +x scripts/add-deb-libegl-dep.sh + scripts/add-deb-libegl-dep.sh desktopApp/build/compose/binaries/main-release/deb/*.deb + - name: Build portable archives (windows + linux-portable) if: matrix.family == 'windows' || matrix.family == 'linux-portable' run: | diff --git a/scripts/add-deb-libegl-dep.sh b/scripts/add-deb-libegl-dep.sh new file mode 100755 index 0000000000..09b778052c --- /dev/null +++ b/scripts/add-deb-libegl-dep.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Ensure a jpackage-built desktop .deb declares libegl1 as a runtime dep on +# arm64. +# +# Why this is needed: the compose-desktop skiko native shipped in +# ${app}/lib/app/libskiko-linux-arm64.so has libEGL.so.1 in DT_NEEDED (the +# aarch64 build uses EGL alongside GLX, unlike the x86_64 skiko which only +# links libGL.so.1). jpackage's --type deb only auto-generates Depends from +# dpkg-shlibdeps against the bundled JRE under ${app}/lib/runtime/, NOT the +# ${app}/lib/app/ tree — so libegl1 never makes it into the arm64 .deb. +# +# On most desktop Linux systems libegl1 is already installed as a transitive +# of the desktop environment. But minimal aarch64 installs (Armbian Server + +# a lightweight WM, Raspberry Pi OS Lite + LXDE, etc.) can miss it. Without +# libegl1 the app dies at startup with: +# +# Exception in thread "main" org.jetbrains.skiko.LibraryLoadException: +# Failed to loade library …/libskiko-linux-arm64.so +# Caused by: java.lang.UnsatisfiedLinkError: +# libEGL.so.1: cannot open shared object file: No such file or directory +# +# Neither jpackage nor the Compose Multiplatform 1.11 DSL exposes a way to +# override the auto-generated Depends, so we rewrite the .deb after the fact +# (same approach as scripts/relax-deb-libicu.sh). +# +# Usage: add-deb-libegl-dep.sh [ ...] +set -euo pipefail + +for deb in "$@"; do + if [[ ! -f "$deb" ]]; then + echo "skip: not a file: $deb" >&2 + continue + fi + + work="$(mktemp -d)" + trap 'rm -rf "$work"' EXIT + dpkg-deb -R "$deb" "$work/pkg" + control="$work/pkg/DEBIAN/control" + + # Only touch .debs whose payload actually contains the arm64 skiko native. + # Applying this to x64 .debs is harmless but the whole point is to be + # surgical. + if ! find "$work/pkg" -type f -name 'libskiko-linux-arm64.so' | grep -q .; then + echo "No arm64 skiko in payload, leaving as-is: $deb" + rm -rf "$work" + trap - EXIT + continue + fi + + if grep -qE '(^| )libegl1( |,|$)' "$control"; then + echo "libegl1 already in Depends, leaving as-is: $deb" + rm -rf "$work" + trap - EXIT + continue + fi + + # Append libegl1 to the Depends line. jpackage-generated lines are single + # physical lines, e.g. + # Depends: libasound2t64, ..., zlib1g + # We insert `, libegl1` before the trailing newline. + sed -i -E 's/^(Depends: .*[^,[:space:]])[[:space:]]*$/\1, libegl1/' "$control" + + dpkg-deb --root-owner-group -Zxz -b "$work/pkg" "$deb" >/dev/null + echo "Added libegl1 dep: $deb" + + rm -rf "$work" + trap - EXIT +done From dc45477deb43bd01a63b426394b528ee0eec47dc Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 14:08:01 +0000 Subject: [PATCH 032/132] fix: don't glue quotes onto detected urls A bare host wrapped in quotes ("relay.momostr.pink") was detected with the opening quote attached, so the rendered link read `"relay.momostr.pink` and pointed at a host that does not exist. The mirror case was also wrong: a quoted url with a path/query/fragment kept the closing quote, because those readers only stop on a space. Quotes are not host characters, so they now end the current token exactly like a space does in readDefault (covering the leading quote and a quote glued to a previous word, e.g. `href="www.google.com"`), and they were added to CANNOT_BEGIN_URLS_WITH / CANNOT_END_URLS_WITH so a trailing quote read as part of a path, query or fragment is stripped on readEnd. The set covers the ascii quotes plus the typographic family, including the guillemets below the international-character threshold that the ascii boundary rule never cut. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GKCAegYMF9V9Nb8FHcMvT7 --- .../richtext/RichTextParserQuotedUrlTest.kt | 76 +++++++++++++++++++ .../commons/richtext/UrlParserTest.kt | 28 +++++++ .../urldetector/detection/UrlDetector.kt | 40 +++++++++- .../urldetector/detection/UriDetectionTest.kt | 32 ++++++++ 4 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserQuotedUrlTest.kt diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserQuotedUrlTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserQuotedUrlTest.kt new file mode 100644 index 0000000000..595e395bbe --- /dev/null +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserQuotedUrlTest.kt @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.richtext + +import com.vitorpamplona.amethyst.commons.model.EmptyTagList +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * A quoted host name (`this bridge-relay "relay.momostr.pink" doesn't appear`) used to be + * detected with the opening quote glued onto it, so the rendered link read + * `"relay.momostr.pink` and pointed at a host that doesn't exist. Quotes are not host + * characters, so they must be left in the surrounding text on both sides. + */ +class RichTextParserQuotedUrlTest { + private fun segmentsOf(text: String) = + RichTextParser() + .parseText(text, EmptyTagList, null) + .paragraphs + .flatMap { it.words } + + @Test + fun quotedSchemelessUrlKeepsQuotesOutOfTheLink() { + val segments = + segmentsOf( + "It seems like this bridge-relay \"relay.momostr.pink\" doesn't appear in the feed", + ).filterIsInstance() + + assertEquals(listOf("relay.momostr.pink"), segments.map { it.segmentText }) + } + + @Test + fun quotedUrlWithSchemeKeepsQuotesOutOfTheLink() { + val segments = + segmentsOf( + "the docs are at \"https://example.com/some/page?a=b\" if you need them", + ).filterIsInstance() + + assertEquals(listOf("https://example.com/some/page?a=b"), segments.map { it.segmentText }) + } + + @Test + fun quotedRelayUrlKeepsQuotesOutOfTheLink() { + val segments = + segmentsOf("add \"wss://relay.momostr.pink\" to your list") + .filterIsInstance() + + assertEquals(listOf("wss://relay.momostr.pink"), segments.map { it.segmentText }) + } + + @Test + fun apostrophesInProseDontCreateLinks() { + val segments = segmentsOf("it doesn't appear until you go into the authors' accounts") + + assertEquals(emptyList(), segments.filterIsInstance()) + assertEquals(emptyList(), segments.filterIsInstance()) + } +} diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/UrlParserTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/UrlParserTest.kt index f1444b88cc..c0c858b47d 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/UrlParserTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/UrlParserTest.kt @@ -349,6 +349,34 @@ class UrlParserTest { Urls(withScheme = setOf("https://test.com")), ) + @Test + fun testQuotedRelayName() = + test( + "It seems like this bridge-relay \"relay.momostr.pink\" doesn't appear in the feed at all", + Urls(withoutScheme = setOf("relay.momostr.pink")), + ) + + @Test + fun testSingleQuotedRelayName() = + test( + "It seems like this bridge-relay 'relay.momostr.pink' doesn't appear in the feed at all", + Urls(withoutScheme = setOf("relay.momostr.pink")), + ) + + @Test + fun testQuotedUrlWithScheme() = + test( + "the docs are at \"https://example.com/some/page?a=b\" if you need them", + Urls(withScheme = setOf("https://example.com/some/page?a=b")), + ) + + @Test + fun testQuotedRelayUrl() = + test( + "add \"wss://relay.momostr.pink\" to your list", + Urls(relayUrls = setOf("wss://relay.momostr.pink")), + ) + @Test fun testBlossom() { val blossom = "blossom:b1674191a88ec5cdd733e4240a81803105dc412d6c6708d53ab94fc248f4f553.pdf?xs=cdn.satellite.earth" diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UrlDetector.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UrlDetector.kt index cf2e8a5f4c..ae05dee3da 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UrlDetector.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UrlDetector.kt @@ -91,7 +91,17 @@ class UrlDetector( while (!reader.eof()) { // read the next char to process. when (val curr = reader.read()) { - ' ' -> { + // A quote can never be part of a host name, so a note that wraps a bare domain in + // quotes (`the relay "relay.example.com" is down`) must not glue the opening quote + // onto the url. Quotes therefore end the current token exactly like a space does. + // Kept as literals (instead of `in QUOTES`) so this hot branch stays a tableswitch; + // the list must mirror [QUOTES], which the punctuation round-trip test enforces. + ' ', '"', '\'', '`', + '\u00AB', '\u00BB', + '\u2018', '\u2019', '\u201A', '\u201B', + '\u201C', '\u201D', '\u201E', '\u201F', + '\u2039', '\u203A', + -> { // space found; if we have a scheme, attempt to read the domain before resetting if (buffer.isNotEmpty() && hasScheme) { reader.goBack() @@ -719,6 +729,30 @@ class UrlDetector( "$it//" } + /** + * Quotes never belong to a url. The opening side is already dropped when [readDefault] + * breaks the token on them, but the closing side can still be swallowed by the path, + * query or fragment readers (which only stop on a space), so it is stripped on [readEnd]. + */ + val QUOTES = + setOf( + '"', + '\'', + '`', + '\u00AB', + '\u00BB', + '\u2018', + '\u2019', + '\u201A', + '\u201B', + '\u201C', + '\u201D', + '\u201E', + '\u201F', + '\u2039', + '\u203A', + ) + val CANNOT_BEGIN_URLS_WITH = setOf( ',', @@ -734,7 +768,7 @@ class UrlDetector( '\u3002', '\uFF0E', '\uFF61', - ) + ) + QUOTES val CANNOT_END_URLS_WITH = setOf( @@ -752,6 +786,6 @@ class UrlDetector( '\u3002', '\uFF0E', '\uFF61', - ) + ) + QUOTES } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UriDetectionTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UriDetectionTest.kt index 76ffc005e8..4a4c45fdc8 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UriDetectionTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UriDetectionTest.kt @@ -882,6 +882,38 @@ class UriDetectionTest { runTest("visit example.com,", "example.com") } + @Test + fun testQuotedUrlsDropTheQuotes() { + // a bare domain wrapped in quotes must not glue the opening quote onto the host. + runTest( + "It seems like this bridge-relay \"relay.momostr.pink\" doesn't appear in the feed", + "relay.momostr.pink", + ) + + UrlDetector.QUOTES.forEach { quote -> + runTest("$quote relay.momostr.pink $quote", "relay.momostr.pink") + runTest("${quote}relay.momostr.pink$quote", "relay.momostr.pink") + runTest("${quote}wss://relay.momostr.pink$quote", "wss://relay.momostr.pink") + + // the closing quote is swallowed by the path/query/fragment readers, which only stop + // on a space, so it has to be stripped at the end of the url instead. + runTest("say ${quote}https://example.com/foo$quote out", "https://example.com/foo") + runTest("say ${quote}https://example.com/foo?a=b$quote out", "https://example.com/foo?a=b") + runTest("say ${quote}https://example.com/foo#b$quote out", "https://example.com/foo#b") + + // a quote glued to the previous word is a boundary too: `href="www.google.com"`. + runTest("href=${quote}www.google.com$quote", "www.google.com") + } + } + + @Test + fun testApostropheStillEndsTheHostForGroupInviteLinks() { + // NIP-29 invite links are `'`; UrlParser recovers the suffix from the + // content, so the detector must keep reporting the relay url alone. + runTest("wss://relay.example.com'groupid", "wss://relay.example.com") + runTest("wss://relay.example.com'groupid?code=xyz", "wss://relay.example.com") + } + private fun runTest( text: String, vararg expected: String?, From 5c5644405482cb515d98b8a97c7362b1e679a757 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 14:28:24 +0000 Subject: [PATCH 033/132] fix: strip the whole punctuation tail from a detected url MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audit follow-up to the quote fix. The path, query and fragment readers only stop on a space, and readEnd dropped a single trailing delimiter, so a quoted link that closed a sentence kept its quote: He linked "https://example.com/some/path". -> https://example.com/some/path" (see "https://example.com/some/path") -> https://example.com/some/path" readEnd now strips the tail in a loop. The balance check runs on every round, so a url that legitimately ends in a matched closer still stops the strip: `[link](…/Bitcoin_(disambiguation)).` keeps `(disambiguation)` and drops the `).` that belongs to the sentence. Differential run over a 4000-string corpus against the previous commit: 8 rows change, every one of them the removal of extra trailing punctuation. No url is gained, lost or truncated mid-string. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GKCAegYMF9V9Nb8FHcMvT7 --- .../urldetector/detection/UrlDetector.kt | 10 ++++++-- .../urldetector/detection/UriDetectionTest.kt | 23 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UrlDetector.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UrlDetector.kt index ae05dee3da..c353f462f9 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UrlDetector.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UrlDetector.kt @@ -659,8 +659,14 @@ class UrlDetector( // if the url is valid and greater then 0 if (state == ReadEndState.ValidUrl && buffer.isNotEmpty()) { var url = buffer.toString() - val last = url.lastOrNull() - if (last != null && last in CANNOT_END_URLS_WITH && !url.endsOnBalancedCloser(last)) { + // Strips the whole punctuation tail, not just its last character: the path, query and + // fragment readers only stop on a space, so a quoted link closing a sentence arrives + // here as `https://host/path".` and a single drop would leave the quote glued on. + // Each round re-checks the balance, so a url that legitimately ends in a matched + // closer (`…/Bitcoin_(disambiguation)`) still stops the strip. + while (true) { + val last = url.lastOrNull() ?: break + if (last !in CANNOT_END_URLS_WITH || url.endsOnBalancedCloser(last)) break url = url.dropLast(1) } if (url.isNotEmpty()) urlList.add(currentUrlMarker.createUrl(url)) diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UriDetectionTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UriDetectionTest.kt index 4a4c45fdc8..da2c80c169 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UriDetectionTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/utils/urldetector/detection/UriDetectionTest.kt @@ -906,6 +906,29 @@ class UriDetectionTest { } } + @Test + fun testWholePunctuationTailIsStripped() { + // The path/query/fragment readers only stop on a space, so a quoted link that closes a + // sentence reaches readEnd as `https://host/path".` — every trailing delimiter has to go, + // not just the last one. + runTest("He linked \"https://example.com/some/path\".", "https://example.com/some/path") + runTest("(see \"https://example.com/some/path\")", "https://example.com/some/path") + runTest("read \"https://example.com/a?b=c\", then go", "https://example.com/a?b=c") + runTest("\"https://example.com/x\"!", "https://example.com/x") + runTest("\"https://example.com/x#frag\"...", "https://example.com/x#frag") + runTest("wait... example.com/path...", "example.com/path") + + // a balanced closer still stops the strip, even behind a longer tail. + runTest( + "(see https://en.wikipedia.org/wiki/Bitcoin_(disambiguation))", + "https://en.wikipedia.org/wiki/Bitcoin_(disambiguation)", + ) + runTest( + "[link](https://en.wikipedia.org/wiki/Bitcoin_(disambiguation)).", + "https://en.wikipedia.org/wiki/Bitcoin_(disambiguation)", + ) + } + @Test fun testApostropheStillEndsTheHostForGroupInviteLinks() { // NIP-29 invite links are `'`; UrlParser recovers the suffix from the From 9ac033effcf5964457984561f53cf34be22ce7fd Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 15:41:29 +0000 Subject: [PATCH 034/132] fix(ime): use the keyboard-aware back handler in the post composers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The thread reply composer (and every other full-screen draft-saving editor) still consumed back with a raw `BackHandler`, so `KeyboardAwareBackHandler` — added for exactly this case — only protected the three chat composers. Popping the screen while the keyboard is still up races the predictive-back window animation against the IME close animation. When the window animation wins, the IME `WindowInsetsAnimationCompat` is cancelled before its terminal zero frame reaches Compose, the shared `WindowInsets.ime` holder stays "animating", and every `Modifier.imePadding()` freezes at keyboard height — the keyboard vanishes but its padding stays behind, even after leaving the screen. Switching these composers to `KeyboardAwareBackHandler` lets the first back (or back-swipe) fall through to the system, which dismisses the keyboard with its own animation that completes cleanly; the next back saves the draft and pops as before. The top bar's cancel arrow remains an always-available exit. Covers `ShortNotePostScreen` (which also backs `PollPostScreen`), `GenericCommentPostScreen`, `LongFormPostScreen`, `NewProductScreen`, `NewPublicMessageScreen`, `NewGoalScreen`, `NewWorkoutScreen` and `AwardBadgeScreen`. `VoiceReplyScreen` keeps the plain handler — it has no text input or `imePadding()`. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LfUMGWYu2uTSyh17JJonfN --- .../ui/note/nip22Comments/GenericCommentPostScreen.kt | 4 ++-- .../ui/screen/loggedIn/badges/award/AwardBadgeScreen.kt | 4 ++-- .../loggedIn/discover/nip23LongForm/LongFormPostScreen.kt | 4 ++-- .../loggedIn/discover/nip99Classifieds/NewProductScreen.kt | 4 ++-- .../amethyst/ui/screen/loggedIn/home/ShortNotePostScreen.kt | 4 ++-- .../ui/screen/loggedIn/home/nip75Goals/NewGoalScreen.kt | 4 ++-- .../notifications/publicMessages/NewPublicMessageScreen.kt | 4 ++-- .../amethyst/ui/screen/loggedIn/workouts/NewWorkoutScreen.kt | 4 ++-- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/nip22Comments/GenericCommentPostScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/nip22Comments/GenericCommentPostScreen.kt index 6baef9f1a2..aec22cb911 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/nip22Comments/GenericCommentPostScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/nip22Comments/GenericCommentPostScreen.kt @@ -20,7 +20,6 @@ */ package com.vitorpamplona.amethyst.ui.note.nip22Comments -import androidx.activity.compose.BackHandler import androidx.compose.foundation.clickable import androidx.compose.foundation.horizontalScroll import androidx.compose.foundation.layout.Box @@ -67,6 +66,7 @@ import com.vitorpamplona.amethyst.ui.actions.uploads.SelectFromGallery import com.vitorpamplona.amethyst.ui.actions.uploads.SelectedMedia import com.vitorpamplona.amethyst.ui.actions.uploads.TakePictureButton import com.vitorpamplona.amethyst.ui.actions.uploads.TakeVideoButton +import com.vitorpamplona.amethyst.ui.navigation.bottombars.KeyboardAwareBackHandler import com.vitorpamplona.amethyst.ui.navigation.navs.Nav import com.vitorpamplona.amethyst.ui.navigation.topbars.PostingTopBar import com.vitorpamplona.amethyst.ui.note.BaseUserPicture @@ -177,7 +177,7 @@ fun GenericCommentPostScreen( StrippingFailureDialog(postViewModel.strippingFailureConfirmation) - BackHandler { + KeyboardAwareBackHandler { accountViewModel.launchSigner { postViewModel.sendDraftSync() postViewModel.cancel() diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/badges/award/AwardBadgeScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/badges/award/AwardBadgeScreen.kt index 2172e18945..bdc768cc3c 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/badges/award/AwardBadgeScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/badges/award/AwardBadgeScreen.kt @@ -20,7 +20,6 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.badges.award -import androidx.activity.compose.BackHandler import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.Spacer @@ -52,6 +51,7 @@ import androidx.lifecycle.viewmodel.compose.viewModel import com.vitorpamplona.amethyst.R import com.vitorpamplona.amethyst.commons.ui.components.Nip05OrPubkeyLine import com.vitorpamplona.amethyst.model.User +import com.vitorpamplona.amethyst.ui.navigation.bottombars.KeyboardAwareBackHandler import com.vitorpamplona.amethyst.ui.navigation.navs.INav import com.vitorpamplona.amethyst.ui.navigation.topbars.SavingTopBar import com.vitorpamplona.amethyst.ui.note.UserPicture @@ -88,7 +88,7 @@ fun AwardBadgeScreen( onDispose { userSuggestions.reset() } } - BackHandler { + KeyboardAwareBackHandler { nav.popBack() } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/discover/nip23LongForm/LongFormPostScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/discover/nip23LongForm/LongFormPostScreen.kt index 2465929d40..12cba33c60 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/discover/nip23LongForm/LongFormPostScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/discover/nip23LongForm/LongFormPostScreen.kt @@ -20,7 +20,6 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.discover.nip23LongForm -import androidx.activity.compose.BackHandler import androidx.compose.foundation.BorderStroke import androidx.compose.foundation.border import androidx.compose.foundation.clickable @@ -96,6 +95,7 @@ import com.vitorpamplona.amethyst.ui.actions.uploads.TakeVideoButton import com.vitorpamplona.amethyst.ui.components.MyAsyncImage import com.vitorpamplona.amethyst.ui.components.ThinPaddingTextField import com.vitorpamplona.amethyst.ui.components.markdown.RenderContentAsMarkdown +import com.vitorpamplona.amethyst.ui.navigation.bottombars.KeyboardAwareBackHandler import com.vitorpamplona.amethyst.ui.navigation.navs.Nav import com.vitorpamplona.amethyst.ui.navigation.topbars.PostingTopBar import com.vitorpamplona.amethyst.ui.note.creators.contentWarning.ContentSensitivityExplainer @@ -149,7 +149,7 @@ fun LongFormPostScreen( StrippingFailureDialog(postViewModel.strippingFailureConfirmation) - BackHandler { + KeyboardAwareBackHandler { accountViewModel.launchSigner { postViewModel.sendDraftSync() postViewModel.cancel() diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/discover/nip99Classifieds/NewProductScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/discover/nip99Classifieds/NewProductScreen.kt index 2e95142599..8cd67d7b46 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/discover/nip99Classifieds/NewProductScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/discover/nip99Classifieds/NewProductScreen.kt @@ -20,7 +20,6 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.discover.nip99Classifieds -import androidx.activity.compose.BackHandler import androidx.compose.foundation.horizontalScroll import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row @@ -53,6 +52,7 @@ import com.vitorpamplona.amethyst.ui.actions.uploads.SelectFromGallery import com.vitorpamplona.amethyst.ui.actions.uploads.SelectedMedia import com.vitorpamplona.amethyst.ui.actions.uploads.TakePictureButton import com.vitorpamplona.amethyst.ui.actions.uploads.TakeVideoButton +import com.vitorpamplona.amethyst.ui.navigation.bottombars.KeyboardAwareBackHandler import com.vitorpamplona.amethyst.ui.navigation.navs.INav import com.vitorpamplona.amethyst.ui.navigation.navs.Nav import com.vitorpamplona.amethyst.ui.navigation.topbars.PostingTopBar @@ -141,7 +141,7 @@ fun NewProductScreen( StrippingFailureDialog(postViewModel.strippingFailureConfirmation) - BackHandler { + KeyboardAwareBackHandler { accountViewModel.launchSigner { postViewModel.sendDraftSync() postViewModel.cancel() diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/ShortNotePostScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/ShortNotePostScreen.kt index db7e95f3be..a4bc2c6bfb 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/ShortNotePostScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/ShortNotePostScreen.kt @@ -23,7 +23,6 @@ package com.vitorpamplona.amethyst.ui.screen.loggedIn.home import android.annotation.SuppressLint import android.content.Intent import android.net.Uri -import androidx.activity.compose.BackHandler import androidx.compose.foundation.clickable import androidx.compose.foundation.horizontalScroll import androidx.compose.foundation.layout.Arrangement @@ -93,6 +92,7 @@ import com.vitorpamplona.amethyst.ui.actions.uploads.VoiceMessagePreview import com.vitorpamplona.amethyst.ui.components.OutlinedThinPaddingTextField import com.vitorpamplona.amethyst.ui.components.ThinPaddingTextField import com.vitorpamplona.amethyst.ui.components.getActivity +import com.vitorpamplona.amethyst.ui.navigation.bottombars.KeyboardAwareBackHandler import com.vitorpamplona.amethyst.ui.navigation.navs.Nav import com.vitorpamplona.amethyst.ui.navigation.routes.Route import com.vitorpamplona.amethyst.ui.navigation.topbars.PostingTopBar @@ -235,7 +235,7 @@ internal fun NewPostScreenInner( StrippingFailureDialog(postViewModel.strippingFailureConfirmation) - BackHandler { + KeyboardAwareBackHandler { accountViewModel.launchSigner { postViewModel.sendDraftSync() postViewModel.cancel() diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/nip75Goals/NewGoalScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/nip75Goals/NewGoalScreen.kt index 5a6348cb6d..f7aabcd6f7 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/nip75Goals/NewGoalScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/nip75Goals/NewGoalScreen.kt @@ -20,7 +20,6 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.home.nip75Goals -import androidx.activity.compose.BackHandler import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.Spacer @@ -48,6 +47,7 @@ import androidx.compose.ui.text.input.KeyboardType import androidx.compose.ui.unit.dp import androidx.lifecycle.viewmodel.compose.viewModel import com.vitorpamplona.amethyst.R +import com.vitorpamplona.amethyst.ui.navigation.bottombars.KeyboardAwareBackHandler import com.vitorpamplona.amethyst.ui.navigation.navs.INav import com.vitorpamplona.amethyst.ui.navigation.navs.Nav import com.vitorpamplona.amethyst.ui.navigation.topbars.PostingTopBar @@ -81,7 +81,7 @@ fun NewGoalScreen( accountViewModel: AccountViewModel, nav: INav, ) { - BackHandler { + KeyboardAwareBackHandler { goalViewModel.cancel() nav.popBack() } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/notifications/publicMessages/NewPublicMessageScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/notifications/publicMessages/NewPublicMessageScreen.kt index 56e1382b6a..f9e277665c 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/notifications/publicMessages/NewPublicMessageScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/notifications/publicMessages/NewPublicMessageScreen.kt @@ -20,7 +20,6 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.notifications.publicMessages -import androidx.activity.compose.BackHandler import androidx.compose.foundation.horizontalScroll import androidx.compose.foundation.layout.Arrangement.Absolute.spacedBy import androidx.compose.foundation.layout.Column @@ -64,6 +63,7 @@ import com.vitorpamplona.amethyst.ui.actions.uploads.SelectFromGallery import com.vitorpamplona.amethyst.ui.actions.uploads.TakePictureButton import com.vitorpamplona.amethyst.ui.actions.uploads.TakeVideoButton import com.vitorpamplona.amethyst.ui.components.ThinPaddingTextField +import com.vitorpamplona.amethyst.ui.navigation.bottombars.KeyboardAwareBackHandler import com.vitorpamplona.amethyst.ui.navigation.navs.INav import com.vitorpamplona.amethyst.ui.navigation.navs.Nav import com.vitorpamplona.amethyst.ui.navigation.topbars.PostingTopBar @@ -134,7 +134,7 @@ fun NewPublicMessageScreen( StrippingFailureDialog(postViewModel.strippingFailureConfirmation) - BackHandler { + KeyboardAwareBackHandler { accountViewModel.launchSigner { postViewModel.sendDraftSync() postViewModel.cancel() diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/workouts/NewWorkoutScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/workouts/NewWorkoutScreen.kt index a3de2013d0..5bc98f4cdb 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/workouts/NewWorkoutScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/workouts/NewWorkoutScreen.kt @@ -20,7 +20,6 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.workouts -import androidx.activity.compose.BackHandler import androidx.compose.animation.Crossfade import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box @@ -61,6 +60,7 @@ import com.vitorpamplona.amethyst.R import com.vitorpamplona.amethyst.commons.icons.symbols.Icon import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbol import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols +import com.vitorpamplona.amethyst.ui.navigation.bottombars.KeyboardAwareBackHandler import com.vitorpamplona.amethyst.ui.navigation.navs.INav import com.vitorpamplona.amethyst.ui.navigation.routes.Route import com.vitorpamplona.amethyst.ui.navigation.topbars.PostingTopBar @@ -80,7 +80,7 @@ fun NewWorkoutScreen( postViewModel.init(accountViewModel) postViewModel.prefill(prefill) - BackHandler { + KeyboardAwareBackHandler { postViewModel.cancel() nav.popBack() } From 42a91ffb790842f23f42dbd9dd4a2f0a7f319dfd Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Wed, 5 Aug 2026 15:50:33 +0000 Subject: [PATCH 035/132] SyncCoverage: one band interval cannot speak for several kinds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A band held ONE created_at interval per (relay, filter). For a filter naming several kinds that is a claim no walk can support: ask for `kinds: [0, 30382]`, find profiles going back years and score cards only from last month, and the band records 2020..now for the pair. The next run then skips that whole interior for BOTH — so score cards written inside it are never asked for again, and nothing anywhere says so. A long-lived kind vouched for a short-lived one. Band.spans is now per kind. Each carries only the evidence actually collected for it, so the profile kind keeps its wide interval and the score kind keeps its narrow one, and legs() re-opens the interior for the second while still skipping it for the first. Three things keep the cost of that where it was: - legs() REGROUPS kinds by the windows they want. Identical coverage — the common case, and the only case until they diverge — collapses back into one ask, so a filter that produced two legs still produces two rather than two per kind. Only a kind whose evidence genuinely differs earns its own. - A finished reconcile needs no per-kind evidence and is given none: negentropy compares the filter's whole id set in one pass, so it covers every kind in the filter or none. Only the PAGED path changed. - Filters naming no kinds keep a single span under ALL_KINDS, which is the same claim as before, correctly scoped to the case where it is the only claim available. record() takes observedByKind, and SyncCoverage.observe() accumulates it as events arrive — replacing the pair of hand-rolled vars each caller kept, and moving the per-event isPlausible guard in with it. A paged walk over a MULTI-kind filter that supplies none earns no band at all, loudly, once: attributing one interval to every kind is exactly the over-claim this removes, and a band that over-claims skips events silently, which is worse than re-reading them. Single-kind filters are untouched — there the aggregate always was the per-kind answer. The state file gains a per-kind `spans` object and keeps `min`/`max` as the outer edges, so a rollback to a binary from before this reads the file and behaves as it always did. A file written BEFORE this loads its one interval under ALL_KINDS — the old, wider claim, kept rather than discarded because discarding it would re-download every upstream's corpus once on upgrade. The first per-kind walk replaces it. All 26 existing SyncCoverage tests pass unchanged, which is the evidence that single-kind behaviour did not move. The five new ones were checked against the pre-fix rule reinstated in place: the two behavioural ones fail there and pass here. Co-Authored-By: Claude Opus 5 --- .../geode/mirror/MirrorWorker.kt | 15 ++ .../geode/mirror/SyncCoverageFile.kt | 47 +++- .../relay/client/accessories/SyncCoverage.kt | 210 +++++++++++++++--- .../client/accessories/SyncCoverageTest.kt | 120 ++++++++++ 4 files changed, 353 insertions(+), 39 deletions(-) diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt index c98e050e37..4d1411593b 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt @@ -498,6 +498,12 @@ class MirrorWorker( val syncStartedAt = TimeUtils.now() var seenMin: Long? = null var seenMax: Long? = null + // Per KIND as well as in aggregate: one interval for a + // multi-kind filter lets a long-lived kind vouch for a + // short-lived one, and the band then skips the interior for + // both. The aggregate is still tracked because the reconcile + // path records against the leg's floor, not per kind. + val seenByKind = mutableMapOf() fun observe(event: Event) { // Same containment as the live path: even a trusted @@ -509,6 +515,7 @@ class MirrorWorker( seenMin = minOf(seenMin ?: event.createdAt, event.createdAt) seenMax = maxOf(seenMax ?: event.createdAt, event.createdAt) } + SyncCoverage.observe(seenByKind, event.kind, event.createdAt) handoff.trySendBlocking(event) } else { filtered.incrementAndGet() @@ -537,6 +544,7 @@ class MirrorWorker( } catch (e: NegentropySyncException) { seenMin = null seenMax = null + seenByKind.clear() // The watchdog matches negentropySync's default rather // than fetchAllPages' shorter one: a paged catch-up // sits behind the same slow upstreams. @@ -558,6 +566,13 @@ class MirrorWorker( seenMin, seenMax?.coerceAtMost(syncStartedAt), paged = true, + // Capped the same way the aggregate is: one + // future-dated event must not lift a kind's ceiling + // past what was actually asked for. + observedByKind = + seenByKind.mapValues { (_, span) -> + SyncCoverage.Span(span.min, span.max.coerceAtMost(syncStartedAt)) + }, ) } else { val legFloor = leg.since ?: initialSince diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt index b31c51a60d..58b0b23310 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt @@ -100,8 +100,7 @@ class SyncCoverageFile( root.mapValues { (_, v) -> val o = v.jsonObject SyncCoverage.Band( - o.getValue("min").jsonPrimitive.long, - o.getValue("max").jsonPrimitive.long, + spansOf(o), o["complete"]?.jsonPrimitive?.boolean ?: false, o["fullAt"]?.jsonPrimitive?.long ?: 0L, ) @@ -112,6 +111,30 @@ class SyncCoverageFile( } } + /** + * The per-kind spans, or the single pre-split span read as covering every + * kind under [SyncCoverage.ALL_KINDS]. + * + * A file written before coverage was tracked per kind carries only + * `min`/`max`, and that is exactly the over-wide claim per-kind spans + * exist to stop — so it is loaded as what it always meant rather than + * discarded, and the first paged walk that reports per kind replaces it. + * Dropping it instead would re-download every upstream's corpus once on + * upgrade, which is the cost bands exist to avoid. + */ + private fun spansOf(o: JsonObject): Map { + o["spans"]?.jsonObject?.let { spans -> + return spans.entries.associate { (kind, v) -> + val span = v.jsonObject + kind.toInt() to SyncCoverage.Span(span.getValue("min").jsonPrimitive.long, span.getValue("max").jsonPrimitive.long) + } + } + return mapOf( + SyncCoverage.ALL_KINDS to + SyncCoverage.Span(o.getValue("min").jsonPrimitive.long, o.getValue("max").jsonPrimitive.long), + ) + } + @Synchronized private fun save() { runCatching { @@ -121,10 +144,30 @@ class SyncCoverageFile( put( key, buildJsonObject { + // min/max are the outer edges across every + // kind, and are written for two readers: a + // human debugging why an upstream re-synced, + // and a ROLLBACK — a binary from before spans + // were per kind reads these and behaves as it + // always did, rather than failing to parse. put("min", band.minCreatedAt) put("max", band.maxCreatedAt) put("complete", band.complete) put("fullAt", band.fullAt) + put( + "spans", + buildJsonObject { + band.spans.forEach { (kind, span) -> + put( + kind.toString(), + buildJsonObject { + put("min", span.min) + put("max", span.max) + }, + ) + } + }, + ) }, ) } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt index 35ba9b330a..4e6de68696 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt @@ -22,6 +22,7 @@ package com.vitorpamplona.quartz.nip01Core.relay.client.accessories import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.utils.Log import com.vitorpamplona.quartz.utils.TimeUtils import com.vitorpamplona.quartz.utils.concurrent.ConcurrentMap @@ -64,23 +65,55 @@ class SyncCoverage( private val now: () -> Long = { TimeUtils.now() }, private val onChange: () -> Unit = {}, ) { + /** A covered `created_at` interval, inclusive at both ends. */ + data class Span( + val min: Long, + val max: Long, + ) { + fun widen(other: Span) = Span(minOf(min, other.min), maxOf(max, other.max)) + } + /** * What is already covered for one (relay, filter) pair. * + * [spans] is PER KIND, and that is the whole point of it. A band used to + * hold one interval for the entire filter, which is a claim no multi-kind + * walk can support: ask for `kinds: [0, 30382]`, see profiles back to 2020 + * and score cards only from 2025, and the band reads 2020..2026 — so the + * next run skips 2020..2025 for BOTH, and the score cards in that interior + * are never asked for again. A long-lived kind vouched for a short-lived + * one. Per kind, each carries only the evidence actually collected for it. + * + * Filters that name no kinds at all cannot be split, so they keep a single + * span under [ALL_KINDS] — the same claim as before, correctly scoped to + * the case where it is the only claim available. + * * [complete] is the difference between "we walked this span" (a paged * fetch) and "we are in sync below this point" (a finished negentropy * reconcile, which compared the whole range). Only a complete band may - * skip its older leg. + * skip its older leg. It is a property of the BAND rather than of a span: + * a reconcile compares the filter's whole id set at once, so it either + * covers every kind in it or none. * * [fullAt] is when the last pass that started from nothing finished — the * clock for the periodic re-walk. */ data class Band( - val minCreatedAt: Long, - val maxCreatedAt: Long, + val spans: Map, val complete: Boolean = false, val fullAt: Long = 0, - ) + ) { + /** The outer edges across every kind — for logging and for the file's compatibility fields. */ + val minCreatedAt: Long get() = spans.values.minOfOrNull { it.min } ?: 0 + val maxCreatedAt: Long get() = spans.values.maxOfOrNull { it.max } ?: 0 + + /** Widen each kind by its counterpart, keeping kinds only one side knows. */ + fun widen(other: Band): Band { + val merged = spans.toMutableMap() + for ((kind, span) in other.spans) merged[kind] = merged[kind]?.widen(span) ?: span + return Band(merged, complete || other.complete, fullAt) + } + } private val bands = ConcurrentMap() @@ -114,31 +147,68 @@ class SyncCoverage( // Time for another full pass: relays gain old events, and without // this the band's claim is never re-tested. if (isStale(band)) return listOf(filter) - val legs = mutableListOf() + if (band.spans.isEmpty()) return listOf(filter) - // Older: up to and including the band's floor, but not past the - // filter's (or, when the filter has no `since`, the caller's - // [floor] — a sync window the filter itself must not carry, or it - // would change the band's key every run). A complete band compared - // its whole range already, but only down to the floor it ran - // against: a caller now reaching deeper — a raised backfill window - // — re-opens the span below the band. + val kinds = filter.kinds + if (kinds.isNullOrEmpty()) { + // Nothing to split by. One span, exactly as before. + return windows(filter, band.spans[ALL_KINDS], band.complete, floor) + .map { (since, until) -> filter.copy(since = since, until = until) } + } + + // Per kind, then REGROUPED by the windows each one wants. Kinds whose + // coverage agrees — the overwhelmingly common case, and the only case + // at all until they diverge — collapse back into one ask, so a filter + // that used to produce two legs still produces two rather than two per + // kind. Only a kind whose evidence genuinely differs earns its own. + val byWindows = LinkedHashMap>, MutableList>() + for (kind in kinds) { + // ALL_KINDS as the fallback: a band written before coverage was + // tracked per kind, restored from such a file. It carries the old, + // wider claim for every kind — the behaviour this replaces — and + // self-corrects on the first paged walk that reports per kind. + val span = band.spans[kind] ?: band.spans[ALL_KINDS] + byWindows.getOrPut(windows(filter, span, band.complete, floor)) { mutableListOf() }.add(kind) + } + return byWindows.flatMap { (windows, group) -> + windows.map { (since, until) -> filter.copy(kinds = group, since = since, until = until) } + } + } + + /** + * The `(since, until)` pairs still outstanding for ONE span — the leg + * arithmetic, with the filter's own bounds applied and nothing else. + * A null [span] means no evidence at all, so the whole filter is wanted. + */ + private fun windows( + filter: Filter, + span: Span?, + complete: Boolean, + floor: Long?, + ): List> { + if (span == null) return listOf(filter.since to filter.until) + val out = mutableListOf>() + + // Older: up to and including the span's floor, but not past the + // filter's (or, when the filter has no `since`, the caller's [floor] — + // a sync window the filter itself must not carry, or it would change + // the band's key every run). A complete band compared its whole range + // already, but only down to the floor it ran against: a caller now + // reaching deeper — a raised backfill window — re-opens the span below. val since = filter.since ?: floor val wantsOlder = - if (band.complete) { - since != null && since < band.minCreatedAt + if (complete) { + since != null && since < span.min } else { - since == null || band.minCreatedAt >= since + since == null || span.min >= since } - if (wantsOlder) { - legs.add(filter.copy(until = minOf(band.minCreatedAt, filter.until ?: Long.MAX_VALUE))) - } + if (wantsOlder) out.add(filter.since to minOf(span.min, filter.until ?: Long.MAX_VALUE)) - // Newer: from the band's ceiling on, but not past the filter's. - if (filter.until == null || band.maxCreatedAt <= filter.until) { - legs.add(filter.copy(since = maxOf(band.maxCreatedAt, filter.since ?: Long.MIN_VALUE))) + // Newer: from the span's ceiling on, but not past the filter's. + if (filter.until == null || span.max <= filter.until) { + out.add(maxOf(span.max, filter.since ?: Long.MIN_VALUE) to filter.until) } - return legs + return out } /** @@ -162,21 +232,59 @@ class SyncCoverage( observedMax: Long?, paged: Boolean, reconciledThrough: Long? = null, + observedByKind: Map? = null, ) { if (reconciledThrough != null) { - put(url, filter, observedMin ?: reconciledThrough, reconciledThrough, complete = true) + // A reconcile compares the filter's whole id set in one pass, so + // the span it earns is the same for every kind the filter names — + // no per-kind evidence needed or possible. + val span = Span(observedMin ?: reconciledThrough, reconciledThrough) + put(url, filter, kindsOf(filter).associateWith { span }, complete = true) return } if (!paged) return + + if (observedByKind != null) { + // Guarded per span for the same reason the aggregate is below. + val plausible = + observedByKind.filterValues { + isPlausible(it.min, now()) && isPlausible(it.max, now()) + } + if (plausible.isEmpty()) return + put(url, filter, plausible, complete = false) + return + } + + // No per-kind evidence. For a filter naming one kind (or none) the + // aggregate IS the per-kind answer and nothing is lost. For a filter + // naming several it is not: attributing one interval to all of them is + // exactly the over-claim [Band.spans] exists to stop, and a band that + // over-claims skips events silently — strictly worse than re-reading + // them. So record nothing and say why, once. The caller resumes as if + // it had no band, which is where it was before bands existed. + val kinds = kindsOf(filter) + if (kinds.size > 1) { + if (!warnedAboutUnattributed) { + warnedAboutUnattributed = true + Log.w("SyncCoverage") { + "paged record for a ${kinds.size}-kind filter with no per-kind spans — no band recorded, so this " + + "walk will not resume. Pass observedByKind (see SyncCoverage.observe) to earn one." + } + } + return + } // Guarded even though callers should filter with [isPlausible] per // event: a 1970 floor or a far-future ceiling would make the band // claim the whole timeline, and the leg outside it would ask for a // range nothing can be in, forever. if (observedMin == null || observedMax == null) return if (!isPlausible(observedMin, now()) || !isPlausible(observedMax, now())) return - put(url, filter, observedMin, observedMax, complete = false) + put(url, filter, kinds.associateWith { Span(observedMin, observedMax) }, complete = false) } + /** The kinds a band is keyed by: the filter's, or [ALL_KINDS] when it names none. */ + private fun kindsOf(filter: Filter): List = filter.kinds?.takeIf { it.isNotEmpty() } ?: listOf(ALL_KINDS) + /** * Widen (or reset) the band. A pass that ran because the previous band * had gone stale REPLACES it: it re-walked the whole filter, so its own @@ -185,22 +293,12 @@ class SyncCoverage( private fun put( url: NormalizedRelayUrl, filter: Filter, - min: Long, - max: Long, + spans: Map, complete: Boolean, ) { - val fresh = Band(min, max, complete, now()) + val fresh = Band(spans, complete, now()) bands.merge(key(url, filter), fresh) { old, new -> - if (isStale(old)) { - new - } else { - Band( - minOf(old.minCreatedAt, new.minCreatedAt), - maxOf(old.maxCreatedAt, new.maxCreatedAt), - old.complete || new.complete, - old.fullAt, - ) - } + if (isStale(old)) new else old.widen(new) } onChange() } @@ -276,7 +374,45 @@ class SyncCoverage( return "${url.url} $fingerprint" } + // One line per process, not per walk: the point is to tell a caller it has + // not been migrated, and repeating it every leg would bury the log it is + // trying to be read in. + private var warnedAboutUnattributed = false + companion object { + /** + * The span key for a filter that names no kinds, and the fallback for + * a band restored from a file written before spans were per kind. + * Negative because NIP-01 kinds are not. + */ + const val ALL_KINDS = -1 + + /** + * Widen [into] with one event's stamp, so a caller can accumulate the + * per-kind evidence [record] wants as events arrive: + * + * val seen = mutableMapOf() + * ... onEvent { SyncCoverage.observe(seen, it.kind, it.createdAt) } + * coverage.record(url, filter, …, paged = true, observedByKind = seen) + * + * Implausible stamps are dropped here rather than by each caller — + * per EVENT, never over a leg's aggregate, because one misdated event + * among hundreds of thousands would otherwise discard the whole band. + * + * Not synchronized: it replaces a pair of plain `var`s at each call + * site and is meant for the same single-consumer callback. + */ + fun observe( + into: MutableMap, + kind: Int, + createdAt: Long, + now: Long = TimeUtils.now(), + ) { + if (!isPlausible(createdAt, now)) return + val one = Span(createdAt, createdAt) + into[kind] = into[kind]?.widen(one) ?: one + } + // More filter instances than any deliberate configuration holds; only // a caller rebuilding filters per cycle ever reaches it. private const val MAX_FINGERPRINTS = 1_000 diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt index 53120624a4..87cac2c977 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt @@ -382,4 +382,124 @@ class SyncCoverageTest { val copy = Filter(kinds = listOf(30382), authors = (1..500).map { it.toString(16).padStart(64, '0') }) assertEquals(1_700_001_000L, c.band(relay, copy)?.minCreatedAt, "identity caching must not change the key") } + + // ---- per-kind spans: one interval cannot speak for several kinds ------- + + private val mixed = Filter(kinds = listOf(0, 30382)) + + /** Does any leg still ask [kind] about the instant [at]? */ + private fun reaches( + legs: List, + kind: Int, + at: Long, + ) = legs.any { + (it.kinds?.contains(kind) ?: true) && + (it.since ?: Long.MIN_VALUE) <= at && + at <= (it.until ?: Long.MAX_VALUE) + } + + @Test + fun `a long-lived kind no longer vouches for a short-lived one`() { + // THE BUG. Ask for profiles and score cards together: the relay has + // profiles going back years and score cards only from last month. One + // interval per band recorded 2020..now for the pair, and the next run + // skipped that whole interior for BOTH — so score cards written inside + // it were never asked for again, and nothing anywhere said so. + val c = SyncCoverage() + c.record( + relay, + mixed, + null, + null, + paged = true, + observedByKind = + mapOf( + 0 to SyncCoverage.Span(1_600_000_000L, 1_700_000_000L), + 30382 to SyncCoverage.Span(1_690_000_000L, 1_700_000_000L), + ), + ) + val legs = c.legs(relay, mixed) + + assertTrue(!reaches(legs, 0, 1_650_000_000L), "kind 0 really was walked there — do not re-read it") + assertTrue(reaches(legs, 30382, 1_650_000_000L), "kind 30382 never was, and must still be asked") + // Both keep the ground they actually earned. + assertTrue(!reaches(legs, 30382, 1_695_000_000L), "…but not its own covered interior") + assertTrue(reaches(legs, 0, 1_500_000_000L), "and both still reach below everything walked") + } + + @Test + fun `kinds whose coverage agrees stay a single ask`() { + // The cost control. Splitting per kind would turn two legs into two + // per kind on every filter, which is the common case made worse to fix + // the rare one. Kinds are regrouped by the windows they want, so + // identical coverage collapses back to exactly what it was before. + val c = SyncCoverage() + val span = SyncCoverage.Span(1_690_000_000L, 1_700_000_000L) + c.record(relay, mixed, null, null, paged = true, observedByKind = mapOf(0 to span, 30382 to span)) + + val legs = c.legs(relay, mixed) + assertEquals(2, legs.size, "two legs, not two per kind") + assertEquals(listOf(0, 30382), legs[0].kinds, "and both kinds ride in one ask") + } + + @Test + fun `a multi-kind paged walk with no per-kind evidence earns no band`() { + // The caller did not say which kind it saw where, so the only band + // available is the over-wide one. Refused: a band that over-claims + // skips events silently, which is worse than re-reading them. The + // walk resumes from nothing, exactly as it did before bands existed. + val c = SyncCoverage() + c.record(relay, mixed, 1_690_000_000L, 1_700_000_000L, paged = true) + + assertNull(c.band(relay, mixed)) + assertEquals(listOf(mixed), c.legs(relay, mixed)) + + // A filter naming ONE kind is unaffected: there, the aggregate IS the + // per-kind answer and nothing was ever ambiguous about it. + c.record(relay, profiles, 1_690_000_000L, 1_700_000_000L, paged = true) + assertEquals(2, c.legs(relay, profiles).size) + } + + @Test + fun `a finished reconcile covers every kind the filter names`() { + // Negentropy compares the filter's whole id set in one pass, so it + // either covers every kind in it or none — no per-kind evidence needed, + // and none invented. + val c = SyncCoverage() + c.record(relay, mixed, null, null, paged = false, reconciledThrough = 1_700_000_000L) + + assertEquals(setOf(0, 30382), c.band(relay, mixed)!!.spans.keys) + val legs = c.legs(relay, mixed) + assertEquals(1, legs.size, "complete: no older leg, and one shared newer one") + assertEquals(1_700_000_000L, legs[0].since) + } + + @Test + fun `a band restored from a pre-split file still narrows every kind`() { + // Files written before spans were per kind carry one interval. It is + // the old, wider claim — loaded as what it always meant rather than + // discarded, because discarding it would re-download every upstream's + // corpus once on upgrade. The first per-kind walk replaces it. + val seed = SyncCoverage() + // A plausible span, or record() correctly drops it and there is no key to read. + seed.record(relay, mixed, null, null, paged = true, observedByKind = mapOf(0 to SyncCoverage.Span(1_690_000_000L, 1_700_000_000L))) + val key = seed.export().keys.single() + + val restored = SyncCoverage() + restored.restore( + mapOf( + key to + SyncCoverage.Band( + mapOf(SyncCoverage.ALL_KINDS to SyncCoverage.Span(1_690_000_000L, 1_700_000_000L)), + complete = false, + fullAt = now(), + ), + ), + ) + + val legs = restored.legs(relay, mixed) + assertEquals(2, legs.size, "one shared pair of legs, which is the old behaviour exactly") + assertEquals(listOf(0, 30382), legs[0].kinds) + assertEquals(1_690_000_000L, legs[0].until) + } } From 74145ee8f3ceda3785c3591e43ff26b9a616dfa2 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Wed, 5 Aug 2026 16:08:56 +0000 Subject: [PATCH 036/132] Code-review fixes: two ways per-kind spans could be recorded and not used MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both found in the review pass over the previous commit, both the same shape — a band written that no lookup can reach, or reaches wrongly. - Spans for kinds the filter never named were stored as given. Inert for legs(), which only looks up the filter's own kinds, but NOT for Band.minCreatedAt — and that is what SyncCoverageFile writes as its rollback-compat `min`/`max`. A relay answering with more than it was asked for (or a caller whose containment check runs against a different filter than the band is keyed by) would push that floor below anything the filter's kinds support, so a binary from before per-kind spans would read the file and over-claim. The fix, undone through the compatibility path it added. - observedByKind on a filter that names NO kinds was stored per kind, while legs() for such a filter reads only ALL_KINDS. The band was recorded, persisted, and never consulted: a resume that silently did not resume. Collapsed to the union, which is the only claim a kind-less filter can make. Why these were not in the initial diff: both live where the new per-kind path meets an OLD assumption — that record()'s input is already scoped to the filter, and that a band's keys are always the filter's kinds. Neither held once callers began supplying the map themselves. Co-Authored-By: Claude Opus 5 --- .../relay/client/accessories/SyncCoverage.kt | 26 ++++++++- .../client/accessories/SyncCoverageTest.kt | 57 +++++++++++++++++++ 2 files changed, 82 insertions(+), 1 deletion(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt index 4e6de68696..74bcddbb3d 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt @@ -251,7 +251,31 @@ class SyncCoverage( isPlausible(it.min, now()) && isPlausible(it.max, now()) } if (plausible.isEmpty()) return - put(url, filter, plausible, complete = false) + val named = filter.kinds + val spans = + if (named.isNullOrEmpty()) { + // A filter naming no kinds cannot be split, so [legs] reads + // ALL_KINDS and nothing else. Storing what the walk saw per + // kind would record a band no lookup can ever reach — it + // would exist and do nothing. Collapse to the union, which + // is the only claim such a filter can make. + mapOf(ALL_KINDS to plausible.values.reduce { a, b -> a.widen(b) }) + } else { + // Only kinds the filter NAMES. A relay may answer with more + // than it was asked for, and a caller whose containment + // check runs against a different filter than the band is + // keyed by passes those straight through. Keeping them + // would be inert for [legs] — which looks up the filter's + // own kinds — but NOT for [Band.minCreatedAt], which the + // state file writes as its rollback-compat `min`/`max`. An + // off-filter kind seen further back would widen those past + // anything the filter's kinds support, so a binary from + // before per-kind spans would read that file and + // over-claim: this fix undone through the compat path. + plausible.filterKeys { it in named } + } + if (spans.isEmpty()) return + put(url, filter, spans, complete = false) return } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt index 87cac2c977..3c26e1891e 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt @@ -502,4 +502,61 @@ class SyncCoverageTest { assertEquals(listOf(0, 30382), legs[0].kinds) assertEquals(1_690_000_000L, legs[0].until) } + + @Test + fun `a kind the filter never asked for cannot widen the band`() { + // A relay may answer with more than it was asked for. Those spans are + // inert for legs(), which only looks up the filter's own kinds — but + // NOT for Band.minCreatedAt, which the state file writes as its + // rollback-compat min/max. Left in, a stray kind seen further back + // would widen that past anything the filter's kinds support, and a + // binary from before per-kind spans would read the file and over-claim. + val c = SyncCoverage() + c.record( + relay, + profiles, + null, + null, + paged = true, + observedByKind = + mapOf( + 0 to SyncCoverage.Span(1_690_000_000L, 1_700_000_000L), + // never asked for, and much older + 1 to SyncCoverage.Span(1_600_000_000L, 1_610_000_000L), + ), + ) + + val band = c.band(relay, profiles)!! + assertEquals(setOf(0), band.spans.keys, "only the kind the filter names") + assertEquals(1_690_000_000L, band.minCreatedAt, "…so the compat floor stays honest") + } + + @Test + fun `per-kind evidence on a filter naming no kinds collapses to one span`() { + // Such a filter cannot be split, so legs() reads ALL_KINDS and nothing + // else. Storing per-kind spans here would record a band no lookup can + // reach — present in the file, doing nothing. + val anyKind = Filter(authors = listOf("a".repeat(64))) + val c = SyncCoverage() + c.record( + relay, + anyKind, + null, + null, + paged = true, + observedByKind = + mapOf( + 0 to SyncCoverage.Span(1_690_000_000L, 1_695_000_000L), + 30382 to SyncCoverage.Span(1_697_000_000L, 1_700_000_000L), + ), + ) + + val band = c.band(relay, anyKind)!! + assertEquals(setOf(SyncCoverage.ALL_KINDS), band.spans.keys) + assertEquals(1_690_000_000L, band.spans.getValue(SyncCoverage.ALL_KINDS).min, "the union, not one of them") + assertEquals(1_700_000_000L, band.spans.getValue(SyncCoverage.ALL_KINDS).max) + // …and it is actually USED, which is the half that was silently missing. + assertEquals(2, c.legs(relay, anyKind).size) + assertEquals(1_690_000_000L, c.legs(relay, anyKind)[0].until) + } } From a3fac4fc085a04a3da344e53c8001d4bdb3c146b Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Wed, 5 Aug 2026 12:07:27 -0400 Subject: [PATCH 037/132] Suspend the incoming-message chain down to SubscriptionListener.onEvent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A consumer that cannot suspend has to block, and blocking here deadlocks the whole client. Measured on a mirror built against this library, twice, ~13 minutes after each start: all 64 shared coroutine workers parked in `runBlocking` beneath `trySendBlocking`, called from the websocket message callback. The consumer draining that channel needed threads from the same pool to reach its store, so it could never make room, so the producers never woke. Every stream, the health reporter, all of it stopped, at 2% CPU with a healthy, idle backend. A full queue was the symptom; producers eating the threads the drain needed was the cause. The coroutine context was already there — BasicOkHttpWebSocket has always processed messages inside `scope.launch { for (message in incomingMessages) }` — so the only thing forcing a blocking hand-off was that the hops in between were declared non-suspend. Now they are not: WebSocketListener.onMessage RelayConnectionListener.onIncomingMessage PoolRequests/PoolCounts/PoolEventOutbox.onIncomingMessage SubscriptionListener.onEvent fetchAllPages / negentropy accessories' onEvent parameter A consumer that fills its buffer now suspends and releases its thread rather than holding it, which is the same reasoning BasicOkHttpWebSocket already documents for keeping its own channel UNLIMITED so a slow consumer cannot block OkHttp reader threads. This extends it one layer down. BLE is the one transport whose callback genuinely cannot suspend — the platform hands notifications to a plain callback — so BleNostrClient gets the same treatment the websocket transport already had: an UNLIMITED hand-off channel so the BLE stack is never blocked, drained by ONE coroutine so message order survives the boundary. Tests that drove these entry points directly now do so from `runTest`, or from `runBlocking` where the call sits inside a raw thread or Runnable that models a platform callback. Co-Authored-By: Claude Opus 5 (1M context) --- .../napplet/NappletLiveSubscriptions.kt | 2 +- .../amethyst/service/ClinkDebitPayer.kt | 2 +- .../amethyst/service/ClinkOfferPayer.kt | 2 +- .../diagnostics/BootRelayDiagnostics.kt | 2 +- .../diagnostics/DmRelayDiagnosticsLogger.kt | 2 +- .../eoseManagers/PerUniqueIdEoseManager.kt | 2 +- .../PerUserAndFollowListEoseManager.kt | 2 +- .../eoseManagers/PerUserEoseManager.kt | 2 +- .../SingleSubNoEoseCacheEoseManager.kt | 2 +- .../notifyCommand/model/NotifyCoordinator.kt | 2 +- .../AccountFollowsLoaderSubAssembler.kt | 2 +- .../AccountNotificationsHistoryEoseManager.kt | 2 +- .../NwcNotificationsEoseManager.kt | 2 +- .../AccountGiftWrapsHistoryEoseManager.kt | 2 +- .../user/watchers/UserWatcherSubAssembler.kt | 2 +- .../speedLogger/RelaySpeedLogger.kt | 2 +- .../resourceusage/RelayUsageListener.kt | 2 +- .../ChatroomNip04HistorySubAssembler.kt | 2 +- .../ConcordChannelHistoryFilterAssembler.kt | 2 +- ...elayGroupOpenChatHistoryFilterAssembler.kt | 2 +- ...yGroupOpenThreadsHistoryFilterAssembler.kt | 2 +- .../ChatroomListNip04HistorySubAssembler.kt | 2 +- .../datasource/ChessFeedFilterSubAssembler.kt | 2 +- .../loggedIn/relays/eventsync/EventSync.kt | 2 +- .../com/vitorpamplona/amethyst/cli/Context.kt | 2 +- .../amethyst/cli/commands/GeochatCommands.kt | 2 +- .../amethyst/cli/commands/NipCommand.kt | 2 +- .../amethyst/cli/commands/NostrConnect.kt | 2 +- .../amethyst/cli/commands/SubscribeCommand.kt | 2 +- .../nip64Chess/ChessRelayFetchHelper.kt | 2 +- .../assemblers/FeedMetadataCoordinator.kt | 6 +- .../eoseManagers/PerKeyEoseManager.kt | 2 +- .../eoseManagers/SingleSubEoseManager.kt | 2 +- .../relays/health/RelayHealthListener.kt | 2 +- .../service/broadcast/BroadcastTracker.kt | 4 +- .../amethyst/commons/wot/OutboxDispatcher.kt | 4 +- .../nip64Chess/ChessEventBroadcaster.kt | 2 +- .../relayClient/paging/WindowLoadTracker.kt | 2 +- .../relays/health/RelayLatencyListener.kt | 2 +- .../nip17Dm/DmInboxRelayResolverOutboxTest.kt | 2 +- .../commons/wot/OutboxDispatcherTest.kt | 2 +- .../vitorpamplona/amethyst/desktop/Main.kt | 4 +- .../desktop/account/AccountManager.kt | 2 +- .../desktop/followpacks/FollowPacksState.kt | 2 +- .../desktop/followpacks/MetadataPrefetch.kt | 2 +- .../desktop/followpacks/ui/FromThePackFeed.kt | 2 +- .../followpacks/ui/RenderFollowPackCard.kt | 2 +- .../desktop/network/RelayConnectionManager.kt | 4 +- .../search/DesktopRelayUserSearchDelegate.kt | 2 +- .../subscriptions/ChessSubscription.kt | 2 +- .../DesktopRelaySubscriptionsCoordinator.kt | 4 +- .../subscriptions/SubscriptionUtils.kt | 2 +- .../desktop/ui/ImportFollowListDialog.kt | 4 +- .../amethyst/desktop/ui/NoteActions.kt | 4 +- .../desktop/ui/chats/ChatroomListState.kt | 2 +- .../desktop/benchmark/LaunchScenario.kt | 2 +- .../testrelay/LaunchFixtureRelayTest.kt | 2 +- .../testrelay/SubscribeBeforeConnectTest.kt | 2 +- .../geode/mirror/MirrorWorker.kt | 2 +- .../geode/GracefulShutdownTest.kt | 2 +- .../com/vitorpamplona/geode/KtorRelayTest.kt | 2 +- .../geode/Nip01ComplianceTest.kt | 12 +- .../vitorpamplona/geode/Nip09DeletionTest.kt | 2 +- .../geode/Nip77NegentropyTest.kt | 2 +- .../mirror/MirrorWorkerTrustOriginTest.kt | 2 +- .../vitorpamplona/geode/perf/LoadBenchmark.kt | 6 +- .../geode/testing/SubscriptionTesting.kt | 2 +- .../graperank/GrapeRankCrawler.kt | 2 +- .../nip01Core/relay/client/NostrClient.kt | 2 +- .../accessories/AdaptiveRelayLimiter.kt | 2 +- .../client/accessories/EventCollector.kt | 2 +- .../client/accessories/NostrClientCountExt.kt | 4 +- .../NostrClientFetchAllPagesExt.kt | 6 +- .../NostrClientFetchAllWithHooksExt.kt | 2 +- .../accessories/NostrClientFetchFirstExt.kt | 2 +- .../NostrClientNegentropyFanOutExt.kt | 2 +- .../NostrClientNegentropySyncExt.kt | 14 +- .../accessories/NostrClientPublishExt.kt | 2 +- .../RelayInsertConfirmationCollector.kt | 2 +- .../relay/client/accessories/RelayLogger.kt | 2 +- .../relay/client/accessories/RelayNotifier.kt | 2 +- .../relay/client/auth/RelayAuthenticator.kt | 2 +- .../client/counts/RelayActiveCountStates.kt | 2 +- .../relay/client/limits/RelayLimitsTracker.kt | 2 +- .../listeners/RedirectConnectionListener.kt | 2 +- .../listeners/RelayConnectionListener.kt | 2 +- .../nip01Core/relay/client/pool/PoolCounts.kt | 2 +- .../relay/client/pool/PoolEventOutbox.kt | 2 +- .../relay/client/pool/PoolRequests.kt | 2 +- .../nip01Core/relay/client/pool/RelayPool.kt | 2 +- .../relay/client/reqs/DynamicSubscription.kt | 2 +- .../client/reqs/NostrClientFetchAsFlowExt.kt | 2 +- .../reqs/NostrClientSubscribeAsFlowExt.kt | 2 +- .../client/reqs/RelayActiveRequestStates.kt | 2 +- .../relay/client/reqs/StaticSubscription.kt | 2 +- .../relay/client/reqs/SubscriptionListener.kt | 2 +- .../relay/client/reqs/stats/RelayReqStats.kt | 2 +- .../client/single/basic/BasicRelayClient.kt | 2 +- .../standalone/StandaloneRelayClient.kt | 2 +- .../relay/client/stats/RelayStats.kt | 2 +- .../relay/sockets/WebSocketListener.kt | 2 +- .../server/NostrConnectSignerService.kt | 2 +- .../reachability/RelayObserver.kt | 2 +- .../reachability/RelayProber.kt | 2 +- .../nip77Negentropy/NegentropyManager.kt | 2 +- .../quartz/nipBEBle/relay/BleMeshManager.kt | 2 +- .../quartz/nipBEBle/relay/BleNostrClient.kt | 40 +- .../client/limits/RelayLimitsTrackerTest.kt | 93 +++-- .../client/pool/PoolEventOutboxAuthTest.kt | 106 ++--- .../client/pool/PoolRequestsRefusalTest.kt | 194 ++++----- .../inprocess/InProcessWebSocketTest.kt | 4 +- .../server/NostrConnectSignerServiceTest.kt | 2 +- .../reachability/RelayObserverTest.kt | 390 +++++++++--------- .../reachability/RelayProberFlowTest.kt | 178 ++++---- .../relay/NostrClientManualSubTest.kt | 2 +- .../relay/NostrClientRepeatSubTest.kt | 2 +- .../relay/PoolRequestsConcurrencyTest.kt | 109 ++--- .../RelayAuthenticatorReauthOnClosedTest.kt | 2 +- .../client/NegentropyRejectionFallbackTest.kt | 41 +- .../relay/prodbench/ByIdFetchBenchmark.kt | 2 +- .../relay/prodbench/DispatchStageBenchmark.kt | 19 +- .../prodbench/NegentropyMultiRelayLiveTest.kt | 2 +- .../relay/prodbench/NegentropyStallRepro.kt | 2 +- .../prodbench/ProductionReceiverBenchmark.kt | 2 +- 124 files changed, 770 insertions(+), 682 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLiveSubscriptions.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLiveSubscriptions.kt index 8429a0b721..1e33c16323 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLiveSubscriptions.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLiveSubscriptions.kt @@ -80,7 +80,7 @@ class NappletLiveSubscriptions { val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/ClinkDebitPayer.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/ClinkDebitPayer.kt index 664b55ed61..3ea9d1678d 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/ClinkDebitPayer.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/ClinkDebitPayer.kt @@ -113,7 +113,7 @@ object ClinkDebitPayer { val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/ClinkOfferPayer.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/ClinkOfferPayer.kt index 28a0d64fb0..a5911dc6c7 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/ClinkOfferPayer.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/ClinkOfferPayer.kt @@ -85,7 +85,7 @@ object ClinkOfferPayer { val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/diagnostics/BootRelayDiagnostics.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/diagnostics/BootRelayDiagnostics.kt index 1966bac786..054f6f855e 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/diagnostics/BootRelayDiagnostics.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/diagnostics/BootRelayDiagnostics.kt @@ -158,7 +158,7 @@ class BootRelayDiagnostics( } } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/diagnostics/DmRelayDiagnosticsLogger.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/diagnostics/DmRelayDiagnosticsLogger.kt index 4aefae6ba6..07f13fbd0b 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/diagnostics/DmRelayDiagnosticsLogger.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/diagnostics/DmRelayDiagnosticsLogger.kt @@ -112,7 +112,7 @@ class DmRelayDiagnosticsLogger( Log.d(TAG) { "[+${at()}ms] REQ -> ${relay.url.url} success=$success ${cmdStr.take(400)}" } } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/PerUniqueIdEoseManager.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/PerUniqueIdEoseManager.kt index 0427380423..61c297887e 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/PerUniqueIdEoseManager.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/PerUniqueIdEoseManager.kt @@ -78,7 +78,7 @@ abstract class PerUniqueIdEoseManager( newEose(key, relay, TimeUtils.now(), forFilters) } - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/PerUserAndFollowListEoseManager.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/PerUserAndFollowListEoseManager.kt index cc55f02d7a..7f53cbd0aa 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/PerUserAndFollowListEoseManager.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/PerUserAndFollowListEoseManager.kt @@ -90,7 +90,7 @@ abstract class PerUserAndFollowListEoseManager( newEose(key, relay, TimeUtils.now(), forFilters) } - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/PerUserEoseManager.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/PerUserEoseManager.kt index b904bf4ab9..c89d53a6e0 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/PerUserEoseManager.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/PerUserEoseManager.kt @@ -77,7 +77,7 @@ abstract class PerUserEoseManager( newEose(key, relay, TimeUtils.now(), forFilters) } - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/SingleSubNoEoseCacheEoseManager.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/SingleSubNoEoseCacheEoseManager.kt index d4df1e5deb..a80357e283 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/SingleSubNoEoseCacheEoseManager.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/SingleSubNoEoseCacheEoseManager.kt @@ -53,7 +53,7 @@ abstract class SingleSubNoEoseCacheEoseManager( } } - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/notifyCommand/model/NotifyCoordinator.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/notifyCommand/model/NotifyCoordinator.kt index 92b40d456e..c7e146b5bf 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/notifyCommand/model/NotifyCoordinator.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/notifyCommand/model/NotifyCoordinator.kt @@ -78,7 +78,7 @@ class NotifyCoordinator( } } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/follows/AccountFollowsLoaderSubAssembler.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/follows/AccountFollowsLoaderSubAssembler.kt index 08ec932e0e..dcda63f127 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/follows/AccountFollowsLoaderSubAssembler.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/follows/AccountFollowsLoaderSubAssembler.kt @@ -116,7 +116,7 @@ class AccountFollowsLoaderSubAssembler( newEose(TimeUtils.now(), relay, forFilters) } - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip01Notifications/AccountNotificationsHistoryEoseManager.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip01Notifications/AccountNotificationsHistoryEoseManager.kt index 485f8fb9f9..5f15938130 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip01Notifications/AccountNotificationsHistoryEoseManager.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip01Notifications/AccountNotificationsHistoryEoseManager.kt @@ -193,7 +193,7 @@ class AccountNotificationsHistoryEoseManager( // cursors so a late callback can't move another account's cursors. newEose runs regardless. val myCursors = key.account.notificationHistory return object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip47WalletConnect/NwcNotificationsEoseManager.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip47WalletConnect/NwcNotificationsEoseManager.kt index 384c89df06..97098d765e 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip47WalletConnect/NwcNotificationsEoseManager.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip47WalletConnect/NwcNotificationsEoseManager.kt @@ -124,7 +124,7 @@ class NwcNotificationsEoseManager( newEose(key, relay, TimeUtils.now(), forFilters) } - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip59GiftWraps/AccountGiftWrapsHistoryEoseManager.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip59GiftWraps/AccountGiftWrapsHistoryEoseManager.kt index 02d351be88..911e6fb132 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip59GiftWraps/AccountGiftWrapsHistoryEoseManager.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip59GiftWraps/AccountGiftWrapsHistoryEoseManager.kt @@ -115,7 +115,7 @@ class AccountGiftWrapsHistoryEoseManager( // cursors so a late callback can't move another account's cursors. newEose runs regardless. val myCursors = key.account.chatroomList.giftWrapHistory return object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/watchers/UserWatcherSubAssembler.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/watchers/UserWatcherSubAssembler.kt index d695cd8e7e..f5381a9be1 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/watchers/UserWatcherSubAssembler.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/watchers/UserWatcherSubAssembler.kt @@ -74,7 +74,7 @@ class UserWatcherSubAssembler( newEose(relay, TimeUtils.now(), forFilters) } - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/speedLogger/RelaySpeedLogger.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/speedLogger/RelaySpeedLogger.kt index 66718a9ea1..32942920eb 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/speedLogger/RelaySpeedLogger.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/speedLogger/RelaySpeedLogger.kt @@ -42,7 +42,7 @@ class RelaySpeedLogger( private val clientListener = object : RelayConnectionListener { - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/RelayUsageListener.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/RelayUsageListener.kt index a8ba773138..5d97ef56cb 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/RelayUsageListener.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/RelayUsageListener.kt @@ -48,7 +48,7 @@ class RelayUsageListener( } } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/privateDM/datasource/ChatroomNip04HistorySubAssembler.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/privateDM/datasource/ChatroomNip04HistorySubAssembler.kt index 4cbe77bffb..c9e30794a4 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/privateDM/datasource/ChatroomNip04HistorySubAssembler.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/privateDM/datasource/ChatroomNip04HistorySubAssembler.kt @@ -116,7 +116,7 @@ class ChatroomNip04HistorySubAssembler( // so a late callback can't move another room's cursors. newEose (framework bookkeeping) runs anyway. val myCursors = cursorsFor(key) return object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/datasource/ConcordChannelHistoryFilterAssembler.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/datasource/ConcordChannelHistoryFilterAssembler.kt index 2720cb1d77..a6cb65f165 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/datasource/ConcordChannelHistoryFilterAssembler.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/datasource/ConcordChannelHistoryFilterAssembler.kt @@ -170,7 +170,7 @@ class ConcordChannelHistorySubAssembler( // cursors so a late callback can't move another channel's cursors. newEose runs regardless. val myCursors = cursorsFor(key) return object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/relayGroup/datasource/RelayGroupOpenChatHistoryFilterAssembler.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/relayGroup/datasource/RelayGroupOpenChatHistoryFilterAssembler.kt index 7b6f7bbcac..f75b814bf7 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/relayGroup/datasource/RelayGroupOpenChatHistoryFilterAssembler.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/relayGroup/datasource/RelayGroupOpenChatHistoryFilterAssembler.kt @@ -126,7 +126,7 @@ class RelayGroupOpenChatHistorySubAssembler( // cursors so a late callback can't move another group's cursors. newEose runs regardless. val myCursors = cursorsFor(key) return object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/relayGroup/datasource/RelayGroupOpenThreadsHistoryFilterAssembler.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/relayGroup/datasource/RelayGroupOpenThreadsHistoryFilterAssembler.kt index ddbef10a9a..cb2d8cc4b3 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/relayGroup/datasource/RelayGroupOpenThreadsHistoryFilterAssembler.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/relayGroup/datasource/RelayGroupOpenThreadsHistoryFilterAssembler.kt @@ -123,7 +123,7 @@ class RelayGroupOpenThreadsHistorySubAssembler( // cursors so a late callback can't move another group's cursors. newEose runs regardless. val myCursors = cursorsFor(key) return object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/datasource/ChatroomListNip04HistorySubAssembler.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/datasource/ChatroomListNip04HistorySubAssembler.kt index bf7f2fee05..60a41ede35 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/datasource/ChatroomListNip04HistorySubAssembler.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/datasource/ChatroomListNip04HistorySubAssembler.kt @@ -108,7 +108,7 @@ class ChatroomListNip04HistorySubAssembler( // cursors so a late callback can't move another account's cursors. newEose runs regardless. val myCursors = key.account.chatroomList.nip04History return object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chess/datasource/ChessFeedFilterSubAssembler.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chess/datasource/ChessFeedFilterSubAssembler.kt index 01616b65a6..0780530666 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chess/datasource/ChessFeedFilterSubAssembler.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chess/datasource/ChessFeedFilterSubAssembler.kt @@ -70,7 +70,7 @@ class ChessFeedFilterSubAssembler( newEose(key, relay, TimeUtils.now(), forFilters) } - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/relays/eventsync/EventSync.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/relays/eventsync/EventSync.kt index 653003ecf4..0918f1a13b 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/relays/eventsync/EventSync.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/relays/eventsync/EventSync.kt @@ -456,7 +456,7 @@ class EventSync( } } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt index b36cb843e3..02b482bebf 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt @@ -668,7 +668,7 @@ class Context( val filters = relays.associateWith { listOf(responseFilter) } val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GeochatCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GeochatCommands.kt index 2547025b63..22c74904ce 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GeochatCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/GeochatCommands.kt @@ -133,7 +133,7 @@ object GeochatCommands { val subId = newSubId() val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/NipCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/NipCommand.kt index 786eefed70..21c69fabf9 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/NipCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/NipCommand.kt @@ -167,7 +167,7 @@ object NipCommand { val remaining = SEARCH_RELAYS.toMutableSet() val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/NostrConnect.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/NostrConnect.kt index 33d51fe403..03629f9922 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/NostrConnect.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/NostrConnect.kt @@ -118,7 +118,7 @@ object NostrConnect { val subId = newSubId() val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/SubscribeCommand.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/SubscribeCommand.kt index b8afce7b90..5779bacf6c 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/SubscribeCommand.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/SubscribeCommand.kt @@ -81,7 +81,7 @@ object SubscribeCommand { val subId = newSubId() val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/nip64Chess/ChessRelayFetchHelper.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/nip64Chess/ChessRelayFetchHelper.kt index f45c499791..2a9c5a407c 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/nip64Chess/ChessRelayFetchHelper.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/nip64Chess/ChessRelayFetchHelper.kt @@ -100,7 +100,7 @@ class ChessRelayFetchHelper( val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/assemblers/FeedMetadataCoordinator.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/assemblers/FeedMetadataCoordinator.kt index 659ca97dcd..2260b279a5 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/assemblers/FeedMetadataCoordinator.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/assemblers/FeedMetadataCoordinator.kt @@ -99,7 +99,7 @@ class FeedMetadataCoordinator( val listener = if (onEvent != null) { object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, @@ -295,7 +295,7 @@ class FeedMetadataCoordinator( val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, @@ -371,7 +371,7 @@ class FeedMetadataCoordinator( val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/eoseManagers/PerKeyEoseManager.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/eoseManagers/PerKeyEoseManager.kt index b609d52dc2..101216858c 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/eoseManagers/PerKeyEoseManager.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/eoseManagers/PerKeyEoseManager.kt @@ -90,7 +90,7 @@ abstract class PerKeyEoseManager( newEose(queryState, relay, TimeUtils.now(), forFilters) } - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/eoseManagers/SingleSubEoseManager.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/eoseManagers/SingleSubEoseManager.kt index fd2c6f4922..da0c3c66d9 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/eoseManagers/SingleSubEoseManager.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/eoseManagers/SingleSubEoseManager.kt @@ -86,7 +86,7 @@ abstract class SingleSubEoseManager( newEose(relay, TimeUtils.now(), forFilters) } - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relays/health/RelayHealthListener.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relays/health/RelayHealthListener.kt index b90d9e1502..6e564e356e 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relays/health/RelayHealthListener.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relays/health/RelayHealthListener.kt @@ -44,7 +44,7 @@ class RelayHealthListener( store.recordConnect(relay.url, TimeUtils.now()) } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/broadcast/BroadcastTracker.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/broadcast/BroadcastTracker.kt index e9f8166cc6..01f4c8a9fb 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/broadcast/BroadcastTracker.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/service/broadcast/BroadcastTracker.kt @@ -118,7 +118,7 @@ class BroadcastTracker { } } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, @@ -294,7 +294,7 @@ class BroadcastTracker { } } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/OutboxDispatcher.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/OutboxDispatcher.kt index 24b6401cb3..7374a54337 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/OutboxDispatcher.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/wot/OutboxDispatcher.kt @@ -371,7 +371,7 @@ class OutboxDispatcher( val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, @@ -415,7 +415,7 @@ class OutboxDispatcher( val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/nip64Chess/ChessEventBroadcaster.kt b/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/nip64Chess/ChessEventBroadcaster.kt index e8630a810a..47a66c3f6c 100644 --- a/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/nip64Chess/ChessEventBroadcaster.kt +++ b/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/nip64Chess/ChessEventBroadcaster.kt @@ -90,7 +90,7 @@ class ChessEventBroadcaster( val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/relayClient/paging/WindowLoadTracker.kt b/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/relayClient/paging/WindowLoadTracker.kt index 345fe58f18..80837de516 100644 --- a/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/relayClient/paging/WindowLoadTracker.kt +++ b/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/relayClient/paging/WindowLoadTracker.kt @@ -212,7 +212,7 @@ fun WindowLoadTracker.trackingListener(forward: (NormalizedRelayUrl, List filter.kinds?.forEach { kind -> script[kind to relay]?.forEach { event -> - listener?.onEvent(event, isLive = false, relay = relay, forFilters = null) + kotlinx.coroutines.runBlocking { listener?.onEvent(event, isLive = false, relay = relay, forFilters = null) } } } } diff --git a/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/wot/OutboxDispatcherTest.kt b/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/wot/OutboxDispatcherTest.kt index 9238b5ce1d..5fe6483199 100644 --- a/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/wot/OutboxDispatcherTest.kt +++ b/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/wot/OutboxDispatcherTest.kt @@ -171,7 +171,7 @@ class OutboxDispatcherTest { filterList.forEach { filter -> filter.kinds?.forEach { kind -> script[kind to relay]?.forEach { event -> - listener?.onEvent(event, isLive = false, relay = relay, forFilters = null) + kotlinx.coroutines.runBlocking { listener?.onEvent(event, isLive = false, relay = relay, forFilters = null) } } } } diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/Main.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/Main.kt index c9966e3442..3c5faabd88 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/Main.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/Main.kt @@ -1786,7 +1786,7 @@ fun MainContent( filters = listOf(filter), listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: com.vitorpamplona.quartz.nip01Core.core.Event, isLive: Boolean, relay: NormalizedRelayUrl, @@ -1845,7 +1845,7 @@ fun MainContent( relays = outbox, listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: com.vitorpamplona.quartz.nip01Core.core.Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/account/AccountManager.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/account/AccountManager.kt index 549847e8b0..6eb0354925 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/account/AccountManager.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/account/AccountManager.kt @@ -249,7 +249,7 @@ class AccountManager internal constructor( } } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/followpacks/FollowPacksState.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/followpacks/FollowPacksState.kt index 80e88355c0..cb506f3282 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/followpacks/FollowPacksState.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/followpacks/FollowPacksState.kt @@ -164,7 +164,7 @@ class FollowPacksState( private fun subscribeToDiscovery() { val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: com.vitorpamplona.quartz.nip01Core.core.Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/followpacks/MetadataPrefetch.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/followpacks/MetadataPrefetch.kt index 240056ddf1..bdeb93104c 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/followpacks/MetadataPrefetch.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/followpacks/MetadataPrefetch.kt @@ -45,7 +45,7 @@ fun RelayConnectionManager.subscribeMetadataFor( val filter = Filter(kinds = listOf(MetadataEvent.KIND), authors = pubkeys) val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/followpacks/ui/FromThePackFeed.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/followpacks/ui/FromThePackFeed.kt index f9b1efc816..de661f4b36 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/followpacks/ui/FromThePackFeed.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/followpacks/ui/FromThePackFeed.kt @@ -92,7 +92,7 @@ fun FromThePackFeed( listOf(filter), listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/followpacks/ui/RenderFollowPackCard.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/followpacks/ui/RenderFollowPackCard.kt index ae82ca7876..3a1bc3f9b6 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/followpacks/ui/RenderFollowPackCard.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/followpacks/ui/RenderFollowPackCard.kt @@ -102,7 +102,7 @@ fun RenderFollowPackCard( listOf(filter), listener = object : com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: com.vitorpamplona.quartz.nip01Core.core.Event, isLive: Boolean, relay: com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl, diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/network/RelayConnectionManager.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/network/RelayConnectionManager.kt index 9e07c9eb21..73057a1619 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/network/RelayConnectionManager.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/network/RelayConnectionManager.kt @@ -200,7 +200,7 @@ open class RelayConnectionManager( filters = filterMap, listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, @@ -264,7 +264,7 @@ open class RelayConnectionManager( } } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/search/DesktopRelayUserSearchDelegate.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/search/DesktopRelayUserSearchDelegate.kt index a58d2bad74..5b74f8eccc 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/search/DesktopRelayUserSearchDelegate.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/search/DesktopRelayUserSearchDelegate.kt @@ -69,7 +69,7 @@ class DesktopRelayUserSearchDelegate( relays = relays, listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/ChessSubscription.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/ChessSubscription.kt index 0d0c645669..6b6602d57f 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/ChessSubscription.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/ChessSubscription.kt @@ -96,7 +96,7 @@ class DesktopChessSubscriptionController( relays = state.relays, listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/DesktopRelaySubscriptionsCoordinator.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/DesktopRelaySubscriptionsCoordinator.kt index ffa8f497ec..3a315d6507 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/DesktopRelaySubscriptionsCoordinator.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/DesktopRelaySubscriptionsCoordinator.kt @@ -303,7 +303,7 @@ class DesktopRelaySubscriptionsCoordinator( val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, @@ -428,7 +428,7 @@ class DesktopRelaySubscriptionsCoordinator( val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/SubscriptionUtils.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/SubscriptionUtils.kt index b9fa38e75b..7fe5bfb77e 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/SubscriptionUtils.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/SubscriptionUtils.kt @@ -81,7 +81,7 @@ fun rememberSubscription( relays = cfg.relays, listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/ImportFollowListDialog.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/ImportFollowListDialog.kt index 52d3983dfb..41b057c510 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/ImportFollowListDialog.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/ImportFollowListDialog.kt @@ -225,7 +225,7 @@ fun ImportFollowListDialog( relays = relays, listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, @@ -277,7 +277,7 @@ fun ImportFollowListDialog( ), listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/NoteActions.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/NoteActions.kt index 1aa9f3a1fe..66770814d0 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/NoteActions.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/NoteActions.kt @@ -858,7 +858,7 @@ private suspend fun fetchMetadataForUsers( relays = relays, listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, @@ -1666,7 +1666,7 @@ private suspend fun fetchUserLightningAddress( relays = relays, listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/chats/ChatroomListState.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/chats/ChatroomListState.kt index d636037723..2b18254d14 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/chats/ChatroomListState.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/chats/ChatroomListState.kt @@ -175,7 +175,7 @@ class ChatroomListState( val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/benchmark/LaunchScenario.kt b/desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/benchmark/LaunchScenario.kt index 79fb062b0e..7edddfa185 100644 --- a/desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/benchmark/LaunchScenario.kt +++ b/desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/benchmark/LaunchScenario.kt @@ -139,7 +139,7 @@ object LaunchScenario { ), listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: com.vitorpamplona.quartz.nip01Core.core.Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/testrelay/LaunchFixtureRelayTest.kt b/desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/testrelay/LaunchFixtureRelayTest.kt index 618704a8ed..95d9c8a8d1 100644 --- a/desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/testrelay/LaunchFixtureRelayTest.kt +++ b/desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/testrelay/LaunchFixtureRelayTest.kt @@ -73,7 +73,7 @@ class LaunchFixtureRelayTest { ), listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/testrelay/SubscribeBeforeConnectTest.kt b/desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/testrelay/SubscribeBeforeConnectTest.kt index 2720941b42..b70fd89c9b 100644 --- a/desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/testrelay/SubscribeBeforeConnectTest.kt +++ b/desktopApp/src/jvmTest/kotlin/com/vitorpamplona/amethyst/desktop/testrelay/SubscribeBeforeConnectTest.kt @@ -72,7 +72,7 @@ class SubscribeBeforeConnectTest { ), listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt index c98e050e37..0557f58afa 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt @@ -686,7 +686,7 @@ class MirrorWorker( val watermark = AtomicLong(initialSince) val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/geode/src/test/kotlin/com/vitorpamplona/geode/GracefulShutdownTest.kt b/geode/src/test/kotlin/com/vitorpamplona/geode/GracefulShutdownTest.kt index 7a55645b7f..3d568e2550 100644 --- a/geode/src/test/kotlin/com/vitorpamplona/geode/GracefulShutdownTest.kt +++ b/geode/src/test/kotlin/com/vitorpamplona/geode/GracefulShutdownTest.kt @@ -125,7 +125,7 @@ class GracefulShutdownTest { val gotEose = Channel(UNLIMITED) val listener = object : RelayConnectionListener { - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/geode/src/test/kotlin/com/vitorpamplona/geode/KtorRelayTest.kt b/geode/src/test/kotlin/com/vitorpamplona/geode/KtorRelayTest.kt index 1b8f84833a..b79f81bf51 100644 --- a/geode/src/test/kotlin/com/vitorpamplona/geode/KtorRelayTest.kt +++ b/geode/src/test/kotlin/com/vitorpamplona/geode/KtorRelayTest.kt @@ -389,7 +389,7 @@ class KtorRelayTest { "close-test", mapOf(server.url.normalizeRelayUrl() to listOf(Filter(kinds = listOf(1)))), object : com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: com.vitorpamplona.quartz.nip01Core.core.Event, isLive: Boolean, relay: com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl, diff --git a/geode/src/test/kotlin/com/vitorpamplona/geode/Nip01ComplianceTest.kt b/geode/src/test/kotlin/com/vitorpamplona/geode/Nip01ComplianceTest.kt index 4851956a13..7f69de4c89 100644 --- a/geode/src/test/kotlin/com/vitorpamplona/geode/Nip01ComplianceTest.kt +++ b/geode/src/test/kotlin/com/vitorpamplona/geode/Nip01ComplianceTest.kt @@ -276,7 +276,7 @@ class Nip01ComplianceTest : RelayClientTest() { "sub-A", mapOf(defaultRelayUrl to listOf(Filter(kinds = listOf(1)))), object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, @@ -297,7 +297,7 @@ class Nip01ComplianceTest : RelayClientTest() { "sub-B", mapOf(defaultRelayUrl to listOf(Filter(kinds = listOf(4)))), object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, @@ -385,7 +385,7 @@ class Nip01ComplianceTest : RelayClientTest() { "live-1", mapOf(defaultRelayUrl to listOf(Filter(kinds = listOf(1)))), object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, @@ -424,7 +424,7 @@ class Nip01ComplianceTest : RelayClientTest() { "live-2", mapOf(defaultRelayUrl to listOf(Filter(kinds = listOf(1)))), object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, @@ -465,7 +465,7 @@ class Nip01ComplianceTest : RelayClientTest() { "eph-1", mapOf(defaultRelayUrl to listOf(Filter(kinds = listOf(20_001)))), object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, @@ -534,7 +534,7 @@ class Nip01ComplianceTest : RelayClientTest() { relayB to listOf(Filter(kinds = listOf(1))), ), object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/geode/src/test/kotlin/com/vitorpamplona/geode/Nip09DeletionTest.kt b/geode/src/test/kotlin/com/vitorpamplona/geode/Nip09DeletionTest.kt index c4e801665a..914da8e385 100644 --- a/geode/src/test/kotlin/com/vitorpamplona/geode/Nip09DeletionTest.kt +++ b/geode/src/test/kotlin/com/vitorpamplona/geode/Nip09DeletionTest.kt @@ -80,7 +80,7 @@ class Nip09DeletionTest { subId, mapOf(relayUrl to listOf(filter)), object : com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/geode/src/test/kotlin/com/vitorpamplona/geode/Nip77NegentropyTest.kt b/geode/src/test/kotlin/com/vitorpamplona/geode/Nip77NegentropyTest.kt index edff7aeb69..1b156225ca 100644 --- a/geode/src/test/kotlin/com/vitorpamplona/geode/Nip77NegentropyTest.kt +++ b/geode/src/test/kotlin/com/vitorpamplona/geode/Nip77NegentropyTest.kt @@ -96,7 +96,7 @@ class Nip77NegentropyTest { compression: Boolean, ) {} - override fun onMessage(text: String) { + override suspend fun onMessage(text: String) { incoming.trySend(text) } diff --git a/geode/src/test/kotlin/com/vitorpamplona/geode/mirror/MirrorWorkerTrustOriginTest.kt b/geode/src/test/kotlin/com/vitorpamplona/geode/mirror/MirrorWorkerTrustOriginTest.kt index 28d7701ebc..634331fdf9 100644 --- a/geode/src/test/kotlin/com/vitorpamplona/geode/mirror/MirrorWorkerTrustOriginTest.kt +++ b/geode/src/test/kotlin/com/vitorpamplona/geode/mirror/MirrorWorkerTrustOriginTest.kt @@ -99,7 +99,7 @@ class MirrorWorkerTrustOriginTest { override fun connect() { connected = true out.onOpen(0, false) - out.onMessage(frame) + kotlinx.coroutines.runBlocking { out.onMessage(frame) } } override fun disconnect() { diff --git a/geode/src/test/kotlin/com/vitorpamplona/geode/perf/LoadBenchmark.kt b/geode/src/test/kotlin/com/vitorpamplona/geode/perf/LoadBenchmark.kt index 771a12784a..7c4acddd4b 100644 --- a/geode/src/test/kotlin/com/vitorpamplona/geode/perf/LoadBenchmark.kt +++ b/geode/src/test/kotlin/com/vitorpamplona/geode/perf/LoadBenchmark.kt @@ -258,7 +258,7 @@ class LoadBenchmark { "fanout-$i", mapOf(relayUrl to listOf(Filter(kinds = listOf(1)))), object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: com.vitorpamplona.quartz.nip01Core.core.Event, isLive: Boolean, relay: NormalizedRelayUrl, @@ -436,7 +436,7 @@ class LoadBenchmark { "fanout-$i", mapOf(relayUrl to listOf(Filter(kinds = listOf(1)))), object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: com.vitorpamplona.quartz.nip01Core.core.Event, isLive: Boolean, relay: NormalizedRelayUrl, @@ -568,7 +568,7 @@ class LoadBenchmark { ), ), object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: com.vitorpamplona.quartz.nip01Core.core.Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/geode/src/testFixtures/kotlin/com/vitorpamplona/geode/testing/SubscriptionTesting.kt b/geode/src/testFixtures/kotlin/com/vitorpamplona/geode/testing/SubscriptionTesting.kt index d2c7ec2d04..fc50d6ef09 100644 --- a/geode/src/testFixtures/kotlin/com/vitorpamplona/geode/testing/SubscriptionTesting.kt +++ b/geode/src/testFixtures/kotlin/com/vitorpamplona/geode/testing/SubscriptionTesting.kt @@ -72,7 +72,7 @@ suspend fun NostrClient.collectUntilEoseMulti( subId, mapOf(relay to filters), object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankCrawler.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankCrawler.kt index 70ef709b72..4cba6e0c9f 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankCrawler.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/experimental/graperank/GrapeRankCrawler.kt @@ -1489,7 +1489,7 @@ class GrapeRankCrawler( val lastEvt = AtomicLong(-1) val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/NostrClient.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/NostrClient.kt index 8378d8140a..08e39201ad 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/NostrClient.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/NostrClient.kt @@ -329,7 +329,7 @@ class NostrClient( listeners.forEach { it.onSent(relay, cmdStr, cmd, success) } } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/AdaptiveRelayLimiter.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/AdaptiveRelayLimiter.kt index 0fcb87907b..be2547ea97 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/AdaptiveRelayLimiter.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/AdaptiveRelayLimiter.kt @@ -137,7 +137,7 @@ class AdaptiveRelayLimiter( if (wait > 0) delay(wait) } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/EventCollector.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/EventCollector.kt index 7c97ec3211..1964d465b5 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/EventCollector.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/EventCollector.kt @@ -37,7 +37,7 @@ class EventCollector( ) { private val clientListener = object : RelayConnectionListener { - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientCountExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientCountExt.kt index 0f19d9d75c..66e502740c 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientCountExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientCountExt.kt @@ -59,7 +59,7 @@ suspend fun INostrClient.count( val listener = object : RelayConnectionListener { - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, @@ -117,7 +117,7 @@ suspend fun INostrClient.count( val listener = object : RelayConnectionListener { - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesExt.kt index f43aed278f..f7b4834515 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesExt.kt @@ -94,7 +94,7 @@ suspend fun INostrClient.fetchAllPages( filters: List, idleTimeoutMs: Long = 30_000L, onNewPage: ((Long) -> Unit)? = null, - onEvent: (Event) -> Unit, + onEvent: suspend (Event) -> Unit, ): Int { var until: Long? = null var totalEvents = 0 @@ -172,7 +172,7 @@ suspend fun INostrClient.fetchAllPages( try { val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, @@ -320,7 +320,7 @@ suspend fun INostrClient.fetchAllPages( filters: List, idleTimeoutMs: Long = 30_000L, onNewPage: ((Long) -> Unit)? = null, - onEvent: (Event) -> Unit, + onEvent: suspend (Event) -> Unit, ): Int = fetchAllPages( relay = RelayUrlNormalizer.normalize(relay), diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllWithHooksExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllWithHooksExt.kt index f3726f5fe2..62d13ce5e9 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllWithHooksExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllWithHooksExt.kt @@ -102,7 +102,7 @@ suspend fun INostrClient.fetchAllWithHooks( val doneReasons = HashMap() val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchFirstExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchFirstExt.kt index 2a80fcfa45..07a3f18e22 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchFirstExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchFirstExt.kt @@ -96,7 +96,7 @@ suspend fun INostrClient.fetchFirst( val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropyFanOutExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropyFanOutExt.kt index cce9d24bf1..7ac08647f5 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropyFanOutExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropyFanOutExt.kt @@ -83,7 +83,7 @@ suspend fun negentropySyncFanOut( idleTimeoutMs: Long = 120_000L, reconcileConcurrency: Int = 2, onProgress: ((needSoFar: Int, downloaded: Int) -> Unit)? = null, - onEvent: (Event) -> Unit, + onEvent: suspend (Event) -> Unit, ): NegentropyFanOutResult { require(clients.isNotEmpty()) { "at least one client is required" } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropySyncExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropySyncExt.kt index 89360facbf..611720876a 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropySyncExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropySyncExt.kt @@ -163,7 +163,7 @@ suspend fun INostrClient.negentropySync( idBufferBatches: Int = maxConcurrentReqs * 4, localEntries: List = emptyList(), onProgress: ((needSoFar: Int, downloaded: Int) -> Unit)? = null, - onEvent: (Event) -> Unit, + onEvent: suspend (Event) -> Unit, ): NegentropySyncResult { val need = AtomicInt(0) val windows = AtomicInt(0) @@ -242,7 +242,7 @@ suspend fun INostrClient.negentropySync( idBufferBatches: Int = maxConcurrentReqs * 4, localEntries: List = emptyList(), onProgress: ((needSoFar: Int, downloaded: Int) -> Unit)? = null, - onEvent: (Event) -> Unit, + onEvent: suspend (Event) -> Unit, ): NegentropySyncResult = negentropySync( relay = RelayUrlNormalizer.normalize(relay), @@ -309,14 +309,14 @@ suspend fun INostrClient.negentropySyncOrFetch( idBufferBatches: Int = maxConcurrentReqs * 4, localEntries: List = emptyList(), onProgress: ((needSoFar: Int, downloaded: Int) -> Unit)? = null, - onEvent: (Event) -> Unit, + onEvent: suspend (Event) -> Unit, ): NegentropyOrFetchResult { val seen = HashSet() var delivered = 0 // Shared dedup + cap across both phases. Returns true if the event was new and // delivered. Both phases run sequentially, so no concurrent access. - fun accept(event: Event): Boolean { + suspend fun accept(event: Event): Boolean { if ((maxEvents <= 0 || delivered < maxEvents) && seen.add(event.id)) { delivered++ onEvent(event) @@ -364,7 +364,7 @@ suspend fun INostrClient.negentropySyncOrFetch( idBufferBatches: Int = maxConcurrentReqs * 4, localEntries: List = emptyList(), onProgress: ((needSoFar: Int, downloaded: Int) -> Unit)? = null, - onEvent: (Event) -> Unit, + onEvent: suspend (Event) -> Unit, ): NegentropyOrFetchResult = negentropySyncOrFetch( relay = RelayUrlNormalizer.normalize(relay), @@ -876,7 +876,7 @@ private suspend fun INostrClient.reconcileStreaming( if (relay.url == targetUrl) clock.bump() } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, @@ -1103,7 +1103,7 @@ internal suspend fun INostrClient.fetchByIds( val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientPublishExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientPublishExt.kt index c1aa8c14a2..fc725dbe84 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientPublishExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientPublishExt.kt @@ -132,7 +132,7 @@ suspend fun INostrClient.publishAndCollectResults( } } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/RelayInsertConfirmationCollector.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/RelayInsertConfirmationCollector.kt index 5014eada6f..6b536de30b 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/RelayInsertConfirmationCollector.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/RelayInsertConfirmationCollector.kt @@ -37,7 +37,7 @@ class RelayInsertConfirmationCollector( ) { private val clientListener = object : RelayConnectionListener { - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/RelayLogger.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/RelayLogger.kt index 78b304edae..86313d9a48 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/RelayLogger.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/RelayLogger.kt @@ -50,7 +50,7 @@ class RelayLogger( private val clientListener = object : RelayConnectionListener { - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/RelayNotifier.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/RelayNotifier.kt index 65f2c6aae8..1853aec5e3 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/RelayNotifier.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/RelayNotifier.kt @@ -40,7 +40,7 @@ class RelayNotifier( private val clientListener = object : RelayConnectionListener { - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/auth/RelayAuthenticator.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/auth/RelayAuthenticator.kt index 1c5bc662de..b45e652421 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/auth/RelayAuthenticator.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/auth/RelayAuthenticator.kt @@ -102,7 +102,7 @@ class RelayAuthenticator( private val clientListener = object : RelayConnectionListener { - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/counts/RelayActiveCountStates.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/counts/RelayActiveCountStates.kt index dba6888dc7..d03077ba2e 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/counts/RelayActiveCountStates.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/counts/RelayActiveCountStates.kt @@ -45,7 +45,7 @@ class RelayActiveCountStates( queryStates.put(relay.url, CountQueryState()) } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/limits/RelayLimitsTracker.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/limits/RelayLimitsTracker.kt index a8ffa37d63..65f61e8057 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/limits/RelayLimitsTracker.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/limits/RelayLimitsTracker.kt @@ -76,7 +76,7 @@ class RelayLimitsTracker( private val clientListener = object : RelayConnectionListener { - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/listeners/RedirectConnectionListener.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/listeners/RedirectConnectionListener.kt index 5d7fba5ffc..75e6ce6363 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/listeners/RedirectConnectionListener.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/listeners/RedirectConnectionListener.kt @@ -48,7 +48,7 @@ open class RedirectConnectionListener( listener.onSent(relay, cmdStr, cmd, success) } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/listeners/RelayConnectionListener.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/listeners/RelayConnectionListener.kt index feefbb532f..8f70a9cc11 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/listeners/RelayConnectionListener.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/listeners/RelayConnectionListener.kt @@ -53,7 +53,7 @@ interface RelayConnectionListener { /** * New error */ - fun onIncomingMessage( + suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolCounts.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolCounts.kt index 2d9a44ea31..97c48dc5fa 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolCounts.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolCounts.kt @@ -123,7 +123,7 @@ class PoolCounts { } } - fun onIncomingMessage( + suspend fun onIncomingMessage( relay: IRelayClient, msg: Message, ) { diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolEventOutbox.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolEventOutbox.kt index c249d2aa3e..928cfd0bc2 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolEventOutbox.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolEventOutbox.kt @@ -178,7 +178,7 @@ class PoolEventOutbox { } } - fun onIncomingMessage( + suspend fun onIncomingMessage( relay: NormalizedRelayUrl, msg: Message, ) { diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolRequests.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolRequests.kt index 7d4b08315b..7c87d14712 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolRequests.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolRequests.kt @@ -238,7 +238,7 @@ class PoolRequests( /** * When a new message is received by the relay, updates the sub */ - fun onIncomingMessage( + suspend fun onIncomingMessage( relay: IRelayClient, msg: Message, ) { diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/RelayPool.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/RelayPool.kt index 22c2e0003b..62fe531d86 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/RelayPool.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/RelayPool.kt @@ -248,7 +248,7 @@ class RelayPool( listener.onDisconnected(relay) } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/DynamicSubscription.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/DynamicSubscription.kt index 1821f4ad86..f6eeb770dd 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/DynamicSubscription.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/DynamicSubscription.kt @@ -34,7 +34,7 @@ class DynamicSubscription( SubscriptionHandle { val subId = RandomInstance.randomChars(10) - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/NostrClientFetchAsFlowExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/NostrClientFetchAsFlowExt.kt index 1cda8b7e7c..6dd2ccaae5 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/NostrClientFetchAsFlowExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/NostrClientFetchAsFlowExt.kt @@ -63,7 +63,7 @@ fun INostrClient.fetchAsFlow( val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/NostrClientSubscribeAsFlowExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/NostrClientSubscribeAsFlowExt.kt index 50b6af68ab..4d7736ac71 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/NostrClientSubscribeAsFlowExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/NostrClientSubscribeAsFlowExt.kt @@ -69,7 +69,7 @@ fun INostrClient.subscribeAsFlow( val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/RelayActiveRequestStates.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/RelayActiveRequestStates.kt index a51d2f1df2..f75dfbe365 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/RelayActiveRequestStates.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/RelayActiveRequestStates.kt @@ -46,7 +46,7 @@ class RelayActiveRequestStates( subStates[relay.url] = RequestSubscriptionState() } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/StaticSubscription.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/StaticSubscription.kt index 1e87560adb..d33e9ff080 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/StaticSubscription.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/StaticSubscription.kt @@ -35,7 +35,7 @@ class StaticSubscription( SubscriptionHandle { val subId = RandomInstance.randomChars(10) - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/SubscriptionListener.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/SubscriptionListener.kt index 45f614237d..15486f55af 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/SubscriptionListener.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/SubscriptionListener.kt @@ -30,7 +30,7 @@ interface SubscriptionListener { forFilters: List?, ) {} - fun onEvent( + suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/stats/RelayReqStats.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/stats/RelayReqStats.kt index 2da8b3ca6d..0dcc1291df 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/stats/RelayReqStats.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/reqs/stats/RelayReqStats.kt @@ -37,7 +37,7 @@ class RelayReqStats( private val clientListener = object : RelayConnectionListener { - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/basic/BasicRelayClient.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/basic/BasicRelayClient.kt index 5005e63c9c..d0f1b7bc16 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/basic/BasicRelayClient.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/basic/BasicRelayClient.kt @@ -156,7 +156,7 @@ open class BasicRelayClient( listener.onConnected(this@BasicRelayClient, pingMillis, compression) } - override fun onMessage(text: String) { + override suspend fun onMessage(text: String) { try { val msg = decoder.decode(text) listener.onIncomingMessage(this@BasicRelayClient, text, msg) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/standalone/StandaloneRelayClient.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/standalone/StandaloneRelayClient.kt index 5063f03319..52f7947b2a 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/standalone/StandaloneRelayClient.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/standalone/StandaloneRelayClient.kt @@ -66,7 +66,7 @@ class StandaloneRelayClient( syncFilters() } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/stats/RelayStats.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/stats/RelayStats.kt index b6548e1a2c..73b41bd000 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/stats/RelayStats.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/stats/RelayStats.kt @@ -83,7 +83,7 @@ class RelayStats( get(relay.url).addBytesSent(cmdStr.bytesUsedInMemory()) } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/sockets/WebSocketListener.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/sockets/WebSocketListener.kt index fef0f36a6b..4f4cdc522b 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/sockets/WebSocketListener.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/sockets/WebSocketListener.kt @@ -30,7 +30,7 @@ interface WebSocketListener { compression: Boolean, ) - fun onMessage(text: String) + suspend fun onMessage(text: String) fun onClosed( code: Int, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip46RemoteSigner/server/NostrConnectSignerService.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip46RemoteSigner/server/NostrConnectSignerService.kt index 4be37a0b84..bc473bcaf1 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip46RemoteSigner/server/NostrConnectSignerService.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip46RemoteSigner/server/NostrConnectSignerService.kt @@ -183,7 +183,7 @@ class NostrConnectSignerService( val subId = newSubId() val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayObserver.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayObserver.kt index 1e293e3d05..d9d40ec0f5 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayObserver.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayObserver.kt @@ -183,7 +183,7 @@ class RelayObserver : RelayConnectionListener { } } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt index 48d33d9185..711c38551a 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt @@ -254,7 +254,7 @@ class RelayProber( val subId = newSubId() val subListener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegentropyManager.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegentropyManager.kt index 3984c1436d..83ad948534 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegentropyManager.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegentropyManager.kt @@ -94,7 +94,7 @@ class NegentropyManager( relay.sendIfConnected(session.close()) } - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nipBEBle/relay/BleMeshManager.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nipBEBle/relay/BleMeshManager.kt index 9e33fb9b91..504d0ff9aa 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nipBEBle/relay/BleMeshManager.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nipBEBle/relay/BleMeshManager.kt @@ -260,7 +260,7 @@ class BleMeshManager( private inner class ClientConnectionListener( val peerUuid: String, ) : RelayConnectionListener { - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nipBEBle/relay/BleNostrClient.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nipBEBle/relay/BleNostrClient.kt index 1a0771d2e7..2f71344a25 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nipBEBle/relay/BleNostrClient.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nipBEBle/relay/BleNostrClient.kt @@ -23,6 +23,7 @@ package com.vitorpamplona.quartz.nipBEBle.relay import com.vitorpamplona.quartz.nip01Core.core.OptimizedJsonMapper import com.vitorpamplona.quartz.nip01Core.relay.client.listeners.RelayConnectionListener import com.vitorpamplona.quartz.nip01Core.relay.client.single.IRelayClient +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.Message import com.vitorpamplona.quartz.nip01Core.relay.commands.toRelay.Command import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nipBEBle.BleConfig @@ -31,8 +32,14 @@ import com.vitorpamplona.quartz.nipBEBle.protocol.BleChunkAssembler import com.vitorpamplona.quartz.nipBEBle.protocol.BleMessageChunker import com.vitorpamplona.quartz.nipBEBle.transport.BleTransport import com.vitorpamplona.quartz.utils.Log +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.launch import kotlin.concurrent.atomics.AtomicBoolean import kotlin.concurrent.atomics.ExperimentalAtomicApi @@ -72,6 +79,31 @@ class BleNostrClient( private val sendQueue = Channel>(Channel.UNLIMITED) private val isSending = AtomicBoolean(false) + /** Parsed messages waiting to reach the suspending listener — see [onChunkReceived]. */ + private val incoming = Channel>(Channel.UNLIMITED) + + private val scope = CoroutineScope(Dispatchers.Default + SupervisorJob()) + + private val pump = + scope.launch { + for ((raw, msg) in incoming) { + try { + listener.onIncomingMessage(this@BleNostrClient, raw, msg) + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + // One peer's bad message must not end the pump for the rest. + Log.e("BleNostrClient", "Failure handling message from ${peer.deviceUuid}: $raw", e) + } + } + } + + /** Stops the [pump]; the client is unusable afterwards, like a closed socket. */ + fun release() { + incoming.close() + scope.cancel() + } + override fun isConnected(): Boolean = connected override fun needsToReconnect(): Boolean = !connected @@ -144,7 +176,13 @@ class BleNostrClient( try { val msg = OptimizedJsonMapper.fromJsonToMessage(message) - listener.onIncomingMessage(this, message, msg) + // The platform hands BLE notifications to a callback that cannot + // suspend, and the listener chain now does — so the message is + // handed off rather than delivered here. Same shape the websocket + // transport already uses: UNLIMITED so this callback never blocks + // the BLE stack, drained by ONE coroutine so message order survives + // the boundary. + incoming.trySend(message to msg) } catch (e: Exception) { Log.e("BleNostrClient", "Failed to parse message from ${peer.deviceUuid}: $message", e) } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/limits/RelayLimitsTrackerTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/limits/RelayLimitsTrackerTest.kt index 6547aeb9bb..39dae9ad12 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/limits/RelayLimitsTrackerTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/limits/RelayLimitsTrackerTest.kt @@ -70,67 +70,72 @@ class RelayLimitsTrackerTest { } @Test - fun cachesLimitsPerRelay() { - val (limits, listener) = setup() - val relay = FakeRelayClient(NormalizedRelayUrl("wss://relay.example/")) + fun cachesLimitsPerRelay() = + kotlinx.coroutines.test.runTest { + val (limits, listener) = setup() + val relay = FakeRelayClient(NormalizedRelayUrl("wss://relay.example/")) - assertNull(limits.get(relay.url), "No limits before any LIMITS message") + assertNull(limits.get(relay.url), "No limits before any LIMITS message") - listener.onIncomingMessage(relay, "", LimitsMessage(canWrite = true, maxLimit = 200)) + listener.onIncomingMessage(relay, "", LimitsMessage(canWrite = true, maxLimit = 200)) - assertEquals(true, limits.get(relay.url)?.canWrite) - assertEquals(200, limits.get(relay.url)?.maxLimit) - assertEquals(limits.get(relay.url), limits.limitsFlow.value[relay.url]) - } + assertEquals(true, limits.get(relay.url)?.canWrite) + assertEquals(200, limits.get(relay.url)?.maxLimit) + assertEquals(limits.get(relay.url), limits.limitsFlow.value[relay.url]) + } @Test - fun laterLimitsReplaceEarlierOnes() { - val (limits, listener) = setup() - val relay = FakeRelayClient(NormalizedRelayUrl("wss://relay.example/")) + fun laterLimitsReplaceEarlierOnes() = + kotlinx.coroutines.test.runTest { + val (limits, listener) = setup() + val relay = FakeRelayClient(NormalizedRelayUrl("wss://relay.example/")) - listener.onIncomingMessage(relay, "", LimitsMessage(canWrite = false, maxLimit = 200)) - listener.onIncomingMessage(relay, "", LimitsMessage(canWrite = true, maxLimit = 500)) + listener.onIncomingMessage(relay, "", LimitsMessage(canWrite = false, maxLimit = 200)) + listener.onIncomingMessage(relay, "", LimitsMessage(canWrite = true, maxLimit = 500)) - // A relay re-advertises LIMITS when rights change (e.g. after AUTH flips can_write). - assertEquals(true, limits.get(relay.url)?.canWrite) - assertEquals(500, limits.get(relay.url)?.maxLimit) - } + // A relay re-advertises LIMITS when rights change (e.g. after AUTH flips can_write). + assertEquals(true, limits.get(relay.url)?.canWrite) + assertEquals(500, limits.get(relay.url)?.maxLimit) + } @Test - fun tracksLimitsForDistinctRelaysIndependently() { - val (limits, listener) = setup() - val relayA = FakeRelayClient(NormalizedRelayUrl("wss://a.example/")) - val relayB = FakeRelayClient(NormalizedRelayUrl("wss://b.example/")) + fun tracksLimitsForDistinctRelaysIndependently() = + kotlinx.coroutines.test.runTest { + val (limits, listener) = setup() + val relayA = FakeRelayClient(NormalizedRelayUrl("wss://a.example/")) + val relayB = FakeRelayClient(NormalizedRelayUrl("wss://b.example/")) - listener.onIncomingMessage(relayA, "", LimitsMessage(maxLimit = 100)) - listener.onIncomingMessage(relayB, "", LimitsMessage(maxLimit = 999)) + listener.onIncomingMessage(relayA, "", LimitsMessage(maxLimit = 100)) + listener.onIncomingMessage(relayB, "", LimitsMessage(maxLimit = 999)) - assertEquals(100, limits.get(relayA.url)?.maxLimit) - assertEquals(999, limits.get(relayB.url)?.maxLimit) - assertEquals(2, limits.snapshot().size) - } + assertEquals(100, limits.get(relayA.url)?.maxLimit) + assertEquals(999, limits.get(relayB.url)?.maxLimit) + assertEquals(2, limits.snapshot().size) + } @Test - fun dropsCachedLimitsOnDisconnect() { - val (limits, listener) = setup() - val relay = FakeRelayClient(NormalizedRelayUrl("wss://relay.example/")) + fun dropsCachedLimitsOnDisconnect() = + kotlinx.coroutines.test.runTest { + val (limits, listener) = setup() + val relay = FakeRelayClient(NormalizedRelayUrl("wss://relay.example/")) - listener.onIncomingMessage(relay, "", LimitsMessage(canRead = true)) - assertTrue(limits.get(relay.url) != null) + listener.onIncomingMessage(relay, "", LimitsMessage(canRead = true)) + assertTrue(limits.get(relay.url) != null) - listener.onDisconnected(relay) - assertNull(limits.get(relay.url), "Limits are connection-scoped and cleared on disconnect") - assertTrue(limits.snapshot().isEmpty()) - } + listener.onDisconnected(relay) + assertNull(limits.get(relay.url), "Limits are connection-scoped and cleared on disconnect") + assertTrue(limits.snapshot().isEmpty()) + } @Test - fun ignoresNonLimitsMessages() { - val (limits, listener) = setup() - val relay = FakeRelayClient(NormalizedRelayUrl("wss://relay.example/")) + fun ignoresNonLimitsMessages() = + kotlinx.coroutines.test.runTest { + val (limits, listener) = setup() + val relay = FakeRelayClient(NormalizedRelayUrl("wss://relay.example/")) - listener.onIncomingMessage(relay, "", EoseMessage("sub1")) + listener.onIncomingMessage(relay, "", EoseMessage("sub1")) - assertNull(limits.get(relay.url)) - assertTrue(limits.snapshot().isEmpty()) - } + assertNull(limits.get(relay.url)) + assertTrue(limits.snapshot().isEmpty()) + } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolEventOutboxAuthTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolEventOutboxAuthTest.kt index f8250eb9a1..8d0fa9b9df 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolEventOutboxAuthTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolEventOutboxAuthTest.kt @@ -62,13 +62,13 @@ class PoolEventOutboxAuthTest { relays.forEach { onSent(it, EventCmd(event)) } } - private fun PoolEventOutbox.nak( + private suspend fun PoolEventOutbox.nak( event: Event, relay: NormalizedRelayUrl, message: String, ) = onIncomingMessage(relay, OkMessage(event.id, false, message)) - private fun PoolEventOutbox.ok( + private suspend fun PoolEventOutbox.ok( event: Event, relay: NormalizedRelayUrl, ) = onIncomingMessage(relay, OkMessage(event.id, true, "")) @@ -104,67 +104,71 @@ class PoolEventOutboxAuthTest { } @Test - fun authRequiredResetsTheTriesBudgetAcrossManyResends() { - val outbox = PoolEventOutbox() - val ev = event("ff".repeat(32)) + fun authRequiredResetsTheTriesBudgetAcrossManyResends() = + kotlinx.coroutines.test.runTest { + val outbox = PoolEventOutbox() + val ev = event("ff".repeat(32)) - outbox.publish(ev, setOf(relay)) // first try + outbox.publish(ev, setOf(relay)) // first try - // A flapping relay / slow AUTH handshake re-pumps the still-pending event many more times - // than the 4-try cap, each NAK'd auth-required. newTry() (the send path) grows `tries` and - // is not auth-aware, so unless auth-required resets the retry budget these sends would trip - // Tries.isDone() and drop the event (with a spurious give-up) before AUTH ever lands. - repeat(8) { i -> - assertNull(outbox.onSent(relay, EventCmd(ev)), "must not give up on re-pump $i") - outbox.nak(ev, relay, "auth-required: authenticate first") + // A flapping relay / slow AUTH handshake re-pumps the still-pending event many more times + // than the 4-try cap, each NAK'd auth-required. newTry() (the send path) grows `tries` and + // is not auth-aware, so unless auth-required resets the retry budget these sends would trip + // Tries.isDone() and drop the event (with a spurious give-up) before AUTH ever lands. + repeat(8) { i -> + assertNull(outbox.onSent(relay, EventCmd(ev)), "must not give up on re-pump $i") + outbox.nak(ev, relay, "auth-required: authenticate first") + } + assertEquals(setOf(relay), outbox.pendingRelaysFor(ev.id)) + + // AUTH finally completes -> the event delivers. + outbox.ok(ev, relay) + assertNull(outbox.pendingRelaysFor(ev.id)) } - assertEquals(setOf(relay), outbox.pendingRelaysFor(ev.id)) - - // AUTH finally completes -> the event delivers. - outbox.ok(ev, relay) - assertNull(outbox.pendingRelaysFor(ev.id)) - } @Test - fun terminalRejectionStillDiscardsImmediately() { - val outbox = PoolEventOutbox() - val ev = event("cc".repeat(32)) + fun terminalRejectionStillDiscardsImmediately() = + kotlinx.coroutines.test.runTest { + val outbox = PoolEventOutbox() + val ev = event("cc".repeat(32)) - outbox.publish(ev, setOf(relay)) - outbox.nak(ev, relay, "invalid: bad signature") + outbox.publish(ev, setOf(relay)) + outbox.nak(ev, relay, "invalid: bad signature") - assertNull(outbox.pendingRelaysFor(ev.id)) - } + assertNull(outbox.pendingRelaysFor(ev.id)) + } @Test - fun givesUpAndSignalsAfterExhaustingTryBudget() { - val outbox = PoolEventOutbox() - val ev = event("ee".repeat(32)) + fun givesUpAndSignalsAfterExhaustingTryBudget() = + kotlinx.coroutines.test.runTest { + val outbox = PoolEventOutbox() + val ev = event("ee".repeat(32)) - // markAsSending + first onSent (1 try). Tries budget is >3 tries. - outbox.publish(ev, setOf(relay)) - // attempts 2, 3 stay under budget and signal nothing. - assertNull(outbox.onSent(relay, EventCmd(ev))) - assertNull(outbox.onSent(relay, EventCmd(ev))) - // the 4th attempt exhausts the budget -> event is returned (gave up) and dropped. - assertEquals(ev.id, outbox.onSent(relay, EventCmd(ev))?.id) - assertNull(outbox.pendingRelaysFor(ev.id)) - } + // markAsSending + first onSent (1 try). Tries budget is >3 tries. + outbox.publish(ev, setOf(relay)) + // attempts 2, 3 stay under budget and signal nothing. + assertNull(outbox.onSent(relay, EventCmd(ev))) + assertNull(outbox.onSent(relay, EventCmd(ev))) + // the 4th attempt exhausts the budget -> event is returned (gave up) and dropped. + assertEquals(ev.id, outbox.onSent(relay, EventCmd(ev))?.id) + assertNull(outbox.pendingRelaysFor(ev.id)) + } @Test - fun ordinaryTransientFailureStillBounded() { - val outbox = PoolEventOutbox() - val ev = event("dd".repeat(32)) + fun ordinaryTransientFailureStillBounded() = + kotlinx.coroutines.test.runTest { + val outbox = PoolEventOutbox() + val ev = event("dd".repeat(32)) - outbox.publish(ev, setOf(relay)) - // 3 non-auth error responses exhaust the retry budget. Responses only - // accumulate here; the drop happens on the next send attempt. - repeat(3) { outbox.nak(ev, relay, "error: rate-limited") } - assertEquals(setOf(relay), outbox.pendingRelaysFor(ev.id)) + outbox.publish(ev, setOf(relay)) + // 3 non-auth error responses exhaust the retry budget. Responses only + // accumulate here; the drop happens on the next send attempt. + repeat(3) { outbox.nak(ev, relay, "error: rate-limited") } + assertEquals(setOf(relay), outbox.pendingRelaysFor(ev.id)) - // The next resend attempt observes the exhausted budget and drops the event - // (unlike auth-required, which never poisons the budget). - outbox.onSent(relay, EventCmd(ev)) - assertNull(outbox.pendingRelaysFor(ev.id)) - } + // The next resend attempt observes the exhausted budget and drops the event + // (unlike auth-required, which never poisons the budget). + outbox.onSent(relay, EventCmd(ev)) + assertNull(outbox.pendingRelaysFor(ev.id)) + } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolRequestsRefusalTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolRequestsRefusalTest.kt index 2e26da853a..91f5c03c4e 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolRequestsRefusalTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolRequestsRefusalTest.kt @@ -75,123 +75,129 @@ class PoolRequestsRefusalTest { return sent } - private fun close( + private suspend fun close( pool: PoolRequests, subId: String, reason: String, ) = pool.onIncomingMessage(FakeRelayClient(relay), ClosedMessage(subId, reason)) @Test - fun stopsReplayingAThriceRefusedFilterAcrossReconnects() { - val pool = PoolRequests(maxRefusalsBeforeSuppress = 3) - pool.addOrUpdate("sub", mapOf(relay to plainFilter()), null) + fun stopsReplayingAThriceRefusedFilterAcrossReconnects() = + kotlinx.coroutines.test.runTest { + val pool = PoolRequests(maxRefusalsBeforeSuppress = 3) + pool.addOrUpdate("sub", mapOf(relay to plainFilter()), null) - // Under the threshold, each reconnect still replays the REQ (giving the relay a chance). - repeat(3) { attempt -> - val sent = reconnectAndSync(pool) - assertEquals(1, sent.filterIsInstance().size, "reconnect #$attempt should replay the REQ") - close(pool, "sub", "unsupported: too many filters") + // Under the threshold, each reconnect still replays the REQ (giving the relay a chance). + repeat(3) { attempt -> + val sent = reconnectAndSync(pool) + assertEquals(1, sent.filterIsInstance().size, "reconnect #$attempt should replay the REQ") + close(pool, "sub", "unsupported: too many filters") + } + + // Once the same filter has been refused [maxRefusalsBeforeSuppress] times, stop replaying it. + val suppressed = reconnectAndSync(pool) + assertTrue(suppressed.filterIsInstance().isEmpty(), "a thrice-refused filter must not be replayed again") } - // Once the same filter has been refused [maxRefusalsBeforeSuppress] times, stop replaying it. - val suppressed = reconnectAndSync(pool) - assertTrue(suppressed.filterIsInstance().isEmpty(), "a thrice-refused filter must not be replayed again") - } - @Test - fun aMeaningfulFilterChangeReEnablesTheReq() { - val pool = PoolRequests(maxRefusalsBeforeSuppress = 2) - pool.addOrUpdate("sub", mapOf(relay to plainFilter(1)), null) + fun aMeaningfulFilterChangeReEnablesTheReq() = + kotlinx.coroutines.test.runTest { + val pool = PoolRequests(maxRefusalsBeforeSuppress = 2) + pool.addOrUpdate("sub", mapOf(relay to plainFilter(1)), null) - repeat(2) { - reconnectAndSync(pool) - close(pool, "sub", "unsupported: too many filters") - } - assertTrue(reconnectAndSync(pool).filterIsInstance().isEmpty(), "refused filter is suppressed") + repeat(2) { + reconnectAndSync(pool) + close(pool, "sub", "unsupported: too many filters") + } + assertTrue(reconnectAndSync(pool).filterIsInstance().isEmpty(), "refused filter is suppressed") - // The app changes the subscription's filter (different kind) — the relay may now accept it. - pool.addOrUpdate("sub", mapOf(relay to plainFilter(30023)), null) - assertEquals(1, reconnectAndSync(pool).filterIsInstance().size, "a changed filter must be tried again") - } - - @Test - fun aSearchOnlyRelayStopsReceivingPlainReqsFromEveryNewSubOnOneConnection() { - // The search.nos.today case: 6 different subscriptions, one connection, each a - // distinct plain feed filter the relay CLOSES with `error: search filter is required`. - // Per-filter memory can't help (the filters differ); the relay-wide block must. - val pool = PoolRequests(relayRefusals = RelayReqRefusals(threshold = 2)) - - val sent = mutableListOf>() // subId -> did a REQ go out - - fun mountSub( - subId: String, - filter: List, - ) { - val affected = pool.addOrUpdate(subId, mapOf(relay to filter), null) - var reqSent = false - pool.sendToRelayIfChanged(subId, affected) { _, cmd -> if (cmd is ReqCmd) reqSent = true } - sent.add(subId to reqSent) - close(pool, subId, "error: search filter is required") + // The app changes the subscription's filter (different kind) — the relay may now accept it. + pool.addOrUpdate("sub", mapOf(relay to plainFilter(30023)), null) + assertEquals(1, reconnectAndSync(pool).filterIsInstance().size, "a changed filter must be tried again") } - mountSub("sub1", plainFilter(1)) - mountSub("sub2", plainFilter(2)) - mountSub("sub3", plainFilter(3)) - mountSub("sub4", plainFilter(4)) - - assertTrue(sent[0].second && sent[1].second, "the first two plain subs are sent (learning the relay is search-only)") - assertTrue(!sent[2].second && !sent[3].second, "after two refusals, further plain subs are not sent to a search-only relay") - } - @Test - fun aCapabilityBlockedRelayIsDroppedFromDesiredRelays() { - // The socket-closing half: once a relay is capability-blocked and no desired sub can - // use it, it leaves the desired-relay set so the pool disconnects it. - val pool = PoolRequests(relayRefusals = RelayReqRefusals(threshold = 2)) - pool.addOrUpdate("sub", mapOf(relay to plainFilter()), null) - assertTrue(relay in pool.desiredRelays.value, "the relay is wanted before it refuses anything") + fun aSearchOnlyRelayStopsReceivingPlainReqsFromEveryNewSubOnOneConnection() = + kotlinx.coroutines.test.runTest { + // The search.nos.today case: 6 different subscriptions, one connection, each a + // distinct plain feed filter the relay CLOSES with `error: search filter is required`. + // Per-filter memory can't help (the filters differ); the relay-wide block must. + val pool = PoolRequests(relayRefusals = RelayReqRefusals(threshold = 2)) - close(pool, "sub", "error: search filter is required") - assertTrue(relay in pool.desiredRelays.value, "one refusal doesn't drop it yet") + val sent = mutableListOf>() // subId -> did a REQ go out - close(pool, "sub", "error: search filter is required") - assertTrue(relay !in pool.desiredRelays.value, "a search-only relay with only plain subs is dropped (socket closes)") - } + suspend fun mountSub( + subId: String, + filter: List, + ) { + val affected = pool.addOrUpdate(subId, mapOf(relay to filter), null) + var reqSent = false + pool.sendToRelayIfChanged(subId, affected) { _, cmd -> if (cmd is ReqCmd) reqSent = true } + sent.add(subId to reqSent) + close(pool, subId, "error: search filter is required") + } - @Test - fun aSearchOnlyRelayStaysWantedWhileASearchSubNeedsIt() { - val pool = PoolRequests(relayRefusals = RelayReqRefusals(threshold = 2)) - pool.addOrUpdate("plain", mapOf(relay to plainFilter()), null) - pool.addOrUpdate("search", mapOf(relay to listOf(Filter(kinds = listOf(1), search = "nostr"))), null) + mountSub("sub1", plainFilter(1)) + mountSub("sub2", plainFilter(2)) + mountSub("sub3", plainFilter(3)) + mountSub("sub4", plainFilter(4)) - close(pool, "plain", "error: search filter is required") - close(pool, "plain", "error: search filter is required") - - assertTrue(relay in pool.desiredRelays.value, "the relay stays wanted: a search sub still has a usable filter for it") - } - - @Test - fun authRequiredAndRateLimitedAreNeverSuppressed() { - val pool = PoolRequests(maxRefusalsBeforeSuppress = 2) - pool.addOrUpdate("sub", mapOf(relay to plainFilter()), null) - - repeat(4) { - reconnectAndSync(pool) - close(pool, "sub", MachineReadablePrefix.AUTH_REQUIRED.format("authenticate first")) + assertTrue(sent[0].second && sent[1].second, "the first two plain subs are sent (learning the relay is search-only)") + assertTrue(!sent[2].second && !sent[3].second, "after two refusals, further plain subs are not sent to a search-only relay") } - assertEquals(1, reconnectAndSync(pool).filterIsInstance().size, "auth-required must keep replaying (auth resolves it)") - val pool2 = PoolRequests(maxRefusalsBeforeSuppress = 2) - pool2.addOrUpdate("sub", mapOf(relay to plainFilter()), null) - repeat(4) { + @Test + fun aCapabilityBlockedRelayIsDroppedFromDesiredRelays() = + kotlinx.coroutines.test.runTest { + // The socket-closing half: once a relay is capability-blocked and no desired sub can + // use it, it leaves the desired-relay set so the pool disconnects it. + val pool = PoolRequests(relayRefusals = RelayReqRefusals(threshold = 2)) + pool.addOrUpdate("sub", mapOf(relay to plainFilter()), null) + assertTrue(relay in pool.desiredRelays.value, "the relay is wanted before it refuses anything") + + close(pool, "sub", "error: search filter is required") + assertTrue(relay in pool.desiredRelays.value, "one refusal doesn't drop it yet") + + close(pool, "sub", "error: search filter is required") + assertTrue(relay !in pool.desiredRelays.value, "a search-only relay with only plain subs is dropped (socket closes)") + } + + @Test + fun aSearchOnlyRelayStaysWantedWhileASearchSubNeedsIt() = + kotlinx.coroutines.test.runTest { + val pool = PoolRequests(relayRefusals = RelayReqRefusals(threshold = 2)) + pool.addOrUpdate("plain", mapOf(relay to plainFilter()), null) + pool.addOrUpdate("search", mapOf(relay to listOf(Filter(kinds = listOf(1), search = "nostr"))), null) + + close(pool, "plain", "error: search filter is required") + close(pool, "plain", "error: search filter is required") + + assertTrue(relay in pool.desiredRelays.value, "the relay stays wanted: a search sub still has a usable filter for it") + } + + @Test + fun authRequiredAndRateLimitedAreNeverSuppressed() = + kotlinx.coroutines.test.runTest { + val pool = PoolRequests(maxRefusalsBeforeSuppress = 2) + pool.addOrUpdate("sub", mapOf(relay to plainFilter()), null) + + repeat(4) { + reconnectAndSync(pool) + close(pool, "sub", MachineReadablePrefix.AUTH_REQUIRED.format("authenticate first")) + } + assertEquals(1, reconnectAndSync(pool).filterIsInstance().size, "auth-required must keep replaying (auth resolves it)") + + val pool2 = PoolRequests(maxRefusalsBeforeSuppress = 2) + pool2.addOrUpdate("sub", mapOf(relay to plainFilter()), null) + repeat(4) { + pool2.onConnecting(relay) + val sent = mutableListOf() + pool2.syncState(relay) { sent.add(it) } + pool2.onIncomingMessage(FakeRelayClient(relay), ClosedMessage("sub", MachineReadablePrefix.RATE_LIMITED.format("slow down"))) + } pool2.onConnecting(relay) val sent = mutableListOf() pool2.syncState(relay) { sent.add(it) } - pool2.onIncomingMessage(FakeRelayClient(relay), ClosedMessage("sub", MachineReadablePrefix.RATE_LIMITED.format("slow down"))) + assertEquals(1, sent.filterIsInstance().size, "rate-limited must keep replaying (the limiter spaces it out)") } - pool2.onConnecting(relay) - val sent = mutableListOf() - pool2.syncState(relay) { sent.add(it) } - assertEquals(1, sent.filterIsInstance().size, "rate-limited must keep replaying (the limiter spaces it out)") - } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/inprocess/InProcessWebSocketTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/inprocess/InProcessWebSocketTest.kt index 2b9cda14cd..4874586ebd 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/inprocess/InProcessWebSocketTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/inprocess/InProcessWebSocketTest.kt @@ -74,7 +74,7 @@ class InProcessWebSocketTest { callbacks.trySend("open") } - override fun onMessage(text: String) { + override suspend fun onMessage(text: String) { callbacks.trySend("message") } @@ -132,7 +132,7 @@ class InProcessWebSocketTest { ) { } - override fun onMessage(text: String) { + override suspend fun onMessage(text: String) { // Answer the AUTH challenge immediately, the way // RelayAuthenticator does. The socket must be fully // wired by the time any server frame is delivered, diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip46RemoteSigner/server/NostrConnectSignerServiceTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip46RemoteSigner/server/NostrConnectSignerServiceTest.kt index 12320fcfc5..3dcf86afd1 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip46RemoteSigner/server/NostrConnectSignerServiceTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip46RemoteSigner/server/NostrConnectSignerServiceTest.kt @@ -141,7 +141,7 @@ class NostrConnectSignerServiceTest { published.add(event) } - fun deliver(event: Event) { + suspend fun deliver(event: Event) { listener?.onEvent(event, isLive = true, relay = RelayUrlNormalizer.normalizeOrNull("wss://relay.example.com")!!, forFilters = null) } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayObserverTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayObserverTest.kt index 6522dd5c64..efa6af497a 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayObserverTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayObserverTest.kt @@ -73,256 +73,276 @@ class RelayObserverTest { // ---- what we measured --------------------------------------------------- @Test - fun `rtt-open is the transport handshake rather than our own queueing`() { - // pingMillis is receivedResponseAtMillis - sentRequestAtMillis: it starts - // when the upgrade request goes out, so it excludes time the call spent - // queued in the client's dispatcher. Timing the enqueue instead published - // our own backlog as the relay's latency — a median of 33.5 SECONDS on a - // 16,507-relay fan-out, against a true minimum of 140ms — into the field - // aggregators rank relays by. - val o = RelayObserver() - o.onConnected(client(url), 140, false) - assertEquals(140L, o.only().rttOpenMs) - } + fun `rtt-open is the transport handshake rather than our own queueing`() = + kotlinx.coroutines.test.runTest { + // pingMillis is receivedResponseAtMillis - sentRequestAtMillis: it starts + // when the upgrade request goes out, so it excludes time the call spent + // queued in the client's dispatcher. Timing the enqueue instead published + // our own backlog as the relay's latency — a median of 33.5 SECONDS on a + // 16,507-relay fan-out, against a true minimum of 140ms — into the field + // aggregators rank relays by. + val o = RelayObserver() + o.onConnected(client(url), 140, false) + assertEquals(140L, o.only().rttOpenMs) + } @Test - fun `a handshake the transport could not time publishes no time`() { - val o = RelayObserver() - o.onConnected(client(url), 0, false) + fun `a handshake the transport could not time publishes no time`() = + kotlinx.coroutines.test.runTest { + val o = RelayObserver() + o.onConnected(client(url), 0, false) - val obs = o.only() - assertTrue(obs.reachable, "it opened, and that much is known") - assertNull(obs.rttOpenMs, "unmeasurable is not zero") - } + val obs = o.only() + assertTrue(obs.reachable, "it opened, and that much is known") + assertNull(obs.rttOpenMs, "unmeasurable is not zero") + } @Test - fun `an opened connection is timed rather than assumed`() { - val o = RelayObserver() - o.onConnecting(client(url)) - o.onConnected(client(url), 1, true) + fun `an opened connection is timed rather than assumed`() = + kotlinx.coroutines.test.runTest { + val o = RelayObserver() + o.onConnecting(client(url)) + o.onConnected(client(url), 1, true) - val obs = o.only() - assertTrue(obs.reachable) - assertNotNull(obs.rttOpenMs, "rtt-open must be measured — aggregators rank on it") - assertNull(obs.error) - } + val obs = o.only() + assertTrue(obs.reachable) + assertNotNull(obs.rttOpenMs, "rtt-open must be measured — aggregators rank on it") + assertNull(obs.error) + } @Test - fun `the read clock runs from the first REQ to the first EOSE`() { - val o = RelayObserver() - o.onConnecting(client(url)) - o.onConnected(client(url), 1, true) - o.onSent(client(url), "", ReqCmd("sub", emptyList()), true) - o.onIncomingMessage(client(url), "", EoseMessage("sub")) + fun `the read clock runs from the first REQ to the first EOSE`() = + kotlinx.coroutines.test.runTest { + val o = RelayObserver() + o.onConnecting(client(url)) + o.onConnected(client(url), 1, true) + o.onSent(client(url), "", ReqCmd("sub", emptyList()), true) + o.onIncomingMessage(client(url), "", EoseMessage("sub")) - assertNotNull(o.only().rttReadMs) - } + assertNotNull(o.only().rttReadMs) + } @Test - fun `the write clock runs from the first EVENT to its OK`() { - val o = RelayObserver() - o.onConnecting(client(url)) - o.onConnected(client(url), 1, true) - o.onIncomingMessage(client(url), "", OkMessage("id", true, "")) + fun `the write clock runs from the first EVENT to its OK`() = + kotlinx.coroutines.test.runTest { + val o = RelayObserver() + o.onConnecting(client(url)) + o.onConnected(client(url), 1, true) + o.onIncomingMessage(client(url), "", OkMessage("id", true, "")) - assertNull(o.collectUnreported().single().rttWriteMs, "an OK with nothing sent behind it times nothing") - } + assertNull(o.collectUnreported().single().rttWriteMs, "an OK with nothing sent behind it times nothing") + } @Test - fun `a non-REQ command does not start the read clock`() { - val o = RelayObserver() - o.onConnecting(client(url)) - o.onConnected(client(url), 1, true) - o.onSent(client(url), "", CloseCmd("sub"), true) - o.onIncomingMessage(client(url), "", EoseMessage("sub")) + fun `a non-REQ command does not start the read clock`() = + kotlinx.coroutines.test.runTest { + val o = RelayObserver() + o.onConnecting(client(url)) + o.onConnected(client(url), 1, true) + o.onSent(client(url), "", CloseCmd("sub"), true) + o.onIncomingMessage(client(url), "", EoseMessage("sub")) - assertNull(o.only().rttReadMs) - } + assertNull(o.only().rttReadMs) + } // ---- what we refuse to claim -------------------------------------------- @Test - fun `a connection that never opened records the reason and no latency`() { - val o = RelayObserver() - o.onConnecting(client(url)) - o.onCannotConnect(client(url), "Expected HTTP 101 response but was '503 Service Unavailable'") + fun `a connection that never opened records the reason and no latency`() = + kotlinx.coroutines.test.runTest { + val o = RelayObserver() + o.onConnecting(client(url)) + o.onCannotConnect(client(url), "Expected HTTP 101 response but was '503 Service Unavailable'") - val obs = o.only() - assertFalse(obs.reachable) - assertNull(obs.rttOpenMs, "nothing opened, so there is nothing to time") - assertTrue(obs.error!!.contains("503")) - } + val obs = o.only() + assertFalse(obs.reachable) + assertNull(obs.rttOpenMs, "nothing opened, so there is nothing to time") + assertTrue(obs.error!!.contains("503")) + } @Test - fun `a relay that answered stays answered through a later failure`() { - // A relay that worked a minute ago and blipped now is not the same thing - // as one that never answered, and only the writer decides which record - // that becomes. A single failure must not erase the success under it. - val o = RelayObserver() - o.onConnecting(client(url)) - o.onConnected(client(url), 1, true) - o.onCannotConnect(client(url), "connection reset") + fun `a relay that answered stays answered through a later failure`() = + kotlinx.coroutines.test.runTest { + // A relay that worked a minute ago and blipped now is not the same thing + // as one that never answered, and only the writer decides which record + // that becomes. A single failure must not erase the success under it. + val o = RelayObserver() + o.onConnecting(client(url)) + o.onConnected(client(url), 1, true) + o.onCannotConnect(client(url), "connection reset") - assertTrue(o.only().reachable, "one bad minute must not bury a relay that answered") - } + assertTrue(o.only().reachable, "one bad minute must not bury a relay that answered") + } @Test - fun `a reconnect clears the previous attempt's error`() { - val o = RelayObserver() - o.onConnecting(client(url)) - o.onCannotConnect(client(url), "timeout") - o.onConnecting(client(url)) + fun `a reconnect clears the previous attempt's error`() = + kotlinx.coroutines.test.runTest { + val o = RelayObserver() + o.onConnecting(client(url)) + o.onCannotConnect(client(url), "timeout") + o.onConnecting(client(url)) - assertNull(o.only().error, "a stale error would report a live relay as broken forever") - } + assertNull(o.only().error, "a stale error would report a live relay as broken forever") + } // ---- AUTH, which is why an anonymous crawl finds a relay empty ------------ @Test - fun `a demand for AUTH is recorded from either shape`() { - val challenged = RelayObserver() - challenged.onIncomingMessage(client(url), "", AuthMessage("challenge")) - assertTrue(challenged.only().authRequired) + fun `a demand for AUTH is recorded from either shape`() = + kotlinx.coroutines.test.runTest { + val challenged = RelayObserver() + challenged.onIncomingMessage(client(url), "", AuthMessage("challenge")) + assertTrue(challenged.only().authRequired) - val closed = RelayObserver() - closed.onIncomingMessage(client(url), "", ClosedMessage("sub", "auth-required: subscribers only")) - val obs = closed.only() - assertTrue(obs.authRequired) - assertEquals("auth-required", obs.closedReason) - } + val closed = RelayObserver() + closed.onIncomingMessage(client(url), "", ClosedMessage("sub", "auth-required: subscribers only")) + val obs = closed.only() + assertTrue(obs.authRequired) + assertEquals("auth-required", obs.closedReason) + } @Test - fun `a CLOSED that is not about auth is categorised rather than misread`() { - val o = RelayObserver() - o.onIncomingMessage(client(url), "", ClosedMessage("sub", "rate-limited: slow down")) + fun `a CLOSED that is not about auth is categorised rather than misread`() = + kotlinx.coroutines.test.runTest { + val o = RelayObserver() + o.onIncomingMessage(client(url), "", ClosedMessage("sub", "rate-limited: slow down")) - val obs = o.only() - assertEquals("rate-limited", obs.closedReason) - assertFalse(obs.authRequired, "only an auth refusal means auth is required") - } + val obs = o.only() + assertEquals("rate-limited", obs.closedReason) + assertFalse(obs.authRequired, "only an auth refusal means auth is required") + } // ---- publishing bookkeeping --------------------------------------------- @Test - fun `an unchanged relay is not re-reported but its measurement survives`() { - // Re-writing a record refreshes its freshness window, so a relay nobody - // re-measured must be left out. But the measurement itself has to stay: - // a long-lived socket fires onConnected once, and if publishing erased - // it, the relays we know best would be the ones we could never describe - // again. - val o = RelayObserver() - o.onConnecting(client(url)) - o.onConnected(client(url), 1, true) + fun `an unchanged relay is not re-reported but its measurement survives`() = + kotlinx.coroutines.test.runTest { + // Re-writing a record refreshes its freshness window, so a relay nobody + // re-measured must be left out. But the measurement itself has to stay: + // a long-lived socket fires onConnected once, and if publishing erased + // it, the relays we know best would be the ones we could never describe + // again. + val o = RelayObserver() + o.onConnecting(client(url)) + o.onConnected(client(url), 1, true) - val first = o.collectUnreported().single() - assertNotNull(first.rttOpenMs) - assertEquals(0, o.collectUnreported().size, "nothing new to say") + val first = o.collectUnreported().single() + assertNotNull(first.rttOpenMs) + assertEquals(0, o.collectUnreported().size, "nothing new to say") - o.onIncomingMessage(client(url), "", NoticeMessage("slow down")) - val second = o.collectUnreported().single() - assertEquals(first.rttOpenMs, second.rttOpenMs, "the last real measurement still stands") - } + o.onIncomingMessage(client(url), "", NoticeMessage("slow down")) + val second = o.collectUnreported().single() + assertEquals(first.rttOpenMs, second.rttOpenMs, "the last real measurement still stands") + } // ---- findings from outside the websocket client ------------------------ @Test - fun `a probe failure is published even though nothing was dialled`() { - // The cheap checks that decide NOT to open a websocket are exactly the - // ones that learn a relay is gone. Without a way in, a listener-only - // observer reports on the small minority it happened to connect to — - // 104 records out of a 16,507-relay list — which is not a census. - val o = RelayObserver() - o.record(url, reachable = false, error = "nodename nor servname provided") + fun `a probe failure is published even though nothing was dialled`() = + kotlinx.coroutines.test.runTest { + // The cheap checks that decide NOT to open a websocket are exactly the + // ones that learn a relay is gone. Without a way in, a listener-only + // observer reports on the small minority it happened to connect to — + // 104 records out of a 16,507-relay list — which is not a census. + val o = RelayObserver() + o.record(url, reachable = false, error = "nodename nor servname provided") - val obs = o.only() - assertFalse(obs.reachable) - assertEquals("nodename nor servname provided", obs.error) - assertNull(obs.rttOpenMs, "a failed probe times nothing") - } + val obs = o.only() + assertFalse(obs.reachable) + assertEquals("nodename nor servname provided", obs.error) + assertNull(obs.rttOpenMs, "a failed probe times nothing") + } @Test - fun `a probe that connected reports its measured time or none at all`() { - val timed = RelayObserver() - timed.record(url, reachable = true, rttOpenMs = 42) - assertEquals(42L, timed.only().rttOpenMs) + fun `a probe that connected reports its measured time or none at all`() = + kotlinx.coroutines.test.runTest { + val timed = RelayObserver() + timed.record(url, reachable = true, rttOpenMs = 42) + assertEquals(42L, timed.only().rttOpenMs) - val untimed = RelayObserver() - untimed.record(url, reachable = true) - val obs = untimed.only() - assertTrue(obs.reachable) - assertNull(obs.rttOpenMs, "reachable without a timing must not invent one") - } + val untimed = RelayObserver() + untimed.record(url, reachable = true) + val obs = untimed.only() + assertTrue(obs.reachable) + assertNull(obs.rttOpenMs, "reachable without a timing must not invent one") + } @Test - fun `a failed probe does not demote a relay that already answered`() { - // Same rule the connection path follows: one bad probe is not death, and - // only the writer decides what record a mixed history becomes. - val o = RelayObserver() - o.onConnecting(client(url)) - o.onConnected(client(url), 1, true) - o.record(url, reachable = false, error = "connect timeout") + fun `a failed probe does not demote a relay that already answered`() = + kotlinx.coroutines.test.runTest { + // Same rule the connection path follows: one bad probe is not death, and + // only the writer decides what record a mixed history becomes. + val o = RelayObserver() + o.onConnecting(client(url)) + o.onConnected(client(url), 1, true) + o.record(url, reachable = false, error = "connect timeout") - assertTrue(o.only().reachable, "it answered; a later probe failure does not erase that") - } + assertTrue(o.only().reachable, "it answered; a later probe failure does not erase that") + } @Test - fun `an out-of-band finding is reported once like any other`() { - val o = RelayObserver() - o.record(url, reachable = false, error = "refused") - assertEquals(1, o.collectUnreported().size) - assertEquals(0, o.collectUnreported().size, "nothing new to say") - } + fun `an out-of-band finding is reported once like any other`() = + kotlinx.coroutines.test.runTest { + val o = RelayObserver() + o.record(url, reachable = false, error = "refused") + assertEquals(1, o.collectUnreported().size) + assertEquals(0, o.collectUnreported().size, "nothing new to say") + } @Test - fun `each relay is observed on its own`() { - val o = RelayObserver() - o.onConnecting(client(url)) - o.onConnected(client(url), 1, true) - o.onConnecting(client(other)) - o.onCannotConnect(client(other), "nodename nor servname provided") + fun `each relay is observed on its own`() = + kotlinx.coroutines.test.runTest { + val o = RelayObserver() + o.onConnecting(client(url)) + o.onConnected(client(url), 1, true) + o.onConnecting(client(other)) + o.onCannotConnect(client(other), "nodename nor servname provided") - val byUrl = o.collectUnreported().associateBy { it.url } - assertTrue(byUrl.getValue(url).reachable) - assertFalse(byUrl.getValue(other).reachable) - } + val byUrl = o.collectUnreported().associateBy { it.url } + assertTrue(byUrl.getValue(url).reachable) + assertFalse(byUrl.getValue(other).reachable) + } // ---- the run-level summary (what RelayDiagnostics used to give) ----------- @Test - fun `the summary tallies feedback across every relay`() { - val o = RelayObserver() - assertFalse(o.hadFeedback()) + fun `the summary tallies feedback across every relay`() = + kotlinx.coroutines.test.runTest { + val o = RelayObserver() + assertFalse(o.hadFeedback()) - o.onIncomingMessage(client(url), "", AuthMessage("c1")) - o.onIncomingMessage(client(other), "", AuthMessage("c2")) - o.onIncomingMessage(client(url), "", ClosedMessage("s", "rate-limited: slow")) - o.onIncomingMessage(client(other), "", ClosedMessage("s", "rate-limited: slow")) - o.onIncomingMessage(client(url), "", NoticeMessage("too many REQs")) + o.onIncomingMessage(client(url), "", AuthMessage("c1")) + o.onIncomingMessage(client(other), "", AuthMessage("c2")) + o.onIncomingMessage(client(url), "", ClosedMessage("s", "rate-limited: slow")) + o.onIncomingMessage(client(other), "", ClosedMessage("s", "rate-limited: slow")) + o.onIncomingMessage(client(url), "", NoticeMessage("too many REQs")) - assertTrue(o.hadFeedback()) - val s = o.summary() - assertEquals(2L, s["auth_challenges"]) - assertEquals(2, s["auth_required_relays"]) - assertEquals(mapOf("rate-limited" to 2L), s["closed_by_reason"]) - assertEquals(1L, s["notices"]) - } + assertTrue(o.hadFeedback()) + val s = o.summary() + assertEquals(2L, s["auth_challenges"]) + assertEquals(2, s["auth_required_relays"]) + assertEquals(mapOf("rate-limited" to 2L), s["closed_by_reason"]) + assertEquals(1L, s["notices"]) + } @Test - fun `the summary outlives publishing`() { - // It answers "how did this run go", which must not be reset by the - // unrelated act of writing records out. - val o = RelayObserver() - o.onIncomingMessage(client(url), "", AuthMessage("c")) - o.collectUnreported() + fun `the summary outlives publishing`() = + kotlinx.coroutines.test.runTest { + // It answers "how did this run go", which must not be reset by the + // unrelated act of writing records out. + val o = RelayObserver() + o.onIncomingMessage(client(url), "", AuthMessage("c")) + o.collectUnreported() - assertTrue(o.hadFeedback(), "a flush must not erase the run's tally") - assertEquals(1L, o.summary()["auth_challenges"]) - } + assertTrue(o.hadFeedback(), "a flush must not erase the run's tally") + assertEquals(1L, o.summary()["auth_challenges"]) + } @Test - fun `a machine-readable prefix is extracted or falls back to other`() { - assertEquals("auth-required", RelayObserver.prefixOf("auth-required: come back signed")) - assertEquals("other", RelayObserver.prefixOf("just some prose")) - assertEquals("other", RelayObserver.prefixOf("")) - } + fun `a machine-readable prefix is extracted or falls back to other`() = + kotlinx.coroutines.test.runTest { + assertEquals("auth-required", RelayObserver.prefixOf("auth-required: come back signed")) + assertEquals("other", RelayObserver.prefixOf("just some prose")) + assertEquals("other", RelayObserver.prefixOf("")) + } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt index 8593489ab2..1cfdfcea00 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt @@ -85,7 +85,7 @@ class RelayProberFlowTest { } /** Plays a relay's OK answer for the published event to every armed listener. */ - fun answerOk( + suspend fun answerOk( relay: NormalizedRelayUrl, success: Boolean, message: String, @@ -235,13 +235,14 @@ class RelayProberFlowTest { } @Test - fun writeTestEventIsEphemeralAndSelfExpiring() { - val template = RelayProbeWriteTest.build(createdAt = 5000) + fun writeTestEventIsEphemeralAndSelfExpiring() = + kotlinx.coroutines.test.runTest { + val template = RelayProbeWriteTest.build(createdAt = 5000) - assertEquals(20166, template.kind) - assertTrue(template.kind in 20000..29999, "the write probe must be an ephemeral kind") - assertTrue(listOf("expiration", "5060") in template.tags.map { it.toList() }) - } + assertEquals(20166, template.kind) + assertTrue(template.kind in 20000..29999, "the write probe must be an ephemeral kind") + assertTrue(listOf("expiration", "5060") in template.tags.map { it.toList() }) + } // ------------------------------------------------------------------ // readWriteCheck — honest read + write measurements, nothing claimed @@ -358,108 +359,117 @@ class RelayProberFlowTest { private fun tagsOf(template: EventTemplate<*>) = template.tags.map { it.toList() } @Test - fun reachableVerdictTemplateCarriesLivenessAndNetwork() { - val template = - RelayProber - .Verdict(fast, reachable = true, rttOpenMs = 150, rttEoseMs = 480, error = null) - .toDiscoveryEventTemplate(createdAt = 1000) + fun reachableVerdictTemplateCarriesLivenessAndNetwork() = + kotlinx.coroutines.test.runTest { + val template = + RelayProber + .Verdict(fast, reachable = true, rttOpenMs = 150, rttEoseMs = 480, error = null) + .toDiscoveryEventTemplate(createdAt = 1000) - val tags = tagsOf(template) - assertEquals(30166, template.kind) - assertEquals(1000, template.createdAt) - assertTrue(listOf("d", fast.url) in tags) - assertTrue(listOf("n", "clearnet") in tags) - assertTrue(listOf("rtt-open", "150") in tags) - // rtt-eose is wave-relative (dial + queue + read) — never published as rtt-read. - assertNull(tags.firstOrNull { it[0] == "rtt-read" }) - } + val tags = tagsOf(template) + assertEquals(30166, template.kind) + assertEquals(1000, template.createdAt) + assertTrue(listOf("d", fast.url) in tags) + assertTrue(listOf("n", "clearnet") in tags) + assertTrue(listOf("rtt-open", "150") in tags) + // rtt-eose is wave-relative (dial + queue + read) — never published as rtt-read. + assertNull(tags.firstOrNull { it[0] == "rtt-read" }) + } @Test - fun deadVerdictTemplateHasNoRttOpen() { - val template = - RelayProber - .Verdict(silent, reachable = false, rttOpenMs = -1, rttEoseMs = -1, error = "cannot:timeout") - .toDiscoveryEventTemplate() + fun deadVerdictTemplateHasNoRttOpen() = + kotlinx.coroutines.test.runTest { + val template = + RelayProber + .Verdict(silent, reachable = false, rttOpenMs = -1, rttEoseMs = -1, error = "cannot:timeout") + .toDiscoveryEventTemplate() - val tags = tagsOf(template) - assertTrue(listOf("d", silent.url) in tags) - // Liveness is the PRESENCE of rtt-open; a dead record must not carry one. - assertNull(tags.firstOrNull { it[0] == "rtt-open" }) - } + val tags = tagsOf(template) + assertTrue(listOf("d", silent.url) in tags) + // Liveness is the PRESENCE of rtt-open; a dead record must not carry one. + assertNull(tags.firstOrNull { it[0] == "rtt-open" }) + } @Test - fun reachableWithoutMeasuredLatencyWritesZeroFlag() { - val template = - RelayProber - .Verdict(fast, reachable = true, rttOpenMs = -1, rttEoseMs = 300, error = null) - .toDiscoveryEventTemplate() + fun reachableWithoutMeasuredLatencyWritesZeroFlag() = + kotlinx.coroutines.test.runTest { + val template = + RelayProber + .Verdict(fast, reachable = true, rttOpenMs = -1, rttEoseMs = 300, error = null) + .toDiscoveryEventTemplate() - // 0 = "reachable, latency not observed": the flag form, never an invented number. - assertTrue(listOf("rtt-open", "0") in tagsOf(template)) - } + // 0 = "reachable, latency not observed": the flag form, never an invented number. + assertTrue(listOf("rtt-open", "0") in tagsOf(template)) + } @Test - fun observedAuthWallBecomesARequirementTag() { - val template = - RelayProber - .Verdict(walled, reachable = true, rttOpenMs = 90, rttEoseMs = -1, error = "closed:auth-required: sign in") - .toDiscoveryEventTemplate() + fun observedAuthWallBecomesARequirementTag() = + kotlinx.coroutines.test.runTest { + val template = + RelayProber + .Verdict(walled, reachable = true, rttOpenMs = 90, rttEoseMs = -1, error = "closed:auth-required: sign in") + .toDiscoveryEventTemplate() - assertTrue(listOf("R", "auth") in tagsOf(template)) - } + assertTrue(listOf("R", "auth") in tagsOf(template)) + } @Test - fun policyClosedIsNotAnAuthRequirement() { - val template = - RelayProber - .Verdict(walled, reachable = true, rttOpenMs = 90, rttEoseMs = -1, error = "closed:blocked: not welcome") - .toDiscoveryEventTemplate() + fun policyClosedIsNotAnAuthRequirement() = + kotlinx.coroutines.test.runTest { + val template = + RelayProber + .Verdict(walled, reachable = true, rttOpenMs = 90, rttEoseMs = -1, error = "closed:blocked: not welcome") + .toDiscoveryEventTemplate() - assertNull(tagsOf(template).firstOrNull { it[0] == "R" }) - } + assertNull(tagsOf(template).firstOrNull { it[0] == "R" }) + } @Test - fun readWriteResultsBecomeRttTags() { - val verdict = RelayProber.Verdict(fast, reachable = true, rttOpenMs = 100, rttEoseMs = 300, error = null) - val readWrite = RelayProber.ReadWriteVerdict(fast, rttReadMs = 40, rttWriteMs = 55, writeAccepted = true, writeMessage = "") + fun readWriteResultsBecomeRttTags() = + kotlinx.coroutines.test.runTest { + val verdict = RelayProber.Verdict(fast, reachable = true, rttOpenMs = 100, rttEoseMs = 300, error = null) + val readWrite = RelayProber.ReadWriteVerdict(fast, rttReadMs = 40, rttWriteMs = 55, writeAccepted = true, writeMessage = "") - val tags = tagsOf(verdict.toDiscoveryEventTemplate(readWrite = readWrite)) - assertTrue(listOf("rtt-read", "40") in tags) - assertTrue(listOf("rtt-write", "55") in tags) - } + val tags = tagsOf(verdict.toDiscoveryEventTemplate(readWrite = readWrite)) + assertTrue(listOf("rtt-read", "40") in tags) + assertTrue(listOf("rtt-write", "55") in tags) + } @Test - fun unobservedReadWriteSidesStayUntagged() { - val verdict = RelayProber.Verdict(fast, reachable = true, rttOpenMs = 100, rttEoseMs = -1, error = null) - val readWrite = RelayProber.ReadWriteVerdict(fast, rttReadMs = -1, rttWriteMs = -1, writeAccepted = null, writeMessage = null) + fun unobservedReadWriteSidesStayUntagged() = + kotlinx.coroutines.test.runTest { + val verdict = RelayProber.Verdict(fast, reachable = true, rttOpenMs = 100, rttEoseMs = -1, error = null) + val readWrite = RelayProber.ReadWriteVerdict(fast, rttReadMs = -1, rttWriteMs = -1, writeAccepted = null, writeMessage = null) - val tags = tagsOf(verdict.toDiscoveryEventTemplate(readWrite = readWrite)) - assertNull(tags.firstOrNull { it[0] == "rtt-read" }) - assertNull(tags.firstOrNull { it[0] == "rtt-write" }) - } + val tags = tagsOf(verdict.toDiscoveryEventTemplate(readWrite = readWrite)) + assertNull(tags.firstOrNull { it[0] == "rtt-read" }) + assertNull(tags.firstOrNull { it[0] == "rtt-write" }) + } @Test - fun writeRejectionReasonsBecomeRequirementTags() { - val verdict = RelayProber.Verdict(walled, reachable = true, rttOpenMs = 100, rttEoseMs = -1, error = null) + fun writeRejectionReasonsBecomeRequirementTags() = + kotlinx.coroutines.test.runTest { + val verdict = RelayProber.Verdict(walled, reachable = true, rttOpenMs = 100, rttEoseMs = -1, error = null) - val pow = RelayProber.ReadWriteVerdict(walled, -1, 30, writeAccepted = false, writeMessage = "pow: 28 bits needed") - assertTrue(listOf("R", "pow") in tagsOf(verdict.toDiscoveryEventTemplate(readWrite = pow))) + val pow = RelayProber.ReadWriteVerdict(walled, -1, 30, writeAccepted = false, writeMessage = "pow: 28 bits needed") + assertTrue(listOf("R", "pow") in tagsOf(verdict.toDiscoveryEventTemplate(readWrite = pow))) - val auth = RelayProber.ReadWriteVerdict(walled, -1, 30, writeAccepted = false, writeMessage = "auth-required: sign in") - assertTrue(listOf("R", "auth") in tagsOf(verdict.toDiscoveryEventTemplate(readWrite = auth))) + val auth = RelayProber.ReadWriteVerdict(walled, -1, 30, writeAccepted = false, writeMessage = "auth-required: sign in") + assertTrue(listOf("R", "auth") in tagsOf(verdict.toDiscoveryEventTemplate(readWrite = auth))) - val blocked = RelayProber.ReadWriteVerdict(walled, -1, 30, writeAccepted = false, writeMessage = "blocked: not welcome") - assertNull(tagsOf(verdict.toDiscoveryEventTemplate(readWrite = blocked)).firstOrNull { it[0] == "R" }) - } + val blocked = RelayProber.ReadWriteVerdict(walled, -1, 30, writeAccepted = false, writeMessage = "blocked: not welcome") + assertNull(tagsOf(verdict.toDiscoveryEventTemplate(readWrite = blocked)).firstOrNull { it[0] == "R" }) + } @Test - fun onionRelayIsTaggedTor() { - val onion = RelayUrlNormalizer.normalize("ws://someonionaddressabcdefghijklmnop.onion") - val template = - RelayProber - .Verdict(onion, reachable = true, rttOpenMs = 900, rttEoseMs = -1, error = null) - .toDiscoveryEventTemplate() + fun onionRelayIsTaggedTor() = + kotlinx.coroutines.test.runTest { + val onion = RelayUrlNormalizer.normalize("ws://someonionaddressabcdefghijklmnop.onion") + val template = + RelayProber + .Verdict(onion, reachable = true, rttOpenMs = 900, rttEoseMs = -1, error = null) + .toDiscoveryEventTemplate() - assertTrue(listOf("n", "tor") in tagsOf(template)) - } + assertTrue(listOf("n", "tor") in tagsOf(template)) + } } diff --git a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientManualSubTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientManualSubTest.kt index 06354f98d2..50dfac2519 100644 --- a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientManualSubTest.kt +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientManualSubTest.kt @@ -47,7 +47,7 @@ class NostrClientManualSubTest : RelayClientTest() { val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientRepeatSubTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientRepeatSubTest.kt index 0c283d4314..7531b70068 100644 --- a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientRepeatSubTest.kt +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientRepeatSubTest.kt @@ -71,7 +71,7 @@ class NostrClientRepeatSubTest : RelayClientTest() { val listener = object : RelayConnectionListener { - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/PoolRequestsConcurrencyTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/PoolRequestsConcurrencyTest.kt index 6d21d526e3..4add1aea15 100644 --- a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/PoolRequestsConcurrencyTest.kt +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/PoolRequestsConcurrencyTest.kt @@ -78,69 +78,70 @@ class PoolRequestsConcurrencyTest { } @Test - fun concurrentEoseResendAndSubscribeSendExactlyOneReq() { - val url = RelayUrlNormalizer.normalize("ws://race/") - val subId = "shared-sub" - val filtersA = listOf(Filter(kinds = listOf(1))) - val filtersB = listOf(Filter(kinds = listOf(2))) - val listener = object : SubscriptionListener {} + fun concurrentEoseResendAndSubscribeSendExactlyOneReq() = + kotlinx.coroutines.test.runTest { + val url = RelayUrlNormalizer.normalize("ws://race/") + val subId = "shared-sub" + val filtersA = listOf(Filter(kinds = listOf(1))) + val filtersB = listOf(Filter(kinds = listOf(2))) + val listener = object : SubscriptionListener {} - // Many episodes so a regression that only sometimes doubles still trips. - repeat(300) { episode -> - val pool = PoolRequests() - val reqBCount = AtomicInteger(0) + // Many episodes so a regression that only sometimes doubles still trips. + repeat(300) { episode -> + val pool = PoolRequests() + val reqBCount = AtomicInteger(0) - fun countReqB(cmd: Command) { - if (cmd is ReqCmd && cmd.filters == filtersB) reqBCount.incrementAndGet() - } - - val fakeRelay = - FakeRelay(url) { cmd -> - // relay-reader auto-resend send path - countReqB(cmd) - pool.onSent(url, cmd) + fun countReqB(cmd: Command) { + if (cmd is ReqCmd && cmd.filters == filtersB) reqBCount.incrementAndGet() } - // Bring the sub to LIVE with filters A. - val setupRelays = pool.addOrUpdate(subId, mapOf(url to filtersA), listener) - pool.sendToRelayIfChanged(subId, setupRelays) { _, cmd -> pool.onSent(url, cmd) } - pool.onIncomingMessage(fakeRelay, EoseMessage(subId)) - - // The desired filters change to B (e.g. the next page of a paged download). - pool.addOrUpdate(subId, mapOf(url to filtersB), listener) - - val appProducedReq = CountDownLatch(1) - val readerDone = CountDownLatch(1) - - val appThread = - thread { - pool.sendToRelayIfChanged(subId, setOf(url)) { _, cmd -> + val fakeRelay = + FakeRelay(url) { cmd -> + // relay-reader auto-resend send path countReqB(cmd) - // App has produced its REQ(B); park before onSent so the - // subscription state is not yet advanced — the exact window - // the race needs. - appProducedReq.countDown() - readerDone.await() pool.onSent(url, cmd) } - } - val readerThread = - thread { - appProducedReq.await() - pool.onIncomingMessage(fakeRelay, EoseMessage(subId)) - readerDone.countDown() - } + // Bring the sub to LIVE with filters A. + val setupRelays = pool.addOrUpdate(subId, mapOf(url to filtersA), listener) + pool.sendToRelayIfChanged(subId, setupRelays) { _, cmd -> pool.onSent(url, cmd) } + pool.onIncomingMessage(fakeRelay, EoseMessage(subId)) - appThread.join() - readerThread.join() + // The desired filters change to B (e.g. the next page of a paged download). + pool.addOrUpdate(subId, mapOf(url to filtersB), listener) - assertEquals( - 1, - reqBCount.get(), - "episode $episode: exactly one REQ must be sent for the changed filters, " + - "never a duplicate from the app + reader race", - ) + val appProducedReq = CountDownLatch(1) + val readerDone = CountDownLatch(1) + + val appThread = + thread { + pool.sendToRelayIfChanged(subId, setOf(url)) { _, cmd -> + countReqB(cmd) + // App has produced its REQ(B); park before onSent so the + // subscription state is not yet advanced — the exact window + // the race needs. + appProducedReq.countDown() + readerDone.await() + pool.onSent(url, cmd) + } + } + + val readerThread = + thread { + appProducedReq.await() + kotlinx.coroutines.runBlocking { pool.onIncomingMessage(fakeRelay, EoseMessage(subId)) } + readerDone.countDown() + } + + appThread.join() + readerThread.join() + + assertEquals( + 1, + reqBCount.get(), + "episode $episode: exactly one REQ must be sent for the changed filters, " + + "never a duplicate from the app + reader race", + ) + } } - } } diff --git a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/auth/RelayAuthenticatorReauthOnClosedTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/auth/RelayAuthenticatorReauthOnClosedTest.kt index e10a5a7c32..e8ae39933b 100644 --- a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/auth/RelayAuthenticatorReauthOnClosedTest.kt +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/auth/RelayAuthenticatorReauthOnClosedTest.kt @@ -100,7 +100,7 @@ class RelayAuthenticatorReauthOnClosedTest { .filterIsInstance() .last() .event - listener.onIncomingMessage(relay, "", OkMessage.accepted(newest.id)) + kotlinx.coroutines.runBlocking { listener.onIncomingMessage(relay, "", OkMessage.accepted(newest.id)) } } @Test diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/NegentropyRejectionFallbackTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/NegentropyRejectionFallbackTest.kt index 230674d4d0..006b8ffe16 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/NegentropyRejectionFallbackTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/NegentropyRejectionFallbackTest.kt @@ -91,11 +91,11 @@ class NegentropyRejectionFallbackTest { when { // The keep-alive REQ and any paging REQ: answer EOSE so the // subscription settles (paging then completes with 0 events). - msg.startsWith("[\"REQ\"") -> subIdOf(msg)?.let { out.onMessage("[\"EOSE\",\"$it\"]") } + msg.startsWith("[\"REQ\"") -> subIdOf(msg)?.let { kotlinx.coroutines.runBlocking { out.onMessage("[\"EOSE\",\"$it\"]") } } // The negentropy handshake: the relay refuses. msg.startsWith("[\"NEG-OPEN\"") -> { negOpens.incrementAndGet() - subIdOf(msg)?.let { out.onMessage(replyToNegOpen(it)) } + subIdOf(msg)?.let { kotlinx.coroutines.runBlocking { out.onMessage(replyToNegOpen(it)) } } } else -> Unit } @@ -140,25 +140,28 @@ class NegentropyRejectionFallbackTest { } @Test - fun strfryNegentropyDisabledFallsBackToPaging() { - negOpenRejectedBy { "[\"NOTICE\",\"ERROR: bad msg: negentropy disabled\"]" } - } + fun strfryNegentropyDisabledFallsBackToPaging() = + kotlinx.coroutines.test.runTest { + negOpenRejectedBy { "[\"NOTICE\",\"ERROR: bad msg: negentropy disabled\"]" } + } @Test - fun purplePagesUnknownEnvelopeFallsBackToPaging() { - negOpenRejectedBy { "[\"NOTICE\",\"failed to parse envelope: unknown envelope label\"]" } - } + fun purplePagesUnknownEnvelopeFallsBackToPaging() = + kotlinx.coroutines.test.runTest { + negOpenRejectedBy { "[\"NOTICE\",\"failed to parse envelope: unknown envelope label\"]" } + } @Test - fun rateLimitNegErrPagesWithoutSplitStorm() { - // A NEG-ERR that does NOT shrink with the window ("too many requests") must not - // be mistaken for a set-too-large overflow: doing so would binary-split the - // created_at range forever. Assert we page after exactly ONE NEG-OPEN. - val relay = negOpenRejectedBy { subId -> "[\"NEG-ERR\",\"$subId\",\"rate-limited: too many requests\"]" } - assertEquals( - 2, - relay.negOpens.get(), - "one NEG-OPEN per phase (sync + syncOrFetch), i.e. no window-split storm; got ${relay.negOpens.get()}", - ) - } + fun rateLimitNegErrPagesWithoutSplitStorm() = + kotlinx.coroutines.test.runTest { + // A NEG-ERR that does NOT shrink with the window ("too many requests") must not + // be mistaken for a set-too-large overflow: doing so would binary-split the + // created_at range forever. Assert we page after exactly ONE NEG-OPEN. + val relay = negOpenRejectedBy { subId -> "[\"NEG-ERR\",\"$subId\",\"rate-limited: too many requests\"]" } + assertEquals( + 2, + relay.negOpens.get(), + "one NEG-OPEN per phase (sync + syncOrFetch), i.e. no window-split storm; got ${relay.negOpens.get()}", + ) + } } diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/ByIdFetchBenchmark.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/ByIdFetchBenchmark.kt index 6dd57131e0..9775a20308 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/ByIdFetchBenchmark.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/ByIdFetchBenchmark.kt @@ -136,7 +136,7 @@ class ByIdFetchBenchmark { val done = Channel(Channel.CONFLATED) val listener = object : SubscriptionListener { - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/DispatchStageBenchmark.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/DispatchStageBenchmark.kt index 8a64c4a0e5..ed0909257e 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/DispatchStageBenchmark.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/DispatchStageBenchmark.kt @@ -197,7 +197,7 @@ class DispatchStageBenchmark { continue } for (subId in subIds) { - client.onIncomingMessage(relayClient, "", EventMessage(subId, event)) + kotlinx.coroutines.runBlocking { client.onIncomingMessage(relayClient, "", EventMessage(subId, event)) } } } } @@ -259,7 +259,7 @@ class DispatchStageBenchmark { Thread { for (event in events) { for (subId in subIds) { - pool.onIncomingMessage(relayClient, EventMessage(subId, event)) + kotlinx.coroutines.runBlocking { pool.onIncomingMessage(relayClient, EventMessage(subId, event)) } } } } @@ -299,12 +299,13 @@ class DispatchStageBenchmark { } @Test - fun dispatchStageBenchmark() { - println("=== DISPATCH STAGE BENCHMARK (post-parse, pre-verify) ===") - println("cores=${Runtime.getRuntime().availableProcessors()} uniqueEvents=$UNIQUE_EVENTS subsPerRelay=$SUBS_PER_RELAY") + fun dispatchStageBenchmark() = + kotlinx.coroutines.test.runTest { + println("=== DISPATCH STAGE BENCHMARK (post-parse, pre-verify) ===") + println("cores=${Runtime.getRuntime().availableProcessors()} uniqueEvents=$UNIQUE_EVENTS subsPerRelay=$SUBS_PER_RELAY") - // warmup pass (JIT), then the measured pass - runAllVariants(print = false) - runAllVariants(print = true) - } + // warmup pass (JIT), then the measured pass + runAllVariants(print = false) + runAllVariants(print = true) + } } diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/NegentropyMultiRelayLiveTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/NegentropyMultiRelayLiveTest.kt index 1c72b53690..8d95420c6b 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/NegentropyMultiRelayLiveTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/NegentropyMultiRelayLiveTest.kt @@ -177,7 +177,7 @@ class NegentropyMultiRelayLiveTest { val listener = object : RelayConnectionListener { - override fun onIncomingMessage( + override suspend fun onIncomingMessage( relay: IRelayClient, msgStr: String, msg: Message, diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/NegentropyStallRepro.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/NegentropyStallRepro.kt index 04d6338582..c85e4fb80a 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/NegentropyStallRepro.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/NegentropyStallRepro.kt @@ -109,7 +109,7 @@ class NegentropyStallRepro { out.onOpen(pingMillis, usingCompression) } - override fun onMessage(text: String) { + override suspend fun onMessage(text: String) { val t = tagger(text) recvCounts.getOrPut(t) { AtomicInteger() }.incrementAndGet() if (t == "NEG-MSG") negMsgBytesIn.addAndGet(text.length.toLong()) diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/ProductionReceiverBenchmark.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/ProductionReceiverBenchmark.kt index b226af0641..eca181e106 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/ProductionReceiverBenchmark.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/ProductionReceiverBenchmark.kt @@ -370,7 +370,7 @@ class ProductionReceiverBenchmark { reqSentAt.putIfAbsent(relay, System.nanoTime() - startNanos) } - override fun onEvent( + override suspend fun onEvent( event: Event, isLive: Boolean, relay: NormalizedRelayUrl, From bb95cad98b96591e45c249e56db1d1a4c05a78b7 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Wed, 5 Aug 2026 17:09:45 +0000 Subject: [PATCH 038/132] Audit fixes: one clock per record, no shared mutable list, pin the file format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deep-audit pass over the branch. Nothing here changes what a band claims; these are the defects that pass tests and bite later. - record() read the clock twice PER KIND. A 40-kind map took 80 readings, and worse, a span's floor and ceiling were judged against two different instants — so a span could be accepted at one end and rejected at the other on a clock tick. One read, one instant, for the whole call. The aggregate path had the same double read and now shares it. - legs() handed the SAME MutableList instance to every Filter in a group, publishing its accumulator through a public return value. Filters are treated as immutable everywhere else; this keeps that true by construction rather than by nobody having tried yet. - The state file's round trip was asserted only for the fields, never for the behaviour. Three tests now pin it: per-kind spans survive a restart AND still narrow per kind afterwards; the ALL_KINDS sentinel survives its negative key through toString/toInt; and a pre-split file (min/max, no spans) loads as the claim it always was. Plus the rollback contract — `min`/`max` must remain the OUTER edges, since a binary from before per-kind spans reads those and would otherwise skip ground it has not covered. Checked and found sound, recorded so the next reader need not re-derive it: ConcurrentMap.snapshot() copies, so export() cannot be mutated under a writer; Band is immutable (widen() copies its map), so a shared Band across threads is safe; merge() keeps old.fullAt, preserving the re-walk clock across widening; and coveringWindow does NOT regress — a paged band gave >1 leg before this change too, and a reconciled band still collapses to one leg and narrows the shared snapshot. Co-Authored-By: Claude Opus 5 --- .../geode/mirror/SyncCoverageFileTest.kt | 132 ++++++++++++++++++ .../relay/client/accessories/SyncCoverage.kt | 15 +- 2 files changed, 144 insertions(+), 3 deletions(-) diff --git a/geode/src/test/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFileTest.kt b/geode/src/test/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFileTest.kt index 4a02507db8..5b6f0387d9 100644 --- a/geode/src/test/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFileTest.kt +++ b/geode/src/test/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFileTest.kt @@ -20,8 +20,17 @@ */ package com.vitorpamplona.geode.mirror +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.SyncCoverage import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer +import com.vitorpamplona.quartz.utils.TimeUtils +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.long +import kotlinx.serialization.json.put import java.io.File import kotlin.test.Test import kotlin.test.assertEquals @@ -127,4 +136,127 @@ class SyncCoverageFileTest { assertEquals(1_500L, clamped.since, "the band's ceiling wins over the window floor") assertEquals(2_000L, clamped.until) } + + @Test + fun `per-kind spans survive a restart, including the kindless sentinel`() { + // The file is the one place a per-kind band can be silently flattened + // back into the single interval it replaced, so the round trip is + // pinned rather than assumed — negative sentinel key included, since + // ALL_KINDS goes through toString()/toInt() like any other kind. + val mixed = Filter(kinds = listOf(0, 30382)) + val anyKind = Filter(authors = listOf("a".repeat(64))) + val f = tempFile() + SyncCoverageFile(f).use { + it.coverage.record( + relay, + mixed, + null, + null, + paged = true, + observedByKind = + mapOf( + 0 to SyncCoverage.Span(1_600_000_000L, 1_700_000_000L), + 30382 to SyncCoverage.Span(1_690_000_000L, 1_700_000_000L), + ), + ) + it.coverage.record( + relay, + anyKind, + null, + null, + paged = true, + observedByKind = mapOf(1 to SyncCoverage.Span(1_690_000_000L, 1_700_000_000L)), + ) + } + + SyncCoverageFile(f).use { reopened -> + val band = reopened.coverage.band(relay, mixed)!! + assertEquals( + mapOf( + 0 to SyncCoverage.Span(1_600_000_000L, 1_700_000_000L), + 30382 to SyncCoverage.Span(1_690_000_000L, 1_700_000_000L), + ), + band.spans, + "each kind keeps its own evidence across the restart", + ) + // …and the restored band still narrows per kind, which is the half + // that would go unnoticed if only the fields round-tripped. + val legs = reopened.coverage.legs(relay, mixed) + assertEquals(4, legs.size, "the two kinds want different windows") + + assertEquals( + mapOf(SyncCoverage.ALL_KINDS to SyncCoverage.Span(1_690_000_000L, 1_700_000_000L)), + reopened.coverage.band(relay, anyKind)!!.spans, + "the kindless sentinel survives its negative key", + ) + } + } + + @Test + fun `a file written before per-kind spans loads as the claim it always was`() { + // Only min/max, no `spans` — what every deployed state file holds today. + // Discarding it would re-download each upstream's corpus once on + // upgrade, so it loads under ALL_KINDS and narrows every kind exactly + // as it did before, until the first per-kind walk replaces it. + val mixed = Filter(kinds = listOf(0, 30382)) + val f = tempFile() + val key = "${relay.url} ${mixed.toJson()}" + f.writeText( + Json.encodeToString( + JsonObject.serializer(), + buildJsonObject { + put( + key, + buildJsonObject { + put("min", 1_690_000_000L) + put("max", 1_700_000_000L) + put("complete", false) + put("fullAt", TimeUtils.now()) + }, + ) + }, + ), + ) + + SyncCoverageFile(f).use { reopened -> + val band = reopened.coverage.band(relay, mixed)!! + assertEquals(mapOf(SyncCoverage.ALL_KINDS to SyncCoverage.Span(1_690_000_000L, 1_700_000_000L)), band.spans) + val legs = reopened.coverage.legs(relay, mixed) + assertEquals(2, legs.size, "one shared pair of legs — the old behaviour, exactly") + assertEquals(listOf(0, 30382), legs[0].kinds) + } + } + + @Test + fun `a rolled-back reader still finds the outer edges it understands`() { + // A binary from before per-kind spans reads `min`/`max` and ignores + // `spans`. Those fields must therefore still be written, and must be + // the OUTER edges — anything narrower would make the old reader skip + // ground it has not covered. + val mixed = Filter(kinds = listOf(0, 30382)) + val f = tempFile() + SyncCoverageFile(f).use { + it.coverage.record( + relay, + mixed, + null, + null, + paged = true, + observedByKind = + mapOf( + 0 to SyncCoverage.Span(1_600_000_000L, 1_695_000_000L), + 30382 to SyncCoverage.Span(1_690_000_000L, 1_700_000_000L), + ), + ) + } + + val written = + Json + .parseToJsonElement(f.readText()) + .jsonObject.values + .single() + .jsonObject + assertEquals(1_600_000_000L, written.getValue("min").jsonPrimitive.long, "the oldest of any kind") + assertEquals(1_700_000_000L, written.getValue("max").jsonPrimitive.long, "the newest of any kind") + } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt index 74bcddbb3d..3eb54ae9fd 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt @@ -171,7 +171,12 @@ class SyncCoverage( byWindows.getOrPut(windows(filter, span, band.complete, floor)) { mutableListOf() }.add(kind) } return byWindows.flatMap { (windows, group) -> - windows.map { (since, until) -> filter.copy(kinds = group, since = since, until = until) } + // toList(): `group` is the mutable accumulator above, and handing + // the same instance to every Filter in the group would publish it + // through a public return value. Filters are treated as immutable + // everywhere else; this keeps that true by construction. + val kindsForGroup = group.toList() + windows.map { (since, until) -> filter.copy(kinds = kindsForGroup, since = since, until = until) } } } @@ -244,11 +249,15 @@ class SyncCoverage( } if (!paged) return + // Read ONCE. `now` is a clock call, and this was invoking it twice per + // entry — so a 40-kind map took 80 readings, and worse, a span's floor + // and ceiling were judged against two different instants. + val at = now() if (observedByKind != null) { // Guarded per span for the same reason the aggregate is below. val plausible = observedByKind.filterValues { - isPlausible(it.min, now()) && isPlausible(it.max, now()) + isPlausible(it.min, at) && isPlausible(it.max, at) } if (plausible.isEmpty()) return val named = filter.kinds @@ -302,7 +311,7 @@ class SyncCoverage( // claim the whole timeline, and the leg outside it would ask for a // range nothing can be in, forever. if (observedMin == null || observedMax == null) return - if (!isPlausible(observedMin, now()) || !isPlausible(observedMax, now())) return + if (!isPlausible(observedMin, at) || !isPlausible(observedMax, at)) return put(url, filter, kinds.associateWith { Span(observedMin, observedMax) }, complete = false) } From 8b4220019afbde5938a56010846b8bdbf3d65b68 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 17:40:03 +0000 Subject: [PATCH 039/132] fix(ime): close the two remaining stuck-padding paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two exits still popped a composer while the IME was mid-animation — the race that strands imePadding() at keyboard height app-wide. 1. Top-bar X and Post. KeyboardAwareBackHandler only guards the back gesture; ActionTopBar wired both buttons straight to nav.popBack(), with nothing dismissing the keyboard first. Tapping either while typing reproduced the original bug exactly. The earlier fix leaned on the back arrow as the "always-available exit" without noticing it was also a race source. 2. A ~250ms hole in the back gate. It read the animated WindowInsets.ime, which stays above zero for the whole close animation — a window in which the IME had already stopped consuming back but the handler was still disabled, so a second back fell through to the NavController and popped without ever running onBack. That silently dropped the draft the handler exists to save: nothing else saves it, onCleared() only closes the writing assistant and there is no autosave. Both are the same underlying requirement — serialize the IME and window animations instead of overlapping them — so both now route through one helper, rememberAfterKeyboardCloses(): keyboard down, the action runs inline and nothing changes; keyboard up, clear focus, hide, wait for the inset to actually reach zero, then act. The wait is bounded so a stale inset (the very failure being guarded) can never trap the user on screen, and re-entrant calls are dropped since the deferral widens the window for a double-tap on Post to fire twice. The back gate now reads WindowInsets.imeAnimationTarget, which flips to zero the moment the hide begins, so back keeps reaching onBack throughout the animation. Re-enabling that early means onBack can fire mid-animation, which is exactly what the helper absorbs. Not covered: this is verified by compile and the unit suite only. The race reproduces on release builds on a device, which this environment cannot run. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LfUMGWYu2uTSyh17JJonfN --- .../ui/navigation/bottombars/KeyboardState.kt | 106 +++++++++++++++--- .../ui/navigation/topbars/ActionTopBar.kt | 9 +- 2 files changed, 98 insertions(+), 17 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/KeyboardState.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/KeyboardState.kt index 1457927832..f576a3556c 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/KeyboardState.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/KeyboardState.kt @@ -21,14 +21,24 @@ package com.vitorpamplona.amethyst.ui.navigation.bottombars import androidx.activity.compose.BackHandler +import androidx.compose.foundation.layout.ExperimentalLayoutApi import androidx.compose.foundation.layout.WindowInsets import androidx.compose.foundation.layout.ime +import androidx.compose.foundation.layout.imeAnimationTarget import androidx.compose.runtime.Composable import androidx.compose.runtime.State import androidx.compose.runtime.derivedStateOf import androidx.compose.runtime.getValue import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.runtime.snapshotFlow import androidx.compose.ui.platform.LocalDensity +import androidx.compose.ui.platform.LocalFocusManager +import androidx.compose.ui.platform.LocalSoftwareKeyboardController +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch +import kotlinx.coroutines.withTimeoutOrNull +import java.util.concurrent.atomic.AtomicBoolean enum class KeyboardState { Opened, @@ -59,28 +69,94 @@ fun keyboardAsState(): State { } } +/** How long to wait for the IME inset to reach zero before running the action anyway. */ +private const val IME_SETTLE_TIMEOUT_MS = 700L + /** - * A [BackHandler] that steps aside while the soft keyboard is on screen. + * Returns a runner that defers an action until the soft keyboard is fully off screen. * - * Chat composers (and draft-saving editors) intercept back to flush a draft and pop the screen. - * When that pop happens while the keyboard is still up, it races the predictive-back window - * animation against the IME's close animation. On release builds — fast enough that the window - * animation wins — the IME [WindowInsetsAnimationCompat][androidx.core.view.WindowInsetsAnimationCompat] - * is cancelled before its terminal (zero) frame reaches Compose, so the shared `WindowInsets.ime` - * holder stays "animating" and every `Modifier.imePadding()` in the app freezes at the keyboard - * height until a later inset pass rebalances it (the "stuck IME padding" that survives leaving the - * screen). + * Popping a screen while the keyboard is still up races the window animation against the IME's + * close animation. On release builds — fast enough that the window animation wins — the IME + * [WindowInsetsAnimationCompat][androidx.core.view.WindowInsetsAnimationCompat] is cancelled before + * its terminal (zero) frame reaches Compose, so the shared `WindowInsets.ime` holder stays + * "animating" and every `Modifier.imePadding()` in the app freezes at the keyboard height until a + * later inset pass rebalances it (the "stuck IME padding" that survives leaving the screen). * - * Gating on [keyboardAsState] fixes it: while the keyboard is visible we do NOT consume back, so the - * system dismisses the keyboard first with its own animation (which completes cleanly). The next - * back — keyboard already down — runs [onBack] as before. The top bar's back arrow stays an - * always-available exit, so this can never trap the user even if the inset reading were itself stale. + * Any exit that leaves a keyboard-bearing screen has to serialize the two animations rather than + * overlap them. With the keyboard already down the action runs inline — same frame, no behavior + * change. With it up we dismiss the keyboard, wait for the inset to actually reach zero, and only + * then act, so the IME animation always completes before the window animation begins. + * + * Re-entrant calls while an action is pending are dropped: the deferral widens the window in which + * a second tap on a Post/Save button would fire the action twice. + * + * [IME_SETTLE_TIMEOUT_MS] bounds the wait — if the inset never reports zero (precisely the failure + * this guards against) the action still runs, so a stale reading can never trap the user on screen. */ @Composable +fun rememberAfterKeyboardCloses(): (() -> Unit) -> Unit { + val density = LocalDensity.current + val imeInsets = WindowInsets.ime + val keyboard = LocalSoftwareKeyboardController.current + val focusManager = LocalFocusManager.current + val scope = rememberCoroutineScope() + val pending = remember { AtomicBoolean(false) } + + return remember(density, imeInsets, keyboard, focusManager, scope, pending) { + { action: () -> Unit -> + if (imeInsets.getBottom(density) <= 0) { + action() + } else if (pending.compareAndSet(false, true)) { + // Clear focus first so nothing re-requests the IME as it retracts. + focusManager.clearFocus(true) + keyboard?.hide() + scope.launch { + try { + withTimeoutOrNull(IME_SETTLE_TIMEOUT_MS) { + snapshotFlow { imeInsets.getBottom(density) }.first { it <= 0 } + } + action() + } finally { + pending.set(false) + } + } + } + } + } +} + +/** + * A [BackHandler] that lets the system dismiss the soft keyboard before it consumes back. + * + * Chat composers (and draft-saving editors) intercept back to flush a draft and pop the screen, + * which is the pop-during-IME-animation race described on [rememberAfterKeyboardCloses]. While the + * keyboard is up we do NOT consume back, so the system dismisses it first with its own animation + * (which completes cleanly, and on recent Android follows the back gesture). The next back runs + * [onBack] as before. + * + * The gate reads [WindowInsets.imeAnimationTarget] — where the IME is *heading* — not the animated + * [WindowInsets.ime]. Gating on the animated value left a hole: it stays above zero for the whole + * close animation, ~250ms in which the IME has already stopped consuming back but this handler was + * still disabled, so a second back fell through to the NavController and popped the screen without + * ever running [onBack] — silently dropping the draft it exists to save. The target flips to zero + * the moment the hide begins, so back keeps reaching [onBack] throughout. + * + * Re-enabling that early means [onBack] can now fire mid-animation, so it is routed through + * [rememberAfterKeyboardCloses] to wait for the inset to settle before popping. + */ +@OptIn(ExperimentalLayoutApi::class) +@Composable fun KeyboardAwareBackHandler( enabled: Boolean = true, onBack: () -> Unit, ) { - val keyboardState by keyboardAsState() - BackHandler(enabled = enabled && keyboardState == KeyboardState.Closed, onBack = onBack) + val density = LocalDensity.current + val imeTarget = WindowInsets.imeAnimationTarget + val afterKeyboardCloses = rememberAfterKeyboardCloses() + + val keyboardIsStaying by remember(density, imeTarget) { + derivedStateOf { imeTarget.getBottom(density) > 0 } + } + + BackHandler(enabled = enabled && !keyboardIsStaying) { afterKeyboardCloses(onBack) } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/topbars/ActionTopBar.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/topbars/ActionTopBar.kt index 84b19c084a..4bd0aba287 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/topbars/ActionTopBar.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/topbars/ActionTopBar.kt @@ -31,6 +31,7 @@ import androidx.compose.ui.Modifier import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.text.style.TextOverflow import com.vitorpamplona.amethyst.R +import com.vitorpamplona.amethyst.ui.navigation.bottombars.rememberAfterKeyboardCloses import com.vitorpamplona.amethyst.ui.note.buttons.CloseButton import com.vitorpamplona.amethyst.ui.stringRes import com.vitorpamplona.amethyst.ui.theme.HalfHorzPadding @@ -45,6 +46,10 @@ fun ActionTopBar( onPost: () -> Unit, additionalActions: @Composable (() -> Unit)? = null, ) { + // Both exits pop the screen, and on a composer the keyboard is up while typing — the same + // pop-during-IME-animation race the back gesture avoids, just reached by a tap instead. + val afterKeyboardCloses = rememberAfterKeyboardCloses() + ShorterTopAppBar( title = { if (titleRes != null) { @@ -60,7 +65,7 @@ fun ActionTopBar( navigationIcon = { CloseButton( modifier = HalfHorzPadding, - onPress = onCancel, + onPress = { afterKeyboardCloses(onCancel) }, ) }, actions = { @@ -70,7 +75,7 @@ fun ActionTopBar( Button( modifier = HalfHorzPadding, enabled = isActive(), - onClick = onPost, + onClick = { afterKeyboardCloses(onPost) }, ) { Text(text = stringRes(postRes)) } From d9ea3ef86adf0e2be5f2c8bf169461ce8563309e Mon Sep 17 00:00:00 2001 From: vitorpamplona <532031+vitorpamplona@users.noreply.github.com> Date: Wed, 5 Aug 2026 18:33:02 +0000 Subject: [PATCH 040/132] chore: sync Crowdin translations and seed translator npub placeholders --- .../src/main/res/values-hi-rIN/strings.xml | 1 + .../src/main/res/values-nl-rNL/strings.xml | 87 +++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/amethyst/src/main/res/values-hi-rIN/strings.xml b/amethyst/src/main/res/values-hi-rIN/strings.xml index 59eea919c8..077e171144 100644 --- a/amethyst/src/main/res/values-hi-rIN/strings.xml +++ b/amethyst/src/main/res/values-hi-rIN/strings.xml @@ -203,6 +203,7 @@ सफल संयोजनों का प्रतिशत पुनःप्रसारक के साथ ढूँढें तथा प्रयेक्ता जोडें पुनःप्रसारक जोडें + मान्य पुनःप्रसारक पता नहीं। एक जालावास नाम का उपयोग करें। अथवा कोष्ठकों में एक अंकीय जालपता उदाहरण [201:d0e:9ba5:8bbc::1]:8080 के जैसे। मेरा @सूचक नाम प्रदर्शन नाम मेरा प्रदर्शन नाम diff --git a/amethyst/src/main/res/values-nl-rNL/strings.xml b/amethyst/src/main/res/values-nl-rNL/strings.xml index 0006542c2a..4519e8c254 100644 --- a/amethyst/src/main/res/values-nl-rNL/strings.xml +++ b/amethyst/src/main/res/values-nl-rNL/strings.xml @@ -203,6 +203,7 @@ Percentage succesvolle verbindingen met deze relay Zoek en voeg gebruiker toe Relay toevoegen + Geen geldig relay-adres. Gebruik een hostnaam, of een IP-adres tussen blokhaken (bijvoorbeeld [201:d0e:9ba5:8bbc::1]:8080). Mijn @naam Weergavenaam Mijn weergavenaam @@ -1929,14 +1930,95 @@ + + %1$s \u00b7 %2$d relay + %1$s \u00b7 %2$d relays + + Bladeren + Media + Hashtags + Onderwerpen + Gesprek + Zoeken + Ontbrekende events zoeken + Events observeren + Haalt events op via hun id waar iets op je scherm naar verwijst maar die je nog niet hebt — een quote, de note waarop een reactie antwoordt, de start van een discussie. + Houdt de events die nu in beeld staan in de gaten voor nieuwe antwoorden, reacties, reposts, zaps en rapportages, zodat de tellers bijwerken terwijl je leest. + Add-ons + Relay-info + Overig + Relaylijsten zoeken + Profielen observeren + Accountgegevens + Startfeed + Relay-groepen + + %1$d groep + %1$d groepen + + Vluchtige chats + Locatiechats + Livestream-chat + NIP-29-groepen waar je lid van bent. Elke groep staat op één host-relay, dus de app verbindt met elke relay die een groep van jou host. + Chatruimtes die geen geschiedenis bewaren — berichten bestaan alleen zolang je verbonden bent, dus deze blijven geabonneerd om überhaupt iets te ontvangen. + Locatiegebonden ruimtes voor de gebieden die je volgt, opgevraagd bij de relays die ze aanbieden. + Chat en zap-doelen die horen bij livestreams die je open hebt staan of volgt. + DM-inbox + Wallet + Nutzap-inbox + Mintoverzicht + Wallet Connect + Community-chats + Community-feeds + Je inbox-relays, plus een kleine wisselende steekproef van de relays waarop de mensen die je volgt plaatsen, voor het geval een vermelding ergens anders is afgeleverd. + Je DM-inbox-relays, waar gift-wrapped berichten worden afgeleverd. + De thuisrelay van elke chat die je open hebt staan of waar je lid van bent. + De relays waarop elke community zijn planes publiceert. + Groepsberichten en sleutelpakketten, op de relays van elke groep. + De relays van de ruimte, zolang die open is. + Je eigen profiel, instellingen en concepten, op je eigen relays. + Profielen van de mensen die nu in beeld zijn. + Zoekt uit naar welke relays iemand publiceert, zodat hun posts van de juiste plek kunnen worden opgehaald. + Volglijsten, gebruikt om je feed en je web-of-trust op te bouwen. + Rapportages die de mensen die je volgt schreven over de profielen die nu in beeld zijn, opgevraagd bij elke relay waarop die mensen plaatsen. + Rapportages van wie je volgt + Je eigen wallet-events, teruggelezen van de relays waarop je ze hebt gepubliceerd. + Luistert op je nutzap-relays plus je inbox- en DM-relays, zodat een betaling er niet langs kan glippen. + Kijkt op alle relays welke mints er bestaan en welke door mensen worden aanbevolen. + Meldingen van je verbonden wallet. + Actieve relay-abonnementen + + %1$d filter + %1$d filters + + + %1$d relay + %1$d relays + + + %1$d filter is nog niet toegewezen + %1$d filters zijn nog niet toegewezen + + %1$s \u00b7 %2$s + Niet toegewezen aan een account + Alles + Iedereen + Mensen die je volgt + Een gekozen lijst mensen + Gedempte mensen + Jouw communities + Een favoriete algo-feed + %1$d%% van alles + abonnementen filters relays verzoeken reqs verbindingen waarom diagnostiek + Posts van de mensen die je volgt, gelezen van de relays waarop ieder van hen publiceert. Verbinden met inbox-relays… Altijd-aan meldingsdienst Houdt een persistente verbinding met je inbox-relays voor directe melding. Toont een permanente notificatie. Gebruikt meer batterij maar zorgt dat je nooit een bericht mist. @@ -2218,8 +2300,10 @@ Alleen locatie-exclusief bericht Alleen volgers van de locatie zien dit bericht. Alleen hashtag-exclusief bericht + Externe inhoud Reageer op een website Reageer op een externe bron + Openen in browser %1$d min lezen %1$d nummer @@ -2531,6 +2615,9 @@ Verzenden naar… Nieuw bericht Nieuwe Highlight + Nieuwe afbeelding + Nieuwe Short + Nieuwe video Nieuwe Highlight Gemarkeerde tekst Wat viel je op? From dda0f0d9e8d6f414fe0a0b3c0332ce1f60279cee Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 30 Jul 2026 16:27:42 +0000 Subject: [PATCH 041/132] Make the left drawer configurable style: apply spotless to the configurable drawer code docs: record why the settings state holders are deliberately unkeyed refactor: address cleanup review of the configurable drawer fix: rename the shared picker section header to avoid an overload clash --- .../vitorpamplona/amethyst/model/Account.kt | 4 + .../amethyst/model/AccountSettings.kt | 14 + .../amethyst/model/AccountSyncedSettings.kt | 18 +- .../model/AccountSyncedSettingsInternal.kt | 9 + .../amethyst/ui/navigation/AppNavigation.kt | 2 + .../ui/navigation/bottombars/NavBarItem.kt | 81 +---- .../ui/navigation/drawer/DrawerContent.kt | 104 +++--- .../navigation/drawer/DrawerItemVisibility.kt | 104 ++++++ .../ui/navigation/drawer/DrawerSections.kt | 149 ++++++++ .../amethyst/ui/navigation/routes/Routes.kt | 2 + .../ui/screen/loggedIn/AccountViewModel.kt | 10 + .../settings/BottomBarSettingsScreen.kt | 305 +++++----------- .../loggedIn/settings/DrawerSettingsScreen.kt | 263 ++++++++++++++ .../loggedIn/settings/DrawerSettingsState.kt | 80 +++++ .../screen/loggedIn/settings/NavPickerUi.kt | 330 ++++++++++++++++++ .../settings/SettingsCatalogBuilder.kt | 1 + amethyst/src/main/res/values/strings.xml | 11 + .../preferences/DrawerPersistenceTest.kt | 83 +++++ .../navigation/DrawerItemVisibilityTest.kt | 159 +++++++++ .../amethyst/navigation/DrawerSectionsTest.kt | 101 ++++++ 20 files changed, 1492 insertions(+), 338 deletions(-) create mode 100644 amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerItemVisibility.kt create mode 100644 amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerSections.kt create mode 100644 amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/DrawerSettingsScreen.kt create mode 100644 amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/DrawerSettingsState.kt create mode 100644 amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/NavPickerUi.kt create mode 100644 amethyst/src/test/java/com/vitorpamplona/amethyst/model/preferences/DrawerPersistenceTest.kt create mode 100644 amethyst/src/test/java/com/vitorpamplona/amethyst/navigation/DrawerItemVisibilityTest.kt create mode 100644 amethyst/src/test/java/com/vitorpamplona/amethyst/navigation/DrawerSectionsTest.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt index 70afe7bbfe..966c01cedf 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt @@ -150,6 +150,7 @@ import com.vitorpamplona.amethyst.service.relayClient.reqCommand.nwc.NWCPaymentF import com.vitorpamplona.amethyst.service.uploads.FileHeader import com.vitorpamplona.amethyst.ui.actions.NewMessageTagger import com.vitorpamplona.amethyst.ui.navigation.bottombars.BottomBarEntry +import com.vitorpamplona.amethyst.ui.navigation.bottombars.NavBarItem import com.vitorpamplona.amethyst.ui.screen.loggedIn.EventProcessor import com.vitorpamplona.quartz.buzz.threading.buzzThread import com.vitorpamplona.quartz.buzz.threading.buzzThreadReply @@ -955,6 +956,9 @@ class Account( */ fun applyBottomBarItems(items: List): Boolean = settings.changeBottomBarItems(items) + /** The drawer counterpart of [applyBottomBarItems] — same synchronous-apply, publish-after contract. */ + fun applyHiddenDrawerItems(items: Set): Boolean = settings.changeHiddenDrawerItems(items) + suspend fun toggleChatroomPin(room: ChatroomKey) { settings.toggleChatroomPin(room) sendNewAppSpecificData() diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountSettings.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountSettings.kt index f2605013c1..9802633ddf 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountSettings.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountSettings.kt @@ -39,6 +39,8 @@ import com.vitorpamplona.amethyst.model.nip60Cashu.CashuPreferences import com.vitorpamplona.amethyst.ui.actions.mediaServers.DEFAULT_MEDIA_SERVERS import com.vitorpamplona.amethyst.ui.actions.mediaServers.ServerName import com.vitorpamplona.amethyst.ui.navigation.bottombars.BottomBarEntry +import com.vitorpamplona.amethyst.ui.navigation.bottombars.NavBarItem +import com.vitorpamplona.amethyst.ui.navigation.drawer.DrawerItemVisibility import com.vitorpamplona.amethyst.ui.screen.FeedDefinition import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEvent import com.vitorpamplona.quartz.experimental.ephemChat.list.EphemeralChatListEvent @@ -501,6 +503,18 @@ class AccountSettings( return false } + fun changeHiddenDrawerItems(newItems: Set): Boolean { + // Sanitize on the way in as well as on the way out: a caller must never be able to persist + // Settings as hidden, which would leave no route back to the screen that hides rows. + val sanitized = DrawerItemVisibility.sanitize(newItems) + if (syncedSettings.navigation.hiddenDrawerItems.value != sanitized) { + syncedSettings.navigation.hiddenDrawerItems.tryEmit(sanitized) + saveAccountSettings() + return true + } + return false + } + /** The selected default spend rail across both NWC wallets and CLINK debits. */ fun defaultPaymentSource(): PaymentSource? = PaymentSourceResolver.resolveDefault(nwcWallets.value, clinkDebitWallets.value, defaultPaymentSourceId.value) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountSyncedSettings.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountSyncedSettings.kt index a33799dd40..5fa6adc7ec 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountSyncedSettings.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountSyncedSettings.kt @@ -25,6 +25,10 @@ import com.vitorpamplona.amethyst.commons.audio.VisualizerStyle import com.vitorpamplona.amethyst.commons.service.pow.PoWCategory import com.vitorpamplona.amethyst.commons.service.pow.PoWPolicy import com.vitorpamplona.amethyst.ui.navigation.bottombars.BottomBarEntry +import com.vitorpamplona.amethyst.ui.navigation.bottombars.NavBarItem +import com.vitorpamplona.amethyst.ui.navigation.bottombars.navBarItemsFromNames +import com.vitorpamplona.amethyst.ui.navigation.bottombars.toNames +import com.vitorpamplona.amethyst.ui.navigation.drawer.DrawerItemVisibility import com.vitorpamplona.amethyst.ui.screen.loggedIn.notifications.equalImmutableLists import com.vitorpamplona.quartz.nip17Dm.base.ChatroomKey import com.vitorpamplona.quartz.nip57Zaps.LnZapEvent @@ -83,6 +87,7 @@ class AccountSyncedSettings( val navigation = AccountNavigationPreferences( MutableStateFlow(internalSettings.navigation.bottomBarItems), + MutableStateFlow(DrawerItemVisibility.sanitize(navBarItemsFromNames(internalSettings.navigation.hiddenDrawerItems))), ) fun toInternal(): AccountSyncedSettingsInternal = @@ -124,7 +129,11 @@ class AccountSyncedSettings( .map { it.id } .sorted(), ), - navigation = AccountNavigationPreferencesInternal(navigation.bottomBarItems.value), + navigation = + AccountNavigationPreferencesInternal( + navigation.bottomBarItems.value, + navigation.hiddenDrawerItems.value.toNames(), + ), ) fun updateFrom(syncedSettingsInternal: AccountSyncedSettingsInternal) { @@ -221,6 +230,11 @@ class AccountSyncedSettings( if (navigation.bottomBarItems.value != newBottomBarItems) { navigation.bottomBarItems.tryEmit(newBottomBarItems) } + + val newHiddenDrawerItems = DrawerItemVisibility.sanitize(navBarItemsFromNames(syncedSettingsInternal.navigation.hiddenDrawerItems)) + if (navigation.hiddenDrawerItems.value != newHiddenDrawerItems) { + navigation.hiddenDrawerItems.tryEmit(newHiddenDrawerItems) + } } fun dontTranslateFromFilteredBySpokenLanguages(): Set = languages.dontTranslateFrom.value - getLanguagesSpokenByUser() @@ -322,6 +336,8 @@ class AccountMediaPreferences( @Stable class AccountNavigationPreferences( val bottomBarItems: MutableStateFlow>, + /** Drawer rows switched off by the user. Empty = the stock drawer; see DrawerItemVisibility. */ + val hiddenDrawerItems: MutableStateFlow>, ) @Stable diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountSyncedSettingsInternal.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountSyncedSettingsInternal.kt index 4f3f51ae95..32b3900cb0 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountSyncedSettingsInternal.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountSyncedSettingsInternal.kt @@ -170,6 +170,15 @@ class AccountNavigationPreferencesInternal( // favorite apps, and individual joined chats/groups). Defaulted so blobs // written before this field existed decode to the app's current defaults. var bottomBarItems: List = DefaultBottomBarEntries, + // The drawer (side menu) rows the user switched off, as NavBarItem *names*. + // Empty by default, which is what makes a newly shipped destination visible + // to everyone without a migration — see DrawerItemVisibility. + // + // Stored as strings rather than the enum on purpose: an id written by a + // newer client would fail the enum decoder and take the whole synced-settings + // blob down with it, so unknown names are dropped on read instead (the same + // approach AccountPoWPreferencesInternal.enabledCategories takes). + var hiddenDrawerItems: List = emptyList(), ) @Serializable diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/AppNavigation.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/AppNavigation.kt index 1b10a93069..ee1ace2d8d 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/AppNavigation.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/AppNavigation.kt @@ -263,6 +263,7 @@ import com.vitorpamplona.amethyst.ui.screen.loggedIn.settings.BlockedUsersScreen import com.vitorpamplona.amethyst.ui.screen.loggedIn.settings.BottomBarSettingsScreen import com.vitorpamplona.amethyst.ui.screen.loggedIn.settings.CallSettingsScreen import com.vitorpamplona.amethyst.ui.screen.loggedIn.settings.ComposeSettingsScreen +import com.vitorpamplona.amethyst.ui.screen.loggedIn.settings.DrawerSettingsScreen import com.vitorpamplona.amethyst.ui.screen.loggedIn.settings.HiddenWordsScreen import com.vitorpamplona.amethyst.ui.screen.loggedIn.settings.HomeTabsSettingsScreen import com.vitorpamplona.amethyst.ui.screen.loggedIn.settings.MessagesSettingsScreen @@ -578,6 +579,7 @@ fun BuildNavigation( composableFromEnd { MessagesSettingsScreen(accountViewModel, nav) } composableFromEnd { AudioVisualizerSettingsScreen(accountViewModel, nav) } composableFromEnd { BottomBarSettingsScreen(accountViewModel, nav) } + composableFromEnd { DrawerSettingsScreen(accountViewModel, nav) } composableFromEnd { HomeTabsSettingsScreen(accountViewModel, nav) } composableFromEnd { ProfileUiSettingsScreen(accountViewModel, nav) } composableFromEnd { VideoPlayerSettingsScreen(accountViewModel, nav) } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/NavBarItem.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/NavBarItem.kt index 51f545d95c..2209c1bec6 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/NavBarItem.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/NavBarItem.kt @@ -20,7 +20,6 @@ */ package com.vitorpamplona.amethyst.ui.navigation.bottombars -import android.os.Build import com.vitorpamplona.amethyst.R import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbol import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols @@ -29,8 +28,9 @@ import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel import kotlinx.serialization.Serializable /** - * Stable identifiers for every drawer destination that the user can pin to the bottom bar. - * Order in this enum has no semantic meaning — the user picks a subset and an order at runtime. + * Stable identifiers for every destination the navigation surfaces can show — the bottom bar pins a + * subset in a user-chosen order, the drawer lists them under fixed headings (see DrawerSections). + * Order in this enum has no semantic meaning. */ @Serializable enum class NavBarItem { @@ -84,6 +84,18 @@ enum class NavBarItem { FAVORITE_ALGO_FEEDS, } +private val NavBarItemsByName = NavBarItem.entries.associateBy { it.name } + +/** + * Parses persisted [NavBarItem] names, silently dropping any this build doesn't know — a settings + * blob synced from a newer client can name a destination that doesn't exist here yet, and that must + * degrade to "ignore this one row" rather than failing the decode of the whole blob. + */ +fun navBarItemsFromNames(names: Collection): Set = names.mapNotNullTo(mutableSetOf()) { NavBarItemsByName[it] } + +/** The inverse of [navBarItemsFromNames]; sorted so the serialized form is deterministic. */ +fun Set.toNames(): List = map { it.name }.sorted() + data class NavBarItemDef( val id: NavBarItem, val labelRes: Int, @@ -443,34 +455,6 @@ val DefaultBottomBarItems: List = /** The default bottom bar as unified entries (all built-in; favorites are added by the user). */ val DefaultBottomBarEntries: List = DefaultBottomBarItems.map { BottomBarEntry.BuiltIn(it) } -// Ordered membership lists for each drawer section. The drawer renders these by looking up -// each id in NavBarCatalog, so adding a new screen only requires editing the catalog + the -// matching section list below — not two separate files. -val DrawerNavigateItems: List = - listOf( - NavBarItem.HOME, - NavBarItem.MESSAGES, - NavBarItem.VIDEO, - NavBarItem.BROWSER, - NavBarItem.DISCOVER, - NavBarItem.NOTIFICATIONS, - ) - -val DrawerYouItems: List = - listOf( - NavBarItem.PROFILE, - NavBarItem.MY_LISTS, - NavBarItem.BOOKMARKS, - NavBarItem.WEB_BOOKMARKS, - NavBarItem.DRAFTS, - NavBarItem.SCHEDULED_POSTS, - NavBarItem.INTEREST_SETS, - NavBarItem.BLOSSOM_DATA, - NavBarItem.EMOJI_PACKS, - NavBarItem.WALLET, - NavBarItem.NOSTR_SIGNER, - ) - /** * A titled, collapsible group of selectable destinations in the bottom-bar settings picker. The * catalog's [linkedMapOf] insertion order is hand-maintained and reads as scattered in the flat @@ -568,38 +552,3 @@ val BottomBarCategories: List = ), ), ) - -val DrawerFeedsItems: List = - listOfNotNull( - NavBarItem.ARTICLES, - NavBarItem.PICTURES, - NavBarItem.SHORTS, - NavBarItem.LONGS, - NavBarItem.PODCAST_EPISODES, - NavBarItem.PODCASTS, - NavBarItem.MUSIC_TRACKS, - NavBarItem.MUSIC_PLAYLISTS, - NavBarItem.POLLS, - NavBarItem.PRODUCTS, - NavBarItem.WORKOUTS, - NavBarItem.GIT_REPOSITORIES, - NavBarItem.HIGHLIGHTS, - NavBarItem.LIVE_STREAMS, - NavBarItem.NESTS, - NavBarItem.COMMUNITIES, - NavBarItem.PUBLIC_CHATS, - NavBarItem.RELAY_GROUPS, - NavBarItem.CONCORD, - NavBarItem.GEOHASH_CHATS, - NavBarItem.CALENDARS, - NavBarItem.CALENDAR_COLLECTIONS, - NavBarItem.SOFTWARE_APPS, - // Favorites can be pinned as inline tabs that render on a cross-process surface - // (SurfaceControlViewHost), which needs API 30+. Gate the whole grid on R+ for that reason. - NavBarItem.FAVORITE_APPS.takeIf { Build.VERSION.SDK_INT >= Build.VERSION_CODES.R }, - NavBarItem.NAPPLETS, - NavBarItem.NSITES, - NavBarItem.FOLLOW_PACKS, - NavBarItem.BADGES, - NavBarItem.EMOJI_SETS, - ) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerContent.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerContent.kt index f0221e563d..a179e627c8 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerContent.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerContent.kt @@ -63,6 +63,7 @@ import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.derivedStateOf import androidx.compose.runtime.getValue +import androidx.compose.runtime.key import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.setValue @@ -105,9 +106,6 @@ import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.observeUse import com.vitorpamplona.amethyst.ui.components.CreateTextWithEmoji import com.vitorpamplona.amethyst.ui.components.RobohashFallbackAsyncImage import com.vitorpamplona.amethyst.ui.layouts.PermanentDrawerWidth -import com.vitorpamplona.amethyst.ui.navigation.bottombars.DrawerFeedsItems -import com.vitorpamplona.amethyst.ui.navigation.bottombars.DrawerNavigateItems -import com.vitorpamplona.amethyst.ui.navigation.bottombars.DrawerYouItems import com.vitorpamplona.amethyst.ui.navigation.bottombars.NavBarCatalog import com.vitorpamplona.amethyst.ui.navigation.bottombars.NavBarItem import com.vitorpamplona.amethyst.ui.navigation.bottombars.NavBarItemDef @@ -584,42 +582,17 @@ fun ListContent( accountViewModel: AccountViewModel, nav: INav, ) { + // Per-account, synced through the NIP-78 app-specific data event, and edited on the + // Side Menu settings screen. Empty (the default) means the full stock drawer. + val hidden by accountViewModel.hiddenDrawerItemsFlow().collectAsStateWithLifecycle() + Column(modifier) { - CatalogSection(R.string.drawer_section_you, DrawerYouItems, accountViewModel, nav) - CatalogSection(R.string.drawer_section_navigate, DrawerNavigateItems, accountViewModel, nav) - CatalogSection(R.string.drawer_section_feeds, DrawerFeedsItems, accountViewModel, nav) - - CollapsibleSection(title = R.string.drawer_section_create) { - NavigationRow( - title = R.string.share_hls_video, - icon = MaterialSymbols.SettingsInputAntenna, - tint = MaterialTheme.colorScheme.onBackground, - nav = nav, - route = Route.NewHlsVideo, - ) - - if (isDebug) { - NavigationRow( - title = R.string.route_chess, - icon = MaterialSymbols.ChessKnight, - tint = MaterialTheme.colorScheme.onBackground, - nav = nav, - route = Route.Chess, - ) - } - } - - CollapsibleSection(title = R.string.drawer_section_system) { - IconRowRelays( - accountViewModel = accountViewModel, - onClick = { - nav.closeDrawer() - nav.nav(Route.EditRelays) - }, - ) - - NavBarCatalog[NavBarItem.SETTINGS]?.let { - CatalogNavigationRow(it, MaterialTheme.colorScheme.onBackground, accountViewModel, nav) + DrawerSections.forEach { section -> + // Keyed by section: hiding the last row of a section removes it from the drawer + // entirely, and without a key the sections below would slide up into its slots and + // inherit its CollapsibleSection expanded/collapsed state. + key(section.id) { + CatalogSection(section, hidden, accountViewModel, nav) } } @@ -634,22 +607,65 @@ fun ListContent( } } +/** The Create section's rows — composer entry points, none of which is a catalog destination. */ +@Composable +private fun CreateRows(nav: INav) { + NavigationRow( + title = R.string.share_hls_video, + icon = MaterialSymbols.SettingsInputAntenna, + tint = MaterialTheme.colorScheme.onBackground, + nav = nav, + route = Route.NewHlsVideo, + ) + + if (isDebug) { + NavigationRow( + title = R.string.route_chess, + icon = MaterialSymbols.ChessKnight, + tint = MaterialTheme.colorScheme.onBackground, + nav = nav, + route = Route.Chess, + ) + } +} + /** - * Renders a drawer section by iterating [ids] and looking each one up in [NavBarCatalog]. - * Profile gets the primary-colored tint; every other item uses onBackground. + * Renders one drawer section: its fixed rows, if it has any, then the catalog rows the user hasn't + * switched off. Profile gets the primary-colored tint; every other item uses onBackground. + * + * A section with nothing left to show renders nothing at all — an empty, permanently collapsed + * heading is just noise. Two sections always have something: Create is entirely fixed rows, and + * System carries the relay-status row (not a catalog destination — it shows a live counter). */ @Composable fun CatalogSection( - titleRes: Int, - ids: List, + section: DrawerSection, + hidden: Set, accountViewModel: AccountViewModel, nav: INav, ) { val primary = MaterialTheme.colorScheme.primary val onBackground = MaterialTheme.colorScheme.onBackground - CollapsibleSection(title = titleRes) { - ids.forEach { id -> + val visible = remember(section, hidden) { DrawerItemVisibility.visibleItems(section, hidden) } + val hasFixedRows = section.id == DrawerSectionId.CREATE || section.id == DrawerSectionId.SYSTEM + if (visible.isEmpty() && !hasFixedRows) return + + CollapsibleSection(title = section.titleRes) { + when (section.id) { + DrawerSectionId.CREATE -> CreateRows(nav) + DrawerSectionId.SYSTEM -> + IconRowRelays( + accountViewModel = accountViewModel, + onClick = { + nav.closeDrawer() + nav.nav(Route.EditRelays) + }, + ) + else -> {} + } + + visible.forEach { id -> NavBarCatalog[id]?.let { def -> val tint = if (def.id == NavBarItem.PROFILE) primary else onBackground if (def.id == NavBarItem.SCHEDULED_POSTS) { diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerItemVisibility.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerItemVisibility.kt new file mode 100644 index 0000000000..5c649a9a60 --- /dev/null +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerItemVisibility.kt @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.ui.navigation.drawer + +import com.vitorpamplona.amethyst.ui.navigation.bottombars.NavBarItem + +/** + * Which drawer rows the user cannot hide. + * + * Settings is the only one, and it is mandatory for a specific reason: it is the route back to the + * screen that hides rows in the first place. Hiding it would let a user lock themselves out of their + * own configuration. Everything else the drawer always shows — the profile header, the relay-status + * row, the account switcher and the version/QR footer — is fixed chrome rather than a catalog row, + * so it is present by construction and never appears in the hidden set. + */ +val MandatoryDrawerItems: Set = setOf(NavBarItem.SETTINGS) + +/** + * Pure show/hide rules for the drawer's catalog rows, kept free of Compose and Android so they are + * exercised directly by unit tests (DrawerItemVisibilityTest) rather than only through the UI. + * + * The per-account preference stores the **hidden** items rather than the visible ones. That choice is + * what makes a newly added destination appear for everyone automatically: a row nobody has ever + * hidden simply isn't in the set, so it renders. Storing the visible list instead would freeze each + * account's drawer at the moment they first touched the setting, and every later release would have + * to migrate saved lists to introduce a screen. + */ +object DrawerItemVisibility { + fun isVisible( + hidden: Set, + item: NavBarItem, + ): Boolean = item in MandatoryDrawerItems || item !in hidden + + /** Hides [item] if shown, shows it if hidden. Mandatory items never change (see [MandatoryDrawerItems]). */ + fun toggle( + hidden: Set, + item: NavBarItem, + ): Set = + when { + item in MandatoryDrawerItems -> hidden + item in hidden -> hidden - item + else -> hidden + item + } + + /** + * Drops mandatory rows from the set. The persistence layer is the single place this is enforced — + * it runs on decode, on an external sync, and on every write — so a value synced from another + * client (or from a build where the row wasn't mandatory yet) can't strand Settings as hidden. + * + * Ids that no section renders are deliberately *kept*: on a device where a row is gated off (see + * DrawerFeedsItems' API-30 gate on Favorite Apps) it matches nothing and costs nothing, and + * preserving it means editing the drawer on that device doesn't silently clear the choice the + * user made on another one. + */ + fun sanitize(hidden: Set): Set = hidden - MandatoryDrawerItems + + /** The rows of [section] to render, in the section's fixed order. */ + fun visibleItems( + section: DrawerSection, + hidden: Set, + ): List = section.items.filter { isVisible(hidden, it) } + + /** How many of [section]'s rows are currently hidden — shown on the collapsed section header. */ + fun hiddenCount( + section: DrawerSection, + hidden: Set, + ): Int = section.items.count { !isVisible(hidden, it) } + + /** Whether [section] has any row the user is allowed to switch off — gates its bulk actions. */ + fun hasHideableRows(section: DrawerSection): Boolean = section.items.any { it !in MandatoryDrawerItems } + + /** Hides every row of [section] that can be hidden, leaving the mandatory ones. */ + fun hideAll( + hidden: Set, + section: DrawerSection, + ): Set = hidden + section.items.filter { it !in MandatoryDrawerItems } + + /** Shows every row of [section] again. */ + fun showAll( + hidden: Set, + section: DrawerSection, + ): Set = hidden - section.items.toSet() + + /** Total hidden rows across every section — the count the settings screen shows at the top. */ + fun totalHidden(hidden: Set): Int = DrawerSections.sumOf { hiddenCount(it, hidden) } +} diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerSections.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerSections.kt new file mode 100644 index 0000000000..4e591f71f9 --- /dev/null +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerSections.kt @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.ui.navigation.drawer + +import android.os.Build +import androidx.compose.runtime.Immutable +import com.vitorpamplona.amethyst.R +import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbol +import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols +import com.vitorpamplona.amethyst.ui.navigation.bottombars.NavBarCatalog +import com.vitorpamplona.amethyst.ui.navigation.bottombars.NavBarItem + +/** + * The drawer's layout: which destinations it lists, under which heading, in which order. + * + * One list drives two screens — [ListContent] renders the visible rows of each section, and the Side + * Menu settings screen renders the same sections as its show/hide catalog. Adding a destination to a + * section's list therefore surfaces it in the drawer *and* in its configuration screen without + * touching either, and DrawerSectionsTest fails the build if a newly added [NavBarCatalog] id isn't + * filed into exactly one section. + * + * Section order and within-section order are fixed and not user-editable: the drawer is a menu, and a + * menu whose headings move around is harder to learn, not easier. The only per-account choice is + * which rows are visible — see [DrawerItemVisibility]. + */ +@Immutable +data class DrawerSection( + val id: DrawerSectionId, + val titleRes: Int, + val icon: MaterialSymbol, + val items: List, +) + +/** + * Identifies a section for the handful of rendering rules that are specific to one. Matching on this + * rather than on a section's object identity keeps those rules working if the list is ever mapped or + * copied — a `DrawerSections.map { it.copy(...) }` would silently defeat an `===` check, with no + * compile error and nothing to fail a test. + */ +enum class DrawerSectionId { + YOU, + NAVIGATE, + FEEDS, + + /** Composer entry points. Carries no catalog destinations, so nothing in it is configurable. */ + CREATE, + + /** Also renders the relay-status row, which isn't a catalog destination (it shows a live counter). */ + SYSTEM, +} + +private val DrawerNavigateItems: List = + listOf( + NavBarItem.HOME, + NavBarItem.MESSAGES, + NavBarItem.VIDEO, + NavBarItem.BROWSER, + NavBarItem.DISCOVER, + NavBarItem.NOTIFICATIONS, + ) + +private val DrawerYouItems: List = + listOf( + NavBarItem.PROFILE, + NavBarItem.MY_LISTS, + NavBarItem.BOOKMARKS, + NavBarItem.WEB_BOOKMARKS, + NavBarItem.DRAFTS, + NavBarItem.SCHEDULED_POSTS, + NavBarItem.INTEREST_SETS, + NavBarItem.FAVORITE_ALGO_FEEDS, + NavBarItem.BLOSSOM_DATA, + NavBarItem.EMOJI_PACKS, + NavBarItem.WALLET, + NavBarItem.NOSTR_SIGNER, + ) + +private val DrawerFeedsItems: List = + listOfNotNull( + NavBarItem.ARTICLES, + NavBarItem.PICTURES, + NavBarItem.SHORTS, + NavBarItem.LONGS, + NavBarItem.PODCAST_EPISODES, + NavBarItem.PODCASTS, + NavBarItem.MUSIC_TRACKS, + NavBarItem.MUSIC_PLAYLISTS, + NavBarItem.POLLS, + NavBarItem.PRODUCTS, + NavBarItem.WORKOUTS, + NavBarItem.GIT_REPOSITORIES, + NavBarItem.HIGHLIGHTS, + NavBarItem.LIVE_STREAMS, + NavBarItem.NESTS, + NavBarItem.COMMUNITIES, + NavBarItem.PUBLIC_CHATS, + NavBarItem.RELAY_GROUPS, + NavBarItem.CONCORD, + NavBarItem.GEOHASH_CHATS, + NavBarItem.CALENDARS, + NavBarItem.CALENDAR_COLLECTIONS, + NavBarItem.SOFTWARE_APPS, + // Favorites can be pinned as inline tabs that render on a cross-process surface + // (SurfaceControlViewHost), which needs API 30+. Gate the whole grid on R+ for that reason. + NavBarItem.FAVORITE_APPS.takeIf { Build.VERSION.SDK_INT >= Build.VERSION_CODES.R }, + NavBarItem.NAPPLETS, + NavBarItem.NSITES, + NavBarItem.FOLLOW_PACKS, + NavBarItem.BADGES, + NavBarItem.EMOJI_SETS, + ) + +val DrawerSections: List = + listOf( + DrawerSection(DrawerSectionId.YOU, R.string.drawer_section_you, MaterialSymbols.AccountCircle, DrawerYouItems), + DrawerSection(DrawerSectionId.NAVIGATE, R.string.drawer_section_navigate, MaterialSymbols.Home, DrawerNavigateItems), + DrawerSection(DrawerSectionId.FEEDS, R.string.drawer_section_feeds, MaterialSymbols.Subscriptions, DrawerFeedsItems), + DrawerSection(DrawerSectionId.CREATE, R.string.drawer_section_create, MaterialSymbols.Edit, emptyList()), + DrawerSection(DrawerSectionId.SYSTEM, R.string.drawer_section_system, MaterialSymbols.Settings, listOf(NavBarItem.SETTINGS)), + ) + +fun drawerSection(id: DrawerSectionId): DrawerSection = DrawerSections.first { it.id == id } + +/** + * Catalog ids deliberately absent from every [DrawerSections] list, with the reason. Only Favorite + * Apps qualifies: [DrawerFeedsItems] gates it on API 30+ (its inline tabs need SurfaceControlViewHost), + * so on older devices the row simply doesn't exist. DrawerSectionsTest allows exactly these to be + * missing, and fails on anything else — that's what keeps a newly added destination from silently + * skipping both the drawer and its settings screen. + */ +val SdkGatedDrawerItems: Set = setOf(NavBarItem.FAVORITE_APPS) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/routes/Routes.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/routes/Routes.kt index 399db4dc84..b1afa1c5b9 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/routes/Routes.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/routes/Routes.kt @@ -457,6 +457,8 @@ sealed class Route { @Serializable object BottomBarSettings : Route() + @Serializable object DrawerSettings : Route() + @Serializable object HomeTabsSettings : Route() @Serializable object ProfileUiSettings : Route() diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/AccountViewModel.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/AccountViewModel.kt index 7963b56c27..f27b3215c9 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/AccountViewModel.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/AccountViewModel.kt @@ -93,6 +93,7 @@ import com.vitorpamplona.amethyst.ui.actions.MediaSaverToDisk import com.vitorpamplona.amethyst.ui.actions.NewMessageTagger import com.vitorpamplona.amethyst.ui.components.toasts.ToastManager import com.vitorpamplona.amethyst.ui.navigation.bottombars.BottomBarEntry +import com.vitorpamplona.amethyst.ui.navigation.bottombars.NavBarItem import com.vitorpamplona.amethyst.ui.navigation.routes.Route import com.vitorpamplona.amethyst.ui.note.ZapAmountCommentNotification import com.vitorpamplona.amethyst.ui.note.ZapraiserStatus @@ -1986,6 +1987,15 @@ class AccountViewModel( fun bottomBarItemsFlow(): StateFlow> = account.settings.syncedSettings.navigation.bottomBarItems + fun hiddenDrawerItemsFlow(): StateFlow> = account.settings.syncedSettings.navigation.hiddenDrawerItems + + /** Same ordering contract as [changeBottomBarItems]: apply on the caller's thread, publish off it. */ + fun changeHiddenDrawerItems(items: Set) { + if (account.applyHiddenDrawerItems(items)) { + launchSigner { account.sendNewAppSpecificData() } + } + } + fun changeBottomBarItems(items: List) { // Apply to the reactive flow synchronously on the caller (UI) thread so rapid edits stay // ordered — launchSigner dispatches on a multi-threaded pool, so wrapping the emit too would diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/BottomBarSettingsScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/BottomBarSettingsScreen.kt index 7131c5c891..f23ace6d4b 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/BottomBarSettingsScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/BottomBarSettingsScreen.kt @@ -22,11 +22,6 @@ package com.vitorpamplona.amethyst.ui.screen.loggedIn.settings import androidx.compose.animation.AnimatedVisibility import androidx.compose.animation.core.animateFloatAsState -import androidx.compose.animation.expandVertically -import androidx.compose.animation.fadeIn -import androidx.compose.animation.fadeOut -import androidx.compose.animation.shrinkVertically -import androidx.compose.foundation.BorderStroke import androidx.compose.foundation.background import androidx.compose.foundation.clickable import androidx.compose.foundation.gestures.detectDragGestures @@ -50,7 +45,6 @@ import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Scaffold import androidx.compose.material3.Surface import androidx.compose.material3.Text -import androidx.compose.material3.TextButton import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue @@ -97,7 +91,6 @@ import com.vitorpamplona.amethyst.ui.navigation.topbars.TopBarWithBackButton import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel import com.vitorpamplona.amethyst.ui.screen.loggedIn.mockAccountViewModel import com.vitorpamplona.amethyst.ui.stringRes -import com.vitorpamplona.amethyst.ui.theme.Size20dp import com.vitorpamplona.amethyst.ui.theme.ThemeComparisonRow import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEntry import com.vitorpamplona.quartz.nip51Lists.simpleGroupList.GroupTag @@ -116,11 +109,6 @@ private val ExpandableItems = /** Soft guidance, not a hard cap: a Material bottom bar reads best at ~5 tabs. */ private const val RECOMMENDED_SLOTS = 5 -// Reveal expandable sections by unrolling straight down from the top edge (the default AnimatedVisibility -// enter also expands horizontally from the bottom-end, which reads as a diagonal slide from the top-left). -private val SectionExpand = expandVertically(expandFrom = Alignment.Top) + fadeIn() -private val SectionCollapse = shrinkVertically(shrinkTowards = Alignment.Top) + fadeOut() - @Composable @Preview(device = "spec:width=2100px,height=2340px,dpi=440") fun BottomBarSettingsScreenPreview() { @@ -157,6 +145,13 @@ fun BottomBarSettingsContent(accountViewModel: AccountViewModel) { // All pin/unpin/reorder logic lives in the holder (unit-tested); the composable only renders and // forwards events. Each persist republishes the account's NIP-78 settings event. syncFrom re-seeds // when the saved list changes elsewhere without clobbering a drag. + // + // Deliberately unkeyed. The holder captures this `accountViewModel` in its persist lambda, so a + // holder that outlived an account switch would write account A's edits to account B. It cannot: + // SetAccountCentricViewModelStore wraps the whole logged-in tree in `key(account.signer.pubKey)`, + // so a switch disposes this composable (and the NavController with it) and re-runs this remember + // against the new account's ViewModel. Keying on accountViewModel here would be a no-op that + // implies the subtree survives a switch — if that ever becomes true, this comment is the bug. val state = remember { BottomBarSettingsState(savedItems) { accountViewModel.changeBottomBarItems(it) } } LaunchedEffect(savedItems) { state.syncFrom(savedItems) } @@ -177,19 +172,12 @@ fun BottomBarSettingsContent(accountViewModel: AccountViewModel) { // --- The editable bar: a real preview you drag to reorder and tap ✕ to remove from. --- EditableBarCard(state, pinned, accountViewModel) - Row( - modifier = Modifier.fillMaxWidth().padding(horizontal = Size20dp), - horizontalArrangement = Arrangement.End, - ) { - TextButton(onClick = { state.restoreDefault() }) { - Text(stringRes(R.string.bottom_bar_settings_restore_default)) - } - } + RestoreDefaultRow(onClick = { state.restoreDefault() }) Spacer(Modifier.height(4.dp)) // --- Available catalogue, grouped into collapsible category cards. --- - SectionHeader(title = stringRes(R.string.bottom_bar_settings_available)) + PickerSectionHeader(title = stringRes(R.string.bottom_bar_settings_available)) BottomBarCategories.forEach { category -> CategoryCard( @@ -217,60 +205,43 @@ private fun EditableBarCard( pinned: List, accountViewModel: AccountViewModel, ) { - val accent = MaterialTheme.colorScheme.primary - Surface( - shape = RoundedCornerShape(22.dp), - color = accent.copy(alpha = 0.07f), - border = BorderStroke(1.dp, accent.copy(alpha = 0.22f)), - modifier = Modifier.fillMaxWidth().padding(horizontal = Size20dp, vertical = 4.dp), - ) { - Column(Modifier.padding(14.dp)) { - Row( - modifier = Modifier.fillMaxWidth().padding(bottom = 10.dp), - horizontalArrangement = Arrangement.SpaceBetween, - verticalAlignment = Alignment.CenterVertically, - ) { - Text( - text = stringRes(R.string.bottom_bar_settings_pinned), - style = MaterialTheme.typography.labelMedium, - color = accent, - fontWeight = FontWeight.Bold, - ) - Text( - text = "${pinned.size} / $RECOMMENDED_SLOTS", - style = MaterialTheme.typography.labelMedium, - color = if (pinned.size > RECOMMENDED_SLOTS) MaterialTheme.colorScheme.error else accent, - fontWeight = FontWeight.Bold, - ) - } - - Surface( - shape = RoundedCornerShape(16.dp), - color = MaterialTheme.colorScheme.background, - shadowElevation = 3.dp, - modifier = Modifier.fillMaxWidth(), - ) { - if (pinned.isEmpty()) { - Box(Modifier.fillMaxWidth().height(60.dp), contentAlignment = Alignment.Center) { - Text( - stringRes(R.string.bottom_bar_settings_pinned_empty), - style = MaterialTheme.typography.bodySmall, - color = MaterialTheme.colorScheme.onSurfaceVariant, - modifier = Modifier.padding(horizontal = 16.dp), - ) - } - } else { - EditableBar(state, pinned, accountViewModel) - } - } - + PickerHeroCard( + title = stringRes(R.string.bottom_bar_settings_pinned), + trailing = { Text( - text = stringRes(R.string.bottom_bar_settings_reorder_hint), - style = MaterialTheme.typography.labelSmall, - color = MaterialTheme.colorScheme.onSurfaceVariant, - modifier = Modifier.padding(top = 8.dp), + text = "${pinned.size} / $RECOMMENDED_SLOTS", + style = MaterialTheme.typography.labelMedium, + color = if (pinned.size > RECOMMENDED_SLOTS) MaterialTheme.colorScheme.error else MaterialTheme.colorScheme.primary, + fontWeight = FontWeight.Bold, ) + }, + ) { + Surface( + shape = RoundedCornerShape(16.dp), + color = MaterialTheme.colorScheme.background, + shadowElevation = 3.dp, + modifier = Modifier.fillMaxWidth(), + ) { + if (pinned.isEmpty()) { + Box(Modifier.fillMaxWidth().height(60.dp), contentAlignment = Alignment.Center) { + Text( + stringRes(R.string.bottom_bar_settings_pinned_empty), + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.padding(horizontal = 16.dp), + ) + } + } else { + EditableBar(state, pinned, accountViewModel) + } } + + Text( + text = stringRes(R.string.bottom_bar_settings_reorder_hint), + style = MaterialTheme.typography.labelSmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.padding(top = 8.dp), + ) } } @@ -470,72 +441,33 @@ private fun CategoryCard( accountViewModel: AccountViewModel, onTogglePin: (BottomBarEntry) -> Unit, ) { - Surface( - shape = RoundedCornerShape(16.dp), - color = MaterialTheme.colorScheme.surface, - border = BorderStroke(1.dp, MaterialTheme.colorScheme.outlineVariant), - modifier = Modifier.fillMaxWidth().padding(horizontal = Size20dp, vertical = 5.dp), + CatalogCard( + icon = categoryIcon(category.titleRes), + title = stringRes(category.titleRes), + expanded = expanded, + onToggleExpand = onToggleExpand, ) { - Column { - Row( - modifier = Modifier.fillMaxWidth().clickable(onClick = onToggleExpand).padding(13.dp), - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.spacedBy(12.dp), - ) { - Box( - modifier = - Modifier - .size(34.dp) - .clip(RoundedCornerShape(11.dp)) - .background(MaterialTheme.colorScheme.surfaceVariant), - contentAlignment = Alignment.Center, + category.items.forEach { item -> + val def = NavBarCatalog[item] ?: return@forEach + val entry = BottomBarEntry.BuiltIn(item) + if (item in ExpandableItems) { + ExpandableAvailableRow( + icon = def.icon, + label = stringRes(def.labelRes), + pinned = entry.stableKey in pinnedKeys, + expanded = expandedItems[item] ?: false, + onTogglePin = { onTogglePin(entry) }, + onToggleExpand = { expandedItems[item] = !(expandedItems[item] ?: false) }, ) { - Icon( - symbol = categoryIcon(category.titleRes), - contentDescription = null, - modifier = Modifier.size(20.dp), - tint = MaterialTheme.colorScheme.onSurfaceVariant, - ) + PickerChildren(item, pinnedKeys, accountViewModel, onTogglePin) } - Text( - text = stringRes(category.titleRes), - style = MaterialTheme.typography.titleSmall, - modifier = Modifier.weight(1f), + } else { + AvailableRow( + leading = { LeadingGlyph(def.icon) }, + label = stringRes(def.labelRes), + pinned = entry.stableKey in pinnedKeys, + onToggle = { onTogglePin(entry) }, ) - Icon( - symbol = if (expanded) MaterialSymbols.ExpandLess else MaterialSymbols.ExpandMore, - contentDescription = null, - modifier = Modifier.size(24.dp), - tint = MaterialTheme.colorScheme.onSurfaceVariant, - ) - } - - AnimatedVisibility(visible = expanded, enter = SectionExpand, exit = SectionCollapse) { - Column(Modifier.padding(bottom = 6.dp)) { - category.items.forEach { item -> - val def = NavBarCatalog[item] ?: return@forEach - val entry = BottomBarEntry.BuiltIn(item) - if (item in ExpandableItems) { - ExpandableAvailableRow( - icon = def.icon, - label = stringRes(def.labelRes), - pinned = entry.stableKey in pinnedKeys, - expanded = expandedItems[item] ?: false, - onTogglePin = { onTogglePin(entry) }, - onToggleExpand = { expandedItems[item] = !(expandedItems[item] ?: false) }, - ) { - PickerChildren(item, pinnedKeys, accountViewModel, onTogglePin) - } - } else { - AvailableRow( - leading = { LeadingGlyph(def.icon) }, - label = stringRes(def.labelRes), - pinned = entry.stableKey in pinnedKeys, - onToggle = { onTogglePin(entry) }, - ) - } - } - } } } } @@ -757,18 +689,6 @@ private fun ConcordServerPickerGroup( // Rows & shared bits // ------------------------------------------------------------------------------------------------ -/** - * Start padding per nesting depth: 0 = a top-level catalog row, 1 = an item under an expandable - * category (a favorite, or a relay/community "server" row), 2 = a room nested under its server (a - * NIP-29 group under its relay, or a Concord channel under its community). - */ -private fun indentPadding(level: Int) = - when (level) { - 0 -> 13.dp - 1 -> 24.dp - else -> 40.dp - } - @Composable private fun AvailableRow( leading: @Composable () -> Unit, @@ -777,23 +697,12 @@ private fun AvailableRow( onToggle: () -> Unit, indentLevel: Int = 0, ) { - Row( - modifier = - Modifier - .fillMaxWidth() - .clickable(onClick = onToggle) - .padding(start = indentPadding(indentLevel), end = 13.dp, top = 7.dp, bottom = 7.dp), - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.spacedBy(12.dp), + CatalogRow( + leading = leading, + label = label, + onToggle = onToggle, + indentLevel = indentLevel, ) { - leading() - Text( - text = label, - style = MaterialTheme.typography.bodyLarge, - maxLines = 1, - overflow = TextOverflow.Ellipsis, - modifier = Modifier.weight(1f), - ) AddPill(added = pinned, onClick = onToggle) } } @@ -838,52 +747,18 @@ private fun ExpandableAvailableRow( } } -/** - * Outlined "Add" that fills to "Added" once pinned — states the action and its result. Both states - * share one Row body (only color/border/tint differ) so the pill keeps a constant height and the rows - * stay aligned whether an item is added or not. - */ +/** Outlined "Add" that fills to "Added" once pinned — states the action and its result. */ @Composable private fun AddPill( added: Boolean, onClick: () -> Unit, ) { - val accent = MaterialTheme.colorScheme.primary - val content = if (added) MaterialTheme.colorScheme.onPrimary else accent - Surface( - shape = CircleShape, - color = if (added) accent else Color.Transparent, - border = if (added) null else BorderStroke(1.dp, accent), - ) { - Row( - modifier = Modifier.clickable(onClick = onClick).padding(horizontal = 14.dp, vertical = 7.dp), - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.spacedBy(4.dp), - ) { - Icon( - symbol = if (added) MaterialSymbols.Check else MaterialSymbols.Add, - contentDescription = null, - modifier = Modifier.size(15.dp), - tint = content, - ) - Text( - text = stringRes(if (added) R.string.bottom_bar_settings_added else R.string.bottom_bar_settings_add), - style = MaterialTheme.typography.labelLarge, - color = content, - ) - } - } -} - -/** A category/destination glyph in a soft accent-tinted circle. */ -@Composable -private fun LeadingGlyph(icon: MaterialSymbol) { - Box( - modifier = Modifier.size(34.dp).clip(CircleShape).background(MaterialTheme.colorScheme.primary.copy(alpha = 0.12f)), - contentAlignment = Alignment.Center, - ) { - Icon(symbol = icon, contentDescription = null, modifier = Modifier.size(19.dp), tint = MaterialTheme.colorScheme.primary) - } + TogglePill( + on = added, + label = stringRes(if (added) R.string.bottom_bar_settings_added else R.string.bottom_bar_settings_add), + icon = if (added) MaterialSymbols.Check else MaterialSymbols.Add, + onClick = onClick, + ) } /** A favorite web-app / nsite / napplet's real favicon in a tinted circle (glyph fallback). */ @@ -902,30 +777,6 @@ private fun FavoriteLeading(app: FavoriteApp) { } } -@Composable -private fun SectionHeader(title: String) { - Text( - text = title, - style = MaterialTheme.typography.labelMedium, - color = MaterialTheme.colorScheme.onSurfaceVariant, - fontWeight = FontWeight.Bold, - modifier = Modifier.padding(start = Size20dp, end = Size20dp, top = 18.dp, bottom = 6.dp), - ) -} - -@Composable -private fun EmptyChildHint( - textRes: Int, - indentLevel: Int = 1, -) { - Text( - text = stringRes(textRes), - style = MaterialTheme.typography.bodySmall, - color = MaterialTheme.colorScheme.onSurfaceVariant, - modifier = Modifier.padding(start = indentPadding(indentLevel), end = 13.dp, top = 6.dp, bottom = 6.dp), - ) -} - private fun categoryIcon(titleRes: Int): MaterialSymbol = when (titleRes) { R.string.bottom_bar_category_main -> MaterialSymbols.Home diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/DrawerSettingsScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/DrawerSettingsScreen.kt new file mode 100644 index 0000000000..9b1d7cdb48 --- /dev/null +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/DrawerSettingsScreen.kt @@ -0,0 +1,263 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.ui.screen.loggedIn.settings + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.verticalScroll +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Scaffold +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.derivedStateOf +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateMapOf +import androidx.compose.runtime.remember +import androidx.compose.ui.Modifier +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.tooling.preview.Preview +import androidx.compose.ui.unit.dp +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.vitorpamplona.amethyst.R +import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols +import com.vitorpamplona.amethyst.ui.navigation.bottombars.NavBarCatalog +import com.vitorpamplona.amethyst.ui.navigation.drawer.DrawerItemVisibility +import com.vitorpamplona.amethyst.ui.navigation.drawer.DrawerSection +import com.vitorpamplona.amethyst.ui.navigation.drawer.DrawerSectionId +import com.vitorpamplona.amethyst.ui.navigation.drawer.DrawerSections +import com.vitorpamplona.amethyst.ui.navigation.drawer.MandatoryDrawerItems +import com.vitorpamplona.amethyst.ui.navigation.navs.EmptyNav +import com.vitorpamplona.amethyst.ui.navigation.navs.INav +import com.vitorpamplona.amethyst.ui.navigation.topbars.TopBarWithBackButton +import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel +import com.vitorpamplona.amethyst.ui.screen.loggedIn.mockAccountViewModel +import com.vitorpamplona.amethyst.ui.stringRes +import com.vitorpamplona.amethyst.ui.theme.ThemeComparisonRow + +@Composable +@Preview(device = "spec:width=2100px,height=2340px,dpi=440") +fun DrawerSettingsScreenPreview() { + ThemeComparisonRow { + DrawerSettingsScreen( + mockAccountViewModel(), + EmptyNav(), + ) + } +} + +@Composable +fun DrawerSettingsScreen( + accountViewModel: AccountViewModel, + nav: INav, +) { + Scaffold( + topBar = { + TopBarWithBackButton(stringRes(id = R.string.drawer_settings), nav) + }, + ) { padding -> + Column(Modifier.padding(padding)) { + DrawerSettingsContent(accountViewModel) + } + } +} + +/** + * Show/hide editor for the side menu's rows. It renders [DrawerSections] directly — the very list the + * drawer renders — so a destination added to a section shows up here with no work, and a row that + * exists here always exists there. + */ +@Composable +fun DrawerSettingsContent(accountViewModel: AccountViewModel) { + // Per-account, synced through the NIP-78 app-specific data event. + val savedHidden by accountViewModel.hiddenDrawerItemsFlow().collectAsStateWithLifecycle() + + // All show/hide logic lives in the holder (unit-tested); the composable only renders and forwards + // events. Each edit republishes the account's NIP-78 settings event. syncFrom re-seeds when the + // saved set changes elsewhere. + // + // Deliberately unkeyed. The holder captures this `accountViewModel` in its persist lambda, so a + // holder that outlived an account switch would write account A's edits to account B. It cannot: + // SetAccountCentricViewModelStore wraps the whole logged-in tree in `key(account.signer.pubKey)`, + // so a switch disposes this composable (and the NavController with it) and re-runs this remember + // against the new account's ViewModel. Keying on accountViewModel here would be a no-op that + // implies the subtree survives a switch — if that ever becomes true, this comment is the bug. + val state = remember { DrawerSettingsState(savedHidden) { accountViewModel.changeHiddenDrawerItems(it) } } + LaunchedEffect(savedHidden) { state.syncFrom(savedHidden) } + + // Sections start collapsed: expanded, they are ~50 rows of scrolling. The header's hidden + // counter is what tells the user which one to open. + val expandedSections = remember { mutableStateMapOf() } + + // derivedStateOf so a toggle that leaves this total unchanged doesn't re-run the whole screen + // body — every SectionCard below reads the same coarse `hidden` state. + val totalHidden by remember { derivedStateOf { state.totalHidden() } } + + Column( + modifier = + Modifier + .fillMaxSize() + .verticalScroll(rememberScrollState()), + ) { + Spacer(Modifier.height(12.dp)) + + SummaryCard(totalHidden) + + RestoreDefaultRow(onClick = { state.restoreDefault() }) + + Spacer(Modifier.height(4.dp)) + + PickerSectionHeader(title = stringRes(R.string.drawer_settings_sections)) + + // A section with no catalog rows has nothing to configure (Create is composer entry points), + // so it isn't listed here even though the drawer renders it. + DrawerSections.forEach { section -> + if (section.items.isEmpty()) return@forEach + SectionCard( + section = section, + state = state, + expanded = expandedSections[section.id] ?: false, + onToggleExpand = { expandedSections[section.id] = !(expandedSections[section.id] ?: false) }, + ) + } + + Spacer(Modifier.height(24.dp)) + } +} + +/** What the setting does and how far from stock the menu currently is. */ +@Composable +private fun SummaryCard(totalHidden: Int) { + PickerHeroCard( + title = stringRes(R.string.drawer_settings_title), + trailing = { + Text( + text = stringRes(R.string.drawer_settings_hidden_count, totalHidden), + style = MaterialTheme.typography.labelMedium, + color = MaterialTheme.colorScheme.primary, + fontWeight = FontWeight.Bold, + ) + }, + ) { + Text( + text = stringRes(R.string.drawer_settings_description), + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } +} + +@Composable +private fun SectionCard( + section: DrawerSection, + state: DrawerSettingsState, + expanded: Boolean, + onToggleExpand: () -> Unit, +) { + // Each card reads the same coarse `hidden` state, so without derivedStateOf a toggle in one + // section would recompose (and re-count) all of them. + val hiddenHere by remember(section) { derivedStateOf { state.hiddenCount(section) } } + + CatalogCard( + icon = section.icon, + title = stringRes(section.titleRes), + expanded = expanded, + onToggleExpand = onToggleExpand, + trailing = { + if (hiddenHere > 0) { + Text( + text = stringRes(R.string.drawer_settings_hidden_count, hiddenHere), + style = MaterialTheme.typography.labelMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + }, + ) { + // Bulk actions: turning ~29 feed rows off one at a time is the kind of chore that makes + // people give up halfway and leave the menu in a worse state than they found it. + if (DrawerItemVisibility.hasHideableRows(section)) { + Row( + modifier = Modifier.fillMaxWidth().padding(start = 6.dp, end = 6.dp), + horizontalArrangement = Arrangement.End, + ) { + TextButton(onClick = { state.showAll(section) }) { + Text(stringRes(R.string.drawer_settings_show_all)) + } + TextButton(onClick = { state.hideAll(section) }) { + Text(stringRes(R.string.drawer_settings_hide_all)) + } + } + } + + section.items.forEach { item -> + val def = NavBarCatalog[item] ?: return@forEach + val mandatory = item in MandatoryDrawerItems + val visible = state.isVisible(item) + CatalogRow( + leading = { LeadingGlyph(def.icon) }, + label = stringRes(def.labelRes), + onToggle = if (mandatory) null else ({ state.toggle(item) }), + ) { + VisibilityPill(visible = visible, mandatory = mandatory, onClick = { state.toggle(item) }) + } + } + } +} + +/** + * Filled "Visible" / outlined "Hidden" — the bottom bar's Add/Added pill, saying what this screen + * says instead. A mandatory row gets a locked "Always on" badge: it reads as deliberately fixed + * rather than as a control that ignores taps. + */ +@Composable +private fun VisibilityPill( + visible: Boolean, + mandatory: Boolean, + onClick: () -> Unit, +) { + TogglePill( + on = visible, + label = + stringRes( + when { + mandatory -> R.string.drawer_settings_always_on + visible -> R.string.drawer_settings_visible + else -> R.string.drawer_settings_hidden + }, + ), + icon = + when { + mandatory -> MaterialSymbols.Lock + visible -> MaterialSymbols.Visibility + else -> MaterialSymbols.VisibilityOff + }, + enabled = !mandatory, + onClick = onClick, + ) +} diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/DrawerSettingsState.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/DrawerSettingsState.kt new file mode 100644 index 0000000000..727058fd4f --- /dev/null +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/DrawerSettingsState.kt @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.ui.screen.loggedIn.settings + +import androidx.compose.runtime.Stable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.setValue +import com.vitorpamplona.amethyst.ui.navigation.bottombars.NavBarItem +import com.vitorpamplona.amethyst.ui.navigation.drawer.DrawerItemVisibility +import com.vitorpamplona.amethyst.ui.navigation.drawer.DrawerSection + +/** + * State holder for the Side Menu settings screen: owns the set of switched-off drawer rows and the + * show / hide / restore-default operations, so the composable only renders and forwards events. + * + * The rules themselves live in [DrawerItemVisibility] (pure, unit-tested); this adds only the Compose + * state and the write-through to the account's synced settings. Unlike the bottom bar there is no + * transient/commit split — a toggle is a single discrete edit, not a drag, so every change persists + * immediately. + * + * No sanitizing here: every value in is either already sanitized by the persistence layer or produced + * by a [DrawerItemVisibility] operation that can't introduce a mandatory row, and the write side + * sanitizes again anyway. One authority, not three. + */ +@Stable +class DrawerSettingsState( + initial: Set, + private val persist: (Set) -> Unit, +) { + var hidden by mutableStateOf(initial) + private set + + fun isVisible(item: NavBarItem): Boolean = DrawerItemVisibility.isVisible(hidden, item) + + fun toggle(item: NavBarItem) = update(DrawerItemVisibility.toggle(hidden, item)) + + fun hiddenCount(section: DrawerSection): Int = DrawerItemVisibility.hiddenCount(section, hidden) + + fun totalHidden(): Int = DrawerItemVisibility.totalHidden(hidden) + + fun showAll(section: DrawerSection) = update(DrawerItemVisibility.showAll(hidden, section)) + + fun hideAll(section: DrawerSection) = update(DrawerItemVisibility.hideAll(hidden, section)) + + /** Back to the stock drawer: nothing hidden. */ + fun restoreDefault() = update(emptySet()) + + /** + * Re-seed from an external change (the saved settings flow emitted) without re-persisting. A no-op + * when equal, so the echo of our own [persist] doesn't fight an in-progress edit. + */ + fun syncFrom(items: Set) { + if (items != hidden) hidden = items + } + + private fun update(newHidden: Set) { + if (newHidden == hidden) return + hidden = newHidden + persist(newHidden) + } +} diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/NavPickerUi.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/NavPickerUi.kt new file mode 100644 index 0000000000..0793edcf81 --- /dev/null +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/NavPickerUi.kt @@ -0,0 +1,330 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.ui.screen.loggedIn.settings + +import androidx.compose.animation.AnimatedVisibility +import androidx.compose.animation.expandVertically +import androidx.compose.animation.fadeIn +import androidx.compose.animation.fadeOut +import androidx.compose.animation.shrinkVertically +import androidx.compose.foundation.BorderStroke +import androidx.compose.foundation.background +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp +import com.vitorpamplona.amethyst.R +import com.vitorpamplona.amethyst.commons.icons.symbols.Icon +import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbol +import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols +import com.vitorpamplona.amethyst.ui.stringRes +import com.vitorpamplona.amethyst.ui.theme.SimpleImage35Modifier +import com.vitorpamplona.amethyst.ui.theme.Size10dp +import com.vitorpamplona.amethyst.ui.theme.Size12dp +import com.vitorpamplona.amethyst.ui.theme.Size13dp +import com.vitorpamplona.amethyst.ui.theme.Size14dp +import com.vitorpamplona.amethyst.ui.theme.Size15Modifier +import com.vitorpamplona.amethyst.ui.theme.Size18dp +import com.vitorpamplona.amethyst.ui.theme.Size19Modifier +import com.vitorpamplona.amethyst.ui.theme.Size20Modifier +import com.vitorpamplona.amethyst.ui.theme.Size20dp +import com.vitorpamplona.amethyst.ui.theme.Size22dp +import com.vitorpamplona.amethyst.ui.theme.Size24Modifier +import com.vitorpamplona.amethyst.ui.theme.Size24dp +import com.vitorpamplona.amethyst.ui.theme.Size34dp +import com.vitorpamplona.amethyst.ui.theme.Size40dp +import com.vitorpamplona.amethyst.ui.theme.Size6dp + +/** + * The shared visual language of the navigation-configuration screens — the Bottom Navigation Bar + * picker and the Side Menu picker. Both present the same shape (collapsible cards of catalog rows, + * each row a glyph + label + a pill stating its current state), so the pieces live here once and + * each screen supplies only its own semantics: the bottom bar pins and reorders entries, the side + * menu switches rows on and off. + * + * [SectionExpand]/[SectionCollapse] reveal expandable sections by unrolling straight down from the + * top edge (the default AnimatedVisibility enter also expands horizontally from the bottom-end, + * which reads as a diagonal slide from the top-left). + */ +val SectionExpand = expandVertically(expandFrom = Alignment.Top) + fadeIn() +val SectionCollapse = shrinkVertically(shrinkTowards = Alignment.Top) + fadeOut() + +/** + * Start padding per nesting depth: 0 = a top-level catalog row, 1 = an item under an expandable + * category (a favorite, or a relay/community "server" row), 2 = a room nested under its server (a + * NIP-29 group under its relay, or a Concord channel under its community). + */ +fun indentPadding(level: Int) = + when (level) { + 0 -> Size13dp + 1 -> Size24dp + else -> Size40dp + } + +@Composable +fun PickerSectionHeader(title: String) { + Text( + text = title, + style = MaterialTheme.typography.labelMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + fontWeight = FontWeight.Bold, + modifier = Modifier.padding(start = Size20dp, end = Size20dp, top = Size18dp, bottom = Size6dp), + ) +} + +/** A category/destination glyph in a soft accent-tinted circle. */ +@Composable +fun LeadingGlyph(icon: MaterialSymbol) { + Box( + modifier = SimpleImage35Modifier.background(MaterialTheme.colorScheme.primary.copy(alpha = 0.12f)), + contentAlignment = Alignment.Center, + ) { + Icon(symbol = icon, contentDescription = null, modifier = Size19Modifier, tint = MaterialTheme.colorScheme.primary) + } +} + +@Composable +fun EmptyChildHint( + textRes: Int, + indentLevel: Int = 1, +) { + Text( + text = stringRes(textRes), + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.padding(start = indentPadding(indentLevel), end = Size13dp, top = Size6dp, bottom = Size6dp), + ) +} + +/** + * One catalog row: leading visual, label, and a caller-supplied [trailing] state control. Tapping + * anywhere on the row runs [onToggle]; pass null for a row whose state can't change (a mandatory + * side-menu item), which also drops the ripple so the row doesn't advertise an action it won't take. + */ +@Composable +fun CatalogRow( + leading: @Composable () -> Unit, + label: String, + onToggle: (() -> Unit)?, + indentLevel: Int = 0, + trailing: @Composable () -> Unit, +) { + Row( + modifier = + Modifier + .fillMaxWidth() + .let { if (onToggle != null) it.clickable(onClick = onToggle) else it } + .padding(start = indentPadding(indentLevel), end = Size13dp, top = 7.dp, bottom = 7.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(Size12dp), + ) { + leading() + Text( + text = label, + style = MaterialTheme.typography.bodyLarge, + maxLines = 1, + overflow = TextOverflow.Ellipsis, + modifier = Modifier.weight(1f), + ) + trailing() + } +} + +/** + * The state pill at the end of a catalog row: outlined in the "off" state, filled in the "on" state — + * so it states both the current state and, by contrast, that it can be changed. Both states share one + * Row body (only color/border/tint differ) so the pill keeps a constant height and rows stay aligned. + * + * [enabled] false renders the pill as a locked, non-interactive badge — used for a row the user isn't + * allowed to switch off. + */ +@Composable +fun TogglePill( + on: Boolean, + label: String, + icon: MaterialSymbol, + enabled: Boolean = true, + onClick: () -> Unit, +) { + val accent = MaterialTheme.colorScheme.primary + val container = if (enabled) accent else MaterialTheme.colorScheme.surfaceVariant + val content = + when { + !enabled -> MaterialTheme.colorScheme.onSurfaceVariant + on -> MaterialTheme.colorScheme.onPrimary + else -> accent + } + Surface( + shape = CircleShape, + color = if (on) container else Color.Transparent, + border = if (on) null else BorderStroke(1.dp, content), + ) { + Row( + modifier = + Modifier + .let { if (enabled) it.clickable(onClick = onClick) else it } + .padding(horizontal = Size14dp, vertical = 7.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(4.dp), + ) { + Icon( + symbol = icon, + contentDescription = null, + modifier = Size15Modifier, + tint = content, + ) + Text( + text = label, + style = MaterialTheme.typography.labelLarge, + color = content, + ) + } + } +} + +/** + * A collapsible card holding catalog rows. [trailing] renders between the title and the chevron — + * the side menu puts its "n hidden" counter there; the bottom bar leaves it empty. + */ +@Composable +fun CatalogCard( + icon: MaterialSymbol, + title: String, + expanded: Boolean, + onToggleExpand: () -> Unit, + trailing: @Composable () -> Unit = {}, + content: @Composable () -> Unit, +) { + Surface( + shape = RoundedCornerShape(16.dp), + color = MaterialTheme.colorScheme.surface, + border = BorderStroke(1.dp, MaterialTheme.colorScheme.outlineVariant), + modifier = Modifier.fillMaxWidth().padding(horizontal = Size20dp, vertical = 5.dp), + ) { + Column { + Row( + modifier = Modifier.fillMaxWidth().clickable(onClick = onToggleExpand).padding(Size13dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(Size12dp), + ) { + Box( + modifier = + Modifier + .size(Size34dp) + .clip(RoundedCornerShape(11.dp)) + .background(MaterialTheme.colorScheme.surfaceVariant), + contentAlignment = Alignment.Center, + ) { + Icon( + symbol = icon, + contentDescription = null, + modifier = Size20Modifier, + tint = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + Text( + text = title, + style = MaterialTheme.typography.titleSmall, + modifier = Modifier.weight(1f), + ) + trailing() + Icon( + symbol = if (expanded) MaterialSymbols.ExpandLess else MaterialSymbols.ExpandMore, + contentDescription = null, + modifier = Size24Modifier, + tint = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + + AnimatedVisibility(visible = expanded, enter = SectionExpand, exit = SectionCollapse) { + Column(Modifier.padding(bottom = Size6dp)) { content() } + } + } + } +} + +/** + * The accent-tinted card each picker opens with: a bold title, an optional [trailing] status, and a + * body. The bottom bar puts its editable preview bar in the body; the side menu puts its description. + */ +@Composable +fun PickerHeroCard( + title: String, + trailing: @Composable () -> Unit = {}, + content: @Composable () -> Unit, +) { + val accent = MaterialTheme.colorScheme.primary + Surface( + shape = RoundedCornerShape(Size22dp), + color = accent.copy(alpha = 0.07f), + border = BorderStroke(1.dp, accent.copy(alpha = 0.22f)), + modifier = Modifier.fillMaxWidth().padding(horizontal = Size20dp, vertical = 4.dp), + ) { + Column(Modifier.padding(Size14dp)) { + Row( + modifier = Modifier.fillMaxWidth().padding(bottom = Size10dp), + horizontalArrangement = Arrangement.SpaceBetween, + verticalAlignment = Alignment.CenterVertically, + ) { + Text( + text = title, + style = MaterialTheme.typography.labelMedium, + color = accent, + fontWeight = FontWeight.Bold, + ) + trailing() + } + + content() + } + } +} + +/** The end-aligned "Restore Default" action both pickers put under their hero card. */ +@Composable +fun RestoreDefaultRow(onClick: () -> Unit) { + Row( + modifier = Modifier.fillMaxWidth().padding(horizontal = Size20dp), + horizontalArrangement = Arrangement.End, + ) { + TextButton(onClick = onClick) { + Text(stringRes(R.string.bottom_bar_settings_restore_default)) + } + } +} diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/SettingsCatalogBuilder.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/SettingsCatalogBuilder.kt index d9a2d09b87..d164a3deba 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/SettingsCatalogBuilder.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/SettingsCatalogBuilder.kt @@ -72,6 +72,7 @@ fun buildSettingsCatalog( symEntry(R.string.reactions_settings, MaterialSymbols.ThumbUp, R.string.reactions_settings_search_keywords, Route.ReactionsSettings), symEntry(R.string.messages_settings, MaterialSymbols.Mail, R.string.messages_settings_search_keywords, Route.MessagesSettings), symEntry(R.string.bottom_bar_settings, MaterialSymbols.Dashboard, R.string.bottom_bar_search_keywords, Route.BottomBarSettings), + symEntry(R.string.drawer_settings, MaterialSymbols.AutoMirrored.ViewList, R.string.drawer_search_keywords, Route.DrawerSettings), symEntry(R.string.video_player_settings, MaterialSymbols.VideoSettings, R.string.video_player_search_keywords, Route.VideoPlayerSettings), symEntry(R.string.audio_visualizer_settings, MaterialSymbols.MusicNote, R.string.audio_visualizer_search_keywords, Route.AudioVisualizerSettings), symEntry(R.string.favorite_dvms_title, MaterialSymbols.AutoAwesome, R.string.favorite_dvms_search_keywords, Route.EditFavoriteAlgoFeeds), diff --git a/amethyst/src/main/res/values/strings.xml b/amethyst/src/main/res/values/strings.xml index 2221700aea..4f0003f2ba 100644 --- a/amethyst/src/main/res/values/strings.xml +++ b/amethyst/src/main/res/values/strings.xml @@ -2378,6 +2378,7 @@ draft, posting, editor, auto-save, signature, proof of work, pow, mining, nip-13 emoji, reactions, like navigation, tabs, nav bar + side menu, drawer, hamburger, sections, hide, show tabs, feeds, threads, conversations profile, layout nsec, private key, seed, mnemonic, export @@ -3512,6 +3513,16 @@ Feeds Apps & Web Other + Side Menu + Your side menu + Open a section and switch off the rows you never use. Settings always stays visible, so you can always get back here. Section order is fixed. + Sections + %1$d hidden + Visible + Hidden + Always on + Show all + Hide all Home Tabs Pick which tabs appear on the Home screen. When only one tab is active the tab bar is hidden. Everything diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/model/preferences/DrawerPersistenceTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/model/preferences/DrawerPersistenceTest.kt new file mode 100644 index 0000000000..d686cadf79 --- /dev/null +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/model/preferences/DrawerPersistenceTest.kt @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.model.preferences + +import com.vitorpamplona.amethyst.model.AccountNavigationPreferencesInternal +import com.vitorpamplona.amethyst.ui.navigation.bottombars.NavBarItem +import com.vitorpamplona.amethyst.ui.navigation.bottombars.navBarItemsFromNames +import com.vitorpamplona.amethyst.ui.navigation.bottombars.toNames +import com.vitorpamplona.quartz.nip01Core.core.JsonMapper +import org.junit.Assert.assertEquals +import org.junit.Test + +/** + * Locks the per-account side-menu persistence. The hidden rows ride along in the same NIP-78 + * app-specific data blob as the bottom bar, so every account keeps its own menu and it syncs across + * the user's devices. + */ +class DrawerPersistenceTest { + @Test + fun defaultIsAnUntouchedMenu() { + val decoded = JsonMapper.fromJson(JsonMapper.toJson(AccountNavigationPreferencesInternal())) + + assertEquals(emptyList(), decoded.hiddenDrawerItems) + } + + @Test + fun blobWrittenBeforeTheFieldExistedDecodesToAnUntouchedMenu() { + // Every existing account is in this state, and so is every account that never opens the + // screen — which is exactly why the preference stores hidden rows rather than visible ones. + val decoded = JsonMapper.fromJson("{}") + + assertEquals(emptyList(), decoded.hiddenDrawerItems) + } + + @Test + fun hiddenRowsRoundTripThroughTheSyncedSettingsBlob() { + val hidden = setOf(NavBarItem.DRAFTS, NavBarItem.BADGES) + + val json = JsonMapper.toJson(AccountNavigationPreferencesInternal(hiddenDrawerItems = hidden.toNames())) + val decoded = JsonMapper.fromJson(json) + + assertEquals(hidden, navBarItemsFromNames(decoded.hiddenDrawerItems)) + } + + @Test + fun serializedFormIsDeterministic() { + // Two equal sets must produce byte-identical JSON, or a republish that changed nothing would + // still look like a change and churn the account's NIP-78 event. + val a = JsonMapper.toJson(AccountNavigationPreferencesInternal(hiddenDrawerItems = setOf(NavBarItem.DRAFTS, NavBarItem.BADGES).toNames())) + val b = JsonMapper.toJson(AccountNavigationPreferencesInternal(hiddenDrawerItems = setOf(NavBarItem.BADGES, NavBarItem.DRAFTS).toNames())) + + assertEquals(a, b) + } + + @Test + fun anIdFromANewerClientIsDroppedInsteadOfFailingTheWholeBlob() { + // The names are stored as strings precisely for this: decoding them as the enum would throw + // and take every other synced setting down with it. + val json = """{"hiddenDrawerItems":["DRAFTS","SOME_SCREEN_FROM_THE_FUTURE"]}""" + + val decoded = JsonMapper.fromJson(json) + + assertEquals(setOf(NavBarItem.DRAFTS), navBarItemsFromNames(decoded.hiddenDrawerItems)) + } +} diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/navigation/DrawerItemVisibilityTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/navigation/DrawerItemVisibilityTest.kt new file mode 100644 index 0000000000..d5fcf5f47a --- /dev/null +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/navigation/DrawerItemVisibilityTest.kt @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.navigation + +import com.vitorpamplona.amethyst.ui.navigation.bottombars.NavBarItem +import com.vitorpamplona.amethyst.ui.navigation.drawer.DrawerItemVisibility +import com.vitorpamplona.amethyst.ui.navigation.drawer.DrawerSectionId +import com.vitorpamplona.amethyst.ui.navigation.drawer.drawerSection +import com.vitorpamplona.amethyst.ui.screen.loggedIn.settings.DrawerSettingsState +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * The show/hide rules behind the Side Menu settings screen, exercised without the UI. + * + * The preference stores what is *hidden*, so the interesting cases are the empty set (a stock drawer, + * and the state every new install and every newly shipped destination starts in) and the mandatory + * rows, which no code path may switch off. + */ +class DrawerItemVisibilityTest { + private val you = drawerSection(DrawerSectionId.YOU) + private val system = drawerSection(DrawerSectionId.SYSTEM) + + @Test + fun nothingHiddenMeansEverythingVisible() { + val visible = DrawerItemVisibility.visibleItems(you, emptySet()) + + assertEquals(you.items, visible) + assertEquals(0, DrawerItemVisibility.hiddenCount(you, emptySet())) + } + + @Test + fun toggleHidesThenShows() { + val once = DrawerItemVisibility.toggle(emptySet(), NavBarItem.DRAFTS) + assertEquals(setOf(NavBarItem.DRAFTS), once) + assertFalse(DrawerItemVisibility.isVisible(once, NavBarItem.DRAFTS)) + + val twice = DrawerItemVisibility.toggle(once, NavBarItem.DRAFTS) + assertEquals(emptySet(), twice) + assertTrue(DrawerItemVisibility.isVisible(twice, NavBarItem.DRAFTS)) + } + + @Test + fun aHiddenRowDropsOutOfItsSectionKeepingTheOrderOfTheRest() { + val hidden = setOf(NavBarItem.DRAFTS) + val visible = DrawerItemVisibility.visibleItems(you, hidden) + + assertEquals(you.items.filter { it != NavBarItem.DRAFTS }, visible) + assertEquals(1, DrawerItemVisibility.hiddenCount(you, hidden)) + } + + @Test + fun settingsCannotBeHidden() { + // The escape hatch: hiding Settings would leave no route back to the screen that hides rows. + assertEquals(emptySet(), DrawerItemVisibility.toggle(emptySet(), NavBarItem.SETTINGS)) + assertTrue(DrawerItemVisibility.isVisible(setOf(NavBarItem.SETTINGS), NavBarItem.SETTINGS)) + } + + @Test + fun sanitizeStripsMandatoryItemsSyncedFromElsewhere() { + // Another client (or an older build) could put Settings in the set; reading it back must not + // strand the row as hidden-yet-unhideable. + val sanitized = DrawerItemVisibility.sanitize(setOf(NavBarItem.SETTINGS, NavBarItem.DRAFTS)) + + assertEquals(setOf(NavBarItem.DRAFTS), sanitized) + } + + @Test + fun sanitizeKeepsIdsThisDeviceDoesNotRender() { + // Favorite Apps is gated off below API 30. Editing the menu on such a device must not clear + // the choice the same account made on a newer one. + val sanitized = DrawerItemVisibility.sanitize(setOf(NavBarItem.FAVORITE_APPS)) + + assertEquals(setOf(NavBarItem.FAVORITE_APPS), sanitized) + } + + @Test + fun hideAllLeavesTheMandatoryRowsOfASection() { + val hidden = DrawerItemVisibility.hideAll(emptySet(), system) + + assertTrue(DrawerItemVisibility.isVisible(hidden, NavBarItem.SETTINGS)) + assertEquals(0, DrawerItemVisibility.hiddenCount(system, hidden)) + } + + @Test + fun aSectionOfOnlyMandatoryRowsHasNothingToHide() { + // What gates the section's bulk Show all / Hide all actions. + assertFalse(DrawerItemVisibility.hasHideableRows(system)) + assertTrue(DrawerItemVisibility.hasHideableRows(you)) + } + + @Test + fun hideAllThenShowAllRoundTripsASection() { + val hidden = DrawerItemVisibility.hideAll(emptySet(), you) + assertEquals(you.items.size, DrawerItemVisibility.hiddenCount(you, hidden)) + + val shown = DrawerItemVisibility.showAll(hidden, you) + assertEquals(emptySet(), shown) + } + + @Test + fun showAllOnlyTouchesItsOwnSection() { + val hidden = DrawerItemVisibility.hideAll(DrawerItemVisibility.hideAll(emptySet(), you), system) + + val shown = DrawerItemVisibility.showAll(hidden, system) + + assertEquals(you.items.size, DrawerItemVisibility.hiddenCount(you, shown)) + } + + @Test + fun stateHolderPersistsEveryEditAndRestoresDefaults() { + val saved = mutableListOf>() + val state = DrawerSettingsState(emptySet()) { saved.add(it) } + + state.toggle(NavBarItem.DRAFTS) + state.toggle(NavBarItem.BOOKMARKS) + + assertEquals(listOf(setOf(NavBarItem.DRAFTS), setOf(NavBarItem.DRAFTS, NavBarItem.BOOKMARKS)), saved) + assertEquals(2, state.totalHidden()) + + state.restoreDefault() + + assertEquals(emptySet(), state.hidden) + assertEquals(0, state.totalHidden()) + assertEquals(emptySet(), saved.last()) + } + + @Test + fun stateHolderDoesNotRepublishANoOpEdit() { + // Tapping a mandatory row must not republish the account's NIP-78 settings event. + val saved = mutableListOf>() + val state = DrawerSettingsState(emptySet()) { saved.add(it) } + + state.toggle(NavBarItem.SETTINGS) + state.restoreDefault() + + assertTrue(saved.isEmpty()) + } +} diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/navigation/DrawerSectionsTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/navigation/DrawerSectionsTest.kt new file mode 100644 index 0000000000..3cf1687a83 --- /dev/null +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/navigation/DrawerSectionsTest.kt @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.navigation + +import com.vitorpamplona.amethyst.ui.navigation.bottombars.NavBarCatalog +import com.vitorpamplona.amethyst.ui.navigation.drawer.DrawerSectionId +import com.vitorpamplona.amethyst.ui.navigation.drawer.DrawerSections +import com.vitorpamplona.amethyst.ui.navigation.drawer.MandatoryDrawerItems +import com.vitorpamplona.amethyst.ui.navigation.drawer.SdkGatedDrawerItems +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * The drawer and its settings screen both render [DrawerSections], so a destination missing from + * every section is invisible in both places at once — no compiler error, no crash, just a screen + * nobody can reach from the menu. These pin the invariants that keep that from happening quietly. + */ +class DrawerSectionsTest { + @Test + fun everyCatalogItemAppearsInADrawerSection() { + val sectioned = DrawerSections.flatMap { it.items }.toSet() + val missing = NavBarCatalog.keys - sectioned + + assertEquals( + "a catalog destination is in no drawer section — add it to one in NavBarItem.kt, " + + "or to SdkGatedDrawerItems with the reason it can't be there", + emptySet(), + missing - SdkGatedDrawerItems, + ) + } + + @Test + fun noItemIsListedInTwoSections() { + val sectioned = DrawerSections.flatMap { it.items } + + assertEquals("an item is listed in more than one drawer section", sectioned.size, sectioned.toSet().size) + } + + @Test + fun everySectionedItemResolvesInTheCatalog() { + // The drawer looks each id up in NavBarCatalog and skips misses, so an id with no catalog + // entry would silently render nothing while still occupying a row in the settings screen. + DrawerSections.forEach { section -> + section.items.forEach { item -> + assertTrue("$item has no NavBarCatalog entry", NavBarCatalog.containsKey(item)) + } + } + } + + @Test + fun sectionsAreOrderedWithCreateBetweenFeedsAndSystem() { + // The drawer renders DrawerSections in order, so this list *is* the menu's layout. Create sits + // between the feeds and System, and is the one section with nothing configurable in it. + assertEquals( + listOf( + DrawerSectionId.YOU, + DrawerSectionId.NAVIGATE, + DrawerSectionId.FEEDS, + DrawerSectionId.CREATE, + DrawerSectionId.SYSTEM, + ), + DrawerSections.map { it.id }, + ) + assertEquals(emptyList(), DrawerSections.first { it.id == DrawerSectionId.CREATE }.items) + } + + @Test + fun everySectionHasItsOwnId() { + val ids = DrawerSections.map { it.id } + + assertEquals("two sections share a DrawerSectionId", ids.size, ids.toSet().size) + } + + @Test + fun mandatoryItemsAreActuallyRenderedByASection() { + // A mandatory item that no section renders would be unhideable *and* invisible — the worst + // of both. Settings is mandatory precisely because it is the way back to this configuration. + val sectioned = DrawerSections.flatMap { it.items }.toSet() + + assertTrue("a mandatory drawer item is in no section", sectioned.containsAll(MandatoryDrawerItems)) + } +} From 93fe3ac727b2cf41ead25e85a954c5d784fb6170 Mon Sep 17 00:00:00 2001 From: davotoula Date: Sun, 2 Aug 2026 09:16:35 +0200 Subject: [PATCH 042/132] Code review: - fold the pickers onto shared row/expand-state UI --- .../ui/navigation/bottombars/NavBarItem.kt | 7 +++ .../ui/navigation/drawer/DrawerContent.kt | 3 +- .../ui/navigation/drawer/DrawerSections.kt | 12 +++-- .../settings/BottomBarSettingsScreen.kt | 50 ++++++------------- .../loggedIn/settings/DrawerSettingsScreen.kt | 35 ++++++------- .../screen/loggedIn/settings/NavPickerUi.kt | 24 ++++++++- .../navigation/DrawerItemVisibilityTest.kt | 6 +-- .../amethyst/navigation/DrawerSectionsTest.kt | 16 ++++++ 8 files changed, 86 insertions(+), 67 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/NavBarItem.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/NavBarItem.kt index 2209c1bec6..5959cb5c25 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/NavBarItem.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/NavBarItem.kt @@ -463,6 +463,7 @@ val DefaultBottomBarEntries: List = DefaultBottomBarItems.map { */ data class NavBarCategory( val titleRes: Int, + val icon: MaterialSymbol, val items: List, ) @@ -475,6 +476,7 @@ val BottomBarCategories: List = listOf( NavBarCategory( R.string.bottom_bar_category_main, + MaterialSymbols.Home, listOf( NavBarItem.HOME, NavBarItem.MESSAGES, @@ -485,6 +487,7 @@ val BottomBarCategories: List = ), NavBarCategory( R.string.bottom_bar_category_chats, + MaterialSymbols.Group, listOf( NavBarItem.PUBLIC_CHATS, NavBarItem.RELAY_GROUPS, @@ -494,6 +497,7 @@ val BottomBarCategories: List = ), NavBarCategory( R.string.bottom_bar_category_you, + MaterialSymbols.AccountCircle, listOf( NavBarItem.PROFILE, NavBarItem.MY_LISTS, @@ -511,6 +515,7 @@ val BottomBarCategories: List = ), NavBarCategory( R.string.bottom_bar_category_feeds, + MaterialSymbols.Subscriptions, listOf( NavBarItem.ARTICLES, NavBarItem.LONGS, @@ -537,6 +542,7 @@ val BottomBarCategories: List = ), NavBarCategory( R.string.bottom_bar_category_apps, + MaterialSymbols.Apps, listOf( NavBarItem.BROWSER, NavBarItem.FAVORITE_APPS, @@ -547,6 +553,7 @@ val BottomBarCategories: List = ), NavBarCategory( R.string.bottom_bar_category_other, + MaterialSymbols.Settings, listOf( NavBarItem.SETTINGS, ), diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerContent.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerContent.kt index a179e627c8..6009583b38 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerContent.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerContent.kt @@ -648,8 +648,7 @@ fun CatalogSection( val onBackground = MaterialTheme.colorScheme.onBackground val visible = remember(section, hidden) { DrawerItemVisibility.visibleItems(section, hidden) } - val hasFixedRows = section.id == DrawerSectionId.CREATE || section.id == DrawerSectionId.SYSTEM - if (visible.isEmpty() && !hasFixedRows) return + if (visible.isEmpty() && !section.hasFixedRows) return CollapsibleSection(title = section.titleRes) { when (section.id) { diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerSections.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerSections.kt index 4e591f71f9..2cf323c39b 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerSections.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/drawer/DrawerSections.kt @@ -47,6 +47,12 @@ data class DrawerSection( val titleRes: Int, val icon: MaterialSymbol, val items: List, + /** + * True for a section that renders rows of its own on top of its catalog items (see [CatalogSection]). + * Such a section stays in the drawer even with every catalog row switched off, and — since a fixed + * row is not a catalog destination — it never appears in the Side Menu settings screen's counts. + */ + val hasFixedRows: Boolean = false, ) /** @@ -133,12 +139,10 @@ val DrawerSections: List = DrawerSection(DrawerSectionId.YOU, R.string.drawer_section_you, MaterialSymbols.AccountCircle, DrawerYouItems), DrawerSection(DrawerSectionId.NAVIGATE, R.string.drawer_section_navigate, MaterialSymbols.Home, DrawerNavigateItems), DrawerSection(DrawerSectionId.FEEDS, R.string.drawer_section_feeds, MaterialSymbols.Subscriptions, DrawerFeedsItems), - DrawerSection(DrawerSectionId.CREATE, R.string.drawer_section_create, MaterialSymbols.Edit, emptyList()), - DrawerSection(DrawerSectionId.SYSTEM, R.string.drawer_section_system, MaterialSymbols.Settings, listOf(NavBarItem.SETTINGS)), + DrawerSection(DrawerSectionId.CREATE, R.string.drawer_section_create, MaterialSymbols.Edit, emptyList(), hasFixedRows = true), + DrawerSection(DrawerSectionId.SYSTEM, R.string.drawer_section_system, MaterialSymbols.Settings, listOf(NavBarItem.SETTINGS), hasFixedRows = true), ) -fun drawerSection(id: DrawerSectionId): DrawerSection = DrawerSections.first { it.id == id } - /** * Catalog ids deliberately absent from every [DrawerSections] list, with the reason. Only Favorite * Apps qualifies: [DrawerFeedsItems] gates it on API 30+ (its inline tabs need SurfaceControlViewHost), diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/BottomBarSettingsScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/BottomBarSettingsScreen.kt index f23ace6d4b..b3d24014fa 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/BottomBarSettingsScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/BottomBarSettingsScreen.kt @@ -54,7 +54,6 @@ import androidx.compose.runtime.mutableIntStateOf import androidx.compose.runtime.mutableStateMapOf import androidx.compose.runtime.remember import androidx.compose.runtime.setValue -import androidx.compose.runtime.snapshots.SnapshotStateMap import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip @@ -91,6 +90,7 @@ import com.vitorpamplona.amethyst.ui.navigation.topbars.TopBarWithBackButton import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel import com.vitorpamplona.amethyst.ui.screen.loggedIn.mockAccountViewModel import com.vitorpamplona.amethyst.ui.stringRes +import com.vitorpamplona.amethyst.ui.theme.Size22Modifier import com.vitorpamplona.amethyst.ui.theme.ThemeComparisonRow import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEntry import com.vitorpamplona.quartz.nip51Lists.simpleGroupList.GroupTag @@ -158,8 +158,8 @@ fun BottomBarSettingsContent(accountViewModel: AccountViewModel) { val pinned = state.pinned val pinnedKeys = remember(pinned) { state.pinnedKeys() } - val expandedCategories = remember { mutableStateMapOf() } - val expandedItems = remember { mutableStateMapOf() } + val expandedCategories = rememberExpandedKeys() + val expandedItems = rememberExpandedKeys() Column( modifier = @@ -183,8 +183,8 @@ fun BottomBarSettingsContent(accountViewModel: AccountViewModel) { CategoryCard( category = category, pinnedKeys = pinnedKeys, - expanded = expandedCategories[category.titleRes] ?: false, - onToggleExpand = { expandedCategories[category.titleRes] = !(expandedCategories[category.titleRes] ?: false) }, + expanded = expandedCategories.isExpanded(category.titleRes), + onToggleExpand = { expandedCategories.toggle(category.titleRes) }, expandedItems = expandedItems, accountViewModel = accountViewModel, onTogglePin = state::togglePin, @@ -437,12 +437,12 @@ private fun CategoryCard( pinnedKeys: Set, expanded: Boolean, onToggleExpand: () -> Unit, - expandedItems: SnapshotStateMap, + expandedItems: ExpandedKeys, accountViewModel: AccountViewModel, onTogglePin: (BottomBarEntry) -> Unit, ) { CatalogCard( - icon = categoryIcon(category.titleRes), + icon = category.icon, title = stringRes(category.titleRes), expanded = expanded, onToggleExpand = onToggleExpand, @@ -455,9 +455,9 @@ private fun CategoryCard( icon = def.icon, label = stringRes(def.labelRes), pinned = entry.stableKey in pinnedKeys, - expanded = expandedItems[item] ?: false, + expanded = expandedItems.isExpanded(item), onTogglePin = { onTogglePin(entry) }, - onToggleExpand = { expandedItems[item] = !(expandedItems[item] ?: false) }, + onToggleExpand = { expandedItems.toggle(item) }, ) { PickerChildren(item, pinnedKeys, accountViewModel, onTogglePin) } @@ -717,27 +717,15 @@ private fun ExpandableAvailableRow( onToggleExpand: () -> Unit, children: @Composable () -> Unit, ) { - Row( - modifier = - Modifier - .fillMaxWidth() - .clickable(onClick = onToggleExpand) - .padding(start = 13.dp, end = 13.dp, top = 7.dp, bottom = 7.dp), - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.spacedBy(12.dp), + CatalogRow( + leading = { LeadingGlyph(icon) }, + label = label, + onToggle = onToggleExpand, ) { - LeadingGlyph(icon) - Text( - text = label, - style = MaterialTheme.typography.bodyLarge, - maxLines = 1, - overflow = TextOverflow.Ellipsis, - modifier = Modifier.weight(1f), - ) Icon( symbol = if (expanded) MaterialSymbols.ExpandLess else MaterialSymbols.ExpandMore, contentDescription = stringRes(R.string.bottom_bar_settings_expand), - modifier = Modifier.size(22.dp), + modifier = Size22Modifier, tint = MaterialTheme.colorScheme.onSurfaceVariant, ) AddPill(added = pinned, onClick = onTogglePin) @@ -777,16 +765,6 @@ private fun FavoriteLeading(app: FavoriteApp) { } } -private fun categoryIcon(titleRes: Int): MaterialSymbol = - when (titleRes) { - R.string.bottom_bar_category_main -> MaterialSymbols.Home - R.string.bottom_bar_category_chats -> MaterialSymbols.Group - R.string.bottom_bar_category_you -> MaterialSymbols.AccountCircle - R.string.bottom_bar_category_feeds -> MaterialSymbols.Subscriptions - R.string.bottom_bar_category_apps -> MaterialSymbols.Apps - else -> MaterialSymbols.Settings - } - // ------------------------------------------------------------------------------------------------ // Leading/label resolution for a pinned entry (built-in glyph, favorite icon, or group avatar). // Computed once so a group's channel is subscribed at most once per row. diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/DrawerSettingsScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/DrawerSettingsScreen.kt index 9b1d7cdb48..9a32cc8a11 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/DrawerSettingsScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/DrawerSettingsScreen.kt @@ -38,7 +38,6 @@ import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.derivedStateOf import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableStateMapOf import androidx.compose.runtime.remember import androidx.compose.ui.Modifier import androidx.compose.ui.text.font.FontWeight @@ -113,11 +112,9 @@ fun DrawerSettingsContent(accountViewModel: AccountViewModel) { // Sections start collapsed: expanded, they are ~50 rows of scrolling. The header's hidden // counter is what tells the user which one to open. - val expandedSections = remember { mutableStateMapOf() } + val expandedSections = rememberExpandedKeys() - // derivedStateOf so a toggle that leaves this total unchanged doesn't re-run the whole screen - // body — every SectionCard below reads the same coarse `hidden` state. - val totalHidden by remember { derivedStateOf { state.totalHidden() } } + val totalHidden = state.totalHidden() Column( modifier = @@ -142,8 +139,8 @@ fun DrawerSettingsContent(accountViewModel: AccountViewModel) { SectionCard( section = section, state = state, - expanded = expandedSections[section.id] ?: false, - onToggleExpand = { expandedSections[section.id] = !(expandedSections[section.id] ?: false) }, + expanded = expandedSections.isExpanded(section.id), + onToggleExpand = { expandedSections.toggle(section.id) }, ) } @@ -241,22 +238,18 @@ private fun VisibilityPill( mandatory: Boolean, onClick: () -> Unit, ) { + // One branch decides both halves of the pill, so a label can't drift away from its glyph. + val (labelRes, icon) = + when { + mandatory -> R.string.drawer_settings_always_on to MaterialSymbols.Lock + visible -> R.string.drawer_settings_visible to MaterialSymbols.Visibility + else -> R.string.drawer_settings_hidden to MaterialSymbols.VisibilityOff + } + TogglePill( on = visible, - label = - stringRes( - when { - mandatory -> R.string.drawer_settings_always_on - visible -> R.string.drawer_settings_visible - else -> R.string.drawer_settings_hidden - }, - ), - icon = - when { - mandatory -> MaterialSymbols.Lock - visible -> MaterialSymbols.Visibility - else -> MaterialSymbols.VisibilityOff - }, + label = stringRes(labelRes), + icon = icon, enabled = !mandatory, onClick = onClick, ) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/NavPickerUi.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/NavPickerUi.kt index 0793edcf81..78be2e8f20 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/NavPickerUi.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/NavPickerUi.kt @@ -42,6 +42,9 @@ import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.material3.TextButton import androidx.compose.runtime.Composable +import androidx.compose.runtime.Stable +import androidx.compose.runtime.mutableStateMapOf +import androidx.compose.runtime.remember import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip @@ -90,13 +93,32 @@ val SectionCollapse = shrinkVertically(shrinkTowards = Alignment.Top) + fadeOut( * category (a favorite, or a relay/community "server" row), 2 = a room nested under its server (a * NIP-29 group under its relay, or a Concord channel under its community). */ -fun indentPadding(level: Int) = +private fun indentPadding(level: Int) = when (level) { 0 -> Size13dp 1 -> Size24dp else -> Size40dp } +/** + * Which collapsible rows of a picker are currently open, keyed by whatever identifies a row (a + * section id, a string-resource id, a catalog item). Absent means collapsed, so the initial state + * costs nothing and no list has to be seeded. + */ +@Stable +class ExpandedKeys { + private val open = mutableStateMapOf() + + fun isExpanded(key: K): Boolean = open[key] == true + + fun toggle(key: K) { + open[key] = !isExpanded(key) + } +} + +@Composable +fun rememberExpandedKeys(): ExpandedKeys = remember { ExpandedKeys() } + @Composable fun PickerSectionHeader(title: String) { Text( diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/navigation/DrawerItemVisibilityTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/navigation/DrawerItemVisibilityTest.kt index d5fcf5f47a..3d193b7f08 100644 --- a/amethyst/src/test/java/com/vitorpamplona/amethyst/navigation/DrawerItemVisibilityTest.kt +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/navigation/DrawerItemVisibilityTest.kt @@ -23,7 +23,7 @@ package com.vitorpamplona.amethyst.navigation import com.vitorpamplona.amethyst.ui.navigation.bottombars.NavBarItem import com.vitorpamplona.amethyst.ui.navigation.drawer.DrawerItemVisibility import com.vitorpamplona.amethyst.ui.navigation.drawer.DrawerSectionId -import com.vitorpamplona.amethyst.ui.navigation.drawer.drawerSection +import com.vitorpamplona.amethyst.ui.navigation.drawer.DrawerSections import com.vitorpamplona.amethyst.ui.screen.loggedIn.settings.DrawerSettingsState import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse @@ -38,8 +38,8 @@ import org.junit.Test * rows, which no code path may switch off. */ class DrawerItemVisibilityTest { - private val you = drawerSection(DrawerSectionId.YOU) - private val system = drawerSection(DrawerSectionId.SYSTEM) + private val you = DrawerSections.first { it.id == DrawerSectionId.YOU } + private val system = DrawerSections.first { it.id == DrawerSectionId.SYSTEM } @Test fun nothingHiddenMeansEverythingVisible() { diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/navigation/DrawerSectionsTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/navigation/DrawerSectionsTest.kt index 3cf1687a83..f77c3d7523 100644 --- a/amethyst/src/test/java/com/vitorpamplona/amethyst/navigation/DrawerSectionsTest.kt +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/navigation/DrawerSectionsTest.kt @@ -90,6 +90,22 @@ class DrawerSectionsTest { assertEquals("two sections share a DrawerSectionId", ids.size, ids.toSet().size) } + @Test + fun aSectionWithNoCatalogItemsRendersFixedRowsOrNothingAtAll() { + // hasFixedRows is declared on the section but consumed by CatalogSection's `when (section.id)`, + // in another file — so the flag and the branch that honours it can drift apart with no compile + // error. A section that carries neither is unreachable in both directions at once: the settings + // screen skips it on items.isEmpty(), and CatalogSection returns before rendering a heading. + val unreachable = DrawerSections.filter { it.items.isEmpty() && !it.hasFixedRows } + + assertEquals( + "a drawer section has no catalog items and no fixed rows, so it renders nowhere — " + + "give it items, set hasFixedRows and a branch in CatalogSection, or delete it", + emptyList(), + unreachable.map { it.id }, + ) + } + @Test fun mandatoryItemsAreActuallyRenderedByASection() { // A mandatory item that no section renders would be unhideable *and* invisible — the worst From 64019dbd7c9d5048b93935daeaa9372fa21f7899 Mon Sep 17 00:00:00 2001 From: davotoula Date: Wed, 5 Aug 2026 22:21:47 +0200 Subject: [PATCH 043/132] update cs,pt,de,sv --- amethyst/src/main/res/values-cs/strings.xml | 90 +++++++++++++++++++ .../src/main/res/values-de-rDE/strings.xml | 78 ++++++++++++++++ .../src/main/res/values-pt-rBR/strings.xml | 79 ++++++++++++++++ .../src/main/res/values-sv-rSE/strings.xml | 79 ++++++++++++++++ 4 files changed, 326 insertions(+) diff --git a/amethyst/src/main/res/values-cs/strings.xml b/amethyst/src/main/res/values-cs/strings.xml index 876d263166..66fa414df9 100644 --- a/amethyst/src/main/res/values-cs/strings.xml +++ b/amethyst/src/main/res/values-cs/strings.xml @@ -4811,4 +4811,94 @@ URL avataru (volitelné) Publikování… Publikovat personu + Vše + Oblíbený algoritmický zdroj + Vaše komunity + Vybraný seznam lidí + Lidé, které sledujete + Všichni + Ztlumení lidé + odběry subscriptions filtry relaye relay požadavky reqs připojení proč diagnostika + %1$d %% ze všech + Aktivní odběry relayů + Nepřiřazeno k žádnému účtu + Váš vlastní profil, nastavení a koncepty na vašich domovských relayích. + Relaye, na které každá komunita publikuje své roviny. + Vaše relaye pro schránku DM, kam se doručují zprávy zabalené v gift-wrapu. + Skupinové zprávy a balíčky klíčů na relayích každé skupiny. + Sleduje právě zobrazené události kvůli novým odpovědím, reakcím, sdílením, zapům a nahlášením, takže se počty aktualizují během čtení. + Chatovací místnosti bez historie — zprávy existují jen po dobu vašeho připojení, proto zůstávají odebírané, aby vůbec něco přišlo. + Seznamy sledovaných, ze kterých se sestavuje váš zdroj a vaše síť důvěry. + Místnosti podle polohy pro oblasti, které sledujete, dotazované na relayích, které je nesou. + Příspěvky lidí, které sledujete, čtené z relayů, na které každý z nich publikuje. + Chat a zapovací cíle připojené k živým vysíláním, která máte otevřená nebo sledujete. + Relaye místnosti, dokud je otevřená. + Prohledává relaye, které minty existují a které lidé doporučují. + Nahlášení, která vaši sledovaní napsali o profilech právě na obrazovce, dotazovaná na každém relayi, kam tito sledovaní publikují. + Vaše relaye pro příjem a k tomu malý rotující vzorek relayů, kam publikují vaši sledovaní, pro případ, že by zmínka byla doručena jinam. + Naslouchá na vašich nutzap relayích a také na relayích pro příjem a DM, aby vám neunikla žádná platba. + Upozornění z vaší připojené peněženky. + Profily lidí právě na obrazovce. + Domovský relay každého chatu, který máte otevřený nebo do kterého jste se připojili. + Načítá podle ID události, na které se něco na obrazovce odkazuje, ale zatím je nemáte — citaci, rodiče odpovědi, kořen vlákna. + Skupiny NIP-29, do kterých jste vstoupili. Každá skupina žije na jednom hostitelském relayi, takže se aplikace připojí ke každému relayi, který hostí některou vaši skupinu. + Zjišťuje, na které relaye každý člověk publikuje, aby se jeho příspěvky daly načíst na správném místě. + Události vaší vlastní peněženky, čtené zpět z relayů, na které jste je publikovali. + Doplňky + Procházení + Chaty komunit + Zdroje komunit + Schránka DM + Sledování událostí + Mizící chaty + Chaty podle místa + Domovský zdroj + Chat živého vysílání + Média + Adresář mintů + Schránka nutzapů + Sledování profilů + Ostatní + Hledání chybějících událostí + Relay skupiny + Informace o relayi + Vyhledávač seznamů relayů + Nahlášení od sledovaných + Hledání + Hashtagy + Konverzace + Témata + Data účtu + Peněženka + Neplatná adresa relaye. Použijte název hostitele nebo IP adresu v hranatých závorkách (například [201:d0e:9ba5:8bbc::1]:8080). + + %1$d filtr + %1$d filtry + %1$d filtru + %1$d filtrů + + + %1$d skupina + %1$d skupiny + %1$d skupiny + %1$d skupin + + + %1$d relay + %1$d relaye + %1$d relaye + %1$d relayů + + + %1$d filtr zatím není přiřazen + %1$d filtry zatím nejsou přiřazeny + %1$d filtru zatím není přiřazeno + %1$d filtrů zatím není přiřazeno + + + %1$s \u00b7 %2$d relay + %1$s \u00b7 %2$d relaye + %1$s \u00b7 %2$d relaye + %1$s \u00b7 %2$d relayů + diff --git a/amethyst/src/main/res/values-de-rDE/strings.xml b/amethyst/src/main/res/values-de-rDE/strings.xml index 4b96c7eddc..45e9c1cb5a 100644 --- a/amethyst/src/main/res/values-de-rDE/strings.xml +++ b/amethyst/src/main/res/values-de-rDE/strings.xml @@ -4635,4 +4635,82 @@ Avatar-URL (optional) Wird veröffentlicht… Persona veröffentlichen + Alle + Ein bevorzugter Feed-Algorithmus + Deine Communitys + Eine ausgewählte Liste von Personen + Personen, denen du folgst + Jeder + Stummgeschaltete Personen + abonnements subscriptions filter relays anfragen reqs verbindungen warum diagnose + %1$d %% von allen + Aktive Relay-Abonnements + Keinem Konto zugeordnet + Dein eigenes Profil, deine Einstellungen und Entwürfe auf deinen Heim-Relays. + Die Relays, auf denen jede Community ihre Planes veröffentlicht. + Deine DM-Posteingangs-Relays, an die Gift-Wrap-Nachrichten zugestellt werden. + Gruppennachrichten und Schlüsselpakete auf den Relays der jeweiligen Gruppe. + Beobachtet die gerade angezeigten Events auf neue Antworten, Reaktionen, Reposts, Zaps und Meldungen, damit die Zähler beim Lesen aktuell bleiben. + Chaträume ohne Verlauf — Nachrichten existieren nur, solange du verbunden bist, deshalb bleiben sie abonniert, damit überhaupt etwas ankommt. + Folgelisten, aus denen dein Feed und dein Web of Trust aufgebaut werden. + Standortbasierte Räume für die Gebiete, denen du folgst, abgefragt bei den Relays, die sie führen. + Beiträge von Personen, denen du folgst, gelesen von den Relays, auf denen jede von ihnen veröffentlicht. + Chat und Zap-Ziele, die an Live-Streams hängen, die du geöffnet hast oder denen du folgst. + Die Relays des Raums, solange er geöffnet ist. + Sucht über Relays hinweg, welche Mints existieren und welche empfohlen werden. + Meldungen, die deine Gefolgten über die gerade angezeigten Profile geschrieben haben, abgefragt bei jedem Relay, auf dem diese Gefolgten veröffentlichen. + Deine Posteingangs-Relays plus eine kleine, rotierende Stichprobe der Relays, auf denen deine Gefolgten veröffentlichen, falls eine Erwähnung woanders zugestellt wurde. + Lauscht auf deinen Nutzap-Relays sowie deinen Posteingangs- und DM-Relays, damit keine Zahlung durchrutscht. + Benachrichtigungen von deiner verbundenen Wallet. + Profile der gerade angezeigten Personen. + Das Heim-Relay jedes Chats, den du geöffnet hast oder dem du beigetreten bist. + Holt Events per ID, auf die etwas auf deinem Bildschirm verweist, die du aber noch nicht hast — ein Zitat, die übergeordnete Antwort, eine Thread-Wurzel. + NIP-29-Gruppen, denen du beigetreten bist. Jede Gruppe liegt auf einem Host-Relay, daher verbindet sich die App mit jedem Relay, das eine deiner Gruppen beherbergt. + Findet heraus, auf welchen Relays jede Person veröffentlicht, damit ihre Beiträge an der richtigen Stelle abgerufen werden können. + Deine eigenen Wallet-Events, zurückgelesen von den Relays, auf denen du sie veröffentlicht hast. + Erweiterungen + Stöbern + Community-Chats + Community-Feeds + DM-Posteingang + Events beobachten + Verschwindende Chats + Standort-Chats + Startseiten-Feed + Live-Stream-Chat + Medien + Mint-Verzeichnis + Nutzap-Posteingang + Profile beobachten + Sonstiges + Fehlende Events finden + Relay-Gruppen + Relay-Info + Relay-Listen-Finder + Meldungen von Gefolgten + Suche + Unterhaltung + Themen + Kontodaten + Keine gültige Relay-Adresse. Verwende einen Hostnamen oder eine IP-Adresse in Klammern (zum Beispiel [201:d0e:9ba5:8bbc::1]:8080). + + %1$d Filter + %1$d Filter + + + %1$d Gruppe + %1$d Gruppen + + + %1$d Relay + %1$d Relays + + + %1$d Filter ist noch nicht zugeordnet + %1$d Filter sind noch nicht zugeordnet + + + %1$s \u00b7 %2$d Relay + %1$s \u00b7 %2$d Relays + diff --git a/amethyst/src/main/res/values-pt-rBR/strings.xml b/amethyst/src/main/res/values-pt-rBR/strings.xml index ba71f48a49..17b4737653 100644 --- a/amethyst/src/main/res/values-pt-rBR/strings.xml +++ b/amethyst/src/main/res/values-pt-rBR/strings.xml @@ -4640,4 +4640,83 @@ URL do avatar (opcional) Publicando… Publicar persona + Tudo + Um algoritmo de feed favorito + Suas comunidades + Uma lista escolhida de pessoas + Pessoas que você segue + Todos + Pessoas silenciadas + assinaturas subscriptions filtros relays requisições reqs conexões por que diagnóstico + %1$d%% de todos + Assinaturas de relay ativas + Não atribuído a nenhuma conta + Seu próprio perfil, configurações e rascunhos, nos seus relays de origem. + Os relays em que cada comunidade publica seus planos. + Os relays da sua caixa de entrada de DM, para onde as mensagens em gift wrap são entregues. + Mensagens de grupo e pacotes de chaves, nos relays de cada grupo. + Observa os eventos exibidos no momento em busca de novas respostas, reações, repostagens, zaps e denúncias, para que as contagens sejam atualizadas enquanto você lê. + Salas de chat que não guardam histórico — as mensagens existem apenas enquanto você está conectado, então elas continuam assinadas para que algo chegue. + Listas de seguindo, usadas para montar seu feed e sua rede de confiança. + Salas baseadas em localização para as áreas que você segue, consultadas nos relays que as hospedam. + Publicações de pessoas que você segue, lidas dos relays em que cada uma delas publica. + Chat e metas de zap ligados às transmissões ao vivo que você tem abertas ou segue. + Os relays da sala, enquanto ela estiver aberta. + Procura pelos relays quais mints existem e quais as pessoas recomendam. + Denúncias que as pessoas que você segue escreveram sobre os perfis atualmente na sua tela, consultadas em cada relay em que essas pessoas publicam. + Os relays da sua caixa de entrada, mais uma pequena amostra rotativa dos relays em que quem você segue publica, caso uma menção tenha sido entregue em outro lugar. + Escuta nos seus relays de nutzap, além dos relays da caixa de entrada e de DM, para que nenhum pagamento passe despercebido. + Notificações da sua carteira conectada. + Perfis das pessoas atualmente na tela. + O relay de origem de cada chat que você abriu ou do qual participa. + Busca por id os eventos a que algo na sua tela se refere, mas que você ainda não tem — uma citação, o pai de uma resposta, a raiz de uma conversa. + Grupos NIP-29 dos quais você participa. Cada grupo vive em um relay hospedeiro, então o app se conecta a todo relay que hospeda um grupo seu. + Descobre em quais relays cada pessoa publica, para que as publicações dela possam ser buscadas no lugar certo. + Os eventos da sua própria carteira, lidos de volta dos relays em que você os publicou. + Complementos + Navegação + Chats de comunidades + Feeds de comunidades + Caixa de entrada de DM + Observando eventos + Chats efêmeros + Chats por localização + Feed inicial + Chat de transmissão ao vivo + Mídia + Diretório de mints + Caixa de entrada de nutzaps + Observando perfis + Outros + Encontrando eventos faltantes + Grupos de relay + Informações do relay + Localizador de listas de relays + Denúncias de quem você segue + Pesquisa + Conversa + Tópicos + Dados da conta + Carteira + Endereço de relay inválido. Use um nome de host ou um endereço IP entre colchetes (por exemplo [201:d0e:9ba5:8bbc::1]:8080). + + %1$d filtro + %1$d filtros + + + %1$d grupo + %1$d grupos + + + %1$d relay + %1$d relays + + + %1$d filtro ainda não foi atribuído + %1$d filtros ainda não foram atribuídos + + + %1$s \u00b7 %2$d relay + %1$s \u00b7 %2$d relays + diff --git a/amethyst/src/main/res/values-sv-rSE/strings.xml b/amethyst/src/main/res/values-sv-rSE/strings.xml index d2a555d0f3..085d398a4c 100644 --- a/amethyst/src/main/res/values-sv-rSE/strings.xml +++ b/amethyst/src/main/res/values-sv-rSE/strings.xml @@ -4643,4 +4643,83 @@ Avatar-URL (valfritt) Publicerar… Publicera persona + Allt + En favorit-flödesalgoritm + Dina gemenskaper + En vald lista med personer + Personer du följer + Alla + Tystade personer + prenumerationer subscriptions filter reläer relay förfrågningar reqs anslutningar varför diagnostik + %1$d %% av alla + Aktiva reläprenumerationer + Inte kopplat till något konto + Din egen profil, dina inställningar och utkast, på dina hemreläer. + Reläerna som varje gemenskap publicerar sina plan till. + Dina DM-inkorgsreläer, dit gift-wrap-meddelanden levereras. + Gruppmeddelanden och nyckelpaket, på varje grupps reläer. + Bevakar de händelser som visas just nu efter nya svar, reaktioner, återinlägg, zaps och rapporter, så att räknarna uppdateras medan du läser. + Chattrum som inte sparar någon historik — meddelanden finns bara medan du är ansluten, så dessa förblir prenumererade för att något alls ska komma fram. + Följerlistor, som används för att bygga ditt flöde och ditt förtroendenät. + Platsbaserade rum för de områden du följer, efterfrågade från de reläer som bär dem. + Inlägg från personer du följer, lästa från de reläer var och en av dem publicerar till. + Chatt och zap-mål kopplade till livesändningar du har öppna eller följer. + Rummets reläer, medan det är öppet. + Söker över reläer efter vilka mints som finns och vilka folk rekommenderar. + Rapporter som personer du följer har skrivit om profilerna som just nu visas på skärmen, efterfrågade från varje relä dessa personer publicerar till. + Dina inkorgsreläer, plus ett litet roterande urval av de reläer personer du följer publicerar till, ifall ett omnämnande levererades någon annanstans. + Lyssnar på dina nutzap-reläer plus dina inkorgs- och DM-reläer, så att en betalning inte kan slinka förbi. + Aviseringar från din anslutna plånbok. + Profiler för personerna som just nu visas på skärmen. + Hemrelät för varje chatt du har öppen eller har gått med i. + Hämtar händelser via id som något på din skärm hänvisar till men som du inte har ännu — ett citat, ett svars förälder, en trådrot. + NIP-29-grupper du gått med i. Varje grupp bor på ett värdrelä, så appen ansluter till varje relä som är värd för en av dina grupper. + Hittar vilka reläer varje person publicerar till, så att deras inlägg kan hämtas från rätt ställe. + Dina egna plånbokshändelser, lästa tillbaka från de reläer du publicerade dem till. + Tillägg + Bläddring + Gemenskapschattar + Gemenskapsflöden + DM-inkorg + Observerar händelser + Försvinnande chattar + Platschattar + Hemflöde + Livesändningschatt + Mint-katalog + Nutzap-inkorg + Observerar profiler + Övrigt + Hittar saknade händelser + Relägrupper + Reläinfo + Relälistsökare + Rapporter från personer du följer + Sök + Hashtaggar + Konversation + Ämnen + Kontots data + Plånbok + Inte en giltig reläadress. Använd ett värdnamn eller en IP-adress inom hakparenteser (till exempel [201:d0e:9ba5:8bbc::1]:8080). + + %1$d filter + %1$d filter + + + %1$d grupp + %1$d grupper + + + %1$d relä + %1$d reläer + + + %1$d filter är inte kopplat ännu + %1$d filter är inte kopplade ännu + + + %1$s \u00b7 %2$d relä + %1$s \u00b7 %2$d reläer + From 0cf1534861b0792d6e72d44bc04fb44f1c718abb Mon Sep 17 00:00:00 2001 From: davotoula Date: Wed, 5 Aug 2026 21:54:04 +0200 Subject: [PATCH 044/132] fix(media): stop rendering non-media NIP-94 files as video MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A kind-1063 file header was classified by a binary `isImage` test: anything that wasn't an image fell through to MediaUrlVideo. A webxdc app (application/x-webxdc, a zip) therefore reached ExoPlayer and buffered forever, as did every archive, installer and — since MediaUrlPdf was never constructed here — every NIP-94 PDF. --- .../ui/components/FileAttachmentCard.kt | 142 ++++++++++++++++++ .../ui/components/pdf/PdfPreviewCard.kt | 45 +----- .../amethyst/ui/note/types/FileHeader.kt | 142 ++++++++++++------ .../amethyst/ui/note/types/Video.kt | 5 +- .../amethyst/ui/note/types/VideoDisplay.kt | 5 +- .../loggedIn/shorts/VideoCardCompose.kt | 5 +- .../loggedIn/video/FileHeaderCardCompose.kt | 48 ++---- .../commons/richtext/MediaContentKind.kt | 37 +++++ .../commons/richtext/RichTextParser.kt | 83 +++++----- .../commons/richtext/ClassifyMediaTest.kt | 138 +++++++++++++++++ 10 files changed, 492 insertions(+), 158 deletions(-) create mode 100644 amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/FileAttachmentCard.kt create mode 100644 commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/MediaContentKind.kt create mode 100644 commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/ClassifyMediaTest.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/FileAttachmentCard.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/FileAttachmentCard.kt new file mode 100644 index 0000000000..759d5e7e83 --- /dev/null +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/FileAttachmentCard.kt @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.ui.components + +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.remember +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalUriHandler +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp +import com.vitorpamplona.amethyst.commons.icons.symbols.Icon +import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbol +import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols +import com.vitorpamplona.amethyst.commons.util.countToHumanReadableBytes +import com.vitorpamplona.amethyst.ui.components.pdf.extractFilename +import com.vitorpamplona.amethyst.ui.note.types.prettyMime +import com.vitorpamplona.amethyst.ui.theme.DoubleVertSpacer +import com.vitorpamplona.amethyst.ui.theme.MaxWidthWithHorzPadding +import com.vitorpamplona.amethyst.ui.theme.Size20Modifier +import com.vitorpamplona.amethyst.ui.theme.innerPostModifier + +/** + * The renderer for a declared file that none of the media viewers can display — a webxdc app, + * an archive, an installer, any MIME [com.vitorpamplona.amethyst.commons.richtext.RichTextParser.classifyMedia] + * returns null for. + * + * It exists so those files have somewhere to land other than the video player: an unknown blob + * used to fall through an image-or-else-video branch into ExoPlayer, which buffers forever on a + * zip. Everything shown here comes off the event's own tags (NIP-94 `alt`, `m`, `size`), so the + * card costs no network round-trip — unlike routing the URL through the OpenGraph previewer, + * which would try to download the blob just to rediscover the type the event already declared. + */ +@Composable +fun FileAttachmentCard( + url: String, + description: String?, + mimeType: String?, + sizeInBytes: Long?, +) { + val uriHandler = LocalUriHandler.current + val filename = remember(url) { extractFilename(url) } + val subtitle = remember(mimeType, sizeInBytes) { fileSubtitle(mimeType, sizeInBytes) } + + Column( + modifier = + MaterialTheme.colorScheme.innerPostModifier + .fillMaxWidth() + .clickable { uriHandler.openUri(url) }, + ) { + FileAttachmentRow( + symbol = MaterialSymbols.AttachFile, + // The alt/content text names the file for a human ("Webxdc app: Quake"); + // the hashed URL basename is the fallback when the event omits it. + title = description?.ifBlank { null } ?: filename, + subtitle = subtitle, + titleMaxLines = 2, + ) + + Spacer(modifier = DoubleVertSpacer) + } +} + +/** + * The icon + title + subtitle row shared by every card that stands in for a file it can't + * render inline: this one and the PDF placeholder/skeleton in + * [com.vitorpamplona.amethyst.ui.components.pdf.PdfPreviewCard]. + */ +@Composable +internal fun FileAttachmentRow( + symbol: MaterialSymbol, + title: String, + subtitle: String?, + titleMaxLines: Int = 1, +) { + Row( + modifier = MaxWidthWithHorzPadding.padding(vertical = 8.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + Icon( + symbol = symbol, + contentDescription = null, + modifier = Size20Modifier, + tint = MaterialTheme.colorScheme.onSurfaceVariant, + ) + + Column(modifier = Modifier.weight(1f)) { + Text( + text = title, + style = MaterialTheme.typography.bodyMedium, + maxLines = titleMaxLines, + overflow = TextOverflow.Ellipsis, + ) + if (subtitle != null) { + Text( + text = subtitle, + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + maxLines = 1, + ) + } + } + } +} + +/** "APK · 16 MB", dropping either half when the event doesn't declare it. */ +private fun fileSubtitle( + mimeType: String?, + sizeInBytes: Long?, +): String? = + listOfNotNull( + mimeType?.ifBlank { null }?.let(::prettyMime), + sizeInBytes?.takeIf { it > 0 }?.let(::countToHumanReadableBytes), + ).joinToString(" · ").ifEmpty { null } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/pdf/PdfPreviewCard.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/pdf/PdfPreviewCard.kt index c352077e42..0b975ede46 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/pdf/PdfPreviewCard.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/pdf/PdfPreviewCard.kt @@ -26,38 +26,29 @@ import android.os.ParcelFileDescriptor import androidx.compose.foundation.ExperimentalFoundationApi import androidx.compose.foundation.Image import androidx.compose.foundation.combinedClickable -import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Column -import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.Spacer import androidx.compose.foundation.layout.aspectRatio import androidx.compose.foundation.layout.fillMaxWidth -import androidx.compose.foundation.layout.padding import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.produceState import androidx.compose.runtime.remember -import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.FilterQuality import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.platform.LocalWindowInfo -import androidx.compose.ui.text.style.TextOverflow -import androidx.compose.ui.unit.dp import androidx.core.graphics.createBitmap -import com.vitorpamplona.amethyst.commons.icons.symbols.Icon import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols import com.vitorpamplona.amethyst.commons.richtext.MediaUrlPdf import com.vitorpamplona.amethyst.ui.components.ClickableUrl +import com.vitorpamplona.amethyst.ui.components.FileAttachmentRow import com.vitorpamplona.amethyst.ui.components.ShareMediaAction import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel import com.vitorpamplona.amethyst.ui.theme.DoubleVertSpacer -import com.vitorpamplona.amethyst.ui.theme.MaxWidthWithHorzPadding -import com.vitorpamplona.amethyst.ui.theme.Size20Modifier import com.vitorpamplona.amethyst.ui.theme.innerPostModifier import com.vitorpamplona.quartz.utils.Log import kotlinx.coroutines.CancellationException @@ -207,35 +198,11 @@ private fun PdfSkeletonCard(filename: String) { private fun FilenameRow( filename: String, subtitle: String, -) { - Row( - modifier = MaxWidthWithHorzPadding.padding(vertical = 8.dp), - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.spacedBy(8.dp), - ) { - Icon( - symbol = MaterialSymbols.PictureAsPdf, - contentDescription = null, - modifier = Size20Modifier, - tint = MaterialTheme.colorScheme.onSurfaceVariant, - ) - - Column(modifier = Modifier.weight(1f)) { - Text( - text = filename, - style = MaterialTheme.typography.bodyMedium, - maxLines = 1, - overflow = TextOverflow.Ellipsis, - ) - Text( - text = subtitle, - style = MaterialTheme.typography.bodySmall, - color = MaterialTheme.colorScheme.onSurfaceVariant, - maxLines = 1, - ) - } - } -} +) = FileAttachmentRow( + symbol = MaterialSymbols.PictureAsPdf, + title = filename, + subtitle = subtitle, +) private fun renderFirstPage( file: java.io.File, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/types/FileHeader.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/types/FileHeader.kt index 62d91d39aa..96730249ff 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/types/FileHeader.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/types/FileHeader.kt @@ -24,10 +24,13 @@ import androidx.compose.runtime.Composable import androidx.compose.runtime.remember import androidx.compose.ui.layout.ContentScale import com.vitorpamplona.amethyst.commons.richtext.BaseMediaContent +import com.vitorpamplona.amethyst.commons.richtext.MediaContentKind import com.vitorpamplona.amethyst.commons.richtext.MediaUrlImage +import com.vitorpamplona.amethyst.commons.richtext.MediaUrlPdf import com.vitorpamplona.amethyst.commons.richtext.MediaUrlVideo import com.vitorpamplona.amethyst.commons.richtext.RichTextParser import com.vitorpamplona.amethyst.model.Note +import com.vitorpamplona.amethyst.ui.components.FileAttachmentCard import com.vitorpamplona.amethyst.ui.components.SensitivityWarning import com.vitorpamplona.amethyst.ui.components.ZoomableContentView import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel @@ -43,50 +46,103 @@ fun FileHeaderDisplay( ) { val event = (note.event as? FileHeaderEvent) ?: return val fullUrl = event.url() ?: return + val mimeType = remember(note) { event.mimeType() } + val content = remember(note) { event.toMediaContent(note, fullUrl, mimeType) } - val content: BaseMediaContent = - remember(note) { - val blurHash = event.blurhash() - val thumbHash = event.thumbhash() - val hash = event.hash() - val dimensions = event.dimensions() - val description = event.content.ifEmpty { null } ?: event.alt() - val isImage = event.mimeType()?.startsWith("image/") == true || RichTextParser.isImageUrl(fullUrl) - val uri = note.toNostrUri() - val mimeType = event.mimeType() - - if (isImage) { - MediaUrlImage( - url = fullUrl, - description = description, - hash = hash, - blurhash = blurHash, - dim = dimensions, - uri = uri, - mimeType = mimeType, - thumbhash = thumbHash, - ) - } else { - MediaUrlVideo( - url = fullUrl, - description = description, - hash = hash, - blurhash = blurHash, - dim = dimensions, - uri = uri, - authorName = note.author?.toBestDisplayName(), - mimeType = mimeType, - thumbhash = thumbHash, - ) - } - } - + // The sensitivity gate wraps both branches: a content warning is about the file, not about + // which viewer happens to render it, so an NSFW-tagged archive stays behind the same gate. SensitivityWarning(note = note, accountViewModel = accountViewModel) { - ZoomableContentView( - content = content, - roundedCorner = roundedCorner, - contentScale = contentScale, - accountViewModel = accountViewModel, - ) + if (content == null) { + FileHeaderAttachmentCard(event, fullUrl, mimeType) + } else { + ZoomableContentView( + content = content, + roundedCorner = roundedCorner, + contentScale = contentScale, + accountViewModel = accountViewModel, + ) + } } } + +/** + * Builds the viewer for a kind-1063 header, or **null** when no viewer can show the blob. + * + * Kind 1063 is a *generic* file container — its `m` tag can name any type, so unlike a NIP-71 + * video event the kind itself asserts nothing about how to render the payload. A null here means + * the file belongs in [FileHeaderAttachmentCard] rather than being pushed into the video player. + */ +internal fun FileHeaderEvent.toMediaContent( + note: Note, + url: String, + mimeType: String?, +): BaseMediaContent? { + val blurHash = blurhash() + val thumbHash = thumbhash() + val hash = hash() + val dimensions = dimensions() + val description = fileDescription() + val uri = note.toNostrUri() + + return when (RichTextParser.classifyMedia(url, mimeType)) { + MediaContentKind.IMAGE -> + MediaUrlImage( + url = url, + description = description, + hash = hash, + blurhash = blurHash, + dim = dimensions, + uri = uri, + mimeType = mimeType, + thumbhash = thumbHash, + ) + + MediaContentKind.VIDEO -> + MediaUrlVideo( + url = url, + description = description, + hash = hash, + blurhash = blurHash, + dim = dimensions, + uri = uri, + authorName = note.author?.toBestDisplayName(), + mimeType = mimeType, + thumbhash = thumbHash, + ) + + MediaContentKind.PDF -> + MediaUrlPdf( + url = url, + description = description, + hash = hash, + blurhash = blurHash, + dim = dimensions, + uri = uri, + mimeType = mimeType, + thumbhash = thumbHash, + ) + + null -> null + } +} + +/** The link card a kind-1063 header falls back to when [toMediaContent] returns null. */ +@Composable +internal fun FileHeaderAttachmentCard( + event: FileHeaderEvent, + url: String, + mimeType: String?, +) { + val description = remember(event) { event.fileDescription() } + val sizeInBytes = remember(event) { event.size()?.toLong() } + + FileAttachmentCard( + url = url, + description = description, + mimeType = mimeType, + sizeInBytes = sizeInBytes, + ) +} + +/** The human-facing name of the file: NIP-94 `content` when present, else the `alt` tag. */ +private fun FileHeaderEvent.fileDescription(): String? = content.ifEmpty { null } ?: alt() diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/types/Video.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/types/Video.kt index d00bd6d2e8..1d8cdcc3d8 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/types/Video.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/types/Video.kt @@ -43,6 +43,7 @@ import com.vitorpamplona.amethyst.R import com.vitorpamplona.amethyst.commons.model.EmptyTagList import com.vitorpamplona.amethyst.commons.model.toImmutableListOfLists import com.vitorpamplona.amethyst.commons.richtext.BaseMediaContent +import com.vitorpamplona.amethyst.commons.richtext.MediaContentKind import com.vitorpamplona.amethyst.commons.richtext.MediaUrlImage import com.vitorpamplona.amethyst.commons.richtext.MediaUrlVideo import com.vitorpamplona.amethyst.commons.richtext.RichTextParser @@ -88,7 +89,9 @@ fun VideoDisplay( val content: BaseMediaContent = remember(note) { val description = videoEvent.content.ifBlank { null } ?: event.alt() - val isImage = imeta.mimeType?.startsWith("image/") == true || RichTextParser.isImageUrl(imeta.url) + // A NIP-71 event asserts its own type, so only an explicit image imeta diverts to the + // viewer; an unclassifiable one still belongs in the player. See classifyMedia. + val isImage = RichTextParser.classifyMedia(imeta.url, imeta.mimeType) == MediaContentKind.IMAGE val uri = note.toNostrUri() if (isImage) { diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/types/VideoDisplay.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/types/VideoDisplay.kt index 0abc16ac93..4f5c661810 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/types/VideoDisplay.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/types/VideoDisplay.kt @@ -26,6 +26,7 @@ import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.ui.layout.ContentScale import com.vitorpamplona.amethyst.commons.richtext.BaseMediaContent +import com.vitorpamplona.amethyst.commons.richtext.MediaContentKind import com.vitorpamplona.amethyst.commons.richtext.MediaUrlImage import com.vitorpamplona.amethyst.commons.richtext.MediaUrlVideo import com.vitorpamplona.amethyst.commons.richtext.RichTextParser @@ -55,7 +56,9 @@ fun JustVideoDisplay( val imeta = videoEvent.imetaTags().getOrNull(0) ?: return val isSensitive = remember(note) { event.isSensitiveOrNSFW() } val reasons = remember(note) { collectContentWarningReasons(event) } - val isImage = remember(note) { imeta.mimeType?.startsWith("image/") == true || RichTextParser.isImageUrl(imeta.url) } + // A NIP-71 event asserts its own type, so only an explicit image imeta diverts to the + // viewer; an unclassifiable one still belongs in the player. See classifyMedia. + val isImage = remember(note) { RichTextParser.classifyMedia(imeta.url, imeta.mimeType) == MediaContentKind.IMAGE } val content by remember(note) { diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/shorts/VideoCardCompose.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/shorts/VideoCardCompose.kt index 6171a82067..9e52ec0362 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/shorts/VideoCardCompose.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/shorts/VideoCardCompose.kt @@ -39,6 +39,7 @@ import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp import com.vitorpamplona.amethyst.commons.richtext.BaseMediaContent +import com.vitorpamplona.amethyst.commons.richtext.MediaContentKind import com.vitorpamplona.amethyst.commons.richtext.MediaUrlImage import com.vitorpamplona.amethyst.commons.richtext.MediaUrlVideo import com.vitorpamplona.amethyst.commons.richtext.RichTextParser @@ -104,7 +105,9 @@ private fun VideoCardImage( val imeta = videoEvent.imetaTags().getOrNull(0) ?: return val isSensitive = remember(note) { event.isSensitiveOrNSFW() } val reasons = remember(note) { collectContentWarningReasons(event) } - val isImage = remember(note) { imeta.mimeType?.startsWith("image/") == true || RichTextParser.isImageUrl(imeta.url) } + // A NIP-71 event asserts its own type, so only an explicit image imeta diverts to the + // viewer; an unclassifiable one still belongs in the player. See classifyMedia. + val isImage = remember(note) { RichTextParser.classifyMedia(imeta.url, imeta.mimeType) == MediaContentKind.IMAGE } val content by remember(note) { diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/video/FileHeaderCardCompose.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/video/FileHeaderCardCompose.kt index ac3778152f..f47613e775 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/video/FileHeaderCardCompose.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/video/FileHeaderCardCompose.kt @@ -29,7 +29,6 @@ import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.MutableState -import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.ui.Modifier @@ -38,10 +37,7 @@ import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp -import com.vitorpamplona.amethyst.commons.richtext.BaseMediaContent import com.vitorpamplona.amethyst.commons.richtext.MediaUrlImage -import com.vitorpamplona.amethyst.commons.richtext.MediaUrlVideo -import com.vitorpamplona.amethyst.commons.richtext.RichTextParser import com.vitorpamplona.amethyst.model.MediaAspectRatioCache import com.vitorpamplona.amethyst.model.Note import com.vitorpamplona.amethyst.ui.components.BlurhashBackdrop @@ -51,9 +47,10 @@ import com.vitorpamplona.amethyst.ui.components.collectContentWarningReasons import com.vitorpamplona.amethyst.ui.components.mediaSizingModifier import com.vitorpamplona.amethyst.ui.navigation.navs.INav import com.vitorpamplona.amethyst.ui.note.ReactionsRow +import com.vitorpamplona.amethyst.ui.note.types.FileHeaderAttachmentCard +import com.vitorpamplona.amethyst.ui.note.types.toMediaContent import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel import com.vitorpamplona.quartz.nip01Core.core.Event -import com.vitorpamplona.quartz.nip31Alts.alt import com.vitorpamplona.quartz.nip36SensitiveContent.isSensitiveOrNSFW import com.vitorpamplona.quartz.nip94FileMetadata.FileHeaderEvent @@ -101,45 +98,22 @@ private fun FileHeaderCardImage( val isSensitive = remember(note) { event.isSensitiveOrNSFW() } val reasons = remember(note) { collectContentWarningReasons(event) } - val isImage = remember(note) { event.mimeType()?.startsWith("image/") == true || RichTextParser.isImageUrl(fullUrl) } + val mimeType = remember(note) { event.mimeType() } val blurHash = remember(note) { event.blurhash() } val thumbHash = remember(note) { event.thumbhash() } val dimensions = remember(note) { event.dimensions() } - val content by remember(note) { - val hash = event.hash() - val description = event.content.ifEmpty { null } ?: event.alt() - val uri = note.toNostrUri() - val mimeType = event.mimeType() + val content = remember(note) { event.toMediaContent(note, fullUrl, mimeType) } - mutableStateOf( - if (isImage) { - MediaUrlImage( - url = fullUrl, - description = description, - hash = hash, - blurhash = blurHash, - dim = dimensions, - uri = uri, - mimeType = mimeType, - thumbhash = thumbHash, - ) - } else { - MediaUrlVideo( - url = fullUrl, - description = description, - hash = hash, - blurhash = blurHash, - dim = dimensions, - uri = uri, - authorName = note.author?.toBestDisplayName(), - mimeType = mimeType, - thumbhash = thumbHash, - ) - }, - ) + // VideoFeedFilter only admits image/video MIMEs and extensions, so a non-media blob should + // never reach this card. Render it as a link anyway rather than keeping an "unknown → video" + // default around: that default is what put a webxdc app into ExoPlayer in the note renderer. + if (content == null) { + FileHeaderAttachmentCard(event, fullUrl, mimeType) + return } + val isImage = content is MediaUrlImage val ratio = dimensions?.aspectRatio() ?: MediaAspectRatioCache.get(fullUrl) ContentWarningGate( diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/MediaContentKind.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/MediaContentKind.kt new file mode 100644 index 0000000000..57cce902b2 --- /dev/null +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/MediaContentKind.kt @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.richtext + +/** + * Which player/viewer can render a declared blob, as resolved by + * [RichTextParser.classifyMedia]. + * + * The set is deliberately closed: it enumerates the renderers [BaseMediaContent] actually has + * (`MediaUrlImage`, `MediaUrlVideo`, `MediaUrlPdf`), so "no constant fits" — a `null` + * classification — is the honest answer for every other file type rather than a bucket some + * caller has to invent a default for. Audio folds into [VIDEO] because both play through the + * same pipeline; see `RichTextParser.videoExt`. + */ +enum class MediaContentKind { + IMAGE, + VIDEO, + PDF, +} diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt index 60f04f7578..0ecc3b82f0 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt @@ -61,41 +61,12 @@ class RichTextParser { val contentType = frags[MimeTypeTag.TAG_NAME] ?: tags[MimeTypeTag.TAG_NAME]?.firstOrNull() - var isImage = false - var isVideo = false - var isPdf = false + // Returning null here drops the URL to a plain link, discarding the imeta's `dim`/blurhash + // and forcing a URL-preview round-trip to rediscover a type the imeta already declared — + // which is why classifyMedia falls back to the extension before giving up. + val kind = classifyMedia(fullUrl, contentType) - if (contentType != null) { - isImage = contentType.startsWith("image/") - // HLS playlists are advertised with a non-`video/*` MIME (`application/vnd.apple.mpegurl` - // and three legacy aliases). Without these, an imeta-described `.m3u8` falls into the - // null bucket below and the renderer drops back to a plain hyperlink — even though - // the matching extension would have routed it to MediaUrlVideo. Mirror the canonical - // list used by MediaItemCache.toExoPlayerMimeType / GalleryThumb.isHlsMimeType. - isVideo = contentType.startsWith("video/") || contentType.startsWith("audio/") || isHlsMimeType(contentType) - isPdf = contentType.startsWith("application/pdf") - } else if (fullUrl.startsWith("data:")) { - isImage = fullUrl.startsWith("data:image/") - isVideo = fullUrl.startsWith("data:video/") || fullUrl.startsWith("data:audio/") - isPdf = fullUrl.startsWith("data:application/pdf") - } - - // Fall back to file-extension detection when the type is still unknown. This covers both - // the no-MIME case and a *malformed* imeta MIME — e.g. Primal iOS emits `m jpeg` instead - // of `m image/jpeg`, which matches none of the `startsWith` prefixes above. Without this - // fallback such a URL returns null and drops to a plain link: that discards the imeta - // `dim`/blurhash (so the loading placeholder can't reserve the image's height and the - // feed jumps once the bitmap arrives) and forces a needless URL-preview network - // round-trip just to rediscover the type the imeta already declared. `data:` URIs carry - // their type in the prefix, so a miss there is genuine — don't extension-probe them. - if (!isImage && !isVideo && !isPdf && !fullUrl.startsWith("data:")) { - val removedParamsFromUrl = removeQueryParamsForExtensionComparison(fullUrl) - isImage = imageExtensions.any { removedParamsFromUrl.endsWith(it) } - isVideo = videoExtensions.any { removedParamsFromUrl.endsWith(it) } - isPdf = pdfExtensions.any { removedParamsFromUrl.endsWith(it) } - } - - return if (isImage) { + return if (kind == MediaContentKind.IMAGE) { MediaUrlImage( url = fullUrl, description = description ?: frags[AltTag.TAG_NAME] ?: tags[AltTag.TAG_NAME]?.firstOrNull(), @@ -108,7 +79,7 @@ class RichTextParser { thumbhash = frags[ThumbhashTag.TAG_NAME] ?: tags[ThumbhashTag.TAG_NAME]?.firstOrNull(), authorPubKey = authorPubKey, ) - } else if (isVideo) { + } else if (kind == MediaContentKind.VIDEO) { MediaUrlVideo( url = fullUrl, description = description ?: frags[AltTag.TAG_NAME] ?: tags[AltTag.TAG_NAME]?.firstOrNull(), @@ -125,7 +96,7 @@ class RichTextParser { thumbhash = frags[ThumbhashTag.TAG_NAME] ?: tags[ThumbhashTag.TAG_NAME]?.firstOrNull(), authorPubKey = authorPubKey, ) - } else if (isPdf) { + } else if (kind == MediaContentKind.PDF) { MediaUrlPdf( url = fullUrl, description = description ?: frags[AltTag.TAG_NAME] ?: tags[AltTag.TAG_NAME]?.firstOrNull(), @@ -582,6 +553,46 @@ class RichTextParser { return pdfExtensions.any { removedParamsFromUrl.endsWith(it) } } + /** + * Resolves which renderer can display a declared blob — the single decision every media + * renderer must make, from a NIP-94 `m` tag, a NIP-92 imeta, or a bare URL. + * + * A declared MIME type wins; the URL extension is the fallback both for the no-MIME case + * and for a *malformed* MIME (Primal iOS emits `m jpeg` rather than `m image/jpeg`, which + * matches no prefix below). `data:` URIs carry their type in the prefix, so a miss there is + * genuine and the base64 payload is never extension-probed. + * + * Returns **null** when nothing can render the file. Callers must not substitute a media + * kind for that null: handing an arbitrary blob — a webxdc app, a zip, an APK — to the + * video player yields a permanently-buffering ExoPlayer where a plain link belongs. The one + * defensible default is on kinds whose *event* already asserts the type (a NIP-71 video + * event is a video however odd its imeta), and those call sites say so explicitly. + */ + fun classifyMedia( + url: String, + mimeType: String?, + ): MediaContentKind? { + if (mimeType != null) { + if (mimeType.startsWith("image/")) return MediaContentKind.IMAGE + // HLS playlists are advertised with a non-`video/*` MIME; see [isHlsMimeType]. + if (mimeType.startsWith("video/") || mimeType.startsWith("audio/") || isHlsMimeType(mimeType)) return MediaContentKind.VIDEO + if (mimeType.startsWith("application/pdf")) return MediaContentKind.PDF + } else if (url.startsWith("data:")) { + if (url.startsWith("data:image/")) return MediaContentKind.IMAGE + if (url.startsWith("data:video/") || url.startsWith("data:audio/")) return MediaContentKind.VIDEO + if (url.startsWith("data:application/pdf")) return MediaContentKind.PDF + } + + if (url.startsWith("data:")) return null + + val removedParamsFromUrl = removeQueryParamsForExtensionComparison(url) + if (imageExtensions.any { removedParamsFromUrl.endsWith(it) }) return MediaContentKind.IMAGE + if (videoExtensions.any { removedParamsFromUrl.endsWith(it) }) return MediaContentKind.VIDEO + if (pdfExtensions.any { removedParamsFromUrl.endsWith(it) }) return MediaContentKind.PDF + + return null + } + fun isValidURL(url: String?): Boolean = isValidUrl(url) fun parseImageOrVideo(fullUrl: String): BaseMediaContent { diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/ClassifyMediaTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/ClassifyMediaTest.kt new file mode 100644 index 0000000000..2482a8ae80 --- /dev/null +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/ClassifyMediaTest.kt @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.richtext + +import com.vitorpamplona.quartz.nip92IMeta.IMetaTag +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull + +class ClassifyMediaTest { + @Test + fun webxdcAppIsNotMedia() { + // Regression: a NIP-94 header for a webxdc app (a zip bundle) used to reach the + // ExoPlayer branch, because the only test was `isImage` and everything else fell + // through to video. https://blossom.ditto.pub/.xdc, m=application/x-webxdc + assertNull( + RichTextParser.classifyMedia( + "https://blossom.ditto.pub/d810ba7873d710b197fc402c0573cd95ce7d44fff7f904e8f58e48af3a47c107.xdc", + "application/x-webxdc", + ), + ) + } + + @Test + fun unknownTypesAreNotMedia() { + assertNull(RichTextParser.classifyMedia("https://x.com/app.apk", "application/vnd.android.package-archive")) + assertNull(RichTextParser.classifyMedia("https://x.com/archive.zip", "application/zip")) + assertNull(RichTextParser.classifyMedia("https://x.com/notes.txt", "text/plain")) + // No mime at all and an extension we don't render. + assertNull(RichTextParser.classifyMedia("https://x.com/file.xdc", null)) + assertNull(RichTextParser.classifyMedia("https://x.com/no-extension-at-all", null)) + } + + @Test + fun declaredMimeTypesClassify() { + assertEquals(MediaContentKind.IMAGE, RichTextParser.classifyMedia("https://x.com/a", "image/png")) + assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/a", "video/mp4")) + assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/a", "audio/mpeg")) + assertEquals(MediaContentKind.PDF, RichTextParser.classifyMedia("https://x.com/a", "application/pdf")) + } + + @Test + fun hlsPlaylistMimesAreVideo() { + assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/a", "application/vnd.apple.mpegurl")) + assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/a", "application/x-mpegURL")) + assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/a", "audio/mpegurl")) + } + + @Test + fun extensionIsUsedWhenMimeIsAbsent() { + assertEquals(MediaContentKind.IMAGE, RichTextParser.classifyMedia("https://x.com/a.jpg", null)) + assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/a.mp4", null)) + assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/a.m3u8", null)) + assertEquals(MediaContentKind.PDF, RichTextParser.classifyMedia("https://x.com/a.pdf", null)) + assertEquals(MediaContentKind.IMAGE, RichTextParser.classifyMedia("https://x.com/a.PNG", null)) + } + + @Test + fun extensionRescuesAMalformedMime() { + // Primal iOS emits `m jpeg` instead of `m image/jpeg`; the extension must still win + // over "unknown". Preserves the behaviour createMediaContent already documented. + assertEquals(MediaContentKind.IMAGE, RichTextParser.classifyMedia("https://x.com/a.jpg", "jpeg")) + } + + @Test + fun queryStringsAndFragmentsAreStripped() { + assertEquals(MediaContentKind.IMAGE, RichTextParser.classifyMedia("https://x.com/a.jpg?token=1", null)) + assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/a.mp4#t=10", null)) + assertNull(RichTextParser.classifyMedia("https://x.com/a.xdc?token=1", null)) + } + + @Test + fun dataUrisAreClassifiedByTheirPrefixOnly() { + assertEquals(MediaContentKind.IMAGE, RichTextParser.classifyMedia("data:image/png;base64,AAAA", null)) + assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("data:video/mp4;base64,AAAA", null)) + assertEquals(MediaContentKind.PDF, RichTextParser.classifyMedia("data:application/pdf;base64,AAAA", null)) + // A data: URI carries its type in the prefix, so a miss there is genuine — the + // payload must never be extension-probed (base64 can end in any letters). + assertNull(RichTextParser.classifyMedia("data:application/zip;base64,AAAAmp4", null)) + } + + @Test + fun classifyMediaAgreesWithCreateMediaContent() { + // createMediaContent is the long-standing reference for this decision; the two must + // not drift, since half the renderers call one and half the other. + val cases = + listOf( + "https://x.com/a.jpg" to null, + "https://x.com/a" to "image/png", + "https://x.com/a" to "video/mp4", + "https://x.com/a" to "audio/mpeg", + "https://x.com/a" to "application/pdf", + "https://x.com/a" to "application/vnd.apple.mpegurl", + "https://x.com/a.xdc" to "application/x-webxdc", + "https://x.com/a.zip" to "application/zip", + "https://x.com/a.jpg" to "jpeg", + "data:image/png;base64,AAAA" to null, + "data:application/zip;base64,AAAAmp4" to null, + ) + + cases.forEach { (url, mime) -> + val tags = mime?.let { mapOf(url to imeta(url, it)) } ?: emptyMap() + val expected = + when (RichTextParser().createMediaContent(url, tags, null)) { + is MediaUrlImage -> MediaContentKind.IMAGE + is MediaUrlVideo -> MediaContentKind.VIDEO + is MediaUrlPdf -> MediaContentKind.PDF + null -> null + else -> error("unexpected content type for $url / $mime") + } + + assertEquals(expected, RichTextParser.classifyMedia(url, mime), "disagreement on $url / $mime") + } + } + + private fun imeta( + url: String, + mimeType: String, + ) = IMetaTag(url = url, properties = mapOf("m" to listOf(mimeType))) +} From 3565f75847c1de37046361525e2c7274413f2f88 Mon Sep 17 00:00:00 2001 From: davotoula Date: Wed, 5 Aug 2026 22:16:14 +0200 Subject: [PATCH 045/132] Code review: - gate the file-attachment card - move prettyMime to commons - add tests --- .../ui/components/FileAttachmentCard.kt | 2 +- .../amethyst/ui/note/types/SoftwareApp.kt | 22 +------- .../loggedIn/video/FileHeaderCardCompose.kt | 19 +++++-- .../amethyst/commons/util/MimeTypeLabels.kt | 52 +++++++++++++++++++ .../commons/richtext/ClassifyMediaTest.kt | 23 ++++++++ 5 files changed, 92 insertions(+), 26 deletions(-) create mode 100644 commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/util/MimeTypeLabels.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/FileAttachmentCard.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/FileAttachmentCard.kt index 759d5e7e83..ed4aba59d0 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/FileAttachmentCard.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/FileAttachmentCard.kt @@ -40,8 +40,8 @@ import com.vitorpamplona.amethyst.commons.icons.symbols.Icon import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbol import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols import com.vitorpamplona.amethyst.commons.util.countToHumanReadableBytes +import com.vitorpamplona.amethyst.commons.util.prettyMime import com.vitorpamplona.amethyst.ui.components.pdf.extractFilename -import com.vitorpamplona.amethyst.ui.note.types.prettyMime import com.vitorpamplona.amethyst.ui.theme.DoubleVertSpacer import com.vitorpamplona.amethyst.ui.theme.MaxWidthWithHorzPadding import com.vitorpamplona.amethyst.ui.theme.Size20Modifier diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/types/SoftwareApp.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/types/SoftwareApp.kt index 059fa68747..8e999414f7 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/types/SoftwareApp.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/types/SoftwareApp.kt @@ -65,6 +65,7 @@ import coil3.compose.AsyncImage import com.vitorpamplona.amethyst.R import com.vitorpamplona.amethyst.commons.richtext.MediaUrlImage import com.vitorpamplona.amethyst.commons.ui.components.ClickableTextPrimary +import com.vitorpamplona.amethyst.commons.util.prettyMime import com.vitorpamplona.amethyst.model.LocalCache import com.vitorpamplona.amethyst.model.MediaAspectRatioCache import com.vitorpamplona.amethyst.model.Note @@ -766,27 +767,6 @@ fun RenderSoftwareAsset( } } -internal fun prettyMime(mime: String): String = - when (mime) { - "application/vnd.android.package-archive" -> "APK" - "application/vnd.apple.ipa" -> "IPA" - "application/x-apple-diskimage" -> "DMG" - "application/vnd.apple.installer+xml" -> "PKG" - "application/x-msi" -> "MSI" - "application/vnd.appimage" -> "AppImage" - "application/vnd.flatpak" -> "Flatpak" - "application/vnd.oci.image.manifest.v1+json" -> "OCI" - "application/x-executable" -> "ELF" - "application/x-mach-binary" -> "Mach-O" - "application/vnd.microsoft.portable-executable" -> "EXE" - "application/vsix" -> "VSIX" - "application/x-chrome-extension" -> "CRX" - "application/x-xpinstall" -> "XPI" - "application/wasm" -> "WASM" - "application/webbundle" -> "Web Bundle" - else -> mime - } - internal fun formatBytes(bytes: Long): String { if (bytes < 1024L) return "$bytes B" val kb = bytes / 1024.0 diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/video/FileHeaderCardCompose.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/video/FileHeaderCardCompose.kt index f47613e775..d5b18976f2 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/video/FileHeaderCardCompose.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/video/FileHeaderCardCompose.kt @@ -105,11 +105,22 @@ private fun FileHeaderCardImage( val content = remember(note) { event.toMediaContent(note, fullUrl, mimeType) } - // VideoFeedFilter only admits image/video MIMEs and extensions, so a non-media blob should - // never reach this card. Render it as a link anyway rather than keeping an "unknown → video" - // default around: that default is what put a webxdc app into ExoPlayer in the note renderer. + // Reachable despite VideoFeedFilter admitting only image/video types: the filter accepts on + // `urls().any { … }` while this card renders `url()`, the first tag — so a multi-mirror event + // whose first URL is unrenderable lands here. The gate wraps it for the same reason it wraps + // the viewer in FileHeaderDisplay: a content warning is about the file, and the card still + // spells out its filename, alt text, MIME and size. Sizing stays on the gate's defaults + // (fillMaxWidth, no backdrop) — a link card has no aspect ratio to reserve and no blurhash + // to show behind it. if (content == null) { - FileHeaderAttachmentCard(event, fullUrl, mimeType) + ContentWarningGate( + isSensitive = isSensitive, + reasons = reasons, + preloadUrls = emptyList(), + accountViewModel = accountViewModel, + ) { + FileHeaderAttachmentCard(event, fullUrl, mimeType) + } return } diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/util/MimeTypeLabels.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/util/MimeTypeLabels.kt new file mode 100644 index 0000000000..1911a22019 --- /dev/null +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/util/MimeTypeLabels.kt @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.util + +/** + * The short label a user recognises for a distributable file type — "APK", not + * "application/vnd.android.package-archive". + * + * Unmapped types return the raw MIME unchanged, which is the honest fallback: a bare + * `application/x-webxdc` still tells the reader more than an invented label would. + * + * Used by the NIP-82 software-app chips and by the file-attachment card that stands in for any + * blob no viewer can render. + */ +fun prettyMime(mime: String): String = + when (mime) { + "application/vnd.android.package-archive" -> "APK" + "application/vnd.apple.ipa" -> "IPA" + "application/x-apple-diskimage" -> "DMG" + "application/vnd.apple.installer+xml" -> "PKG" + "application/x-msi" -> "MSI" + "application/vnd.appimage" -> "AppImage" + "application/vnd.flatpak" -> "Flatpak" + "application/vnd.oci.image.manifest.v1+json" -> "OCI" + "application/x-executable" -> "ELF" + "application/x-mach-binary" -> "Mach-O" + "application/vnd.microsoft.portable-executable" -> "EXE" + "application/vsix" -> "VSIX" + "application/x-chrome-extension" -> "CRX" + "application/x-xpinstall" -> "XPI" + "application/wasm" -> "WASM" + "application/webbundle" -> "Web Bundle" + else -> mime + } diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/ClassifyMediaTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/ClassifyMediaTest.kt index 2482a8ae80..240d4f0baf 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/ClassifyMediaTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/ClassifyMediaTest.kt @@ -73,6 +73,29 @@ class ClassifyMediaTest { assertEquals(MediaContentKind.IMAGE, RichTextParser.classifyMedia("https://x.com/a.PNG", null)) } + @Test + fun aDeclaredMimeBeatsAContradictingExtension() { + // The check this replaced was an OR — `mime.startsWith("image/") || isImageUrl(url)` — + // so a poster-named video URL classified as an image. A declared MIME is the publisher + // stating the type; the extension is only a guess for when they didn't. Pins the + // precedence against a future "simplification" back to OR-semantics. + assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/thumb.jpg", "video/mp4")) + assertEquals(MediaContentKind.IMAGE, RichTextParser.classifyMedia("https://x.com/clip.mp4", "image/png")) + assertEquals(MediaContentKind.PDF, RichTextParser.classifyMedia("https://x.com/scan.png", "application/pdf")) + } + + @Test + fun anUnrecognisedMimeDefersToTheExtensionRatherThanVetoingIt() { + // Precedence applies only to MIMEs we recognise. An unrecognised one means "no usable + // declaration", not "declared unrenderable" — the two are indistinguishable here, and + // treating them alike is what lets [extensionRescuesAMalformedMime] work. So a real + // video mislabelled `application/x-webxdc` still plays… + assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/bundle.mp4", "application/x-webxdc")) + // …while the webxdc app that motivated this class stays unrenderable, because nothing + // rescues it: `.xdc` is in no extension list either. + assertNull(RichTextParser.classifyMedia("https://x.com/bundle.xdc", "application/x-webxdc")) + } + @Test fun extensionRescuesAMalformedMime() { // Primal iOS emits `m jpeg` instead of `m image/jpeg`; the extension must still win From 5c18cee6ad917eb785d9bcb1f6753de6583415ed Mon Sep 17 00:00:00 2001 From: davotoula <1747287+davotoula@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:29:50 +0000 Subject: [PATCH 046/132] chore: sync Crowdin translations and seed translator npub placeholders --- amethyst/src/main/res/values-cs/strings.xml | 168 ++++++++---------- .../src/main/res/values-de-rDE/strings.xml | 156 ++++++++-------- .../src/main/res/values-pt-rBR/strings.xml | 150 ++++++++-------- .../src/main/res/values-sv-rSE/strings.xml | 154 ++++++++-------- docs/changelog/translators.json | 20 +-- 5 files changed, 312 insertions(+), 336 deletions(-) diff --git a/amethyst/src/main/res/values-cs/strings.xml b/amethyst/src/main/res/values-cs/strings.xml index 66fa414df9..6a86477be8 100644 --- a/amethyst/src/main/res/values-cs/strings.xml +++ b/amethyst/src/main/res/values-cs/strings.xml @@ -211,6 +211,7 @@ Procento úspěšných připojení k relé Vyhledat a přidat uživatele Přidat přeposílání + Neplatná adresa relaye. Použijte název hostitele nebo IP adresu v hranatých závorkách (například [201:d0e:9ba5:8bbc::1]:8080). Moje @tag jméno Zobrazované jméno Moje zobrazované jméno @@ -2024,14 +2025,91 @@ + Procházení + Média + Hashtagy + Témata + Konverzace + Hledání + Hledání chybějících událostí + Sledování událostí + Načítá podle ID události, na které se něco na obrazovce odkazuje, ale zatím je nemáte — citaci, rodiče odpovědi, kořen vlákna. + Sleduje právě zobrazené události kvůli novým odpovědím, reakcím, sdílením, zapům a nahlášením, takže se počty aktualizují během čtení. + Doplňky + Informace o relayi + Ostatní + Vyhledávač seznamů relayů + Sledování profilů + Data účtu + Domovský zdroj + Relay skupiny + + %1$d skupina + %1$d skupiny + %1$d skupiny + %1$d skupin + + Mizící chaty + Chaty podle místa + Chat živého vysílání + Skupiny NIP-29, do kterých jste vstoupili. Každá skupina žije na jednom hostitelském relayi, takže se aplikace připojí ke každému relayi, který hostí některou vaši skupinu. + Chatovací místnosti bez historie — zprávy existují jen po dobu vašeho připojení, proto zůstávají odebírané, aby vůbec něco přišlo. + Místnosti podle polohy pro oblasti, které sledujete, dotazované na relayích, které je nesou. + Chat a zapovací cíle připojené k živým vysíláním, která máte otevřená nebo sledujete. + Schránka DM + Peněženka + Schránka nutzapů + Adresář mintů + Chaty komunit + Zdroje komunit + Vaše relaye pro příjem a k tomu malý rotující vzorek relayů, kam publikují vaši sledovaní, pro případ, že by zmínka byla doručena jinam. + Vaše relaye pro schránku DM, kam se doručují zprávy zabalené v gift-wrapu. + Domovský relay každého chatu, který máte otevřený nebo do kterého jste se připojili. + Relaye, na které každá komunita publikuje své roviny. + Skupinové zprávy a balíčky klíčů na relayích každé skupiny. + Relaye místnosti, dokud je otevřená. + Váš vlastní profil, nastavení a koncepty na vašich domovských relayích. + Profily lidí právě na obrazovce. + Zjišťuje, na které relaye každý člověk publikuje, aby se jeho příspěvky daly načíst na správném místě. + Seznamy sledovaných, ze kterých se sestavuje váš zdroj a vaše síť důvěry. + Nahlášení, která vaši sledovaní napsali o profilech právě na obrazovce, dotazovaná na každém relayi, kam tito sledovaní publikují. + Nahlášení od sledovaných + Události vaší vlastní peněženky, čtené zpět z relayů, na které jste je publikovali. + Naslouchá na vašich nutzap relayích a také na relayích pro příjem a DM, aby vám neunikla žádná platba. + Prohledává relaye, které minty existují a které lidé doporučují. + Upozornění z vaší připojené peněženky. + Aktivní odběry relayů + + %1$d filtr + %1$d filtry + %1$d filtru + %1$d filtrů + + + %1$d filtr zatím není přiřazen + %1$d filtry zatím nejsou přiřazeny + %1$d filtru zatím není přiřazeno + %1$d filtrů zatím není přiřazeno + + Nepřiřazeno k žádnému účtu + Vše + Všichni + Lidé, které sledujete + Vybraný seznam lidí + Ztlumení lidé + Vaše komunity + Oblíbený algoritmický zdroj + %1$d %% ze všech + odběry subscriptions filtry relaye relay požadavky reqs připojení proč diagnostika + Příspěvky lidí, které sledujete, čtené z relayů, na které každý z nich publikuje. Připojování k inbox relayím\u2026 Služba trvalých oznámení Udržuje trvalé připojení k vašim inbox relayím pro okamžité doručování oznámení. Zobrazuje průběžné oznámení. Spotřebovává více baterie, ale zajišťuje, že nezmeškáte žádnou zprávu. @@ -4811,94 +4889,4 @@ URL avataru (volitelné) Publikování… Publikovat personu - Vše - Oblíbený algoritmický zdroj - Vaše komunity - Vybraný seznam lidí - Lidé, které sledujete - Všichni - Ztlumení lidé - odběry subscriptions filtry relaye relay požadavky reqs připojení proč diagnostika - %1$d %% ze všech - Aktivní odběry relayů - Nepřiřazeno k žádnému účtu - Váš vlastní profil, nastavení a koncepty na vašich domovských relayích. - Relaye, na které každá komunita publikuje své roviny. - Vaše relaye pro schránku DM, kam se doručují zprávy zabalené v gift-wrapu. - Skupinové zprávy a balíčky klíčů na relayích každé skupiny. - Sleduje právě zobrazené události kvůli novým odpovědím, reakcím, sdílením, zapům a nahlášením, takže se počty aktualizují během čtení. - Chatovací místnosti bez historie — zprávy existují jen po dobu vašeho připojení, proto zůstávají odebírané, aby vůbec něco přišlo. - Seznamy sledovaných, ze kterých se sestavuje váš zdroj a vaše síť důvěry. - Místnosti podle polohy pro oblasti, které sledujete, dotazované na relayích, které je nesou. - Příspěvky lidí, které sledujete, čtené z relayů, na které každý z nich publikuje. - Chat a zapovací cíle připojené k živým vysíláním, která máte otevřená nebo sledujete. - Relaye místnosti, dokud je otevřená. - Prohledává relaye, které minty existují a které lidé doporučují. - Nahlášení, která vaši sledovaní napsali o profilech právě na obrazovce, dotazovaná na každém relayi, kam tito sledovaní publikují. - Vaše relaye pro příjem a k tomu malý rotující vzorek relayů, kam publikují vaši sledovaní, pro případ, že by zmínka byla doručena jinam. - Naslouchá na vašich nutzap relayích a také na relayích pro příjem a DM, aby vám neunikla žádná platba. - Upozornění z vaší připojené peněženky. - Profily lidí právě na obrazovce. - Domovský relay každého chatu, který máte otevřený nebo do kterého jste se připojili. - Načítá podle ID události, na které se něco na obrazovce odkazuje, ale zatím je nemáte — citaci, rodiče odpovědi, kořen vlákna. - Skupiny NIP-29, do kterých jste vstoupili. Každá skupina žije na jednom hostitelském relayi, takže se aplikace připojí ke každému relayi, který hostí některou vaši skupinu. - Zjišťuje, na které relaye každý člověk publikuje, aby se jeho příspěvky daly načíst na správném místě. - Události vaší vlastní peněženky, čtené zpět z relayů, na které jste je publikovali. - Doplňky - Procházení - Chaty komunit - Zdroje komunit - Schránka DM - Sledování událostí - Mizící chaty - Chaty podle místa - Domovský zdroj - Chat živého vysílání - Média - Adresář mintů - Schránka nutzapů - Sledování profilů - Ostatní - Hledání chybějících událostí - Relay skupiny - Informace o relayi - Vyhledávač seznamů relayů - Nahlášení od sledovaných - Hledání - Hashtagy - Konverzace - Témata - Data účtu - Peněženka - Neplatná adresa relaye. Použijte název hostitele nebo IP adresu v hranatých závorkách (například [201:d0e:9ba5:8bbc::1]:8080). - - %1$d filtr - %1$d filtry - %1$d filtru - %1$d filtrů - - - %1$d skupina - %1$d skupiny - %1$d skupiny - %1$d skupin - - - %1$d relay - %1$d relaye - %1$d relaye - %1$d relayů - - - %1$d filtr zatím není přiřazen - %1$d filtry zatím nejsou přiřazeny - %1$d filtru zatím není přiřazeno - %1$d filtrů zatím není přiřazeno - - - %1$s \u00b7 %2$d relay - %1$s \u00b7 %2$d relaye - %1$s \u00b7 %2$d relaye - %1$s \u00b7 %2$d relayů - diff --git a/amethyst/src/main/res/values-de-rDE/strings.xml b/amethyst/src/main/res/values-de-rDE/strings.xml index 45e9c1cb5a..16abc0f2e7 100644 --- a/amethyst/src/main/res/values-de-rDE/strings.xml +++ b/amethyst/src/main/res/values-de-rDE/strings.xml @@ -203,6 +203,7 @@ Prozentsatz erfolgreicher Verbindungen zum Relay Benutzer suchen und hinzufügen Relay hinzufügen + Keine gültige Relay-Adresse. Verwende einen Hostnamen oder eine IP-Adresse in Klammern (zum Beispiel [201:d0e:9ba5:8bbc::1]:8080). Mein @tag-Name Anzeigename Mein Anzeigename @@ -1942,14 +1943,91 @@ + + %1$s \u00b7 %2$d Relay + %1$s \u00b7 %2$d Relays + + Stöbern + Medien + Themen + Unterhaltung + Suche + Fehlende Events finden + Events beobachten + Holt Events per ID, auf die etwas auf deinem Bildschirm verweist, die du aber noch nicht hast — ein Zitat, die übergeordnete Antwort, eine Thread-Wurzel. + Beobachtet die gerade angezeigten Events auf neue Antworten, Reaktionen, Reposts, Zaps und Meldungen, damit die Zähler beim Lesen aktuell bleiben. + Erweiterungen + Relay-Info + Sonstiges + Relay-Listen-Finder + Profile beobachten + Kontodaten + Startseiten-Feed + Relay-Gruppen + + %1$d Gruppe + %1$d Gruppen + + Verschwindende Chats + Standort-Chats + Live-Stream-Chat + NIP-29-Gruppen, denen du beigetreten bist. Jede Gruppe liegt auf einem Host-Relay, daher verbindet sich die App mit jedem Relay, das eine deiner Gruppen beherbergt. + Chaträume ohne Verlauf — Nachrichten existieren nur, solange du verbunden bist, deshalb bleiben sie abonniert, damit überhaupt etwas ankommt. + Standortbasierte Räume für die Gebiete, denen du folgst, abgefragt bei den Relays, die sie führen. + Chat und Zap-Ziele, die an Live-Streams hängen, die du geöffnet hast oder denen du folgst. + DM-Posteingang + Nutzap-Posteingang + Mint-Verzeichnis + Community-Chats + Community-Feeds + Deine Posteingangs-Relays plus eine kleine, rotierende Stichprobe der Relays, auf denen deine Gefolgten veröffentlichen, falls eine Erwähnung woanders zugestellt wurde. + Deine DM-Posteingangs-Relays, an die Gift-Wrap-Nachrichten zugestellt werden. + Das Heim-Relay jedes Chats, den du geöffnet hast oder dem du beigetreten bist. + Die Relays, auf denen jede Community ihre Planes veröffentlicht. + Gruppennachrichten und Schlüsselpakete auf den Relays der jeweiligen Gruppe. + Die Relays des Raums, solange er geöffnet ist. + Dein eigenes Profil, deine Einstellungen und Entwürfe auf deinen Heim-Relays. + Profile der gerade angezeigten Personen. + Findet heraus, auf welchen Relays jede Person veröffentlicht, damit ihre Beiträge an der richtigen Stelle abgerufen werden können. + Folgelisten, aus denen dein Feed und dein Web of Trust aufgebaut werden. + Meldungen, die deine Gefolgten über die gerade angezeigten Profile geschrieben haben, abgefragt bei jedem Relay, auf dem diese Gefolgten veröffentlichen. + Meldungen von Gefolgten + Deine eigenen Wallet-Events, zurückgelesen von den Relays, auf denen du sie veröffentlicht hast. + Lauscht auf deinen Nutzap-Relays sowie deinen Posteingangs- und DM-Relays, damit keine Zahlung durchrutscht. + Sucht über Relays hinweg, welche Mints existieren und welche empfohlen werden. + Benachrichtigungen von deiner verbundenen Wallet. + Aktive Relay-Abonnements + + %1$d Filter + %1$d Filter + + + %1$d Relay + %1$d Relays + + + %1$d Filter ist noch nicht zugeordnet + %1$d Filter sind noch nicht zugeordnet + + Keinem Konto zugeordnet + Alle + Jeder + Personen, denen du folgst + Eine ausgewählte Liste von Personen + Stummgeschaltete Personen + Deine Communitys + Ein bevorzugter Feed-Algorithmus + %1$d %% von allen + abonnements subscriptions filter relays anfragen reqs verbindungen warum diagnose + Beiträge von Personen, denen du folgst, gelesen von den Relays, auf denen jede von ihnen veröffentlicht. Verbinde mit Inbox-Relays\u2026 Dauerhafter Benachrichtigungsdienst Hält eine dauerhafte Verbindung zu deinen Inbox-Relays für sofortige Benachrichtigungen aufrecht. Zeigt eine fortlaufende Benachrichtigung an. Verbraucht mehr Akku, stellt aber sicher, dass du keine Nachricht verpasst. @@ -4635,82 +4713,4 @@ Avatar-URL (optional) Wird veröffentlicht… Persona veröffentlichen - Alle - Ein bevorzugter Feed-Algorithmus - Deine Communitys - Eine ausgewählte Liste von Personen - Personen, denen du folgst - Jeder - Stummgeschaltete Personen - abonnements subscriptions filter relays anfragen reqs verbindungen warum diagnose - %1$d %% von allen - Aktive Relay-Abonnements - Keinem Konto zugeordnet - Dein eigenes Profil, deine Einstellungen und Entwürfe auf deinen Heim-Relays. - Die Relays, auf denen jede Community ihre Planes veröffentlicht. - Deine DM-Posteingangs-Relays, an die Gift-Wrap-Nachrichten zugestellt werden. - Gruppennachrichten und Schlüsselpakete auf den Relays der jeweiligen Gruppe. - Beobachtet die gerade angezeigten Events auf neue Antworten, Reaktionen, Reposts, Zaps und Meldungen, damit die Zähler beim Lesen aktuell bleiben. - Chaträume ohne Verlauf — Nachrichten existieren nur, solange du verbunden bist, deshalb bleiben sie abonniert, damit überhaupt etwas ankommt. - Folgelisten, aus denen dein Feed und dein Web of Trust aufgebaut werden. - Standortbasierte Räume für die Gebiete, denen du folgst, abgefragt bei den Relays, die sie führen. - Beiträge von Personen, denen du folgst, gelesen von den Relays, auf denen jede von ihnen veröffentlicht. - Chat und Zap-Ziele, die an Live-Streams hängen, die du geöffnet hast oder denen du folgst. - Die Relays des Raums, solange er geöffnet ist. - Sucht über Relays hinweg, welche Mints existieren und welche empfohlen werden. - Meldungen, die deine Gefolgten über die gerade angezeigten Profile geschrieben haben, abgefragt bei jedem Relay, auf dem diese Gefolgten veröffentlichen. - Deine Posteingangs-Relays plus eine kleine, rotierende Stichprobe der Relays, auf denen deine Gefolgten veröffentlichen, falls eine Erwähnung woanders zugestellt wurde. - Lauscht auf deinen Nutzap-Relays sowie deinen Posteingangs- und DM-Relays, damit keine Zahlung durchrutscht. - Benachrichtigungen von deiner verbundenen Wallet. - Profile der gerade angezeigten Personen. - Das Heim-Relay jedes Chats, den du geöffnet hast oder dem du beigetreten bist. - Holt Events per ID, auf die etwas auf deinem Bildschirm verweist, die du aber noch nicht hast — ein Zitat, die übergeordnete Antwort, eine Thread-Wurzel. - NIP-29-Gruppen, denen du beigetreten bist. Jede Gruppe liegt auf einem Host-Relay, daher verbindet sich die App mit jedem Relay, das eine deiner Gruppen beherbergt. - Findet heraus, auf welchen Relays jede Person veröffentlicht, damit ihre Beiträge an der richtigen Stelle abgerufen werden können. - Deine eigenen Wallet-Events, zurückgelesen von den Relays, auf denen du sie veröffentlicht hast. - Erweiterungen - Stöbern - Community-Chats - Community-Feeds - DM-Posteingang - Events beobachten - Verschwindende Chats - Standort-Chats - Startseiten-Feed - Live-Stream-Chat - Medien - Mint-Verzeichnis - Nutzap-Posteingang - Profile beobachten - Sonstiges - Fehlende Events finden - Relay-Gruppen - Relay-Info - Relay-Listen-Finder - Meldungen von Gefolgten - Suche - Unterhaltung - Themen - Kontodaten - Keine gültige Relay-Adresse. Verwende einen Hostnamen oder eine IP-Adresse in Klammern (zum Beispiel [201:d0e:9ba5:8bbc::1]:8080). - - %1$d Filter - %1$d Filter - - - %1$d Gruppe - %1$d Gruppen - - - %1$d Relay - %1$d Relays - - - %1$d Filter ist noch nicht zugeordnet - %1$d Filter sind noch nicht zugeordnet - - - %1$s \u00b7 %2$d Relay - %1$s \u00b7 %2$d Relays - diff --git a/amethyst/src/main/res/values-pt-rBR/strings.xml b/amethyst/src/main/res/values-pt-rBR/strings.xml index 17b4737653..33467f8882 100644 --- a/amethyst/src/main/res/values-pt-rBR/strings.xml +++ b/amethyst/src/main/res/values-pt-rBR/strings.xml @@ -203,6 +203,7 @@ Porcentagem de conexões bem-sucedidas ao relay Pesquisar e adicionar usuário Adicionar um Relay + Endereço de relay inválido. Use um nome de host ou um endereço IP entre colchetes (por exemplo [201:d0e:9ba5:8bbc::1]:8080). Meu nome de @tag Nome de Exibição Meu nome de exibição @@ -1940,14 +1941,84 @@ + Navegação + Mídia + Tópicos + Conversa + Pesquisa + Encontrando eventos faltantes + Observando eventos + Busca por id os eventos a que algo na sua tela se refere, mas que você ainda não tem — uma citação, o pai de uma resposta, a raiz de uma conversa. + Observa os eventos exibidos no momento em busca de novas respostas, reações, repostagens, zaps e denúncias, para que as contagens sejam atualizadas enquanto você lê. + Complementos + Informações do relay + Outros + Localizador de listas de relays + Observando perfis + Dados da conta + Feed inicial + Grupos de relay + + %1$d grupo + %1$d grupos + + Chats efêmeros + Chats por localização + Chat de transmissão ao vivo + Grupos NIP-29 dos quais você participa. Cada grupo vive em um relay hospedeiro, então o app se conecta a todo relay que hospeda um grupo seu. + Salas de chat que não guardam histórico — as mensagens existem apenas enquanto você está conectado, então elas continuam assinadas para que algo chegue. + Salas baseadas em localização para as áreas que você segue, consultadas nos relays que as hospedam. + Chat e metas de zap ligados às transmissões ao vivo que você tem abertas ou segue. + Caixa de entrada de DM + Carteira + Caixa de entrada de nutzaps + Diretório de mints + Chats de comunidades + Feeds de comunidades + Os relays da sua caixa de entrada, mais uma pequena amostra rotativa dos relays em que quem você segue publica, caso uma menção tenha sido entregue em outro lugar. + Os relays da sua caixa de entrada de DM, para onde as mensagens em gift wrap são entregues. + O relay de origem de cada chat que você abriu ou do qual participa. + Os relays em que cada comunidade publica seus planos. + Mensagens de grupo e pacotes de chaves, nos relays de cada grupo. + Os relays da sala, enquanto ela estiver aberta. + Seu próprio perfil, configurações e rascunhos, nos seus relays de origem. + Perfis das pessoas atualmente na tela. + Descobre em quais relays cada pessoa publica, para que as publicações dela possam ser buscadas no lugar certo. + Listas de seguindo, usadas para montar seu feed e sua rede de confiança. + Denúncias que as pessoas que você segue escreveram sobre os perfis atualmente na sua tela, consultadas em cada relay em que essas pessoas publicam. + Denúncias de quem você segue + Os eventos da sua própria carteira, lidos de volta dos relays em que você os publicou. + Escuta nos seus relays de nutzap, além dos relays da caixa de entrada e de DM, para que nenhum pagamento passe despercebido. + Procura pelos relays quais mints existem e quais as pessoas recomendam. + Notificações da sua carteira conectada. + Assinaturas de relay ativas + + %1$d filtro + %1$d filtros + + + %1$d filtro ainda não foi atribuído + %1$d filtros ainda não foram atribuídos + + Não atribuído a nenhuma conta + Tudo + Todos + Pessoas que você segue + Uma lista escolhida de pessoas + Pessoas silenciadas + Suas comunidades + Um algoritmo de feed favorito + %1$d%% de todos + assinaturas subscriptions filtros relays requisições reqs conexões por que diagnóstico + Publicações de pessoas que você segue, lidas dos relays em que cada uma delas publica. Conectando aos relays de caixa de entrada\u2026 Serviço de notificações sempre ativo Mantém uma conexão persistente com seus relays de caixa de entrada para entrega instantânea de notificações. Mostra uma notificação contínua. Usa mais bateria, mas garante que você nunca perca uma mensagem. @@ -4640,83 +4711,4 @@ URL do avatar (opcional) Publicando… Publicar persona - Tudo - Um algoritmo de feed favorito - Suas comunidades - Uma lista escolhida de pessoas - Pessoas que você segue - Todos - Pessoas silenciadas - assinaturas subscriptions filtros relays requisições reqs conexões por que diagnóstico - %1$d%% de todos - Assinaturas de relay ativas - Não atribuído a nenhuma conta - Seu próprio perfil, configurações e rascunhos, nos seus relays de origem. - Os relays em que cada comunidade publica seus planos. - Os relays da sua caixa de entrada de DM, para onde as mensagens em gift wrap são entregues. - Mensagens de grupo e pacotes de chaves, nos relays de cada grupo. - Observa os eventos exibidos no momento em busca de novas respostas, reações, repostagens, zaps e denúncias, para que as contagens sejam atualizadas enquanto você lê. - Salas de chat que não guardam histórico — as mensagens existem apenas enquanto você está conectado, então elas continuam assinadas para que algo chegue. - Listas de seguindo, usadas para montar seu feed e sua rede de confiança. - Salas baseadas em localização para as áreas que você segue, consultadas nos relays que as hospedam. - Publicações de pessoas que você segue, lidas dos relays em que cada uma delas publica. - Chat e metas de zap ligados às transmissões ao vivo que você tem abertas ou segue. - Os relays da sala, enquanto ela estiver aberta. - Procura pelos relays quais mints existem e quais as pessoas recomendam. - Denúncias que as pessoas que você segue escreveram sobre os perfis atualmente na sua tela, consultadas em cada relay em que essas pessoas publicam. - Os relays da sua caixa de entrada, mais uma pequena amostra rotativa dos relays em que quem você segue publica, caso uma menção tenha sido entregue em outro lugar. - Escuta nos seus relays de nutzap, além dos relays da caixa de entrada e de DM, para que nenhum pagamento passe despercebido. - Notificações da sua carteira conectada. - Perfis das pessoas atualmente na tela. - O relay de origem de cada chat que você abriu ou do qual participa. - Busca por id os eventos a que algo na sua tela se refere, mas que você ainda não tem — uma citação, o pai de uma resposta, a raiz de uma conversa. - Grupos NIP-29 dos quais você participa. Cada grupo vive em um relay hospedeiro, então o app se conecta a todo relay que hospeda um grupo seu. - Descobre em quais relays cada pessoa publica, para que as publicações dela possam ser buscadas no lugar certo. - Os eventos da sua própria carteira, lidos de volta dos relays em que você os publicou. - Complementos - Navegação - Chats de comunidades - Feeds de comunidades - Caixa de entrada de DM - Observando eventos - Chats efêmeros - Chats por localização - Feed inicial - Chat de transmissão ao vivo - Mídia - Diretório de mints - Caixa de entrada de nutzaps - Observando perfis - Outros - Encontrando eventos faltantes - Grupos de relay - Informações do relay - Localizador de listas de relays - Denúncias de quem você segue - Pesquisa - Conversa - Tópicos - Dados da conta - Carteira - Endereço de relay inválido. Use um nome de host ou um endereço IP entre colchetes (por exemplo [201:d0e:9ba5:8bbc::1]:8080). - - %1$d filtro - %1$d filtros - - - %1$d grupo - %1$d grupos - - - %1$d relay - %1$d relays - - - %1$d filtro ainda não foi atribuído - %1$d filtros ainda não foram atribuídos - - - %1$s \u00b7 %2$d relay - %1$s \u00b7 %2$d relays - diff --git a/amethyst/src/main/res/values-sv-rSE/strings.xml b/amethyst/src/main/res/values-sv-rSE/strings.xml index 085d398a4c..f39e33d025 100644 --- a/amethyst/src/main/res/values-sv-rSE/strings.xml +++ b/amethyst/src/main/res/values-sv-rSE/strings.xml @@ -203,6 +203,7 @@ Andel lyckade anslutningar till reläet Sök och lägg till användare Lägg till Relä + Inte en giltig reläadress. Använd ett värdnamn eller en IP-adress inom hakparenteser (till exempel [201:d0e:9ba5:8bbc::1]:8080). Mitt @tag-namn Visningsnamn Mitt visningsnamn @@ -1940,14 +1941,88 @@ + + %1$s \u00b7 %2$d relä + %1$s \u00b7 %2$d reläer + + Bläddring + Hashtaggar + Ämnen + Konversation + Sök + Hittar saknade händelser + Observerar händelser + Hämtar händelser via id som något på din skärm hänvisar till men som du inte har ännu — ett citat, ett svars förälder, en trådrot. + Bevakar de händelser som visas just nu efter nya svar, reaktioner, återinlägg, zaps och rapporter, så att räknarna uppdateras medan du läser. + Tillägg + Reläinfo + Övrigt + Relälistsökare + Observerar profiler + Kontots data + Hemflöde + Relägrupper + + %1$d grupp + %1$d grupper + + Försvinnande chattar + Platschattar + Livesändningschatt + NIP-29-grupper du gått med i. Varje grupp bor på ett värdrelä, så appen ansluter till varje relä som är värd för en av dina grupper. + Chattrum som inte sparar någon historik — meddelanden finns bara medan du är ansluten, så dessa förblir prenumererade för att något alls ska komma fram. + Platsbaserade rum för de områden du följer, efterfrågade från de reläer som bär dem. + Chatt och zap-mål kopplade till livesändningar du har öppna eller följer. + DM-inkorg + Plånbok + Nutzap-inkorg + Mint-katalog + Gemenskapschattar + Gemenskapsflöden + Dina inkorgsreläer, plus ett litet roterande urval av de reläer personer du följer publicerar till, ifall ett omnämnande levererades någon annanstans. + Dina DM-inkorgsreläer, dit gift-wrap-meddelanden levereras. + Hemrelät för varje chatt du har öppen eller har gått med i. + Reläerna som varje gemenskap publicerar sina plan till. + Gruppmeddelanden och nyckelpaket, på varje grupps reläer. + Rummets reläer, medan det är öppet. + Din egen profil, dina inställningar och utkast, på dina hemreläer. + Profiler för personerna som just nu visas på skärmen. + Hittar vilka reläer varje person publicerar till, så att deras inlägg kan hämtas från rätt ställe. + Följerlistor, som används för att bygga ditt flöde och ditt förtroendenät. + Rapporter som personer du följer har skrivit om profilerna som just nu visas på skärmen, efterfrågade från varje relä dessa personer publicerar till. + Rapporter från personer du följer + Dina egna plånbokshändelser, lästa tillbaka från de reläer du publicerade dem till. + Lyssnar på dina nutzap-reläer plus dina inkorgs- och DM-reläer, så att en betalning inte kan slinka förbi. + Söker över reläer efter vilka mints som finns och vilka folk rekommenderar. + Aviseringar från din anslutna plånbok. + Aktiva reläprenumerationer + + %1$d relä + %1$d reläer + + + %1$d filter är inte kopplat ännu + %1$d filter är inte kopplade ännu + + Inte kopplat till något konto + Allt + Alla + Personer du följer + En vald lista med personer + Tystade personer + Dina gemenskaper + En favorit-flödesalgoritm + %1$d %% av alla + prenumerationer subscriptions filter reläer relay förfrågningar reqs anslutningar varför diagnostik + Inlägg från personer du följer, lästa från de reläer var och en av dem publicerar till. Ansluter till inbox-relän\u2026 Alltid på-notifieringstjänst Upprätthåller en konstant anslutning till dina inbox-relän för omedelbar leverans av notifieringar. Visar en pågående notifiering. Använder mer batteri men säkerställer att du aldrig missar ett meddelande. @@ -4643,83 +4718,4 @@ Avatar-URL (valfritt) Publicerar… Publicera persona - Allt - En favorit-flödesalgoritm - Dina gemenskaper - En vald lista med personer - Personer du följer - Alla - Tystade personer - prenumerationer subscriptions filter reläer relay förfrågningar reqs anslutningar varför diagnostik - %1$d %% av alla - Aktiva reläprenumerationer - Inte kopplat till något konto - Din egen profil, dina inställningar och utkast, på dina hemreläer. - Reläerna som varje gemenskap publicerar sina plan till. - Dina DM-inkorgsreläer, dit gift-wrap-meddelanden levereras. - Gruppmeddelanden och nyckelpaket, på varje grupps reläer. - Bevakar de händelser som visas just nu efter nya svar, reaktioner, återinlägg, zaps och rapporter, så att räknarna uppdateras medan du läser. - Chattrum som inte sparar någon historik — meddelanden finns bara medan du är ansluten, så dessa förblir prenumererade för att något alls ska komma fram. - Följerlistor, som används för att bygga ditt flöde och ditt förtroendenät. - Platsbaserade rum för de områden du följer, efterfrågade från de reläer som bär dem. - Inlägg från personer du följer, lästa från de reläer var och en av dem publicerar till. - Chatt och zap-mål kopplade till livesändningar du har öppna eller följer. - Rummets reläer, medan det är öppet. - Söker över reläer efter vilka mints som finns och vilka folk rekommenderar. - Rapporter som personer du följer har skrivit om profilerna som just nu visas på skärmen, efterfrågade från varje relä dessa personer publicerar till. - Dina inkorgsreläer, plus ett litet roterande urval av de reläer personer du följer publicerar till, ifall ett omnämnande levererades någon annanstans. - Lyssnar på dina nutzap-reläer plus dina inkorgs- och DM-reläer, så att en betalning inte kan slinka förbi. - Aviseringar från din anslutna plånbok. - Profiler för personerna som just nu visas på skärmen. - Hemrelät för varje chatt du har öppen eller har gått med i. - Hämtar händelser via id som något på din skärm hänvisar till men som du inte har ännu — ett citat, ett svars förälder, en trådrot. - NIP-29-grupper du gått med i. Varje grupp bor på ett värdrelä, så appen ansluter till varje relä som är värd för en av dina grupper. - Hittar vilka reläer varje person publicerar till, så att deras inlägg kan hämtas från rätt ställe. - Dina egna plånbokshändelser, lästa tillbaka från de reläer du publicerade dem till. - Tillägg - Bläddring - Gemenskapschattar - Gemenskapsflöden - DM-inkorg - Observerar händelser - Försvinnande chattar - Platschattar - Hemflöde - Livesändningschatt - Mint-katalog - Nutzap-inkorg - Observerar profiler - Övrigt - Hittar saknade händelser - Relägrupper - Reläinfo - Relälistsökare - Rapporter från personer du följer - Sök - Hashtaggar - Konversation - Ämnen - Kontots data - Plånbok - Inte en giltig reläadress. Använd ett värdnamn eller en IP-adress inom hakparenteser (till exempel [201:d0e:9ba5:8bbc::1]:8080). - - %1$d filter - %1$d filter - - - %1$d grupp - %1$d grupper - - - %1$d relä - %1$d reläer - - - %1$d filter är inte kopplat ännu - %1$d filter är inte kopplade ännu - - - %1$s \u00b7 %2$d relä - %1$s \u00b7 %2$d reläer - diff --git a/docs/changelog/translators.json b/docs/changelog/translators.json index 87f8b8a5ae..ef7dced2fc 100644 --- a/docs/changelog/translators.json +++ b/docs/changelog/translators.json @@ -90,6 +90,16 @@ "Hungarian" ] }, + { + "user": "vitorpamplona", + "languages": [ + "Czech", + "German", + "Polish", + "Portuguese, Brazilian", + "Swedish" + ] + }, { "user": "maxblake2015", "languages": [ @@ -102,16 +112,6 @@ "Hindi" ] }, - { - "user": "vitorpamplona", - "languages": [ - "Czech", - "German", - "Polish", - "Portuguese, Brazilian", - "Swedish" - ] - }, { "user": "greenart7c3", "languages": [] From cd2ce05ee8b0c2a7f3faf4ff91cf673dc64a6425 Mon Sep 17 00:00:00 2001 From: mstrofnone Date: Wed, 5 Aug 2026 15:45:39 +1000 Subject: [PATCH 047/132] ci: publish windows-arm64 desktop + windows amy/geode release assets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the linux-arm64 CI leg (feat/release-linux-arm64). Extends the release matrix to Windows in three places, all on free public-repo hosted GitHub runners: * build-desktop: adds windows-11-arm (arm64) alongside the existing windows-latest (x64). jpackage/jlink on Windows arm64 produce arm64 MSIs natively; the same packageReleaseMsi + createReleaseDistributable task list is used unchanged and the portable-archive step already parameterises on ${{ matrix.arch }}. * build-cli: adds windows-latest (x64) and windows-11-arm (arm64) legs running :cli:amyImage. Windows has no jpackageDeb/Rpm and MSI-for-CLI is deferred (portable zip is the documented Windows install path); the headless-lib assertion runs unchanged under git-bash. amyImage now emits both a POSIX `bin/amy` shell launcher AND a Windows `bin/amy.bat` launcher into the flat image so the tree layout is uniform regardless of build host. The .bat pins UTF-8 (chcp 65001) for sun.jnu.encoding, same reason the installDist .bat was already patched. * build-geode: adds windows-latest + windows-11-arm legs running :geode:geodeImage. The existing --port smoke test is generalised to pick bin/geode.bat on Windows; NIP-11 fetch via curl works unchanged under git-bash on GH windows runners. Same dual-launcher pattern as amy. scripts/asset-name.sh: collect_cli_assets and collect_geode_assets now package the flat image as .zip on Windows (7z when available, falling back to `zip`, then a portable python3 zipfile.ZipFile invocation). Every other OS continues to use tar.gz. Adds the expected Windows examples to the header block. BUILDING.md: mentions the windows-11-arm runner and updates the asset count in the Release runbook. No asset-naming contract changes — the existing amethyst-desktop--windows-., amy--windows-.zip, and geode--windows-.zip shapes were already in scope, they just weren't produced by any CI leg before. Local validation on macOS arm64 (build host: JDK 21, gradle 9.5.0): ./gradlew :cli:amyImage -> bin/amy + bin/amy.bat both present ./gradlew :geode:geodeImage -> bin/geode + bin/geode.bat both present ./bin/amy --help -> parses (unix launcher unbroken) ./bin/geode --port 17447 -> NIP-11 served, "supported_nips" present collect_cli_assets windows arm64 ... -> valid .zip with bin/amy.bat collect_geode_assets windows x64 ... -> valid .zip with bin/geode.bat actionlint .github/workflows/create-release.yml -> no new findings Cross-compile is impossible for jlink/jpackage, so end-to-end Windows-runtime validation still happens on GH CI on the first PR build; nothing in this change can be verified any harder locally. --- .github/workflows/create-release.yml | 49 +++++++++++---- BUILDING.md | 19 +++--- cli/build.gradle.kts | 37 ++++++++++-- geode/build.gradle.kts | 30 ++++++++-- scripts/asset-name.sh | 89 +++++++++++++++++++++++----- 5 files changed, 185 insertions(+), 39 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 0dfea9bfcd..ec8dd01ac0 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -44,11 +44,14 @@ jobs: matrix: # Linux legs run on x64 and arm64 GitHub-hosted runners (the # ubuntu-24.04-arm label is a standard free public-repo runner as of - # early 2025). jpackage / jlink / Compose Multiplatform 1.11 all - # produce host-native artifacts — no cross-compilation needed. + # early 2025). Windows arm64 uses windows-11-arm, added to the free + # public-repo runner catalogue in 2025 (4 vCPU / 16 GB / arm64). + # jpackage / jlink / Compose Multiplatform 1.11 all produce + # host-native artifacts — no cross-compilation needed. include: - { os: macos-14, arch: arm64, family: macos, tasks: "packageReleaseDmg" } - { os: windows-latest, arch: x64, family: windows, tasks: "packageReleaseMsi createReleaseDistributable" } + - { os: windows-11-arm, arch: arm64, family: windows, tasks: "packageReleaseMsi createReleaseDistributable" } - { os: ubuntu-latest, arch: x64, family: linux, tasks: "packageReleaseDeb packageReleaseRpm" } - { os: ubuntu-24.04-arm, arch: arm64, family: linux, tasks: "packageReleaseDeb packageReleaseRpm" } - { os: ubuntu-latest, arch: x64, family: linux-portable, tasks: "createReleaseAppImage createReleaseDistributable" } @@ -369,9 +372,17 @@ jobs: fail-fast: false matrix: include: - - { os: macos-14, arch: arm64, family: macos, tasks: "amyImage" } - - { os: ubuntu-latest, arch: x64, family: linux, tasks: "amyImage jpackageDeb jpackageRpm" } - - { os: ubuntu-24.04-arm, arch: arm64, family: linux, tasks: "amyImage jpackageDeb jpackageRpm" } + - { os: macos-14, arch: arm64, family: macos, tasks: "amyImage" } + - { os: ubuntu-latest, arch: x64, family: linux, tasks: "amyImage jpackageDeb jpackageRpm" } + - { os: ubuntu-24.04-arm, arch: arm64, family: linux, tasks: "amyImage jpackageDeb jpackageRpm" } + # Windows legs: only amyImage. .deb/.rpm are Linux-only jpackage types + # and jpackageMsi for a CLI is deferred (the portable zip is the + # documented Windows install path). The launcher script writes both + # `bin/amy` (sh) and `bin/amy.bat`, and the assertion below runs + # under bash on GH windows runners (git-bash is on PATH). collect_cli_assets + # zips the image on Windows instead of tar.gz. + - { os: windows-latest, arch: x64, family: windows, tasks: "amyImage" } + - { os: windows-11-arm, arch: arm64, family: windows, tasks: "amyImage" } runs-on: ${{ matrix.os }} timeout-minutes: 45 # macOS leg also codesigns + notarizes the jlink image defaults: @@ -619,9 +630,16 @@ jobs: fail-fast: false matrix: include: - - { os: macos-14, arch: arm64, family: macos, tasks: "geodeImage" } - - { os: ubuntu-latest, arch: x64, family: linux, tasks: "geodeImage jpackageDeb jpackageRpm" } - - { os: ubuntu-24.04-arm, arch: arm64, family: linux, tasks: "geodeImage jpackageDeb jpackageRpm" } + - { os: macos-14, arch: arm64, family: macos, tasks: "geodeImage" } + - { os: ubuntu-latest, arch: x64, family: linux, tasks: "geodeImage jpackageDeb jpackageRpm" } + - { os: ubuntu-24.04-arm, arch: arm64, family: linux, tasks: "geodeImage jpackageDeb jpackageRpm" } + # Windows legs: geodeImage only. The .deb/.rpm are Linux-only; MSI is + # deferred (portable zip covers the primary use — operators still + # deploy geode via the Docker image or the tarball on Linux). The + # image writes both `bin/geode` (sh) and `bin/geode.bat`, and the + # smoke test below runs under bash on the windows runner. + - { os: windows-latest, arch: x64, family: windows, tasks: "geodeImage" } + - { os: windows-11-arm, arch: arm64, family: windows, tasks: "geodeImage" } runs-on: ${{ matrix.os }} timeout-minutes: 45 # macOS leg also codesigns + notarizes the jlink image defaults: @@ -676,12 +694,23 @@ jobs: # module list is complete for the real relay path (Ktor CIO + SQLite + # NIP-11 serialization) — a too-tight module list links fine but fails # here with NoClassDefFound instead of on an operator's machine. + # + # On the Windows legs we invoke bin/geode.bat instead of bin/geode. The + # tmp path also differs between git-bash on Windows (which resolves /tmp + # to a mingw path that curl -o accepts) and POSIX runners; kept identical + # because the workflow's `defaults.run.shell: bash` uses git-bash on + # Windows and /tmp is a valid mingw path there. - name: Smoke-test the geode image run: | set -euo pipefail IMG="geode/build/geode-image/geode" - "$IMG/bin/geode" --version - "$IMG/bin/geode" --port 17447 & + if [[ "${{ matrix.family }}" == "windows" ]]; then + LAUNCHER="$IMG/bin/geode.bat" + else + LAUNCHER="$IMG/bin/geode" + fi + "$LAUNCHER" --version + "$LAUNCHER" --port 17447 & PID=$! ok=0 for i in $(seq 1 20); do diff --git a/BUILDING.md b/BUILDING.md index b931926c00..680d4caad3 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -35,7 +35,9 @@ All platforms: Platform-specific: - **macOS**: Xcode Command Line Tools (`xcode-select --install`) -- **Windows**: WiX Toolset 3.x on PATH (for MSI). `winget install WiXToolset.WiXToolset` +- **Windows**: WiX Toolset 3.x on PATH (for MSI). `winget install WiXToolset.WiXToolset`. + Windows arm64 builds run on the free public-repo `windows-11-arm` GitHub runner — + jpackage on Windows arm64 produces arm64 MSIs natively; no cross-compilation. - **Linux (all)**: nothing extra for `.deb`; `rpm` + `fakeroot` for `.rpm`; `appimagetool` + `desktop-file-utils` for AppImage; `flatpak` + `flatpak-builder` for the Flatpak bundle (see @@ -328,14 +330,17 @@ Quartz library in one pipeline. 3. **Wait** for the `Create Release Assets` workflow to finish (~25–30 min). -4. **Verify** — the GH Release should hold **31 assets**: - - **8 desktop** — `dmg` (macOS arm64), `msi` + `zip` (Windows), `deb`, `rpm`, - `AppImage`, `flatpak`, `tar.gz` (Linux). There is **no Intel/x64 macOS - DMG** — `jpackage` cannot cross-compile and no Intel runner leg is - configured, so macOS ships arm64-only. +4. **Verify** — the GH Release should hold **37 assets**: + - **10 desktop** — `dmg` (macOS arm64); `msi` + `zip` per Windows arch + (x64 and arm64, 4 files); `deb`, `rpm`, `AppImage`, `flatpak`, `tar.gz` + for Linux x64+arm64 (5 formats × 2 arches shipped as one merged set of + 5 in the current layout — see the previous release for the exact + enumeration). There is **no Intel/x64 macOS DMG** — `jpackage` cannot + cross-compile and no Intel runner leg is configured, so macOS ships + arm64-only. - **13 Android** — 5 Google Play APKs + 5 F-Droid APKs + 2 AABs + the F-Droid `.apks` set built for Accrescent. - - **5 amy** + **5 geode** bundles. + - **7 amy** + **7 geode** bundles (5 unix + 2 Windows portable zips each). - Asset sizes look sane (see §Enforce asset size budget — CI auto-fails at 1 GB/asset) - Android flow unchanged diff --git a/cli/build.gradle.kts b/cli/build.gradle.kts index d92227363e..6aa88d026f 100644 --- a/cli/build.gradle.kts +++ b/cli/build.gradle.kts @@ -254,7 +254,14 @@ val jlinkRuntime = } // Flat app-image: bin/amy launcher + lib/*.jar + runtime/ (the jlink'd JRE). -// Cross-platform — the release workflow tars this up on every OS. +// Cross-platform — the release workflow archives this on every OS (tar.gz on +// unix, zip on Windows). We write BOTH a POSIX `amy` shell launcher AND a +// Windows `amy.bat` launcher into `bin/` unconditionally so the same tree is +// runnable on any target after extraction, regardless of which OS built it. +// (The bundled jlink runtime is host-native — you still need to unzip a +// Windows-built image on Windows to actually launch it — but the launcher +// scripts themselves are host-agnostic, which keeps the layout uniform and +// makes ad-hoc cross-machine inspection painless.) val amyImage = tasks.register("amyImage") { group = "distribution" @@ -282,13 +289,35 @@ val amyImage = DIR="${'$'}(cd "${'$'}(dirname "${'$'}0")/.." && pwd)" exec "${'$'}DIR/runtime/bin/java" -Djava.awt.headless=true -cp "${'$'}DIR/lib/*" $mainClass "${'$'}@" """.trimIndent() + "\n" + // Windows launcher. Uses %~dp0 (drive+path of this .bat, always ending in + // a backslash) so it resolves the app root without depending on CWD, then + // execs the bundled JRE against lib\*. `chcp 65001` pins the console to + // UTF-8 so `sun.jnu.encoding` isn't the OS OEM code page — same rationale + // as the installDist launcher patch above. CRLF line endings so cmd.exe + // parses it correctly. + // + // The `for %%i in (...) do set DIR=%%~fi` trick canonicalises `\bin\..` + // out of DIR to the parent directory — same idiom Gradle's own + // installDist .bat uses to resolve APP_HOME. Java tolerates the `..` + // segment but canonicalising once here keeps every classpath entry and + // error message clean (and matches the loose-directory layout users see + // after unzipping the release archive). + val windowsLauncher = + "@echo off\r\n" + + "chcp 65001 > NUL 2>&1\r\n" + + "setlocal\r\n" + + "set \"DIR=%~dp0..\"\r\n" + + "for %%i in (\"%DIR%\") do set \"DIR=%%~fi\"\r\n" + + "\"%DIR%\\runtime\\bin\\java.exe\" -Djava.awt.headless=true -cp \"%DIR%\\lib\\*\" $mainClass %*\r\n" doLast { val binDir = amyImageDir.get().asFile.resolve("bin") binDir.mkdirs() - val launcher = binDir.resolve("amy") - launcher.writeText(unixLauncher) - launcher.setExecutable(true, false) + val unix = binDir.resolve("amy") + unix.writeText(unixLauncher) + unix.setExecutable(true, false) + val windows = binDir.resolve("amy.bat") + windows.writeText(windowsLauncher) } } diff --git a/geode/build.gradle.kts b/geode/build.gradle.kts index 70254af99c..2c8efa600f 100644 --- a/geode/build.gradle.kts +++ b/geode/build.gradle.kts @@ -236,7 +236,9 @@ val jlinkRuntime = // Flat app-image: bin/geode launcher + lib/*.jar + runtime/ (the jlink'd JRE) + // share/geode/ (config.example.toml + the systemd unit). Cross-platform — the -// release workflow tars this up on every OS. +// release workflow archives it on every OS (tar.gz on unix, zip on Windows). +// Both a POSIX shell launcher and a Windows .bat launcher are written so the +// tree layout is uniform regardless of build host. val geodeImage = tasks.register("geodeImage") { group = "distribution" @@ -271,13 +273,33 @@ val geodeImage = DIR="${'$'}(cd "${'$'}(dirname "${'$'}0")/.." && pwd)" exec "${'$'}DIR/runtime/bin/java" -cp "${'$'}DIR/lib/*" $mainClass "${'$'}@" """.trimIndent() + "\n" + // Windows launcher: %~dp0 anchors on the .bat's own directory (drive+ + // path, always trailing backslash) so geode.bat works no matter where + // it's invoked from. CRLF for cmd.exe. We do NOT force UTF-8 here — the + // relay is a network daemon that logs and speaks JSON over sockets, and + // its stdout is machine-readable; leaving the console code page alone + // matches the geode launcher on POSIX which similarly doesn't touch + // LANG. + // + // The `for %%i in (...) do set DIR=%%~fi` trick canonicalises `\bin\..` + // out of DIR to the parent directory — same idiom Gradle's own + // installDist .bat uses to resolve APP_HOME. See the matching comment on + // the amy launcher for the rationale (log cleanliness, not correctness). + val windowsLauncher = + "@echo off\r\n" + + "setlocal\r\n" + + "set \"DIR=%~dp0..\"\r\n" + + "for %%i in (\"%DIR%\") do set \"DIR=%%~fi\"\r\n" + + "\"%DIR%\\runtime\\bin\\java.exe\" -cp \"%DIR%\\lib\\*\" $mainClass %*\r\n" doLast { val binDir = geodeImageDir.get().asFile.resolve("bin") binDir.mkdirs() - val launcher = binDir.resolve("geode") - launcher.writeText(unixLauncher) - launcher.setExecutable(true, false) + val unix = binDir.resolve("geode") + unix.writeText(unixLauncher) + unix.setExecutable(true, false) + val windows = binDir.resolve("geode.bat") + windows.writeText(windowsLauncher) } } diff --git a/scripts/asset-name.sh b/scripts/asset-name.sh index ee7b2bda98..100e7247b2 100755 --- a/scripts/asset-name.sh +++ b/scripts/asset-name.sh @@ -24,6 +24,8 @@ # amethyst-desktop-1.08.0-macos-arm64.dmg # amethyst-desktop-1.08.0-windows-x64.msi # amethyst-desktop-1.08.0-windows-x64.zip +# amethyst-desktop-1.08.0-windows-arm64.msi +# amethyst-desktop-1.08.0-windows-arm64.zip # amethyst-desktop-1.08.0-linux-x64.deb # amethyst-desktop-1.08.0-linux-x64.rpm # amethyst-desktop-1.08.0-linux-x64.AppImage @@ -42,6 +44,8 @@ # amy-1.08.0-linux-arm64.tar.gz # amy-1.08.0-linux-arm64.deb # amy-1.08.0-linux-arm64.rpm +# amy-1.08.0-windows-x64.zip +# amy-1.08.0-windows-arm64.zip # geode-1.08.0-macos-arm64.tar.gz # geode-1.08.0-linux-x64.tar.gz # geode-1.08.0-linux-x64.deb @@ -49,6 +53,8 @@ # geode-1.08.0-linux-arm64.tar.gz # geode-1.08.0-linux-arm64.deb # geode-1.08.0-linux-arm64.rpm +# geode-1.08.0-windows-x64.zip +# geode-1.08.0-windows-arm64.zip # # Two assets break the family/arch shape on purpose: the no-JRE jar bundles for # Homebrew-core are pure JVM bytecode (no bundled runtime), so a single @@ -121,10 +127,14 @@ collect_assets() { # # Expected inputs: # cli/build/amy-image/amy/ flat app-image built by :cli:amyImage -# (bin/amy + lib/*.jar + runtime/) +# (bin/amy + bin/amy.bat + lib/*.jar + runtime/) # cli/build/jpackage/*.deb from :cli:jpackageDeb (Linux only) # cli/build/jpackage/*.rpm from :cli:jpackageRpm (Linux only) # +# On Windows the flat image is packaged as .zip (native archive format, +# preserves file layout without requiring a tar tool at install time). Every +# other OS uses tar.gz. +# # Usage: collect_cli_assets collect_cli_assets() { local family="$1" arch="$2" version="$3" dest="$4" @@ -133,14 +143,39 @@ collect_cli_assets() { abs_dest="$(cd "$dest" && pwd)" shopt -s nullglob - # 1. Tar the flat app-image into amy---.tar.gz. - # This is the portable-across-OS asset — macOS runners produce only - # this one. + # 1. Archive the flat app-image into amy---.. + # macOS runners produce only this one. Windows uses .zip; every other + # OS uses tar.gz. local app_image="cli/build/amy-image/amy" if [ -d "$app_image" ]; then - local tarball="$abs_dest/$(cli_asset_name "$family" "$arch" "$version" tar.gz)" - ( cd "$(dirname "$app_image")" && tar czf "$tarball" "$(basename "$app_image")" ) - echo "Collected: $tarball" + if [ "$family" = "windows" ]; then + local zipfile="$abs_dest/$(cli_asset_name "$family" "$arch" "$version" zip)" + # Prefer 7z when available (bash+7zip is standard on GH windows runners), + # else fall back to a portable python3 zipfile. `zip` itself is not always + # present on GH windows runners. + if command -v 7z >/dev/null 2>&1; then + ( cd "$(dirname "$app_image")" && 7z a -tzip "$zipfile" "$(basename "$app_image")/" >/dev/null ) + elif command -v zip >/dev/null 2>&1; then + ( cd "$(dirname "$app_image")" && zip -qr "$zipfile" "$(basename "$app_image")" ) + else + python3 - "$app_image" "$zipfile" <<'PY' +import os, sys, zipfile +src, dst = sys.argv[1], sys.argv[2] +root = os.path.dirname(src) +base = os.path.basename(src) +with zipfile.ZipFile(dst, "w", zipfile.ZIP_DEFLATED) as zf: + for dirpath, _dirs, files in os.walk(src): + for f in files: + p = os.path.join(dirpath, f) + zf.write(p, os.path.relpath(p, root)) +PY + fi + echo "Collected: $zipfile" + else + local tarball="$abs_dest/$(cli_asset_name "$family" "$arch" "$version" tar.gz)" + ( cd "$(dirname "$app_image")" && tar czf "$tarball" "$(basename "$app_image")" ) + echo "Collected: $tarball" + fi fi # 2. Linux native installers (.deb, .rpm). jpackage writes them directly @@ -166,10 +201,14 @@ collect_cli_assets() { # # Expected inputs: # geode/build/geode-image/geode/ flat app-image built by :geode:geodeImage -# (bin/geode + lib/*.jar + runtime/ + share/) +# (bin/geode + bin/geode.bat + lib/*.jar +# + runtime/ + share/) # geode/build/jpackage/*.deb from :geode:jpackageDeb (Linux only) # geode/build/jpackage/*.rpm from :geode:jpackageRpm (Linux only) # +# On Windows the flat image is packaged as .zip; every other OS uses tar.gz. +# See the matching comment in collect_cli_assets for the tool-selection order. +# # Usage: collect_geode_assets collect_geode_assets() { local family="$1" arch="$2" version="$3" dest="$4" @@ -178,14 +217,36 @@ collect_geode_assets() { abs_dest="$(cd "$dest" && pwd)" shopt -s nullglob - # 1. Tar the flat app-image into geode---.tar.gz. - # This is the portable-across-OS asset — macOS runners produce only - # this one. + # 1. Archive the flat app-image into geode---.. + # macOS runners produce only this one. Windows uses .zip; every other + # OS uses tar.gz. local app_image="geode/build/geode-image/geode" if [ -d "$app_image" ]; then - local tarball="$abs_dest/$(geode_asset_name "$family" "$arch" "$version" tar.gz)" - ( cd "$(dirname "$app_image")" && tar czf "$tarball" "$(basename "$app_image")" ) - echo "Collected: $tarball" + if [ "$family" = "windows" ]; then + local zipfile="$abs_dest/$(geode_asset_name "$family" "$arch" "$version" zip)" + if command -v 7z >/dev/null 2>&1; then + ( cd "$(dirname "$app_image")" && 7z a -tzip "$zipfile" "$(basename "$app_image")/" >/dev/null ) + elif command -v zip >/dev/null 2>&1; then + ( cd "$(dirname "$app_image")" && zip -qr "$zipfile" "$(basename "$app_image")" ) + else + python3 - "$app_image" "$zipfile" <<'PY' +import os, sys, zipfile +src, dst = sys.argv[1], sys.argv[2] +root = os.path.dirname(src) +base = os.path.basename(src) +with zipfile.ZipFile(dst, "w", zipfile.ZIP_DEFLATED) as zf: + for dirpath, _dirs, files in os.walk(src): + for f in files: + p = os.path.join(dirpath, f) + zf.write(p, os.path.relpath(p, root)) +PY + fi + echo "Collected: $zipfile" + else + local tarball="$abs_dest/$(geode_asset_name "$family" "$arch" "$version" tar.gz)" + ( cd "$(dirname "$app_image")" && tar czf "$tarball" "$(basename "$app_image")" ) + echo "Collected: $tarball" + fi fi # 2. Linux native installers (.deb, .rpm). jpackage writes them directly From f3104f0a6c4f45b30b7a941befd04f9c7efeee24 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Wed, 5 Aug 2026 17:29:51 -0400 Subject: [PATCH 048/132] ci(release): build windows-arm64 desktop as a portable zip only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The windows-11-arm leg added by the previous commit runs `packageReleaseMsi`, which cannot succeed on that runner: jpackage --type msi shells out to WiX 3's heat.exe / candle.exe / light.exe (JDK 21 jpackage guide names WiX 3.11.1), and the Windows 11 Arm64 runner image ships no WiX at all. Verified against actions/runner-images: images/windows/Windows2025-Readme.md -> "WiX Toolset 3.14.1.8722" images/windows/Windows11-Arm64-Readme.md -> no WiX entry (7zip 26.02, Python 3.13 and Java 21 aarch64 ARE present on the arm64 image, so the rest of the leg — createReleaseDistributable, the 7z portable zip, collect_assets — is unaffected.) Installing WiX in the job instead was the alternative and is worse: wixtoolset/wix3 was archived in Feb 2025, WiX 4+ replaced the candle/light CLI that jpackage drives with `wix build`, and the WiX 3 binaries are x86-only (emulated on arm64). That would mean pulling an archived, unpinned third-party toolchain into the job that publishes signed release assets, for one asset we already ship in portable form. So: arm64 Windows gets the portable .zip, which is already the documented Windows install path for amy and geode. The x64 leg is untouched and still produces the MSI. collect_assets needs no change — it globs with nullglob and skips the absent msi/ directory. BUILDING.md: replace the release-verification asset count, which this branch had left vague ("5 formats x 2 arches shipped as one merged set of 5 ... see the previous release"), with a per-leg enumeration counted off the matrix: 14 desktop + 13 Android + 10 amy + 10 geode = 47. Also corrects the Windows prerequisites note, which claimed CI produces arm64 MSIs natively. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/create-release.yml | 14 ++++++++++- BUILDING.md | 35 +++++++++++++++++++--------- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index ec8dd01ac0..72643e9e40 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -48,10 +48,22 @@ jobs: # public-repo runner catalogue in 2025 (4 vCPU / 16 GB / arm64). # jpackage / jlink / Compose Multiplatform 1.11 all produce # host-native artifacts — no cross-compilation needed. + # + # The arm64 Windows leg builds the portable .zip ONLY — no MSI. + # jpackage --type msi shells out to WiX 3's heat/candle/light, and the + # windows-11-arm runner image ships no WiX (the windows-latest image + # has WiX 3.14 preinstalled, which is why the x64 leg can package an + # MSI). Installing it here would mean pulling an archived, x86-only + # toolchain (wixtoolset/wix3 was archived in Feb 2025; WiX 4+ dropped + # the candle/light CLI that JDK 21's jpackage requires) into the job + # that publishes signed release assets. The portable zip is the + # documented Windows install path for amy/geode already, so arm64 + # Windows users get that until either the runner image gains WiX or + # jpackage learns the WiX 4+ CLI. include: - { os: macos-14, arch: arm64, family: macos, tasks: "packageReleaseDmg" } - { os: windows-latest, arch: x64, family: windows, tasks: "packageReleaseMsi createReleaseDistributable" } - - { os: windows-11-arm, arch: arm64, family: windows, tasks: "packageReleaseMsi createReleaseDistributable" } + - { os: windows-11-arm, arch: arm64, family: windows, tasks: "createReleaseDistributable" } - { os: ubuntu-latest, arch: x64, family: linux, tasks: "packageReleaseDeb packageReleaseRpm" } - { os: ubuntu-24.04-arm, arch: arm64, family: linux, tasks: "packageReleaseDeb packageReleaseRpm" } - { os: ubuntu-latest, arch: x64, family: linux-portable, tasks: "createReleaseAppImage createReleaseDistributable" } diff --git a/BUILDING.md b/BUILDING.md index 680d4caad3..aa29c7ba11 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -36,8 +36,11 @@ Platform-specific: - **macOS**: Xcode Command Line Tools (`xcode-select --install`) - **Windows**: WiX Toolset 3.x on PATH (for MSI). `winget install WiXToolset.WiXToolset`. - Windows arm64 builds run on the free public-repo `windows-11-arm` GitHub runner — - jpackage on Windows arm64 produces arm64 MSIs natively; no cross-compilation. + Windows arm64 builds run on the free public-repo `windows-11-arm` GitHub runner + and produce the portable `.zip` only — that image ships no WiX, so CI cannot + package an arm64 MSI. Locally you *can* build one on an arm64 Windows box with + WiX 3.x installed (jpackage produces host-native artifacts; the WiX 3 binaries + themselves are x86 and run under emulation). - **Linux (all)**: nothing extra for `.deb`; `rpm` + `fakeroot` for `.rpm`; `appimagetool` + `desktop-file-utils` for AppImage; `flatpak` + `flatpak-builder` for the Flatpak bundle (see @@ -330,17 +333,27 @@ Quartz library in one pipeline. 3. **Wait** for the `Create Release Assets` workflow to finish (~25–30 min). -4. **Verify** — the GH Release should hold **37 assets**: - - **10 desktop** — `dmg` (macOS arm64); `msi` + `zip` per Windows arch - (x64 and arm64, 4 files); `deb`, `rpm`, `AppImage`, `flatpak`, `tar.gz` - for Linux x64+arm64 (5 formats × 2 arches shipped as one merged set of - 5 in the current layout — see the previous release for the exact - enumeration). There is **no Intel/x64 macOS DMG** — `jpackage` cannot - cross-compile and no Intel runner leg is configured, so macOS ships - arm64-only. +4. **Verify** — the GH Release should hold **47 assets**: + - **14 desktop**, one per matrix leg × format: + - macOS arm64: `dmg` (1) + - Windows x64: `msi` + portable `zip` (2) + - Windows arm64: portable `zip` only (1) — **no arm64 MSI**, see below + - Linux x64 / arm64: `deb` + `rpm` (4) + - Linux-portable x64 / arm64: `AppImage` + `tar.gz` + `flatpak` (6) + + There is **no Intel/x64 macOS DMG** — `jpackage` cannot cross-compile + and no Intel runner leg is configured, so macOS ships arm64-only. + There is **no Windows arm64 MSI**: `jpackage --type msi` shells out to + WiX 3's `heat`/`candle`/`light`, and the `windows-11-arm` runner image + ships no WiX (`windows-latest` has WiX 3.14 preinstalled, which is why + the x64 leg gets an MSI). Revisit if that image gains WiX, or if + jpackage learns the WiX 4+ `wix build` CLI. - **13 Android** — 5 Google Play APKs + 5 F-Droid APKs + 2 AABs + the F-Droid `.apks` set built for Accrescent. - - **7 amy** + **7 geode** bundles (5 unix + 2 Windows portable zips each). + - **10 amy** — `tar.gz` (macOS arm64, Linux x64, Linux arm64), + `deb` + `rpm` per Linux arch, portable `zip` per Windows arch, and the + one arch-independent no-JRE `amy--jvm.tar.gz` for Homebrew-core. + - **10 geode** — same shape as amy. - Asset sizes look sane (see §Enforce asset size budget — CI auto-fails at 1 GB/asset) - Android flow unchanged From ff9855aad614661bccfd84b0a7358f3320bf56d2 Mon Sep 17 00:00:00 2001 From: mstrofnone Date: Wed, 29 Jul 2026 09:01:04 +1000 Subject: [PATCH 049/132] feat(gitRepositories): add ngit-specific search on the Git Repositories screen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a client-side filter for the Git Repositories page that only searches fields relevant to ngit repository announcements (kind:30617 NIP-34): repo name, `d` identifier, description, hashtags/topics, clone URLs, web URLs, maintainer relays, maintainer/author pubkeys (accepting both hex and `npub…` bech32 in the query), and the earliest-unique-commit hash. Before this change, the only search affordance on the screen was the generic Nostr search icon that navigated away to `Route.Search`, which matches people, notes, hashtags, and channels — none of which are ngit repositories. Users who wanted to find a repo they'd already discovered had to scroll through the full follow-list-scoped feed. UX: - A filter icon in the top bar toggles an inline `OutlinedTextField` directly above the feed. First-appearance focus opens the keyboard without a second tap. - The general search icon is preserved beside the filter icon so outbound searches still work. - While filtering, results render with the same `NoteCompose` cells the feed uses so every affordance (bookmark, open, share) still works. - Filtered rendering uses a scoped `LazyListState` because the item-key set of the filtered list is not stable against the feed's cached scroll offset; sharing them would jump the user to an unrelated repo. - Closing the filter icon clears the query, restoring the full feed in one tap. Filter semantics: - Whitespace-separated terms are ANDed against each repo (`amethyst nostr` matches only repos that carry both terms in some indexed field). - Case-insensitive substring match on each indexed field. - `npub1…` queries are decoded to hex before matching, so a maintainer can be found by either encoding. Tests: `GitRepositorySearchMatcherTest` (17 hermetic cases) pins every indexed field, plus the "empty query returns nothing / filter blank returns everything" contract that the caller relies on to skip the filter path. Build check: `./gradlew :amethyst:compileFdroidDebugKotlin :amethyst:testFdroidDebugUnitTest --tests 'com.vitorpamplona.amethyst.ui.screen.loggedIn.gitRepositories.GitRepositorySearchMatcherTest' :amethyst:spotlessCheck` all green. --- .../gitRepositories/GitRepositoriesScreen.kt | 218 +++++++++++++++++- .../gitRepositories/GitRepositoriesTopBar.kt | 92 +++++++- .../GitRepositorySearchMatcher.kt | 134 +++++++++++ amethyst/src/main/res/values/strings.xml | 4 + .../GitRepositorySearchMatcherTest.kt | 194 ++++++++++++++++ 5 files changed, 622 insertions(+), 20 deletions(-) create mode 100644 amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositorySearchMatcher.kt create mode 100644 amethyst/src/test/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositorySearchMatcherTest.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositoriesScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositoriesScreen.kt index 3b03b5ba4c..6dcee4d39b 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositoriesScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositoriesScreen.kt @@ -20,11 +20,35 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.gitRepositories +import androidx.compose.foundation.ExperimentalFoundationApi +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.itemsIndexed +import androidx.compose.foundation.lazy.rememberLazyListState +import androidx.compose.material3.HorizontalDivider +import androidx.compose.material3.IconButton +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedTextField +import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.saveable.rememberSaveable +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.focus.FocusRequester +import androidx.compose.ui.focus.focusRequester +import androidx.compose.ui.unit.dp import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.vitorpamplona.amethyst.R import com.vitorpamplona.amethyst.commons.ui.feeds.FeedContentState +import com.vitorpamplona.amethyst.commons.ui.layouts.rememberFeedContentPadding import com.vitorpamplona.amethyst.ui.feeds.RefresheableBox import com.vitorpamplona.amethyst.ui.feeds.RenderFeedContentState import com.vitorpamplona.amethyst.ui.feeds.SaveableFeedContentState @@ -34,8 +58,17 @@ import com.vitorpamplona.amethyst.ui.layouts.DisappearingScaffold import com.vitorpamplona.amethyst.ui.navigation.bottombars.AppBottomBar import com.vitorpamplona.amethyst.ui.navigation.navs.INav import com.vitorpamplona.amethyst.ui.navigation.routes.Route +import com.vitorpamplona.amethyst.ui.note.ClearTextIcon +import com.vitorpamplona.amethyst.ui.note.NoteCompose +import com.vitorpamplona.amethyst.ui.note.SearchIcon import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel import com.vitorpamplona.amethyst.ui.screen.loggedIn.gitRepositories.datasource.GitRepositoriesFilterAssemblerSubscription +import com.vitorpamplona.amethyst.ui.stringRes +import com.vitorpamplona.amethyst.ui.theme.DividerThickness +import com.vitorpamplona.amethyst.ui.theme.FeedPadding +import com.vitorpamplona.amethyst.ui.theme.Size20Modifier +import com.vitorpamplona.amethyst.ui.theme.placeholderText +import com.vitorpamplona.quartz.nip34Git.repository.GitRepositoryEvent @Composable fun GitRepositoriesScreen( @@ -59,10 +92,31 @@ fun GitRepositoriesScreen( WatchAccountForGitRepositoriesScreen(gitRepositoriesFeedContentState = gitRepositoriesFeedContentState, accountViewModel = accountViewModel) GitRepositoriesFilterAssemblerSubscription(accountViewModel) + // Search UI state is remembered across configuration changes so the + // user doesn't lose their query when rotating; scoped to this screen, + // not persisted to disk (unlike the follow-list filter above). + var isSearchOpen by rememberSaveable { mutableStateOf(false) } + var searchQuery by rememberSaveable { mutableStateOf("") } + DisappearingScaffold( isInvertedLayout = false, topBar = { - GitRepositoriesTopBar(accountViewModel, nav) + GitRepositoriesTopBar( + isSearchOpen = isSearchOpen, + onToggleSearch = { + // Closing collapses the field AND clears the query so + // the feed is fully restored — the icon acts as a + // one-tap "reset" once the user has narrowed the view. + if (isSearchOpen) { + searchQuery = "" + isSearchOpen = false + } else { + isSearchOpen = true + } + }, + accountViewModel = accountViewModel, + nav = nav, + ) }, bottomBar = { AppBottomBar(Route.GitRepositories, nav, accountViewModel) { route -> @@ -75,20 +129,166 @@ fun GitRepositoriesScreen( }, accountViewModel = accountViewModel, ) { - RefresheableBox(gitRepositoriesFeedContentState, true) { - SaveableFeedContentState(gitRepositoriesFeedContentState, scrollStateKey = ScrollStateKeys.GIT_REPOSITORIES_SCREEN) { listState -> - RenderFeedContentState( - feedContentState = gitRepositoriesFeedContentState, - accountViewModel = accountViewModel, - listState = listState, - nav = nav, - routeForLastRead = "GitRepositoriesFeed", + Column(Modifier.fillMaxSize()) { + if (isSearchOpen) { + GitRepositorySearchField( + query = searchQuery, + onQueryChange = { searchQuery = it }, + onClearQuery = { searchQuery = "" }, ) + HorizontalDivider(thickness = DividerThickness) + } + RefresheableBox(gitRepositoriesFeedContentState, true) { + SaveableFeedContentState(gitRepositoriesFeedContentState, scrollStateKey = ScrollStateKeys.GIT_REPOSITORIES_SCREEN) { listState -> + val query = searchQuery + if (query.isBlank()) { + RenderFeedContentState( + feedContentState = gitRepositoriesFeedContentState, + accountViewModel = accountViewModel, + listState = listState, + nav = nav, + routeForLastRead = "GitRepositoriesFeed", + ) + } else { + // When the filter is active we can't reuse the shared + // scroll state because the filtered list has a different + // set of item keys — using the same LazyListState would + // make Compose try to restore an index that no longer + // exists and jump the user to an unrelated repo. We + // scope a fresh, per-query LazyListState so scrolling + // stays inside the filtered view. + RenderFilteredFeed( + feedContentState = gitRepositoriesFeedContentState, + query = query, + accountViewModel = accountViewModel, + nav = nav, + ) + } + } } } } } +/** + * Inline text field that drives the client-side ngit-repository search. Sits + * directly under the top bar and above the feed so the user can see the + * result of every keystroke narrow the list beneath it. + */ +@Composable +private fun GitRepositorySearchField( + query: String, + onQueryChange: (String) -> Unit, + onClearQuery: () -> Unit, +) { + val focusRequester = remember { FocusRequester() } + LaunchedEffect(Unit) { + // Focus on first appearance so the keyboard opens without a second + // tap. Subsequent recompositions inside the same session don't re- + // request focus, which would fight with the user pressing "back to + // the feed" via the field's clear-text icon. + focusRequester.requestFocus() + } + + Row(Modifier.fillMaxWidth().padding(horizontal = 12.dp, vertical = 4.dp)) { + OutlinedTextField( + value = query, + onValueChange = onQueryChange, + modifier = Modifier.fillMaxWidth().focusRequester(focusRequester), + placeholder = { + Text( + text = stringRes(R.string.git_repositories_search_placeholder), + color = MaterialTheme.colorScheme.placeholderText, + ) + }, + leadingIcon = { SearchIcon(modifier = Size20Modifier, MaterialTheme.colorScheme.placeholderText) }, + trailingIcon = { + if (query.isNotEmpty()) { + IconButton(onClick = onClearQuery) { + ClearTextIcon() + } + } + }, + singleLine = true, + ) + } +} + +/** + * Renders the ngit repositories the user is already subscribed to, filtered + * by [query]. Loading and error states are delegated to the shared + * [RenderFeedContentState] via the appropriate branches; the loaded branch + * is intercepted so we can filter the notes without touching the shared + * feed model (which other screens also observe). + */ +@OptIn(ExperimentalFoundationApi::class) +@Composable +private fun RenderFilteredFeed( + feedContentState: FeedContentState, + query: String, + accountViewModel: AccountViewModel, + nav: INav, +) { + val filteredListState = rememberLazyListState() + + RenderFeedContentState( + feedContentState = feedContentState, + accountViewModel = accountViewModel, + listState = filteredListState, + nav = nav, + routeForLastRead = "GitRepositoriesFeed", + onLoaded = { loaded -> + val loadedItems by loaded.feed.collectAsStateWithLifecycle() + + val filtered = + remember(loadedItems, query) { + loadedItems.list.filter { note -> + val event = note.event as? GitRepositoryEvent ?: return@filter false + GitRepositorySearchMatcher.matches(event, query) + } + } + + if (filtered.isEmpty()) { + Column( + modifier = Modifier.fillMaxSize().padding(24.dp), + ) { + Text( + text = stringRes(R.string.git_repositories_search_no_results), + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + } else { + LazyColumn( + contentPadding = rememberFeedContentPadding(FeedPadding), + state = filteredListState, + modifier = Modifier.fillMaxSize(), + ) { + itemsIndexed( + filtered, + key = { _, item -> item.idHex }, + contentType = { _, item -> item.event?.kind ?: -1 }, + ) { _, item -> + Row(Modifier.fillMaxWidth().animateItem()) { + NoteCompose( + item, + modifier = Modifier.fillMaxWidth(), + routeForLastRead = "GitRepositoriesFeed", + isBoostedNote = false, + isHiddenFeed = loadedItems.showHidden, + quotesLeft = 3, + accountViewModel = accountViewModel, + nav = nav, + ) + } + HorizontalDivider(thickness = DividerThickness) + } + } + } + }, + ) +} + @Composable fun WatchAccountForGitRepositoriesScreen( gitRepositoriesFeedContentState: FeedContentState, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositoriesTopBar.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositoriesTopBar.kt index 4c30842a95..2d15903c9b 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositoriesTopBar.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositoriesTopBar.kt @@ -20,35 +20,105 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.gitRepositories +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.IconButton +import androidx.compose.material3.MaterialTheme import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier import androidx.lifecycle.compose.collectAsStateWithLifecycle import com.vitorpamplona.amethyst.R +import com.vitorpamplona.amethyst.commons.icons.symbols.Icon +import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols import com.vitorpamplona.amethyst.model.TopFilter import com.vitorpamplona.amethyst.ui.navigation.navs.INav +import com.vitorpamplona.amethyst.ui.navigation.routes.Route import com.vitorpamplona.amethyst.ui.navigation.topbars.FeedFilterSpinner -import com.vitorpamplona.amethyst.ui.navigation.topbars.UserDrawerSearchTopBar +import com.vitorpamplona.amethyst.ui.navigation.topbars.ShorterTopAppBar +import com.vitorpamplona.amethyst.ui.navigation.topbars.TopBarNavigationIcon +import com.vitorpamplona.amethyst.ui.note.SearchIcon import com.vitorpamplona.amethyst.ui.screen.FeedDefinition import com.vitorpamplona.amethyst.ui.screen.TopNavFilterState import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel import com.vitorpamplona.amethyst.ui.stringRes +import com.vitorpamplona.amethyst.ui.theme.Size22Modifier +import com.vitorpamplona.amethyst.ui.theme.placeholderText +/** + * Top bar for the ngit repositories discovery screen. + * + * Two search affordances live side-by-side in the actions row: + * + * 1. A **repository filter** (magnifier-with-a-plus icon) that toggles + * an inline text field over the loaded feed. This is the ngit-specific + * search — it matches the fields NIP-34 announcements carry: name, + * identifier, description, hashtags, clone/web/relay URLs, and + * maintainer pubkeys. It filters what the user is already looking + * at without touching relays. + * + * 2. The **generic Nostr search** (plain magnifier) that navigates to + * the global [Route.Search] screen, matching the affordance on + * every other top-level screen. + * + * Splitting them this way makes it obvious which magnifier does what: the + * inline one narrows the current list, the outbound one opens the fleet- + * wide search that also queries people, notes, hashtags, etc. + */ +@OptIn(ExperimentalMaterial3Api::class) @Composable fun GitRepositoriesTopBar( + isSearchOpen: Boolean, + onToggleSearch: () -> Unit, accountViewModel: AccountViewModel, nav: INav, ) { - UserDrawerSearchTopBar(accountViewModel, nav) { - val list by accountViewModel.account.settings.defaultGitRepositoriesFollowList - .collectAsStateWithLifecycle() + ShorterTopAppBar( + title = { + Column( + modifier = Modifier.fillMaxWidth(), + horizontalAlignment = Alignment.CenterHorizontally, + verticalArrangement = Arrangement.Center, + ) { + val list by accountViewModel.account.settings.defaultGitRepositoriesFollowList + .collectAsStateWithLifecycle() - GitRepositoriesTopNavFilterBar( - followListsModel = accountViewModel.feedStates.feedListOptions, - listName = list, - accountViewModel = accountViewModel, - onChange = accountViewModel.account.settings::changeDefaultGitRepositoriesFollowList, - ) - } + GitRepositoriesTopNavFilterBar( + followListsModel = accountViewModel.feedStates.feedListOptions, + listName = list, + accountViewModel = accountViewModel, + onChange = accountViewModel.account.settings::changeDefaultGitRepositoriesFollowList, + ) + } + }, + navigationIcon = { TopBarNavigationIcon(accountViewModel, nav) }, + actions = { + IconButton(onClick = onToggleSearch) { + Icon( + symbol = + if (isSearchOpen) { + MaterialSymbols.Close + } else { + MaterialSymbols.FilterAlt + }, + contentDescription = + stringRes( + if (isSearchOpen) { + R.string.git_repositories_search_close + } else { + R.string.git_repositories_search_open + }, + ), + ) + } + IconButton(onClick = { nav.nav(Route.Search) }) { + SearchIcon(modifier = Size22Modifier, MaterialTheme.colorScheme.placeholderText) + } + }, + ) } @Composable diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositorySearchMatcher.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositorySearchMatcher.kt new file mode 100644 index 0000000000..4f40260a88 --- /dev/null +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositorySearchMatcher.kt @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.ui.screen.loggedIn.gitRepositories + +import com.vitorpamplona.quartz.nip19Bech32.Nip19Parser +import com.vitorpamplona.quartz.nip19Bech32.entities.NPub +import com.vitorpamplona.quartz.nip34Git.repository.GitRepositoryEvent + +/** + * Local, in-memory matcher for the Git Repositories screen search box. + * + * The screen already loads the full set of ngit repository announcements + * (`kind:30617` `GitRepositoryEvent`) the user has subscribed to via their + * follow lists / follow set, so a client-side filter avoids issuing an + * extra NIP-50 relay query for the common "I know its name/topic/host" + * lookup. It also matches on fields the generic NIP-50 search would + * ignore — clone/web URLs, maintainer npubs, the ngit `d` identifier + * — which are exactly what someone browsing repos on gitworkshop / + * ngit tends to remember. + * + * The query is split on whitespace so `"amethyst nostr"` requires each + * term to appear in at least one indexed field of the same repository. + * Every term match is case-insensitive. + * + * Indexed fields: + * - repo name (`name` tag) + * - repo identifier (`d` tag; what appears in the ngit URL path) + * - description + * - hashtags/topics (`t` tags) + * - clone URLs (`clone` tag values) + * - web URLs (`web` tag values) + * - relay URLs the maintainers listen on (`relays` tag values) + * - maintainer pubkeys (both hex and NIP-19 `npub…` form) + * - repo author pubkey (hex and npub) + * - earliest-unique-commit hash (`r … euc`) — lets you paste a commit + * hash from a nostr:naddr and land on the repo + */ +object GitRepositorySearchMatcher { + /** + * @return `true` when [event] matches every whitespace-separated term + * in [query]. An empty query matches nothing (callers should skip the + * filter path in that case). + */ + fun matches( + event: GitRepositoryEvent, + query: String, + ): Boolean { + val terms = query.trim().split(WHITESPACE).filter { it.isNotEmpty() } + if (terms.isEmpty()) return false + + val haystack = buildHaystack(event) + return terms.all { term -> + val needle = term.lowercase() + // Support "npub1…" queries by resolving them to hex; the hex + // form is already in the haystack via authorNpubs / dTag / + // maintainers. + val hexFromBech32 = tryDecodeNpubToHex(needle) + haystack.any { field -> field.contains(needle) } || + (hexFromBech32 != null && haystack.any { field -> field.contains(hexFromBech32) }) + } + } + + /** + * Same as [matches] but returns the list of unique repositories + * ordered by the caller-provided iteration order. Duplicate `d` + * tags collapse to the newest event, because a maintainer publishing + * two revisions of `amethyst` is still one repo. + */ + fun filter( + events: Sequence, + query: String, + ): List { + if (query.isBlank()) return events.toList() + return events.filter { matches(it, query) }.toList() + } + + private fun buildHaystack(event: GitRepositoryEvent): List { + val out = ArrayList(16) + event.name()?.lowercase()?.let(out::add) + event + .dTag() + .takeIf { it.isNotEmpty() } + ?.lowercase() + ?.let(out::add) + event.description()?.lowercase()?.let(out::add) + event.hashtags().forEach { out.add(it.lowercase()) } + event.clones().forEach { out.add(it.lowercase()) } + event.webs().forEach { out.add(it.lowercase()) } + event.relays().forEach { out.add(it.lowercase()) } + // Maintainers as hex + npub. Author is an implicit maintainer per + // NIP-34, so include it in both forms too. + val authors = HashSet() + authors.add(event.pubKey) + authors.addAll(event.maintainers()) + authors.forEach { hex -> + out.add(hex.lowercase()) + hexToNpub(hex)?.let { out.add(it.lowercase()) } + } + event.earliestUniqueCommit()?.lowercase()?.let(out::add) + return out + } + + private val WHITESPACE = Regex("\\s+") + + private fun tryDecodeNpubToHex(candidate: String): String? { + if (!candidate.startsWith("npub1")) return null + return runCatching { + when (val parsed = Nip19Parser.uriToRoute(candidate)?.entity) { + is NPub -> parsed.hex + else -> null + } + }.getOrNull() + } + + private fun hexToNpub(hex: String): String? = runCatching { NPub.create(hex) }.getOrNull() +} diff --git a/amethyst/src/main/res/values/strings.xml b/amethyst/src/main/res/values/strings.xml index 4f0003f2ba..b6b362ed51 100644 --- a/amethyst/src/main/res/values/strings.xml +++ b/amethyst/src/main/res/values/strings.xml @@ -3930,6 +3930,10 @@ Save Git Repositories Highlights + Filter repositories + Close filter + Filter by name, topic, host, maintainer… + No repositories in the current feed match this search. nSite: %1$s nApplet: %1$s Permissions: diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositorySearchMatcherTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositorySearchMatcherTest.kt new file mode 100644 index 0000000000..7703d9f853 --- /dev/null +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositorySearchMatcherTest.kt @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.ui.screen.loggedIn.gitRepositories + +import com.vitorpamplona.quartz.nip19Bech32.entities.NPub +import com.vitorpamplona.quartz.nip34Git.repository.GitRepositoryEvent +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * Pins the ngit-repository-relevant search matcher used by + * `GitRepositoriesScreen`. Every field the matcher promises to index is + * exercised once here so a future refactor that drops one (e.g. relays) + * shows up as a red test. + */ +class GitRepositorySearchMatcherTest { + private val ownerHex = "aa".repeat(32) + private val maintainerHex = "bb".repeat(32) + private val ownerNpub = NPub.create(ownerHex) + private val maintainerNpub = NPub.create(maintainerHex) + + private fun repo( + name: String = "amethyst", + dTag: String = "amethyst", + description: String? = "A Nostr client for Android", + clones: List = listOf("https://github.com/vitorpamplona/amethyst.git"), + webs: List = listOf("https://amethyst.social"), + relays: List = listOf("wss://relay.ngit.dev"), + maintainers: List = listOf(maintainerHex), + hashtags: List = listOf("nostr", "android"), + euc: String? = "99614f07e4ffa99dff4143d7457be8923690bbba", + pubKey: String = ownerHex, + ): GitRepositoryEvent { + val tags = mutableListOf>() + tags += arrayOf("d", dTag) + tags += arrayOf("name", name) + description?.let { tags += arrayOf("description", it) } + clones.forEach { tags += arrayOf("clone", it) } + webs.forEach { tags += arrayOf("web", it) } + if (relays.isNotEmpty()) tags += arrayOf("relays", *relays.toTypedArray()) + if (maintainers.isNotEmpty()) tags += arrayOf("maintainers", *maintainers.toTypedArray()) + hashtags.forEach { tags += arrayOf("t", it) } + euc?.let { tags += arrayOf("r", it, "euc") } + return GitRepositoryEvent( + id = "00".repeat(32), + pubKey = pubKey, + createdAt = 0L, + tags = tags.toTypedArray(), + content = "", + sig = "00", + ) + } + + @Test + fun emptyQueryMatchesNothing() { + // Callers must skip the filter path themselves — the matcher is + // conservative and refuses to accept an empty term list. + assertFalse(GitRepositorySearchMatcher.matches(repo(), "")) + assertFalse(GitRepositorySearchMatcher.matches(repo(), " ")) + } + + @Test + fun matchesRepoNameCaseInsensitive() { + assertTrue(GitRepositorySearchMatcher.matches(repo(name = "Amethyst"), "amethyst")) + assertTrue(GitRepositorySearchMatcher.matches(repo(name = "Amethyst"), "AMET")) + } + + @Test + fun matchesRepoIdentifierDTag() { + assertTrue(GitRepositorySearchMatcher.matches(repo(dTag = "ngit-cli"), "ngit-cli")) + } + + @Test + fun matchesDescription() { + assertTrue( + GitRepositorySearchMatcher.matches( + repo(description = "A private Nostr messenger"), + "messenger", + ), + ) + } + + @Test + fun matchesHashtag() { + assertTrue(GitRepositorySearchMatcher.matches(repo(hashtags = listOf("kotlin", "mobile")), "kotlin")) + } + + @Test + fun matchesCloneUrl() { + assertTrue( + GitRepositorySearchMatcher.matches( + repo(clones = listOf("https://relay.ngit.dev/npub1abc/foo.git")), + "relay.ngit.dev", + ), + ) + } + + @Test + fun matchesWebUrl() { + assertTrue(GitRepositorySearchMatcher.matches(repo(webs = listOf("https://gitworkshop.dev/x")), "gitworkshop")) + } + + @Test + fun matchesRelayHost() { + assertTrue( + GitRepositorySearchMatcher.matches( + repo(relays = listOf("wss://relay.damus.io")), + "damus.io", + ), + ) + } + + @Test + fun matchesMaintainerHex() { + assertTrue(GitRepositorySearchMatcher.matches(repo(), maintainerHex)) + } + + @Test + fun matchesMaintainerNpub() { + // The `bb…` npub is a valid bech32 pubkey; supplying it as a + // query must resolve to the same hex the tag carries. + assertTrue(GitRepositorySearchMatcher.matches(repo(), maintainerNpub)) + } + + @Test + fun matchesAuthorHex() { + assertTrue(GitRepositorySearchMatcher.matches(repo(), ownerHex)) + } + + @Test + fun matchesAuthorNpub() { + assertTrue(GitRepositorySearchMatcher.matches(repo(), ownerNpub)) + } + + @Test + fun matchesEarliestUniqueCommit() { + // Full euc must match; a prefix that lives inside it should too. + assertTrue(GitRepositorySearchMatcher.matches(repo(euc = "99614f07e4ff"), "99614f07")) + } + + @Test + fun multipleTermsMustAllMatch() { + val target = repo(name = "amethyst", hashtags = listOf("nostr", "android")) + assertTrue(GitRepositorySearchMatcher.matches(target, "amethyst android")) + // "kotlin" isn't in this repo's fields, so the AND fails. + assertFalse(GitRepositorySearchMatcher.matches(target, "amethyst kotlin")) + } + + @Test + fun invalidNpubTreatedAsRawText() { + // "npub1notreallybech32" is not a decodable npub; the matcher + // should still let it match as a raw substring of e.g. the + // description, without throwing. + val target = repo(description = "npub1notreallybech32 is a placeholder") + assertTrue(GitRepositorySearchMatcher.matches(target, "npub1notreallybech32")) + } + + @Test + fun filterReturnsAllMatches() { + val a = repo(name = "amethyst") + val b = repo(name = "ngit-cli", dTag = "ngit-cli", hashtags = listOf("rust", "git")) + val c = repo(name = "shakespeare", dTag = "shakespeare") + val hits = GitRepositorySearchMatcher.filter(sequenceOf(a, b, c), "rust") + assertEquals(listOf(b), hits) + } + + @Test + fun filterBlankQueryReturnsEverything() { + val a = repo(name = "amethyst") + val b = repo(name = "ngit-cli") + val hits = GitRepositorySearchMatcher.filter(sequenceOf(a, b), " ") + assertEquals(listOf(a, b), hits) + } +} From 221214e545d168c66796606b725d3eab55034957 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Wed, 5 Aug 2026 18:13:31 -0400 Subject: [PATCH 050/132] refactor(commons): move GitRepositorySearchMatcher to commons/search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The matcher is pure platform-agnostic Kotlin over a Quartz event type — no Compose, no Android, no platform APIs — so per the sharing philosophy it belongs in commons rather than amethyst. commons/ARCHITECTURE.md assigns "event search filtering/ranking" to the `search` package (non-UI, CLI-safe), which is where it lands. The desktop Git Repositories screen can now use the same matcher instead of growing its own copy. The test moves to commons/src/commonTest and swaps org.junit for kotlin.test, matching every other test in that source set. That gains iOS coverage for free, and keeps the source set compiling for the native targets — commonTest is built for iosArm64/iosSimulatorArm64 too, so a JUnit import there is a build break, not a style nit. The test builds GitRepositoryEvent from raw tags and never signs, so it needs no secp256k1 binding. Also drops `filter()`. It had no caller — the screen filters the loaded feed itself with `matches` — and its KDoc promised behaviour it never implemented ("duplicate `d` tags collapse to the newest event"; it did no deduplication at all). Better to delete the unused API than to ship a dedup nobody asked for or a doc comment that lies. Its two tests go with it; the empty-query contract the screen does rely on stays covered. Verified: :commons:jvmTest (15/15 in the new location), :commons:compileTestKotlinIosSimulatorArm64, :commons:verifyKmpPurity, :amethyst:compileFdroidDebugKotlin, spotlessApply — all green. Co-Authored-By: Claude Opus 5 (1M context) --- .../gitRepositories/GitRepositoriesScreen.kt | 1 + .../search}/GitRepositorySearchMatcher.kt | 16 +----------- .../search}/GitRepositorySearchMatcherTest.kt | 26 +++---------------- 3 files changed, 6 insertions(+), 37 deletions(-) rename {amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories => commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/search}/GitRepositorySearchMatcher.kt (89%) rename {amethyst/src/test/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories => commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/search}/GitRepositorySearchMatcherTest.kt (88%) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositoriesScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositoriesScreen.kt index 6dcee4d39b..ee32e6e1b4 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositoriesScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositoriesScreen.kt @@ -47,6 +47,7 @@ import androidx.compose.ui.focus.focusRequester import androidx.compose.ui.unit.dp import androidx.lifecycle.compose.collectAsStateWithLifecycle import com.vitorpamplona.amethyst.R +import com.vitorpamplona.amethyst.commons.search.GitRepositorySearchMatcher import com.vitorpamplona.amethyst.commons.ui.feeds.FeedContentState import com.vitorpamplona.amethyst.commons.ui.layouts.rememberFeedContentPadding import com.vitorpamplona.amethyst.ui.feeds.RefresheableBox diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositorySearchMatcher.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/search/GitRepositorySearchMatcher.kt similarity index 89% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositorySearchMatcher.kt rename to commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/search/GitRepositorySearchMatcher.kt index 4f40260a88..90d5ed1f5c 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositorySearchMatcher.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/search/GitRepositorySearchMatcher.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.ui.screen.loggedIn.gitRepositories +package com.vitorpamplona.amethyst.commons.search import com.vitorpamplona.quartz.nip19Bech32.Nip19Parser import com.vitorpamplona.quartz.nip19Bech32.entities.NPub @@ -78,20 +78,6 @@ object GitRepositorySearchMatcher { } } - /** - * Same as [matches] but returns the list of unique repositories - * ordered by the caller-provided iteration order. Duplicate `d` - * tags collapse to the newest event, because a maintainer publishing - * two revisions of `amethyst` is still one repo. - */ - fun filter( - events: Sequence, - query: String, - ): List { - if (query.isBlank()) return events.toList() - return events.filter { matches(it, query) }.toList() - } - private fun buildHaystack(event: GitRepositoryEvent): List { val out = ArrayList(16) event.name()?.lowercase()?.let(out::add) diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositorySearchMatcherTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/search/GitRepositorySearchMatcherTest.kt similarity index 88% rename from amethyst/src/test/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositorySearchMatcherTest.kt rename to commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/search/GitRepositorySearchMatcherTest.kt index 7703d9f853..592eeb7ce4 100644 --- a/amethyst/src/test/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/gitRepositories/GitRepositorySearchMatcherTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/search/GitRepositorySearchMatcherTest.kt @@ -18,14 +18,13 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.ui.screen.loggedIn.gitRepositories +package com.vitorpamplona.amethyst.commons.search import com.vitorpamplona.quartz.nip19Bech32.entities.NPub import com.vitorpamplona.quartz.nip34Git.repository.GitRepositoryEvent -import org.junit.Assert.assertEquals -import org.junit.Assert.assertFalse -import org.junit.Assert.assertTrue -import org.junit.Test +import kotlin.test.Test +import kotlin.test.assertFalse +import kotlin.test.assertTrue /** * Pins the ngit-repository-relevant search matcher used by @@ -174,21 +173,4 @@ class GitRepositorySearchMatcherTest { val target = repo(description = "npub1notreallybech32 is a placeholder") assertTrue(GitRepositorySearchMatcher.matches(target, "npub1notreallybech32")) } - - @Test - fun filterReturnsAllMatches() { - val a = repo(name = "amethyst") - val b = repo(name = "ngit-cli", dTag = "ngit-cli", hashtags = listOf("rust", "git")) - val c = repo(name = "shakespeare", dTag = "shakespeare") - val hits = GitRepositorySearchMatcher.filter(sequenceOf(a, b, c), "rust") - assertEquals(listOf(b), hits) - } - - @Test - fun filterBlankQueryReturnsEverything() { - val a = repo(name = "amethyst") - val b = repo(name = "ngit-cli") - val hits = GitRepositorySearchMatcher.filter(sequenceOf(a, b), " ") - assertEquals(listOf(a, b), hits) - } } From bcbf78d8ea42d01e210360450363fd1520cd6e39 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 23:04:09 +0000 Subject: [PATCH 051/132] fix(ime): settle the keyboard in Nav so every screen is covered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Leaving a screen while the soft keyboard is still animating strands `imePadding()` at keyboard height for the whole app — `WindowInsets.ime` is a single shared holder, so the padding survives leaving the screen that caused it. PR #3864 fixed this for the post composers, at their call sites. That was the wrong altitude: Search strands it too, and Search has no BackHandler and no top bar of ours. Search is the clearest case: it focuses its field on arrival, so the keyboard is up before the user has done anything, and every way out is a navigation — a bottom-nav tab, a tapped result, back. Any destination that can focus a text field can strand the padding on the way out. There are 174 files with text input in this module; enumerating the screens was never going to converge. Two facts make a central fix possible: every in-app navigation goes through INav (there is not one `controller.navigate` outside navigation/navs/, and nothing touches OnBackPressedDispatcher, navigateUp or popBackStack directly), and every Nav method already runs inside `navigationScope.launch`. So Nav awaits an ImeSettler before each transition: keyboard down, it returns immediately and nothing changes; keyboard up, it clears focus, hides the IME and waits for the inset to actually reach zero, bounded, so the two animations never overlap. ObservableNav delegates to Nav and inherits it. That subsumes #3864's call-site patches, so they are removed rather than left as a second mechanism: ActionTopBar goes back to plain callbacks (which also drops the composition-scoped deferral of onPost, so posting no longer depends on the top bar staying composed), and KeyboardAwareBackHandler keeps only its imeAnimationTarget gate — the part that stops back falling through and silently dropping a draft. It is now a UX preference (let the system animate the dismissal) rather than the safety mechanism. NavImeSettleTest pins the ordering: each transition must settle before it navigates, and a settler that suspends must hold the navigation back rather than run alongside it. All four fail with the settle calls removed. Known gap: on a screen with no BackHandler the system's back pops through the NavController directly, not Nav.popBack(), so a second back landing inside the ~250ms retraction can still race. Closing it needs a shell-level handler registered after the NavHost to outrank its back callback, which is a composition-order dependency subtle enough to break silently — worth a deliberate decision rather than smuggling in here. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LfUMGWYu2uTSyh17JJonfN --- .../ui/navigation/bottombars/KeyboardState.kt | 89 ++----------- .../amethyst/ui/navigation/navs/ImeSettler.kt | 89 +++++++++++++ .../amethyst/ui/navigation/navs/Nav.kt | 13 ++ .../ui/navigation/navs/RememberNavs.kt | 5 +- .../ui/navigation/topbars/ActionTopBar.kt | 9 +- .../ui/navigation/NavImeSettleTest.kt | 118 ++++++++++++++++++ 6 files changed, 238 insertions(+), 85 deletions(-) create mode 100644 amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/navs/ImeSettler.kt create mode 100644 amethyst/src/test/java/com/vitorpamplona/amethyst/ui/navigation/NavImeSettleTest.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/KeyboardState.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/KeyboardState.kt index f576a3556c..78ea948462 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/KeyboardState.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/KeyboardState.kt @@ -30,15 +30,7 @@ import androidx.compose.runtime.State import androidx.compose.runtime.derivedStateOf import androidx.compose.runtime.getValue import androidx.compose.runtime.remember -import androidx.compose.runtime.rememberCoroutineScope -import androidx.compose.runtime.snapshotFlow import androidx.compose.ui.platform.LocalDensity -import androidx.compose.ui.platform.LocalFocusManager -import androidx.compose.ui.platform.LocalSoftwareKeyboardController -import kotlinx.coroutines.flow.first -import kotlinx.coroutines.launch -import kotlinx.coroutines.withTimeoutOrNull -import java.util.concurrent.atomic.AtomicBoolean enum class KeyboardState { Opened, @@ -69,80 +61,26 @@ fun keyboardAsState(): State { } } -/** How long to wait for the IME inset to reach zero before running the action anyway. */ -private const val IME_SETTLE_TIMEOUT_MS = 700L - -/** - * Returns a runner that defers an action until the soft keyboard is fully off screen. - * - * Popping a screen while the keyboard is still up races the window animation against the IME's - * close animation. On release builds — fast enough that the window animation wins — the IME - * [WindowInsetsAnimationCompat][androidx.core.view.WindowInsetsAnimationCompat] is cancelled before - * its terminal (zero) frame reaches Compose, so the shared `WindowInsets.ime` holder stays - * "animating" and every `Modifier.imePadding()` in the app freezes at the keyboard height until a - * later inset pass rebalances it (the "stuck IME padding" that survives leaving the screen). - * - * Any exit that leaves a keyboard-bearing screen has to serialize the two animations rather than - * overlap them. With the keyboard already down the action runs inline — same frame, no behavior - * change. With it up we dismiss the keyboard, wait for the inset to actually reach zero, and only - * then act, so the IME animation always completes before the window animation begins. - * - * Re-entrant calls while an action is pending are dropped: the deferral widens the window in which - * a second tap on a Post/Save button would fire the action twice. - * - * [IME_SETTLE_TIMEOUT_MS] bounds the wait — if the inset never reports zero (precisely the failure - * this guards against) the action still runs, so a stale reading can never trap the user on screen. - */ -@Composable -fun rememberAfterKeyboardCloses(): (() -> Unit) -> Unit { - val density = LocalDensity.current - val imeInsets = WindowInsets.ime - val keyboard = LocalSoftwareKeyboardController.current - val focusManager = LocalFocusManager.current - val scope = rememberCoroutineScope() - val pending = remember { AtomicBoolean(false) } - - return remember(density, imeInsets, keyboard, focusManager, scope, pending) { - { action: () -> Unit -> - if (imeInsets.getBottom(density) <= 0) { - action() - } else if (pending.compareAndSet(false, true)) { - // Clear focus first so nothing re-requests the IME as it retracts. - focusManager.clearFocus(true) - keyboard?.hide() - scope.launch { - try { - withTimeoutOrNull(IME_SETTLE_TIMEOUT_MS) { - snapshotFlow { imeInsets.getBottom(density) }.first { it <= 0 } - } - action() - } finally { - pending.set(false) - } - } - } - } - } -} - /** * A [BackHandler] that lets the system dismiss the soft keyboard before it consumes back. * - * Chat composers (and draft-saving editors) intercept back to flush a draft and pop the screen, - * which is the pop-during-IME-animation race described on [rememberAfterKeyboardCloses]. While the - * keyboard is up we do NOT consume back, so the system dismisses it first with its own animation - * (which completes cleanly, and on recent Android follows the back gesture). The next back runs - * [onBack] as before. + * Chat composers (and draft-saving editors) intercept back to flush a draft and pop the screen. + * While the keyboard is up we do NOT consume back, so the system dismisses it first with its own + * animation — which completes cleanly, and on recent Android follows the back gesture rather than + * snapping. The next back runs [onBack] as before. + * + * This is a UX preference, not the safety mechanism: the actual pop-during-IME-animation race is + * handled for every exit in the app by + * [ImeSettler][com.vitorpamplona.amethyst.ui.navigation.navs.ImeSettler] on `Nav`, so [onBack] is + * safe to run whenever it fires. * * The gate reads [WindowInsets.imeAnimationTarget] — where the IME is *heading* — not the animated * [WindowInsets.ime]. Gating on the animated value left a hole: it stays above zero for the whole * close animation, ~250ms in which the IME has already stopped consuming back but this handler was * still disabled, so a second back fell through to the NavController and popped the screen without - * ever running [onBack] — silently dropping the draft it exists to save. The target flips to zero - * the moment the hide begins, so back keeps reaching [onBack] throughout. - * - * Re-enabling that early means [onBack] can now fire mid-animation, so it is routed through - * [rememberAfterKeyboardCloses] to wait for the inset to settle before popping. + * ever running [onBack] — silently dropping the draft it exists to save, since nothing else saves + * one. The target flips to zero the moment the hide begins, so back keeps reaching [onBack] + * throughout the animation. */ @OptIn(ExperimentalLayoutApi::class) @Composable @@ -152,11 +90,10 @@ fun KeyboardAwareBackHandler( ) { val density = LocalDensity.current val imeTarget = WindowInsets.imeAnimationTarget - val afterKeyboardCloses = rememberAfterKeyboardCloses() val keyboardIsStaying by remember(density, imeTarget) { derivedStateOf { imeTarget.getBottom(density) > 0 } } - BackHandler(enabled = enabled && !keyboardIsStaying) { afterKeyboardCloses(onBack) } + BackHandler(enabled = enabled && !keyboardIsStaying, onBack = onBack) } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/navs/ImeSettler.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/navs/ImeSettler.kt new file mode 100644 index 0000000000..c36e3bc30b --- /dev/null +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/navs/ImeSettler.kt @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.ui.navigation.navs + +import androidx.compose.foundation.layout.WindowInsets +import androidx.compose.foundation.layout.ime +import androidx.compose.runtime.Composable +import androidx.compose.runtime.remember +import androidx.compose.runtime.snapshotFlow +import androidx.compose.ui.platform.LocalDensity +import androidx.compose.ui.platform.LocalFocusManager +import androidx.compose.ui.platform.LocalSoftwareKeyboardController +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.withTimeoutOrNull + +/** How long to wait for the IME inset to reach zero before navigating anyway. */ +const val IME_SETTLE_TIMEOUT_MS = 700L + +/** + * Waits for the soft keyboard to be fully off screen. Installed on [Nav] so that every navigation + * in the app serializes the IME and window animations instead of overlapping them. + * + * Navigating while the keyboard is up races the window animation against the IME's close animation. + * On release builds — fast enough that the window animation wins — the IME + * [WindowInsetsAnimationCompat][androidx.core.view.WindowInsetsAnimationCompat] is cancelled before + * its terminal (zero) frame reaches Compose. `WindowInsets.ime` is a single app-wide holder, so it + * stays "animating" and every `Modifier.imePadding()` in the app — not just the screen being left — + * freezes at the keyboard height until some later inset pass happens to rebalance it. + * + * This is not a composer-screen problem, which is why it lives here rather than in the screens. + * Any destination that can hold focus in a text field can strand the padding on the way out, by any + * exit: a back gesture, a top-bar button, a bottom-nav tab, or tapping a result. Search is the + * clearest case — it focuses its field on arrival, so the keyboard is already up before the user + * has done anything, and every way out of it is a navigation. + */ +fun interface ImeSettler { + suspend fun settle() + + companion object { + /** For [EmptyNav] and previews, where there is no window to read insets from. */ + val None = ImeSettler { } + } +} + +/** + * Reads the same animated `WindowInsets.ime` that drives `Modifier.imePadding()`, so the settler + * and the padding can never disagree about whether the keyboard is gone. + * + * Focus is cleared before hiding so nothing re-requests the IME as it retracts. The wait is bounded + * by [IME_SETTLE_TIMEOUT_MS] — if the inset never reports zero, which is precisely the failure this + * guards against, navigation still proceeds rather than stranding the user on the screen. + */ +@Composable +fun rememberImeSettler(): ImeSettler { + val density = LocalDensity.current + val imeInsets = WindowInsets.ime + val keyboard = LocalSoftwareKeyboardController.current + val focusManager = LocalFocusManager.current + + return remember(density, imeInsets, keyboard, focusManager) { + ImeSettler { + if (imeInsets.getBottom(density) > 0) { + focusManager.clearFocus(true) + keyboard?.hide() + withTimeoutOrNull(IME_SETTLE_TIMEOUT_MS) { + snapshotFlow { imeInsets.getBottom(density) }.first { it <= 0 } + } + } + } + } +} diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/navs/Nav.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/navs/Nav.kt index 1526d0be72..d937109b10 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/navs/Nav.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/navs/Nav.kt @@ -44,6 +44,13 @@ import kotlin.reflect.KClass class Nav( val controller: NavHostController, override val navigationScope: CoroutineScope, + /** + * Awaited before every transition below. Leaving a screen while the soft keyboard is still + * animating strands `imePadding()` app-wide; see [ImeSettler]. Every in-app navigation goes + * through this class, so this is the one place that has to get it right — no screen, top bar + * or back handler needs to think about the keyboard on its way out. + */ + private val ime: ImeSettler = ImeSettler.None, ) : INav { override val drawerState = DrawerState(DrawerValue.Closed) @@ -63,6 +70,7 @@ class Nav( override fun nav(route: Route) { navigationScope.launch { + ime.settle() if (getRouteWithArguments(route::class, controller) != route) { controller.navigate(route) } @@ -71,6 +79,7 @@ class Nav( override fun nav(computeRoute: suspend () -> Route?) { navigationScope.launch { + ime.settle() val route = computeRoute() if (route != null && getRouteWithArguments(route::class, controller) != route) { controller.navigate(route) @@ -80,6 +89,7 @@ class Nav( override fun newStack(route: Route) { navigationScope.launch { + ime.settle() controller.navigate(route) { popUpTo(route) { inclusive = true @@ -91,6 +101,7 @@ class Nav( override fun navBottomBar(route: Route) { navigationScope.launch { + ime.settle() controller.navigate(route) { // Clear sibling bottom-nav entries but keep Home (the start // destination) below, so back-swipe from any tab returns to @@ -149,6 +160,7 @@ class Nav( override fun popBack() { navigationScope.launch { + ime.settle() controller.navigateUp() } } @@ -159,6 +171,7 @@ class Nav( klass: KClass, ) { navigationScope.launch { + ime.settle() controller.navigate(route) { popUpTo(klass) { inclusive = true } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/navs/RememberNavs.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/navs/RememberNavs.kt index f6c42c0aa3..8b9a2d0e40 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/navs/RememberNavs.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/navs/RememberNavs.kt @@ -29,9 +29,10 @@ import androidx.navigation.compose.rememberNavController fun rememberNav(): Nav { val navController = rememberNavController() val scope = rememberCoroutineScope() + val ime = rememberImeSettler() - return remember(navController, scope) { - Nav(navController, scope) + return remember(navController, scope, ime) { + Nav(navController, scope, ime) } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/topbars/ActionTopBar.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/topbars/ActionTopBar.kt index 4bd0aba287..84b19c084a 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/topbars/ActionTopBar.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/topbars/ActionTopBar.kt @@ -31,7 +31,6 @@ import androidx.compose.ui.Modifier import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.text.style.TextOverflow import com.vitorpamplona.amethyst.R -import com.vitorpamplona.amethyst.ui.navigation.bottombars.rememberAfterKeyboardCloses import com.vitorpamplona.amethyst.ui.note.buttons.CloseButton import com.vitorpamplona.amethyst.ui.stringRes import com.vitorpamplona.amethyst.ui.theme.HalfHorzPadding @@ -46,10 +45,6 @@ fun ActionTopBar( onPost: () -> Unit, additionalActions: @Composable (() -> Unit)? = null, ) { - // Both exits pop the screen, and on a composer the keyboard is up while typing — the same - // pop-during-IME-animation race the back gesture avoids, just reached by a tap instead. - val afterKeyboardCloses = rememberAfterKeyboardCloses() - ShorterTopAppBar( title = { if (titleRes != null) { @@ -65,7 +60,7 @@ fun ActionTopBar( navigationIcon = { CloseButton( modifier = HalfHorzPadding, - onPress = { afterKeyboardCloses(onCancel) }, + onPress = onCancel, ) }, actions = { @@ -75,7 +70,7 @@ fun ActionTopBar( Button( modifier = HalfHorzPadding, enabled = isActive(), - onClick = { afterKeyboardCloses(onPost) }, + onClick = onPost, ) { Text(text = stringRes(postRes)) } diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/ui/navigation/NavImeSettleTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/ui/navigation/NavImeSettleTest.kt new file mode 100644 index 0000000000..1b362dc062 --- /dev/null +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/ui/navigation/NavImeSettleTest.kt @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.ui.navigation + +import androidx.navigation.NavHostController +import androidx.navigation.NavOptionsBuilder +import com.vitorpamplona.amethyst.ui.navigation.navs.ImeSettler +import com.vitorpamplona.amethyst.ui.navigation.navs.Nav +import com.vitorpamplona.amethyst.ui.navigation.routes.Route +import io.mockk.every +import io.mockk.mockk +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.delay +import kotlinx.coroutines.test.advanceUntilIdle +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Test + +/** + * Leaving a screen while the soft keyboard is still animating strands `imePadding()` at keyboard + * height for the whole app, because `WindowInsets.ime` is a single shared holder. The fix is that + * [Nav] waits for the IME to be gone before it moves, so these assert the ordering rather than any + * visual result: every transition must settle the keyboard *first*. + * + * Without the settle calls in [Nav] each of these records only "navigate" and fails. + */ +@OptIn(ExperimentalCoroutinesApi::class) +class NavImeSettleTest { + private fun controllerRecording(order: MutableList): NavHostController = + mockk(relaxed = true) { + every { navigate(any(), any Unit>()) } answers + { order.add("navigate") } + every { navigate(any()) } answers { order.add("navigate") } + every { navigateUp() } answers { + order.add("navigate") + true + } + } + + @Test + fun popBackSettlesTheKeyboardBeforeNavigating() = + runTest { + val order = mutableListOf() + val nav = Nav(controllerRecording(order), this, ImeSettler { order.add("settle") }) + + nav.popBack() + advanceUntilIdle() + + assertEquals(listOf("settle", "navigate"), order) + } + + @Test + fun bottomBarSettlesTheKeyboardBeforeNavigating() = + runTest { + // The search tab focuses its field on arrival, so the keyboard is already up when the + // user taps another tab — the exit that has no BackHandler and no top bar to guard it. + val order = mutableListOf() + val nav = Nav(controllerRecording(order), this, ImeSettler { order.add("settle") }) + + nav.navBottomBar(Route.Home) + advanceUntilIdle() + + assertEquals(listOf("settle", "navigate"), order) + } + + @Test + fun newStackSettlesTheKeyboardBeforeNavigating() = + runTest { + val order = mutableListOf() + val nav = Nav(controllerRecording(order), this, ImeSettler { order.add("settle") }) + + nav.newStack(Route.Home) + advanceUntilIdle() + + assertEquals(listOf("settle", "navigate"), order) + } + + @Test + fun aSlowKeyboardStillHoldsTheNavigationBack() = + runTest { + // The real settler suspends for the length of the IME close animation. Navigation must + // wait for it, not fire alongside it — that overlap is the bug. + val order = mutableListOf() + val nav = + Nav( + controllerRecording(order), + this, + ImeSettler { + delay(250) + order.add("settle") + }, + ) + + nav.popBack() + assertEquals(emptyList(), order) + + advanceUntilIdle() + assertEquals(listOf("settle", "navigate"), order) + } +} From 72d05e2eb8bd7579fc797ff94f5f00b4a22051bb Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 23:58:24 +0000 Subject: [PATCH 052/132] refactor(ime): drop KeyboardAwareBackHandler now that Nav settles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Its job was to stop a composer's pop from racing the IME close animation, and that pop is `nav.popBack()` in all 11 call sites — exactly what the ImeSettler on Nav now serializes. So it no longer carries the fix; a plain BackHandler reaches the same place safely. What it did still provide was the two-back convention: first back dismisses the keyboard (via the system's own animation, which on recent Android follows the gesture), second back leaves. That came at a price it did not used to have. The mechanism is to NOT consume back while the keyboard is up and let the IME consume it instead — so on any device or API level where the IME does not, back reaches the NavController, which pops without ever running onBack and silently drops the draft, since nothing else saves one. Now that Nav settles, that failure would also be invisible: no stranded padding to hint at it, just a missing draft. A plain BackHandler has no such failure mode. It always consumes, so the draft is always flushed, on the first back rather than the second. KeyboardState.kt keeps keyboardAsState(), which is a separate concern — AppBottomBar uses it to hide the bottom bar while typing. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01LfUMGWYu2uTSyh17JJonfN --- .../ui/navigation/bottombars/KeyboardState.kt | 41 ------------------- .../nip22Comments/GenericCommentPostScreen.kt | 4 +- .../loggedIn/badges/award/AwardBadgeScreen.kt | 4 +- .../chats/privateDM/send/NewGroupDMScreen.kt | 4 +- .../send/PrivateMessageEditFieldRow.kt | 4 +- .../chats/publicChannels/send/EditFieldRow.kt | 4 +- .../nip23LongForm/LongFormPostScreen.kt | 4 +- .../nip99Classifieds/NewProductScreen.kt | 4 +- .../loggedIn/home/ShortNotePostScreen.kt | 4 +- .../loggedIn/home/nip75Goals/NewGoalScreen.kt | 4 +- .../publicMessages/NewPublicMessageScreen.kt | 4 +- .../loggedIn/workouts/NewWorkoutScreen.kt | 4 +- 12 files changed, 22 insertions(+), 63 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/KeyboardState.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/KeyboardState.kt index 78ea948462..1fc4a00534 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/KeyboardState.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/bottombars/KeyboardState.kt @@ -20,15 +20,11 @@ */ package com.vitorpamplona.amethyst.ui.navigation.bottombars -import androidx.activity.compose.BackHandler -import androidx.compose.foundation.layout.ExperimentalLayoutApi import androidx.compose.foundation.layout.WindowInsets import androidx.compose.foundation.layout.ime -import androidx.compose.foundation.layout.imeAnimationTarget import androidx.compose.runtime.Composable import androidx.compose.runtime.State import androidx.compose.runtime.derivedStateOf -import androidx.compose.runtime.getValue import androidx.compose.runtime.remember import androidx.compose.ui.platform.LocalDensity @@ -60,40 +56,3 @@ fun keyboardAsState(): State { } } } - -/** - * A [BackHandler] that lets the system dismiss the soft keyboard before it consumes back. - * - * Chat composers (and draft-saving editors) intercept back to flush a draft and pop the screen. - * While the keyboard is up we do NOT consume back, so the system dismisses it first with its own - * animation — which completes cleanly, and on recent Android follows the back gesture rather than - * snapping. The next back runs [onBack] as before. - * - * This is a UX preference, not the safety mechanism: the actual pop-during-IME-animation race is - * handled for every exit in the app by - * [ImeSettler][com.vitorpamplona.amethyst.ui.navigation.navs.ImeSettler] on `Nav`, so [onBack] is - * safe to run whenever it fires. - * - * The gate reads [WindowInsets.imeAnimationTarget] — where the IME is *heading* — not the animated - * [WindowInsets.ime]. Gating on the animated value left a hole: it stays above zero for the whole - * close animation, ~250ms in which the IME has already stopped consuming back but this handler was - * still disabled, so a second back fell through to the NavController and popped the screen without - * ever running [onBack] — silently dropping the draft it exists to save, since nothing else saves - * one. The target flips to zero the moment the hide begins, so back keeps reaching [onBack] - * throughout the animation. - */ -@OptIn(ExperimentalLayoutApi::class) -@Composable -fun KeyboardAwareBackHandler( - enabled: Boolean = true, - onBack: () -> Unit, -) { - val density = LocalDensity.current - val imeTarget = WindowInsets.imeAnimationTarget - - val keyboardIsStaying by remember(density, imeTarget) { - derivedStateOf { imeTarget.getBottom(density) > 0 } - } - - BackHandler(enabled = enabled && !keyboardIsStaying, onBack = onBack) -} diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/nip22Comments/GenericCommentPostScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/nip22Comments/GenericCommentPostScreen.kt index aec22cb911..6baef9f1a2 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/nip22Comments/GenericCommentPostScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/nip22Comments/GenericCommentPostScreen.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.amethyst.ui.note.nip22Comments +import androidx.activity.compose.BackHandler import androidx.compose.foundation.clickable import androidx.compose.foundation.horizontalScroll import androidx.compose.foundation.layout.Box @@ -66,7 +67,6 @@ import com.vitorpamplona.amethyst.ui.actions.uploads.SelectFromGallery import com.vitorpamplona.amethyst.ui.actions.uploads.SelectedMedia import com.vitorpamplona.amethyst.ui.actions.uploads.TakePictureButton import com.vitorpamplona.amethyst.ui.actions.uploads.TakeVideoButton -import com.vitorpamplona.amethyst.ui.navigation.bottombars.KeyboardAwareBackHandler import com.vitorpamplona.amethyst.ui.navigation.navs.Nav import com.vitorpamplona.amethyst.ui.navigation.topbars.PostingTopBar import com.vitorpamplona.amethyst.ui.note.BaseUserPicture @@ -177,7 +177,7 @@ fun GenericCommentPostScreen( StrippingFailureDialog(postViewModel.strippingFailureConfirmation) - KeyboardAwareBackHandler { + BackHandler { accountViewModel.launchSigner { postViewModel.sendDraftSync() postViewModel.cancel() diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/badges/award/AwardBadgeScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/badges/award/AwardBadgeScreen.kt index bdc768cc3c..2172e18945 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/badges/award/AwardBadgeScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/badges/award/AwardBadgeScreen.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.badges.award +import androidx.activity.compose.BackHandler import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.Spacer @@ -51,7 +52,6 @@ import androidx.lifecycle.viewmodel.compose.viewModel import com.vitorpamplona.amethyst.R import com.vitorpamplona.amethyst.commons.ui.components.Nip05OrPubkeyLine import com.vitorpamplona.amethyst.model.User -import com.vitorpamplona.amethyst.ui.navigation.bottombars.KeyboardAwareBackHandler import com.vitorpamplona.amethyst.ui.navigation.navs.INav import com.vitorpamplona.amethyst.ui.navigation.topbars.SavingTopBar import com.vitorpamplona.amethyst.ui.note.UserPicture @@ -88,7 +88,7 @@ fun AwardBadgeScreen( onDispose { userSuggestions.reset() } } - KeyboardAwareBackHandler { + BackHandler { nav.popBack() } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/privateDM/send/NewGroupDMScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/privateDM/send/NewGroupDMScreen.kt index ce87e19328..d87f54987d 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/privateDM/send/NewGroupDMScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/privateDM/send/NewGroupDMScreen.kt @@ -21,6 +21,7 @@ package com.vitorpamplona.amethyst.ui.screen.loggedIn.chats.privateDM.send import android.net.Uri +import androidx.activity.compose.BackHandler import androidx.compose.foundation.horizontalScroll import androidx.compose.foundation.layout.Arrangement.Absolute.spacedBy import androidx.compose.foundation.layout.Box @@ -86,7 +87,6 @@ import com.vitorpamplona.amethyst.ui.actions.uploads.TakePictureButton import com.vitorpamplona.amethyst.ui.actions.uploads.TakeVideoButton import com.vitorpamplona.amethyst.ui.components.ThinPaddingTextField import com.vitorpamplona.amethyst.ui.components.ZoomableContentView -import com.vitorpamplona.amethyst.ui.navigation.bottombars.KeyboardAwareBackHandler import com.vitorpamplona.amethyst.ui.navigation.navs.INav import com.vitorpamplona.amethyst.ui.navigation.navs.Nav import com.vitorpamplona.amethyst.ui.navigation.routes.routeToMessage @@ -169,7 +169,7 @@ fun NewGroupDMScreen( WatchAndLoadMyEmojiList(accountViewModel) - KeyboardAwareBackHandler { + BackHandler { accountViewModel.launchSigner { postViewModel.sendDraftSync() postViewModel.cancel() diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/privateDM/send/PrivateMessageEditFieldRow.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/privateDM/send/PrivateMessageEditFieldRow.kt index 390d718d9b..fc64efe0ab 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/privateDM/send/PrivateMessageEditFieldRow.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/privateDM/send/PrivateMessageEditFieldRow.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.chats.privateDM.send +import androidx.activity.compose.BackHandler import androidx.compose.foundation.background import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box @@ -59,7 +60,6 @@ import com.vitorpamplona.amethyst.ui.actions.UrlUserTagOutputTransformation import com.vitorpamplona.amethyst.ui.actions.uploads.SelectFromGallery import com.vitorpamplona.amethyst.ui.actions.uploads.SelectedMedia import com.vitorpamplona.amethyst.ui.components.ThinPaddingTextField -import com.vitorpamplona.amethyst.ui.navigation.bottombars.KeyboardAwareBackHandler import com.vitorpamplona.amethyst.ui.navigation.navs.EmptyNav import com.vitorpamplona.amethyst.ui.navigation.navs.INav import com.vitorpamplona.amethyst.ui.navigation.routes.routeFor @@ -110,7 +110,7 @@ fun PrivateMessageEditFieldRow( onSendNewMessage: () -> Unit, nav: INav, ) { - KeyboardAwareBackHandler { + BackHandler { if (channelScreenModel.message.text.isNotBlank()) { accountViewModel.launchSigner { channelScreenModel.sendDraftSync() diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/send/EditFieldRow.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/send/EditFieldRow.kt index f14cd1d59e..8176dc4d5b 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/send/EditFieldRow.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/send/EditFieldRow.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.chats.publicChannels.send +import androidx.activity.compose.BackHandler import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.fillMaxWidth @@ -53,7 +54,6 @@ import com.vitorpamplona.amethyst.ui.actions.UrlUserTagOutputTransformation import com.vitorpamplona.amethyst.ui.actions.uploads.SelectFromGallery import com.vitorpamplona.amethyst.ui.actions.uploads.SelectedMedia import com.vitorpamplona.amethyst.ui.components.ThinPaddingTextField -import com.vitorpamplona.amethyst.ui.navigation.bottombars.KeyboardAwareBackHandler import com.vitorpamplona.amethyst.ui.navigation.navs.INav import com.vitorpamplona.amethyst.ui.note.creators.userSuggestions.ShowUserSuggestionList import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel @@ -78,7 +78,7 @@ fun EditFieldRow( onSendNewMessage: suspend () -> Unit, nav: INav, ) { - KeyboardAwareBackHandler { + BackHandler { accountViewModel.launchSigner { channelScreenModel.sendDraftSync() channelScreenModel.cancel() diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/discover/nip23LongForm/LongFormPostScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/discover/nip23LongForm/LongFormPostScreen.kt index 12cba33c60..2465929d40 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/discover/nip23LongForm/LongFormPostScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/discover/nip23LongForm/LongFormPostScreen.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.discover.nip23LongForm +import androidx.activity.compose.BackHandler import androidx.compose.foundation.BorderStroke import androidx.compose.foundation.border import androidx.compose.foundation.clickable @@ -95,7 +96,6 @@ import com.vitorpamplona.amethyst.ui.actions.uploads.TakeVideoButton import com.vitorpamplona.amethyst.ui.components.MyAsyncImage import com.vitorpamplona.amethyst.ui.components.ThinPaddingTextField import com.vitorpamplona.amethyst.ui.components.markdown.RenderContentAsMarkdown -import com.vitorpamplona.amethyst.ui.navigation.bottombars.KeyboardAwareBackHandler import com.vitorpamplona.amethyst.ui.navigation.navs.Nav import com.vitorpamplona.amethyst.ui.navigation.topbars.PostingTopBar import com.vitorpamplona.amethyst.ui.note.creators.contentWarning.ContentSensitivityExplainer @@ -149,7 +149,7 @@ fun LongFormPostScreen( StrippingFailureDialog(postViewModel.strippingFailureConfirmation) - KeyboardAwareBackHandler { + BackHandler { accountViewModel.launchSigner { postViewModel.sendDraftSync() postViewModel.cancel() diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/discover/nip99Classifieds/NewProductScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/discover/nip99Classifieds/NewProductScreen.kt index 8cd67d7b46..2e95142599 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/discover/nip99Classifieds/NewProductScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/discover/nip99Classifieds/NewProductScreen.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.discover.nip99Classifieds +import androidx.activity.compose.BackHandler import androidx.compose.foundation.horizontalScroll import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row @@ -52,7 +53,6 @@ import com.vitorpamplona.amethyst.ui.actions.uploads.SelectFromGallery import com.vitorpamplona.amethyst.ui.actions.uploads.SelectedMedia import com.vitorpamplona.amethyst.ui.actions.uploads.TakePictureButton import com.vitorpamplona.amethyst.ui.actions.uploads.TakeVideoButton -import com.vitorpamplona.amethyst.ui.navigation.bottombars.KeyboardAwareBackHandler import com.vitorpamplona.amethyst.ui.navigation.navs.INav import com.vitorpamplona.amethyst.ui.navigation.navs.Nav import com.vitorpamplona.amethyst.ui.navigation.topbars.PostingTopBar @@ -141,7 +141,7 @@ fun NewProductScreen( StrippingFailureDialog(postViewModel.strippingFailureConfirmation) - KeyboardAwareBackHandler { + BackHandler { accountViewModel.launchSigner { postViewModel.sendDraftSync() postViewModel.cancel() diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/ShortNotePostScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/ShortNotePostScreen.kt index a4bc2c6bfb..db7e95f3be 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/ShortNotePostScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/ShortNotePostScreen.kt @@ -23,6 +23,7 @@ package com.vitorpamplona.amethyst.ui.screen.loggedIn.home import android.annotation.SuppressLint import android.content.Intent import android.net.Uri +import androidx.activity.compose.BackHandler import androidx.compose.foundation.clickable import androidx.compose.foundation.horizontalScroll import androidx.compose.foundation.layout.Arrangement @@ -92,7 +93,6 @@ import com.vitorpamplona.amethyst.ui.actions.uploads.VoiceMessagePreview import com.vitorpamplona.amethyst.ui.components.OutlinedThinPaddingTextField import com.vitorpamplona.amethyst.ui.components.ThinPaddingTextField import com.vitorpamplona.amethyst.ui.components.getActivity -import com.vitorpamplona.amethyst.ui.navigation.bottombars.KeyboardAwareBackHandler import com.vitorpamplona.amethyst.ui.navigation.navs.Nav import com.vitorpamplona.amethyst.ui.navigation.routes.Route import com.vitorpamplona.amethyst.ui.navigation.topbars.PostingTopBar @@ -235,7 +235,7 @@ internal fun NewPostScreenInner( StrippingFailureDialog(postViewModel.strippingFailureConfirmation) - KeyboardAwareBackHandler { + BackHandler { accountViewModel.launchSigner { postViewModel.sendDraftSync() postViewModel.cancel() diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/nip75Goals/NewGoalScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/nip75Goals/NewGoalScreen.kt index f7aabcd6f7..5a6348cb6d 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/nip75Goals/NewGoalScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/home/nip75Goals/NewGoalScreen.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.home.nip75Goals +import androidx.activity.compose.BackHandler import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.Spacer @@ -47,7 +48,6 @@ import androidx.compose.ui.text.input.KeyboardType import androidx.compose.ui.unit.dp import androidx.lifecycle.viewmodel.compose.viewModel import com.vitorpamplona.amethyst.R -import com.vitorpamplona.amethyst.ui.navigation.bottombars.KeyboardAwareBackHandler import com.vitorpamplona.amethyst.ui.navigation.navs.INav import com.vitorpamplona.amethyst.ui.navigation.navs.Nav import com.vitorpamplona.amethyst.ui.navigation.topbars.PostingTopBar @@ -81,7 +81,7 @@ fun NewGoalScreen( accountViewModel: AccountViewModel, nav: INav, ) { - KeyboardAwareBackHandler { + BackHandler { goalViewModel.cancel() nav.popBack() } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/notifications/publicMessages/NewPublicMessageScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/notifications/publicMessages/NewPublicMessageScreen.kt index f9e277665c..56e1382b6a 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/notifications/publicMessages/NewPublicMessageScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/notifications/publicMessages/NewPublicMessageScreen.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.notifications.publicMessages +import androidx.activity.compose.BackHandler import androidx.compose.foundation.horizontalScroll import androidx.compose.foundation.layout.Arrangement.Absolute.spacedBy import androidx.compose.foundation.layout.Column @@ -63,7 +64,6 @@ import com.vitorpamplona.amethyst.ui.actions.uploads.SelectFromGallery import com.vitorpamplona.amethyst.ui.actions.uploads.TakePictureButton import com.vitorpamplona.amethyst.ui.actions.uploads.TakeVideoButton import com.vitorpamplona.amethyst.ui.components.ThinPaddingTextField -import com.vitorpamplona.amethyst.ui.navigation.bottombars.KeyboardAwareBackHandler import com.vitorpamplona.amethyst.ui.navigation.navs.INav import com.vitorpamplona.amethyst.ui.navigation.navs.Nav import com.vitorpamplona.amethyst.ui.navigation.topbars.PostingTopBar @@ -134,7 +134,7 @@ fun NewPublicMessageScreen( StrippingFailureDialog(postViewModel.strippingFailureConfirmation) - KeyboardAwareBackHandler { + BackHandler { accountViewModel.launchSigner { postViewModel.sendDraftSync() postViewModel.cancel() diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/workouts/NewWorkoutScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/workouts/NewWorkoutScreen.kt index 5bc98f4cdb..a3de2013d0 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/workouts/NewWorkoutScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/workouts/NewWorkoutScreen.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.workouts +import androidx.activity.compose.BackHandler import androidx.compose.animation.Crossfade import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box @@ -60,7 +61,6 @@ import com.vitorpamplona.amethyst.R import com.vitorpamplona.amethyst.commons.icons.symbols.Icon import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbol import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols -import com.vitorpamplona.amethyst.ui.navigation.bottombars.KeyboardAwareBackHandler import com.vitorpamplona.amethyst.ui.navigation.navs.INav import com.vitorpamplona.amethyst.ui.navigation.routes.Route import com.vitorpamplona.amethyst.ui.navigation.topbars.PostingTopBar @@ -80,7 +80,7 @@ fun NewWorkoutScreen( postViewModel.init(accountViewModel) postViewModel.prefill(prefill) - KeyboardAwareBackHandler { + BackHandler { postViewModel.cancel() nav.popBack() } From f4fc9917f8e6df5473febda56bde4494365bff19 Mon Sep 17 00:00:00 2001 From: vitorpamplona <532031+vitorpamplona@users.noreply.github.com> Date: Thu, 6 Aug 2026 00:48:28 +0000 Subject: [PATCH 053/132] chore: sync Crowdin translations and seed translator npub placeholders --- amethyst/src/main/res/values-cs/strings.xml | 16 ++++++++++++++++ amethyst/src/main/res/values-sv-rSE/strings.xml | 7 +++++++ docs/changelog/translators.json | 11 +++++++---- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/amethyst/src/main/res/values-cs/strings.xml b/amethyst/src/main/res/values-cs/strings.xml index 6a86477be8..854354592d 100644 --- a/amethyst/src/main/res/values-cs/strings.xml +++ b/amethyst/src/main/res/values-cs/strings.xml @@ -2025,6 +2025,12 @@ + + %1$s \u00b7 %2$d relay + %1$s \u00b7 %2$d relaye + %1$s \u00b7 %2$d relaye + %1$s \u00b7 %2$d relayů + Procházení Média Hashtagy @@ -2063,6 +2069,7 @@ Peněženka Schránka nutzapů Adresář mintů + Wallet Connect Chaty komunit Zdroje komunit + + %1$d filter + %1$d filter + %1$d relä %1$d reläer @@ -2012,6 +2018,7 @@ %1$d filter är inte kopplat ännu %1$d filter är inte kopplade ännu + %1$s \u00b7 %2$s Inte kopplat till något konto Allt Alla diff --git a/docs/changelog/translators.json b/docs/changelog/translators.json index ef7dced2fc..a322a2a269 100644 --- a/docs/changelog/translators.json +++ b/docs/changelog/translators.json @@ -112,6 +112,13 @@ "Hindi" ] }, + { + "user": "davotoula", + "languages": [ + "Czech", + "Swedish" + ] + }, { "user": "greenart7c3", "languages": [] @@ -180,10 +187,6 @@ "user": "adhrasreoshiathoi", "languages": [] }, - { - "user": "davotoula", - "languages": [] - }, { "user": "crackadoo", "languages": [] From d6b8a54d8a7e6aa3f4800bbda2c62505e563b35e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 16:21:32 +0000 Subject: [PATCH 054/132] NEG-ERR: state the relay's max_sync_events on an overflow refusal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A client that is refused for matching too much has exactly one thing to decide — how much smaller to ask next time — and no way to find out. NIP-11 has no field for max_sync_events, so the only route to a window the relay will answer is to guess and halve, and every wrong guess costs the relay the snapshot scan that produces the refusal. strfry already states the number in its rejection text; this makes it a first-class part of the frame. ["NEG-ERR", , ] unchanged, still what NIP-77 says ["NEG-ERR", , , ] when the refusal is about size Both mappers write the fourth element only when there is one, so a refusal with nothing to state is byte-identical to before, and both tolerate a non-numeric fourth element from someone else's relay. NegErrMessage.statedCap reads either form — the wire field or strfry's "(2431002 > 1000000)" prose — but only for a refusal that is about SIZE. That gate is the point of the property: a rate limit or a quota can carry numbers too, and it does not shrink when the window shrinks, so a client that mistook one for a cap would shrink its windows forever against a relay that has no size limit at all. The relay side sends its own configured cap for the same reason it is cheap: it had to know the number to refuse. --- .../kotlinSerialization/MessageKSerializer.kt | 9 ++ .../relay/server/NegSessionRegistry.kt | 8 +- .../quartz/nip77Negentropy/NegErrMessage.kt | 51 ++++++++++ .../nip77Negentropy/NegentropySettings.kt | 4 +- .../nip77Negentropy/NegErrMessageTest.kt | 96 +++++++++++++++++++ .../commands/toClient/MessageDeserializer.kt | 16 +++- .../commands/toClient/MessageSerializer.kt | 1 + .../nip77Negentropy/Nip77SerializationTest.kt | 64 +++++++++++++ 8 files changed, 243 insertions(+), 6 deletions(-) create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegErrMessageTest.kt diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/kotlinSerialization/MessageKSerializer.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/kotlinSerialization/MessageKSerializer.kt index adba985212..93c0c360cb 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/kotlinSerialization/MessageKSerializer.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/kotlinSerialization/MessageKSerializer.kt @@ -46,6 +46,7 @@ import kotlinx.serialization.json.buildJsonArray import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.longOrNull object MessageKSerializer : KSerializer { override val descriptor: SerialDescriptor = @@ -112,6 +113,10 @@ object MessageKSerializer : KSerializer { is NegErrMessage -> { add(JsonPrimitive(value.subId)) add(JsonPrimitive(value.reason)) + // Only written when there is one: a three-element + // NEG-ERR is what NIP-77 describes, and that is what a + // refusal with nothing to state stays. + value.cap?.let { add(JsonPrimitive(it)) } } } } @@ -184,6 +189,10 @@ object MessageKSerializer : KSerializer { NegErrMessage( subId = array[1].jsonPrimitive.content, reason = if (array.size > 2) array[2].jsonPrimitive.content else "", + // Optional, and only a number: a relay that puts something + // else there is telling us nothing rather than breaking the + // frame. + cap = if (array.size > 3) array[3].jsonPrimitive.longOrNull else null, ) } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/NegSessionRegistry.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/NegSessionRegistry.kt index 66e1675e32..18b4a31925 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/NegSessionRegistry.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/NegSessionRegistry.kt @@ -104,7 +104,13 @@ class NegSessionRegistry( // `null` = matching set exceeds the cap (strfry-parity error). val sealedStorage = store.sealedNegentropyStorage(filters, maxEntries = settings.maxSyncEvents) if (sealedStorage == null) { - send(NegErrMessage(cmd.subId, "blocked: too many query results")) + // The cap rides along with the refusal. A client cannot discover + // this number any other way — NIP-11 has no field for it — so + // without it the only route to a window we WILL answer is guessing, + // halving, one refused NEG-OPEN at a time. Every one of those costs + // us the snapshot scan that produced this rejection, which makes + // stating it cheaper for the relay than staying quiet. + send(NegErrMessage(cmd.subId, "blocked: too many query results", settings.maxSyncEvents.toLong())) return } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegErrMessage.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegErrMessage.kt index a81e1ded1f..e191598d97 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegErrMessage.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegErrMessage.kt @@ -22,13 +22,64 @@ package com.vitorpamplona.quartz.nip77Negentropy import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.Message +/** + * `["NEG-ERR", , ]`, optionally followed by the relay's own + * `max_sync_events` when the refusal is about result-set size. + * + * That fourth element is not in NIP-77, but it is the only way a client learns + * the one number that decides how to ask again — no NIP-11 field carries it — + * and it is free for the relay to send, since it must know its own cap to have + * refused. strfry states it in the prose (`… too many records (2431002 > + * 1000000)`); [statedCap] reads either form. + * + * @property cap the fourth wire element, when present. + */ class NegErrMessage( val subId: String, val reason: String, + val cap: Long? = null, ) : Message { override fun label() = LABEL + /** + * The relay's negentropy cap if this refusal states one, from the wire + * field or from the prose, in that order. + * + * Only read for a refusal that is about SIZE ([isOverflow]). A quota or + * rate-limit refusal can carry numbers too, and sizing future windows + * against one of those would shrink every ask against a relay that has no + * size limit at all — while the limit that actually refused does not move + * however small the window gets. + */ + val statedCap: Long? + get() = if (!isOverflow(reason)) null else cap?.takeIf { it > 0 } ?: capInReason(reason) + companion object { const val LABEL = "NEG-ERR" + + /** `(2431002 > 1000000)` — the cap is the right-hand side. */ + private val COMPARISON = Regex("""\(\s*\d+\s*>\s*(\d+)\s*\)""") + + /** + * Does this reason mean "your query matched more than I will + * reconcile"? — as opposed to any other refusal, which no amount of + * window splitting will get past. + */ + fun isOverflow(reason: String): Boolean = + reason.contains("too many records", ignoreCase = true) || + reason.contains("too many results", ignoreCase = true) || + reason.contains("too many query results", ignoreCase = true) || + reason.contains("result set too large", ignoreCase = true) || + reason.contains("results too large", ignoreCase = true) || + reason.contains("max_sync_events", ignoreCase = true) + + /** The cap strfry writes into the refusal text, when it is there. */ + fun capInReason(reason: String): Long? = + COMPARISON + .find(reason) + ?.groupValues + ?.get(1) + ?.toLongOrNull() + ?.takeIf { it > 0 } } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegentropySettings.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegentropySettings.kt index 896ec7c472..0bd1c1027d 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegentropySettings.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegentropySettings.kt @@ -32,7 +32,9 @@ package com.vitorpamplona.quartz.nip77Negentropy * unlimited). * @param maxSyncEvents Hard cap on the snapshot size for a single * NEG-OPEN. Mirrors strfry's `relay__negentropy__maxSyncEvents`. - * Overflow returns NEG-ERR `"blocked: too many query results"`. + * Overflow returns NEG-ERR `"blocked: too many query results"` + * carrying this number as its fourth element, so a client can size + * its next window instead of halving its way down to one. * @param maxSessionsPerConnection Cap on concurrent NEG sessions * held by one connection. strfry shares 200 with REQ subs; we * count NEG independently. Overflow sends NOTICE diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegErrMessageTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegErrMessageTest.kt new file mode 100644 index 0000000000..9adafce838 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip77Negentropy/NegErrMessageTest.kt @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip77Negentropy + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * A stated cap is acted on — it sizes the next NEG-OPEN — so reading one out of + * a refusal that is not about size is worse than reading none at all: a quota or + * rate limit does not shrink when the window shrinks, so a client that mistook + * one for a cap would shrink its windows forever against a relay that has no + * size limit. + */ +class NegErrMessageTest { + @Test + fun capComesFromTheWireField() { + assertEquals(1_000_000L, NegErrMessage("s", "blocked: too many query results", 1_000_000L).statedCap) + } + + @Test + fun capComesFromStrfrysProseWhenTheFieldIsAbsent() { + val msg = NegErrMessage("s", "blocked: query matches too many records (2431002 > 1000000)") + assertEquals(1_000_000L, msg.statedCap) + } + + @Test + fun theWireFieldWinsOverTheProse() { + val msg = NegErrMessage("s", "blocked: too many records (5 > 10)", 1_000L) + assertEquals(1_000L, msg.statedCap) + } + + @Test + fun anOverflowWithNoNumberStatesNothing() { + assertNull(NegErrMessage("s", "blocked: too many query results").statedCap) + } + + @Test + fun aRateLimitIsNotACapHoweverManyNumbersItCarries() { + assertFalse(NegErrMessage.isOverflow("rate-limited: too many requests (30 > 10)")) + assertNull(NegErrMessage("s", "rate-limited: too many requests (30 > 10)", 10L).statedCap) + } + + @Test + fun refusalsThatAreNotAboutSizeStateNothing() { + listOf( + "auth-required: we only serve negentropy to authenticated users", + "blocked: pubkey is banned", + "error: negentropy disabled", + "closed: unknown subscription handle", + ).forEach { + assertFalse(NegErrMessage.isOverflow(it), "read as an overflow: $it") + assertNull(NegErrMessage("s", it, 42L).statedCap, "read a cap from: $it") + } + } + + @Test + fun theWordingsThatDoMeanOverflow() { + listOf( + "blocked: query matches too many records (5 > 1)", + "blocked: too many query results", + "error: result set too large", + "blocked: results too large", + "blocked: max_sync_events exceeded", + ).forEach { assertTrue(NegErrMessage.isOverflow(it), "not read as an overflow: $it") } + } + + @Test + fun aNonsensicalCapIsRefused() { + // Zero would wedge a client at a window that can never fit. + assertNull(NegErrMessage("s", "blocked: too many query results", 0L).statedCap) + assertNull(NegErrMessage("s", "blocked: too many records (5 > 0)").statedCap) + assertNull(NegErrMessage("s", "blocked: too many query results", -1L).statedCap) + } +} diff --git a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/MessageDeserializer.kt b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/MessageDeserializer.kt index 7e45082421..a2d3e10df3 100644 --- a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/MessageDeserializer.kt +++ b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/MessageDeserializer.kt @@ -121,10 +121,18 @@ class MessageDeserializer : StdDeserializer(Message::class.java) { } NegErrMessage.LABEL -> { - NegErrMessage( - subId = jp.nextTextValue(), - reason = jp.nextTextValue() ?: "", - ) + val subId = jp.nextTextValue() + val reason = jp.nextTextValue() ?: "" + // The optional fourth element, the relay's own cap. Read by + // stepping one token: anything that is not a number leaves + // the loop below to drain the frame, as before. + val cap = + if (jp.nextToken() == JsonToken.VALUE_NUMBER_INT) { + jp.longValue + } else { + null + } + NegErrMessage(subId, reason, cap) } else -> { diff --git a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/MessageSerializer.kt b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/MessageSerializer.kt index 86274bf1e6..76671a396f 100644 --- a/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/MessageSerializer.kt +++ b/quartz/src/jvmAndroid/kotlin/com/vitorpamplona/quartz/nip01Core/relay/commands/toClient/MessageSerializer.kt @@ -129,6 +129,7 @@ class MessageSerializer : StdSerializer(Message::class.java) { is NegErrMessage -> { gen.writeString(msg.subId) gen.writeString(msg.reason) + msg.cap?.let { gen.writeNumber(it) } } } diff --git a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip77Negentropy/Nip77SerializationTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip77Negentropy/Nip77SerializationTest.kt index 8b3bce89ba..7a71cc9a1b 100644 --- a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip77Negentropy/Nip77SerializationTest.kt +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip77Negentropy/Nip77SerializationTest.kt @@ -124,6 +124,70 @@ class Nip77SerializationTest { assertEquals(msg.reason, jacksonDeserialized.reason) } + @Test + fun serializeNegErrMessageWithCap_matchesJackson() { + val msg = NegErrMessage("neg-sub1", "blocked: too many query results", 1_000_000L) + val jacksonJson = JacksonMapper.toJson(msg) + val kotlinJson = KotlinSerializationMapper.toJson(msg) + + assertEquals(jacksonJson, kotlinJson) + assertEquals("""["NEG-ERR","neg-sub1","blocked: too many query results",1000000]""", kotlinJson) + } + + @Test + fun serializeNegErrMessageWithoutCap_staysThreeElements() { + // NIP-77 describes a three-element NEG-ERR. A refusal with no cap to + // state must stay exactly that, rather than growing a fourth element + // every existing reader then has to tolerate. + val msg = NegErrMessage("neg-sub1", "closed: timeout") + assertEquals("""["NEG-ERR","neg-sub1","closed: timeout"]""", KotlinSerializationMapper.toJson(msg)) + assertEquals("""["NEG-ERR","neg-sub1","closed: timeout"]""", JacksonMapper.toJson(msg)) + } + + @Test + fun deserializeNegErrMessageWithCap_bothMappers() { + val json = """["NEG-ERR","neg-sub1","blocked: too many query results",1000000]""" + + val jackson = JacksonMapper.fromJsonToMessage(json) + assertTrue(jackson is NegErrMessage) + assertEquals(1_000_000L, jackson.cap) + assertEquals(1_000_000L, jackson.statedCap) + + val kotlin = KotlinSerializationMapper.fromJsonToMessage(json) + assertTrue(kotlin is NegErrMessage) + assertEquals(1_000_000L, kotlin.cap) + } + + @Test + fun deserializeNegErrMessageWithGarbageFourthElement_bothMappers() { + // A relay that puts something else there is telling us nothing; it must + // not break the frame that carries the reason. + val json = """["NEG-ERR","neg-sub1","blocked: too many query results","soon"]""" + + val jackson = JacksonMapper.fromJsonToMessage(json) + assertTrue(jackson is NegErrMessage) + assertEquals("blocked: too many query results", jackson.reason) + assertEquals(null, jackson.cap) + + val kotlin = KotlinSerializationMapper.fromJsonToMessage(json) + assertTrue(kotlin is NegErrMessage) + assertEquals("blocked: too many query results", kotlin.reason) + assertEquals(null, kotlin.cap) + } + + @Test + fun negErrMessageWithCap_crossDeserialization() { + val msg = NegErrMessage("neg-sub1", "blocked: too many records", 500_000L) + + val kotlinDeserialized = KotlinSerializationMapper.fromJsonToMessage(JacksonMapper.toJson(msg)) + assertTrue(kotlinDeserialized is NegErrMessage) + assertEquals(500_000L, kotlinDeserialized.cap) + + val jacksonDeserialized = JacksonMapper.fromJsonToMessage(KotlinSerializationMapper.toJson(msg)) + assertTrue(jacksonDeserialized is NegErrMessage) + assertEquals(500_000L, jacksonDeserialized.cap) + } + // ========================================================================= // NEG-OPEN Command (client-to-relay) Tests // ========================================================================= From 18998c897c655ad30195ff2282d656eb55e0e8d9 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 16:42:46 +0000 Subject: [PATCH 055/132] negentropy: size reconcile windows from the caller's own index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A NEG-OPEN is all-or-nothing at both ends of the wire and neither end can see the other's size. The relay half has been handled since windowing landed — refuse, halve, retry. The client half has not: localEntries has to hold every matching (created_at, id) pair before the first NEG-OPEN goes out, so peak memory is a property of the CORPUS, not of the window. On a multi-million-event filter that list is the sync's high-water mark, and it is built even when the sync then splits into windows that each touch a fraction of it. NegentropyLocalIndex is that half. A caller whose store answers by range passes an index instead of a list, and the engine reads a window's worth at a time. count() is what makes it work: a window is sized BEFORE the round trip, so entriesFor() is only ever asked for something bounded. Callers that pass a list are unchanged — internally the list becomes an index that sorts once and binary-searches per window, exactly what the engine did inline before. targetWindow (0 = off, the old behaviour) turns the two signals into one loop. Our count splits a window before asking; their refusal shrinks the target — straight to the relay's stated cap where there is one, halved where there isn't — and windows that reconcile in one piece grow it back toward, never past, the caller's number. Neither side knows anything about the other and the same work queue absorbs both, which is what makes it adapt rather than need tuning. peerCap carries the relay's number back out, so a caller can persist it and start the NEXT sync at a window that fits. The local pre-split deliberately does NOT count against MAX_WINDOWS: that backstop exists for an overflow loop that never converges, while this split is driven by a number that provably halves with the range. Also here, because it is the same loop: page the window that overflowed rather than the whole filter. A second dense enough to exceed the cap is reachable — created_at has second granularity and is author-controlled — and negentropySyncOrFetch used to answer it by re-paging everything, including every window that had already reconciled cleanly. reconcileWindows now takes onUnreconcilableWindow and hands that window over; the sweep carries on with the rest of the range, so a dense second costs that second. Raw negentropySync/negentropyReconcile callers that pass no hook still get the exception, unchanged. pagedFallback stays conservative and now means "any part of this range came over REQ rather than a reconcile", with pagedWindows saying how much — the distinction matters to anyone recording coverage, since a paged walk booked as a completed reconcile would claim a range nothing compared. The existing over-cap test is updated rather than deleted: its ten events share one created_at, so the whole filter IS the un-reconcilable window — same events, now via the window path instead of by abandoning the sync. Its sibling test, that raw negentropySync still throws, is untouched. --- .../accessories/NegentropyLocalIndex.kt | 118 +++++++ .../accessories/NegentropySyncException.kt | 4 + .../NostrClientNegentropyFanOutExt.kt | 9 +- .../NostrClientNegentropySyncExt.kt | 330 +++++++++++++----- .../accessories/NegentropyLocalIndexTest.kt | 90 +++++ .../relay/NostrClientNegentropySyncTest.kt | 169 ++++++++- 6 files changed, 622 insertions(+), 98 deletions(-) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropyLocalIndex.kt create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropyLocalIndexTest.kt diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropyLocalIndex.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropyLocalIndex.kt new file mode 100644 index 0000000000..6ed4d51619 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropyLocalIndex.kt @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay.client.accessories + +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.store.IdAndTime + +/** + * The caller's own matching set, read one `created_at` window at a time. + * + * The list overloads of [negentropySync] / [negentropyReconcile] need every + * matching `(created_at, id)` pair before the first NEG-OPEN goes out, which + * makes peak memory a property of the corpus: a multi-million-event filter is + * a multi-million-entry list held for the whole sync, whether or not the sync + * ends up splitting into windows that each touch a fraction of it. + * + * A caller whose store can answer by range doesn't need that. Passing an index + * instead lets the window engine ask for a window's worth at a time, so the + * high-water mark becomes the size of one window — which the engine also sizes, + * from [count], before spending a round trip on it. + * + * Both methods are called on the reconciler coroutines, possibly concurrently + * when `reconcileConcurrency > 1`, and possibly more than once for the same + * window (a window that overflows is re-asked as halves). Implementations + * should be cheap and side-effect free; a store-backed one usually is, since + * these are index scans. + */ +interface NegentropyLocalIndex { + /** + * How many local events fall inside [window], or null when the store + * cannot answer cheaply. + * + * This is what lets the engine split a window BEFORE asking the relay for + * it — the only signal available about our own side, and the one that + * bounds what [entriesFor] will have to materialise. Null disables that + * pre-split for the window; the relay's own refusal is then the only thing + * that shrinks it, exactly as before this method existed. + */ + suspend fun count(window: Filter): Int? + + /** The `(created_at, id)` pairs inside [window]. Order does not matter. */ + suspend fun entriesFor(window: Filter): List + + companion object { + /** Nothing held locally: the sync downloads the relay's whole matched set. */ + val Empty: NegentropyLocalIndex = + object : NegentropyLocalIndex { + override suspend fun count(window: Filter) = 0 + + override suspend fun entriesFor(window: Filter) = emptyList() + } + + /** + * An index over a list already in memory — what the list overloads use, + * so they behave exactly as they did: sorted once, then binary-searched + * per window. + */ + fun of(entries: List): NegentropyLocalIndex = if (entries.isEmpty()) Empty else SortedListIndex(entries.sortedBy { it.createdAt }) + } +} + +private class SortedListIndex( + private val sorted: List, +) : NegentropyLocalIndex { + override suspend fun count(window: Filter): Int = slice(window).size + + override suspend fun entriesFor(window: Filter): List = slice(window) + + /** + * The `createdAt`-range slice of [sorted] (ascending by `createdAt`) that + * belongs to `[since, until]` (both inclusive, NIP-01 semantics). + * Binary-searched so window splits stay O(log n) over multi-million sets. + */ + private fun slice(window: Filter): List { + val since = window.since + val until = window.until + if (sorted.isEmpty() || (since == null && until == null)) return sorted + + val lo = since ?: 0L + val hi = until ?: Long.MAX_VALUE + + // first index with createdAt >= lo + var start = 0 + var e = sorted.size + while (start < e) { + val mid = (start + e) ushr 1 + if (sorted[mid].createdAt < lo) start = mid + 1 else e = mid + } + + // first index with createdAt > hi + var end = start + e = sorted.size + while (end < e) { + val mid = (end + e) ushr 1 + if (sorted[mid].createdAt <= hi) end = mid + 1 else e = mid + } + + return if (start >= end) emptyList() else sorted.subList(start, end) + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropySyncException.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropySyncException.kt index f6018a8098..1a6dacc98d 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropySyncException.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropySyncException.kt @@ -43,12 +43,16 @@ import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl * @property window the filter slice that failed. * @property reason machine-readable category — branch on this to recover. * @property detail the underlying specifics (a relay's `NEG-ERR` text, `timeout`, …). + * @property cap the relay's own `max_sync_events` when its refusal stated one + * (see [com.vitorpamplona.quartz.nip77Negentropy.NegErrMessage.statedCap]). + * Worth persisting per relay: it is what sizes the first window next time. */ class NegentropySyncException( val relay: NormalizedRelayUrl, val window: Filter, val reason: Reason, val detail: String, + val cap: Long? = null, ) : Exception("NIP-77 sync of $relay failed ($reason): $detail") { enum class Reason { /** diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropyFanOutExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropyFanOutExt.kt index 7ac08647f5..e941af4417 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropyFanOutExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropyFanOutExt.kt @@ -77,6 +77,8 @@ suspend fun negentropySyncFanOut( relay: NormalizedRelayUrl, filter: Filter, localEntries: List = emptyList(), + localIndex: NegentropyLocalIndex? = null, + targetWindow: Int = 0, maxEvents: Int = 0, reqsPerClient: Int = 10, fetchBatch: Int = 250, @@ -135,8 +137,6 @@ suspend fun negentropySyncFanOut( val producer = launch { try { - val sorted = - if (localEntries.size > 1) localEntries.sortedBy { it.createdAt } else localEntries // Windows reconcile round-robin ACROSS the clients so // server-side snapshot builds parallelize per connection // (a single connection produced ids at only ~9k/s and @@ -145,17 +145,18 @@ suspend fun negentropySyncFanOut( clients = clients, relay = relay, filter = filter, - localEntries = sorted, + local = localIndex ?: NegentropyLocalIndex.of(localEntries), idleTimeoutMs = idleTimeoutMs, batchSize = fetchBatch, reconcileConcurrency = reconcileConcurrency, + targetWindow = targetWindow, onWindow = { windows.incrementAndFetch() }, onNeed = { need.addAndFetch(it) }, onHave = { have.addAndFetch(it) }, sendNeedBatch = { batch -> idBatches.send(batch) }, - sendHaveBatch = if (localEntries.isEmpty()) null else { _ -> }, + sendHaveBatch = if (localEntries.isEmpty() && localIndex == null) null else { _ -> }, ) } finally { idBatches.close() diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropySyncExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropySyncExt.kt index 611720876a..6595cc763e 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropySyncExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropySyncExt.kt @@ -66,12 +66,17 @@ import kotlin.math.min * @property downloaded distinct events actually delivered through `onEvent`. * @property windows number of `created_at` windows the matched set was split * into (`1` when the relay reconciled the whole filter in one shot). + * @property peerCap the relay's own `max_sync_events`, when a refusal during + * this sync stated one. Worth persisting per relay: it is the number that + * sizes the first window of the NEXT sync, and it is not discoverable any + * other way. */ class NegentropySyncResult( val needCount: Int, val haveCount: Int, val downloaded: Int, val windows: Int, + val peerCap: Long? = null, ) /** @@ -162,12 +167,16 @@ suspend fun INostrClient.negentropySync( reconcileConcurrency: Int = 1, idBufferBatches: Int = maxConcurrentReqs * 4, localEntries: List = emptyList(), + localIndex: NegentropyLocalIndex? = null, + targetWindow: Int = 0, + onUnreconcilableWindow: (suspend (Filter) -> Unit)? = null, onProgress: ((needSoFar: Int, downloaded: Int) -> Unit)? = null, onEvent: suspend (Event) -> Unit, ): NegentropySyncResult { val need = AtomicInt(0) val windows = AtomicInt(0) var downloaded = 0 + var peerCap: Long? = null // Pin the relay in the pool's "desired" set for the whole sync. A NEG-OPEN is not // a REQ, so during a reconcile round (before that window's first download REQ @@ -195,8 +204,11 @@ suspend fun INostrClient.negentropySync( maxConcurrentReqs = maxConcurrentReqs, reconcileConcurrency = reconcileConcurrency, idBufferBatches = idBufferBatches, - localEntries = localEntries, + local = localIndex ?: NegentropyLocalIndex.of(localEntries), + targetWindow = targetWindow, + onUnreconcilableWindow = onUnreconcilableWindow, onWindow = { windows.incrementAndFetch() }, + onPeerCap = { peerCap = it }, // Only accumulate here; progress is reported from the // single consumer loop below so the user callback is never // invoked from two coroutines at once. @@ -228,6 +240,7 @@ suspend fun INostrClient.negentropySync( haveCount = 0, downloaded = downloaded, windows = windows.load(), + peerCap = peerCap, ) } @@ -241,6 +254,9 @@ suspend fun INostrClient.negentropySync( reconcileConcurrency: Int = 1, idBufferBatches: Int = maxConcurrentReqs * 4, localEntries: List = emptyList(), + localIndex: NegentropyLocalIndex? = null, + targetWindow: Int = 0, + onUnreconcilableWindow: (suspend (Filter) -> Unit)? = null, onProgress: ((needSoFar: Int, downloaded: Int) -> Unit)? = null, onEvent: suspend (Event) -> Unit, ): NegentropySyncResult = @@ -254,6 +270,9 @@ suspend fun INostrClient.negentropySync( reconcileConcurrency = reconcileConcurrency, idBufferBatches = idBufferBatches, localEntries = localEntries, + localIndex = localIndex, + targetWindow = targetWindow, + onUnreconcilableWindow = onUnreconcilableWindow, onProgress = onProgress, onEvent = onEvent, ) @@ -263,16 +282,28 @@ suspend fun INostrClient.negentropySync( * * @property downloaded distinct events delivered through `onEvent` (across whichever * path ran). - * @property pagedFallback `true` if negentropy could not reconcile and the events - * came from [fetchAllPages] instead. + * @property pagedFallback `true` if ANY part of the range came from + * [fetchAllPages] rather than a reconcile — either the whole filter (the + * relay could not reconcile at all) or the individual windows counted by + * [pagedWindows]. Deliberately conservative: a caller recording what it has + * covered must not book a paged walk as a completed reconcile, and one + * un-reconcilable second in the range is enough to make that claim untrue. * @property negentropy the negentropy outcome when it succeeded; `null` on fallback. - * @property fallbackCause why negentropy was abandoned; `null` when it succeeded. + * @property fallbackCause why negentropy was abandoned for the WHOLE filter; + * `null` when it was not — including when individual windows were paged, which + * have no single cause between them. + * @property pagedWindows how many individual `created_at` windows were paged + * inside an otherwise-successful negentropy sync — seconds so dense the relay + * would not reconcile them at any window size. `0` for almost every sync; + * non-zero means part of the range came over REQ and is subject to a paged + * walk's limits rather than a reconcile's guarantees. */ class NegentropyOrFetchResult( val downloaded: Int, val pagedFallback: Boolean, val negentropy: NegentropySyncResult?, val fallbackCause: NegentropySyncException?, + val pagedWindows: Int = 0, ) /** @@ -308,11 +339,14 @@ suspend fun INostrClient.negentropySyncOrFetch( reconcileConcurrency: Int = 1, idBufferBatches: Int = maxConcurrentReqs * 4, localEntries: List = emptyList(), + localIndex: NegentropyLocalIndex? = null, + targetWindow: Int = 0, onProgress: ((needSoFar: Int, downloaded: Int) -> Unit)? = null, onEvent: suspend (Event) -> Unit, ): NegentropyOrFetchResult { val seen = HashSet() var delivered = 0 + var pagedWindows = 0 // Shared dedup + cap across both phases. Returns true if the event was new and // delivered. Both phases run sequentially, so no concurrent access. @@ -337,9 +371,32 @@ suspend fun INostrClient.negentropySyncOrFetch( reconcileConcurrency = reconcileConcurrency, idBufferBatches = idBufferBatches, localEntries = localEntries, + localIndex = localIndex, + targetWindow = targetWindow, + // One second the relay will not reconcile at any size costs + // that second, not the sync. Without this the exception below + // catches it and re-pages the WHOLE filter — every window that + // already reconciled cleanly walked again over REQ, which on a + // large corpus is the entire cost negentropy was there to save. + onUnreconcilableWindow = { window -> + pagedWindows++ + val pageTimeoutMs = if (idleTimeoutMs > 0) idleTimeoutMs else DEFAULT_DOWNLOAD_IDLE_MS + fetchAllPages(relay, listOf(window), pageTimeoutMs) { event -> + if (accept(event)) onProgress?.invoke(delivered, delivered) + } + }, onProgress = onProgress, ) { accept(it) } - NegentropyOrFetchResult(delivered, pagedFallback = false, negentropy = result, fallbackCause = null) + NegentropyOrFetchResult( + delivered, + // Any paged window makes this not a clean reconcile — see the + // property doc: under-reporting it would let a caller record + // coverage it never compared. + pagedFallback = pagedWindows > 0, + negentropy = result, + fallbackCause = null, + pagedWindows = pagedWindows, + ) } catch (e: NegentropySyncException) { // Negentropy couldn't enumerate the set — page the whole filter instead, // skipping anything the negentropy attempt already delivered. fetchAllPages @@ -349,7 +406,13 @@ suspend fun INostrClient.negentropySyncOrFetch( fetchAllPages(relay, listOf(pageFilter), pageTimeoutMs) { event -> if (accept(event)) onProgress?.invoke(delivered, delivered) } - NegentropyOrFetchResult(delivered, pagedFallback = true, negentropy = null, fallbackCause = e) + NegentropyOrFetchResult( + delivered, + pagedFallback = true, + negentropy = null, + fallbackCause = e, + pagedWindows = pagedWindows, + ) } } @@ -363,6 +426,8 @@ suspend fun INostrClient.negentropySyncOrFetch( reconcileConcurrency: Int = 1, idBufferBatches: Int = maxConcurrentReqs * 4, localEntries: List = emptyList(), + localIndex: NegentropyLocalIndex? = null, + targetWindow: Int = 0, onProgress: ((needSoFar: Int, downloaded: Int) -> Unit)? = null, onEvent: suspend (Event) -> Unit, ): NegentropyOrFetchResult = @@ -376,6 +441,8 @@ suspend fun INostrClient.negentropySyncOrFetch( reconcileConcurrency = reconcileConcurrency, idBufferBatches = idBufferBatches, localEntries = localEntries, + localIndex = localIndex, + targetWindow = targetWindow, onProgress = onProgress, onEvent = onEvent, ) @@ -411,9 +478,12 @@ private suspend fun INostrClient.syncPipeline( maxConcurrentReqs: Int, reconcileConcurrency: Int, idBufferBatches: Int, - localEntries: List, + local: NegentropyLocalIndex, + targetWindow: Int, onWindow: () -> Unit, onNeed: (Int) -> Unit, + onPeerCap: ((Long) -> Unit)?, + onUnreconcilableWindow: (suspend (Filter) -> Unit)?, deliver: suspend (Event) -> Unit, ) = coroutineScope { val idBatches = Channel>(idBufferBatches.coerceAtLeast(1)) @@ -430,21 +500,20 @@ private suspend fun INostrClient.syncPipeline( } } - // reconcileWindows needs the local set sorted by createdAt (it binary-searches - // each window's slice). Empty/singleton sets are already trivially sorted. - val sortedLocal = if (localEntries.size > 1) localEntries.sortedBy { it.createdAt } else localEntries - reconcileWindows( clients = listOf(this@syncPipeline), relay = relay, filter = filter, - localEntries = sortedLocal, + local = local, idleTimeoutMs = idleTimeoutMs, batchSize = fetchBatch, reconcileConcurrency = reconcileConcurrency, + targetWindow = targetWindow, onWindow = onWindow, onNeed = onNeed, onHave = {}, + onPeerCap = onPeerCap, + onUnreconcilableWindow = onUnreconcilableWindow, sendNeedBatch = { batch -> idBatches.send(batch) }, sendHaveBatch = null, ) @@ -455,16 +524,29 @@ private suspend fun INostrClient.syncPipeline( /** * The shared window engine behind [negentropySync] and [negentropyReconcile]: - * reconciles [filter] against [localEntries], splitting into `created_at` - * windows whenever the relay rejects the set as too large, with up to - * [reconcileConcurrency] windows reconciling at once from a shared work - * queue. Each window's local subset is sliced out of [localEntries] (which - * MUST be sorted by `createdAt`) so both sides always reconcile the same - * slice of the timeline. + * reconciles [filter] against [local], splitting into `created_at` windows, + * with up to [reconcileConcurrency] windows reconciling at once from a shared + * work queue. Each window reconciles against that window's slice of [local], so + * both sides always compare the same slice of the timeline. + * + * Two independent things split a window, and the same queue absorbs both: + * + * - **The relay refuses it** (strfry's `max_sync_events`). Known only after a + * round trip, and the only signal available about THEIR size. + * - **We hold more than [targetWindow] in it**, per [NegentropyLocalIndex.count], + * which is known before the round trip and is what bounds the entries this + * engine asks [local] to materialise. Off when [targetWindow] is `0` (the + * default), which is the pre-existing behaviour: one window until refused. + * + * Neither side can see the other's size, so [targetWindow] adapts within the + * sync: a refusal shrinks it — straight to the relay's own cap when the refusal + * states one ([NegErrMessage.statedCap]), halved when it does not — and windows + * that reconcile in one piece grow it back toward, never past, the caller's + * number. * * Throws [NegentropySyncException] for any window negentropy cannot reconcile - * (a minimal window still over the cap, or an unavailable/erroring relay); the - * failure cancels the whole scope. + * (a minimal window still over the cap with no [onUnreconcilableWindow] to hand + * it to, or an unavailable/erroring relay); the failure cancels the whole scope. */ @OptIn(ExperimentalAtomicApi::class) internal suspend fun reconcileWindows( @@ -474,13 +556,19 @@ internal suspend fun reconcileWindows( clients: List, relay: NormalizedRelayUrl, filter: Filter, - localEntries: List, + local: NegentropyLocalIndex, idleTimeoutMs: Long, batchSize: Int, reconcileConcurrency: Int, + targetWindow: Int = 0, onWindow: () -> Unit, onNeed: (Int) -> Unit, onHave: (Int) -> Unit, + onPeerCap: ((Long) -> Unit)? = null, + // Given a minimal window the relay will not reconcile at any size, instead + // of throwing. The caller drains it however it can (paging it over REQ) and + // the sweep carries on with the rest of the filter. + onUnreconcilableWindow: (suspend (Filter) -> Unit)? = null, sendNeedBatch: suspend (List) -> Unit, sendHaveBatch: (suspend (List) -> Unit)?, ) = coroutineScope { @@ -503,6 +591,33 @@ internal suspend fun reconcileWindows( // the tens–hundreds, so the cap is orders of magnitude above any real sync. val totalWindows = AtomicInt(1) + // The largest window this sync will ask for, in events. Shrinks on a + // refusal, recovers toward the caller's number on clean windows, and is + // read only where a local count exists to compare it against — with + // targetWindow at 0 nothing below this line does anything. + val budget = AtomicInt(targetWindow) + + // Splits a window in two and queues both halves. Returns false when the + // window is already minimal — `created_at` is in seconds, so that is the + // floor, not a tuning choice. + fun splitInto( + pendingWindow: Filter, + lo: Long, + hi: Long, + ): Boolean { + if (hi - lo <= MIN_WINDOW_SECONDS) return false + val mid = lo + (hi - lo) / 2 + remaining.incrementAndFetch() + // The lower child gets the finite midpoint; the upper child KEEPS this + // window's original `until` (which may be null = unbounded). Replacing + // null with `now()` here would drop every event dated after now() + // (clock skew) once any split happens, while the un-split path would + // have included them. + pending.trySend(pendingWindow.copy(since = lo, until = mid)) + pending.trySend(pendingWindow.copy(since = mid + 1, until = pendingWindow.until)) + return true + } + val reconcilers = List(reconcileConcurrency.coerceAtLeast(1)) { reconcilerIndex -> launch { @@ -510,11 +625,27 @@ internal suspend fun reconcileWindows( for (window in pending) { coroutineContext.ensureActive() + val lo = window.since ?: 0L + val hi = window.until ?: TimeUtils.now() + + // Our own side, before the round trip. Deliberately NOT + // counted against MAX_WINDOWS: that backstop guards against + // an overflow loop that never converges, while this split is + // driven by a number that provably halves with the range. + val ceiling = budget.load() + if (ceiling > 0 && hi - lo > MIN_WINDOW_SECONDS) { + val mine = local.count(window) + if (mine != null && mine > ceiling) { + splitInto(window, lo, hi) + continue + } + } + val outcome = client.reconcileStreaming( relay = relay, filter = window, - localEntries = entriesForWindow(localEntries, window.since, window.until), + localEntries = local.entriesFor(window), idleTimeoutMs = idleTimeoutMs, fetchBatch = batchSize, onNeed = onNeed, @@ -526,21 +657,53 @@ internal suspend fun reconcileWindows( when (outcome) { is ReconcileOutcome.Complete -> { onWindow() + // A window that fitted is evidence the budget can + // recover — gently, and never past what the caller + // asked for, so a sync that met one dense stretch + // does not stay small for the rest of the timeline. + if (targetWindow > 0) { + val now = budget.load() + if (now < targetWindow) { + budget.store(minOf(targetWindow, (now * BUDGET_GROWTH).toInt().coerceAtLeast(now + 1))) + } + } if (remaining.decrementAndFetch() == 0) pending.close() } is ReconcileOutcome.Overflow -> { - val lo = window.since ?: 0L - val hi = window.until ?: TimeUtils.now() + // What they will take, when they said so: one step + // instead of a halving ladder, for this sync and — + // via onPeerCap — for whatever the caller persists. + outcome.cap?.let { cap -> + onPeerCap?.invoke(cap) + if (targetWindow > 0) { + val fitted = (cap * CAP_MARGIN).toInt().coerceAtLeast(1) + if (fitted < budget.load()) budget.store(fitted) + } + } + if (outcome.cap == null && targetWindow > 0) { + // No number to go on: halve and find out. + budget.store((budget.load() / 2).coerceAtLeast(1)) + } if (hi - lo <= MIN_WINDOW_SECONDS) { - // A minimal window that still overflows: negentropy - // genuinely can't enumerate this slice. Surface it — - // paging is the caller's call. + // A minimal window that still overflows: + // negentropy genuinely can't enumerate this + // slice. Hand it to the caller if it has a way + // to drain it, otherwise surface it — paging is + // the caller's call either way. + val fallback = onUnreconcilableWindow + if (fallback != null) { + fallback(window) + onWindow() + if (remaining.decrementAndFetch() == 0) pending.close() + continue + } throw NegentropySyncException( relay = relay, window = window, reason = NegentropySyncException.Reason.OVER_MAX_SYNC_EVENTS, detail = "created_at window [$lo, $hi] still exceeds the relay's max_sync_events", + cap = outcome.cap, ) } if (totalWindows.addAndFetch(2) > MAX_WINDOWS) { @@ -554,15 +717,7 @@ internal suspend fun reconcileWindows( detail = "created_at window split exceeded $MAX_WINDOWS windows without converging; the relay likely rejects negentropy with an overflow-looking error", ) } - val mid = lo + (hi - lo) / 2 - remaining.incrementAndFetch() - // The lower child gets the finite midpoint; the upper child - // KEEPS this window's original `until` (which may be null = - // unbounded). Replacing null with `now()` here would drop - // every event dated after now() (clock skew) once any split - // happens, while the un-split path would have included them. - pending.send(window.copy(since = lo, until = mid)) - pending.send(window.copy(since = mid + 1, until = window.until)) + splitInto(window, lo, hi) } is ReconcileOutcome.Failed -> @@ -580,46 +735,17 @@ internal suspend fun reconcileWindows( reconcilers.joinAll() } -/** - * The `createdAt`-range slice of [sorted] (ascending by `createdAt`) that - * belongs to the window `[since, until]` (both inclusive, NIP-01 semantics). - * Binary-searched so window splits stay O(log n) over multi-million local sets. - */ -private fun entriesForWindow( - sorted: List, - since: Long?, - until: Long?, -): List { - if (sorted.isEmpty() || (since == null && until == null)) return sorted - - val lo = since ?: 0L - val hi = until ?: Long.MAX_VALUE - - // first index with createdAt >= lo - var start = 0 - var e = sorted.size - while (start < e) { - val mid = (start + e) ushr 1 - if (sorted[mid].createdAt < lo) start = mid + 1 else e = mid - } - - // first index with createdAt > hi - var end = start - e = sorted.size - while (end < e) { - val mid = (end + e) ushr 1 - if (sorted[mid].createdAt <= hi) end = mid + 1 else e = mid - } - - return if (start >= end) emptyList() else sorted.subList(start, end) -} - private sealed interface ReconcileOutcome { /** Reconciliation completed; every id was streamed to the downloader. */ object Complete : ReconcileOutcome - /** Relay rejected the set as too large (strfry `max_sync_events`). */ - object Overflow : ReconcileOutcome + /** + * Relay rejected the set as too large (strfry `max_sync_events`). + * [cap] is the relay's own limit when the refusal stated one. + */ + class Overflow( + val cap: Long?, + ) : ReconcileOutcome /** Reconciliation could not complete; [detail] says why. */ class Failed( @@ -633,11 +759,14 @@ private sealed interface ReconcileOutcome { * @property needCount ids the relay has that the local set lacks (streamed to `onNeedIds`). * @property haveCount ids the local set has that the relay lacks (streamed to `onHaveIds`). * @property windows number of `created_at` windows the reconcile split into. + * @property peerCap the relay's own `max_sync_events`, when a refusal during + * this reconcile stated one. */ class NegentropyReconcileResult( val needCount: Int, val haveCount: Int, val windows: Int, + val peerCap: Long? = null, ) /** @@ -682,15 +811,19 @@ suspend fun INostrClient.negentropyReconcile( relay: NormalizedRelayUrl, filter: Filter, localEntries: List = emptyList(), + localIndex: NegentropyLocalIndex? = null, + targetWindow: Int = 0, batchSize: Int = 500, idleTimeoutMs: Long = 120_000L, reconcileConcurrency: Int = 1, + onUnreconcilableWindow: (suspend (Filter) -> Unit)? = null, onHaveIds: (suspend (List) -> Unit)? = null, onNeedIds: suspend (List) -> Unit, ): NegentropyReconcileResult { val need = AtomicInt(0) val have = AtomicInt(0) val windows = AtomicInt(0) + var peerCap: Long? = null // Same connection-pinning trick as negentropySync: a NEG-OPEN is not a REQ, // so without a live subscription the pool would consider the relay unwanted @@ -698,24 +831,20 @@ suspend fun INostrClient.negentropyReconcile( val keepAliveSubId = newSubId() subscribe(keepAliveSubId, mapOf(relay to listOf(Filter(ids = listOf(KEEP_ALIVE_ID)))), null) try { - val sorted = - if (localEntries.size > 1) { - localEntries.sortedBy { it.createdAt } - } else { - localEntries - } - reconcileWindows( clients = listOf(this), relay = relay, filter = filter, - localEntries = sorted, + local = localIndex ?: NegentropyLocalIndex.of(localEntries), idleTimeoutMs = idleTimeoutMs, batchSize = batchSize, reconcileConcurrency = reconcileConcurrency, + targetWindow = targetWindow, onWindow = { windows.incrementAndFetch() }, onNeed = { need.addAndFetch(it) }, onHave = { have.addAndFetch(it) }, + onPeerCap = { peerCap = it }, + onUnreconcilableWindow = onUnreconcilableWindow, sendNeedBatch = onNeedIds, sendHaveBatch = onHaveIds, ) @@ -727,6 +856,7 @@ suspend fun INostrClient.negentropyReconcile( needCount = need.load(), haveCount = have.load(), windows = windows.load(), + peerCap = peerCap, ) } @@ -734,9 +864,12 @@ suspend fun INostrClient.negentropyReconcile( relay: String, filter: Filter, localEntries: List = emptyList(), + localIndex: NegentropyLocalIndex? = null, + targetWindow: Int = 0, batchSize: Int = 500, idleTimeoutMs: Long = 120_000L, reconcileConcurrency: Int = 1, + onUnreconcilableWindow: (suspend (Filter) -> Unit)? = null, onHaveIds: (suspend (List) -> Unit)? = null, onNeedIds: suspend (List) -> Unit, ): NegentropyReconcileResult = @@ -744,9 +877,12 @@ suspend fun INostrClient.negentropyReconcile( relay = RelayUrlNormalizer.normalize(relay), filter = filter, localEntries = localEntries, + localIndex = localIndex, + targetWindow = targetWindow, batchSize = batchSize, idleTimeoutMs = idleTimeoutMs, reconcileConcurrency = reconcileConcurrency, + onUnreconcilableWindow = onUnreconcilableWindow, onHaveIds = onHaveIds, onNeedIds = onNeedIds, ) @@ -895,7 +1031,7 @@ private suspend fun INostrClient.reconcileStreaming( clock.bump() if (msg.subId == subId) { sawNegFrame = true - incoming.trySend(NegFrame.Err(msg.reason)) + incoming.trySend(NegFrame.Err(msg.reason, msg.statedCap)) } } @@ -965,7 +1101,11 @@ private suspend fun INostrClient.reconcileStreaming( when (frame) { is NegFrame.Err -> - return if (isOverflow(frame.reason)) ReconcileOutcome.Overflow else ReconcileOutcome.Failed(frame.reason) + return if (isOverflow(frame.reason)) { + ReconcileOutcome.Overflow(frame.cap) + } else { + ReconcileOutcome.Failed(frame.reason) + } is NegFrame.Msg -> { val result = session.processMessage(frame.payload) @@ -1014,6 +1154,8 @@ private sealed interface NegFrame { class Err( val reason: String, + // The relay's own max_sync_events, when the refusal stated one. + val cap: Long? = null, ) : NegFrame } @@ -1041,13 +1183,7 @@ private sealed interface NegFrame { * [reconcileWindows] also caps the total window count as a wording-independent * backstop, so a novel overflow-looking-but-not-shrinking error can never storm. */ -internal fun isOverflow(reason: String): Boolean = - reason.contains("too many records", ignoreCase = true) || - reason.contains("too many results", ignoreCase = true) || - reason.contains("too many query results", ignoreCase = true) || - reason.contains("result set too large", ignoreCase = true) || - reason.contains("results too large", ignoreCase = true) || - reason.contains("max_sync_events", ignoreCase = true) +internal fun isOverflow(reason: String): Boolean = NegErrMessage.isOverflow(reason) /** * A relay that advertises NIP-77 but refuses it at runtime signals the refusal with @@ -1159,6 +1295,22 @@ private const val MIN_WINDOW_SECONDS = 1L */ private const val MAX_WINDOWS = 100_000 +/** + * How much of a relay's stated `max_sync_events` a window actually aims for. + * The margin absorbs what the relay gains between stating that number and + * answering the next NEG-OPEN — asking for exactly the cap would be refused + * again by anything still being written to. + */ +private const val CAP_MARGIN = 0.8 + +/** + * How fast a shrunk window grows back toward the caller's target, per window + * that reconciled in one piece. Multiplicative and gentle on purpose: too small + * costs an extra round trip, too big costs a refused NEG-OPEN plus the snapshot + * scan the relay did before refusing it. + */ +private const val BUDGET_GROWTH = 1.25 + /** Bounded buffer between the download workers and the single delivery consumer. */ private const val DELIVERY_BUFFER = 256 diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropyLocalIndexTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropyLocalIndexTest.kt new file mode 100644 index 0000000000..7967d1695d --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropyLocalIndexTest.kt @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay.client.accessories + +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.store.IdAndTime +import kotlinx.coroutines.test.runTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * The list-backed index is what the `localEntries` overloads become, so its + * slicing has to keep NIP-01's inclusive `since`/`until` exactly: a window that + * dropped its boundary second would leave events neither side ever compares, + * and the two sides of a reconcile would disagree about what the window holds. + */ +class NegentropyLocalIndexTest { + private fun idAt(second: Long) = IdAndTime(second, second.toString().padStart(64, '0')) + + private val index = NegentropyLocalIndex.of((1000L..1009L).map { idAt(it) }) + + private fun window( + since: Long?, + until: Long?, + ) = Filter(kinds = listOf(1), since = since, until = until) + + @Test + fun bothBoundsAreInclusive() = + runTest { + assertEquals(3, index.count(window(1002, 1004))) + assertEquals(listOf(1002L, 1003L, 1004L), index.entriesFor(window(1002, 1004)).map { it.createdAt }) + } + + @Test + fun anUnboundedSideReachesTheEnd() = + runTest { + assertEquals(5, index.count(window(1005, null))) + assertEquals(6, index.count(window(null, 1005))) + assertEquals(10, index.count(window(null, null))) + } + + @Test + fun aWindowOutsideEverythingIsEmpty() = + runTest { + assertEquals(0, index.count(window(2000, 3000))) + assertTrue(index.entriesFor(window(2000, 3000)).isEmpty()) + } + + @Test + fun aSingleSecondWindowHoldsThatSecond() = + runTest { + assertEquals(1, index.count(window(1007, 1007))) + assertEquals(listOf(1007L), index.entriesFor(window(1007, 1007)).map { it.createdAt }) + } + + @Test + fun entriesNeedNotArriveSorted() = + runTest { + val shuffled = NegentropyLocalIndex.of(listOf(idAt(1005), idAt(1001), idAt(1009), idAt(1003))) + assertEquals(2, shuffled.count(window(1001, 1003))) + assertEquals(listOf(1001L, 1003L), shuffled.entriesFor(window(1001, 1003)).map { it.createdAt }) + } + + @Test + fun theEmptyIndexAnswersZeroForEveryWindow() = + runTest { + assertEquals(0, NegentropyLocalIndex.Empty.count(window(1000, 2000))) + assertTrue(NegentropyLocalIndex.Empty.entriesFor(window(1000, 2000)).isEmpty()) + assertEquals(0, NegentropyLocalIndex.of(emptyList()).count(window(null, null))) + } +} diff --git a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientNegentropySyncTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientNegentropySyncTest.kt index 52aea9a8b4..6f4b121146 100644 --- a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientNegentropySyncTest.kt +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientNegentropySyncTest.kt @@ -26,6 +26,7 @@ import com.vitorpamplona.geode.testing.RelayClientTest import com.vitorpamplona.geode.testing.preload import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.client.NostrClient +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.NegentropyLocalIndex import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.NegentropySyncException import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.fetchAllPages import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.negentropySync @@ -36,6 +37,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer import com.vitorpamplona.quartz.nip01Core.relay.server.policies.PassThroughPolicy import com.vitorpamplona.quartz.nip01Core.relay.server.policies.PolicyResult +import com.vitorpamplona.quartz.nip01Core.store.IdAndTime import com.vitorpamplona.quartz.nip77Negentropy.NegentropySettings import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -48,6 +50,8 @@ import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFailsWith import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull import kotlin.test.assertTrue class NostrClientNegentropySyncTest : RelayClientTest() { @@ -252,6 +256,11 @@ class NostrClientNegentropySyncTest : RelayClientTest() { * The "try negentropy, else page" combinator: against the same over-cap relay * where raw [negentropySync] throws, [negentropySyncOrFetch] transparently pages * and delivers every event, reporting that it fell back. + * + * Every event here shares one `created_at`, so the whole filter IS the + * un-reconcilable window: it is drained as one paged window rather than by + * abandoning the sync, which is why `fallbackCause` is null. On a filter + * spanning more than this second, everything outside it still reconciles. */ @Test fun orFetchPagesWhenNegentropyCannotReconcile() = @@ -275,11 +284,9 @@ class NostrClientNegentropySyncTest : RelayClientTest() { assertEquals(10, got.map { it.id }.toSet().size, "all events delivered via the paging fallback") assertEquals(10, result.downloaded) - assertTrue(result.pagedFallback, "it should have fallen back to paging") - assertEquals( - NegentropySyncException.Reason.OVER_MAX_SYNC_EVENTS, - result.fallbackCause?.reason, - ) + assertTrue(result.pagedFallback, "part of the range came over REQ, so this was not a clean reconcile") + assertEquals(1, result.pagedWindows, "exactly the one un-reconcilable window was paged") + assertNull(result.fallbackCause, "the sync was not abandoned — one window was drained by paging") } finally { client.disconnect() scope.cancel() @@ -375,4 +382,156 @@ class NostrClientNegentropySyncTest : RelayClientTest() { assertFalse(result.pagedFallback, "negentropy should have handled it") assertEquals(8, result.negentropy?.downloaded) } + + /** + * The caller's own count splits a window BEFORE the relay is asked for it. + * + * Nothing here overflows — the relay would have reconciled the whole filter + * in one NEG-OPEN — so every split is driven by [NegentropyLocalIndex.count] + * against `targetWindow`. That is what bounds the entries a caller has to + * materialise: without it the first (and only) window is the whole filter, + * and the local set for it is the whole corpus. + */ + @Test + fun targetWindowSplitsFromTheLocalCountAlone() = + runBlocking { + // 40 seconds of history, one event each; we already hold the even ones. + val all = (0 until 40).map { SyntheticEvents.fakeEvent(idSeed = it + 1, kind = 1, createdAt = 1000L + it) } + defaultRelay.preload(all) + val ours = all.filterIndexed { i, _ -> i % 2 == 0 }.map { IdAndTime(it.createdAt, it.id) } + + val asked = mutableListOf() + val index = + object : NegentropyLocalIndex { + val inner = NegentropyLocalIndex.of(ours) + + override suspend fun count(window: Filter): Int { + asked += window + return inner.count(window) ?: 0 + } + + override suspend fun entriesFor(window: Filter) = inner.entriesFor(window) + } + + val got = mutableListOf() + val result = + withTimeout(60_000) { + client.negentropySync( + relay = defaultRelayUrl, + filter = Filter(kinds = listOf(1)), + localIndex = index, + targetWindow = 5, + ) { got.add(it) } + } + + assertEquals(20, got.map { it.id }.toSet().size, "only the half we lacked comes down") + assertTrue(result.windows > 1, "the local count alone must have split the filter") + assertTrue(asked.isNotEmpty(), "windows must be counted before they are asked for") + assertNull(result.peerCap, "nothing was refused, so there is no cap to report") + } + + /** Passing no target keeps the old shape: one window until the relay objects. */ + @Test + fun withoutATargetTheLocalCountIsNeverConsulted() = + runBlocking { + defaultRelay.preload(SyntheticEvents.batch(20, kind = 1)) + var counted = 0 + val index = + object : NegentropyLocalIndex { + override suspend fun count(window: Filter): Int { + counted++ + return 1_000_000 + } + + override suspend fun entriesFor(window: Filter) = emptyList() + } + + val result = + withTimeout(20_000) { + client.negentropySync( + relay = defaultRelayUrl, + filter = Filter(kinds = listOf(1)), + localIndex = index, + ) { } + } + + assertEquals(0, counted, "targetWindow = 0 must not ask the store anything") + assertEquals(1, result.windows) + assertEquals(20, result.downloaded) + } + + /** + * A relay that refuses for size states its cap, and the client reports it — + * so the next sync can start at a window that fits instead of rediscovering + * it by halving. + */ + @Test + fun theRelaysCapIsReportedBack() = + runBlocking { + val hub = InProcessRelays(negentropySettings = NegentropySettings(maxSyncEvents = 3)) + val scope = CoroutineScope(Dispatchers.Default + SupervisorJob()) + val client = NostrClient(hub, scope) + try { + val url = RelayUrlNormalizer.normalize("ws://127.0.0.1:7786/") + hub.getOrCreate(url).preload((0 until 12).map { SyntheticEvents.fakeEvent(idSeed = it + 1, kind = 1, createdAt = 1000L + it) }) + + val result = + withTimeout(60_000) { + client.negentropySync(relay = url, filter = Filter(kinds = listOf(1))) { } + } + + assertEquals(12, result.downloaded) + assertTrue(result.windows > 1) + assertEquals(3L, result.peerCap, "the relay stated its own max_sync_events") + } finally { + client.disconnect() + scope.cancel() + hub.close() + } + } + + /** + * One second the relay will not reconcile at any window size costs that + * second, not the sync. + * + * The whole point of the [NegentropyOrFetchResult.pagedWindows] path: the + * dense second is drained over REQ while everything around it still + * reconciles. Before, the exception from that one window abandoned the whole + * sync and re-paged the entire filter — on a large corpus, exactly the cost + * negentropy was there to avoid. + */ + @Test + fun oneUnreconcilableSecondDoesNotCostTheRestOfTheFilter() = + runBlocking { + val hub = InProcessRelays(negentropySettings = NegentropySettings(maxSyncEvents = 3)) + val scope = CoroutineScope(Dispatchers.Default + SupervisorJob()) + val client = NostrClient(hub, scope) + try { + val url = RelayUrlNormalizer.normalize("ws://127.0.0.1:7787/") + // Ten events crammed into one second — no created_at window can + // separate them — plus five ordinary seconds around them. + val dense = (1..10).map { SyntheticEvents.fakeEvent(idSeed = it, kind = 1, createdAt = 1000L) } + val sparse = (0 until 5).map { SyntheticEvents.fakeEvent(idSeed = 100 + it, kind = 1, createdAt = 2000L + it) } + hub.getOrCreate(url).preload(dense + sparse) + + val got = mutableListOf() + val result = + withTimeout(60_000) { + client.negentropySyncOrFetch( + relay = url, + filter = Filter(kinds = listOf(1)), + ) { got.add(it) } + } + + assertEquals(15, got.map { it.id }.toSet().size, "everything is delivered, by whichever route") + assertEquals(1, result.pagedWindows, "only the dense second is paged") + assertNull(result.fallbackCause, "the sync itself was never abandoned") + val negentropy = assertNotNull(result.negentropy, "the rest of the range still reconciled") + assertTrue(negentropy.windows > 1) + } finally { + client.disconnect() + scope.cancel() + hub.close() + } + } } From 8555309492a6fc54039cc06d7d08fa3077cd09bb Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 6 Aug 2026 20:12:36 +0000 Subject: [PATCH 056/132] negentropy: audit fixes over the windowing change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A read-back over the two commits before this, rather than a failure — which is the only way these would have turned up, since every one of them lives on a path that runs when something has already gone wrong. **accept() is no longer single-threaded, and its comment said it was.** "Both phases run sequentially, so no concurrent access" was true right up until a paged window started running on a reconciler coroutine while the sync's own delivery consumer was still calling accept(). An unguarded HashSet between two coroutines can corrupt, and the delivered counter can lose updates. Now behind a Mutex — with onEvent kept INSIDE it, because callers are promised it never runs concurrently with itself and some of them keep unsynchronised state in that callback. pagedWindows becomes an AtomicInt for the same reason. **The kotlinx cap parse could take down the whole frame.** `.jsonPrimitive` throws on an object or array, so a relay putting something structured in the fourth element would have failed the NEG-ERR and lost the reason with it — where before that element existed, anything extra was simply ignored. `as?` restores that. Both mappers are now tested against a structured fourth element as well as a string one. **Int overflow in the split fan-out.** `mine + ceiling - 1` wraps when a window holds close to Int.MAX events, which is reachable on exactly the corpora this targets; done in Long now. **The count-driven split cuts N ways, not two.** The work queue is FIFO, so halving means every internal node's count() runs before the first NEG-OPEN goes out: on a corpus ~30,000 windows wide that is ~30,000 store counts of dead time with nothing downloading. Cutting into ceil(count/budget) pieces (capped at 32) reaches the same corpus in about three levels instead of fifteen, and pieces that guess wrong are re-split by the same rule. **The budget moves by CAS.** With reconcileConcurrency > 1 two reconcilers adjust it at once, and a lost SHRINK is the one that costs something real: the next window is then asked at a size the relay has already refused. --- .../kotlinSerialization/MessageKSerializer.kt | 6 +- .../client/accessories/NegentropyStoreSync.kt | 47 +++++- .../NostrClientNegentropySyncExt.kt | 143 +++++++++++++----- .../nip77Negentropy/Nip77SerializationTest.kt | 18 +++ 4 files changed, 173 insertions(+), 41 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/kotlinSerialization/MessageKSerializer.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/kotlinSerialization/MessageKSerializer.kt index 93c0c360cb..c0a7972e10 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/kotlinSerialization/MessageKSerializer.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/kotlinSerialization/MessageKSerializer.kt @@ -191,8 +191,10 @@ object MessageKSerializer : KSerializer { reason = if (array.size > 2) array[2].jsonPrimitive.content else "", // Optional, and only a number: a relay that puts something // else there is telling us nothing rather than breaking the - // frame. - cap = if (array.size > 3) array[3].jsonPrimitive.longOrNull else null, + // frame. `as?` rather than `.jsonPrimitive`, which THROWS on + // an object or array — that would fail the whole message and + // lose the reason, where before this element was ignored. + cap = if (array.size > 3) (array[3] as? JsonPrimitive)?.longOrNull else null, ) } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropyStoreSync.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropyStoreSync.kt index a0272abecc..02bb576509 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropyStoreSync.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropyStoreSync.kt @@ -26,6 +26,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.store.IEventStore +import com.vitorpamplona.quartz.nip01Core.store.IdAndTime import com.vitorpamplona.quartz.nip01Core.store.verifyAndInsert import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll @@ -92,6 +93,15 @@ class NegentropyStoreSync( * @param concurrency relays synced at once by [sync] (a relay's own filters stay sequential). * @param idleTimeoutMs idle watchdog for reconciles / fetches / pages. * @param publishTimeoutSecs OK-confirmation wait per uploaded event. + * @param targetWindow events per reconcile window, or `0` to snapshot the + * whole filter up front (the default, and what this class always did). + * + * Above zero, the store is read one `created_at` window at a time through + * a [NegentropyLocalIndex] instead: the id snapshot stops being O(matched + * set) — it is the largest thing this class holds — at the price of an + * indexed count + range read per window. Worth turning on exactly when the + * filter matches more than fits comfortably in memory; pointless below + * that, where one snapshot shared by the whole group is cheaper. */ class Config( val down: Boolean = true, @@ -105,6 +115,7 @@ class NegentropyStoreSync( val concurrency: Int = 4, val idleTimeoutMs: Long = 30_000L, val publishTimeoutSecs: Long = 15, + val targetWindow: Int = 0, ) /** Outcome of one `(relay, filter)` group. `error` is null on success. */ @@ -166,7 +177,14 @@ class NegentropyStoreSync( // events (~40 B/entry vs ~1 KB), which matters when a relay hosts a large // matched set. The events the reconcile decides to UP-publish (the small // residual haves) are fetched by id on demand in the uploader below. - val localEntries = store.snapshotIdsForNegentropy(listOf(filter)) + // + // With a targetWindow, even those 40 B/entry are read per window rather + // than for the whole filter — on a large store that snapshot is the + // biggest thing this class allocates, and it is allocated before the + // first frame goes out. + val windowed = config.targetWindow > 0 + val localIndex = if (windowed) StoreWindowIndex(store) else null + val localEntries = if (windowed) emptyList() else store.snapshotIdsForNegentropy(listOf(filter)) val downloaded = AtomicInt(0) val uploaded = AtomicInt(0) @@ -210,6 +228,8 @@ class NegentropyStoreSync( relay = relay, filter = filter, localEntries = localEntries, + localIndex = localIndex, + targetWindow = config.targetWindow, batchSize = config.idChunk, idleTimeoutMs = config.idleTimeoutMs, reconcileConcurrency = config.reconcileConcurrency, @@ -311,3 +331,28 @@ class NegentropyStoreSync( return stored.load() } } + +/** + * [NegentropyLocalIndex] over an [IEventStore]: the window engine's per-window + * reads answered straight from the store's `created_at` index. + * + * The windows handed here are the caller's own filter with `since`/`until` + * narrowed, so they can go to the store as-is. A count the store cannot answer + * comes back null rather than throwing — the engine then simply stops + * pre-splitting that window and lets the relay's refusal decide, which is the + * behaviour without an index at all. + */ +private class StoreWindowIndex( + private val store: IEventStore, +) : NegentropyLocalIndex { + override suspend fun count(window: Filter): Int? = + try { + store.count(window) + } catch (e: CancellationException) { + throw e + } catch (_: Exception) { + null + } + + override suspend fun entriesFor(window: Filter): List = store.snapshotIdsForNegentropy(listOf(window)) +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropySyncExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropySyncExt.kt index 6595cc763e..51941e1856 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropySyncExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropySyncExt.kt @@ -329,6 +329,7 @@ class NegentropyOrFetchResult( * Use [negentropySync] directly if you want to decide the fallback yourself (try * another relay, narrow the filter, abort, …) instead of always paging. */ +@OptIn(ExperimentalAtomicApi::class) suspend fun INostrClient.negentropySyncOrFetch( relay: NormalizedRelayUrl, filter: Filter, @@ -346,18 +347,29 @@ suspend fun INostrClient.negentropySyncOrFetch( ): NegentropyOrFetchResult { val seen = HashSet() var delivered = 0 - var pagedWindows = 0 + val pagedWindows = AtomicInt(0) - // Shared dedup + cap across both phases. Returns true if the event was new and - // delivered. Both phases run sequentially, so no concurrent access. - suspend fun accept(event: Event): Boolean { - if ((maxEvents <= 0 || delivered < maxEvents) && seen.add(event.id)) { - delivered++ - onEvent(event) - return true + // Shared dedup + cap across every path that delivers. + // + // The lock is not optional. The two phases used to run strictly one after + // the other, but a paged window now runs DURING the negentropy phase, on a + // reconciler coroutine, while the sync's own delivery consumer is calling + // this too — an unguarded HashSet between them can corrupt, and the count + // can lose updates. onEvent stays INSIDE the lock deliberately: callers are + // promised it never runs concurrently with itself, and some of them keep + // unsynchronised state in it. + val gate = Mutex() + + suspend fun accept(event: Event): Boolean = + gate.withLock { + if ((maxEvents <= 0 || delivered < maxEvents) && seen.add(event.id)) { + delivered++ + onEvent(event) + true + } else { + false + } } - return false - } return try { val result = @@ -379,7 +391,7 @@ suspend fun INostrClient.negentropySyncOrFetch( // already reconciled cleanly walked again over REQ, which on a // large corpus is the entire cost negentropy was there to save. onUnreconcilableWindow = { window -> - pagedWindows++ + pagedWindows.incrementAndFetch() val pageTimeoutMs = if (idleTimeoutMs > 0) idleTimeoutMs else DEFAULT_DOWNLOAD_IDLE_MS fetchAllPages(relay, listOf(window), pageTimeoutMs) { event -> if (accept(event)) onProgress?.invoke(delivered, delivered) @@ -392,10 +404,10 @@ suspend fun INostrClient.negentropySyncOrFetch( // Any paged window makes this not a clean reconcile — see the // property doc: under-reporting it would let a caller record // coverage it never compared. - pagedFallback = pagedWindows > 0, + pagedFallback = pagedWindows.load() > 0, negentropy = result, fallbackCause = null, - pagedWindows = pagedWindows, + pagedWindows = pagedWindows.load(), ) } catch (e: NegentropySyncException) { // Negentropy couldn't enumerate the set — page the whole filter instead, @@ -411,7 +423,7 @@ suspend fun INostrClient.negentropySyncOrFetch( pagedFallback = true, negentropy = null, fallbackCause = e, - pagedWindows = pagedWindows, + pagedWindows = pagedWindows.load(), ) } } @@ -567,7 +579,9 @@ internal suspend fun reconcileWindows( onPeerCap: ((Long) -> Unit)? = null, // Given a minimal window the relay will not reconcile at any size, instead // of throwing. The caller drains it however it can (paging it over REQ) and - // the sweep carries on with the rest of the filter. + // the sweep carries on with the rest of the filter. It runs ON the reconciler + // that hit the window, so a slow drain holds that reconciler — with + // reconcileConcurrency = 1 the rest of the sweep waits for it. onUnreconcilableWindow: (suspend (Filter) -> Unit)? = null, sendNeedBatch: suspend (List) -> Unit, sendHaveBatch: (suspend (List) -> Unit)?, @@ -597,25 +611,57 @@ internal suspend fun reconcileWindows( // targetWindow at 0 nothing below this line does anything. val budget = AtomicInt(targetWindow) - // Splits a window in two and queues both halves. Returns false when the - // window is already minimal — `created_at` is in seconds, so that is the - // floor, not a tuning choice. - fun splitInto( + // Every budget move goes through here. With reconcileConcurrency > 1 two + // reconcilers adjust it at once, and read-then-store can drop one of them — + // a lost SHRINK being the one that costs something real, since the next + // window is then asked at a size the relay has already refused. + fun budgetTo(next: (Int) -> Int) { + while (true) { + val now = budget.load() + val want = next(now) + if (want == now || budget.compareAndSet(now, want)) return + } + } + + /** + * Cuts `[lo, hi]` into [pieces] equal spans of time and queues them all. A + * window already at the floor is left alone — `created_at` is in seconds, so + * that is where splitting ends, not a tuning choice. Both callers check that + * themselves; the guard here is so a third one cannot silently lose a window. + * + * [pieces] > 2 exists for the count-driven split, where we know HOW FAR over + * the budget a window is and can land near the right size in one step. + * Halving instead costs a store count per level of a tree that can be ~15 + * deep on a large corpus, and — since the queue is FIFO — every one of those + * counts happens before the first window is reconciled at all. + */ + suspend fun splitInto( pendingWindow: Filter, lo: Long, hi: Long, - ): Boolean { - if (hi - lo <= MIN_WINDOW_SECONDS) return false - val mid = lo + (hi - lo) / 2 - remaining.incrementAndFetch() - // The lower child gets the finite midpoint; the upper child KEEPS this - // window's original `until` (which may be null = unbounded). Replacing - // null with `now()` here would drop every event dated after now() - // (clock skew) once any split happens, while the un-split path would - // have included them. - pending.trySend(pendingWindow.copy(since = lo, until = mid)) - pending.trySend(pendingWindow.copy(since = mid + 1, until = pendingWindow.until)) - return true + pieces: Int = 2, + ) { + if (hi - lo <= MIN_WINDOW_SECONDS) return + val span = hi - lo + 1 + // Never more pieces than there are seconds to give them. + val n = pieces.toLong().coerceIn(2L, minOf(span, MAX_SPLIT_FANOUT.toLong())).toInt() + val step = span / n + remaining.addAndFetch(n - 1) + var start = lo + repeat(n) { i -> + val last = i == n - 1 + // The top piece KEEPS this window's original `until` (which may be + // null = unbounded). Replacing null with `now()` here would drop + // every event dated after now() (clock skew) once any split happens, + // while the un-split path would have included them. + if (last) { + pending.send(pendingWindow.copy(since = start, until = pendingWindow.until)) + } else { + val end = start + step - 1 + pending.send(pendingWindow.copy(since = start, until = end)) + start = end + 1 + } + } } val reconcilers = @@ -636,7 +682,14 @@ internal suspend fun reconcileWindows( if (ceiling > 0 && hi - lo > MIN_WINDOW_SECONDS) { val mine = local.count(window) if (mine != null && mine > ceiling) { - splitInto(window, lo, hi) + // How many windows this one is worth, not just "two": + // the count says how far over budget we are, and + // uneven density is corrected by the same check on + // each piece. + // Long arithmetic: `mine` can be near Int.MAX on a + // corpus this size, and the +ceiling would wrap. + val over = (mine.toLong() + ceiling - 1) / ceiling + splitInto(window, lo, hi, pieces = over.coerceAtMost(MAX_SPLIT_FANOUT.toLong()).toInt()) continue } } @@ -662,9 +715,12 @@ internal suspend fun reconcileWindows( // asked for, so a sync that met one dense stretch // does not stay small for the rest of the timeline. if (targetWindow > 0) { - val now = budget.load() - if (now < targetWindow) { - budget.store(minOf(targetWindow, (now * BUDGET_GROWTH).toInt().coerceAtLeast(now + 1))) + budgetTo { now -> + if (now >= targetWindow) { + now + } else { + minOf(targetWindow, (now * BUDGET_GROWTH).toInt().coerceAtLeast(now + 1)) + } } } if (remaining.decrementAndFetch() == 0) pending.close() @@ -677,13 +733,16 @@ internal suspend fun reconcileWindows( outcome.cap?.let { cap -> onPeerCap?.invoke(cap) if (targetWindow > 0) { - val fitted = (cap * CAP_MARGIN).toInt().coerceAtLeast(1) - if (fitted < budget.load()) budget.store(fitted) + val fitted = + (cap * CAP_MARGIN) + .coerceIn(1.0, Int.MAX_VALUE.toDouble()) + .toInt() + budgetTo { now -> minOf(now, fitted) } } } if (outcome.cap == null && targetWindow > 0) { // No number to go on: halve and find out. - budget.store((budget.load() / 2).coerceAtLeast(1)) + budgetTo { now -> (now / 2).coerceAtLeast(1) } } if (hi - lo <= MIN_WINDOW_SECONDS) { // A minimal window that still overflows: @@ -1295,6 +1354,14 @@ private const val MIN_WINDOW_SECONDS = 1L */ private const val MAX_WINDOWS = 100_000 +/** + * Most pieces one count-driven split may cut a window into. Bounds both the + * queue and the depth: with 32, a corpus 30,000 windows wide is reached in + * three levels instead of fifteen, and the pieces that guessed wrong are + * re-split by the same rule. + */ +private const val MAX_SPLIT_FANOUT = 32 + /** * How much of a relay's stated `max_sync_events` a window actually aims for. * The margin absorbs what the relay gains between stating that number and diff --git a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip77Negentropy/Nip77SerializationTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip77Negentropy/Nip77SerializationTest.kt index 7a71cc9a1b..d8d0bcc50b 100644 --- a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip77Negentropy/Nip77SerializationTest.kt +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip77Negentropy/Nip77SerializationTest.kt @@ -175,6 +175,24 @@ class Nip77SerializationTest { assertEquals(null, kotlin.cap) } + @Test + fun deserializeNegErrMessageWithStructuredFourthElement_bothMappers() { + // A fourth element that is an object or array must degrade to no cap, + // NOT fail the frame — the reason is the part that matters, and before + // this element existed any extra was simply ignored. + val json = """["NEG-ERR","neg-sub1","blocked: too many query results",{"max":10}]""" + + val jackson = JacksonMapper.fromJsonToMessage(json) + assertTrue(jackson is NegErrMessage) + assertEquals("blocked: too many query results", jackson.reason) + assertEquals(null, jackson.cap) + + val kotlin = KotlinSerializationMapper.fromJsonToMessage(json) + assertTrue(kotlin is NegErrMessage) + assertEquals("blocked: too many query results", kotlin.reason) + assertEquals(null, kotlin.cap) + } + @Test fun negErrMessageWithCap_crossDeserialization() { val msg = NegErrMessage("neg-sub1", "blocked: too many records", 500_000L) From 36a79d74dee753bd153e46cd86a5aa34ca4802c7 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Thu, 6 Aug 2026 17:09:34 -0400 Subject: [PATCH 057/132] Stop the outbox getting slower with every publish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PoolEventOutbox kept its pending publishes in an immutable map and rebuilt it on every send: eventOutbox = eventOutbox + Pair(event.id, PoolEventOutboxState(...)) That copies every entry, per event, so publishing N events copies 1 + 2 + … + N. The relay-set bookkeeping alongside it was the same shape — needsToUpdateRelays() and updateRelays() each walk every value, and both ran on every send. Measured on a bulk push against a relay with ~970k entries resident: 22.7ms per event, of which ~20.5ms was the outbox. The store fetch feeding the same loop cost 1.2ms and the configured pace 1ms, so the map was ~90% of the budget — and the rate decayed as the backlog grew, 45.6 -> 44.6 -> 43.2 ev/s across three windows. The map is now LargeCache (ConcurrentHashMap on JVM/Android), so put/get/ remove are O(1) and the cross-thread visibility that @Volatile republishing provided comes from the map itself. The relay set is now maintained asymmetrically, because the two directions are not equally expensive. Adding is exact and cheap: union the event's own relays, touching the flow only when it actually changes. Deciding a relay may LEAVE means asking whether any remaining entry still wants it, which is inherently O(outbox) — so it is swept every SWEEP_EVERY removals, and always when the outbox empties. Keeping a relay a little too long costs an idle connection; scanning a million entries to retire it promptly costs the push. The test asserts the SHAPE of the cost, not a wall-clock budget: equal windows at the start and end of a 60k-publish run, where the late window carries ~29x the backlog. Halves were not enough — over 20k publishes the average backlog only grows 7k to 17k, a 2.4x expected ratio that hid inside JIT noise, and the first version of this test passed against the very code it was written to catch. Co-Authored-By: Claude Opus 5 (1M context) --- .../relay/client/pool/PoolEventOutbox.kt | 109 +++++++++++----- .../client/pool/PoolEventOutboxScaleTest.kt | 121 ++++++++++++++++++ 2 files changed, 201 insertions(+), 29 deletions(-) create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolEventOutboxScaleTest.kt diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolEventOutbox.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolEventOutbox.kt index 928cfd0bc2..a563229115 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolEventOutbox.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolEventOutbox.kt @@ -27,32 +27,61 @@ import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.OkMessage import com.vitorpamplona.quartz.nip01Core.relay.commands.toRelay.Command import com.vitorpamplona.quartz.nip01Core.relay.commands.toRelay.EventCmd import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.utils.cache.LargeCache import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.update import kotlin.concurrent.Volatile class PoolEventOutbox { - // @Volatile so the polling path (INostrClient.pendingPublishRelaysFor) - // sees current state from threads that didn't write the map. Mutations - // still happen on NostrClient's IO scope; this only closes the - // visibility gap for cross-thread readers. - @Volatile - private var eventOutbox = mapOf() + /** + * Pending publishes, keyed by event id. + * + * A concurrent map, NOT a copy-on-write immutable one. It used to be + * `@Volatile var eventOutbox = mapOf(...)` reassigned with + * `eventOutbox + Pair(...)`, which copies EVERY entry on EVERY publish — + * so publishing N events cost O(N^2). Measured on a bulk push with ~970k + * entries resident: 22.7ms per event, of which ~20.5ms was this map, and + * the rate decayed as the outbox grew (45.6 -> 44.6 -> 43.2 ev/s across + * three windows). The store fetch behind the same loop cost 1.2ms. + * + * [LargeCache] is ConcurrentHashMap on JVM/Android, so put/get/remove are + * O(1) and cross-thread visibility no longer needs the volatile republish. + */ + private val eventOutbox = LargeCache() val relays = MutableStateFlow(setOf()) + /** + * Removals since the relay set was last rebuilt. + * + * Deciding whether a relay may leave [relays] means asking whether ANY + * remaining entry still wants it — O(outbox), and doing that per publish + * is the second half of the quadratic. Additions stay exact and cheap (a + * union of the event's own relays); removals are swept in batches, because + * keeping a relay in the set slightly too long only means holding a + * connection a little longer, while scanning a million entries to retire + * it promptly costs the whole push. + */ + @Volatile + private var pendingSweep = 0 + + companion object { + /** Removals between full relay-set rebuilds — see [pendingSweep]. */ + private const val SWEEP_EVERY = 256 + } + fun needsToUpdateRelays(): Boolean { val currentRelays = relays.value var relaysToRemoveCounter = 0 currentRelays.forEach { currentRelay -> - if (eventOutbox.values.none { currentRelay in it.relaysRemaining }) { + if (eventOutbox.values().none { currentRelay in it.relaysRemaining }) { relaysToRemoveCounter++ } } var relaysToAddCounter = 0 - eventOutbox.values.forEach { outboxState -> + eventOutbox.values().forEach { outboxState -> if (outboxState.relaysRemaining.any { it !in currentRelays }) { relaysToAddCounter++ } @@ -67,13 +96,13 @@ class PoolEventOutbox { val relaysToRemove = mutableSetOf() currentRelays.forEach { currentRelay -> - if (eventOutbox.values.none { currentRelay in it.relaysRemaining }) { + if (eventOutbox.values().none { currentRelay in it.relaysRemaining }) { relaysToRemove.add(currentRelay) } } val relaysToAdd = mutableSetOf() - eventOutbox.values.forEach { outboxState -> + eventOutbox.values().forEach { outboxState -> outboxState.relaysRemaining.forEach { relay -> if (relay !in relaysToAdd && relay !in currentRelays) { relaysToAdd.add(relay) @@ -88,7 +117,7 @@ class PoolEventOutbox { fun activeOutboxCacheFor(url: NormalizedRelayUrl): Set { val myEvents = mutableSetOf() - eventOutbox.forEach { (eventId, outboxCache) -> + eventOutbox.forEach { eventId, outboxCache -> if (url in outboxCache.relaysRemaining) { myEvents.add(eventId) } @@ -103,7 +132,7 @@ class PoolEventOutbox { */ fun activeOutboxEventsFor(url: NormalizedRelayUrl): List { val myEvents = mutableListOf() - eventOutbox.forEach { (_, outboxCache) -> + eventOutbox.forEach { _, outboxCache -> if (url in outboxCache.relaysRemaining) { myEvents.add(outboxCache.event) } @@ -117,20 +146,41 @@ class PoolEventOutbox { * Callers can poll this after publish to detect when relays ack: the set shrinks * as OKs arrive, then the entry is removed from the outbox (returns null). */ - fun pendingRelaysFor(eventId: HexKey): Set? = eventOutbox[eventId]?.relaysLeft() + fun pendingRelaysFor(eventId: HexKey): Set? = eventOutbox.get(eventId)?.relaysLeft() fun markAsSending( event: Event, relays: Set, ): Set { - val currentOutbox = eventOutbox[event.id] + val currentOutbox = eventOutbox.get(event.id) if (currentOutbox == null) { - eventOutbox = eventOutbox + Pair(event.id, PoolEventOutboxState(event, relays)) + eventOutbox.put(event.id, PoolEventOutboxState(event, relays)) } else { currentOutbox.updateRelays(relays) } - updateRelays() - return eventOutbox[event.id]?.remainingRelays() ?: emptySet() + // Additions only, and only what is genuinely new: the union is over + // this event's relays, never over the whole outbox. + addRelays(relays) + return eventOutbox.get(event.id)?.remainingRelays() ?: emptySet() + } + + /** Union [wanted] into [relays], touching the flow only when it actually changes. */ + private fun addRelays(wanted: Set) { + val missing = wanted - relays.value + if (missing.isNotEmpty()) relays.update { it + missing } + } + + /** + * An entry left the outbox. Retiring its relays needs a full scan, so that + * is amortised across [SWEEP_EVERY] removals — and always run once the + * outbox empties, which is the case that must not linger. + */ + private fun onRemoved() { + pendingSweep++ + if (pendingSweep >= SWEEP_EVERY || eventOutbox.isEmpty()) { + pendingSweep = 0 + updateRelays() + } } /** Records a send attempt. Returns the event if this attempt exhausted its retry budget for @@ -139,11 +189,11 @@ class PoolEventOutbox { id: HexKey, url: NormalizedRelayUrl, ): Event? { - val waiting = eventOutbox[id] ?: return null + val waiting = eventOutbox.get(id) ?: return null val gaveUp = waiting.newTry(url) if (waiting.isDone()) { - eventOutbox = eventOutbox - waiting.event.id - updateRelays() + eventOutbox.remove(waiting.event.id) + onRemoved() } return if (gaveUp) waiting.event else null } @@ -154,12 +204,12 @@ class PoolEventOutbox { success: Boolean, message: String, ) { - val waiting = eventOutbox[id] + val waiting = eventOutbox.get(id) if (waiting != null) { waiting.newResponse(url, success, message) if (waiting.isDone()) { - eventOutbox = eventOutbox - waiting.event.id - updateRelays() + eventOutbox.remove(waiting.event.id) + onRemoved() } } } @@ -171,8 +221,8 @@ class PoolEventOutbox { relay: NormalizedRelayUrl, sync: (Command) -> Unit, ) { - eventOutbox.forEach { - it.value.forEachUnsentEvent(relay) { + eventOutbox.forEach { _, outboxCache -> + outboxCache.forEachUnsentEvent(relay) { sync(EventCmd(it)) } } @@ -210,15 +260,16 @@ class PoolEventOutbox { relay: NormalizedRelayUrl, errorMessage: String, ) { - eventOutbox.forEach { - if (relay in it.value.relaysRemaining) { - newResponse(it.key, relay, false, errorMessage) + eventOutbox.forEach { id, outboxCache -> + if (relay in outboxCache.relaysRemaining) { + newResponse(id, relay, false, errorMessage) } } } fun destroy() { - eventOutbox = emptyMap() + eventOutbox.clear() + pendingSweep = 0 relays.tryEmit(emptySet()) } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolEventOutboxScaleTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolEventOutboxScaleTest.kt new file mode 100644 index 0000000000..03058c51ff --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/pool/PoolEventOutboxScaleTest.kt @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay.client.pool + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue +import kotlin.time.TimeSource + +/** + * The outbox must not get slower as it fills. + * + * It used to: the map was immutable and every `markAsSending` rebuilt it with + * `eventOutbox + Pair(...)`, so publishing N events copied 1 + 2 + … + N + * entries. Measured on a real bulk push at ~970k entries resident, that was + * ~20.5ms of the 22.7ms each event cost, and the rate visibly decayed as the + * backlog grew (45.6 -> 44.6 -> 43.2 ev/s over three windows). The relay-set + * bookkeeping was the other half — two full scans of every entry, per publish. + * + * This asserts the SHAPE of the cost rather than a wall-clock budget: a + * quadratic makes the second half of a run dramatically slower than the first, + * whatever the machine. A constant factor cannot be pinned in a unit test, but + * a growth curve can. + */ +class PoolEventOutboxScaleTest { + private val relay = NormalizedRelayUrl("wss://scale.relay.test") + + private fun event(i: Int) = + Event( + id = i.toString(16).padStart(64, '0'), + pubKey = "00".repeat(32), + createdAt = 1_700_000_000L, + kind = 1, + tags = emptyArray(), + content = "hello", + sig = "00".repeat(64), + ) + + @Test + fun `publishing stays flat as the outbox fills`() { + val outbox = PoolEventOutbox() + val relays = setOf(relay) + val clock = TimeSource.Monotonic + val sample = 2_000 + val total = 60_000 + + fun publishRange( + from: Int, + until: Int, + ) { + for (i in from until until) outbox.markAsSending(event(i), relays) + } + + // Equal-sized windows at the START and the END of a long run. Halves + // would not do: over 20k publishes the average backlog only grows from + // ~7k to ~17k, a 2.4x expected ratio that hides inside JIT noise. Here + // the late window carries ~29x the backlog of the early one, so a + // per-entry cost shows up as a per-entry cost. + repeat(sample) { outbox.markAsSending(event(it), relays) } // warm up + val early = + clock.markNow().let { start -> + publishRange(sample, sample * 2) + start.elapsedNow() + } + publishRange(sample * 2, total - sample) + val late = + clock.markNow().let { start -> + publishRange(total - sample, total) + start.elapsedNow() + } + + assertEquals(total, outbox.activeOutboxCacheFor(relay).size, "every publish is tracked") + + val ratio = late.inWholeMicroseconds.toDouble() / early.inWholeMicroseconds.coerceAtLeast(1) + assertTrue( + ratio < 5.0, + "cost per publish must not grow with the backlog: first $sample took ${early.inWholeMilliseconds}ms at " + + "~$sample entries, last $sample took ${late.inWholeMilliseconds}ms at ~$total entries (ratio $ratio)", + ) + } + + @Test + fun `the relay set still reflects what is pending`() { + val outbox = PoolEventOutbox() + val a = NormalizedRelayUrl("wss://a.relay.test") + val b = NormalizedRelayUrl("wss://b.relay.test") + + outbox.markAsSending(event(1), setOf(a)) + assertEquals(setOf(a), outbox.relays.value, "a publish adds its relay immediately") + + outbox.markAsSending(event(2), setOf(b)) + assertEquals(setOf(a, b), outbox.relays.value, "a second relay joins without a rebuild") + + // Draining every entry must clear the set — the sweep is batched, but + // emptying the outbox forces it, so a finished push does not strand a + // connection open forever. + outbox.newResponse(event(1).id, a, true, "") + outbox.newResponse(event(2).id, b, true, "") + assertEquals(emptySet(), outbox.relays.value, "an empty outbox wants no relays") + } +} From d7226b70213a7e13beecacebe43c0acfff686af3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 6 Aug 2026 19:06:25 -0500 Subject: [PATCH 058/132] =?UTF-8?q?concord:=20implement=20CORD-02=20=C2=A7?= =?UTF-8?q?2=20staff-held=20control=5Froot=20write=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Track the spec change in concord2 96f0647 (CORD-01 Write-Restricted Streams) and its review follow-up bbc67b6: the Control Plane's stream key splits, the signer keypair deriving from a new control_root held only by the owner and staff (concord/control-signer), while every member holds the delivered control_pk to subscribe and verify, reading under the community_root-derived read key the old concord/control derivation still yields. - ControlPlaneKeys models the three views of an epoch: staff (signer held), member (address held, read-only), legacy (pre-split, one key). ConcordStreamEnvelope gains write-restricted wrap/open forms; wrapping without the write key fails loudly instead of missigning. - Genesis mints the control_root beside the community_root; invites, the kind-13302 join material, and held roots carry control_pk (and, staff-side, control_root) since a split address is held, never derivable. A same-epoch list merge fills either side's missing key material, so a holder's own second device converges (CORD-02 §8); across epochs it is never inherited, being stale by construction. - Promotion delivers the secret inside the staff-making Grant itself: GrantEntity.control_wrap, a 40-byte epoch_be8‖control_root pairwise ciphertext (ControlRootWrap), adopted only when it derives to the held control_pk — fails closed. PIN_MESSAGES claims frozen bit 11 and the staff set is the six Control-writing bits, a normative list. - Refoundings roll the pair: base rekey blobs are now width-per-form (72 channel/legacy, 104 member +control_pk, 136 staff +control_root), a mismatched staff pair is refused, and a legacy 72-byte base blob is honored when reading old rotations, never minted anew — so a legacy community upgrades as a side effect of its next base rotation. - Sessions, the plane registry, the subscription planner, amy, and the app read the plane by held address per epoch; moderation verbs take ControlPlaneKeys, and both amy and the app refuse a Control write without the secret rather than throwing out of the envelope. Rank and possession diverge for as long as a promotee waits on delivery, so the app gates its mod affordances on the write key too. Stored control material only ever backstops its own epoch. The account drains staff-making Grants on the revision tick. Possession stays a spam gate, never authority: every edition is still judged by its sealed actor's rank in the owner-rooted Roster. --- .../vitorpamplona/amethyst/model/Account.kt | 3 + .../amethyst/model/AccountConcordActions.kt | 206 +++++++++++++-- .../concord/ConcordChannelListScreen.kt | 10 +- .../cli/commands/ConcordChannelCommands.kt | 12 +- .../amethyst/cli/commands/ConcordCommands.kt | 44 +++- .../cli/commands/ConcordModCommands.kt | 50 +++- .../amethyst/cli/stores/ConcordStore.kt | 8 + .../commons/actions/ConcordActions.kt | 93 ++++++- .../commons/actions/ConcordModeration.kt | 78 +++++- .../actions/ConcordSubscriptionPlanner.kt | 11 +- .../model/concord/ConcordCommunitySession.kt | 44 +++- .../model/concord/ConcordPlaneRegistry.kt | 40 ++- .../commons/actions/ConcordActionsTest.kt | 32 ++- .../commons/actions/ConcordModerationTest.kt | 82 ++++++ .../actions/ConcordSubscriptionPlannerTest.kt | 20 +- .../concord/ConcordCommunitySessionTest.kt | 6 +- .../model/concord/ConcordPlaneRegistryTest.kt | 2 + .../model/concord/ConcordRollbackFloorTest.kt | 33 ++- .../concord/ConcordSessionManagerTest.kt | 22 +- .../concord/ConcordSessionRegistryTest.kt | 6 +- .../ConcordCommunityFactory.kt | 25 +- .../cord02Community/ConcordCommunityList.kt | 121 ++++++++- .../concord/cord04Roles/AuthorityResolver.kt | 14 ++ .../concord/cord04Roles/ConcordPermissions.kt | 19 +- .../concord/cord04Roles/ControlEntities.kt | 9 + .../concord/cord04Roles/ControlRootWrap.kt | 115 +++++++++ .../concord/cord05Invites/CommunityInvite.kt | 13 + .../cord05Invites/ConcordStrandedRecovery.kt | 8 +- .../concord/cord06Rekey/ConcordRefounding.kt | 139 ++++++++--- .../concord/cord06Rekey/ConcordRekey.kt | 46 +++- .../quartz/concord/cord06Rekey/RekeyBlob.kt | 62 ++++- .../concord/crypto/ConcordKeyDerivation.kt | 22 +- .../quartz/concord/crypto/ConcordLabels.kt | 12 +- .../quartz/concord/crypto/ControlPlaneKeys.kt | 127 ++++++++++ .../concord/envelope/ConcordStreamEnvelope.kt | 99 +++++++- .../ConcordCommunityFactoryTest.kt | 15 +- .../ConcordCommunityListTest.kt | 53 +++- .../cord02Community/ControlPlaneSplitTest.kt | 194 ++++++++++++++ .../cord04Roles/AuthorityResolverTest.kt | 33 +++ .../cord04Roles/ControlRootWrapTest.kt | 164 ++++++++++++ .../ConcordInviteJoinFlowTest.kt | 14 +- .../cord06Rekey/ConcordRefoundingTest.kt | 31 ++- .../cord06Rekey/ControlRootRotationTest.kt | 236 ++++++++++++++++++ 43 files changed, 2158 insertions(+), 215 deletions(-) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlRootWrap.kt create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/crypto/ControlPlaneKeys.kt create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ControlPlaneSplitTest.kt create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlRootWrapTest.kt create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ControlRootRotationTest.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt index 966c01cedf..57d6f64e5e 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt @@ -3563,6 +3563,9 @@ class Account( refreshConcordChannelIndex() // A revision also bumps when a base-rotation rekey lands; adopt ours if present. runCatching { concord.drainConcordRekeys() }.onFailure { Log.w("Concord", "rekey drain failed", it) } + // A promotion to staff delivers the Control Plane write key inside the Grant + // itself (CORD-04 §3), so the fold that seats the role is also when it arrives. + runCatching { concord.drainConcordStaffGrants() }.onFailure { Log.w("Concord", "staff grant drain failed", it) } // A rotation we were *excluded* from produces no rekey to drain, so it can only be // found by re-resolving the invite link we joined through. Rate-limited internally. runCatching { concord.recoverStrandedConcordCommunities() }.onFailure { Log.w("Concord", "stranded recovery failed", it) } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt index be17eefef0..fba41f5e01 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt @@ -24,20 +24,28 @@ import com.vitorpamplona.amethyst.commons.actions.ConcordActions import com.vitorpamplona.amethyst.commons.actions.ConcordModeration import com.vitorpamplona.amethyst.commons.actions.ConcordSubscriptionPlanner import com.vitorpamplona.amethyst.commons.model.concord.ConcordChannel +import com.vitorpamplona.amethyst.commons.model.concord.ConcordCommunitySession import com.vitorpamplona.amethyst.commons.viewmodels.ReplyMode import com.vitorpamplona.amethyst.ui.screen.loggedIn.chats.publicChannels.concord.concordChannelLastReadRoute +import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityList.withControlRoot import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEntry import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEvent import com.vitorpamplona.quartz.concord.cord02Community.HeldRoot import com.vitorpamplona.quartz.concord.cord02Community.ImagePointer import com.vitorpamplona.quartz.concord.cord03Channels.ChannelChat import com.vitorpamplona.quartz.concord.cord04Roles.ChannelEntity +import com.vitorpamplona.quartz.concord.cord04Roles.ConcordJson import com.vitorpamplona.quartz.concord.cord04Roles.ConcordPermissions +import com.vitorpamplona.quartz.concord.cord04Roles.ControlEntityKind +import com.vitorpamplona.quartz.concord.cord04Roles.ControlRootWrap +import com.vitorpamplona.quartz.concord.cord04Roles.GrantEntity import com.vitorpamplona.quartz.concord.cord04Roles.MetadataEntity import com.vitorpamplona.quartz.concord.cord04Roles.RoleEntity import com.vitorpamplona.quartz.concord.cord05Invites.CommunityInvite import com.vitorpamplona.quartz.concord.cord05Invites.InviteBundleStatus import com.vitorpamplona.quartz.concord.cord05Invites.InviteRelayDictionary +import com.vitorpamplona.quartz.concord.crypto.ConcordKeyDerivation +import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys import com.vitorpamplona.quartz.concord.crypto.GroupKey import com.vitorpamplona.quartz.concord.envelope.ConcordStreamEnvelope import com.vitorpamplona.quartz.nip01Core.core.Event @@ -138,6 +146,10 @@ class AccountConcordActions( ownerSalt = community.ownerSalt.toHexKey(), root = community.communityRoot.toHexKey(), rootEpoch = community.rootEpoch, + // The creator is the founding staff member (CORD-02 §2): it keeps the write + // secret and publishes only the derived pubkey to everyone else. + controlPk = community.controlPkHex, + controlRoot = community.controlRoot.toHexKey(), relays = relayUrls, name = name, addedAt = TimeUtils.now() * 1000, @@ -168,6 +180,9 @@ class AccountConcordActions( rootEpoch = entry.rootEpoch, name = entry.name, relays = entry.relays, + // The joiner can never derive the Control Plane address, so the bundle carries + // it (CORD-05 §1). Null on a legacy community, which has none to carry. + controlPk = entry.controlPk, ) val minted = ConcordActions.mintInviteLink(base, invite, TimeUtils.now(), entry.relays) @@ -245,6 +260,9 @@ class AccountConcordActions( ownerSalt = bundle.ownerSalt, root = bundle.communityRoot, rootEpoch = bundle.rootEpoch, + // Read access to the Control Plane, never write (CORD-05 §1). Absent = the + // community is still pre-split, so we fold it at the legacy address. + controlPk = bundle.controlPk, relays = bundle.relays, name = bundle.name, addedAt = TimeUtils.now() * 1000, @@ -451,6 +469,24 @@ class AccountConcordActions( // every client's AuthorityResolver, so a call by someone who doesn't outrank the // target is simply dropped on fold. Owner-authored calls always take effect. + /** + * The Control Plane keys for a moderation write, or null when this account cannot + * publish there: on a split epoch only `control_root` holders can mint a wrap that + * verifies at the plane's address (CORD-02 §2), and wrapping without the secret + * throws rather than missigning. Rank and key possession can diverge — a freshly + * promoted staffer writes only once their `control_wrap` is adopted (CORD-04 §3), + * and the UI gates on rank — so every moderation verb no-ops through this check + * instead of crashing on a rank-gated action. + */ + private fun controlKeysForWrite(session: ConcordCommunitySession): ControlPlaneKeys? { + val cp = session.controlPlaneKeys() + if (!cp.canWrite) { + Log.w("Concord") { "Control write refused for ${session.entry.id}: control_root not held at epoch ${session.entry.rootEpoch} (CORD-02 §2)" } + return null + } + return cp + } + /** Grant [member] exactly [roleIds] (empty list revokes their roles). */ suspend fun grantConcordRole( communityId: String, @@ -459,7 +495,23 @@ class AccountConcordActions( ): Boolean { val session = account.concordSessions.sessionFor(communityId) ?: return false if (!account.isWriteable()) return false - val wrap = ConcordModeration.grant(account.signer, session.controlPlaneKey(), communityId.hexToByteArray(), member, roleIds, session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) + val cp = controlKeysForWrite(session) ?: return false + // A Grant that first makes its member staff must deliver the control_root in the same + // edition (CORD-04 §3) — grantWithStaffDelivery attaches the pairwise wrap when the + // roles carry a Control-writing bit and we hold the secret to hand over. + val wrap = + ConcordModeration.grantWithStaffDelivery( + actor = account.signer, + controlPlane = cp, + communityId = communityId.hexToByteArray(), + member = member, + roleIds = roleIds, + current = session.controlEditions(), + createdAt = TimeUtils.now(), + owner = session.entry.owner, + controlRoot = session.entry.controlRoot?.hexToByteArray(), + epoch = session.entry.rootEpoch, + ) publishConcordWrap(session.entry, wrap) return true } @@ -494,11 +546,11 @@ class AccountConcordActions( val author = note.author?.pubkeyHex ?: note.event?.pubKey ?: return null if (author == account.signer.pubKey) return null val communityId = channel.channelId.communityId - val state = - account.concordSessions - .sessionFor(communityId) - ?.state - ?.value ?: return null + val session = account.concordSessions.sessionFor(communityId) ?: return null + val state = session.state.value ?: return null + // Rank alone isn't enough on a split epoch: the Grant edition takes the control_root + // (CORD-02 §2), so don't offer an action the verb would refuse. + if (!session.controlPlaneKeys().canWrite) return null if (state.authority.isOwner(author) || !state.authority.isOwner(account.signer.pubKey)) return null val adminRoleId = state.roles.entries @@ -515,7 +567,7 @@ class AccountConcordActions( ): Boolean { val session = account.concordSessions.sessionFor(communityId) ?: return false if (!account.isWriteable()) return false - val cp = session.controlPlaneKey() + val cp = controlKeysForWrite(session) ?: return false val existing = session.state.value @@ -530,7 +582,22 @@ class AccountConcordActions( roleId.toHexKey() } - val grantWrap = ConcordModeration.grant(account.signer, cp, communityId.hexToByteArray(), member, listOf(roleIdHex), session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) + // Admin carries every management bit, so this Grant makes its member staff: it must + // deliver the control_root alongside the rank (CORD-04 §3), or the new admin holds + // authority it cannot publish under. + val grantWrap = + ConcordModeration.grantWithStaffDelivery( + actor = account.signer, + controlPlane = cp, + communityId = communityId.hexToByteArray(), + member = member, + roleIds = listOf(roleIdHex), + current = session.controlEditions(), + createdAt = TimeUtils.now(), + owner = session.entry.owner, + controlRoot = session.entry.controlRoot?.hexToByteArray(), + epoch = session.entry.rootEpoch, + ) publishConcordWrap(session.entry, grantWrap) return true } @@ -542,7 +609,8 @@ class AccountConcordActions( ): Boolean { val session = account.concordSessions.sessionFor(communityId) ?: return false if (!account.isWriteable()) return false - val grantWrap = ConcordModeration.grant(account.signer, session.controlPlaneKey(), communityId.hexToByteArray(), member, emptyList(), session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) + val cp = controlKeysForWrite(session) ?: return false + val grantWrap = ConcordModeration.grant(account.signer, cp, communityId.hexToByteArray(), member, emptyList(), session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) publishConcordWrap(session.entry, grantWrap) return true } @@ -568,12 +636,11 @@ class AccountConcordActions( val author = note.author?.pubkeyHex ?: note.event?.pubKey ?: return null if (author == account.signer.pubKey) return null val communityId = channel.channelId.communityId - val authority = - account.concordSessions - .sessionFor(communityId) - ?.state - ?.value - ?.authority ?: return null + val session = account.concordSessions.sessionFor(communityId) ?: return null + val authority = session.state.value?.authority ?: return null + // Rank alone isn't enough on a split epoch: the banlist edition takes the control_root + // (CORD-02 §2), so don't offer an action the verb would refuse. + if (!session.controlPlaneKeys().canWrite) return null if (authority.isOwner(author)) return null // The owner short-circuits rather than going through canActOn: canActOn starts at // hasPermission, which is false while banned, and a rogue BAN holder *can* currently put @@ -590,7 +657,8 @@ class AccountConcordActions( ): Boolean { val session = account.concordSessions.sessionFor(communityId) ?: return false if (!account.isWriteable()) return false - val wrap = ConcordModeration.ban(account.signer, session.controlPlaneKey(), communityId.hexToByteArray(), member, session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) + val cp = controlKeysForWrite(session) ?: return false + val wrap = ConcordModeration.ban(account.signer, cp, communityId.hexToByteArray(), member, session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) publishConcordWrap(session.entry, wrap) return true } @@ -602,7 +670,8 @@ class AccountConcordActions( ): Boolean { val session = account.concordSessions.sessionFor(communityId) ?: return false if (!account.isWriteable()) return false - val wrap = ConcordModeration.unban(account.signer, session.controlPlaneKey(), communityId.hexToByteArray(), member, session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) + val cp = controlKeysForWrite(session) ?: return false + val wrap = ConcordModeration.unban(account.signer, cp, communityId.hexToByteArray(), member, session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) publishConcordWrap(session.entry, wrap) return true } @@ -636,12 +705,16 @@ class AccountConcordActions( if (!iCanBan) return false val removedLower = removed.mapTo(HashSet()) { it.lowercase() } if (removedLower.isEmpty() || removedLower.any { authority.isOwner(it) }) return false + // A Refounding writes the current plane (the pre-rotation bans) and the new one (the + // compaction), so on a split epoch it takes the current control_root (CORD-02 §2). A + // rank-qualified refounder whose secret hasn't arrived yet must wait for re-delivery. + val cp = controlKeysForWrite(session) ?: return false // 1. Ban the removed members on the current Control Plane so the compacted snapshot — // and thus the new epoch — carries the ban. publishConcordWrap folds it in locally // first, so each subsequent edition chains onto the updated banlist head. for (target in removedLower) { - val banWrap = ConcordModeration.ban(account.signer, session.controlPlaneKey(), communityId.hexToByteArray(), target, session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) + val banWrap = ConcordModeration.ban(account.signer, cp, communityId.hexToByteArray(), target, session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) publishConcordWrap(session.entry, banWrap) } @@ -667,16 +740,27 @@ class AccountConcordActions( // 3. Build the refounding: new root, compacted Control Plane, per-recipient rekey blobs. val entry = session.entry val newRoot = RandomInstance.bytes(32) + // A fresh control_root is minted beside the new root at every Refounding (CORD-02 §2), + // so a demoted staffer's retained secret dies with the epoch — and a legacy community + // upgrades to the split as a side effect of its next ban (CORD-06 §3). + val newControlRoot = RandomInstance.bytes(32) + // The staff set the new secret goes to: the owner plus everyone holding a + // Control-writing bit (CORD-04 §3). They get the 136-byte blob, every other + // recipient the 104-byte one carrying the pubkey alone. (The builder mints a + // blob per recipient, so staff who aren't recipients are simply never reached.) + val staff = authority.staffMembers() val build = ConcordActions.buildRefounding( rotatorSigner = account.signer, communityId = communityId, priorRoot = entry.root.hexToByteArray(), newRoot = newRoot, + newControlRoot = newControlRoot, rootEpoch = entry.rootEpoch, priorControlWraps = session.controlPlaneWraps(), - priorControlKey = session.controlPlaneKey(), + priorControlKeys = cp, recipientsXOnly = recipients, + staffXOnly = staff, createdAt = TimeUtils.now(), ) @@ -690,7 +774,7 @@ class AccountConcordActions( // 5. Adopt the new epoch ourselves. This rebuilds our session under the new root and // re-folds the compacted Control Plane (with the ban), dropping the removed members. - adoptConcordRoot(entry, newRoot, build.newEpoch) + adoptConcordRoot(entry, newRoot, build.newEpoch, build.newControlKeys.address.hexToByteArray(), newControlRoot) return true } @@ -710,9 +794,14 @@ class AccountConcordActions( entry: ConcordCommunityListEntry, newRoot: ByteArray, newEpoch: Long, + newControlPk: ByteArray? = null, + newControlRoot: ByteArray? = null, ) { if (!adoptedConcordRotations.add("${entry.id}:$newEpoch")) return - val held = (entry.heldRoots + HeldRoot(entry.rootEpoch, entry.root)).distinctBy { it.epoch } + // The epoch we're leaving is banked with the address it was folded at, so its Control + // Plane stays subscribable for the anti-rollback floor (a split epoch's address can + // never be re-derived, only remembered — CORD-02 §2). + val held = (entry.heldRoots + HeldRoot(entry.rootEpoch, entry.root, entry.controlPk, entry.controlRoot)).distinctBy { it.epoch } val next = ConcordCommunityListEntry( id = entry.id, @@ -720,6 +809,11 @@ class AccountConcordActions( ownerSalt = entry.ownerSalt, root = newRoot.toHexKey(), rootEpoch = newEpoch, + // A rotation that delivered no control material is a legacy, pre-split one + // (CORD-06 §3): the new epoch keeps folding at the legacy address, and the + // stale prior-epoch values must NOT be carried into it. + controlPk = newControlPk?.toHexKey(), + controlRoot = newControlRoot?.toHexKey(), heldRoots = held, privateChannels = entry.privateChannels, relays = entry.relays, @@ -769,6 +863,7 @@ class AccountConcordActions( wraps = wraps, baseRekey = session.nextBaseRekeyKey(), recipientSigner = account.signer, + communityId = entry.id, priorRoot = entry.root.hexToByteArray(), rootEpoch = entry.rootEpoch, ) ?: continue @@ -779,7 +874,62 @@ class AccountConcordActions( // who has themselves been banned could still rotate the whole community. val authorized = authority.isOwner(received.rotator) || authority.hasPermission(received.rotator, ConcordPermissions.BAN) if (!authorized) continue - adoptConcordRoot(entry, received.newRoot, received.newEpoch) + adoptConcordRoot(entry, received.newRoot, received.newEpoch, received.newControlPk, received.newControlRoot) + } + } + + /** + * Adopt a `control_root` delivered to us by a staff-making Grant (CORD-04 §3): the + * promoting edition carries the secret in `control_wrap`, NIP-44-encrypted under the + * granter↔member pairwise key, so promotion and key delivery are one signed edition + * with nothing separate to watch an inbox for. + * + * Adoption is gated twice and fails closed both times. The secret is adopted only if + * it derives to exactly the `control_pk` we already hold for the named epoch — a + * garbage wrap is attributable griefing, nothing worse — and only from a Grant our own + * fold honors, so a rogue cannot feed us a key by minting an edition nobody accepts. + * The epoch check matters because compaction re-wraps a Grant head verbatim across + * Refoundings, so a folded head can legitimately carry a wrap minted for a prior epoch. + * + * Idempotent: once the entry holds the secret there is nothing to adopt. Runs on the + * revision tick, like the rekey drain. + */ + internal suspend fun drainConcordStaffGrants() { + if (!account.isWriteable()) return + val me = account.signer.pubKey.lowercase() + for (session in account.concordSessions.sessions()) { + val entry = session.entry + // Already staff at this epoch, or a legacy community with no split to join. + val heldControlPk = entry.controlPk + if (entry.controlRoot != null || heldControlPk == null) continue + val state = session.state.value ?: continue + // Only a Grant our fold honors can deliver: an unauthorized edition hands us nothing. + if (!state.authority.isStaff(me)) continue + + val myGrantCoordinate = + ConcordKeyDerivation + .grantCoordinate(entry.id.hexToByteArray(), me.hexToByteArray()) + .toHexKey() + val delivered = + session + .controlEditions() + .filter { it.entityKind == ControlEntityKind.GRANT && it.entityIdHex == myGrantCoordinate } + // Newest first: a re-issued Grant (a lost key, a head superseded before we + // fetched it) carries the fresher wrap. + .sortedByDescending { it.version } + .firstNotNullOfOrNull { edition -> + val wrap = ConcordJson.decodeOrNull(edition.content)?.controlWrap ?: return@firstNotNullOfOrNull null + val opened = ControlRootWrap.openOrNull(wrap, account.signer, edition.author) ?: return@firstNotNullOfOrNull null + if (opened.epoch != entry.rootEpoch) return@firstNotNullOfOrNull null + // Fails closed: a secret that doesn't derive to the pk we hold is dropped, + // never adopted — we will not split ourselves off from the plane's readers. + if (!ControlRootWrap.derivesTo(opened.controlRoot, entry.id.hexToByteArray(), entry.rootEpoch, heldControlPk)) return@firstNotNullOfOrNull null + opened.controlRoot + } ?: continue + + account.sendMyPublicAndPrivateOutbox( + account.concordChannelList.follow(entry.withControlRoot(delivered.toHexKey())), + ) } } @@ -859,8 +1009,9 @@ class AccountConcordActions( ): Boolean { val session = account.concordSessions.sessionFor(communityId) ?: return false if (!account.isWriteable()) return false + val cp = controlKeysForWrite(session) ?: return false val metadata = MetadataEntity(name = name, icon = icon, banner = banner, description = description, relays = relays) - val wrap = ConcordModeration.editMetadata(account.signer, session.controlPlaneKey(), communityId.hexToByteArray(), metadata, session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) + val wrap = ConcordModeration.editMetadata(account.signer, cp, communityId.hexToByteArray(), metadata, session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) publishConcordWrap(session.entry, wrap) return true } @@ -876,9 +1027,10 @@ class AccountConcordActions( ): Boolean { val session = account.concordSessions.sessionFor(communityId) ?: return false if (!account.isWriteable()) return false + val cp = controlKeysForWrite(session) ?: return false val channelId = RandomInstance.bytes(32) val channel = ChannelEntity(name = name.trim()) - val wrap = ConcordModeration.defineChannel(account.signer, session.controlPlaneKey(), channelId, channel, session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) + val wrap = ConcordModeration.defineChannel(account.signer, cp, channelId, channel, session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) publishConcordWrap(session.entry, wrap) return true } @@ -891,6 +1043,7 @@ class AccountConcordActions( ): Boolean { val session = account.concordSessions.sessionFor(communityId) ?: return false if (!account.isWriteable()) return false + val cp = controlKeysForWrite(session) ?: return false // Carry the standing definition forward and change only the name. A ChannelEntity built from // scratch defaults `private` and `voice` to false, so renaming a private channel used to // publish an edition declaring it PUBLIC — and a voice channel became a text channel. @@ -900,7 +1053,7 @@ class AccountConcordActions( ?.get(channelIdHex) ?.definition val channel = ChannelEntity(name = name.trim(), private = standing?.private ?: false, voice = standing?.voice ?: false) - val wrap = ConcordModeration.defineChannel(account.signer, session.controlPlaneKey(), channelIdHex.hexToByteArray(), channel, session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) + val wrap = ConcordModeration.defineChannel(account.signer, cp, channelIdHex.hexToByteArray(), channel, session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) publishConcordWrap(session.entry, wrap) return true } @@ -913,6 +1066,7 @@ class AccountConcordActions( ): Boolean { val session = account.concordSessions.sessionFor(communityId) ?: return false if (!account.isWriteable()) return false + val cp = controlKeysForWrite(session) ?: return false // Same as rename: preserve the standing flags so a tombstone does not also silently // reclassify the channel it retires. val standing = @@ -921,7 +1075,7 @@ class AccountConcordActions( ?.get(channelIdHex) ?.definition val channel = ChannelEntity(name = name.trim(), private = standing?.private ?: false, voice = standing?.voice ?: false, deleted = true) - val wrap = ConcordModeration.defineChannel(account.signer, session.controlPlaneKey(), channelIdHex.hexToByteArray(), channel, session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) + val wrap = ConcordModeration.defineChannel(account.signer, cp, channelIdHex.hexToByteArray(), channel, session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) publishConcordWrap(session.entry, wrap) return true } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordChannelListScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordChannelListScreen.kt index 1b40e49820..d8c94f5ace 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordChannelListScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordChannelListScreen.kt @@ -167,11 +167,15 @@ fun ConcordChannelListScreen( // Channel create/rename/delete are gated on MANAGE_CHANNELS (or owner) — the same predicate the // fold enforces, so an unauthorized action would be a silent no-op we shouldn't even offer. + // Rank alone isn't enough on a split epoch: publishing any Control edition also takes the + // control_root (CORD-02 §2), which a freshly promoted staffer may not hold yet (CORD-04 §3), + // so the affordance waits for the key too. val canManageChannels = state?.authority?.let { it.isOwner(account.signer.pubKey) || it.effectivePermissions(account.signer.pubKey).has(ConcordPermissions.MANAGE_CHANNELS) - } == true + } == true && + session?.controlPlaneKeys()?.canWrite == true // channelIdHex == null → create; else → rename that channel. var channelEditor by remember { mutableStateOf(null) } @@ -234,11 +238,13 @@ fun ConcordChannelListScreen( } }, actions = { + // Rank + the Control write key (CORD-02 §2), like [canManageChannels] above. val canEdit = state?.authority?.let { it.isOwner(account.signer.pubKey) || it.effectivePermissions(account.signer.pubKey).has(ConcordPermissions.MANAGE_METADATA) - } == true + } == true && + session?.controlPlaneKeys()?.canWrite == true IconButton(onClick = { nav.nav(Route.ConcordMembers(communityId)) }) { SymbolIcon(symbol = MaterialSymbols.Group, contentDescription = stringRes(com.vitorpamplona.amethyst.R.string.concord_members_title)) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordChannelCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordChannelCommands.kt index 7f52a986a8..618ae59581 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordChannelCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordChannelCommands.kt @@ -150,12 +150,14 @@ object ConcordChannelCommands { ctx: Context, sc: StoredCommunity, ): ConcordCommunityState { - val controlPlane = ConcordActions.controlPlane(sc.root.hexToByteArray(), sc.communityId.hexToByteArray(), sc.rootEpoch) + val controlPlane = ConcordCommands.controlPlaneKeysFor(sc) val relays = ConcordCommands.relaysFor(ctx, sc) - // The relays gate the plane's kind-1059 behind NIP-42 as the derived stream key — register - // it so the drain's AUTH challenge is answered as the control plane, not the account. - ctx.registerConcordStreamKeys(relays, listOf(controlPlane.secretKey)) - val wraps = ctx.drain(relays.associateWith { listOf(ConcordActions.planeFilter(controlPlane.publicKeyHex)) }, pendingOnAuthRequired = true).map { it.second } + // The relays gate the plane's kind-1059 behind NIP-42 as the stream key — register it so + // the drain's AUTH challenge is answered as the control plane, not the account. On a split + // epoch only staff hold that secret (CORD-02 §2); a plain member registers nothing and + // relies on the relay serving the plane unauthenticated. + ctx.registerConcordStreamKeys(relays, listOfNotNull(controlPlane.signer?.secretKey)) + val wraps = ctx.drain(relays.associateWith { listOf(ConcordActions.planeFilter(controlPlane.address)) }, pendingOnAuthRequired = true).map { it.second } return ConcordActions.foldCommunity(wraps, controlPlane, sc.owner) } diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt index 7b41a02f8f..ff151176ef 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt @@ -29,6 +29,7 @@ import com.vitorpamplona.amethyst.cli.stores.StoredCommunity import com.vitorpamplona.amethyst.cli.stores.StoredHeldRoot import com.vitorpamplona.amethyst.commons.actions.ConcordActions import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEvent +import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray import com.vitorpamplona.quartz.nip01Core.core.toHexKey import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl @@ -119,6 +120,10 @@ object ConcordCommands { ownerSalt = community.ownerSalt.toHexKey(), root = community.communityRoot.toHexKey(), rootEpoch = community.rootEpoch, + // The creator is the founding staff member: it keeps the write secret and + // publishes the pubkey to everyone else (CORD-02 §2). + controlPk = community.controlPkHex, + controlRoot = community.controlRoot.toHexKey(), generalChannelId = community.generalChannelIdHex, relays = relays, ), @@ -179,6 +184,14 @@ object ConcordCommands { val imported = entries.map { e -> val prior = existing[e.id] + // Control key material is per-epoch (CORD-02 §2): a stored value may only + // backstop a list entry from the SAME epoch (e.g. another client republished + // the list without the extension fields). Across a rotation the old pair is + // stale — a prior-epoch control_root would derive a wrong address entirely, + // and a prior-epoch control_pk would shadow a legacy rotation's address — so + // it must never be carried forward (the invariant adoption enforces with its + // derive-check, which this path has no way to run). + val priorSameEpoch = prior?.takeIf { it.rootEpoch == e.rootEpoch } store.upsert( StoredCommunity( name = e.name.ifBlank { prior?.name ?: "" }, @@ -187,15 +200,23 @@ object ConcordCommands { ownerSalt = e.ownerSalt, root = e.root, rootEpoch = e.rootEpoch, + // Carried straight from the list entry: the Control Plane address is + // delivered, never derivable (CORD-02 §2), and the write secret only + // rides the list when this account is staff. Both blank on a legacy + // community, which keeps its old single-key plane. + controlPk = e.controlPk ?: priorSameEpoch?.controlPk ?: "", + controlRoot = e.controlRoot ?: priorSameEpoch?.controlRoot ?: "", generalChannelId = prior?.generalChannelId ?: "", relays = e.relays, - heldRoots = e.heldRoots.map { StoredHeldRoot(it.epoch, it.key) }, + heldRoots = e.heldRoots.map { StoredHeldRoot(it.epoch, it.key, it.controlPk ?: "") }, ), ) mapOf( "name" to e.name, "community_id" to e.id, "root_epoch" to e.rootEpoch, + "control_pk" to (e.controlPk ?: ""), + "staff" to (e.controlRoot != null), "held_roots" to e.heldRoots.map { mapOf("epoch" to it.epoch, "root" to it.key) }, ) } @@ -216,7 +237,9 @@ object ConcordCommands { val sc = ConcordStore(dataDir.concordFile).find(handle) ?: return notFound(handle) Context.open(dataDir).use { ctx -> ctx.prepare() - val invite = ConcordActions.inviteFor(sc.communityId, sc.owner, sc.ownerSalt, sc.root, sc.rootEpoch, sc.name, sc.relays) + // The joiner cannot derive the Control Plane address, so the invite carries it + // (CORD-05 §1); omitted for a legacy community, which has none to carry. + val invite = ConcordActions.inviteFor(sc.communityId, sc.owner, sc.ownerSalt, sc.root, sc.rootEpoch, sc.name, sc.relays, sc.controlPk.ifBlank { null }) val minted = ConcordActions.mintInviteLink(base, invite, TimeUtils.now(), sc.relays) val ack = ctx.publish(minted.bundleEvent, relaysFor(ctx, sc)) RawEventSupport.publishGuard(ack, minted.bundleEvent.id)?.let { return it } @@ -257,6 +280,9 @@ object ConcordCommands { ownerSalt = bundle.ownerSalt, root = bundle.communityRoot, rootEpoch = bundle.rootEpoch, + // Read access to the Control Plane, never write (CORD-05 §1). Absent = the + // community is still pre-split and folds at the legacy address. + controlPk = bundle.controlPk ?: "", relays = bundle.relays, ), ) @@ -276,6 +302,20 @@ object ConcordCommands { sc: StoredCommunity, ): Set = normalize(sc.relays).ifEmpty { ctx.outboxRelays() } + /** + * The Control Plane keys for [sc] as this account holds them (CORD-02 §5): staff + * (write key held), member (address held, read-only), or legacy (pre-split, keyed + * by the `community_root` alone). + */ + fun controlPlaneKeysFor(sc: StoredCommunity) = + ConcordActions.controlPlaneKeys( + communityRoot = sc.root.hexToByteArray(), + communityId = sc.communityId.hexToByteArray(), + rootEpoch = sc.rootEpoch, + controlPk = sc.controlPk.ifBlank { null }, + controlRoot = sc.controlRoot.ifBlank { null }, + ) + fun notFound(handle: String): Int { Output.error("not_found", "no joined community matching '$handle' — run `amy concord list`") return 1 diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt index 16fc1b2757..3633b24dbe 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt @@ -32,7 +32,7 @@ import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityState import com.vitorpamplona.quartz.concord.cord04Roles.ConcordPermissions import com.vitorpamplona.quartz.concord.cord04Roles.ControlEdition import com.vitorpamplona.quartz.concord.cord04Roles.RoleEntity -import com.vitorpamplona.quartz.concord.crypto.GroupKey +import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray import com.vitorpamplona.quartz.nip01Core.core.toHexKey import com.vitorpamplona.quartz.utils.RandomInstance @@ -93,6 +93,7 @@ object ConcordModCommands { Context.open(dataDir).use { ctx -> ctx.prepare() val (cp, editions) = load(ctx, sc) + writeGuard(cp)?.let { return it } val roleId = RandomInstance.bytes(32) val role = RoleEntity(name = name, position = position, permissions = ConcordPermissions.of(*permBits.toIntArray()).toWire()) val wrap = ConcordModeration.defineRole(ctx.signer, cp, roleId, role, editions, TimeUtils.now(), owner = sc.owner) @@ -119,7 +120,23 @@ object ConcordModCommands { ctx.prepare() val member = ctx.requireUserHex(userRef) val (cp, editions) = load(ctx, sc) - val wrap = ConcordModeration.grant(ctx.signer, cp, sc.communityId.hexToByteArray(), member, listOf(roleId), editions, TimeUtils.now(), owner = sc.owner) + writeGuard(cp)?.let { return it } + // A Grant that first makes its member staff must carry the write secret in the same + // edition (CORD-04 §3); ConcordModeration wraps it pairwise when the granted roles + // hold a Control-writing bit and we hold the secret to deliver. + val wrap = + ConcordModeration.grantWithStaffDelivery( + actor = ctx.signer, + controlPlane = cp, + communityId = sc.communityId.hexToByteArray(), + member = member, + roleIds = listOf(roleId), + current = editions, + createdAt = TimeUtils.now(), + owner = sc.owner, + controlRoot = sc.controlRoot.ifBlank { null }?.hexToByteArray(), + epoch = sc.rootEpoch, + ) val ack = ctx.publish(wrap, ConcordCommands.relaysFor(ctx, sc)) RawEventSupport.publishGuard(ack, wrap.id)?.let { return it } Output.emit(mapOf("member" to member, "roles" to listOf(roleId)) + RawEventSupport.ackFields(ack)) @@ -154,6 +171,7 @@ object ConcordModCommands { ctx.prepare() val member = ctx.requireUserHex(userRef) val (cp, editions) = load(ctx, sc) + writeGuard(cp)?.let { return it } val cid = sc.communityId.hexToByteArray() val wrap = if (ban) { @@ -168,20 +186,34 @@ object ConcordModCommands { } } - /** Drain the control plane and return its key + current editions to chain onto. */ + /** Drain the control plane and return its keys + current editions to chain onto. */ private suspend fun load( ctx: Context, sc: StoredCommunity, - ): Pair> { - val cp = ConcordActions.controlPlane(sc.root.hexToByteArray(), sc.communityId.hexToByteArray(), sc.rootEpoch) + ): Pair> { + val cp = ConcordCommands.controlPlaneKeysFor(sc) val relays = ConcordCommands.relaysFor(ctx, sc) - // Concord relays serve the plane's kind-1059 only to a connection AUTHed as the derived - // stream key — register the control key so the drain isn't refused (else the fold is empty). - ctx.registerConcordStreamKeys(relays, listOf(cp.secretKey)) - val wraps = ctx.drain(relays.associateWith { listOf(ConcordActions.planeFilter(cp.publicKeyHex)) }, pendingOnAuthRequired = true).map { it.second } + // Concord relays serve the plane's kind-1059 only to a connection AUTHed as the stream + // key — register it so the drain isn't refused (else the fold is empty). On a split epoch + // that secret is staff-only (CORD-02 §2), and a member simply has nothing to register. + ctx.registerConcordStreamKeys(relays, listOfNotNull(cp.signer?.secretKey)) + val wraps = ctx.drain(relays.associateWith { listOf(ConcordActions.planeFilter(cp.address)) }, pendingOnAuthRequired = true).map { it.second } return cp to ConcordActions.controlEditions(wraps, cp) } + /** + * Refuses a moderation command that this account cannot publish: on a split epoch + * only `control_root` holders can mint a wrap the plane accepts (CORD-02 §2), so a + * member would otherwise sign an edition every relay and reader drops. Possession is + * a spam gate, never authority — holding the key still does not make the action + * honored, which the Roster decides at fold (CORD-04 §5). + */ + private fun writeGuard(cp: ControlPlaneKeys): Int? { + if (cp.canWrite) return null + Output.error("forbidden", "this account holds no control_root for the community, so it cannot publish Control Plane editions (CORD-02 §2) — ask a staff member to grant you a Control-writing role") + return 1 + } + private fun permByName(name: String): Int? = when (name.uppercase()) { "MANAGE_ROLES" -> ConcordPermissions.MANAGE_ROLES diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt index 379dc44b83..7ae731a877 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt @@ -37,6 +37,12 @@ data class StoredCommunity( val ownerSalt: String = "", val root: String = "", val rootEpoch: Long = 0, + // The Control Plane signer's pubkey at [rootEpoch] (CORD-02 §2): read access, never write. + // Blank = a legacy, pre-split community, whose Control Plane is keyed the old way. + val controlPk: String = "", + // The staff write key at [rootEpoch], held only when this account is the owner or staff + // (CORD-02 §2). Blank for a regular member, who can read the plane but not publish to it. + val controlRoot: String = "", val generalChannelId: String = "", val relays: List = emptyList(), // Past access roots kept per epoch (CORD-06 Refounding rotates the root). Lets `read --epoch ` @@ -48,6 +54,8 @@ data class StoredCommunity( data class StoredHeldRoot( val epoch: Long = 0, val root: String = "", + /** That epoch's Control Plane address; blank for a legacy, pre-split epoch (CORD-02 §5). */ + val controlPk: String = "", ) /** diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt index fa23a7ee33..d99679263f 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt @@ -45,6 +45,7 @@ import com.vitorpamplona.quartz.concord.cord06Rekey.ConcordRefounding import com.vitorpamplona.quartz.concord.cord06Rekey.ReceivedRefounding import com.vitorpamplona.quartz.concord.cord06Rekey.RefoundingBuild import com.vitorpamplona.quartz.concord.crypto.ConcordKeyDerivation +import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys import com.vitorpamplona.quartz.concord.crypto.GroupKey import com.vitorpamplona.quartz.concord.envelope.ConcordStreamEnvelope import com.vitorpamplona.quartz.nip01Core.core.Event @@ -89,12 +90,52 @@ data class HistoricalChannelPlane( object ConcordActions { // ---- plane key derivation ------------------------------------------------- + /** + * The **legacy** Control Plane group key (pre-split epochs, CORD-06 §3), which + * doubles as the split epochs' *read* key derivation. For anything that opens + * or writes the Control Plane, prefer [controlPlaneKeys]. + */ fun controlPlane( communityRoot: ByteArray, communityId: ByteArray, rootEpoch: Long, ): GroupKey = ConcordKeyDerivation.controlPlaneKey(communityRoot, communityId, rootEpoch) + /** + * The Control Plane keys as this account holds them (CORD-02 §5): staff when + * [controlRoot] is held, read-only member when only [controlPk] is, and the + * legacy single-key plane when neither (a pre-split epoch). + */ + fun controlPlaneKeys( + communityRoot: ByteArray, + communityId: ByteArray, + rootEpoch: Long, + controlPk: HexKey? = null, + controlRoot: HexKey? = null, + ): ControlPlaneKeys = ControlPlaneKeys.of(communityRoot, communityId, rootEpoch, controlPk, controlRoot) + + /** The Control Plane keys described by a joined-list [entry]. */ + fun controlPlaneKeysFor(entry: ConcordCommunityListEntry): ControlPlaneKeys = + controlPlaneKeys( + entry.root.hexToByteArray(), + entry.id.hexToByteArray(), + entry.rootEpoch, + entry.controlPk, + entry.controlRoot, + ) + + /** + * The Control Plane's stream address for a subscription: the held `control_pk` + * on a split epoch, else the legacy derived address. Cheaper than + * [controlPlaneKeys] when only the address is needed. + */ + fun controlPlaneAddress( + communityRoot: ByteArray, + communityId: ByteArray, + rootEpoch: Long, + controlPk: HexKey?, + ): HexKey = controlPk?.lowercase() ?: controlPlane(communityRoot, communityId, rootEpoch).publicKeyHex + fun publicChannel( communityRoot: ByteArray, channelId: ByteArray, @@ -178,7 +219,7 @@ object ConcordActions { /** Opens the control-plane [wraps] into their [ControlEdition]s (drops any that don't open/parse). */ fun controlEditions( wraps: List, - controlPlane: GroupKey, + controlPlane: ControlPlaneKeys, ): List = wraps.mapNotNull { wrap -> ConcordStreamEnvelope.openOrNull(wrap, controlPlane)?.let { ControlEdition.fromRumor(it.rumor) } @@ -187,7 +228,7 @@ object ConcordActions { /** Opens the control-plane [wraps] and folds them into the live community state. */ fun foldCommunity( wraps: List, - controlPlane: GroupKey, + controlPlane: ControlPlaneKeys, ownerPubKey: HexKey, ): ConcordCommunityState = ConcordCommunityState.fold(controlEditions(wraps, controlPlane), ownerPubKey) @@ -360,7 +401,12 @@ object ConcordActions { // ---- invites -------------------------------------------------------------- - /** Builds a [CommunityInvite] from a freshly created (or joined) community's public info. */ + /** + * Builds a [CommunityInvite] from a freshly created (or joined) community's + * public info. [controlPk] is the Control Plane's signer pubkey at [rootEpoch] + * (CORD-05 §1) — read access for the joiner, never write; null only for a + * legacy, pre-split community. + */ fun inviteFor( communityIdHex: HexKey, ownerPubKey: HexKey, @@ -369,6 +415,7 @@ object ConcordActions { rootEpoch: Long, name: String, relays: List, + controlPk: HexKey? = null, ): CommunityInvite = CommunityInvite( communityId = communityIdHex, @@ -376,6 +423,7 @@ object ConcordActions { ownerSalt = ownerSaltHex, communityRoot = communityRootHex, rootEpoch = rootEpoch, + controlPk = controlPk, relays = relays, name = name, ) @@ -427,8 +475,18 @@ object ConcordActions { nowMs: Long = TimeUtils.nowMillis(), ): InviteBundleStatus = ConcordInviteBundle.classify(wraps, token, nowMs) - /** Derives the control plane described by a redeemed [invite] so the joiner can read it. */ - fun controlPlaneFor(invite: CommunityInvite): GroupKey = controlPlane(invite.communityRoot.hexToByteArray(), invite.communityId.hexToByteArray(), invite.rootEpoch) + /** + * The Control Plane keys described by a redeemed [invite] so the joiner can + * read it: the bundle's `control_pk` on a split community, the legacy plane + * when absent (CORD-05 §1). Never a writer — an invite delivers no secret. + */ + fun controlPlaneFor(invite: CommunityInvite): ControlPlaneKeys = + controlPlaneKeys( + invite.communityRoot.hexToByteArray(), + invite.communityId.hexToByteArray(), + invite.rootEpoch, + controlPk = invite.controlPk, + ) // ---- guestbook (CORD-02 §5) ---------------------------------------------- @@ -468,19 +526,23 @@ object ConcordActions { /** * Builds a whole-community Refounding (CORD-06 §3): the compacted Control Plane - * re-sealed under [newRoot] plus the base-rotation rekey blobs delivering - * [newRoot] to [recipientsXOnly]. Pure — the caller sources the recipient set - * and owns publish + persistence. + * re-sealed at the new epoch's split Control address plus the base-rotation + * rekey blobs delivering [newRoot] + the new `control_pk` to [recipientsXOnly] + * — the [staffXOnly] subset also receiving [newControlRoot] (CORD-06 §1). Pure + * — the caller sources the recipient and staff sets (the folded Roster's + * `staffMembers()`, CORD-04 §3) and owns publish + persistence. */ suspend fun buildRefounding( rotatorSigner: NostrSigner, communityId: HexKey, priorRoot: ByteArray, newRoot: ByteArray, + newControlRoot: ByteArray, rootEpoch: Long, priorControlWraps: List, - priorControlKey: GroupKey, + priorControlKeys: ControlPlaneKeys, recipientsXOnly: List, + staffXOnly: Set, createdAt: Long, ): RefoundingBuild = ConcordRefounding.build( @@ -488,24 +550,29 @@ object ConcordActions { communityId = communityId.hexToByteArray(), priorRoot = priorRoot, newRoot = newRoot, + newControlRoot = newControlRoot, rootEpoch = rootEpoch, priorControlWraps = priorControlWraps, - priorControlKey = priorControlKey, + priorControlKeys = priorControlKeys, recipientsXOnly = recipientsXOnly, + staffXOnly = staffXOnly, createdAt = createdAt, ) /** * Receives an inbound base rotation for the member behind [recipientSigner]: * finds the delivered new root across the buffered kind-3303 [wraps], verifying - * scope, epoch and continuity against the [priorRoot] the member holds. Returns - * the new root + rotator (for the caller to authorize) or null if not re-keyed. + * scope, epoch and continuity against the [priorRoot] the member holds — and, + * on a staff blob, that the delivered `control_root` derives to the delivered + * `control_pk` (CORD-06 §1). Returns the new root + Control keys + rotator + * (for the caller to authorize) or null if not re-keyed. */ suspend fun openBaseRekey( wraps: List, baseRekey: GroupKey, recipientSigner: NostrSigner, + communityId: HexKey, priorRoot: ByteArray, rootEpoch: Long, - ): ReceivedRefounding? = ConcordRefounding.findNewRoot(wraps, baseRekey, recipientSigner, priorRoot, rootEpoch) + ): ReceivedRefounding? = ConcordRefounding.findNewRoot(wraps, baseRekey, recipientSigner, communityId.hexToByteArray(), priorRoot, rootEpoch) } diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordModeration.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordModeration.kt index 3e3e9b1de0..e5ba1e55d1 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordModeration.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordModeration.kt @@ -25,14 +25,16 @@ import com.vitorpamplona.quartz.concord.cord04Roles.AuthorityCitation import com.vitorpamplona.quartz.concord.cord04Roles.AuthorityResolver import com.vitorpamplona.quartz.concord.cord04Roles.ChannelEntity import com.vitorpamplona.quartz.concord.cord04Roles.ConcordJson +import com.vitorpamplona.quartz.concord.cord04Roles.ConcordPermissions import com.vitorpamplona.quartz.concord.cord04Roles.ControlEdition import com.vitorpamplona.quartz.concord.cord04Roles.ControlEditionBuilder import com.vitorpamplona.quartz.concord.cord04Roles.ControlEntityKind +import com.vitorpamplona.quartz.concord.cord04Roles.ControlRootWrap import com.vitorpamplona.quartz.concord.cord04Roles.GrantEntity import com.vitorpamplona.quartz.concord.cord04Roles.MetadataEntity import com.vitorpamplona.quartz.concord.cord04Roles.RoleEntity import com.vitorpamplona.quartz.concord.crypto.ConcordKeyDerivation -import com.vitorpamplona.quartz.concord.crypto.GroupKey +import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys import com.vitorpamplona.quartz.concord.envelope.ConcordStreamEnvelope import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey @@ -97,7 +99,7 @@ object ConcordModeration { private suspend fun wrap( actor: NostrSigner, - controlPlane: GroupKey, + controlPlane: ControlPlaneKeys, kind: ControlEntityKind, entityId: ByteArray, version: Long, @@ -116,7 +118,7 @@ object ConcordModeration { */ suspend fun defineRole( actor: NostrSigner, - controlPlane: GroupKey, + controlPlane: ControlPlaneKeys, roleId: ByteArray, role: RoleEntity, current: List, @@ -138,7 +140,7 @@ object ConcordModeration { */ suspend fun defineChannel( actor: NostrSigner, - controlPlane: GroupKey, + controlPlane: ControlPlaneKeys, channelId: ByteArray, channel: ChannelEntity, current: List, @@ -159,7 +161,7 @@ object ConcordModeration { */ suspend fun editMetadata( actor: NostrSigner, - controlPlane: GroupKey, + controlPlane: ControlPlaneKeys, communityId: ByteArray, metadata: MetadataEntity, current: List, @@ -172,10 +174,18 @@ object ConcordModeration { return wrap(actor, controlPlane, ControlEntityKind.METADATA, communityId, version, prev, content, createdAt, citation) } - /** Grants [member] exactly [roleIds] (replaces their prior grant). Empty list revokes all roles. */ + /** + * Grants [member] exactly [roleIds] (replaces their prior grant). Empty list revokes all roles. + * + * A Grant that first makes its member **staff** must deliver the current + * `control_root` in the same edition (CORD-04 §3): pass [controlWrap] built with + * [ControlRootWrap.build] for the current epoch. A current staffer may also + * re-issue a Grant with a fresh wrap to re-deliver (a lost key, a superseded + * head). Leave null for a non-staff grant, a revoke, or a legacy community. + */ suspend fun grant( actor: NostrSigner, - controlPlane: GroupKey, + controlPlane: ControlPlaneKeys, communityId: ByteArray, member: HexKey, roleIds: List, @@ -183,17 +193,63 @@ object ConcordModeration { createdAt: Long, citation: AuthorityCitation? = null, owner: HexKey, + controlWrap: String? = null, ): Event { val entityId = ConcordKeyDerivation.grantCoordinate(communityId, member.hexToByteArray()) val (version, prev) = versioning(current, entityId, owner) - val content = ConcordJson.instance.encodeToString(GrantEntity.serializer(), GrantEntity(member = member, roleIds = roleIds)) + val content = ConcordJson.instance.encodeToString(GrantEntity.serializer(), GrantEntity(member = member, roleIds = roleIds, controlWrap = controlWrap)) return wrap(actor, controlPlane, ControlEntityKind.GRANT, entityId, version, prev, content, createdAt, citation) } + /** + * [grant], deciding the staff delivery for the caller: when [roleIds] hands the + * member any Control-writing bit ([ConcordPermissions.STAFF_BITS]) and we hold the + * [controlRoot] to deliver, the edition carries a `control_wrap` fresh for [epoch] + * (CORD-04 §3). A non-staff grant, a revoke, a legacy community, or a granter who + * does not hold the secret all produce a plain Grant. + * + * The role bits are read off the same authority-gated fold the readers use, so a + * role a reader would drop never triggers a delivery — and a role we cannot resolve + * yet (its edition unseen) conservatively doesn't either, which the spec's re-issue + * path covers: any current staffer MAY re-issue a Grant with a fresh wrap. + */ + suspend fun grantWithStaffDelivery( + actor: NostrSigner, + controlPlane: ControlPlaneKeys, + communityId: ByteArray, + member: HexKey, + roleIds: List, + current: List, + createdAt: Long, + citation: AuthorityCitation? = null, + owner: HexKey, + controlRoot: ByteArray?, + epoch: Long, + ): Event { + val wrap = + if (controlRoot != null && makesStaff(roleIds, current, owner)) { + ControlRootWrap.build(actor, member, epoch, controlRoot) + } else { + null + } + return grant(actor, controlPlane, communityId, member, roleIds, current, createdAt, citation, owner, wrap) + } + + /** True when any of [roleIds] resolves to a role carrying a Control-writing bit (CORD-04 §3). */ + fun makesStaff( + roleIds: List, + current: List, + owner: HexKey, + ): Boolean { + if (roleIds.isEmpty()) return false + val roles = AuthorityResolver.resolve(current, owner).roles() + return roleIds.any { roles[it]?.permissionBits()?.hasAny(ConcordPermissions.STAFF_BITS) == true } + } + /** Adds [member] to the banlist (union with the current head). */ suspend fun ban( actor: NostrSigner, - controlPlane: GroupKey, + controlPlane: ControlPlaneKeys, communityId: ByteArray, member: HexKey, current: List, @@ -205,7 +261,7 @@ object ConcordModeration { /** Removes [member] from the banlist. */ suspend fun unban( actor: NostrSigner, - controlPlane: GroupKey, + controlPlane: ControlPlaneKeys, communityId: ByteArray, member: HexKey, current: List, @@ -231,7 +287,7 @@ object ConcordModeration { private suspend fun setBanlist( actor: NostrSigner, - controlPlane: GroupKey, + controlPlane: ControlPlaneKeys, communityId: ByteArray, banned: Set, current: List, diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordSubscriptionPlanner.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordSubscriptionPlanner.kt index 52b4436475..213121d0d4 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordSubscriptionPlanner.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordSubscriptionPlanner.kt @@ -72,17 +72,20 @@ object ConcordSubscriptionPlanner { entries.flatMap { e -> val communityId = e.id.hexToByteArray() val relays = normalize(e.relays) - val cp = ConcordActions.controlPlane(e.root.hexToByteArray(), communityId, e.rootEpoch) + // The address is the held `control_pk` on a split epoch and the legacy derivation + // otherwise (CORD-02 §5) — a member can never derive the former, so it is read off + // the entry, per epoch, exactly as it was delivered. + val cp = ConcordActions.controlPlaneKeysFor(e) val historical = e.heldRoots .filter { it.epoch < e.rootEpoch } .sortedByDescending { it.epoch } .take(ConcordActions.MAX_BACKFILL_EPOCHS) .mapNotNull { held -> - val key = runCatching { ConcordActions.controlPlane(held.key.hexToByteArray(), communityId, held.epoch) }.getOrNull() ?: return@mapNotNull null - ConcordPlaneSub(channelId = null, pubKeyHex = key.publicKeyHex, relays = relays) + val keys = runCatching { ConcordActions.controlPlaneKeys(held.key.hexToByteArray(), communityId, held.epoch, held.controlPk, held.controlRoot) }.getOrNull() ?: return@mapNotNull null + ConcordPlaneSub(channelId = null, pubKeyHex = keys.address, relays = relays) } - listOf(ConcordPlaneSub(channelId = null, pubKeyHex = cp.publicKeyHex, relays = relays)) + historical + listOf(ConcordPlaneSub(channelId = null, pubKeyHex = cp.address, relays = relays)) + historical } /** diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt index 49acfd9b1e..32869407e1 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt @@ -29,6 +29,7 @@ import com.vitorpamplona.quartz.concord.cord03Channels.ChannelChat import com.vitorpamplona.quartz.concord.cord04Roles.ControlEdition import com.vitorpamplona.quartz.concord.cord04Roles.EditionFold import com.vitorpamplona.quartz.concord.cord04Roles.EntityFloor +import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys import com.vitorpamplona.quartz.concord.crypto.GroupKey import com.vitorpamplona.quartz.concord.envelope.ConcordStreamEnvelope import com.vitorpamplona.quartz.nip01Core.core.Event @@ -103,7 +104,12 @@ class ConcordCommunitySession( private val root = entry.root.hexToByteArray() private val communityIdBytes = entry.id.hexToByteArray() - private val controlPlaneKey: GroupKey = ConcordActions.controlPlane(root, communityIdBytes, entry.rootEpoch) + /** + * The Control Plane as this account holds it (CORD-02 §5): split when the entry + * carries a `control_pk` (plus the write key when this account is staff and + * holds the `control_root`), legacy single-key otherwise. + */ + private val controlKeys: ControlPlaneKeys = ConcordActions.controlPlaneKeysFor(entry) /** The Guestbook Plane at this epoch — where member join/leave motions ride (CORD-02 §5). */ private val guestbookKey: GroupKey = ConcordActions.guestbookPlane(root, communityIdBytes, entry.rootEpoch) @@ -116,7 +122,7 @@ class ConcordCommunitySession( private val nextBaseRekeyKey: GroupKey = ConcordActions.nextBaseRekeyPlane(root, communityIdBytes, entry.rootEpoch) /** The Control Plane stream address to subscribe to (known from the entry alone). */ - val controlPlaneAddress: HexKey get() = controlPlaneKey.publicKeyHex + val controlPlaneAddress: HexKey get() = controlKeys.address /** The Guestbook Plane stream address to subscribe to (known from the entry alone). */ val guestbookAddress: HexKey get() = guestbookKey.publicKeyHex @@ -136,14 +142,16 @@ class ConcordCommunitySession( * `heldRoots` is already persisted in the kind-13302 community list, that memory * survives a process restart without any new storage. */ - private val historicalControlKeys: Map> = + private val historicalControlKeys: Map> = entry.heldRoots .filter { it.epoch < entry.rootEpoch } .sortedByDescending { it.epoch } .take(ConcordActions.MAX_BACKFILL_EPOCHS) .mapNotNull { held -> - val key = runCatching { ConcordActions.controlPlane(held.key.hexToByteArray(), communityIdBytes, held.epoch) }.getOrNull() ?: return@mapNotNull null - key.publicKeyHex to (key to held.epoch) + // A held split epoch's address is the banked control_pk (held, never derivable); + // a held legacy epoch derives its address from the root as it always did. + val keys = runCatching { ConcordActions.controlPlaneKeys(held.key.hexToByteArray(), communityIdBytes, held.epoch, held.controlPk, held.controlRoot) }.getOrNull() ?: return@mapNotNull null + keys.address to (keys to held.epoch) }.toMap() /** The prior-epoch Control Plane addresses to subscribe to, so the rollback floor can be rebuilt. */ @@ -289,13 +297,19 @@ class ConcordCommunitySession( * NOT included here: mixing them into the shared control/channel AUTH set starved the * subscription on relays that gate a REQ on stream-key AUTH (control stopped folding, * channels went empty). They AUTH on their own isolated subscription instead. + * + * A **split** Control Plane epoch contributes a key only when this account is staff + * (CORD-02 §2): a regular member holds the `control_pk` but not the secret behind it, + * so it cannot answer an AUTH challenge as the plane — which is the write-restriction + * working as designed, not a gap to paper over. A legacy epoch still contributes, its + * member-held derivation being address and signer at once (CORD-02 §5). */ fun streamKeys(): List = lock.withLock { - listOf(controlPlaneKey) + + listOfNotNull(controlKeys.signer) + // Prior-epoch Control Planes: the anti-rollback floor is folded from them, so the // gated relays must serve their wraps too. - historicalControlKeys.values.map { it.first } + + historicalControlKeys.values.mapNotNull { it.first.signer } + channelKeysByAddress.values.map { it.second } + // Prior-epoch channel stream keys so the gated relays serve their older wraps too. historicalChannelKeysByAddress.values.map { it.second } @@ -305,13 +319,17 @@ class ConcordCommunitySession( fun auxStreamKeys(): List = listOf(guestbookKey, nextBaseRekeyKey) /** The community's current Control Plane editions — the input a moderation edition chains onto. */ - fun controlEditions(): List = lock.withLock { editionsLocked(controlWraps.values.toList(), controlPlaneKey) } + fun controlEditions(): List = lock.withLock { editionsLocked(controlWraps.values.toList(), controlKeys) } /** The raw Control Plane wraps buffered so far — the input a Refounding compacts (CORD-06 §3). */ fun controlPlaneWraps(): List = lock.withLock { controlWraps.values.toList() } - /** The Control Plane key, for authoring moderation editions. */ - fun controlPlaneKey(): GroupKey = controlPlaneKey + /** + * The Control Plane keys as this account holds them, for authoring moderation + * editions. [ControlPlaneKeys.canWrite] is false for a regular member on a split + * epoch (CORD-02 §2) — the caller must not attempt to publish an edition then. + */ + fun controlPlaneKeys(): ControlPlaneKeys = controlKeys /** This account's standing, from the current fold. */ fun membership(): ConcordMembership { @@ -447,7 +465,7 @@ class ConcordCommunitySession( val wraps = controlWraps.values.toList() val folded = ConcordCommunityState.fold( - editionsLocked(wraps, controlPlaneKey), + editionsLocked(wraps, controlKeys), entry.owner, controlFloorsLocked(), ) @@ -486,13 +504,13 @@ class ConcordCommunitySession( */ private fun editionsLocked( wraps: Collection, - planeKey: GroupKey, + planeKeys: ControlPlaneKeys, ): List = wraps.mapNotNull { wrap -> if (editionByWrapId.containsKey(wrap.id)) { editionByWrapId[wrap.id] } else { - val edition = ConcordStreamEnvelope.openOrNull(wrap, planeKey)?.let { ControlEdition.fromRumor(it.rumor) } + val edition = ConcordStreamEnvelope.openOrNull(wrap, planeKeys)?.let { ControlEdition.fromRumor(it.rumor) } editionByWrapId[wrap.id] = edition edition } diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordPlaneRegistry.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordPlaneRegistry.kt index 86699776e7..cd0cbe7979 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordPlaneRegistry.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordPlaneRegistry.kt @@ -20,12 +20,12 @@ */ package com.vitorpamplona.amethyst.commons.model.concord +import com.vitorpamplona.amethyst.commons.actions.ConcordActions import com.vitorpamplona.amethyst.commons.util.KmpLock import com.vitorpamplona.amethyst.commons.util.withLock import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEntry import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityState import com.vitorpamplona.quartz.concord.cord03Channels.ConcordChannelId -import com.vitorpamplona.quartz.concord.crypto.ConcordKeyDerivation import com.vitorpamplona.quartz.concord.crypto.GroupKey import com.vitorpamplona.quartz.concord.envelope.ConcordStreamEnvelope import com.vitorpamplona.quartz.concord.envelope.OpenedStreamEvent @@ -39,13 +39,31 @@ enum class ConcordPlaneKind { CHANNEL, } -/** A known Concord plane: its kind, community, optional channel, and the key to open its wraps. */ +/** + * A known Concord plane: its kind, community, optional channel, and what it takes + * to open its wraps — the stream [address] they must be authored by, and the + * [readConversationKey] their content decrypts under. + * + * The two are separate fields rather than one [GroupKey] because a split Control + * Plane separates them (CORD-01, Write-Restricted Streams): the address is the + * staff-held signer's pubkey, while the read key derives from the `community_root` + * every member holds. On a channel plane and a legacy Control Plane they are the + * two halves of the same key. + */ class ConcordPlane( val kind: ConcordPlaneKind, val communityId: HexKey, val channelId: ConcordChannelId?, - val key: GroupKey, -) + val address: HexKey, + val readConversationKey: ByteArray, +) { + constructor( + kind: ConcordPlaneKind, + communityId: HexKey, + channelId: ConcordChannelId?, + key: GroupKey, + ) : this(kind, communityId, channelId, key.publicKeyHex, key.conversationKey) +} /** The routed result of opening an inbound wrap that belonged to a known plane. */ class RoutedRumor( @@ -72,12 +90,18 @@ class ConcordPlaneRegistry { private val lock = KmpLock() private val planes = HashMap() - /** Registers every joined community's Control Plane address. Idempotent. */ + /** + * Registers every joined community's Control Plane address. Idempotent. + * + * On a split epoch the address is the entry's held `control_pk` and the wraps + * still decrypt under the `community_root`-derived read key (CORD-02 §5); on a + * legacy entry (no `control_pk`) both come from the old single derivation. + */ fun registerControlPlanes(entries: List) = lock.withLock { for (e in entries) { - val cp = ConcordKeyDerivation.controlPlaneKey(e.root.hexToByteArray(), e.id.hexToByteArray(), e.rootEpoch) - planes[cp.publicKeyHex] = ConcordPlane(ConcordPlaneKind.CONTROL, e.id, null, cp) + val cp = ConcordActions.controlPlaneKeysFor(e) + planes[cp.address] = ConcordPlane(ConcordPlaneKind.CONTROL, e.id, null, cp.address, cp.readKey.conversationKey) } } @@ -106,7 +130,7 @@ class ConcordPlaneRegistry { */ fun route(wrap: Event): RoutedRumor? { val plane = planeFor(wrap.pubKey) ?: return null - val opened = ConcordStreamEnvelope.openOrNull(wrap, plane.key) ?: return null + val opened = ConcordStreamEnvelope.openOrNull(wrap, plane.address, plane.readConversationKey) ?: return null return RoutedRumor(plane, opened) } diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActionsTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActionsTest.kt index 15c73f51d0..a5d8ff772d 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActionsTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActionsTest.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.amethyst.commons.actions +import com.vitorpamplona.quartz.nip01Core.core.toHexKey import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal import kotlinx.coroutines.test.runTest @@ -63,6 +64,9 @@ class ConcordActionsTest { rootEpoch = community.rootEpoch, name = "Nostrichs", relays = listOf("wss://r.example"), + // Without this the joiner has no Control Plane address to fold at — the + // bundle is the only place it can come from (CORD-05 §1). + controlPk = community.controlPkHex, ) val minted = ConcordActions.mintInviteLink("https://vector.chat", invite, createdAt = 1L) @@ -103,29 +107,47 @@ class ConcordActionsTest { val carol = NostrSignerInternal(KeyPair()) // removed val newRoot = ByteArray(32) { 0x33 } + // A fresh control_root is minted beside the new root at every Refounding (CORD-02 §2). + val newControlRoot = ByteArray(32) { 0x44 } val build = ConcordActions.buildRefounding( rotatorSigner = owner, communityId = community.communityIdHex, priorRoot = community.communityRoot, newRoot = newRoot, + newControlRoot = newControlRoot, rootEpoch = community.rootEpoch, priorControlWraps = community.genesisWraps, - priorControlKey = community.controlPlane, + priorControlKeys = community.controlPlane, recipientsXOnly = listOf(owner.pubKey, alice.pubKey), + // Only the owner is staff, so only the owner's blob carries the secret. + staffXOnly = setOf(owner.pubKey), createdAt = 5L, ) val baseRekey = ConcordActions.nextBaseRekeyPlane(community.communityRoot, community.communityId, community.rootEpoch) - val aliceGot = ConcordActions.openBaseRekey(build.rekeyWraps, baseRekey, alice, community.communityRoot, community.rootEpoch) - val carolGot = ConcordActions.openBaseRekey(build.rekeyWraps, baseRekey, carol, community.communityRoot, community.rootEpoch) + val aliceGot = ConcordActions.openBaseRekey(build.rekeyWraps, baseRekey, alice, community.communityIdHex, community.communityRoot, community.rootEpoch) + val carolGot = ConcordActions.openBaseRekey(build.rekeyWraps, baseRekey, carol, community.communityIdHex, community.communityRoot, community.rootEpoch) assertNotNull(aliceGot) assertEquals(community.rootEpoch + 1, aliceGot.newEpoch) assertTrue(carolGot == null) - // The compacted Control Plane folds identically under the new root. - val newControl = ConcordActions.controlPlane(aliceGot.newRoot, community.communityId, aliceGot.newEpoch) + // Alice is a plain member: her blob carries the new control_pk to read with, never the + // secret to write with (CORD-06 §1). + val deliveredControlPk = aliceGot.newControlPk + assertNotNull(deliveredControlPk) + assertTrue(aliceGot.newControlRoot == null, "a member's base blob must not carry the write key") + + // The compacted Control Plane folds identically at the new epoch, opened the way a + // member does: the delivered address plus the derived read key. + val newControl = + ConcordActions.controlPlaneKeys( + communityRoot = aliceGot.newRoot, + communityId = community.communityId, + rootEpoch = aliceGot.newEpoch, + controlPk = deliveredControlPk.toHexKey(), + ) val state = ConcordActions.foldCommunity(build.controlWraps, newControl, community.ownerPubKey) assertEquals("Test", state.metadata?.name) assertTrue(state.channels.isNotEmpty()) diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordModerationTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordModerationTest.kt index 77ed81f1ae..c5ae3af6a9 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordModerationTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordModerationTest.kt @@ -22,10 +22,13 @@ package com.vitorpamplona.amethyst.commons.actions import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityFactory import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityState +import com.vitorpamplona.quartz.concord.cord04Roles.ConcordJson import com.vitorpamplona.quartz.concord.cord04Roles.ConcordPermissions import com.vitorpamplona.quartz.concord.cord04Roles.ControlEdition import com.vitorpamplona.quartz.concord.cord04Roles.ControlEntityKind +import com.vitorpamplona.quartz.concord.cord04Roles.ControlRootWrap import com.vitorpamplona.quartz.concord.cord04Roles.EditionFold +import com.vitorpamplona.quartz.concord.cord04Roles.GrantEntity import com.vitorpamplona.quartz.concord.cord04Roles.RoleEntity import com.vitorpamplona.quartz.nip01Core.core.toHexKey import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair @@ -34,6 +37,8 @@ import kotlinx.coroutines.test.runTest import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull import kotlin.test.assertTrue class ConcordModerationTest { @@ -214,4 +219,81 @@ class ConcordModerationTest { assertTrue(state.authority.isBanned(troll.pubKey), "the forged unban must not free the troll") assertTrue(state.authority.isBanned(stranger.pubKey)) } + + /** + * The staff-delivery decision (CORD-04 §3): a Grant that hands out a Control-writing + * bit must carry the `control_root` in the same edition (`control_wrap`), and every + * other shape of Grant must not — a non-staff role, a revoke, an unresolvable role, + * or a granter who holds no secret to deliver. + */ + @Test + fun aStaffMakingGrantDeliversTheControlRootAndNothingElseDoes() = + runTest { + val community = ConcordCommunityFactory.create(owner, "Nostrichs", createdAt = 1L, relays = listOf("wss://r.example")) + val cp = community.controlPlane + val communityId = community.communityId + val editions = ConcordActions.controlEditions(community.genesisWraps, cp).toMutableList() + + // A staff role (BAN writes Control editions) and a non-staff one (KICK writes the Guestbook). + val staffRoleId = ByteArray(32) { 0x51 } + val staffRoleIdHex = staffRoleId.toHexKey() + val kickRoleId = ByteArray(32) { 0x52 } + val kickRoleIdHex = kickRoleId.toHexKey() + editions += + ConcordActions.controlEditions( + listOf( + ConcordModeration.defineRole(owner, cp, staffRoleId, RoleEntity(name = "Mod", position = 2, permissions = ConcordPermissions.of(ConcordPermissions.BAN).toWire()), editions, createdAt = 2L, owner = community.ownerPubKey), + ), + cp, + ) + editions += + ConcordActions.controlEditions( + listOf( + ConcordModeration.defineRole(owner, cp, kickRoleId, RoleEntity(name = "Bouncer", position = 3, permissions = ConcordPermissions.of(ConcordPermissions.KICK).toWire()), editions, createdAt = 3L, owner = community.ownerPubKey), + ), + cp, + ) + + assertTrue(ConcordModeration.makesStaff(listOf(staffRoleIdHex), editions, community.ownerPubKey)) + assertFalse(ConcordModeration.makesStaff(listOf(kickRoleIdHex), editions, community.ownerPubKey)) + assertFalse(ConcordModeration.makesStaff(emptyList(), editions, community.ownerPubKey), "a revoke hands out nothing") + assertFalse(ConcordModeration.makesStaff(listOf("ee".repeat(32)), editions, community.ownerPubKey), "an unresolvable role must not trigger a delivery") + + suspend fun grantEntity( + roleIds: List, + controlRoot: ByteArray?, + ): GrantEntity { + val wrap = + ConcordModeration.grantWithStaffDelivery( + actor = owner, + controlPlane = cp, + communityId = communityId, + member = admin.pubKey, + roleIds = roleIds, + current = editions, + createdAt = 4L, + owner = community.ownerPubKey, + controlRoot = controlRoot, + epoch = community.rootEpoch, + ) + val edition = ConcordActions.controlEditions(listOf(wrap), cp).single() + return ConcordJson.decodeOrNull(edition.content)!! + } + + // A staff-making Grant carries the wrap; the promotee opens it and it derives to the + // control_pk every member holds for the epoch — the adoption gate (CORD-04 §3). + val staffGrant = grantEntity(listOf(staffRoleIdHex), community.controlRoot) + val controlWrap = staffGrant.controlWrap + assertNotNull(controlWrap, "a staff-making Grant must deliver the write key in the same edition") + val opened = ControlRootWrap.openOrNull(controlWrap, admin, owner.pubKey) + assertNotNull(opened, "the promotee must be able to open the delivery") + assertEquals(community.rootEpoch, opened.epoch, "the wrap must be fresh for the current epoch") + assertTrue(ControlRootWrap.derivesTo(opened.controlRoot, communityId, community.rootEpoch, community.controlPkHex)) + + // Every other shape is a plain Grant. + assertNull(grantEntity(listOf(kickRoleIdHex), community.controlRoot).controlWrap, "a Guestbook-writing role needs no key") + assertNull(grantEntity(emptyList(), community.controlRoot).controlWrap, "a revoke delivers nothing") + assertNull(grantEntity(listOf("ee".repeat(32)), community.controlRoot).controlWrap, "an unresolved role conservatively delivers nothing") + assertNull(grantEntity(listOf(staffRoleIdHex), controlRoot = null).controlWrap, "no held secret, no delivery (legacy community or keyless granter)") + } } diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordSubscriptionPlannerTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordSubscriptionPlannerTest.kt index d7e02a724e..0412057a24 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordSubscriptionPlannerTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordSubscriptionPlannerTest.kt @@ -48,6 +48,8 @@ class ConcordSubscriptionPlannerTest { ownerSalt = community.ownerSalt.toHexKey(), root = community.communityRoot.toHexKey(), rootEpoch = community.rootEpoch, + controlPk = community.controlPkHex, + controlRoot = community.controlRoot.toHexKey(), heldRoots = listOf( com.vitorpamplona.quartz.concord.cord02Community @@ -78,6 +80,8 @@ class ConcordSubscriptionPlannerTest { ownerSalt = community.ownerSalt.toHexKey(), root = community.communityRoot.toHexKey(), rootEpoch = community.rootEpoch, + controlPk = community.controlPkHex, + controlRoot = community.controlRoot.toHexKey(), relays = listOf("wss://r.example"), name = "Nostrichs", ) @@ -85,7 +89,7 @@ class ConcordSubscriptionPlannerTest { // Control-plane sub address must equal the derived control plane pk. val controlSubs = ConcordSubscriptionPlanner.controlPlaneSubs(listOf(entry)) assertEquals(1, controlSubs.size) - assertEquals(community.controlPlane.publicKeyHex, controlSubs[0].pubKeyHex) + assertEquals(community.controlPlane.address, controlSubs[0].pubKeyHex) assertTrue(controlSubs[0].channelId == null) // Channel-plane subs cover the folded #general channel. @@ -103,7 +107,7 @@ class ConcordSubscriptionPlannerTest { assertEquals(1, filters.size) // single relay val filter = filters.values.first().first() assertEquals(listOf(1059), filter.kinds) - assertTrue(filter.authors!!.contains(community.controlPlane.publicKeyHex)) + assertTrue(filter.authors!!.contains(community.controlPlane.address)) assertTrue(filter.authors!!.contains(general.pubKeyHex)) } @@ -118,6 +122,8 @@ class ConcordSubscriptionPlannerTest { ownerSalt = community.ownerSalt.toHexKey(), root = community.communityRoot.toHexKey(), rootEpoch = community.rootEpoch, + controlPk = community.controlPkHex, + controlRoot = community.controlRoot.toHexKey(), relays = listOf("wss://r.example"), name = "Nostrichs", ) @@ -147,6 +153,8 @@ class ConcordSubscriptionPlannerTest { ownerSalt = community.ownerSalt.toHexKey(), root = community.communityRoot.toHexKey(), rootEpoch = community.rootEpoch, + controlPk = community.controlPkHex, + controlRoot = community.controlRoot.toHexKey(), relays = listOf("wss://r.example"), name = "Nostrichs", ) @@ -176,6 +184,8 @@ class ConcordSubscriptionPlannerTest { ownerSalt = community.ownerSalt.toHexKey(), root = community.communityRoot.toHexKey(), rootEpoch = community.rootEpoch, + controlPk = community.controlPkHex, + controlRoot = community.controlRoot.toHexKey(), relays = listOf("wss://r.example"), name = "Nostrichs", ) @@ -189,7 +199,7 @@ class ConcordSubscriptionPlannerTest { assertEquals(relay, filters[0].relay) assertEquals(listOf(1059, 21059), filters[0].filter.kinds) assertEquals(1234L, filters[0].filter.since) - assertTrue(filters[0].filter.authors!!.contains(community.controlPlane.publicKeyHex)) + assertTrue(filters[0].filter.authors!!.contains(community.controlPlane.address)) // No planes resolve to a relay -> nothing to subscribe. assertNull(ConcordSubscriptionPlanner.relayBasedFilters(emptyList(), null)) @@ -210,12 +220,14 @@ class ConcordSubscriptionPlannerTest { ownerSalt = community.ownerSalt.toHexKey(), root = community.communityRoot.toHexKey(), rootEpoch = community.rootEpoch, + controlPk = community.controlPkHex, + controlRoot = community.controlRoot.toHexKey(), relays = listOf("wss://r.example"), name = "Nostrichs", ) val state = ConcordActions.foldCommunity(community.genesisWraps, community.controlPlane, community.ownerPubKey) - val controlPk = community.controlPlane.publicKeyHex + val controlPk = community.controlPlane.address val guestbookPk = ConcordActions.guestbookPlane(community.communityRoot, community.communityId, community.rootEpoch).publicKeyHex val generalPk = ConcordActions.publicChannel(community.communityRoot, community.generalChannelId, community.rootEpoch).publicKeyHex diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySessionTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySessionTest.kt index 7f7882a14d..95e7721ff8 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySessionTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySessionTest.kt @@ -53,6 +53,8 @@ class ConcordCommunitySessionTest { ownerSalt = community.ownerSalt.toHexKey(), root = community.communityRoot.toHexKey(), rootEpoch = community.rootEpoch, + controlPk = community.controlPkHex, + controlRoot = community.controlRoot.toHexKey(), heldRoots = listOf(HeldRoot(priorEpoch, priorRoot.toHexKey())), relays = listOf("wss://r.example"), name = "Nostrichs", @@ -98,13 +100,15 @@ class ConcordCommunitySessionTest { ownerSalt = community.ownerSalt.toHexKey(), root = community.communityRoot.toHexKey(), rootEpoch = community.rootEpoch, + controlPk = community.controlPkHex, + controlRoot = community.controlRoot.toHexKey(), relays = listOf("wss://r.example"), name = "Nostrichs", ) val captured = mutableListOf>() val session = ConcordCommunitySession(entry, owner.pubKey) { communityId, channelIdHex, rumor, _ -> captured += Triple(communityId, channelIdHex, rumor) } - assertEquals(community.controlPlane.publicKeyHex, session.controlPlaneAddress) + assertEquals(community.controlPlane.address, session.controlPlaneAddress) // Feed the genesis control wraps → state folds, channels + membership resolve. A fold is // STRUCTURAL (it moves the subscription set), so it's allowed to bump the revision. diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordPlaneRegistryTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordPlaneRegistryTest.kt index 6f395de18e..25b14d7fd1 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordPlaneRegistryTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordPlaneRegistryTest.kt @@ -48,6 +48,8 @@ class ConcordPlaneRegistryTest { ownerSalt = community.ownerSalt.toHexKey(), root = community.communityRoot.toHexKey(), rootEpoch = community.rootEpoch, + controlPk = community.controlPkHex, + controlRoot = community.controlRoot.toHexKey(), relays = listOf("wss://r.example"), name = "Nostrichs", ) diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordRollbackFloorTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordRollbackFloorTest.kt index 978b17130f..d3c1a50a75 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordRollbackFloorTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordRollbackFloorTest.kt @@ -28,6 +28,7 @@ import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityState import com.vitorpamplona.quartz.concord.cord02Community.HeldRoot import com.vitorpamplona.quartz.concord.cord04Roles.MetadataEntity import com.vitorpamplona.quartz.concord.cord06Rekey.ConcordRefounding +import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys import com.vitorpamplona.quartz.nip01Core.core.toHexKey import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal @@ -73,7 +74,10 @@ class ConcordRollbackFloorTest { // silently dropped. Every wrap it publishes is a genuine, owner-signed edition. val newRoot = ByteArray(32) { 0x33 } val newEpoch = community.rootEpoch + 1 - val newControl = ConcordActions.controlPlane(newRoot, community.communityId, newEpoch) + // The rotator mints a fresh control_root beside the new root (CORD-02 §2), so the + // new epoch's plane is split and addressed by the derived signer, not the root. + val newControlRoot = ByteArray(32) { 0x44 } + val newControl = ControlPlaneKeys.forStaff(newRoot, community.communityId, newEpoch, newControlRoot) val rolledBack = ConcordRefounding.compactControlPlane(community.genesisWraps, community.controlPlane, newControl) val entry = @@ -83,7 +87,11 @@ class ConcordRollbackFloorTest { ownerSalt = community.ownerSalt.toHexKey(), root = newRoot.toHexKey(), rootEpoch = newEpoch, - heldRoots = listOf(HeldRoot(community.rootEpoch, community.communityRoot.toHexKey())), + controlPk = newControl.address, + controlRoot = newControlRoot.toHexKey(), + // The prior epoch is banked with the address it was folded at: a split epoch's + // Control Plane can never be re-derived, only remembered (CORD-02 §2). + heldRoots = listOf(HeldRoot(community.rootEpoch, community.communityRoot.toHexKey(), community.controlPkHex, community.controlRoot.toHexKey())), relays = listOf("wss://r.example"), name = "Nostrichs", ) @@ -92,11 +100,11 @@ class ConcordRollbackFloorTest { // The prior epoch's Control Plane is subscribed and AUTHed for — that is where the floor // comes from, and without it the client has no memory to check the rotator against. assertTrue( - session.historicalControlPlaneAddresses().contains(community.controlPlane.publicKeyHex), + session.historicalControlPlaneAddresses().contains(community.controlPlane.address), "prior-epoch control plane not subscribed", ) assertTrue( - session.streamKeys().any { it.publicKeyHex == community.controlPlane.publicKeyHex }, + session.streamKeys().any { it.publicKeyHex == community.controlPlane.address }, "prior-epoch control plane not AUTHed", ) @@ -132,7 +140,10 @@ class ConcordRollbackFloorTest { val newRoot = ByteArray(32) { 0x33 } val newEpoch = community.rootEpoch + 1 - val newControl = ConcordActions.controlPlane(newRoot, community.communityId, newEpoch) + // The rotator mints a fresh control_root beside the new root (CORD-02 §2), so the + // new epoch's plane is split and addressed by the derived signer, not the root. + val newControlRoot = ByteArray(32) { 0x44 } + val newControl = ControlPlaneKeys.forStaff(newRoot, community.communityId, newEpoch, newControlRoot) // Honest: compacted from the FULL prior plane, so each entity's head (metadata v1) survives. val honest = ConcordRefounding.compactControlPlane(epoch0Wraps, community.controlPlane, newControl) @@ -143,7 +154,11 @@ class ConcordRollbackFloorTest { ownerSalt = community.ownerSalt.toHexKey(), root = newRoot.toHexKey(), rootEpoch = newEpoch, - heldRoots = listOf(HeldRoot(community.rootEpoch, community.communityRoot.toHexKey())), + controlPk = newControl.address, + controlRoot = newControlRoot.toHexKey(), + // The prior epoch is banked with the address it was folded at: a split epoch's + // Control Plane can never be re-derived, only remembered (CORD-02 §2). + heldRoots = listOf(HeldRoot(community.rootEpoch, community.communityRoot.toHexKey(), community.controlPkHex, community.controlRoot.toHexKey())), relays = listOf("wss://r.example"), name = "Nostrichs", ) @@ -167,7 +182,10 @@ class ConcordRollbackFloorTest { val community = ConcordCommunityFactory.create(owner, "Nostrichs", createdAt = 1L, relays = listOf("wss://r.example")) val newRoot = ByteArray(32) { 0x33 } val newEpoch = community.rootEpoch + 1 - val newControl = ConcordActions.controlPlane(newRoot, community.communityId, newEpoch) + // The rotator mints a fresh control_root beside the new root (CORD-02 §2), so the + // new epoch's plane is split and addressed by the derived signer, not the root. + val newControlRoot = ByteArray(32) { 0x44 } + val newControl = ControlPlaneKeys.forStaff(newRoot, community.communityId, newEpoch, newControlRoot) val compacted = ConcordRefounding.compactControlPlane(community.genesisWraps, community.controlPlane, newControl) val entry = @@ -177,6 +195,7 @@ class ConcordRollbackFloorTest { ownerSalt = community.ownerSalt.toHexKey(), root = newRoot.toHexKey(), rootEpoch = newEpoch, + controlPk = newControl.address, relays = listOf("wss://r.example"), name = "Nostrichs", ) diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionManagerTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionManagerTest.kt index 58b0c55253..10a771942e 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionManagerTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionManagerTest.kt @@ -47,6 +47,8 @@ class ConcordSessionManagerTest { ownerSalt = community.ownerSalt.toHexKey(), root = community.communityRoot.toHexKey(), rootEpoch = community.rootEpoch, + controlPk = community.controlPkHex, + controlRoot = community.controlRoot.toHexKey(), relays = listOf("wss://r.example"), name = name, ) @@ -61,7 +63,7 @@ class ConcordSessionManagerTest { testScheduler.runCurrent() // The joined community produced a session, and its control plane is in the subscribe set. - assertTrue(manager.subscribeAddresses().contains(alpha.controlPlane.publicKeyHex)) + assertTrue(manager.subscribeAddresses().contains(alpha.controlPlane.address)) val revAfterSync = manager.revision.value assertTrue(revAfterSync > 0) @@ -87,7 +89,7 @@ class ConcordSessionManagerTest { val beta = ConcordCommunityFactory.create(owner, "Beta", createdAt = 1L, relays = listOf("wss://r.example")) communities.value = listOf(entryFor(alpha, "Alpha"), entryFor(beta, "Beta")) testScheduler.runCurrent() - assertTrue(manager.subscribeAddresses().contains(beta.controlPlane.publicKeyHex)) + assertTrue(manager.subscribeAddresses().contains(beta.controlPlane.address)) // Alpha's fold survived the re-sync. assertEquals( "Alpha", @@ -114,7 +116,13 @@ class ConcordSessionManagerTest { // Before any fold, only the control-plane key must AUTH — and only on the community's relay. val beforeFold = manager.streamAuthSecretsFor(hosted).map { it.toHexKey() } - assertTrue(beforeFold.contains(alpha.controlPlane.secretKey.toHexKey())) + assertTrue( + beforeFold.contains( + alpha.controlPlane.signer!! + .secretKey + .toHexKey(), + ), + ) assertTrue(manager.streamAuthSecretsFor(elsewhere).isEmpty()) // relay-scoped // After the Control Plane folds, the #general channel key joins the AUTH set. @@ -122,7 +130,13 @@ class ConcordSessionManagerTest { testScheduler.runCurrent() val general = ConcordActions.publicChannel(alpha.communityRoot, alpha.generalChannelId, alpha.rootEpoch) val afterFold = manager.streamAuthSecretsFor(hosted).map { it.toHexKey() } - assertTrue(afterFold.contains(alpha.controlPlane.secretKey.toHexKey())) + assertTrue( + afterFold.contains( + alpha.controlPlane.signer!! + .secretKey + .toHexKey(), + ), + ) assertTrue(afterFold.contains(general.secretKey.toHexKey())) assertFalse(manager.streamAuthSecretsFor(elsewhere).any { it.toHexKey() == general.secretKey.toHexKey() }) } diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionRegistryTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionRegistryTest.kt index 0419520238..a8bedadc73 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionRegistryTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionRegistryTest.kt @@ -46,6 +46,8 @@ class ConcordSessionRegistryTest { ownerSalt = community.ownerSalt.toHexKey(), root = community.communityRoot.toHexKey(), rootEpoch = community.rootEpoch, + controlPk = community.controlPkHex, + controlRoot = community.controlRoot.toHexKey(), relays = listOf("wss://r.example"), name = name, ) @@ -66,8 +68,8 @@ class ConcordSessionRegistryTest { assertNotNull(registry.sessionFor(beta.communityIdHex)) // Both control-plane addresses are in the subscribe set from the entries alone. - assertTrue(registry.subscribeAddresses().contains(alpha.controlPlane.publicKeyHex)) - assertTrue(registry.subscribeAddresses().contains(beta.controlPlane.publicKeyHex)) + assertTrue(registry.subscribeAddresses().contains(alpha.controlPlane.address)) + assertTrue(registry.subscribeAddresses().contains(beta.controlPlane.address)) // A genesis control wrap routes to Alpha's session and folds it (STRUCTURAL). alpha.genesisWraps.forEach { assertEquals(ConcordIngestOutcome.STRUCTURAL_FOLD, registry.ingest(it)) } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityFactory.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityFactory.kt index f7d320c5ff..bf3dd5fb98 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityFactory.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityFactory.kt @@ -27,7 +27,7 @@ import com.vitorpamplona.quartz.concord.cord04Roles.ControlEditionBuilder import com.vitorpamplona.quartz.concord.cord04Roles.ControlEntityKind import com.vitorpamplona.quartz.concord.cord04Roles.MetadataEntity import com.vitorpamplona.quartz.concord.crypto.ConcordKeyDerivation -import com.vitorpamplona.quartz.concord.crypto.GroupKey +import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys import com.vitorpamplona.quartz.concord.envelope.ConcordStreamEnvelope import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray @@ -45,9 +45,12 @@ class NewConcordCommunity( val ownerPubKey: String, val ownerSalt: ByteArray, val communityRoot: ByteArray, + /** The staff write key (CORD-02 §2): held by the owner and staff only, never members. */ + val controlRoot: ByteArray, val rootEpoch: Long, val generalChannelId: ByteArray, - val controlPlane: GroupKey, + /** The split Control Plane keys (owner view: signer held, [ControlPlaneKeys.canWrite] true). */ + val controlPlane: ControlPlaneKeys, /** The kind-1059 control-plane wraps to publish (metadata + #general). */ val genesisWraps: List, /** The same editions as parsed [ControlEdition]s, for immediate local folding. */ @@ -55,6 +58,9 @@ class NewConcordCommunity( ) { val communityIdHex: String get() = communityId.toHexKey() val generalChannelIdHex: String get() = generalChannelId.toHexKey() + + /** The Control Plane address (`control_pk`) members hold to subscribe/verify/read. */ + val controlPkHex: String get() = controlPlane.address } /** @@ -63,9 +69,11 @@ class NewConcordCommunity( * `create` mints a random `owner_salt`, derives the self-certifying * `community_id = sha256("concord/community" ‖ owner ‖ salt)`, generates an * independent random `community_root` (so access can rotate while identity stays - * fixed), and emits exactly two owner-signed genesis editions — the community - * metadata and a public `#general` channel — as plaintext-seal wraps on the - * Control Plane at epoch 0. + * fixed) plus the staff-held `control_root` write key (CORD-02 §2), and emits + * exactly two owner-signed genesis editions — the community metadata and a public + * `#general` channel — as plaintext-seal wraps on the split Control Plane at + * epoch 0: signed by the `control_root`-derived signer, readable under the + * `community_root`-derived read key (CORD-02 §5). */ object ConcordCommunityFactory { const val GENERAL_CHANNEL_NAME = "general" @@ -82,9 +90,13 @@ object ConcordCommunityFactory { val ownerSalt = ConcordKeyDerivation.newOwnerSalt() val communityId = ConcordKeyDerivation.communityId(ownerXOnly, ownerSalt) val communityRoot = RandomInstance.bytes(32) + // The staff write key, minted alongside the community_root and kept deliberately + // apart from it (CORD-02 §2): members derive the Control read key from the root, + // but only control_root holders can mint a wrap at the plane's address. + val controlRoot = RandomInstance.bytes(32) val generalChannelId = RandomInstance.bytes(32) val rootEpoch = 0L - val controlPlane = ConcordKeyDerivation.controlPlaneKey(communityRoot, communityId, rootEpoch) + val controlPlane = ControlPlaneKeys.forStaff(communityRoot, communityId, rootEpoch, controlRoot) val metadataJson = ConcordJson.instance.encodeToString( @@ -127,6 +139,7 @@ object ConcordCommunityFactory { ownerPubKey = ownerSigner.pubKey, ownerSalt = ownerSalt, communityRoot = communityRoot, + controlRoot = controlRoot, rootEpoch = rootEpoch, generalChannelId = generalChannelId, controlPlane = controlPlane, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityList.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityList.kt index e77709ecdf..11ce25695e 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityList.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityList.kt @@ -44,6 +44,13 @@ val NoExtras: JsonObject = JsonObject(emptyMap()) /** * A past root key for a specific epoch, kept so historical channel keys stay derivable. * + * [controlPk] is that epoch's Control Plane address (CORD-02 §5) when the epoch was + * split — a `control_pk` is held, never derivable from the root, so without keeping + * it a prior split epoch's Control Plane could not be re-subscribed for the + * anti-rollback floor. Null for a legacy (pre-split) epoch, whose address the root + * still derives. A client-extension field on the wire (`control_pk` inside the held + * root object), preserved verbatim by extras-honoring peers. + * * [extras] carries any key another client wrote inside this held root that we do not * model, so a read-modify-write does not delete it (see [ConcordCommunityList]). */ @@ -51,6 +58,15 @@ val NoExtras: JsonObject = JsonObject(emptyMap()) class HeldRoot( val epoch: Long, val key: String, + val controlPk: String? = null, + /** + * That epoch's staff write key, banked only if we held it. It buys nothing on the + * wire — the epoch is frozen, so writing to it is pointless — but a relay that gates + * a plane's REQ on NIP-42 AUTH as the stream key will not serve the old Control Plane + * without it, and those wraps are what rebuild the anti-rollback floor. A member who + * was never staff simply has none, and reads the epoch wherever the relay allows. + */ + val controlRoot: String? = null, val extras: JsonObject = NoExtras, ) @@ -148,6 +164,21 @@ class ConcordCommunityListEntry( val ownerSalt: String, val root: String, val rootEpoch: Long = 0, + /** + * The Control Plane signer's pubkey at [rootEpoch] (CORD-02 §2/§8): read + * access, never write. Null = a legacy, pre-split epoch — fold Control at the + * legacy address (CORD-06 §3). + */ + val controlPk: String? = null, + /** + * The staff write key at [rootEpoch] (CORD-02 §2), when this account is staff + * and has adopted it (community creation, a Grant's `control_wrap`, or a + * 136-byte base rekey blob). Null for a plain member or a legacy epoch. Part + * of the join material since CORD-02 §8 ("plus `control_root` when the member + * holds it") — the List carries every private key its holder has, so a + * staffer's own devices can write. + */ + val controlRoot: String? = null, val heldRoots: List = emptyList(), val privateChannels: List = emptyList(), val relays: List = emptyList(), @@ -250,6 +281,8 @@ object ConcordCommunityList { private class WireHeldRoot( val epoch: Long, val key: String, + @SerialName("control_pk") val controlPk: String? = null, + @SerialName("control_root") val controlRoot: String? = null, @SerialName(EXTRAS) val extras: JsonObject = NoExtras, ) @@ -260,6 +293,8 @@ object ConcordCommunityList { @SerialName("owner_salt") val ownerSalt: String, @SerialName("community_root") val communityRoot: String, @SerialName("root_epoch") val rootEpoch: Long, + @SerialName("control_pk") val controlPk: String? = null, + @SerialName("control_root") val controlRoot: String? = null, val channels: List< @Serializable(WireChannelSerializer::class) WireChannel, @@ -315,10 +350,12 @@ object ConcordCommunityList { ownerSalt = ownerSalt, communityRoot = root, rootEpoch = rootEpoch, + controlPk = controlPk, + controlRoot = controlRoot, channels = privateChannels.map { WireChannel(it.channelId, it.key, it.epoch, it.name, it.extras) }, relays = relays, name = name, - heldRoots = heldRoots.map { WireHeldRoot(it.epoch, it.key, it.extras) }, + heldRoots = heldRoots.map { WireHeldRoot(it.epoch, it.key, it.controlPk, it.controlRoot, it.extras) }, extras = residue.currentExtras, ) @@ -333,7 +370,9 @@ object ConcordCommunityList { ownerSalt = ownerSalt, root = communityRoot, rootEpoch = rootEpoch, - heldRoots = heldRoots.map { HeldRoot(it.epoch, it.key, it.extras) }, + controlPk = controlPk, + controlRoot = controlRoot, + heldRoots = heldRoots.map { HeldRoot(it.epoch, it.key, it.controlPk, it.controlRoot, it.extras) }, privateChannels = channels.map { PrivateChannelKey(it.id, it.key, it.epoch, it.name, it.extras) }, relays = relays, name = name, @@ -499,19 +538,53 @@ object ConcordCommunityList { val byId = LinkedHashMap() for (e in a + b) { val existing = byId[e.id] - if (existing == null) { - byId[e.id] = e - } else if (e.rootEpoch > existing.rootEpoch) { - // A winner without an invite_ref inherits the loser's: that link is the only anchor - // stranded recovery has, and dropping it on a merge would disarm recovery forever. - byId[e.id] = if (e.inviteRef == null) e.withInviteRef(existing.inviteRef) else e - } else if (existing.inviteRef == null && e.inviteRef != null) { - byId[e.id] = existing.withInviteRef(e.inviteRef) - } + byId[e.id] = + when { + existing == null -> e + // A winner without an invite_ref inherits the loser's: that link is the only anchor + // stranded recovery has, and dropping it on a merge would disarm recovery forever. + // Control key material is NOT inherited across epochs — a lower epoch's + // control_pk/control_root is stale for the winner's planes (CORD-06). + e.rootEpoch > existing.rootEpoch -> e.copyWith(inviteRef = e.inviteRef ?: existing.inviteRef) + e.rootEpoch < existing.rootEpoch -> existing.copyWith(inviteRef = existing.inviteRef ?: e.inviteRef) + else -> + // Same epoch: both sides describe the same planes, so fill whatever key + // material either is missing — e.g. one device was promoted to staff + // (control_root via a Grant's control_wrap) or joined through a newer + // bundle (control_pk) while the other holds the invite anchor. + existing.copyWith( + controlPk = existing.controlPk ?: e.controlPk, + controlRoot = existing.controlRoot ?: e.controlRoot, + inviteRef = existing.inviteRef ?: e.inviteRef, + ) + } } return byId.values.toList() } + /** Field-selective copy (the entry is not a data class). Defaults keep every field. */ + private fun ConcordCommunityListEntry.copyWith( + controlPk: String? = this.controlPk, + controlRoot: String? = this.controlRoot, + inviteRef: String? = this.inviteRef, + ) = ConcordCommunityListEntry( + id = id, + owner = owner, + ownerSalt = ownerSalt, + root = root, + rootEpoch = rootEpoch, + controlPk = controlPk, + controlRoot = controlRoot, + heldRoots = heldRoots, + privateChannels = privateChannels, + relays = relays, + name = name, + addedAt = addedAt, + inviteRef = inviteRef, + excludedAtEpoch = excludedAtEpoch, + residue = residue, + ) + /** Copy of this entry carrying [inviteRef]; every other field untouched. */ fun ConcordCommunityListEntry.withInviteRef(inviteRef: String?) = ConcordCommunityListEntry( @@ -520,6 +593,32 @@ object ConcordCommunityList { ownerSalt = ownerSalt, root = root, rootEpoch = rootEpoch, + controlPk = controlPk, + controlRoot = controlRoot, + heldRoots = heldRoots, + privateChannels = privateChannels, + relays = relays, + name = name, + addedAt = addedAt, + inviteRef = inviteRef, + excludedAtEpoch = excludedAtEpoch, + residue = residue, + ) + + /** + * Copy of this entry holding [controlRoot] — the staff write key, adopted after a + * promotion delivered it (CORD-04 §3) or a base rotation carried it (CORD-06 §1). + * Every other field untouched. + */ + fun ConcordCommunityListEntry.withControlRoot(controlRoot: String?) = + ConcordCommunityListEntry( + id = id, + owner = owner, + ownerSalt = ownerSalt, + root = root, + rootEpoch = rootEpoch, + controlPk = controlPk, + controlRoot = controlRoot, heldRoots = heldRoots, privateChannels = privateChannels, relays = relays, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt index bff6b54070..039a63e7ca 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt @@ -99,6 +99,20 @@ data class AuthorityResolver private constructor( bit: Int, ): Boolean = !isBanned(pubKey) && effectivePermissions(pubKey).has(bit) + /** + * True if the member is **staff** (CORD-04 §3): the owner, or any non-banned + * holder of a Control-writing bit ([ConcordPermissions.STAFF_BITS]) — the set + * that holds the `control_root` (CORD-02 §2). + */ + fun isStaff(pubKey: String): Boolean = isOwner(pubKey) || (!isBanned(pubKey) && effectivePermissions(pubKey).hasAny(ConcordPermissions.STAFF_BITS)) + + /** + * The staff roster (lowercase hex): the owner plus every role-holder whose + * effective permissions carry a staff bit. This is the recipient set that gets + * the `control_root` in a base rotation's 136-byte blobs (CORD-06 §1). + */ + fun staffMembers(): Set = memberRoles.keys.filterTo(hashSetOf(ownerLower)) { isStaff(it) } + /** * Whether [actor] may take the action guarded by permission [bit] against * [target]. Requires: actor not banned, actor holds [bit], the owner is never diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ConcordPermissions.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ConcordPermissions.kt index 13782071cf..4753cdf64c 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ConcordPermissions.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ConcordPermissions.kt @@ -42,6 +42,9 @@ value class ConcordPermissions( /** True if every bit set in [other] is also set here (used for role-vs-actor checks). */ fun hasAll(other: ConcordPermissions): Boolean = (bits and other.bits) == other.bits + /** True if at least one bit set in [other] is also set here. */ + fun hasAny(other: ConcordPermissions): Boolean = (bits and other.bits) != 0uL + infix fun union(other: ConcordPermissions): ConcordPermissions = ConcordPermissions(bits or other.bits) fun with(bit: Int): ConcordPermissions = ConcordPermissions(bits or (1uL shl bit)) @@ -71,7 +74,21 @@ value class ConcordPermissions( const val VIEW_AUDIT_LOG = 8 const val MENTION_EVERYONE = 9 - // bits 10-12 reserved + // bits 10 and 12 reserved (MANAGE_EMOJI, MANAGE_EVENTS) + + const val PIN_MESSAGES = 11 + + /** + * The **staff** bits (CORD-04 §3): the six permissions whose actions land as + * Control Plane editions. A member holding any of them, plus always the + * owner, is staff — the set that holds the `control_root` (CORD-02 §2). + * `KICK` writes to the Guestbook and `MANAGE_MESSAGES` to Chat planes, so + * neither is here. The spec's list is **normative**: a future CORD + * introducing a permission whose actions are Control editions MUST amend it + * explicitly, so no implementation judges membership of the set for itself — + * extend this constant only when the spec's list changes. + */ + val STAFF_BITS: ConcordPermissions get() = of(MANAGE_ROLES, MANAGE_CHANNELS, MANAGE_METADATA, BAN, CREATE_INVITE, PIN_MESSAGES) fun of(vararg bits: Int): ConcordPermissions { var acc = 0uL diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlEntities.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlEntities.kt index 71696ba8c9..33c3aba220 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlEntities.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlEntities.kt @@ -92,11 +92,20 @@ data class RoleEntity( * A Grant's content (CORD-04): maps a [member] to the set of [roleIds] they hold. * Honored only if the granting actor outranks every assigned Role and the chain * terminates at the owner (see [AuthorityResolver]). + * + * A staff-making Grant also delivers the Control Plane write secret in + * [controlWrap] (CORD-04 §3): the `control_root` NIP-44-encrypted under the + * granter↔member pairwise conversation key, its plaintext the fixed-width 40 + * bytes `epoch_be[8] ‖ control_root[32]` (see [ControlRootWrap]). Delivery, never + * authority — every reader but the member treats it as opaque bytes, and the + * member adopts the secret only if it derives to the `control_pk` they hold for + * the named epoch. */ @Serializable data class GrantEntity( val member: String = "", @SerialName("role_ids") val roleIds: List = emptyList(), + @SerialName("control_wrap") val controlWrap: String? = null, ) /** diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlRootWrap.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlRootWrap.kt new file mode 100644 index 0000000000..a08f16b4bf --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlRootWrap.kt @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.concord.cord04Roles + +import com.vitorpamplona.quartz.concord.crypto.ConcordKeyDerivation +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner +import kotlin.io.encoding.Base64 +import kotlin.io.encoding.ExperimentalEncodingApi + +/** The opened `control_wrap` payload: the `control_root` delivered for [epoch]. */ +class DeliveredControlRoot( + val epoch: Long, + val controlRoot: ByteArray, +) + +/** + * The staff write-key delivery riding a Grant (CORD-04 §3): a staff-making Grant + * carries the current `control_root` in [GrantEntity.controlWrap], NIP-44-encrypted + * under the granter↔member pairwise conversation key — one ECDH either side can + * compute, so a NIP-46 bunker account opens it with a single `nip44Decrypt`. + * + * The plaintext is fixed-width, the rekey-blob discipline (CORD-06 §1): + * `epoch_be[8] ‖ control_root[32]`, 40 bytes. The epoch rides *inside* the + * ciphertext because staleness is structural — compaction re-wraps a Grant head + * verbatim across Refoundings, so a folded head can carry a wrap minted for a + * prior epoch's key. Harmless: the recipient adopts the secret only if it derives + * to exactly the `control_pk` they hold for the named epoch ([derivesTo]), and any + * mismatch is dropped, never adopted. + */ +object ControlRootWrap { + /** `epoch_be[8] ‖ control_root[32]` */ + const val SIZE = 40 + + fun encodePlaintext( + epoch: Long, + controlRoot: ByteArray, + ): ByteArray { + require(controlRoot.size == 32) { "controlRoot must be 32 bytes" } + val out = ByteArray(SIZE) + ConcordKeyDerivation.writeBe64(out, 0, epoch) + controlRoot.copyInto(out, 8) + return out + } + + fun decodePlaintext(bytes: ByteArray): DeliveredControlRoot? { + if (bytes.size != SIZE) return null + var epoch = 0L + for (i in 0 until 8) epoch = (epoch shl 8) or (bytes[i].toLong() and 0xFF) + return DeliveredControlRoot(epoch, bytes.copyOfRange(8, SIZE)) + } + + /** + * Builds the `control_wrap` value a staff-making Grant carries: the 40-byte + * plaintext, base64'd, then NIP-44-encrypted by [granterSigner] to + * [memberPubKey]. A staff-making edition MUST carry a wrap fresh for the + * current [epoch]. + */ + @OptIn(ExperimentalEncodingApi::class) + suspend fun build( + granterSigner: NostrSigner, + memberPubKey: HexKey, + epoch: Long, + controlRoot: ByteArray, + ): String = granterSigner.nip44Encrypt(Base64.Default.encode(encodePlaintext(epoch, controlRoot)), memberPubKey) + + /** + * Opens a received `control_wrap` with the member's own [memberSigner] against + * the Grant edition's author ([granterPubKey]). Null on any failure — a garbage + * wrap is attributable griefing, nothing worse. The caller MUST still gate + * adoption on [derivesTo] against the `control_pk` it holds for the returned + * epoch. + */ + @OptIn(ExperimentalEncodingApi::class) + suspend fun openOrNull( + controlWrap: String, + memberSigner: NostrSigner, + granterPubKey: HexKey, + ): DeliveredControlRoot? = + try { + decodePlaintext(Base64.Default.decode(memberSigner.nip44Decrypt(controlWrap, granterPubKey))) + } catch (_: Exception) { + null + } + + /** + * The adoption check (CORD-04 §3): true when [controlRoot] derives to exactly + * the [heldControlPk] this member holds for [epoch] (CORD-02 §5). A mismatch is + * dropped, never adopted — the check fails closed. + */ + fun derivesTo( + controlRoot: ByteArray, + communityId: ByteArray, + epoch: Long, + heldControlPk: HexKey, + ): Boolean = ConcordKeyDerivation.controlSignerKey(controlRoot, communityId, epoch).publicKeyHex == heldControlPk.lowercase() +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/CommunityInvite.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/CommunityInvite.kt index 10bc1c2fbd..d652c70143 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/CommunityInvite.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/CommunityInvite.kt @@ -58,6 +58,19 @@ class CommunityInvite( @SerialName("owner_salt") val ownerSalt: String, @SerialName("community_root") val communityRoot: String, @SerialName("root_epoch") val rootEpoch: Long = 0, + /** + * The Control Plane's signer pubkey at [rootEpoch] (CORD-02 §5): subscribe, + * verify, read — never write. Absent = a legacy, pre-split Community; the + * joiner folds Control at the legacy address instead (CORD-06 §3). + * + * Taken on trust in a way the other fields are not: it derives from a secret + * the joiner will never hold, so nothing in the bundle can prove it. A wrong + * one is eclipse-class self-harm by the inviter (a stale or empty Control + * read), the same trust class as a hostile [relays] list — never forged + * authority, since every edition still verifies against the owner-rooted + * Roster, and a later base rotation re-delivers the true key. + */ + @SerialName("control_pk") val controlPk: String? = null, val channels: List = emptyList(), val relays: List = emptyList(), val name: String = "", diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordStrandedRecovery.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordStrandedRecovery.kt index 1baf91cb5f..4f6e02d447 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordStrandedRecovery.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordStrandedRecovery.kt @@ -76,7 +76,10 @@ object ConcordStrandedRecovery { ): ConcordCommunityListEntry? { if (!isStranded(entry, bundle)) return null - val held = (entry.heldRoots + HeldRoot(entry.rootEpoch, entry.root)).distinctBy { it.epoch } + // Bank the epoch we are leaving with its control_pk, so its Control Plane + // stays re-subscribable for the anti-rollback floor (a split epoch's address + // is held, never derivable — CORD-02 §2). + val held = (entry.heldRoots + HeldRoot(entry.rootEpoch, entry.root, entry.controlPk, entry.controlRoot)).distinctBy { it.epoch } return ConcordCommunityListEntry( id = entry.id, @@ -84,6 +87,9 @@ object ConcordStrandedRecovery { ownerSalt = entry.ownerSalt, root = bundle.communityRoot, rootEpoch = bundle.rootEpoch, + // The re-minted bundle carries the new epoch's control_pk (CORD-05 §1); + // absent means the community is (still) legacy at that epoch. + controlPk = bundle.controlPk, heldRoots = held, privateChannels = entry.privateChannels, relays = if (bundle.relays.isNotEmpty()) bundle.relays else entry.relays, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefounding.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefounding.kt index 7a7bb34899..18df7e3c46 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefounding.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefounding.kt @@ -22,6 +22,7 @@ package com.vitorpamplona.quartz.concord.cord06Rekey import com.vitorpamplona.quartz.concord.cord04Roles.ControlEdition import com.vitorpamplona.quartz.concord.crypto.ConcordKeyDerivation +import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys import com.vitorpamplona.quartz.concord.crypto.GroupKey import com.vitorpamplona.quartz.concord.envelope.ConcordStreamEnvelope import com.vitorpamplona.quartz.nip01Core.core.Event @@ -34,25 +35,48 @@ import com.vitorpamplona.quartz.nip59Giftwrap.rumors.RumorAssembler /** * The events a Refounding produces (CORD-06 §3): the [controlWraps] (the current - * Control Plane, compacted to its per-entity head editions and re-sealed under the - * fresh [newRoot] at [newEpoch]) and the [rekeyWraps] (kind-3303 base-rotation - * blobs, sealed under the **prior** root, that deliver [newRoot] to every retained - * member and to nobody else). Publish [controlWraps] first (the new epoch's state) - * then [rekeyWraps] (the key that unlocks it). + * Control Plane, compacted to its per-entity head editions and re-sealed at the + * new epoch's split Control address — signed by the fresh `control_root`-derived + * signer, readable under the fresh [newRoot]-derived read key) and the + * [rekeyWraps] (kind-3303 base-rotation blobs, sealed under the **prior** root, + * that deliver [newRoot] + the new `control_pk` to every retained member — and + * the [newControlRoot] secret to staff — and to nobody else). Publish + * [controlWraps] first (the new epoch's state) then [rekeyWraps] (the key that + * unlocks it). */ class RefoundingBuild( val newRoot: ByteArray, + /** The fresh staff write key, minted beside [newRoot] (CORD-02 §2). */ + val newControlRoot: ByteArray, val newEpoch: Long, + /** The new epoch's split Control Plane keys (rotator view: signer held). */ + val newControlKeys: ControlPlaneKeys, val controlWraps: List, val rekeyWraps: List, -) +) { + /** The new epoch's Control Plane address, delivered to every member in the base blobs. */ + val newControlPk: ByteArray get() = newControlKeys.address.hexToByteArray() +} -/** A retained member's decrypted rekey result: the [newRoot] delivered at [newEpoch] by [rotator]. */ +/** + * A retained member's decrypted rekey result: the [newRoot] delivered at + * [newEpoch] by [rotator], plus the next epoch's Control Plane keys — the + * [newControlPk] every member's blob carries, and, for a staff recipient, the + * [newControlRoot] write secret (CORD-06 §1). A null [newControlPk] marks a + * legacy, pre-split 72-byte rotation (CORD-06 §3): its acceptor folds that + * epoch's Control at the legacy address, honored when reading old rotations and + * never minted by a compliant Rotator. + */ class ReceivedRefounding( val newRoot: ByteArray, val newEpoch: Long, val rotator: HexKey, -) + val newControlPk: ByteArray? = null, + val newControlRoot: ByteArray? = null, +) { + /** True when this was a legacy pre-split rotation (72-byte base blob). */ + val legacy: Boolean get() = newControlPk == null +} /** * Whole-community Refounding (CORD-06 §3): rotate `community_root` to sever a @@ -60,37 +84,50 @@ class ReceivedRefounding( * derive from the root, so rolling it rotates every plane at once; Private Channels * (independently keyed) are rekeyed separately and are not handled here. * + * A compliant Rotator performing any base rotation MUST mint the `control_root` + * split (CORD-02 §2) — a fresh secret beside the new root, both riding the same + * blobs — so a legacy Community upgrades as a side effect of its next Refounding, + * with nobody deciding to. + * * The builder is pure — the caller sources the retained-recipient set (from the - * Guestbook membership minus the removed/banned) and owns publish + persistence. - * All crypto is signer-based so a NIP-46 bunker owner can refound without exposing - * a raw key. + * Guestbook membership minus the removed/banned) and the staff subset (the folded + * Roster's `staffMembers()`, CORD-04 §3) and owns publish + persistence. All + * crypto is signer-based so a NIP-46 bunker owner can refound without exposing a + * raw key. */ object ConcordRefounding { /** - * Builds a Refounding: compacts the Control Plane under [newRoot] and mints the - * base-rotation rekey blobs delivering [newRoot] to [recipientsXOnly]. + * Builds a Refounding: compacts the Control Plane onto the new epoch's split + * Control address and mints the base-rotation rekey blobs delivering [newRoot] + * + the new `control_pk` to [recipientsXOnly] (the [staffXOnly] subset also + * receiving [newControlRoot]). * * @param priorRoot the community_root being rotated out (at [rootEpoch]) * @param newRoot the freshly generated 32-byte community_root + * @param newControlRoot the freshly minted 32-byte staff write key (CORD-02 §2) * @param priorControlWraps the current Control Plane's kind-1059 wraps (any subset that folds) - * @param priorControlKey the Control Plane group key at [rootEpoch] + * @param priorControlKeys the Control Plane keys at [rootEpoch] (split or legacy) * @param recipientsXOnly the retained members' x-only pubkeys (hex) to re-key + * @param staffXOnly the subset of [recipientsXOnly] that is staff (owner + Control-writing + * permission holders, CORD-04 §3) and receives the 136-byte blob */ suspend fun build( rotatorSigner: NostrSigner, communityId: ByteArray, priorRoot: ByteArray, newRoot: ByteArray, + newControlRoot: ByteArray, rootEpoch: Long, priorControlWraps: List, - priorControlKey: GroupKey, + priorControlKeys: ControlPlaneKeys, recipientsXOnly: List, + staffXOnly: Set, createdAt: Long, ): RefoundingBuild { val newEpoch = rootEpoch + 1 - val newControlKey = ConcordKeyDerivation.controlPlaneKey(newRoot, communityId, newEpoch) + val newControlKeys = ControlPlaneKeys.forStaff(newRoot, communityId, newEpoch, newControlRoot) - val controlWraps = compactControlPlane(priorControlWraps, priorControlKey, newControlKey) + val controlWraps = compactControlPlane(priorControlWraps, priorControlKeys, newControlKeys) val baseRekeyKey = ConcordKeyDerivation.baseRekeyAddress(priorRoot, communityId, newEpoch) val prevCommit = ConcordKeyDerivation.epochKeyCommitment(rootEpoch, priorRoot).toHexKey() @@ -99,33 +136,41 @@ object ConcordRefounding { rotatorSigner = rotatorSigner, baseRekeyKey = baseRekeyKey, recipientsXOnly = recipientsXOnly, + staffXOnly = staffXOnly, newRoot = newRoot, + newControlPk = newControlKeys.address.hexToByteArray(), + newControlRoot = newControlRoot, newEpoch = newEpoch, prevEpoch = rootEpoch, prevCommit = prevCommit, createdAt = createdAt, ) - return RefoundingBuild(newRoot, newEpoch, controlWraps, rekeyWraps) + return RefoundingBuild(newRoot, newControlRoot, newEpoch, newControlKeys, controlWraps, rekeyWraps) } /** - * Compacts [priorWraps] into a slim snapshot re-published under [newControlKey] + * Compacts [priorWraps] into a slim snapshot re-published under [newControlKeys] * (CORD-06 §3): keep only the head (highest-version) edition per entity and * re-wrap its **original plaintext seal** — which carries the original author's - * signature — under the new root. Because Control Plane seals are plaintext - * (CORD-02 §5), re-encryption preserves those signatures, so a fresh joiner - * verifies the compacted state exactly as it verified the full chain. + * signature — at the new epoch's Control address. Because Control Plane seals + * are plaintext (CORD-02 §5), re-encryption preserves those signatures, so a + * fresh joiner verifies the compacted state exactly as it verified the full + * chain. [priorControlKeys] may be legacy (a pre-split epoch's compaction is + * exactly how a Community upgrades to the split) or split; [newControlKeys] + * must hold the new signer. A Rotator MUST NOT mirror editions to the new + * epoch's legacy-derived address to appease stale readers — the mirror + * re-opens exactly the member-writable surface the split closes. */ fun compactControlPlane( priorWraps: List, - priorControlKey: GroupKey, - newControlKey: GroupKey, + priorControlKeys: ControlPlaneKeys, + newControlKeys: ControlPlaneKeys, ): List { // entity coordinate -> (head edition, its verified seal) val heads = HashMap>() for (wrap in priorWraps) { - val opened = ConcordStreamEnvelope.openOrNull(wrap, priorControlKey) ?: continue + val opened = ConcordStreamEnvelope.openOrNull(wrap, priorControlKeys) ?: continue val edition = ControlEdition.fromRumor(opened.rumor) ?: continue val coord = edition.entityKind.wire + ":" + edition.entityIdHex val current = heads[coord] @@ -133,12 +178,14 @@ object ConcordRefounding { heads[coord] = edition to opened.seal } } - return heads.values.map { (_, seal) -> ConcordStreamEnvelope.wrapSeal(seal, newControlKey, createdAt = seal.createdAt) } + return heads.values.map { (_, seal) -> ConcordStreamEnvelope.wrapSeal(seal, newControlKeys, createdAt = seal.createdAt) } } /** - * Mints the base-rotation rekey blobs delivering [newRoot] to [recipientsXOnly], - * chunked at [ConcordRekey.MAX_BLOBS_PER_CHUNK] and wrapped (encrypted seal, + * Mints the base-rotation rekey blobs delivering [newRoot] + [newControlPk] to + * [recipientsXOnly] — the [staffXOnly] subset also receiving [newControlRoot] + * in the 136-byte staff form (CORD-06 §1) — chunked at + * [ConcordRekey.MAX_BLOBS_PER_CHUNK] and wrapped (encrypted seal, * rotator-signed) on the [baseRekeyKey] address so every current member — who * precomputes that address from the prior root — receives it live. */ @@ -146,16 +193,28 @@ object ConcordRefounding { rotatorSigner: NostrSigner, baseRekeyKey: GroupKey, recipientsXOnly: List, + staffXOnly: Set, newRoot: ByteArray, + newControlPk: ByteArray, + newControlRoot: ByteArray, newEpoch: Long, prevEpoch: Long, prevCommit: HexKey, createdAt: Long, ): List { if (recipientsXOnly.isEmpty()) return emptyList() + val staffLower = staffXOnly.mapTo(HashSet()) { it.lowercase() } val blobs = recipientsXOnly.map { recipient -> - ConcordRekey.blobForSigner(rotatorSigner, recipient.hexToByteArray(), ConcordRekey.ROOT_SCOPE, newEpoch, newRoot) + ConcordRekey.blobForSigner( + rotatorSigner = rotatorSigner, + recipientXOnly = recipient.hexToByteArray(), + scopeId = ConcordRekey.ROOT_SCOPE, + newEpoch = newEpoch, + newKey = newRoot, + newControlPk = newControlPk, + newControlRoot = if (recipient.lowercase() in staffLower) newControlRoot else null, + ) } val chunks = blobs.chunked(ConcordRekey.MAX_BLOBS_PER_CHUNK) val total = chunks.size @@ -170,15 +229,19 @@ object ConcordRefounding { * Receives a base rotation for the member behind [recipientSigner]: opens the * kind-3303 [wraps] at the member's next base-rekey address ([baseRekeyKey]), * verifies each is a well-formed root rotation to [newEpoch] whose `prevcommit` - * continues the [priorRoot] the member holds, and returns the delivered new root - * (with the rotator's real pubkey, so the caller can authorize it against the - * folded roster). Null if no chunk carries this member's blob — which only means + * continues the [priorRoot] the member holds, and returns the delivered new + * root and Control Plane keys (with the rotator's real pubkey, so the caller + * can authorize it against the folded roster). A staff blob's delivered secret + * must derive to exactly the delivered `control_pk` (CORD-02 §5) — a + * mismatched pair is refused rather than adopting a plane split from its + * readers. Null if no chunk carries this member's blob — which only means * "removed" once the caller confirms it holds every chunk of the rotation. */ suspend fun findNewRoot( wraps: List, baseRekeyKey: GroupKey, recipientSigner: NostrSigner, + communityId: ByteArray, priorRoot: ByteArray, rootEpoch: Long, ): ReceivedRefounding? { @@ -195,8 +258,16 @@ object ConcordRefounding { val blobs = ConcordRekey.decodeContent(rumor.content) val rotatorXOnly = opened.author.hexToByteArray() - val newRoot = ConcordRekey.findNewKeyWithSigner(blobs, recipientSigner, rotatorXOnly, ConcordRekey.ROOT_SCOPE, newEpoch) ?: continue - return ReceivedRefounding(newRoot, newEpoch, opened.author) + val payload = ConcordRekey.findPayloadWithSigner(blobs, recipientSigner, rotatorXOnly, ConcordRekey.ROOT_SCOPE, newEpoch) ?: continue + val controlRoot = payload.newControlRoot + val controlPk = payload.newControlPk + if (controlRoot != null && controlPk != null) { + // The staff derive-check (CORD-06 §1): refuse a pair whose secret does not + // derive to the pk the other members were handed — fails closed. + val derived = ConcordKeyDerivation.controlSignerKey(controlRoot, communityId, newEpoch).publicKey + if (!derived.contentEquals(controlPk)) continue + } + return ReceivedRefounding(payload.newKey, newEpoch, opened.author, controlPk, controlRoot) } return null } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRekey.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRekey.kt index fed5b15eb9..84d9c1910d 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRekey.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRekey.kt @@ -38,10 +38,12 @@ import kotlin.io.encoding.ExperimentalEncodingApi * * The rotator publishes a kind-3303 rumor whose content is a JSON array of * [RekeyBlob]s, one per remaining member. Each blob's `locator` is the recipient's - * pseudonym (public-input HKDF), and its `wrapped` field is the 72-byte - * [RekeyPayload] (base64 → NIP-44 under the rotator↔recipient pairwise key). A - * recipient computes their own locator, finds the matching blob, and decrypts the - * new key; a member with no matching blob across all chunks of a complete rotation + * pseudonym (public-input HKDF), and its `wrapped` field is the fixed-width + * [RekeyPayload] (base64 → NIP-44 under the rotator↔recipient pairwise key) — + * 72 bytes for a channel rotation, 104/136 for a base rotation's member/staff + * forms carrying the next epoch's Control Plane keys (CORD-02 §2). A recipient + * computes their own locator, finds the matching blob, and decrypts the new + * key(s); a member with no matching blob across all chunks of a complete rotation * has been removed. * * Pinned to the Concord v2 reference client for interop. @@ -57,7 +59,9 @@ object ConcordRekey { val ROOT_SCOPE: ByteArray = ByteArray(32) /** - * Builds a rekey blob delivering [newKey] to one recipient. + * Builds a rekey blob delivering [newKey] to one recipient. On a base rotation + * pass [newControlPk] (every member) and, for a staff recipient, also + * [newControlRoot] (CORD-06 §1) — the widths select the 104/136-byte forms. * * @param rotatorPrivKey the rotator's private key (their real identity) * @param rotatorXOnly the rotator's x-only pubkey @@ -71,9 +75,11 @@ object ConcordRekey { scopeId: ByteArray, newEpoch: Long, newKey: ByteArray, + newControlPk: ByteArray? = null, + newControlRoot: ByteArray? = null, ): RekeyBlob { val locator = ConcordKeyDerivation.recipientLocator(rotatorXOnly, recipientXOnly, scopeId, newEpoch).toHexKey() - val payloadB64 = Base64.Default.encode(RekeyPayload(scopeId, newEpoch, newKey).encode()) + val payloadB64 = Base64.Default.encode(RekeyPayload(scopeId, newEpoch, newKey, newControlPk, newControlRoot).encode()) val convKey = Nip44.v2.getConversationKey(rotatorPrivKey, recipientXOnly) val wrapped = Nip44.v2.encrypt(payloadB64, convKey).encodePayload() return RekeyBlob(locator, wrapped) @@ -125,38 +131,54 @@ object ConcordRekey { scopeId: ByteArray, newEpoch: Long, newKey: ByteArray, + newControlPk: ByteArray? = null, + newControlRoot: ByteArray? = null, ): RekeyBlob { val rotatorXOnly = rotatorSigner.pubKey.hexToByteArray() val locator = ConcordKeyDerivation.recipientLocator(rotatorXOnly, recipientXOnly, scopeId, newEpoch).toHexKey() - val payloadB64 = Base64.Default.encode(RekeyPayload(scopeId, newEpoch, newKey).encode()) + val payloadB64 = Base64.Default.encode(RekeyPayload(scopeId, newEpoch, newKey, newControlPk, newControlRoot).encode()) val wrapped = rotatorSigner.nip44Encrypt(payloadB64, recipientXOnly.toHexKey()) return RekeyBlob(locator, wrapped) } /** - * Finds the recipient's rotated key like [findNewKey], but decrypts the blob via - * [recipientSigner] (bunker-compatible) rather than a raw private key. + * Finds the recipient's whole decrypted [RekeyPayload] (scope and epoch already + * verified against the expectation) via [recipientSigner], or null if no blob + * matches or it fails to open/verify. Base rotations need the full payload — + * the delivered `new_control_pk` / `new_control_root` ride beside the key. */ @OptIn(ExperimentalEncodingApi::class) - suspend fun findNewKeyWithSigner( + suspend fun findPayloadWithSigner( blobs: List, recipientSigner: NostrSigner, rotatorXOnly: ByteArray, scopeId: ByteArray, newEpoch: Long, - ): ByteArray? { + ): RekeyPayload? { val recipientXOnly = recipientSigner.pubKey.hexToByteArray() val myLocator = ConcordKeyDerivation.recipientLocator(rotatorXOnly, recipientXOnly, scopeId, newEpoch).toHexKey() val blob = blobs.firstOrNull { it.locator == myLocator } ?: return null return try { val payload = RekeyPayload.decode(Base64.Default.decode(recipientSigner.nip44Decrypt(blob.wrapped, rotatorXOnly.toHexKey()))) ?: return null if (!payload.scopeId.contentEquals(scopeId) || payload.epoch != newEpoch) return null - payload.newKey + payload } catch (_: Exception) { null } } + /** + * Finds the recipient's rotated key like [findNewKey], but decrypts the blob via + * [recipientSigner] (bunker-compatible) rather than a raw private key. + */ + suspend fun findNewKeyWithSigner( + blobs: List, + recipientSigner: NostrSigner, + rotatorXOnly: ByteArray, + scopeId: ByteArray, + newEpoch: Long, + ): ByteArray? = findPayloadWithSigner(blobs, recipientSigner, rotatorXOnly, scopeId, newEpoch)?.newKey + /** * Finds the recipient's rotated key across the [blobs] of one or more chunks, * or null if they were removed. Computes the recipient's locator, matches it, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/RekeyBlob.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/RekeyBlob.kt index e841183732..41856cbb8b 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/RekeyBlob.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/RekeyBlob.kt @@ -26,8 +26,8 @@ import kotlinx.serialization.Serializable /** * One recipient's entry in a rekey (CORD-06): a [locator] (the recipient's * pseudonym, so only they know it's for them) and the [wrapped] new key (the - * 72-byte payload, base64'd then NIP-44-encrypted under the rotator↔recipient - * pairwise key). + * fixed-width [RekeyPayload], base64'd then NIP-44-encrypted under the + * rotator↔recipient pairwise key). */ @Serializable class RekeyBlob( @@ -36,33 +36,77 @@ class RekeyBlob( ) /** - * The 72-byte rekey payload: `scope_id[32] ‖ epoch_be8 ‖ new_key[32]` - * (CORD-06 §2). Fixed-width so a recipient can verify the scope and epoch it - * decrypts to match what they expected before adopting [newKey]. + * A rekey blob's plaintext (CORD-06 §1), fixed-width per form — the width + * declaring the form: + * + * - **72 bytes** — `scope_id[32] ‖ epoch_be8 ‖ new_key[32]`: a Channel + * rotation's blob, or a legacy pre-split *base* rotation (honored when reading + * old epochs, never minted anew — CORD-06 §3). + * - **104 bytes** — `… ‖ new_control_pk[32]`: a base rotation's member blob, + * also carrying the next epoch's Control Plane address (CORD-02 §2). + * - **136 bytes** — `… ‖ new_control_root[32]`: a base rotation's staff blob, + * additionally delivering the write secret (CORD-04 §3 staff). + * + * Any other width is malformed and the blob is dropped ([decode] returns null). + * The scope and epoch live *inside* the ciphertext so a recipient can verify them + * against the event's tags before adopting anything, making a blob unspliceable; + * a staff recipient additionally requires that [newControlRoot] derive to exactly + * [newControlPk] (CORD-02 §5) before adopting the pair. */ class RekeyPayload( val scopeId: ByteArray, val epoch: Long, val newKey: ByteArray, + /** The next epoch's `control_pk` on a base rotation; null on a channel or legacy blob. */ + val newControlPk: ByteArray? = null, + /** The next epoch's `control_root` on a staff base blob; null otherwise. */ + val newControlRoot: ByteArray? = null, ) { + init { + require(newControlRoot == null || newControlPk != null) { "a control_root is only ever delivered beside its control_pk" } + } + fun encode(): ByteArray { require(scopeId.size == 32) { "scopeId must be 32 bytes" } require(newKey.size == 32) { "newKey must be 32 bytes" } - val out = ByteArray(SIZE) + require(newControlPk == null || newControlPk.size == 32) { "newControlPk must be 32 bytes" } + require(newControlRoot == null || newControlRoot.size == 32) { "newControlRoot must be 32 bytes" } + val size = + when { + newControlRoot != null -> SIZE_BASE_STAFF + newControlPk != null -> SIZE_BASE_MEMBER + else -> SIZE_CHANNEL + } + val out = ByteArray(size) scopeId.copyInto(out, 0) ConcordKeyDerivation.writeBe64(out, 32, epoch) newKey.copyInto(out, 40) + newControlPk?.copyInto(out, 72) + newControlRoot?.copyInto(out, 104) return out } companion object { - const val SIZE = 72 + /** A Channel rotation's blob — also the legacy pre-split base form (CORD-06 §3). */ + const val SIZE_CHANNEL = 72 + + /** A base rotation's member blob: `… ‖ new_control_pk[32]`. */ + const val SIZE_BASE_MEMBER = 104 + + /** A base rotation's staff blob: `… ‖ new_control_root[32]`. */ + const val SIZE_BASE_STAFF = 136 fun decode(bytes: ByteArray): RekeyPayload? { - if (bytes.size != SIZE) return null + if (bytes.size != SIZE_CHANNEL && bytes.size != SIZE_BASE_MEMBER && bytes.size != SIZE_BASE_STAFF) return null var epoch = 0L for (i in 0 until 8) epoch = (epoch shl 8) or (bytes[32 + i].toLong() and 0xFF) - return RekeyPayload(bytes.copyOfRange(0, 32), epoch, bytes.copyOfRange(40, 72)) + return RekeyPayload( + scopeId = bytes.copyOfRange(0, 32), + epoch = epoch, + newKey = bytes.copyOfRange(40, 72), + newControlPk = if (bytes.size >= SIZE_BASE_MEMBER) bytes.copyOfRange(72, 104) else null, + newControlRoot = if (bytes.size >= SIZE_BASE_STAFF) bytes.copyOfRange(104, 136) else null, + ) } } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/crypto/ConcordKeyDerivation.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/crypto/ConcordKeyDerivation.kt index b59827aadd..6ff9df015d 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/crypto/ConcordKeyDerivation.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/crypto/ConcordKeyDerivation.kt @@ -182,13 +182,33 @@ object ConcordKeyDerivation { // ---- Plane keys (CORD-02) ------------------------------------------------- - /** The Control Plane address for a community at [epoch] (holders of the root only). */ + /** + * The Control Plane *read* key for a community at [epoch] (CORD-02 §5): its + * `conversationKey` encrypts the wraps, so every `community_root` holder can read. + * + * On a pre-split (legacy) epoch this derivation alone was the plane — its pk the + * address and wrap signer, its sk held by every member. That use is retained for + * reading legacy epochs (CORD-06 §3); a split epoch's address comes from + * [controlSignerKey] instead. + */ fun controlPlaneKey( communityRoot: ByteArray, communityId: ByteArray, epoch: Long, ): GroupKey = groupKey(ConcordLabels.CONTROL, communityRoot, communityId, epoch) + /** + * The Control Plane *signer* for a community at [epoch] (CORD-02 §5): its pk is + * the plane's address (`control_pk`) and its sk — derivable only from the + * staff-held `control_root` — signs the wraps. Possession is a spam gate, never + * authority: every edition is still judged by its sealed actor's Roster rank. + */ + fun controlSignerKey( + controlRoot: ByteArray, + communityId: ByteArray, + epoch: Long, + ): GroupKey = groupKey(ConcordLabels.CONTROL_SIGNER, controlRoot, communityId, epoch) + /** The Guestbook Plane address for a community at [epoch]. */ fun guestbookPlaneKey( communityRoot: ByteArray, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/crypto/ConcordLabels.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/crypto/ConcordLabels.kt index 0edb36ec9f..dcdd50c616 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/crypto/ConcordLabels.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/crypto/ConcordLabels.kt @@ -35,9 +35,19 @@ object ConcordLabels { /** Per-Channel Chat Plane key (CORD-03). */ const val CHANNEL = "concord/channel" - /** Control Plane key (CORD-02). */ + /** + * Control Plane *read* key (CORD-02 §5): community_root-derived, its conv_key + * encrypts the wraps. Pre-split epochs used its pk/sk as the plane's address and + * signer too — that use is retained for reading legacy epochs (CORD-06 §3). + */ const val CONTROL = "concord/control" + /** + * Control Plane signer (CORD-02 §5): control_root-derived, its pk is the plane's + * address and its staff-only sk signs the wraps (CORD-01, Write-Restricted Streams). + */ + const val CONTROL_SIGNER = "concord/control-signer" + /** Guestbook Plane key (CORD-02). */ const val GUESTBOOK = "concord/guestbook" diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/crypto/ControlPlaneKeys.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/crypto/ControlPlaneKeys.kt new file mode 100644 index 0000000000..6cf6c1d73f --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/crypto/ControlPlaneKeys.kt @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.concord.crypto + +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray + +/** + * The Control Plane's key material at one epoch (CORD-02 §5). + * + * The plane's stream key is split (CORD-01, Write-Restricted Streams): the + * address-and-signer keypair derives from the staff-held `control_root`, while the + * wraps' content is encrypted under the `community_root`-derived read key every + * member holds. So what an account holds depends on its standing: + * + * - **Member**: the [address] (`control_pk`, held — never derivable) plus the + * [readKey]. Enough to subscribe, verify wrap signatures, and decrypt; [signer] + * is null and [canWrite] false. + * - **Staff / owner**: additionally the [signer] (derived from the `control_root`), + * whose sk mints wraps that verify at the address. + * - **Legacy epoch** (pre-split, CORD-06 §3): the `concord/control` derivation + * alone was the plane — its pk the address and signer, every member holding + * both. [legacy] is true and [signer] == [readKey]. + */ +class ControlPlaneKeys( + /** The plane's stream address (`control_pk` on a split epoch): subscribe + verify. */ + val address: HexKey, + /** The `community_root`-derived read key; its `conversationKey` opens the wraps. */ + val readKey: GroupKey, + /** The keypair whose sk signs wraps at [address]. Null when this account cannot write. */ + val signer: GroupKey?, + /** True for a pre-split epoch keyed by the legacy member-held derivation. */ + val legacy: Boolean, +) { + /** True when this account holds the write key for the plane. */ + val canWrite: Boolean get() = signer != null + + companion object { + /** + * A pre-split epoch's Control Plane: the legacy `concord/control` derivation + * is address, signer, and read key at once — every member holds all three. + */ + fun legacy( + communityRoot: ByteArray, + communityId: ByteArray, + epoch: Long, + ): ControlPlaneKeys { + val key = ConcordKeyDerivation.controlPlaneKey(communityRoot, communityId, epoch) + return ControlPlaneKeys(key.publicKeyHex, key, signer = key, legacy = true) + } + + /** + * A split epoch as a regular member holds it: the delivered [controlPk] + * (invite / community list / base rekey blob, CORD-02 §2) plus the derived + * read key. Read-only — a member cannot mint a wrap at the address. + */ + fun forMember( + communityRoot: ByteArray, + communityId: ByteArray, + epoch: Long, + controlPk: HexKey, + ): ControlPlaneKeys = + ControlPlaneKeys( + address = controlPk.lowercase(), + readKey = ConcordKeyDerivation.controlPlaneKey(communityRoot, communityId, epoch), + signer = null, + legacy = false, + ) + + /** + * A split epoch as staff holds it: the signer derives from the held + * [controlRoot], yielding the address and the write key together. + */ + fun forStaff( + communityRoot: ByteArray, + communityId: ByteArray, + epoch: Long, + controlRoot: ByteArray, + ): ControlPlaneKeys { + val signer = ConcordKeyDerivation.controlSignerKey(controlRoot, communityId, epoch) + return ControlPlaneKeys( + address = signer.publicKeyHex, + readKey = ConcordKeyDerivation.controlPlaneKey(communityRoot, communityId, epoch), + signer = signer, + legacy = false, + ) + } + + /** + * Dispatches on what the account holds: the `control_root` secret (staff), + * only the `control_pk` (member), or neither — a legacy, pre-split epoch + * (CORD-06 §3). A held [controlRoot] wins over a held [controlPk]: the pk it + * derives is the plane by definition (a delivered secret is only ever adopted + * after the derive-check, CORD-04 §3). + */ + fun of( + communityRoot: ByteArray, + communityId: ByteArray, + epoch: Long, + controlPk: HexKey? = null, + controlRoot: HexKey? = null, + ): ControlPlaneKeys = + when { + controlRoot != null -> forStaff(communityRoot, communityId, epoch, controlRoot.hexToByteArray()) + controlPk != null -> forMember(communityRoot, communityId, epoch, controlPk) + else -> legacy(communityRoot, communityId, epoch) + } + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/envelope/ConcordStreamEnvelope.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/envelope/ConcordStreamEnvelope.kt index acd8c2a007..7975d3aa37 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/envelope/ConcordStreamEnvelope.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/envelope/ConcordStreamEnvelope.kt @@ -20,8 +20,10 @@ */ package com.vitorpamplona.quartz.concord.envelope +import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys import com.vitorpamplona.quartz.concord.crypto.GroupKey import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.toHexKey import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair import com.vitorpamplona.quartz.nip01Core.crypto.verify @@ -96,14 +98,45 @@ object ConcordStreamEnvelope { stream: GroupKey, ephemeral: Boolean = false, createdAt: Long = TimeUtils.now(), + ): Event = wrapSeal(seal, stream, stream.conversationKey, ephemeral, createdAt) + + /** + * Write-restricted variant (CORD-01, Write-Restricted Streams): the wrap is + * signed by [signerKey] (its pk the stream address, its sk held by the writers + * alone) while the content is encrypted under [readConversationKey], the second + * shared key the full readership holds. Concord's Control Plane wraps this way + * on a split epoch (CORD-02 §5). + */ + fun wrapSeal( + seal: Event, + signerKey: GroupKey, + readConversationKey: ByteArray, + ephemeral: Boolean = false, + createdAt: Long = TimeUtils.now(), ): Event { - val streamSigner = NostrSignerSync(KeyPair(privKey = stream.secretKey)) - val content = Nip44.v2.encrypt(seal.toJson(), stream.conversationKey).encodePayload() + val streamSigner = NostrSignerSync(KeyPair(privKey = signerKey.secretKey)) + val content = Nip44.v2.encrypt(seal.toJson(), readConversationKey).encodePayload() val ephemeralP = KeyPair().pubKey.toHexKey() val kind = if (ephemeral) KIND_WRAP_EPHEMERAL else KIND_WRAP return streamSigner.signNormal(createdAt, kind, arrayOf(arrayOf("p", ephemeralP)), content) } + /** + * Wraps [seal] onto the Control Plane described by [keys]: signed by its signer + * (which the holder must have — throws when [ControlPlaneKeys.canWrite] is + * false), encrypted under its read key. On a legacy epoch signer == read key + * and this is the classic single-key wrap. + */ + fun wrapSeal( + seal: Event, + keys: ControlPlaneKeys, + ephemeral: Boolean = false, + createdAt: Long = TimeUtils.now(), + ): Event { + val signer = requireNotNull(keys.signer) { "This account cannot write to the Control Plane: control_root not held (CORD-02 §2)" } + return wrapSeal(seal, signer, keys.readKey.conversationKey, ephemeral, createdAt) + } + /** Convenience: [seal] then [wrapSeal] in one call. */ suspend fun wrap( rumor: Event, @@ -114,6 +147,20 @@ object ConcordStreamEnvelope { createdAt: Long = TimeUtils.now(), ): Event = wrapSeal(seal(rumor, stream, authorSigner, encrypted), stream, ephemeral, createdAt) + /** + * Convenience for the Control Plane: seals under [keys]' read key (an encrypted + * seal's rumor must decrypt for every reader, not only writers) and wraps with + * its signer. Throws when the account cannot write (see [wrapSeal]). + */ + suspend fun wrap( + rumor: Event, + keys: ControlPlaneKeys, + authorSigner: NostrSigner, + encrypted: Boolean, + ephemeral: Boolean = false, + createdAt: Long = TimeUtils.now(), + ): Event = wrapSeal(seal(rumor, keys.readKey, authorSigner, encrypted), keys, ephemeral, createdAt) + /** * Opens a stream [wrap] for the [stream] plane and returns the verified author * rumor, or throws if any layer fails to validate: @@ -129,16 +176,31 @@ object ConcordStreamEnvelope { fun open( wrap: Event, stream: GroupKey, + ): OpenedStreamEvent = open(wrap, stream.publicKeyHex, stream.conversationKey) + + /** + * Write-restricted variant (CORD-01, Write-Restricted Streams): opening takes + * only the stream [address] (the writers' pubkey, held by every reader) and the + * [readConversationKey] — never the signer's secret. `wrap.verify()` checks the + * signature against `wrap.pubkey`, which the address equality pins to the + * writers' key, so a wrap minted by anyone else fails here. A verifying wrap + * proves only that *a* writer published it; the seal's actor stays the sole + * authority (CORD-04). + */ + fun open( + wrap: Event, + address: HexKey, + readConversationKey: ByteArray, ): OpenedStreamEvent { require(wrap.kind == KIND_WRAP || wrap.kind == KIND_WRAP_EPHEMERAL) { "Not a Concord stream wrap: kind ${wrap.kind}" } - require(wrap.pubKey == stream.publicKeyHex) { - "Wrap author ${wrap.pubKey} is not the stream address ${stream.publicKeyHex}" + require(wrap.pubKey == address) { + "Wrap author ${wrap.pubKey} is not the stream address $address" } require(wrap.verify()) { "Wrap signature/id is invalid" } - val seal = Event.fromJson(Nip44.v2.decrypt(wrap.content, stream.conversationKey)) + val seal = Event.fromJson(Nip44.v2.decrypt(wrap.content, readConversationKey)) require(seal.kind == KIND_SEAL_ENCRYPTED || seal.kind == KIND_SEAL_PLAINTEXT) { "Not a Concord seal: kind ${seal.kind}" } @@ -146,7 +208,7 @@ object ConcordStreamEnvelope { val rumorJson = if (seal.kind == KIND_SEAL_ENCRYPTED) { - Nip44.v2.decrypt(seal.content, stream.conversationKey) + Nip44.v2.decrypt(seal.content, readConversationKey) } else { seal.content } @@ -160,17 +222,40 @@ object ConcordStreamEnvelope { return OpenedStreamEvent(rumor, seal.kind, seal.pubKey, seal) } + /** + * Opens a Control Plane wrap with [keys] (split or legacy): verified against the + * plane's address, decrypted under its read key. Needs no write key, so a regular + * member reads exactly as staff does (CORD-02 §5). + */ + fun open( + wrap: Event, + keys: ControlPlaneKeys, + ): OpenedStreamEvent = open(wrap, keys.address, keys.readKey.conversationKey) + /** Like [open] but returns null instead of throwing on any validation failure. */ fun openOrNull( wrap: Event, stream: GroupKey, + ): OpenedStreamEvent? = openOrNull(wrap, stream.publicKeyHex, stream.conversationKey) + + /** Like the write-restricted [open] but returns null instead of throwing. */ + fun openOrNull( + wrap: Event, + address: HexKey, + readConversationKey: ByteArray, ): OpenedStreamEvent? = try { - open(wrap, stream) + open(wrap, address, readConversationKey) } catch (_: Exception) { null } + /** Opens a Control Plane wrap with [keys] (split or legacy), or null on failure. */ + fun openOrNull( + wrap: Event, + keys: ControlPlaneKeys, + ): OpenedStreamEvent? = openOrNull(wrap, keys.address, keys.readKey.conversationKey) + private val EMPTY_TAGS = emptyArray>() } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityFactoryTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityFactoryTest.kt index 0c234713ca..e42b0e0c79 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityFactoryTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityFactoryTest.kt @@ -31,6 +31,7 @@ import kotlin.test.Test import kotlin.test.assertContentEquals import kotlin.test.assertEquals import kotlin.test.assertFalse +import kotlin.test.assertNotEquals import kotlin.test.assertNotNull import kotlin.test.assertTrue @@ -55,12 +56,22 @@ class ConcordCommunityFactoryTest { community.communityId, ) - // Two genesis wraps, both authored by the Control Plane address. + // Two genesis wraps, both authored by the Control Plane address — which on a fresh + // community is the control_root-derived signer, not the community_root (CORD-02 §5). assertEquals(2, community.genesisWraps.size) community.genesisWraps.forEach { assertEquals(ConcordStreamEnvelope.KIND_WRAP, it.kind) - assertEquals(community.controlPlane.publicKeyHex, it.pubKey) + assertEquals(community.controlPlane.address, it.pubKey) } + assertEquals( + ConcordKeyDerivation.controlSignerKey(community.controlRoot, community.communityId, community.rootEpoch).publicKeyHex, + community.controlPkHex, + ) + // The plane is genuinely split: its address is NOT the legacy community_root derivation. + assertNotEquals( + ConcordKeyDerivation.controlPlaneKey(community.communityRoot, community.communityId, community.rootEpoch).publicKeyHex, + community.controlPkHex, + ) // Genesis wraps open with plaintext (20014) seals, authored by the owner. val opened = community.genesisWraps.map { ConcordStreamEnvelope.open(it, community.controlPlane) } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityListTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityListTest.kt index b6ccaa8f81..4da831f09f 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityListTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ConcordCommunityListTest.kt @@ -35,6 +35,7 @@ import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFalse import kotlin.test.assertNotNull +import kotlin.test.assertNull import kotlin.test.assertTrue class ConcordCommunityListTest { @@ -45,14 +46,20 @@ class ConcordCommunityListTest { id: String, name: String, epoch: Long = 0, + controlPk: String? = null, + controlRoot: String? = null, + inviteRef: String? = null, ) = ConcordCommunityListEntry( id = id, owner = "0f".repeat(32), ownerSalt = "aa".repeat(32), root = "bb".repeat(32), rootEpoch = epoch, + controlPk = controlPk, + controlRoot = controlRoot, relays = listOf("wss://relay.example"), name = name, + inviteRef = inviteRef, ) @Test @@ -417,7 +424,7 @@ class ConcordCommunityListTest { ownerSalt = decoded.ownerSalt, root = decoded.root, rootEpoch = decoded.rootEpoch, - heldRoots = decoded.heldRoots.map { HeldRoot(it.epoch, it.key, buildJsonObject { put("key", "STALE") }) }, + heldRoots = decoded.heldRoots.map { HeldRoot(it.epoch, it.key, it.controlPk, it.controlRoot, buildJsonObject { put("key", "STALE") }) }, privateChannels = decoded.privateChannels.map { PrivateChannelKey(it.channelId, it.key, it.epoch, it.name, buildJsonObject { put("name", "STALE") }) }, relays = decoded.relays, name = "Renamed", @@ -486,4 +493,48 @@ class ConcordCommunityListTest { assertEquals(2, merged.size) assertEquals("New", merged.first { it.id == "11".repeat(32) }.name) // higher epoch wins } + + @Test + fun mergeAtTheSameEpochFillsMissingControlKeyMaterialFromEitherSide() { + // The second-device gap (CORD-02 §8): device A was promoted to staff (it adopted the + // control_root via a Grant's control_wrap), device B holds the invite anchor and the + // delivered control_pk. Both describe the same epoch, so a merge must end holding all + // of it — this is how the write secret reaches a staffer's own other devices. + val id = "11".repeat(32) + val promoted = listOf(entry(id, "Nostrichs", epoch = 2, controlPk = "cc".repeat(32), controlRoot = "dd".repeat(32))) + val joined = listOf(entry(id, "Nostrichs", epoch = 2, controlPk = "cc".repeat(32), inviteRef = "https://vector.chat/i/abc")) + + val merged = ConcordCommunityList.merge(promoted, joined).single() + assertEquals("cc".repeat(32), merged.controlPk) + assertEquals("dd".repeat(32), merged.controlRoot, "the staff write key must reach the holder's other devices") + assertEquals("https://vector.chat/i/abc", merged.inviteRef, "the recovery anchor must survive the fill") + + // Order-independent: the fill works whichever side holds the secret. + val reversed = ConcordCommunityList.merge(joined, promoted).single() + assertEquals("dd".repeat(32), reversed.controlRoot) + assertEquals("https://vector.chat/i/abc", reversed.inviteRef) + } + + @Test + fun mergeNeverInheritsControlKeysAcrossEpochs() { + // A lower epoch's control_pk/control_root is stale for the winner's planes (CORD-06): + // the pair rolls at every Refounding, so carrying it forward would point the client at + // a dead address (pk) or derive a wrong one entirely (root). A winner without control + // material is a legacy rotation and must stay that way. + val id = "11".repeat(32) + val stale = listOf(entry(id, "Old", epoch = 1, controlPk = "cc".repeat(32), controlRoot = "dd".repeat(32), inviteRef = "https://vector.chat/i/abc")) + val rotated = listOf(entry(id, "New", epoch = 2)) + + val merged = ConcordCommunityList.merge(stale, rotated).single() + assertEquals(2, merged.rootEpoch) + assertNull(merged.controlPk, "a prior epoch's control_pk must not shadow the new epoch") + assertNull(merged.controlRoot, "a prior epoch's control_root must die with its epoch") + assertEquals("https://vector.chat/i/abc", merged.inviteRef) // the anchor, and only the anchor, survives + + val reversed = ConcordCommunityList.merge(rotated, stale).single() + assertEquals(2, reversed.rootEpoch) + assertNull(reversed.controlPk) + assertNull(reversed.controlRoot) + assertEquals("https://vector.chat/i/abc", reversed.inviteRef) + } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ControlPlaneSplitTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ControlPlaneSplitTest.kt new file mode 100644 index 0000000000..bbb60b54fc --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord02Community/ControlPlaneSplitTest.kt @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.concord.cord02Community + +import com.vitorpamplona.quartz.concord.cord04Roles.ControlEdition +import com.vitorpamplona.quartz.concord.cord04Roles.ControlEditionBuilder +import com.vitorpamplona.quartz.concord.cord04Roles.ControlEntityKind +import com.vitorpamplona.quartz.concord.crypto.ConcordKeyDerivation +import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys +import com.vitorpamplona.quartz.concord.envelope.ConcordStreamEnvelope +import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal +import com.vitorpamplona.quartz.utils.RandomInstance +import kotlinx.coroutines.test.runTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * The Control Plane's write restriction (CORD-01 Write-Restricted Streams, CORD-02 §2/§5). + * + * Every member holds the derived `control_pk` to subscribe, verify and read under the + * `community_root`-derived read key, but only the owner and staff hold the `control_root` + * the signer derives from — so a member can read every edition and mint none. + */ +class ControlPlaneSplitTest { + private val owner = NostrSignerInternal(KeyPair()) + private val now = 1_700_000_000L + + private val communityRoot = ByteArray(32) { 0x11 } + private val controlRoot = ByteArray(32) { 0x22 } + private val communityId = ByteArray(32) { 0x33 } + private val epoch = 0L + + private fun staffView() = ControlPlaneKeys.forStaff(communityRoot, communityId, epoch, controlRoot) + + private fun memberView() = ControlPlaneKeys.forMember(communityRoot, communityId, epoch, staffView().address) + + private suspend fun edition(createdAt: Long = now) = + ControlEditionBuilder.rumor( + authorPubKey = owner.pubKey, + entityKind = ControlEntityKind.METADATA, + entityId = communityId, + version = 0, + prevHash = null, + content = """{"name":"Nostrichs"}""", + createdAt = createdAt, + ) + + @Test + fun theSignerAndTheReadKeyAreDifferentKeysUnderDifferentLabels() { + val staff = staffView() + val legacy = ControlPlaneKeys.legacy(communityRoot, communityId, epoch) + + // The address derives from the control_root, the read key from the community_root. + assertEquals(ConcordKeyDerivation.controlSignerKey(controlRoot, communityId, epoch).publicKeyHex, staff.address) + assertNotEquals(staff.address, staff.readKey.publicKeyHex) + + // The two schemes never collide: different labels, different addresses (CORD-02 §5). + assertNotEquals(legacy.address, staff.address) + + // A legacy epoch is address, signer and read key at once — every member holds all three. + assertTrue(legacy.legacy) + assertTrue(legacy.canWrite) + assertEquals(legacy.address, legacy.readKey.publicKeyHex) + } + + @Test + fun aMemberReadsEveryEditionButHoldsNoWriteKey() = + runTest { + val staff = staffView() + val member = memberView() + + assertTrue(staff.canWrite) + assertFalse(member.canWrite, "a member must never hold the Control Plane write key") + // Same plane: same address to subscribe to, same conversation key to decrypt with. + assertEquals(staff.address, member.address) + assertContentEqualsHex(staff.readKey.conversationKey, member.readKey.conversationKey) + + val wrap = ConcordStreamEnvelope.wrap(edition(), staff, owner, encrypted = false, createdAt = now) + assertEquals(staff.address, wrap.pubKey) + + val opened = ConcordStreamEnvelope.openOrNull(wrap, member) + assertNotNull(opened, "a member must be able to read a staff-written edition") + assertEquals(owner.pubKey, opened.author) + assertNotNull(ControlEdition.fromRumor(opened.rumor)) + } + + @Test + fun aMemberCannotMintAWrapThatVerifiesAtThePlaneAddress() = + runTest { + val member = memberView() + + // The only stream key a member holds is the community_root-derived read key. Signing + // with it produces a wrap at the WRONG address — the spam gate the split exists for. + val forged = ConcordStreamEnvelope.wrap(edition(), member.readKey, owner, encrypted = false, createdAt = now) + assertNotEquals(member.address, forged.pubKey) + assertNull(ConcordStreamEnvelope.openOrNull(forged, member), "a member-signed wrap must not open at the plane") + assertNull(ConcordStreamEnvelope.openOrNull(forged, staffView())) + } + + @Test + fun aWrapFromAnUnrelatedKeyIsRefusedAtTheAddressCheck() = + runTest { + val staff = staffView() + // A spammer who somehow learned the read key still cannot mint at the address: the + // wrap's author must BE the address, and only control_root holders can produce it. + val strangerSecret = RandomInstance.bytes(32) + val stranger = ConcordKeyDerivation.groupKey("concord/whatever", strangerSecret, communityId, epoch) + val forged = ConcordStreamEnvelope.wrapSeal(ConcordStreamEnvelope.seal(edition(), staff.readKey, owner, encrypted = false), stranger, staff.readKey.conversationKey, createdAt = now) + + assertNull(ConcordStreamEnvelope.openOrNull(forged, staff)) + assertNull(ConcordStreamEnvelope.openOrNull(forged, memberView())) + } + + @Test + fun wrappingWithoutTheWriteKeyIsRefusedRatherThanSilentlyMissigned() = + runTest { + val member = memberView() + val seal = ConcordStreamEnvelope.seal(edition(), member.readKey, owner, encrypted = false) + var threw = false + try { + ConcordStreamEnvelope.wrapSeal(seal, member) + } catch (_: IllegalArgumentException) { + threw = true + } + assertTrue(threw, "wrapping on a plane we cannot write to must fail loudly") + } + + @Test + fun aLegacyEpochStaysReadableAfterTheSplitExists() = + runTest { + // A Community minted before the split keyed its plane by the member-held derivation. + // A client MUST retain that reading (CORD-02 §5) — the upgrade is the next Refounding. + val legacy = ControlPlaneKeys.legacy(communityRoot, communityId, epoch) + val wrap = ConcordStreamEnvelope.wrap(edition(), legacy, owner, encrypted = false, createdAt = now) + + val opened = ConcordStreamEnvelope.openOrNull(wrap, legacy) + assertNotNull(opened) + assertEquals(owner.pubKey, opened.author) + + // And it does not leak into the split scheme: the split plane refuses it. + assertNull(ConcordStreamEnvelope.openOrNull(wrap, staffView())) + } + + @Test + fun heldSecretsSelectTheViewOfAnEpoch() { + val staffAddress = staffView().address + + // Holding the secret: staff view, write key derived. + val asStaff = ControlPlaneKeys.of(communityRoot, communityId, epoch, controlPk = staffAddress, controlRoot = controlRoot.toHex()) + assertTrue(asStaff.canWrite) + assertEquals(staffAddress, asStaff.address) + + // Holding only the address: member view, read-only. + val asMember = ControlPlaneKeys.of(communityRoot, communityId, epoch, controlPk = staffAddress) + assertFalse(asMember.canWrite) + assertEquals(staffAddress, asMember.address) + + // Holding neither: a legacy, pre-split epoch. + val asLegacy = ControlPlaneKeys.of(communityRoot, communityId, epoch) + assertTrue(asLegacy.legacy) + assertNotEquals(staffAddress, asLegacy.address) + } + + private fun assertContentEqualsHex( + a: ByteArray, + b: ByteArray, + ) = assertEquals(a.toHex(), b.toHex()) + + private fun ByteArray.toHex(): String = joinToString("") { (it.toInt() and 0xFF).toString(16).padStart(2, '0') } +} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolverTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolverTest.kt index ec6ecae8c6..1d60fe7d91 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolverTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolverTest.kt @@ -582,4 +582,37 @@ class AuthorityResolverTest { assertFalse(r.isBanned(alice), "the part it does not outrank is dropped") assertTrue(r.isBanned(carol), "the part it does outrank still lands") } + + @Test + fun staffIsTheOwnerPlusEveryControlWritingBitHolder() { + // Staff (CORD-04 §3) is the set that holds the control_root (CORD-02 §2): the owner + // always, plus every non-banned holder of a Control-writing bit. This set decides who + // receives the 136-byte staff blob at a Refounding (CORD-06 §1). + val pinRole = "33".repeat(32) + // PIN_MESSAGES alone (bit 11 = 2048) writes Control editions, so it is a staff bit. + val pinJson = """{"name":"Curator","position":6,"permissions":"2048"}""" + val r = + AuthorityResolver.resolve( + listOf( + role(adminRole, adminJson), // MANAGE_ROLES|KICK|BAN → staff via MANAGE_ROLES/BAN + role(modRole, modJson), // KICK only → Guestbook writer, NOT staff + role(pinRole, pinJson), + grant("31".repeat(32), alice, listOf(adminRole), granter = owner), + grant("32".repeat(32), bob, listOf(modRole), granter = owner), + grant("33".repeat(32), carol, listOf(adminRole), granter = owner), + grant("34".repeat(32), dave, listOf(pinRole), granter = owner), + banlist(carol), // a banned admin loses staff standing with everything else + ), + owner, + ) + + assertTrue(r.isStaff(owner), "the owner is always staff") + assertTrue(r.isStaff(alice), "a Control-writing bit makes staff") + assertTrue(r.isStaff(dave), "PIN_MESSAGES lands as Control editions, so it is a staff bit") + assertFalse(r.isStaff(bob), "KICK writes to the Guestbook, never the Control Plane") + assertFalse(r.isStaff(carol), "a banned member is not staff") + assertFalse(r.isStaff("e5".repeat(32)), "a roleless member is not staff") + + assertEquals(setOf(owner, alice, dave), r.staffMembers()) + } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlRootWrapTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlRootWrapTest.kt new file mode 100644 index 0000000000..5feb9ddeb2 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlRootWrapTest.kt @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.concord.cord04Roles + +import com.vitorpamplona.quartz.concord.crypto.ConcordKeyDerivation +import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal +import kotlinx.coroutines.test.runTest +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * The staff write-key delivery riding a Grant (CORD-04 §3): promotion and key delivery + * are one signed edition, opaque pairwise ciphertext to every other reader, and adopted + * only after the derive-check — which fails closed. + */ +class ControlRootWrapTest { + private val granter = NostrSignerInternal(KeyPair()) + private val member = NostrSignerInternal(KeyPair()) + private val stranger = NostrSignerInternal(KeyPair()) + + private val communityId = ByteArray(32) { 0x33 } + private val controlRoot = ByteArray(32) { 0x22 } + private val epoch = 7L + + private fun controlPkAt( + root: ByteArray = controlRoot, + at: Long = epoch, + ) = ConcordKeyDerivation.controlSignerKey(root, communityId, at).publicKeyHex + + @Test + fun theWirePlaintextIsFortyBytesEpochThenSecret() { + val plaintext = ControlRootWrap.encodePlaintext(epoch, controlRoot) + assertEquals(ControlRootWrap.SIZE, plaintext.size) + assertEquals(40, plaintext.size) + + val decoded = ControlRootWrap.decodePlaintext(plaintext) + assertNotNull(decoded) + assertEquals(epoch, decoded.epoch) + assertContentEquals(controlRoot, decoded.controlRoot) + + // Any other width is malformed — the rekey-blob discipline (CORD-06 §1). + assertNull(ControlRootWrap.decodePlaintext(ByteArray(39))) + assertNull(ControlRootWrap.decodePlaintext(ByteArray(41))) + } + + @Test + fun thePromotedMemberOpensItAndNobodyElseCan() = + runTest { + val wrap = ControlRootWrap.build(granter, member.pubKey, epoch, controlRoot) + + val opened = ControlRootWrap.openOrNull(wrap, member, granter.pubKey) + assertNotNull(opened) + assertEquals(epoch, opened.epoch) + assertContentEquals(controlRoot, opened.controlRoot) + + // Every other reader of the plane sees opaque bytes. + assertNull(ControlRootWrap.openOrNull(wrap, stranger, granter.pubKey)) + } + + @Test + fun theGranterCanReopenItsOwnDeliveryBecauseTheKeyIsPairwise() = + runTest { + // One ECDH either side can compute, so a NIP-46 bunker account opens it with a + // single nip44_decrypt and a re-issuing staffer needs no stored copy. + val wrap = ControlRootWrap.build(granter, member.pubKey, epoch, controlRoot) + val opened = ControlRootWrap.openOrNull(wrap, granter, member.pubKey) + assertNotNull(opened) + assertContentEquals(controlRoot, opened.controlRoot) + } + + @Test + fun adoptionRequiresTheSecretToDeriveToTheHeldAddress() { + assertTrue(ControlRootWrap.derivesTo(controlRoot, communityId, epoch, controlPkAt())) + + // A garbage secret is attributable griefing, nothing worse: it is dropped, never adopted. + assertFalse(ControlRootWrap.derivesTo(ByteArray(32) { 0x77 }, communityId, epoch, controlPkAt())) + + // The epoch binds too — a secret for another epoch derives elsewhere. + assertFalse(ControlRootWrap.derivesTo(controlRoot, communityId, epoch + 1, controlPkAt())) + + // And so does the community: the same secret in another Community is another plane. + assertFalse(ControlRootWrap.derivesTo(controlRoot, ByteArray(32) { 0x44 }, epoch, controlPkAt())) + } + + @Test + fun aWrapMintedForAPriorEpochFailsTheCheckRatherThanBeingAdopted() = + runTest { + // Compaction re-wraps a Grant head verbatim across Refoundings, so a folded head can + // carry a wrap minted for a prior epoch's key. Staleness is structural and harmless. + val staleWrap = ControlRootWrap.build(granter, member.pubKey, epoch, controlRoot) + val opened = ControlRootWrap.openOrNull(staleWrap, member, granter.pubKey) + assertNotNull(opened) + + val currentEpoch = epoch + 1 + val currentControlRoot = ByteArray(32) { 0x55 } + assertEquals(epoch, opened.epoch, "the epoch rides inside the ciphertext, not beside it") + assertFalse( + ControlRootWrap.derivesTo(opened.controlRoot, communityId, currentEpoch, controlPkAt(currentControlRoot, currentEpoch)), + "a stale wrap must fail closed at the current epoch", + ) + } + + @Test + fun aGrantCarriesTheWrapThroughTheWireShapeAndSurvivesARoundTrip() = + runTest { + val wrap = ControlRootWrap.build(granter, member.pubKey, epoch, controlRoot) + val grant = GrantEntity(member = member.pubKey, roleIds = listOf("ab".repeat(32)), controlWrap = wrap) + + val json = ConcordJson.instance.encodeToString(GrantEntity.serializer(), grant) + assertTrue(json.contains("control_wrap"), "the wire field is snake_case (CORD-04 §2)") + + val decoded = ConcordJson.decodeOrNull(json) + assertNotNull(decoded) + assertEquals(wrap, decoded.controlWrap) + + // A plain grant carries none, and a reader must cope with its absence. + val plain = ConcordJson.decodeOrNull("""{"member":"${member.pubKey}","role_ids":[]}""") + assertNotNull(plain) + assertNull(plain.controlWrap) + } + + @Test + fun theStaffBitsAreTheControlWritingPermissions() { + // The six bits whose actions land as Control editions (CORD-04 §3). + val staff = ConcordPermissions.STAFF_BITS + assertTrue(staff.has(ConcordPermissions.MANAGE_ROLES)) + assertTrue(staff.has(ConcordPermissions.MANAGE_CHANNELS)) + assertTrue(staff.has(ConcordPermissions.MANAGE_METADATA)) + assertTrue(staff.has(ConcordPermissions.BAN)) + assertTrue(staff.has(ConcordPermissions.CREATE_INVITE)) + assertTrue(staff.has(ConcordPermissions.PIN_MESSAGES)) + + // KICK writes to the Guestbook and MANAGE_MESSAGES to Chat planes; neither needs the key. + assertFalse(staff.has(ConcordPermissions.KICK)) + assertFalse(staff.has(ConcordPermissions.MANAGE_MESSAGES)) + + // PIN_MESSAGES claims the frozen bit 11 (CORD-04 §3 table). + assertEquals(11, ConcordPermissions.PIN_MESSAGES) + } +} diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteJoinFlowTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteJoinFlowTest.kt index 3d365679c8..5d6d9209ca 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteJoinFlowTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteJoinFlowTest.kt @@ -23,7 +23,7 @@ package com.vitorpamplona.quartz.concord.cord05Invites import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityFactory import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityState import com.vitorpamplona.quartz.concord.cord04Roles.ControlEdition -import com.vitorpamplona.quartz.concord.crypto.ConcordKeyDerivation +import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys import com.vitorpamplona.quartz.concord.envelope.ConcordStreamEnvelope import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray import com.vitorpamplona.quartz.nip01Core.core.toHexKey @@ -52,6 +52,9 @@ class ConcordInviteJoinFlowTest { ownerSalt = community.ownerSalt.toHexKey(), communityRoot = community.communityRoot.toHexKey(), rootEpoch = community.rootEpoch, + // Read access to the Control Plane, never write (CORD-05 §1): the joiner cannot + // derive this address, so the bundle is the only place it can come from. + controlPk = community.controlPkHex, relays = listOf("wss://relay.example"), name = "Nostrichs", ) @@ -73,13 +76,18 @@ class ConcordInviteJoinFlowTest { assertTrue(ConcordInviteBundle.validate(invite)) assertEquals(community.communityIdHex, invite.communityId) - // Reconstruct the root, derive the Control Plane, and read the genesis. + // Reconstruct the root and open the Control Plane as a plain member does: the + // delivered control_pk is the address to verify against, the derived read key + // decrypts (CORD-02 §5). No write key anywhere on this path. + assertNotNull(invite.controlPk) val controlPlane = - ConcordKeyDerivation.controlPlaneKey( + ControlPlaneKeys.forMember( invite.communityRoot.hexToByteArray(), invite.communityId.hexToByteArray(), invite.rootEpoch, + invite.controlPk, ) + assertFalse(controlPlane.canWrite, "an invite must never hand a joiner the write key") val editions = community.genesisWraps.mapNotNull { ControlEdition.fromRumor(ConcordStreamEnvelope.open(it, controlPlane).rumor) } val state = ConcordCommunityState.fold(editions, invite.owner) assertEquals("Nostrichs", state.metadata?.name) diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefoundingTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefoundingTest.kt index fa919e9b14..06778c4c71 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefoundingTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefoundingTest.kt @@ -48,6 +48,9 @@ class ConcordRefoundingTest { private val carol = NostrSignerInternal(KeyPair()) // removed private val newRoot = ByteArray(32) { 0x5A } + + /** The fresh staff write key minted beside [newRoot] at every Refounding (CORD-02 §2). */ + private val newControlRoot = ByteArray(32) { 0x6B } private val now = 1_700_000_000L @Test @@ -64,10 +67,12 @@ class ConcordRefoundingTest { communityId = communityId, priorRoot = priorRoot, newRoot = newRoot, + newControlRoot = newControlRoot, rootEpoch = community.rootEpoch, priorControlWraps = community.genesisWraps, - priorControlKey = priorControl, + priorControlKeys = priorControl, recipientsXOnly = listOf(alice.pubKey, bob.pubKey), + staffXOnly = setOf(owner.pubKey), createdAt = now, ) @@ -77,9 +82,9 @@ class ConcordRefoundingTest { val baseRekeyKey = ConcordKeyDerivation.baseRekeyAddress(priorRoot, communityId, build.newEpoch) // Alice and Bob find the new root; Carol (no blob) does not. - val aliceRoot = ConcordRefounding.findNewRoot(build.rekeyWraps, baseRekeyKey, alice, priorRoot, community.rootEpoch) - val bobRoot = ConcordRefounding.findNewRoot(build.rekeyWraps, baseRekeyKey, bob, priorRoot, community.rootEpoch) - val carolRoot = ConcordRefounding.findNewRoot(build.rekeyWraps, baseRekeyKey, carol, priorRoot, community.rootEpoch) + val aliceRoot = ConcordRefounding.findNewRoot(build.rekeyWraps, baseRekeyKey, alice, communityId, priorRoot, community.rootEpoch) + val bobRoot = ConcordRefounding.findNewRoot(build.rekeyWraps, baseRekeyKey, bob, communityId, priorRoot, community.rootEpoch) + val carolRoot = ConcordRefounding.findNewRoot(build.rekeyWraps, baseRekeyKey, carol, communityId, priorRoot, community.rootEpoch) assertNotNull(aliceRoot) assertContentEquals(newRoot, aliceRoot.newRoot) @@ -101,14 +106,16 @@ class ConcordRefoundingTest { communityId = communityId, priorRoot = community.communityRoot, newRoot = newRoot, + newControlRoot = newControlRoot, rootEpoch = community.rootEpoch, priorControlWraps = community.genesisWraps, - priorControlKey = community.controlPlane, + priorControlKeys = community.controlPlane, recipientsXOnly = listOf(alice.pubKey), + staffXOnly = setOf(owner.pubKey), createdAt = now, ) - val newControl = ConcordKeyDerivation.controlPlaneKey(newRoot, communityId, build.newEpoch) + val newControl = build.newControlKeys // Re-open the compacted wraps under the NEW control key and fold: same authority + metadata. val editions = @@ -170,14 +177,16 @@ class ConcordRefoundingTest { communityId = communityId, priorRoot = community.communityRoot, newRoot = newRoot, + newControlRoot = newControlRoot, rootEpoch = community.rootEpoch, priorControlWraps = priorWraps, - priorControlKey = control, + priorControlKeys = control, recipientsXOnly = listOf(alice.pubKey), + staffXOnly = setOf(owner.pubKey), createdAt = now, ) - val newControl = ConcordKeyDerivation.controlPlaneKey(newRoot, communityId, build.newEpoch) + val newControl = build.newControlKeys val editions = build.controlWraps.mapNotNull { wrap -> ConcordStreamEnvelope.openOrNull(wrap, newControl)?.let { ControlEdition.fromRumor(it.rumor) } @@ -207,16 +216,18 @@ class ConcordRefoundingTest { communityId = community.communityId, priorRoot = community.communityRoot, newRoot = newRoot, + newControlRoot = newControlRoot, rootEpoch = community.rootEpoch, priorControlWraps = community.genesisWraps, - priorControlKey = community.controlPlane, + priorControlKeys = community.controlPlane, recipientsXOnly = listOf(alice.pubKey), + staffXOnly = setOf(owner.pubKey), createdAt = now, ) val baseRekeyKey = ConcordKeyDerivation.baseRekeyAddress(community.communityRoot, community.communityId, build.newEpoch) // Alice claims a different prior root: prevcommit mismatch ⇒ rotation rejected. val wrongRoot = ByteArray(32) { 0x11 } - assertNull(ConcordRefounding.findNewRoot(build.rekeyWraps, baseRekeyKey, alice, wrongRoot, community.rootEpoch)) + assertNull(ConcordRefounding.findNewRoot(build.rekeyWraps, baseRekeyKey, alice, community.communityId, wrongRoot, community.rootEpoch)) } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ControlRootRotationTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ControlRootRotationTest.kt new file mode 100644 index 0000000000..bcd5eec1e5 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ControlRootRotationTest.kt @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.concord.cord06Rekey + +import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityFactory +import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityState +import com.vitorpamplona.quartz.concord.cord04Roles.ControlEdition +import com.vitorpamplona.quartz.concord.crypto.ConcordKeyDerivation +import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys +import com.vitorpamplona.quartz.concord.envelope.ConcordStreamEnvelope +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray +import com.vitorpamplona.quartz.nip01Core.core.toHexKey +import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal +import com.vitorpamplona.quartz.nip59Giftwrap.rumors.RumorAssembler +import kotlinx.coroutines.test.runTest +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +/** + * The `control_root` rolling with the `community_root` at every Refounding + * (CORD-02 §2, CORD-06 §1/§3): base blobs carry the new pk to members and the new + * secret to staff, and the blob's width declares which form it is. + */ +class ControlRootRotationTest { + private val owner = NostrSignerInternal(KeyPair()) // rotator, and staff by definition + private val moderator = NostrSignerInternal(KeyPair()) // staff + private val member = NostrSignerInternal(KeyPair()) // plain member + private val removed = NostrSignerInternal(KeyPair()) + + private val newRoot = ByteArray(32) { 0x5A } + private val newControlRoot = ByteArray(32) { 0x6B } + private val now = 1_700_000_000L + + @Test + fun theBlobWidthDeclaresItsForm() { + val scope = ByteArray(32) { 0x01 } + val key = ByteArray(32) { 0x02 } + val pk = ByteArray(32) { 0x03 } + val secret = ByteArray(32) { 0x04 } + + assertEquals(RekeyPayload.SIZE_CHANNEL, RekeyPayload(scope, 1, key).encode().size) + assertEquals(RekeyPayload.SIZE_BASE_MEMBER, RekeyPayload(scope, 1, key, pk).encode().size) + assertEquals(RekeyPayload.SIZE_BASE_STAFF, RekeyPayload(scope, 1, key, pk, secret).encode().size) + assertEquals(72, RekeyPayload.SIZE_CHANNEL) + assertEquals(104, RekeyPayload.SIZE_BASE_MEMBER) + assertEquals(136, RekeyPayload.SIZE_BASE_STAFF) + + // Round-trips keep exactly what each form carries, and nothing it doesn't. + val channel = RekeyPayload.decode(RekeyPayload(scope, 1, key).encode()) + assertNotNull(channel) + assertNull(channel.newControlPk) + assertNull(channel.newControlRoot) + + val memberBlob = RekeyPayload.decode(RekeyPayload(scope, 1, key, pk).encode()) + assertNotNull(memberBlob) + assertContentEquals(pk, memberBlob.newControlPk) + assertNull(memberBlob.newControlRoot, "a member's blob must never carry the write key") + + val staffBlob = RekeyPayload.decode(RekeyPayload(scope, 1, key, pk, secret).encode()) + assertNotNull(staffBlob) + assertContentEquals(pk, staffBlob.newControlPk) + assertContentEquals(secret, staffBlob.newControlRoot) + + // Any other width is malformed and the blob is dropped. + assertNull(RekeyPayload.decode(ByteArray(71))) + assertNull(RekeyPayload.decode(ByteArray(103))) + assertNull(RekeyPayload.decode(ByteArray(137))) + } + + @Test + fun staffGetTheSecretMembersOnlyThePubkeyAndRemovedNothing() = + runTest { + val community = ConcordCommunityFactory.create(owner, "Test", now) + val build = + ConcordRefounding.build( + rotatorSigner = owner, + communityId = community.communityId, + priorRoot = community.communityRoot, + newRoot = newRoot, + newControlRoot = newControlRoot, + rootEpoch = community.rootEpoch, + priorControlWraps = community.genesisWraps, + priorControlKeys = community.controlPlane, + recipientsXOnly = listOf(owner.pubKey, moderator.pubKey, member.pubKey), + staffXOnly = setOf(owner.pubKey, moderator.pubKey), + createdAt = now, + ) + + val baseRekey = ConcordKeyDerivation.baseRekeyAddress(community.communityRoot, community.communityId, build.newEpoch) + + suspend fun received(who: NostrSignerInternal) = ConcordRefounding.findNewRoot(build.rekeyWraps, baseRekey, who, community.communityId, community.communityRoot, community.rootEpoch) + + val expectedPk = ConcordKeyDerivation.controlSignerKey(newControlRoot, community.communityId, build.newEpoch).publicKey + + val asModerator = received(moderator) + assertNotNull(asModerator) + assertContentEquals(newRoot, asModerator.newRoot) + assertContentEquals(expectedPk, asModerator.newControlPk) + assertContentEquals(newControlRoot, asModerator.newControlRoot, "staff must receive the new write key") + + val asMember = received(member) + assertNotNull(asMember) + assertContentEquals(newRoot, asMember.newRoot) + assertContentEquals(expectedPk, asMember.newControlPk, "every member must receive the new address") + assertNull(asMember.newControlRoot, "a plain member must never receive the write key") + + assertNull(received(removed), "a removed member receives no blob at all") + } + + @Test + fun theRotatedPlaneIsWritableByStaffAndReadableByEveryMember() = + runTest { + val community = ConcordCommunityFactory.create(owner, "Test", now, description = "A place") + val build = + ConcordRefounding.build( + rotatorSigner = owner, + communityId = community.communityId, + priorRoot = community.communityRoot, + newRoot = newRoot, + newControlRoot = newControlRoot, + rootEpoch = community.rootEpoch, + priorControlWraps = community.genesisWraps, + priorControlKeys = community.controlPlane, + recipientsXOnly = listOf(owner.pubKey, member.pubKey), + staffXOnly = setOf(owner.pubKey), + createdAt = now, + ) + + // The rotator's own view writes; a member's view of the same epoch only reads. + assertTrue(build.newControlKeys.canWrite) + val memberView = + ControlPlaneKeys.forMember(newRoot, community.communityId, build.newEpoch, build.newControlKeys.address) + assertFalse(memberView.canWrite) + + // Every compacted wrap sits at the new signer's address and opens for the member. + assertTrue(build.controlWraps.isNotEmpty()) + build.controlWraps.forEach { assertEquals(build.newControlKeys.address, it.pubKey) } + + val editions = + build.controlWraps.mapNotNull { wrap -> + ConcordStreamEnvelope.openOrNull(wrap, memberView)?.let { ControlEdition.fromRumor(it.rumor) } + } + val folded = ConcordCommunityState.fold(editions, owner.pubKey) + assertEquals("Test", folded.metadata?.name) + assertTrue(folded.channels.isNotEmpty()) + } + + @Test + fun aStaffBlobWhoseSecretDoesNotDeriveToItsPubkeyIsRefused() = + runTest { + val community = ConcordCommunityFactory.create(owner, "Test", now) + val newEpoch = community.rootEpoch + 1 + + // A rotator that splits the plane from its own readers: the delivered secret derives + // to a DIFFERENT address than the one every member was handed (CORD-06 §1). + val mismatchedPk = ConcordKeyDerivation.controlSignerKey(ByteArray(32) { 0x7C }, community.communityId, newEpoch).publicKey + val blob = + ConcordRekey.blobForSigner( + rotatorSigner = owner, + recipientXOnly = moderator.pubKey.hexToByteArray(), + scopeId = ConcordRekey.ROOT_SCOPE, + newEpoch = newEpoch, + newKey = newRoot, + newControlPk = mismatchedPk, + newControlRoot = newControlRoot, + ) + + val baseRekey = ConcordKeyDerivation.baseRekeyAddress(community.communityRoot, community.communityId, newEpoch) + val prevCommit = ConcordKeyDerivation.epochKeyCommitment(community.rootEpoch, community.communityRoot).toHexKey() + val tags = ConcordRekey.tags(ConcordRekey.ROOT_SCOPE, newEpoch, community.rootEpoch, prevCommit, 0, 1) + val rumor = + RumorAssembler.assembleRumor(owner.pubKey, now, ConcordRekey.KIND, tags, ConcordRekey.encodeContent(listOf(blob))) + val wrap = ConcordStreamEnvelope.wrap(rumor, baseRekey, owner, encrypted = true, createdAt = now) + + assertNull( + ConcordRefounding.findNewRoot(listOf(wrap), baseRekey, moderator, community.communityId, community.communityRoot, community.rootEpoch), + "a mismatched control pair must be refused rather than adopted", + ) + } + + @Test + fun aLegacySeventyTwoByteBaseBlobStillDeliversItsRoot() = + runTest { + // A pre-split rotation carries no control material; it is honored when reading old + // epochs (CORD-06 §3) and its acceptor keeps folding at the legacy address. + val community = ConcordCommunityFactory.create(owner, "Test", now) + val newEpoch = community.rootEpoch + 1 + + val blob = + ConcordRekey.blobForSigner( + rotatorSigner = owner, + recipientXOnly = member.pubKey.hexToByteArray(), + scopeId = ConcordRekey.ROOT_SCOPE, + newEpoch = newEpoch, + newKey = newRoot, + ) + + val baseRekey = ConcordKeyDerivation.baseRekeyAddress(community.communityRoot, community.communityId, newEpoch) + val prevCommit = ConcordKeyDerivation.epochKeyCommitment(community.rootEpoch, community.communityRoot).toHexKey() + val tags = ConcordRekey.tags(ConcordRekey.ROOT_SCOPE, newEpoch, community.rootEpoch, prevCommit, 0, 1) + val rumor = + RumorAssembler.assembleRumor(owner.pubKey, now, ConcordRekey.KIND, tags, ConcordRekey.encodeContent(listOf(blob))) + val wrap = ConcordStreamEnvelope.wrap(rumor, baseRekey, owner, encrypted = true, createdAt = now) + + val got = ConcordRefounding.findNewRoot(listOf(wrap), baseRekey, member, community.communityId, community.communityRoot, community.rootEpoch) + assertNotNull(got) + assertContentEquals(newRoot, got.newRoot) + assertNull(got.newControlPk, "a legacy base blob announces a pre-split epoch") + assertNull(got.newControlRoot) + } +} From 6472099d3a07ec1bd9f4cdd8bc8804e585523ed3 Mon Sep 17 00:00:00 2001 From: davotoula Date: Fri, 7 Aug 2026 07:49:17 +0200 Subject: [PATCH 059/132] fix(media): repair bare-subtype imeta mimes so sharing works MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A NIP-92 `imeta` is meant to carry a full `type/subtype`, but some clients emit only the subtype — Primal iOS writes `m jpeg` instead of `m image/jpeg`. --- .../amethyst/ui/components/ShareHelper.kt | 23 +++++++ .../ui/components/ZoomableContentView.kt | 6 +- .../amethyst/ui/components/ShareHelperTest.kt | 37 +++++++++++ .../commons/richtext/RichTextParser.kt | 22 ++++++- .../commons/richtext/PdfParserTest.kt | 25 ++++++-- .../RichTextParserMalformedMimeTest.kt | 62 +++++++++++++++++++ 6 files changed, 167 insertions(+), 8 deletions(-) create mode 100644 commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserMalformedMimeTest.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ShareHelper.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ShareHelper.kt index 02592b6a06..1764f6b2a9 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ShareHelper.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ShareHelper.kt @@ -25,6 +25,8 @@ import android.net.Uri import androidx.annotation.VisibleForTesting import androidx.core.content.FileProvider import com.vitorpamplona.amethyst.Amethyst +import com.vitorpamplona.amethyst.commons.richtext.mimeTypeMap +import com.vitorpamplona.amethyst.commons.richtext.normalizeMimeType import com.vitorpamplona.quartz.utils.Log import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.withContext @@ -65,6 +67,27 @@ object ShareHelper { private val MP4_BRAND_MP42 = "mp42".toByteArray() private val MOV_BRAND_QT = "qt ".toByteArray() + /** + * Picks the MIME type to put on an `ACTION_SEND` intent. + * + * `Intent.type` has to be a real `type/subtype`: an `IntentFilter` matches the two halves + * separately, so a slash-less value like `jpeg` matches nothing and the chooser opens empty — + * the share silently does nothing. Events can absolutely carry such a value, because NIP-92 + * `imeta`/NIP-94 `m` tags are author-supplied and some clients write the bare subtype (Primal + * iOS emits `m jpeg`). Treat the declared type as a hint, not as truth. + * + * [fileExtension] is the safer signal — [getMediaExtension] sniffs it from the file's magic + * numbers rather than trusting the event — so it backs up an unusable declaration. + */ + internal fun resolveShareMimeType( + declaredMimeType: String?, + fileExtension: String, + defaultTypePrefix: String, + ): String = + normalizeMimeType(declaredMimeType) + ?: mimeTypeMap[fileExtension.lowercase()] + ?: "$defaultTypePrefix/$fileExtension" + suspend fun getSharableUriFromUrl( context: Context, imageUrl: String, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ZoomableContentView.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ZoomableContentView.kt index 3e560530cc..a09e69278a 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ZoomableContentView.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ZoomableContentView.kt @@ -1103,7 +1103,7 @@ private suspend fun shareImageFile( val (uri, fileExtension) = ShareHelper.getSharableUriFromUrl(context, videoUri) // Determine mime type, use provided or derive from extension - val determinedMimeType = mimeType ?: "image/$fileExtension" + val determinedMimeType = ShareHelper.resolveShareMimeType(mimeType, fileExtension, "image") // Create share intent val shareIntent = @@ -1161,7 +1161,7 @@ private suspend fun shareVideoFile( sharedFile = sharableFile // Determine mime type - val determinedMimeType = mimeType ?: "video/$extension" + val determinedMimeType = ShareHelper.resolveShareMimeType(mimeType, extension, "video") // Create share intent val shareIntent = @@ -1227,7 +1227,7 @@ private suspend fun shareLocalVideoFile( val (uri, extension) = ShareHelper.getSharableUriForLocalVideo(context, localFile) // Determine mime type - val determinedMimeType = mimeType ?: "video/$extension" + val determinedMimeType = ShareHelper.resolveShareMimeType(mimeType, extension, "video") // Create share intent val shareIntent = diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/ui/components/ShareHelperTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/ui/components/ShareHelperTest.kt index 8bbc8a29ae..e9336b351b 100644 --- a/amethyst/src/test/java/com/vitorpamplona/amethyst/ui/components/ShareHelperTest.kt +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/ui/components/ShareHelperTest.kt @@ -162,4 +162,41 @@ class ShareHelperTest { file.writeBytes(bytes) return file } + + // A slash-less Intent.type matches no IntentFilter, so the chooser opens with zero targets and + // the share silently fails. Author-supplied `m` tags can carry exactly that (Primal iOS emits + // `m jpeg`), so a bare subtype must be repaired rather than forwarded. + @Test + fun resolveShareMimeType_bareSubtype_isExpandedToFullMimeType() { + assertEquals("image/jpeg", ShareHelper.resolveShareMimeType("jpeg", "jpg", "image")) + assertEquals("video/mp4", ShareHelper.resolveShareMimeType("mp4", "mp4", "video")) + } + + @Test + fun resolveShareMimeType_wellFormedDeclaration_isPreserved() { + assertEquals("image/png", ShareHelper.resolveShareMimeType("image/png", "png", "image")) + } + + // An unusable declaration falls back to the extension, which is sniffed from the file's magic + // numbers and so is not attacker-controlled. + @Test + fun resolveShareMimeType_unrecognizableDeclaration_fallsBackToSniffedExtension() { + assertEquals("image/png", ShareHelper.resolveShareMimeType("notatype", "png", "image")) + } + + @Test + fun resolveShareMimeType_noDeclaration_usesCanonicalTypeForExtension() { + // Not "image/jpg" -- jpg is not a registered subtype. + assertEquals("image/jpeg", ShareHelper.resolveShareMimeType(null, "jpg", "image")) + assertEquals("video/quicktime", ShareHelper.resolveShareMimeType(null, "mov", "video")) + } + + @Test + fun resolveShareMimeType_alwaysProducesASlashSeparatedType() { + val cases = listOf(null, "", "jpeg", "image/jpeg", "notatype", "JPEG") + cases.forEach { declared -> + val resolved = ShareHelper.resolveShareMimeType(declared, "jpg", "image") + assertTrue("`$declared` resolved to un-matchable type `$resolved`", resolved.contains("/")) + } + } } diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt index 0ecc3b82f0..51765fe3bb 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt @@ -59,7 +59,7 @@ class RichTextParser { val tags = eventTags.get(fullUrl)?.properties ?: emptyMap() - val contentType = frags[MimeTypeTag.TAG_NAME] ?: tags[MimeTypeTag.TAG_NAME]?.firstOrNull() + val contentType = normalizeMimeType(frags[MimeTypeTag.TAG_NAME] ?: tags[MimeTypeTag.TAG_NAME]?.firstOrNull()) // Returning null here drops the URL to a plain link, discarding the imeta's `dim`/blurhash // and forcing a URL-preview round-trip to rediscover a type the imeta already declared — @@ -681,3 +681,23 @@ val mimeTypeMap: Map = // Documents "pdf" to "application/pdf", ) + +/** + * NIP-92's `m` property is meant to carry a full `type/subtype`, but several clients emit the + * bare subtype instead — Primal iOS writes `m jpeg` rather than `m image/jpeg`. That value is + * useless as a MIME type: it matches none of the `startsWith("image/")`-style checks, and once + * it is stored on the media model it travels all the way into Android's `ACTION_SEND` as + * `Intent.type = "jpeg"`. No `` filter matches a type without a slash, + * so the share sheet opens with zero targets and the image cannot be shared at all. + * + * Map a bare subtype back onto its canonical MIME so every downstream consumer (the share + * intent, the gallery entry's published `m` tag, the player's type hint) sees a well-formed + * value. Anything already containing a `/` is passed through untouched, and an unrecognised + * bare token is dropped to null rather than propagated — that leaves the caller's + * extension-based detection to decide, which is strictly better than carrying garbage forward. + */ +fun normalizeMimeType(rawMimeType: String?): String? { + if (rawMimeType == null) return null + if (rawMimeType.contains('/')) return rawMimeType + return mimeTypeMap[rawMimeType.lowercase()] +} diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/PdfParserTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/PdfParserTest.kt index 2dc2f906fa..53c0cfb947 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/PdfParserTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/PdfParserTest.kt @@ -112,10 +112,10 @@ class PdfParserTest { assertTrue(videoMedia is MediaUrlVideo, "Expected MediaUrlVideo despite the malformed `m mp4` mime") } - // A malformed mime on a URL with *no* recognizable extension can't be recovered — it stays a - // link. This documents the boundary of the fallback so it isn't mistaken for a regression. + // A bare-subtype mime is recovered from the imeta itself, so an extensionless URL — the shape + // Blossom hands out, where the imeta is the only type signal there is — still renders. @Test - fun malformedImetaMimeWithoutExtensionStaysUnclassified() { + fun malformedImetaMimeWithoutExtensionIsRecoveredFromTheMime() { val url = "https://files.example.com/abcd1234" val tags = ImmutableListOfLists( @@ -126,6 +126,23 @@ class PdfParserTest { val state = RichTextParser().parseText(url, tags, null) - assertEquals(null, state.mediaForPager[url], "No extension to recover from -> not treated as media") + assertTrue(state.mediaForPager[url] is MediaUrlImage, "`m jpeg` alone is enough to classify the media") + } + + // The boundary: a bare token that maps to no known type, on a URL with no extension, has + // nothing left to recover from. This documents the limit so it isn't mistaken for a regression. + @Test + fun unrecognizableImetaMimeWithoutExtensionStaysUnclassified() { + val url = "https://files.example.com/abcd1234" + val tags = + ImmutableListOfLists( + arrayOf( + arrayOf("imeta", "url $url", "m notarealtype"), + ), + ) + + val state = RichTextParser().parseText(url, tags, null) + + assertEquals(null, state.mediaForPager[url], "Nothing to recover from -> not treated as media") } } diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserMalformedMimeTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserMalformedMimeTest.kt new file mode 100644 index 0000000000..f5067d0d4b --- /dev/null +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserMalformedMimeTest.kt @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.richtext + +import com.vitorpamplona.quartz.nip92IMeta.IMetaTag +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class RichTextParserMalformedMimeTest { + private val url = "https://blossom.primal.net/c27d7b7be6e58d69b29006cc275d29d67967760b1772e50a79d5b24f60d62fc5.jpg" + + private fun parse(mime: String): MediaUrlContent? = + RichTextParser().createMediaContent( + fullUrl = url, + eventTags = + mapOf( + url to + IMetaTag( + url = url, + properties = mapOf("m" to listOf(mime), "dim" to listOf("960.0x1358.0")), + ), + ), + description = null, + ) + + @Test + fun malformedImetaMimeStillRendersAsImage() { + val content = parse("jpeg") + assertTrue(content is MediaUrlImage, "bare `m jpeg` must still route to MediaUrlImage") + } + + @Test + fun malformedImetaMimeIsNormalizedForSharing() { + val content = parse("jpeg") as MediaUrlImage + assertEquals("image/jpeg", content.mimeType) + } + + @Test + fun wellFormedImetaMimeIsUntouched() { + val content = parse("image/jpeg") as MediaUrlImage + assertEquals("image/jpeg", content.mimeType) + } +} From ee4a5e5b8928b5367f9e1b171c1543f8b3db93d9 Mon Sep 17 00:00:00 2001 From: davotoula Date: Fri, 7 Aug 2026 08:23:45 +0200 Subject: [PATCH 060/132] Code review: - fix(media): stop ogg bypassing the ambiguity guard it is listed in - fix(media): don't guess a family for an ambiguous bare subtype - refactor(media): normalize the mime at the chokepoints, not one call site --- .../amethyst/ui/components/ShareHelper.kt | 19 +++--- .../ui/components/ZoomableContentView.kt | 6 +- .../amethyst/ui/components/ShareHelperTest.kt | 23 ++++--- .../commons/richtext/MediaContentModels.kt | 12 +++- .../commons/richtext/RichTextParser.kt | 58 ++++++++++++---- .../commons/richtext/ClassifyMediaTest.kt | 68 ++++++++++++++++++- .../commons/richtext/PdfParserTest.kt | 51 +++++--------- .../RichTextParserMalformedMimeTest.kt | 62 ----------------- 8 files changed, 164 insertions(+), 135 deletions(-) delete mode 100644 commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserMalformedMimeTest.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ShareHelper.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ShareHelper.kt index 1764f6b2a9..9a96186328 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ShareHelper.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ShareHelper.kt @@ -39,6 +39,7 @@ object ShareHelper { private const val DEFAULT_IMAGE_EXTENSION = "jpg" private const val DEFAULT_VIDEO_EXTENSION = "mp4" private const val SHARED_FILE_PREFIX = "shared_media" + private const val GENERIC_BINARY_MIME_TYPE = "application/octet-stream" data class SharableFile( val uri: Uri, @@ -70,23 +71,21 @@ object ShareHelper { /** * Picks the MIME type to put on an `ACTION_SEND` intent. * - * `Intent.type` has to be a real `type/subtype`: an `IntentFilter` matches the two halves - * separately, so a slash-less value like `jpeg` matches nothing and the chooser opens empty — - * the share silently does nothing. Events can absolutely carry such a value, because NIP-92 - * `imeta`/NIP-94 `m` tags are author-supplied and some clients write the bare subtype (Primal - * iOS emits `m jpeg`). Treat the declared type as a hint, not as truth. - * - * [fileExtension] is the safer signal — [getMediaExtension] sniffs it from the file's magic - * numbers rather than trusting the event — so it backs up an unusable declaration. + * `Intent.type` has to be a real `type/subtype` — an `IntentFilter` matches the two halves + * separately, so a slash-less value matches nothing and the chooser opens empty. The declared + * type is author-supplied and may be unusable (see [normalizeMimeType]), so it is treated as a + * hint; [fileExtension] is the safer signal because [getMediaExtension] sniffs it from the + * file's magic numbers rather than trusting the event. */ internal fun resolveShareMimeType( declaredMimeType: String?, fileExtension: String, - defaultTypePrefix: String, ): String = normalizeMimeType(declaredMimeType) ?: mimeTypeMap[fileExtension.lowercase()] - ?: "$defaultTypePrefix/$fileExtension" + // Unreachable today: getMediaExtension only ever returns keys of mimeTypeMap. Kept so + // the return type stays a well-formed MIME if that ever stops holding. + ?: GENERIC_BINARY_MIME_TYPE suspend fun getSharableUriFromUrl( context: Context, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ZoomableContentView.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ZoomableContentView.kt index a09e69278a..1122b73a57 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ZoomableContentView.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/components/ZoomableContentView.kt @@ -1103,7 +1103,7 @@ private suspend fun shareImageFile( val (uri, fileExtension) = ShareHelper.getSharableUriFromUrl(context, videoUri) // Determine mime type, use provided or derive from extension - val determinedMimeType = ShareHelper.resolveShareMimeType(mimeType, fileExtension, "image") + val determinedMimeType = ShareHelper.resolveShareMimeType(mimeType, fileExtension) // Create share intent val shareIntent = @@ -1161,7 +1161,7 @@ private suspend fun shareVideoFile( sharedFile = sharableFile // Determine mime type - val determinedMimeType = ShareHelper.resolveShareMimeType(mimeType, extension, "video") + val determinedMimeType = ShareHelper.resolveShareMimeType(mimeType, extension) // Create share intent val shareIntent = @@ -1227,7 +1227,7 @@ private suspend fun shareLocalVideoFile( val (uri, extension) = ShareHelper.getSharableUriForLocalVideo(context, localFile) // Determine mime type - val determinedMimeType = ShareHelper.resolveShareMimeType(mimeType, extension, "video") + val determinedMimeType = ShareHelper.resolveShareMimeType(mimeType, extension) // Create share intent val shareIntent = diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/ui/components/ShareHelperTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/ui/components/ShareHelperTest.kt index e9336b351b..fc15511007 100644 --- a/amethyst/src/test/java/com/vitorpamplona/amethyst/ui/components/ShareHelperTest.kt +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/ui/components/ShareHelperTest.kt @@ -168,35 +168,38 @@ class ShareHelperTest { // `m jpeg`), so a bare subtype must be repaired rather than forwarded. @Test fun resolveShareMimeType_bareSubtype_isExpandedToFullMimeType() { - assertEquals("image/jpeg", ShareHelper.resolveShareMimeType("jpeg", "jpg", "image")) - assertEquals("video/mp4", ShareHelper.resolveShareMimeType("mp4", "mp4", "video")) + assertEquals("image/jpeg", ShareHelper.resolveShareMimeType("jpeg", "jpg")) + assertEquals("video/mp4", ShareHelper.resolveShareMimeType("mp4", "mp4")) + assertEquals("image/jpeg", ShareHelper.resolveShareMimeType("JPEG", "jpg")) } @Test fun resolveShareMimeType_wellFormedDeclaration_isPreserved() { - assertEquals("image/png", ShareHelper.resolveShareMimeType("image/png", "png", "image")) + assertEquals("image/png", ShareHelper.resolveShareMimeType("image/png", "png")) } // An unusable declaration falls back to the extension, which is sniffed from the file's magic // numbers and so is not attacker-controlled. @Test fun resolveShareMimeType_unrecognizableDeclaration_fallsBackToSniffedExtension() { - assertEquals("image/png", ShareHelper.resolveShareMimeType("notatype", "png", "image")) + assertEquals("image/png", ShareHelper.resolveShareMimeType("notatype", "png")) + assertEquals("image/png", ShareHelper.resolveShareMimeType("", "png")) } @Test fun resolveShareMimeType_noDeclaration_usesCanonicalTypeForExtension() { // Not "image/jpg" -- jpg is not a registered subtype. - assertEquals("image/jpeg", ShareHelper.resolveShareMimeType(null, "jpg", "image")) - assertEquals("video/quicktime", ShareHelper.resolveShareMimeType(null, "mov", "video")) + assertEquals("image/jpeg", ShareHelper.resolveShareMimeType(null, "jpg")) + assertEquals("video/quicktime", ShareHelper.resolveShareMimeType(null, "mov")) } + // The invariant the share sheet depends on, pinned across the whole set of extensions + // getMediaExtension can sniff: whatever the event declared, Intent.type stays matchable. @Test fun resolveShareMimeType_alwaysProducesASlashSeparatedType() { - val cases = listOf(null, "", "jpeg", "image/jpeg", "notatype", "JPEG") - cases.forEach { declared -> - val resolved = ShareHelper.resolveShareMimeType(declared, "jpg", "image") - assertTrue("`$declared` resolved to un-matchable type `$resolved`", resolved.contains("/")) + listOf("jpg", "png", "gif", "webp", "webm", "avi", "mp4", "mov").forEach { extension -> + val resolved = ShareHelper.resolveShareMimeType("notatype", extension) + assertTrue("`$extension` resolved to un-matchable type `$resolved`", resolved.contains("/")) } } } diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/MediaContentModels.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/MediaContentModels.kt index be472293f2..14ab09b18c 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/MediaContentModels.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/MediaContentModels.kt @@ -44,10 +44,18 @@ abstract class MediaUrlContent( dim: DimensionTag? = null, blurhash: String? = null, val uri: String? = null, - val mimeType: String? = null, + mimeType: String? = null, thumbhash: String? = null, val authorPubKey: String? = null, -) : BaseMediaContent(description, dim, blurhash, thumbhash) +) : BaseMediaContent(description, dim, blurhash, thumbhash) { + /** + * Repaired at construction rather than at each of the eight call sites that build a model from + * an author-supplied `m` tag — a bare subtype reaching this field is what puts `Intent.type = + * "jpeg"` on the share sheet (matching no `IntentFilter`) and what republishes the malformed + * tag under the user's own key when media is added to a gallery. See [normalizeMimeType]. + */ + val mimeType: String? = normalizeMimeType(mimeType) +} @Immutable open class MediaUrlImage( diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt index 51765fe3bb..8d5fd30da0 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParser.kt @@ -59,7 +59,7 @@ class RichTextParser { val tags = eventTags.get(fullUrl)?.properties ?: emptyMap() - val contentType = normalizeMimeType(frags[MimeTypeTag.TAG_NAME] ?: tags[MimeTypeTag.TAG_NAME]?.firstOrNull()) + val contentType = frags[MimeTypeTag.TAG_NAME] ?: tags[MimeTypeTag.TAG_NAME]?.firstOrNull() // Returning null here drops the URL to a plain link, discarding the imeta's `dim`/blurhash // and forcing a URL-preview round-trip to rediscover a type the imeta already declared — @@ -557,10 +557,10 @@ class RichTextParser { * Resolves which renderer can display a declared blob — the single decision every media * renderer must make, from a NIP-94 `m` tag, a NIP-92 imeta, or a bare URL. * - * A declared MIME type wins; the URL extension is the fallback both for the no-MIME case - * and for a *malformed* MIME (Primal iOS emits `m jpeg` rather than `m image/jpeg`, which - * matches no prefix below). `data:` URIs carry their type in the prefix, so a miss there is - * genuine and the base64 payload is never extension-probed. + * A declared MIME type wins, after [normalizeMimeType] repairs the bare-subtype form some + * clients emit; the URL extension is the fallback both for the no-MIME case and for a + * declaration too mangled to repair. `data:` URIs carry their type in the prefix, so a miss + * there is genuine and the base64 payload is never extension-probed. * * Returns **null** when nothing can render the file. Callers must not substitute a media * kind for that null: handing an arbitrary blob — a webxdc app, a zip, an APK — to the @@ -570,8 +570,9 @@ class RichTextParser { */ fun classifyMedia( url: String, - mimeType: String?, + rawMimeType: String?, ): MediaContentKind? { + val mimeType = normalizeMimeType(rawMimeType) if (mimeType != null) { if (mimeType.startsWith("image/")) return MediaContentKind.IMAGE // HLS playlists are advertised with a non-`video/*` MIME; see [isHlsMimeType]. @@ -666,6 +667,9 @@ val mimeTypeMap: Map = // Video "mp4" to "video/mp4", "webm" to "video/webm", + // Dead entry: "ogg" is re-keyed under Audio below and mapOf keeps the last, so every + // lookup of it yields audio/ogg. Kept only to show the extension is genuinely ambiguous — + // see [ambiguousMimeSubtypes]. Don't read this line as reachable. "ogg" to "video/ogg", "mov" to "video/quicktime", "avi" to "video/x-msvideo", @@ -682,6 +686,22 @@ val mimeTypeMap: Map = "pdf" to "application/pdf", ) +/** + * Subtypes that name more than one top-level type: `mpeg`, `mp4`, `ogg`, `webm` and `3gpp` all exist + * as both `audio/` and `video/`, so a bare token spelling one of them identifies no family on its + * own. See [normalizeMimeType] for what that costs them. + */ +private val ambiguousMimeSubtypes = setOf("mpeg", "mp4", "ogg", "webm", "3gpp") + +/** + * The subtype half of every MIME in [mimeTypeMap], so a bare token can be looked up as what it + * actually is. [mimeTypeMap] is keyed by *extension*, which only doubles as a subtype index where + * the two spellings coincide — `quicktime`, `x-matroska` and `svg+xml` are subtypes no extension + * spells. Consulted after [mimeTypeMap] so the extension spelling keeps priority where they + * disagree (`mp4` stays `video/mp4` rather than the later `audio/mp4` entry). + */ +private val mimeSubtypeMap: Map = mimeTypeMap.values.associateBy { it.substringAfter('/') } + /** * NIP-92's `m` property is meant to carry a full `type/subtype`, but several clients emit the * bare subtype instead — Primal iOS writes `m jpeg` rather than `m image/jpeg`. That value is @@ -690,14 +710,28 @@ val mimeTypeMap: Map = * `Intent.type = "jpeg"`. No `` filter matches a type without a slash, * so the share sheet opens with zero targets and the image cannot be shared at all. * - * Map a bare subtype back onto its canonical MIME so every downstream consumer (the share - * intent, the gallery entry's published `m` tag, the player's type hint) sees a well-formed - * value. Anything already containing a `/` is passed through untouched, and an unrecognised - * bare token is dropped to null rather than propagated — that leaves the caller's - * extension-based detection to decide, which is strictly better than carrying garbage forward. + * Map a bare subtype back onto its canonical MIME. This is called from the two chokepoints every + * `m` value passes through — [RichTextParser.classifyMedia] for the render decision and + * [MediaUrlContent] for the value the share intent and the gallery entry's republished `m` tag + * read — so consumers see a well-formed type without each having to remember to repair it. + * + * Anything already containing a `/` is passed through untouched, and an unrecognised bare token is + * dropped to null rather than propagated — that leaves the caller's extension-based detection to + * decide, which is strictly better than carrying garbage forward. + * + * The same refusal covers a token in [ambiguousMimeSubtypes] that would land in `audio/`. `audio/` + * is the one destructive family: it is what [RichTextParser.isAudioContent] reads to drop the + * picture, so guessing it for what may be a video loses content, while guessing `video/` for what + * may be audio only costs some chrome. That asymmetry is why the guard is one-sided rather than a + * blanket refusal — `mp4` and `webm` resolve to `video/` and keep their rescue, so an extensionless + * Blossom URL declaring `m mp4` still renders, whereas `ogg` (whose only live [mimeTypeMap] entry + * is `audio/ogg`, its `video/ogg` one being a dead duplicate key) and `mpeg` decline. */ fun normalizeMimeType(rawMimeType: String?): String? { if (rawMimeType == null) return null if (rawMimeType.contains('/')) return rawMimeType - return mimeTypeMap[rawMimeType.lowercase()] + val token = rawMimeType.lowercase() + val resolved = mimeTypeMap[token] ?: mimeSubtypeMap[token] ?: return null + if (token in ambiguousMimeSubtypes && resolved.startsWith("audio/")) return null + return resolved } diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/ClassifyMediaTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/ClassifyMediaTest.kt index 240d4f0baf..551aa7081e 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/ClassifyMediaTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/ClassifyMediaTest.kt @@ -23,7 +23,9 @@ package com.vitorpamplona.amethyst.commons.richtext import com.vitorpamplona.quartz.nip92IMeta.IMetaTag import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertFalse import kotlin.test.assertNull +import kotlin.test.assertTrue class ClassifyMediaTest { @Test @@ -97,10 +99,67 @@ class ClassifyMediaTest { } @Test - fun extensionRescuesAMalformedMime() { - // Primal iOS emits `m jpeg` instead of `m image/jpeg`; the extension must still win - // over "unknown". Preserves the behaviour createMediaContent already documented. + fun aMalformedMimeIsRepairedRatherThanIgnored() { + // Primal iOS emits `m jpeg` instead of `m image/jpeg`. The bare subtype is mapped back to + // its canonical MIME here, so it classifies even on the extensionless URLs Blossom hands + // out, where the imeta is the only type signal there is. assertEquals(MediaContentKind.IMAGE, RichTextParser.classifyMedia("https://x.com/a.jpg", "jpeg")) + assertEquals(MediaContentKind.IMAGE, RichTextParser.classifyMedia("https://x.com/abcd1234", "jpeg")) + assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/abcd1234", "quicktime")) + // A token that maps to no known type has nothing to recover from; the extension decides. + assertNull(RichTextParser.classifyMedia("https://x.com/abcd1234", "notarealtype")) + assertEquals(MediaContentKind.IMAGE, RichTextParser.classifyMedia("https://x.com/a.jpg", "notarealtype")) + } + + // A subtype that names more than one top-level type identifies no family on its own. Guessing + // one is worse than not repairing: `audio/mpeg` classifies as VIDEO either way, but it also + // satisfies isAudioContent, which strips the picture and renders an MPEG video as a bare audio + // track. Leaving it unresolved hands the decision back to the extension, which knows. + @Test + fun anAmbiguousBareSubtypeIsLeftForTheExtensionToDecide() { + assertNull(normalizeMimeType("mpeg"), "`mpeg` names both audio/mpeg and video/mpeg") + // `ogg` reaches the same guard even though mimeTypeMap answers it: the extension table + // holds audio/ogg (its video/ogg entry is a dead duplicate key), so short-circuiting there + // is exactly the audio mis-flag this guard exists to stop. + assertNull(normalizeMimeType("ogg"), "`ogg` names both audio/ogg and video/ogg") + + val url = "https://x.com/clip.mpg" + val media = RichTextParser().createMediaContent(url, mapOf(url to imeta(url, "mpeg")), null) + + assertTrue(media is MediaUrlVideo, "the .mpg extension still classifies it") + assertFalse(RichTextParser.isAudioContent(media.mimeType, url), "an MPEG video is not an audio track") + + // The case that actually reached a user: an OGG video must not be flagged as an audio track. + val ogv = "https://x.com/clip.ogv" + assertFalse( + RichTextParser.isAudioContent(normalizeMimeType("ogg"), ogv), + "an OGG video must not be rendered as a pictureless audio track", + ) + } + + // Declining is reserved for the destructive direction. `audio/` is the one family that strips + // the picture, so an ambiguous token whose extension spelling already resolves to `video/` + // keeps its rescue — an extensionless Blossom URL with `m mp4` still renders. + @Test + fun anAmbiguousSubtypeThatResolvesToVideoKeepsItsRescue() { + assertEquals("video/mp4", normalizeMimeType("mp4")) + assertEquals("video/webm", normalizeMimeType("webm")) + assertEquals(MediaContentKind.VIDEO, RichTextParser.classifyMedia("https://x.com/abcd1234", "mp4")) + + // Unambiguously-audio tokens are untouched by the guard. + assertEquals("audio/mpeg", normalizeMimeType("mp3")) + assertEquals("audio/flac", normalizeMimeType("flac")) + } + + // The unambiguous half must keep working: these are subtypes no extension in the table spells, + // so they resolve only through the subtype index. + @Test + fun anUnambiguousBareSubtypeStillResolves() { + assertEquals("video/quicktime", normalizeMimeType("quicktime")) + assertEquals("video/x-matroska", normalizeMimeType("x-matroska")) + assertEquals("image/svg+xml", normalizeMimeType("svg+xml")) + // An extension spelling that is also a subtype keeps the extension's family. + assertEquals("video/mp4", normalizeMimeType("mp4")) } @Test @@ -135,6 +194,9 @@ class ClassifyMediaTest { "https://x.com/a.xdc" to "application/x-webxdc", "https://x.com/a.zip" to "application/zip", "https://x.com/a.jpg" to "jpeg", + "https://x.com/abcd1234" to "jpeg", + "https://x.com/abcd1234" to "notarealtype", + "https://x.com/clip.mpg" to "mpeg", "data:image/png;base64,AAAA" to null, "data:application/zip;base64,AAAAmp4" to null, ) diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/PdfParserTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/PdfParserTest.kt index 53c0cfb947..6ec4de6de5 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/PdfParserTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/PdfParserTest.kt @@ -67,11 +67,10 @@ class PdfParserTest { assertTrue(RichTextParser.isPdfUrl("https://example.com/doc.pdf?sig=abc")) } - // Primal iOS writes a bare subtype (`m jpeg`) instead of a full MIME (`m image/jpeg`). - // The bare subtype matches none of the `image/`/`video/`/`application/pdf` prefixes, so before - // the extension fallback the whole imeta was dropped: the URL rendered as a plain link, losing - // the `dim` needed to reserve the image's height (feed jump) and forcing a URL-preview fetch. - // The `.jpg` extension must still route it to a MediaUrlImage that carries `dim`. + // Primal iOS writes a bare subtype (`m jpeg`) instead of a full MIME (`m image/jpeg`); see + // normalizeMimeType. End-to-end here because the failure was never just the render decision: + // dropping the imeta also lost the `dim` that reserves the image's height (feed jump) and + // forced a URL-preview fetch to rediscover a type the imeta had already declared. @Test fun detectsImageFromMalformedImetaMimeWithImageExtension() { val url = "https://blossom.primal.net/33e7c01afbea894a64e1db44dece460b09a2426108f47143754e1cf4bfdf747c.jpg" @@ -88,6 +87,9 @@ class PdfParserTest { val imageMedia = state.mediaForPager[url] assertTrue(imageMedia is MediaUrlImage, "Expected MediaUrlImage despite the malformed `m jpeg` mime") + // Repaired on the model too, not just for the render decision: this field becomes + // Intent.type when the image is shared, and a slash-less one matches no IntentFilter. + assertEquals("image/jpeg", imageMedia.mimeType, "The bare subtype must not survive onto the model") assertEquals("1009x680", imageMedia.dim?.toString(), "The imeta dim must survive so the loader can reserve space") assertEquals(1009f / 680f, imageMedia.dim?.aspectRatio()) @@ -112,37 +114,20 @@ class PdfParserTest { assertTrue(videoMedia is MediaUrlVideo, "Expected MediaUrlVideo despite the malformed `m mp4` mime") } - // A bare-subtype mime is recovered from the imeta itself, so an extensionless URL — the shape - // Blossom hands out, where the imeta is the only type signal there is — still renders. + // An extensionless URL is the shape Blossom hands out, where the imeta is the only type signal + // there is: a recognizable bare subtype now carries it, and an unrecognizable one leaves + // nothing to recover from. The second half documents the limit so it isn't read as a + // regression. @Test - fun malformedImetaMimeWithoutExtensionIsRecoveredFromTheMime() { + fun malformedImetaMimeWithoutExtensionIsRecoveredFromTheMimeAlone() { val url = "https://files.example.com/abcd1234" - val tags = - ImmutableListOfLists( - arrayOf( - arrayOf("imeta", "url $url", "m jpeg"), - ), - ) - val state = RichTextParser().parseText(url, tags, null) + fun parse(mime: String) = + RichTextParser() + .parseText(url, ImmutableListOfLists(arrayOf(arrayOf("imeta", "url $url", "m $mime"))), null) + .mediaForPager[url] - assertTrue(state.mediaForPager[url] is MediaUrlImage, "`m jpeg` alone is enough to classify the media") - } - - // The boundary: a bare token that maps to no known type, on a URL with no extension, has - // nothing left to recover from. This documents the limit so it isn't mistaken for a regression. - @Test - fun unrecognizableImetaMimeWithoutExtensionStaysUnclassified() { - val url = "https://files.example.com/abcd1234" - val tags = - ImmutableListOfLists( - arrayOf( - arrayOf("imeta", "url $url", "m notarealtype"), - ), - ) - - val state = RichTextParser().parseText(url, tags, null) - - assertEquals(null, state.mediaForPager[url], "Nothing to recover from -> not treated as media") + assertTrue(parse("jpeg") is MediaUrlImage, "`m jpeg` alone is enough to classify the media") + assertEquals(null, parse("notarealtype"), "Nothing to recover from -> not treated as media") } } diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserMalformedMimeTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserMalformedMimeTest.kt deleted file mode 100644 index f5067d0d4b..0000000000 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/richtext/RichTextParserMalformedMimeTest.kt +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright (c) 2025 Vitor Pamplona - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to use, - * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the - * Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN - * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package com.vitorpamplona.amethyst.commons.richtext - -import com.vitorpamplona.quartz.nip92IMeta.IMetaTag -import kotlin.test.Test -import kotlin.test.assertEquals -import kotlin.test.assertTrue - -class RichTextParserMalformedMimeTest { - private val url = "https://blossom.primal.net/c27d7b7be6e58d69b29006cc275d29d67967760b1772e50a79d5b24f60d62fc5.jpg" - - private fun parse(mime: String): MediaUrlContent? = - RichTextParser().createMediaContent( - fullUrl = url, - eventTags = - mapOf( - url to - IMetaTag( - url = url, - properties = mapOf("m" to listOf(mime), "dim" to listOf("960.0x1358.0")), - ), - ), - description = null, - ) - - @Test - fun malformedImetaMimeStillRendersAsImage() { - val content = parse("jpeg") - assertTrue(content is MediaUrlImage, "bare `m jpeg` must still route to MediaUrlImage") - } - - @Test - fun malformedImetaMimeIsNormalizedForSharing() { - val content = parse("jpeg") as MediaUrlImage - assertEquals("image/jpeg", content.mimeType) - } - - @Test - fun wellFormedImetaMimeIsUntouched() { - val content = parse("image/jpeg") as MediaUrlImage - assertEquals("image/jpeg", content.mimeType) - } -} From 8b17624c458a11275e95acb25502ca4b92b3187d Mon Sep 17 00:00:00 2001 From: vitorpamplona <532031+vitorpamplona@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:27:50 +0000 Subject: [PATCH 061/132] chore: sync Crowdin translations and seed translator npub placeholders --- .../src/main/res/values-hi-rIN/strings.xml | 126 ++++++++++-------- .../src/main/res/values-hu-rHU/strings.xml | 15 +++ .../src/main/res/values-pl-rPL/strings.xml | 16 +++ 3 files changed, 101 insertions(+), 56 deletions(-) diff --git a/amethyst/src/main/res/values-hi-rIN/strings.xml b/amethyst/src/main/res/values-hi-rIN/strings.xml index 077e171144..68dc9e575b 100644 --- a/amethyst/src/main/res/values-hi-rIN/strings.xml +++ b/amethyst/src/main/res/values-hi-rIN/strings.xml @@ -46,8 +46,8 @@ हिंसा अज्ञात लेखक लेख की अनुकृति करें - लेखक विभेदक की अनुकृति करें - टीका विभेदक की अनुकृति करें + लेखक सूचक की अनुकृति करें + टीका सूचक की अनुकृति करें कच्चा जेसोन॰ की अनुकृति करें प्रसारण समयांकन करें @@ -220,7 +220,7 @@ संचारयन्त्र में अभिलेखन करें चित्र का अभिलेखन किया गया चित्रालय क्रमक में चलचित्र अवरोहण आरम्भ हुआ … - अभिलेख अवरोहण आरम्भ हुआ … + श्रव्यदृश्याभिलेख अवरोहण आरम्भ हुआ … ऊपर टाँकें टाँका हटाएँ ऊपर टँकित @@ -271,7 +271,7 @@ " पुनःप्रसारक" जालस्थान लैटनिंग पता - एनसेक॰ विभेदक (आपका गुप्त पारणशब्द) की अनुकृति करता है टाँकाफलक में सुरक्षित रखने के लिए + एनसेक॰ सूचक (आपका गुप्त पारणशब्द) की अनुकृति करता है टाँकाफलक में सुरक्षित रखने के लिए निजी कुंचिका क्यूआर॰ क्रमचित्र दिखाएँ। रहस्यीकृत निजी कुंचिका क्यूआर॰ क्रमचित्र दिखाएँ। सीधा संदेश भेजें @@ -279,7 +279,7 @@ अनुचरण करें प्रत्यानुचरण करें बाधा हटाएँ - उपयोगकर्ता विभेदक की अनुकृति + प्रयोक्ता सूचक की अनुकृति उपयोगकर्ता बाधा हटाएँ प्रयोक्ता बाधित अथवा मौनकृत। उनके पत्र छिपाए गए। "एनपुब॰, उपयोगकर्ता नाम, लेख" @@ -327,7 +327,7 @@ प्रणाली नाम प्रणाली मिटाएँ क्या प्रणाली मिटा दें। - क्या #%1$s को मिटा दें। इसको पूर्ववत नहीं किया जा सकता। तथा उसी विभेदक के साथ प्रणाली का पुनःउत्पादन नहीं किया जा सकता। + क्या #%1$s को मिटा दें। इसको पूर्ववत नहीं किया जा सकता। तथा उसी सूचक के साथ प्रणाली का पुनःउत्पादन नहीं किया जा सकता। मिटाएँ समुदाय छोडें क्या समुदाय छोडें। @@ -510,7 +510,7 @@ पदक पुरस्कार इनको दिया गया टीका लेख की अनुकृति की गई टाँकाफलक में लेखक के @npub की अनुकृति की गई टाँकाफलक में - टीका विभेदक (@note1) की अनुकृति की गई टाँकाफलक में + टीका सूचक (@note1) की अनुकृति की गई टाँकाफलक में "<निजी संदेश का अरहस्यीकरण असफल>\n\nआप का उल्लेख किया गया एक निजी अथवा रहस्यीकृत संवाद में %1$s तथा %2$s के बीच।" नयी लेखा जोडें लेखाएँ @@ -539,14 +539,14 @@ क्यूआर॰ चित्र जिसमें इस टीका का एक जाल योजक समाविष्ट है क्यूआर॰ चित्र जिसमें इस टीका का एक नोस्टर योजक समाविष्ट है अंगुलचित्र छिपाया गया संवेदनशिल विषयवस्तु के कारण - लेखक विभेदक - टीका विभेदक + लेखक सूचक + टीका सूचक लेख की अनुकृति करें मिटाएँ अनुचरण ना करें अनुचरण करें चित्रालय से मिटाएँ - इस अभिलेख को आपके चित्रालय से हटाएँ। + इस श्रव्यदृश्याभिलेख को आपके चित्रालय से हटाएँ। हटाने की याचना अमेथिस्ट अनुरोध करेगा कि आपका टीका मिटा दिया जाए उन पुनःप्रसारकों से जिनके साथ आप अब जुडे हुए हैं। कोई आश्वासन नहीं कि आपका टीका सर्वदा के लिए मिटा दिया जाएगा उन पुनःप्रसारकों से, अथवा अन्य पुनःप्रसारकों में से जहाँ यह रखा गया हो। बाधित करें @@ -638,7 +638,7 @@ %d पुनःप्रसारक तक %d पुनःप्रसारकों तक - टीका विभेदक की अनुकृति की गई + टीका सूचक की अनुकृति की गई मतदान खुला आवृत @@ -650,7 +650,7 @@ विवरण नियम (विकल्पात्मक) आवरण चित्र जोडें - चित्र चुनने के लिए दबाएँ। वह आपके प्रसारसंगणक तक आरोहित किया जाएगा। + चित्र चुनने के लिए दबाएँ। वह आपके श्रव्यदृश्याभिलेख सेवासंगणक तक आरोहित किया जाएगा। नियामक नियामक अनुमति दे सकते हैं पत्र प्रकाशन के लिए। आप सदैव नियामक हैं। नियामक जोडें @@ -846,7 +846,7 @@ उसका अपना निजी भण्डार लैटनिंग चालान का भुगतान जाल तथा ब्लोस्सम॰ संसाधन ले आएँ - आपके प्रसारसंगणक तक अभिलेखों का आरोहण + आपके श्रव्यदृश्याभिलेख सेवासंगणक तक अभिलेखों का आरोहण प्रदर्शनशैली सूचनाएँ सन्देश प्रेषण @@ -870,7 +870,7 @@ सावधान। यह नोस्टरसंलग्नक्रमक एक लैटनिंग चालान का भुगतान करना चाहता है जिसमें कोई मात्रा स्पष्टीकृत नहीं। प्राप्तकर्ता निर्णय करेगा कितना लिया जाएगा। इसे तभी अनुमति दें यदि आप इस पर विश्वास करते हैं। यह नोस्टर संलग्नक्रमक एक जाल संसाधन लाना चाहता है। - यह नोस्टर संलग्नक्रमक एक अभिलेख को आपके प्रसारसंगणक तक आरोहण करना चाहता है। + यह नोस्टर संलग्नक्रमक एक अभिलेख को आपके श्रव्यदृश्याभिलेख सेवासंगणक तक आरोहण करना चाहता है। यह नोस्टर संलग्नक्रमक आपको सूचनाएँ दिखाना चाहता है। यह स्थान आपकी नोस्टर कुंचिका के साथ एक %1$d प्रकार घटना का हस्ताक्षर करना चाहता है। @@ -1376,7 +1376,7 @@ पूर्वावलोकन अम्श जोडें लघु ध्वनि अथवा दृश्य पूर्वीक्षण पूर्वावलोकन शीर्षक - अभिलेख जालपता + श्रव्यदृश्याभिलेख जालपता आपका पुटप्रसार शीर्षकरहित कोई कडी नहीं अब तक। नयी कडी दबाएँ अपने प्रथम कडी प्रकाशित करने के लिए। @@ -1391,7 +1391,7 @@ प्राप्तकर्ता जोडें पता जोडें स्वयम हाथ से नोस्टर प्रयोक्ता जोडें - नाम अथवा @विभेदक द्वारा ढूँढें + नाम अथवा @लेखासूचक द्वारा ढूँढें इस प्रयोक्ता का कोई लैटनिंग॰ पता नहीं प्राप्तकर्ता हटाएँ नाम (विकल्पात्मक) @@ -1564,7 +1564,7 @@ इसका अर्थ क्या है? यह विषयवस्तु वैसे ही है जैसे पत्र प्रकाशन पर यह विषयवस्तु परिवर्तित हुआ है। हो सकता है लेखक ने परिवर्तन देखा नहीं अथवा अनुमति दिया नहीं। - चित्र चलचित्र जोडें + श्रव्यदृश्याभिलेख जोडें चित्र जोडें चलचित्र जोडें पत्र जोडें @@ -1589,8 +1589,8 @@ नामरहित अभिलेख सेवासंगणक इस अभिलेख का आरोहण करने के लिए सेवासंगणक का चयन करें - प्रसारसंगणक - आपके प्रसारसंगणक आद्यताएँ स्थापित करें। + श्रव्यदृश्याभिलेख सेवासंगणक + आपके श्रव्यदृश्याभिलेख सेवासंगणक आद्यताएँ स्थापित करें। स्थानीय ब्लोस्सम॰ द्रुतस्मृति का प्रयोग करें जब एक ब्लोस्सम॰ द्रुतस्मृति चल रही है इस यन्त्र पर (संयोजनद्वार २४२४२) तब चित्र चलचित्र अवरोहण उसके द्वारा करें। स्थानीय द्रुतस्मृति का पता चला संयोजनद्वार २४२४२ पर। @@ -1601,7 +1601,7 @@ आरोहण व्यवहार प्रतिबिम्ब आरोहण आरोहण पश्चात अभिलेख की अनुकृति आपके अन्य ब्लोस्सम सेवासंगणकों में करें जिससे वह उपलब्ध रहेगा एक संगणक असंयोजित होने पर भी। - सेवासंगणक पर अभिलेखों का अनुकूलन + सेवासंगणक पर श्रव्यदृश्याभिलेखों का अनुकूलन आरोहण करें सेवासंगणक के /media अन्तबिन्दु द्वारा जिससे वह अभिलेख के उपतथ्य मिटा सके तथा संकुचित कर सके। रखा गया अभिलेख मूल से पृथक हो सकता है। रखे गए अभिलेखों का प्रबन्धन मेरे ब्लोस्सम॰ अभिलेख @@ -1662,11 +1662,11 @@ %1$d अभिलेख का आयात %1$d अभिलेखों का आयात - अनुशम्सित प्रसारसंगणक + अनुशम्सित श्रव्यदृश्याभिलेख सेवासंगणक अमेथिस्त की मूलविकल्प सूची। आप एक एक करके जोड सकते हैं अथवा सूची जोड सकते हैं। मूलविकल्प सूची का प्रयोग करें - प्रसारसंगणक जोडें - प्रसारसंगणक मिटाएँ + श्रव्यदृश्याभिलेख सेवासंगणक जोडें + श्रव्यदृश्याभिलेख सेवासंगणक मिटाएँ खींचे पुनःव्यवस्थित करने के लिए। आरोहण के लिए प्रत्येक सेवासंगणक के साथ प्रयास किया जाएगा ऊपर से नीचे। आरोहण प्राथमिकता सेवासंगणक जोडें @@ -1787,14 +1787,14 @@ ज्साप लैटनिंग तथा काशयू व्यापार के लिए टोर का प्रयोग अवश्य करें नोस्ट्र पता सत्यापन निप॰-०५ पता सत्यापन के लिए टोर का प्रयोग अवश्य करें - चित्र चलचित्र अभिलेख आरोहण - चित्र चलचित्र अभिलेख आरोहण के लिए टोर का प्रयोग अवश्य करें + श्रव्यदृश्याभिलेख आरोहण + विषयवस्तु आरोहण के लिए टोर का प्रयोग अवश्य करें आन्तरीय ओर्बोट निष्क्रिय मूलभूत मूलविकल्प - ध्वनिदृश्याभिलेख के अतिरिक्त सभी + श्रव्यदृश्याभिलेख के अतिरिक्त सभी सम्पूर्णतः गुप्त विशिष्ट सेवासंगणक की आवश्यकता होने पर टोर का प्रयोग करें @@ -1853,11 +1853,11 @@ %1$s ने आपके पत्र को पुनःप्रकाशित किया नए पुनःप्रकाशन - ध्वनिचित्राभिलेख + श्रव्यदृश्याभिलेख आपको सूचित करता है जब कोई आपका उल्लेख करते हैं चित्र अथवा चलचित्र में %1$s ने चित्र बाँटा %1$s ने चलचित्र बाँटा - नए ध्वनिचित्राभिलेख + नए श्रव्यदृश्याभिलेख निबन्ध तथा उद्दीप्तव्य आपको सूचित करता है जब कोई आपका उल्लेख अथवा आपको उद्दीप्त करते हैं एक निबन्ध में @@ -1948,14 +1948,14 @@ %1$s \u00b7 %2$d पुनःप्रसारक जालभ्रमण - ध्वनिचित्राभिलेख + श्रव्यदृश्याभिलेख विषयसूचक विषय सूची वार्तालाप खोज लुप्त घटनाओं को ढूँढें घटना अवलोकन - घटनाओं को विभेदक अनुसार ले आता है जिसका उल्लेख आपके पटल पर अमुक करता है पर जिसकी प्राप्ती अभी नहीं हुई। एक उद्धरण अथवा एक प्रत्युत्तर का पूर्वपत्र अथवा एक सूत्र का मूल। + घटनाओं को सूचक अनुसार ले आता है जिसका उल्लेख आपके पटल पर अमुक करता है पर जिसकी प्राप्ती अभी नहीं हुई। एक उद्धरण अथवा एक प्रत्युत्तर का पूर्वपत्र अथवा एक सूत्र का मूल। घटनाओं का अवलोकन करता है जो वर्तमान में प्रदर्शित हो रहे हैं नए प्रत्युत्तर प्रतिक्रियाएँ उद्धरण ज्साप तथा वृत्तान्तों के लिए जिससे गिनतियों का नवीकरण होता है जब आप पढ रहे हैं। संलग्न पुनःप्रसारक जानकारी @@ -2124,7 +2124,7 @@ गणना ग्राहकताएँ (%1$d) निर्गतपेटिका घटनाएँ (%1$d) %1$d लेखक - %1$d विभेदक + %1$d सूचक %1$s से %1$s तक सीमा %1$d @@ -2147,7 +2147,7 @@ अधिकतम ग्राहकताएँ अधिकतम छलनियाँ प्रति ग्राहकता अधिकतम सीमा (घटनाएँ प्रतिफलित) - अधिकतम ग्राहकताविभेदक लम्बाई + अधिकतम ग्राहकतासूचक लम्बाई काशयू अक्षरराशि टकसाल : %1$s चुकाएँ @@ -2222,7 +2222,7 @@ अन्धकारमय क्रमक आद्यताएँ स्वरूप - ध्वनिदृश्याभिलेख तथा जानकारी + श्रव्यदृश्याभिलेख तथा जानकारी सामान्य संयोजन टोर॰ के माध्यम से @@ -2333,7 +2333,7 @@ बाहरी चित्र आरोहण करें एक वर्गाकार चित्र का चयन करें इस अभिलेख की कलाकृती के रूप में। ध्वनि अभिलेख आरोहण - एक एमपीत्री॰ अथवा वेव॰ अथवा फ्लाक॰ का चयन करें। इसका आरोहण किया जाएगा आपके प्रसारसंगणक पर अभिलेखन करने पर। + एक एमपीत्री॰ अथवा वेव॰ अथवा फ्लाक॰ का चयन करें। इसका आरोहण किया जाएगा आपके श्रव्यदृश्याभिलेख सेवासंगणक पर अभिलेखन करने पर। ध्वनि अभिलेख आरोहण के लिए तत्पर बाहरी चित्र तथा ध्वनि अभिलेख का आरोहण तथा प्रकाशन चालू। यह चलता रहेगा आप पटल से विगमन करें तो भी। संगीतसूची सम्पादन @@ -2629,7 +2629,7 @@ चिति की अनुकृति करें टाँकाफलक में अनुकृति करें टाँकाफलक में अनुकृत - टाँकाफलक में एन॰परिचय की अनुकृति करें + टाँकाफलक में एनप्रोफैल॰ की अनुकृति करें टाँकाफलक में एनपुब॰ की अनुकृति करें बाँटें अथवा अभिलेखन करें सीधेसन्देश के रूप में भेजें @@ -2647,10 +2647,10 @@ आपके विचार जोडें… उद्दीप्तव्य टाँकाफलक में जालपता की अनुकृति करें - टाँकाफलक में टीका विभेदक की अनुकृति करें - अभिलेख को चित्रालय में जोडें - अभिलेख जोडा गया - अभिलेख जोडा गया आपके परिचय चित्रालय में + टाँकाफलक में टीका सूचक की अनुकृति करें + श्रव्यदृश्याभिलेख को चित्रालय में जोडें + श्रव्यदृश्याभिलेख जोडा गया + श्रव्यदृश्याभिलेख जोडा गया आपके परिचय चित्रालय में तब बनाया गया दिशा निर्देश नियामक @@ -2767,23 +2767,23 @@ आहार विभिन्न अन्य - चित्र चलचित्र आरोहण असफल + श्रव्यदृश्याभिलेख आरोहण असफल संकुचित अभिलेख को खोल नहीं पाए आरोहण अपक्रम : %1$s सेवासंगणक ने आरोहण पश्चात जालपता नहीं दिया - सेवासंगणक से आरोहणकृत चित्र चलचित्र का अवरोहण नहीं कर पाए + सेवासंगणक से आरोहणकृत श्रव्यदृश्याभिलेख का अवरोहण नहीं कर पाए आरोहण पश्चात अवरोहित अभिलेख की जाँच नहीं हो सकी : %1$s %1$s पर आरोहण असफल : %2$s मिटाने में असफल : %1$s - अभिलेख निप॰-९५ के लिए बहुत बडा है + श्रव्यदृश्याभिलेख निप॰-९५ के लिए बहुत बडा है अभिलेख रहस्यीकरण गोपनीयता के लिए अभिलेखों का रहस्यीकरण करें। कुछ सेवासंगणक रहस्यीकृत अभिलेखों को सम्भाव्यतः अस्वीकार कर सकते हैं निःशुल्क लेखाओं के लिए। रहस्यीकृत आरोहण असफल अनेक सेवासंगणक रहस्यीकृत अभिलेखों को स्वीकार नहीं करते निःशुल्क लेखाओं के लिए। आप पुनःप्रयास कर सकते हैं रहस्यीकरण के बिना। रहस्यीकरण के बिना पुनःप्रयास सावधान : रहस्यीकरण के बिना कोई भी विषयवस्तु देख सकेगा अभिलेख योजक के साथ। - अभिलेख गुणस्तर - निम्न गुणस्तर चुनें अपने अभिलेख को अल्प गुणवत्ता युक्त छोटे आकार अभिलेख तक संकुचित करने के लिए अथवा उच्च गुणस्तर चुनें उच्चतर गुणवत्ता युक्त बृहत्तर अभिलेख तक संकुचित करने के लिए। + श्रव्यदृश्याभिलेख गुणस्तर + निम्न गुणस्तर चुनें अपने श्रव्यदृश्याभिलेख को अल्प गुणवत्ता युक्त छोटे आकार अभिलेख तक संकुचित करने के लिए अथवा उच्च गुणस्तर चुनें उच्चतर गुणवत्ता युक्त बृहत्तर श्रव्यदृश्याभिलेख तक संकुचित करने के लिए। निम्न मध्यम उच्च @@ -2793,11 +2793,11 @@ जिफ॰ से एमपी४॰ में परिवर्तित करें चलन्त जिफ॰ से एमपी४॰ में परिवर्तित करता है सूक्ष्मतर अभिलेख आकार तथा अधिक चालन अनुकूलता के लिए। निजी परितथ्य हटाएँ - निजी परितथ्य हटाने का प्रयास करता है आलम्बित चित्रध्वनिदृश्य अभिलेखों से आरोहण पूर्व + निजी परितथ्य हटाने का प्रयास करता है आलम्बित श्रव्यदृश्याभिलेखों से आरोहण पूर्व परितथ्य हटाने में असफल यह अभिलेख प्रारूप परितथ्य हटाने का अवलम्बन नहीं करता। निजी जानकारी जैसे स्थान तथा यन्त्र विवरण समाविष्ट हो सकते हैं। क्या आरोहण करें। आरोहण करें - अभिलेख से निजी परितथ्य हटाने में असफल। आरोहण निरस्त। + श्रव्यदृश्याभिलेख से निजी परितथ्य हटाने में असफल। आरोहण निरस्त। आरोहण निरस्त एविफ॰ से परितथ्य नहीं मिटा सके : %1$s पाण्डुलिपि सम्पादन @@ -3222,6 +3222,16 @@ सूचनावलियाँ क्रमक तथा जाल अन्य + पार्श्व विकल्पसूची + आपकी पार्श्व विकल्पसूची + एक विभाग खोलें तथा उन पंक्तियों को निष्क्रिय करें जिनका उपयोग आप कभी नहीं करते। स्थापना विकल्प सर्वदा दृश्यमान रहते हैं। जिससे कि आप यहाँ कभी भी लौट सकेंगे। विभाग क्रम स्थायी है। + विभाग + %1$d छिपे हुए + दृश्यमान + छिपे हुए + सर्वदा सक्रिय + सभी दिखाएँ + सभी छिपाएँ मुख्यपटल पृष्ठसूचक चयन करें किन पृष्ठसूचक दिखने चाहिए मुख्यपटल पर। जब एक ही सक्रिय है तब पृष्ठसूचक पट्टी छुपाई जाएगी। सभी @@ -3439,7 +3449,7 @@ क्यूआर॰ क्रमचित्र के रूप में एनपुब॰ को दिखाएँ क्यूआर॰ क्रमचित्र के रूप में एन॰परिचय को दिखाएँ अमान्य पता - अमेथिस्ट को एक वैश्विक वस्तु विभेदक प्राप्त हुआ खोलने के लिए परन्तु वह विभेदक अमान्य था : %1$s + अमेथिस्ट को एक वैश्विकवस्तुसूचक प्राप्त हुआ खोलने के लिए परन्तु वह सूचक अमान्य था : %1$s सीधा संदेश आगतपेटिका पुनःप्रसारक आपके निजी आगतपेटिका पुनःप्रसारकों की स्थापना करें यह स्थापना विकल्प सब को सूचित करता है आपको सन्देश भेजने के लिए कौनसे पुनःप्रसारकों का प्रयोग करना चाहिए। इनके बिना आप कुछ सन्देश प्राप्त नहीं कर पाएँगे। @@ -3595,6 +3605,10 @@ अभिलेखन गिट क्रमलेखकोश प्रमुखताएँ + क्रमलेखकोश छलनी + छलनी आवृत + छालन इनके अनुसार नाम विषय निमन्त्रक परिपालक… + कोई क्रमलेखकोश वर्तमान सूचनावली में इस खोज के अनुकूल नहीं। नोस्टरस्थान : %1$s नोस्टर संलग्नक्रमक : %1$s अनुमतियाँ : @@ -3645,7 +3659,7 @@ पूर्वावस्था असफल - अनुरोध के शीर्षक प्रपत्रस्थानों में निर्दिष्ट पूर्वावस्थाओं की पूर्ति में सेवासंगणक असफल भार अत्याधिक - अनुरोध सेवासंगणक के निरूपित सीमाएँ से बडा है, तथा सेवासंगणक ने इस पर काम करना नकार दिया जालपता लम्बाई अत्याधिक - ग्राहक द्वारा अनुरोधित जालपता की लम्बाई सेवासंगणक के काम के लिए अत्याधिक है। - अनावलम्बित माध्यम प्रकार - अनुरोध में प्रयुक्त माध्यम प्रकार सेवासंगणक द्वारा अवलम्बित नहीं + अनावलम्बित श्रव्यदृश्याभिलेख प्रकार - अनुरोध में प्रयुक्त श्रव्यदृश्याभिलेख प्रकार सेवासंगणक द्वारा अवलम्बित नहीं विस्तार असाध्य - अनुरोध के विस्तार शीर्षक प्रपत्रस्थान में सूचित मूल्य की पूर्ति सेवासंगणक द्वारा असाध्य। अपेक्षा असफल - अनुरोध के अपेक्षा शीर्षक प्रपत्रस्थान में सूचित आवश्यकताओं की पूर्ति सेवासंगणक के लिए असाध्य नवीकरण आवश्यक - ग्राहक वर्तमान से भिन्न संचारविधि तक नवीकरण नहीं करता तो सेवासंगणक इसके अनुरोध पर काम नहीं करेगा। @@ -3659,7 +3673,7 @@ स्मृतिस्थान का अभाव - सेवासंगणक में पर्याप्त स्मृतिस्थान उपलब्ध नहीं अनुरोध पर सफलतापूर्वक काम करने के लिए क्रमचक्र दृष्ट - सेवासंगणक को अनन्त क्रमचक्र का पता चला अनुरोध पर काम करते हुए जाल प्रमाणीकरण आवश्यक - जाल उपलब्ध होने के लिए ग्राहक का प्रमाणीकरण अनिवार्य - ब्लोस्सम॰ प्रसारसंगणक + ब्लोस्सम॰ सेवासंगणक सेवासंगणक जितना चाहें जोडें। किस संगणक का उपयोग करना है उसका चयन कर सकते हैं चित्र का आरोहण करते समय निप॰-९६ सेवासंगणक जोडें ब्लोस्सम॰ प्रसारसंगणक जोडें @@ -3669,7 +3683,7 @@ अवरोहण अभिलेख खोलने में असफल कोई अनेकत्रावरोहण क्रमक स्थापित नहीं अभिलेख खोलने तथा अवरोहण करने के लिए। - अभिलेखविभेदक युक्त जालनिर्देशक बनाने के लिए पर्याप्त जानकारी नहीं है घटना में + चुम्बकजाल योजक बनाने के लिए पर्याप्त जानकारी नहीं है घटना में मेरे सूचियाँ सूचनावली छानने के लिए सूची चुनें सूचनावली @@ -3769,7 +3783,7 @@ जाल अनुरोध विकिरणेन्द्रिय जाग उठना अवरोहणों के लिए (अनुमान) सक्रिय स्थानान्तरण समय - ध्वनिदृश्याभिलेख चलन समय + श्रव्यदृश्याभिलेख चलन समय हस्ताक्षर सत्यापित क्रमवर्तक समय उपयुक्त क्रमकाभ्यन्तर समय @@ -3908,7 +3922,7 @@ खेल आवहन चालू\u2026 खेल अप्राप्त सम्भाव्यतः खेल समाप्त अथवा प्रतिपक्ष की प्रतीक्षा में। - खेल विभेदक : %1$s\u2026 + खेल सूचक : %1$s\u2026 तथा %1$d अन्य %1$s हटाएँ @@ -4067,7 +4081,7 @@ अभी के लिए छोडें अनुचरण सूची प्राप्त की जा रही है… कोई अनुचरित नहीं - "किसी मित्र अथवा समूह नेता का परिचय प्रविष्ट करें। उनके एनपुब॰ अथवा निप॰०५ पता अथवा नामरूप्य नाम जैसे कि alice@example.com अथवा id/alice का उपयोग आप कर सकते हैं खण्डश्रृंखला सत्यापित विभेदकों के लिए।" + "किसी मित्र अथवा समूह नेता का परिचय प्रविष्ट करें। उनके एनपुब॰ अथवा निप॰०५ पता अथवा नामरूप्य नाम जैसे कि alice@example.com अथवा id/alice का उपयोग आप कर सकते हैं खण्डश्रृंखला सत्यापित परिचयसूचकों के लिए।" सभी चुनें %1$d%% समय निरन्तर उपलब्ध नामरूप्य स्थापना विकल्प @@ -4183,7 +4197,7 @@ पढा हुआ चिह्नित करें टीका कार्य परिचय कार्य - अभिलेख कार्य + श्रव्यदृश्याभिलेख कार्य चालन स्वचालित @@ -4342,7 +4356,7 @@ घटना प्रतिबन्धित करें प्रकार को अनुमति दें अंकीय जालपता बाधित करें - घटना विभेदक (षोडशांक) + घटना सूचक (षोडशांक) प्रकार संख्या अंकीय जालपता कारण (विकल्पात्मक) @@ -4718,7 +4732,7 @@ नयी व्यक्तिशैली व्यक्तिशैली सम्पादन - सूचकाम्श (व्यक्तिशैली विभेदक) + सूचकाम्श (व्यक्तिशैली सूचक) a-z तथा 0-9 तथा \'-\' तथा \'_\'। परिवर्तनीय नहीं बनाने के पश्चात। प्रदर्शन नाम यन्त्र प्रेरण diff --git a/amethyst/src/main/res/values-hu-rHU/strings.xml b/amethyst/src/main/res/values-hu-rHU/strings.xml index bb09b99b56..f559c7e139 100644 --- a/amethyst/src/main/res/values-hu-rHU/strings.xml +++ b/amethyst/src/main/res/values-hu-rHU/strings.xml @@ -204,6 +204,7 @@ Az átjátszóhoz való sikeres kapcsolatok százalékos aránya Felhasználó keresése és hozzáadása Egy átjátszó hozzáadása + Az átjátszó címe érvénytelen. Használjon gazdagépnevet vagy zárójelben megadott IP-címet (például: [201:d0e:9ba5:8bbc::1]:8080). Saját @említési név Megjelenítendő név Saját megjelenítendő név @@ -3222,6 +3223,16 @@ Hírfolyamok Alkalmazások és web Egyéb + Oldalsó menü + Saját oldalsó menü + Nyisson meg egy szakaszt, és kapcsolja ki azokat a sorokat, amelyeket soha nem használ. A beállítások mindig láthatók maradnak, így bármikor visszatérhet ide. A szakaszok sorrendje rögzített. + Szakaszok + %1$d rejtett + Látható + Rejtett + Mindig bekapcsolva + Összes megjelenítése + Összes elrejtése Kezdőlap lapjai Válassza ki, mely lapok jelenjenek meg a kezdőlapon. Ha csak egy lap aktív, akkor a lapsáv rejtett. Minden @@ -3595,6 +3606,10 @@ Mentés Git-tárolók Kiemelések + Tárolók szűrése + Szűrő bezárása + Szűrés név, téma, kiszolgáló, karbantartó szerint… + A jelenlegi hírcsatornában nincs olyan tároló, amely megfelelne ennek a keresésnek. nOldal: %1$s nKisalkalmazás: %1$s Engedélyek: diff --git a/amethyst/src/main/res/values-pl-rPL/strings.xml b/amethyst/src/main/res/values-pl-rPL/strings.xml index db3045cba8..001bb2d0b4 100644 --- a/amethyst/src/main/res/values-pl-rPL/strings.xml +++ b/amethyst/src/main/res/values-pl-rPL/strings.xml @@ -211,6 +211,8 @@ Odsetek udanych połączeń z transmiterem Szukaj i dodaj użytkownika Dodaj Transmiter + Nieprawidłowy adres transmitera. Użyj nazwy hosta lub adresu IP w nawiasach (na przykład +[201:d0e:9ba5:8bbc::1]:8080). Moje imię @tag Nazwa użytkownika Mój nick @@ -3347,6 +3349,16 @@ Zaplanowane posty z innych kont nie zostaną opublikowane, dopóki to konto jest Kanały Aplikacje & Strony Inne + Menu boczne + Twoje menu boczne + Otwórz sekcję i wyłącz wiersze, których nigdy nie używasz. Ustawienia zawsze pozostają widoczne, więc zawsze możesz tu wrócić. Kolejność sekcji jest ustalona. + Sekcje + %1$d ukrytych + Widoczne + Ukryte + Zawsze włączony + Pokaż wszystkie + Ukryj wszystkie Karty główne Wybierz, które karty pojawiają się na ekranie głównym. Gdy tylko jedna karta jest aktywna, pasek karty jest ukryty. Wszystko @@ -3726,6 +3738,10 @@ Zaplanowane posty z innych kont nie zostaną opublikowane, dopóki to konto jest Zapisz Repozytoria Git Wyróżnienia + Filtruj repozytoria + Zamknij filtr + Filtruj według nazwy, tematu, hosta, opiekuna… + Brak repozytoriów w bieżącym kanale pasujących do tego wyszukiwania. Statyczna Witryna: %1$s nApplet: %1$s Uprawnienia: From 6d7ec4d9376a27e227cea61e19bd9e08213809b9 Mon Sep 17 00:00:00 2001 From: davotoula Date: Fri, 7 Aug 2026 17:55:09 +0200 Subject: [PATCH 062/132] i18n: convert drawer hidden-count to plurals and fill missing cs/de/sv/pt strings --- .../loggedIn/settings/DrawerSettingsScreen.kt | 5 ++-- amethyst/src/main/res/values-cs/strings.xml | 19 ++++++++++++++ .../src/main/res/values-de-rDE/strings.xml | 17 +++++++++++++ .../src/main/res/values-hi-rIN/strings.xml | 5 +++- .../src/main/res/values-hu-rHU/strings.xml | 5 +++- .../src/main/res/values-pl-rPL/strings.xml | 7 +++++- .../src/main/res/values-pt-rBR/strings.xml | 25 +++++++++++++++++++ .../src/main/res/values-sv-rSE/strings.xml | 17 +++++++++++++ amethyst/src/main/res/values/strings.xml | 5 +++- 9 files changed, 99 insertions(+), 6 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/DrawerSettingsScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/DrawerSettingsScreen.kt index 9a32cc8a11..f1f6bef5bc 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/DrawerSettingsScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/DrawerSettingsScreen.kt @@ -40,6 +40,7 @@ import androidx.compose.runtime.derivedStateOf import androidx.compose.runtime.getValue import androidx.compose.runtime.remember import androidx.compose.ui.Modifier +import androidx.compose.ui.res.pluralStringResource import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.tooling.preview.Preview import androidx.compose.ui.unit.dp @@ -155,7 +156,7 @@ private fun SummaryCard(totalHidden: Int) { title = stringRes(R.string.drawer_settings_title), trailing = { Text( - text = stringRes(R.string.drawer_settings_hidden_count, totalHidden), + text = pluralStringResource(R.plurals.drawer_settings_hidden_count, totalHidden, totalHidden), style = MaterialTheme.typography.labelMedium, color = MaterialTheme.colorScheme.primary, fontWeight = FontWeight.Bold, @@ -189,7 +190,7 @@ private fun SectionCard( trailing = { if (hiddenHere > 0) { Text( - text = stringRes(R.string.drawer_settings_hidden_count, hiddenHere), + text = pluralStringResource(R.plurals.drawer_settings_hidden_count, hiddenHere, hiddenHere), style = MaterialTheme.typography.labelMedium, color = MaterialTheme.colorScheme.onSurfaceVariant, ) diff --git a/amethyst/src/main/res/values-cs/strings.xml b/amethyst/src/main/res/values-cs/strings.xml index 854354592d..09e207b16c 100644 --- a/amethyst/src/main/res/values-cs/strings.xml +++ b/amethyst/src/main/res/values-cs/strings.xml @@ -4905,4 +4905,23 @@ URL avataru (volitelné) Publikování… Publikovat personu + Boční nabídka + Vaše boční nabídka + Otevřete sekci a vypněte řádky, které nikdy nepoužíváte. Nastavení zůstává vždy viditelné, takže se sem vždy vrátíte. Pořadí sekcí je pevné. + Sekce + + %1$d skrytá + %1$d skryté + %1$d skrytých + %1$d skrytých + + Viditelné + Skryté + Vždy zapnuto + Zobrazit vše + Skrýt vše + Filtrovat repozitáře + Zavřít filtr + Filtrovat podle názvu, tématu, hostitele, správce… + Tomuto hledání neodpovídají žádné repozitáře v aktuálním kanálu. diff --git a/amethyst/src/main/res/values-de-rDE/strings.xml b/amethyst/src/main/res/values-de-rDE/strings.xml index 16abc0f2e7..e7728b8f71 100644 --- a/amethyst/src/main/res/values-de-rDE/strings.xml +++ b/amethyst/src/main/res/values-de-rDE/strings.xml @@ -4713,4 +4713,21 @@ Avatar-URL (optional) Wird veröffentlicht… Persona veröffentlichen + Seitenmenü + Dein Seitenmenü + Öffne einen Bereich und schalte die Zeilen aus, die du nie nutzt. Einstellungen bleibt immer sichtbar, damit du jederzeit hierher zurückkommst. Die Reihenfolge der Bereiche ist fest. + Bereiche + + %1$d ausgeblendet + %1$d ausgeblendet + + Sichtbar + Ausgeblendet + Immer an + Alle anzeigen + Alle ausblenden + Repositories filtern + Filter schließen + Nach Name, Thema, Host, Betreuer filtern… + Keine Repositories im aktuellen Feed passen zu dieser Suche. diff --git a/amethyst/src/main/res/values-hi-rIN/strings.xml b/amethyst/src/main/res/values-hi-rIN/strings.xml index 68dc9e575b..6faae86163 100644 --- a/amethyst/src/main/res/values-hi-rIN/strings.xml +++ b/amethyst/src/main/res/values-hi-rIN/strings.xml @@ -3226,7 +3226,10 @@ आपकी पार्श्व विकल्पसूची एक विभाग खोलें तथा उन पंक्तियों को निष्क्रिय करें जिनका उपयोग आप कभी नहीं करते। स्थापना विकल्प सर्वदा दृश्यमान रहते हैं। जिससे कि आप यहाँ कभी भी लौट सकेंगे। विभाग क्रम स्थायी है। विभाग - %1$d छिपे हुए + + %1$d छिपा हुआ + %1$d छिपे हुए + दृश्यमान छिपे हुए सर्वदा सक्रिय diff --git a/amethyst/src/main/res/values-hu-rHU/strings.xml b/amethyst/src/main/res/values-hu-rHU/strings.xml index f559c7e139..ec099bd65c 100644 --- a/amethyst/src/main/res/values-hu-rHU/strings.xml +++ b/amethyst/src/main/res/values-hu-rHU/strings.xml @@ -3227,7 +3227,10 @@ Saját oldalsó menü Nyisson meg egy szakaszt, és kapcsolja ki azokat a sorokat, amelyeket soha nem használ. A beállítások mindig láthatók maradnak, így bármikor visszatérhet ide. A szakaszok sorrendje rögzített. Szakaszok - %1$d rejtett + + %1$d rejtett + %1$d rejtett + Látható Rejtett Mindig bekapcsolva diff --git a/amethyst/src/main/res/values-pl-rPL/strings.xml b/amethyst/src/main/res/values-pl-rPL/strings.xml index 001bb2d0b4..4d613a32e3 100644 --- a/amethyst/src/main/res/values-pl-rPL/strings.xml +++ b/amethyst/src/main/res/values-pl-rPL/strings.xml @@ -3353,7 +3353,12 @@ Zaplanowane posty z innych kont nie zostaną opublikowane, dopóki to konto jest Twoje menu boczne Otwórz sekcję i wyłącz wiersze, których nigdy nie używasz. Ustawienia zawsze pozostają widoczne, więc zawsze możesz tu wrócić. Kolejność sekcji jest ustalona. Sekcje - %1$d ukrytych + + %1$d ukryta + %1$d ukryte + %1$d ukrytych + %1$d ukrytych + Widoczne Ukryte Zawsze włączony diff --git a/amethyst/src/main/res/values-pt-rBR/strings.xml b/amethyst/src/main/res/values-pt-rBR/strings.xml index 33467f8882..380c186b11 100644 --- a/amethyst/src/main/res/values-pt-rBR/strings.xml +++ b/amethyst/src/main/res/values-pt-rBR/strings.xml @@ -4711,4 +4711,29 @@ URL do avatar (opcional) Publicando… Publicar persona + Menu lateral + Seu menu lateral + Abra uma seção e desligue as linhas que você nunca usa. Configurações permanece sempre visível, então você sempre pode voltar aqui. A ordem das seções é fixa. + Seções + + %1$d oculta + %1$d ocultas + + Visíveis + Ocultas + Sempre ativo + Mostrar tudo + Ocultar tudo + Filtrar repositórios + Fechar filtro + Filtrar por nome, tópico, host, mantenedor… + Nenhum repositório no feed atual corresponde a esta pesquisa. + + %1$d relay + %1$d relays + + + %1$s \u00b7 %2$d relay + %1$s \u00b7 %2$d relays + diff --git a/amethyst/src/main/res/values-sv-rSE/strings.xml b/amethyst/src/main/res/values-sv-rSE/strings.xml index 17d4fe1f2d..32a34b6eac 100644 --- a/amethyst/src/main/res/values-sv-rSE/strings.xml +++ b/amethyst/src/main/res/values-sv-rSE/strings.xml @@ -4725,4 +4725,21 @@ Avatar-URL (valfritt) Publicerar… Publicera persona + Sidomeny + Din sidomeny + Öppna en sektion och stäng av raderna du aldrig använder. Inställningar förblir alltid synligt, så du kan alltid ta dig tillbaka hit. Sektionernas ordning är fast. + Sektioner + + %1$d dold + %1$d dolda + + Synliga + Dolda + Alltid på + Visa alla + Dölj alla + Filtrera repositories + Stäng filter + Filtrera på namn, ämne, värd, underhållare… + Inga repositories i det aktuella flödet matchar den här sökningen. diff --git a/amethyst/src/main/res/values/strings.xml b/amethyst/src/main/res/values/strings.xml index b6b362ed51..da24418325 100644 --- a/amethyst/src/main/res/values/strings.xml +++ b/amethyst/src/main/res/values/strings.xml @@ -3517,7 +3517,10 @@ Your side menu Open a section and switch off the rows you never use. Settings always stays visible, so you can always get back here. Section order is fixed. Sections - %1$d hidden + + %1$d hidden + %1$d hidden + Visible Hidden Always on From 8b0ea7389c17baae6532fba2a4873a6aaa14d6dd Mon Sep 17 00:00:00 2001 From: davotoula <1747287+davotoula@users.noreply.github.com> Date: Fri, 7 Aug 2026 16:14:52 +0000 Subject: [PATCH 063/132] chore: sync Crowdin translations and seed translator npub placeholders --- amethyst/src/main/res/values-cs/strings.xml | 38 ++++++++--------- .../src/main/res/values-de-rDE/strings.xml | 34 +++++++-------- .../src/main/res/values-pt-rBR/strings.xml | 42 ++++++++----------- .../src/main/res/values-sv-rSE/strings.xml | 34 +++++++-------- docs/changelog/translators.json | 2 + 5 files changed, 72 insertions(+), 78 deletions(-) diff --git a/amethyst/src/main/res/values-cs/strings.xml b/amethyst/src/main/res/values-cs/strings.xml index 09e207b16c..2fce7e78c2 100644 --- a/amethyst/src/main/res/values-cs/strings.xml +++ b/amethyst/src/main/res/values-cs/strings.xml @@ -3347,6 +3347,21 @@ Kanály Aplikace a web Ostatní + Boční nabídka + Vaše boční nabídka + Otevřete sekci a vypněte řádky, které nikdy nepoužíváte. Nastavení zůstává vždy viditelné, takže se sem vždy vrátíte. Pořadí sekcí je pevné. + Sekce + + %1$d skrytá + %1$d skryté + %1$d skrytých + %1$d skrytých + + Viditelné + Skryté + Vždy zapnuto + Zobrazit vše + Skrýt vše Záložky domova Vyberte, které záložky se zobrazí na domovské obrazovce. Pokud je aktivní jen jedna záložka, lišta záložek se skryje. Vše @@ -3726,6 +3741,10 @@ Uložit Git repozitáře Zvýraznění + Filtrovat repozitáře + Zavřít filtr + Filtrovat podle názvu, tématu, hostitele, správce… + Tomuto hledání neodpovídají žádné repozitáře v aktuálním kanálu. Statický web: %1$s nApplet: %1$s Oprávnění: @@ -4905,23 +4924,4 @@ URL avataru (volitelné) Publikování… Publikovat personu - Boční nabídka - Vaše boční nabídka - Otevřete sekci a vypněte řádky, které nikdy nepoužíváte. Nastavení zůstává vždy viditelné, takže se sem vždy vrátíte. Pořadí sekcí je pevné. - Sekce - - %1$d skrytá - %1$d skryté - %1$d skrytých - %1$d skrytých - - Viditelné - Skryté - Vždy zapnuto - Zobrazit vše - Skrýt vše - Filtrovat repozitáře - Zavřít filtr - Filtrovat podle názvu, tématu, hostitele, správce… - Tomuto hledání neodpovídají žádné repozitáře v aktuálním kanálu. diff --git a/amethyst/src/main/res/values-de-rDE/strings.xml b/amethyst/src/main/res/values-de-rDE/strings.xml index e7728b8f71..b8f2041354 100644 --- a/amethyst/src/main/res/values-de-rDE/strings.xml +++ b/amethyst/src/main/res/values-de-rDE/strings.xml @@ -3212,6 +3212,19 @@ Feeds Apps & Web Sonstiges + Seitenmenü + Dein Seitenmenü + Öffne einen Bereich und schalte die Zeilen aus, die du nie nutzt. Einstellungen bleibt immer sichtbar, damit du jederzeit hierher zurückkommst. Die Reihenfolge der Bereiche ist fest. + Bereiche + + %1$d ausgeblendet + %1$d ausgeblendet + + Sichtbar + Ausgeblendet + Immer an + Alle anzeigen + Alle ausblenden Startseiten-Tabs Wähle, welche Tabs auf der Startseite erscheinen. Wenn nur ein Tab aktiv ist, wird die Tab-Leiste ausgeblendet. Alles @@ -3584,6 +3597,10 @@ Themen (durch Komma getrennt) Speichern Git Repositories + Repositories filtern + Filter schließen + Nach Name, Thema, Host, Betreuer filtern… + Keine Repositories im aktuellen Feed passen zu dieser Suche. Statische Website: %1$s nApplet: %1$s Berechtigungen: @@ -4713,21 +4730,4 @@ Avatar-URL (optional) Wird veröffentlicht… Persona veröffentlichen - Seitenmenü - Dein Seitenmenü - Öffne einen Bereich und schalte die Zeilen aus, die du nie nutzt. Einstellungen bleibt immer sichtbar, damit du jederzeit hierher zurückkommst. Die Reihenfolge der Bereiche ist fest. - Bereiche - - %1$d ausgeblendet - %1$d ausgeblendet - - Sichtbar - Ausgeblendet - Immer an - Alle anzeigen - Alle ausblenden - Repositories filtern - Filter schließen - Nach Name, Thema, Host, Betreuer filtern… - Keine Repositories im aktuellen Feed passen zu dieser Suche. diff --git a/amethyst/src/main/res/values-pt-rBR/strings.xml b/amethyst/src/main/res/values-pt-rBR/strings.xml index 380c186b11..1ceae75a2e 100644 --- a/amethyst/src/main/res/values-pt-rBR/strings.xml +++ b/amethyst/src/main/res/values-pt-rBR/strings.xml @@ -3206,6 +3206,19 @@ Feeds Apps e Web Outros + Menu lateral + Seu menu lateral + Abra uma seção e desligue as linhas que você nunca usa. Configurações permanece sempre visível, então você sempre pode voltar aqui. A ordem das seções é fixa. + Seções + + %1$d oculta + %1$d ocultas + + Visíveis + Ocultas + Sempre ativo + Mostrar tudo + Ocultar tudo Abas da Tela Inicial Escolha quais abas aparecem na Tela Inicial. Quando apenas uma aba está ativa, a barra de abas fica oculta. Tudo @@ -3579,6 +3592,10 @@ Salvar Repositórios Git Destaques + Filtrar repositórios + Fechar filtro + Filtrar por nome, tópico, host, mantenedor… + Nenhum repositório no feed atual corresponde a esta pesquisa. Site Estático: %1$s nApplet: %1$s Permissões: @@ -4711,29 +4728,4 @@ URL do avatar (opcional) Publicando… Publicar persona - Menu lateral - Seu menu lateral - Abra uma seção e desligue as linhas que você nunca usa. Configurações permanece sempre visível, então você sempre pode voltar aqui. A ordem das seções é fixa. - Seções - - %1$d oculta - %1$d ocultas - - Visíveis - Ocultas - Sempre ativo - Mostrar tudo - Ocultar tudo - Filtrar repositórios - Fechar filtro - Filtrar por nome, tópico, host, mantenedor… - Nenhum repositório no feed atual corresponde a esta pesquisa. - - %1$d relay - %1$d relays - - - %1$s \u00b7 %2$d relay - %1$s \u00b7 %2$d relays - diff --git a/amethyst/src/main/res/values-sv-rSE/strings.xml b/amethyst/src/main/res/values-sv-rSE/strings.xml index 32a34b6eac..59ec28bf1d 100644 --- a/amethyst/src/main/res/values-sv-rSE/strings.xml +++ b/amethyst/src/main/res/values-sv-rSE/strings.xml @@ -3219,6 +3219,19 @@ Flöden Appar & webb Övrigt + Sidomeny + Din sidomeny + Öppna en sektion och stäng av raderna du aldrig använder. Inställningar förblir alltid synligt, så du kan alltid ta dig tillbaka hit. Sektionernas ordning är fast. + Sektioner + + %1$d dold + %1$d dolda + + Synliga + Dolda + Alltid på + Visa alla + Dölj alla Hemflikar Välj vilka flikar som visas på startskärmen. När endast en flik är aktiv döljs flikraden. Allt @@ -3592,6 +3605,10 @@ Spara Git-repositories Höjdpunkter + Filtrera repositories + Stäng filter + Filtrera på namn, ämne, värd, underhållare… + Inga repositories i det aktuella flödet matchar den här sökningen. Statisk webbplats: %1$s nApplet: %1$s Behörigheter: @@ -4725,21 +4742,4 @@ Avatar-URL (valfritt) Publicerar… Publicera persona - Sidomeny - Din sidomeny - Öppna en sektion och stäng av raderna du aldrig använder. Inställningar förblir alltid synligt, så du kan alltid ta dig tillbaka hit. Sektionernas ordning är fast. - Sektioner - - %1$d dold - %1$d dolda - - Synliga - Dolda - Alltid på - Visa alla - Dölj alla - Filtrera repositories - Stäng filter - Filtrera på namn, ämne, värd, underhållare… - Inga repositories i det aktuella flödet matchar den här sökningen. diff --git a/docs/changelog/translators.json b/docs/changelog/translators.json index a322a2a269..78f466826a 100644 --- a/docs/changelog/translators.json +++ b/docs/changelog/translators.json @@ -95,6 +95,8 @@ "languages": [ "Czech", "German", + "Hindi", + "Hungarian", "Polish", "Portuguese, Brazilian", "Swedish" From 3f087e5c6080f09c1b3294c9f8716ef131c7b998 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 7 Aug 2026 23:46:30 +0000 Subject: [PATCH 064/132] Quartz: give SyncCoverage's persistence a typed band key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `export`/`restore` handed back `Map` where the string was the INTERNAL key — `" "`. That is fine for a file layer that writes the key back verbatim, and nothing else. A layer that wants its own layout — one object per relay, or per filter, or nested by both — had to split the key apart, and the separator was folklore it could only learn by reading this class. Two of them now do. So the key is a pair, with the joined form kept here as `encode`/`decode` for a file that does want one key per line. geode keeps its format byte-for-byte and stops pattern-matching on somebody else's string. It is also faster on the path that matters. `key()` built a new string per lookup, so a `legs()` over a fan-out COPIED the filter's json — tens of thousands of characters for an author-scoped filter — once per relay per cycle, then hashed all of it, since a freshly built string carries no cached hash. The pair hashes two halves it already holds: the url, and the fingerprint instance the cache above it already returns. No behaviour change: the same pairs key the same bands, a file written before this reads back through `decode`, and the format on disk is untouched. --- .../geode/mirror/SyncCoverageFile.kt | 25 +++++---- .../relay/client/accessories/SyncCoverage.kt | 51 ++++++++++++++++--- .../client/accessories/SyncCoverageTest.kt | 20 ++++++++ 3 files changed, 80 insertions(+), 16 deletions(-) diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt index 58b0b23310..ea5f9aad21 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt @@ -96,15 +96,22 @@ class SyncCoverageFile( if (!file.isFile) return runCatching { val root = Json.parseToJsonElement(file.readText()).jsonObject + // The file's key is the joined form, decoded by the class that + // mints it — this layer never has to know the separator. A key it + // cannot read names no pair and is dropped, which costs one + // upstream's re-walk rather than the whole file. coverage.restore( - root.mapValues { (_, v) -> - val o = v.jsonObject - SyncCoverage.Band( - spansOf(o), - o["complete"]?.jsonPrimitive?.boolean ?: false, - o["fullAt"]?.jsonPrimitive?.long ?: 0L, - ) - }, + root.entries + .mapNotNull { (k, v) -> + val key = SyncCoverage.BandKey.decode(k) ?: return@mapNotNull null + val o = v.jsonObject + key to + SyncCoverage.Band( + spansOf(o), + o["complete"]?.jsonPrimitive?.boolean ?: false, + o["fullAt"]?.jsonPrimitive?.long ?: 0L, + ) + }.toMap(), ) }.onFailure { Log.w("SyncCoverageFile") { "could not read ${file.path} (${it.message}); starting fresh" } @@ -142,7 +149,7 @@ class SyncCoverageFile( buildJsonObject { coverage.export().forEach { (key, band) -> put( - key, + key.encode(), buildJsonObject { // min/max are the outer edges across every // kind, and are written for two readers: a diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt index 3eb54ae9fd..c8e2e3aed8 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt @@ -49,8 +49,10 @@ import com.vitorpamplona.quartz.utils.concurrent.ConcurrentMap * occasional and self-heal on the next filter change or full re-walk. * * Persistence is the caller's: [export] the map on a schedule and [restore] - * it at startup. [onChange] fires whenever a band changes, so a persistence - * layer can mark itself dirty without polling. + * it at startup, both keyed by [BandKey] so a file layer can lay the two + * halves out however it likes without having to know how a key is spelled. + * [onChange] fires whenever a band changes, so a persistence layer can mark + * itself dirty without polling. * * Not to be confused with the `relay.client.paging` package: its * `RelayLoadingCursors` are in-memory POSITIONS for demand-driven UI paging @@ -65,6 +67,41 @@ class SyncCoverage( private val now: () -> Long = { TimeUtils.now() }, private val onChange: () -> Unit = {}, ) { + /** + * What one band is about: the relay's url, and the filter as [Filter.toJson] + * renders it. + * + * A pair, not a joined string, for two reasons. A persistence layer needs + * the halves — one that lays its file out by relay, or by filter, or by + * both, had to split the key back apart, and the separator was folklore it + * could only learn by reading this class. And on the hot path a joined key + * COPIES the filter's json on every lookup: `legs()` runs once per relay + * per cycle, an author-scoped filter's json runs to tens of thousands of + * characters, and a fan-out over thousands of relays paid that copy — plus + * a fresh hash over all of it, since a newly built string has none cached — + * on every one of them. A pair hashes the two halves it already holds. + * + * [encode] and [decode] are the joined form, kept HERE so a file that wants + * one key per line still gets the separator from the class that mints it. + * A normalized relay url contains no space, which is what makes splitting + * at the first one exact. + */ + data class BandKey( + val relay: String, + val filter: String, + ) { + fun encode(): String = "$relay $filter" + + companion object { + /** The inverse of [encode], or null for a key that names no pair. */ + fun decode(key: String): BandKey? { + val at = key.indexOf(' ') + if (at <= 0 || at == key.length - 1) return null + return BandKey(key.substring(0, at), key.substring(at + 1)) + } + } + } + /** A covered `created_at` interval, inclusive at both ends. */ data class Span( val min: Long, @@ -115,7 +152,7 @@ class SyncCoverage( } } - private val bands = ConcurrentMap() + private val bands = ConcurrentMap() // filter -> its canonical json. Filter.toJson() runs to tens of thousands // of characters for author-scoped filters, and a fan-out keys once per @@ -379,10 +416,10 @@ class SyncCoverage( fun size(): Int = bands.size() /** A point-in-time copy of every band, for a persistence layer to write out. */ - fun export(): Map = bands.snapshot() + fun export(): Map = bands.snapshot() /** Load previously [export]ed bands, e.g. at startup. */ - fun restore(entries: Map) { + fun restore(entries: Map) { for ((key, band) in entries) bands[key] = band } @@ -395,7 +432,7 @@ class SyncCoverage( private fun key( url: NormalizedRelayUrl, filter: Filter, - ): String { + ): BandKey { val fingerprint = fingerprints[filter] ?: filter.toJson().also { @@ -404,7 +441,7 @@ class SyncCoverage( // pays the toJson each time instead of growing the heap. if (fingerprints.size() < MAX_FINGERPRINTS) fingerprints[filter] = it } - return "${url.url} $fingerprint" + return BandKey(url.url, fingerprint) } // One line per process, not per walk: the point is to tell a caller it has diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt index 3c26e1891e..abf1113f02 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt @@ -354,6 +354,26 @@ class SyncCoverageTest { assertEquals(1_700_002_000L, band.maxCreatedAt) } + @Test + fun `a band key round-trips through the joined form a file writes`() { + // A file that wants one key per line joins and splits with these, so + // the separator stays in the class that mints the key instead of being + // rediscovered by every persistence layer downstream. + val c = SyncCoverage() + c.record(relay, profiles, 1_700_001_000L, 1_700_002_000L, paged = true) + val key = c.export().keys.single() + + assertEquals(relay.url, key.relay) + assertEquals(profiles.toJson(), key.filter) + assertEquals(key, SyncCoverage.BandKey.decode(key.encode())) + + // A key naming no pair is refused rather than read as a relay with an + // empty filter, which would key a band nothing can ever look up. + assertNull(SyncCoverage.BandKey.decode("no-space-here")) + assertNull(SyncCoverage.BandKey.decode(" {\"kinds\":[0]}")) + assertNull(SyncCoverage.BandKey.decode("wss://relay.example/ ")) + } + @Test fun `onChange fires when a band changes so persistence can mark dirty`() { var changes = 0 From dc03209bb544fbc784da84d9df1b6cb1c26f3663 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 00:41:02 +0000 Subject: [PATCH 065/132] fix: silence Kotlin override-parameter-name and redundant-!! warnings LocalCache implements both Dao and ICacheProvider, which disagreed on the parameter names of getOrCreateUser (hex vs pubkey) and getOrCreateAddressableNote (address vs key), so every override warned about named-argument mismatches. Align both interfaces on pubkey/address and update the implementations that used the other name. Also drop the non-null assertions the compiler already smart-casts away in LimitsPolicy.capLimits and RelayProberFlowTest, and match the WebSocketListener parameter names in NegentropyStallRepro. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017X7C797zGYsiui5yj1JQcY --- .../java/com/vitorpamplona/amethyst/model/Dao.kt | 2 +- .../com/vitorpamplona/amethyst/model/LocalCache.kt | 6 +++--- .../amethyst/ui/screen/loggedIn/AccountViewModel.kt | 2 +- .../amethyst/NewMessageTaggerKeyParseTest.kt | 2 +- .../amethyst/commons/model/cache/ICacheProvider.kt | 2 +- .../amethyst/commons/model/ThreadAssemblerTest.kt | 2 +- .../model/concord/ConcordChannelListLeaveTest.kt | 2 +- .../model/concord/ConcordListLateArrivalTest.kt | 2 +- .../amethyst/commons/ui/note/ReplyContextTest.kt | 2 +- .../amethyst/desktop/cache/DesktopLocalCache.kt | 6 +++--- .../nip01Core/relay/server/policies/LimitsPolicy.kt | 4 ++-- .../reachability/RelayProberFlowTest.kt | 6 +++--- .../relay/prodbench/NegentropyStallRepro.kt | 12 ++++++------ 13 files changed, 25 insertions(+), 25 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Dao.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Dao.kt index c1e3443d94..00062654eb 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Dao.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Dao.kt @@ -29,7 +29,7 @@ import com.vitorpamplona.quartz.nip01Core.core.HexKey * needing the full [LocalCache] API. */ interface Dao { - fun getOrCreateUser(hex: HexKey): User + fun getOrCreateUser(pubkey: HexKey): User fun getOrCreateNote(hex: HexKey): Note diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/LocalCache.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/LocalCache.kt index 7ffa080688..9df341c771 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/LocalCache.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/LocalCache.kt @@ -685,12 +685,12 @@ object LocalCache : ILocalCache, ICacheProvider, Dao { fun load(keys: Set): Set = keys.mapNotNullTo(mutableSetOf(), ::checkGetOrCreateUser) - override fun getOrCreateUser(hex: HexKey): User { - require(isValidHex(key = hex)) { "$hex is not a valid hex" } + override fun getOrCreateUser(pubkey: HexKey): User { + require(isValidHex(key = pubkey)) { "$pubkey is not a valid hex" } // Pass `this` as the UserContext — User now resolves each pinned // addressable note (kind:10002 / 10050 / 10019) lazily on first // read, instead of all-or-nothing at construction time. - return users.getOrCreate(hex) { User(it, userContext) } + return users.getOrCreate(pubkey) { User(it, userContext) } } /** [UserContext] bridge to this cache's addressable lookup. */ diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/AccountViewModel.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/AccountViewModel.kt index f27b3215c9..490f853b80 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/AccountViewModel.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/AccountViewModel.kt @@ -2106,7 +2106,7 @@ class AccountViewModel( fun checkGetOrCreateUser(key: HexKey): User? = LocalCache.checkGetOrCreateUser(key) - override fun getOrCreateUser(hex: HexKey): User = LocalCache.getOrCreateUser(hex) + override fun getOrCreateUser(pubkey: HexKey): User = LocalCache.getOrCreateUser(pubkey) fun getUserIfExists(hex: HexKey): User? = LocalCache.getUserIfExists(hex) diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/NewMessageTaggerKeyParseTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/NewMessageTaggerKeyParseTest.kt index dba6e87238..b73dbfd3fc 100644 --- a/amethyst/src/test/java/com/vitorpamplona/amethyst/NewMessageTaggerKeyParseTest.kt +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/NewMessageTaggerKeyParseTest.kt @@ -39,7 +39,7 @@ import org.junit.Test class NewMessageTaggerKeyParseTest { val dao: Dao = object : Dao { - override fun getOrCreateUser(hex: String): User = User(hex) { addr -> getOrCreateAddressableNoteInternal(addr) } + override fun getOrCreateUser(pubkey: String): User = User(pubkey) { addr -> getOrCreateAddressableNoteInternal(addr) } override fun getOrCreateNote(hex: String) = com.vitorpamplona.amethyst.model diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/cache/ICacheProvider.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/cache/ICacheProvider.kt index e4b3ccd9ae..6bdb38532e 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/cache/ICacheProvider.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/cache/ICacheProvider.kt @@ -93,7 +93,7 @@ interface ICacheProvider { * @param address The note's ID in address format * @return The AddressableNote (existing or newly created) */ - fun getOrCreateAddressableNote(key: Address): AddressableNote + fun getOrCreateAddressableNote(address: Address): AddressableNote /** * Gets the event stream for cache updates. diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/ThreadAssemblerTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/ThreadAssemblerTest.kt index e676dd6dc6..e2a25246a9 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/ThreadAssemblerTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/ThreadAssemblerTest.kt @@ -181,7 +181,7 @@ class ThreadAssemblerTest { override fun checkGetOrCreateNote(hexKey: HexKey): Note? = notesById[hexKey] - override fun getOrCreateAddressableNote(key: Address): AddressableNote = error("not used by ThreadAssembler in this test") + override fun getOrCreateAddressableNote(address: Address): AddressableNote = error("not used by ThreadAssembler in this test") override fun getEventStream(): ICacheEventStream = error("not used by ThreadAssembler in this test") diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordChannelListLeaveTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordChannelListLeaveTest.kt index 2b4682326f..600a56a38b 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordChannelListLeaveTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordChannelListLeaveTest.kt @@ -90,7 +90,7 @@ class ConcordChannelListLeaveTest { override fun checkGetOrCreateNote(hexKey: HexKey): Note? = null - override fun getOrCreateAddressableNote(key: Address): AddressableNote = AddressableNote(key) + override fun getOrCreateAddressableNote(address: Address): AddressableNote = AddressableNote(address) override fun getEventStream(): ICacheEventStream = error("not used") diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordListLateArrivalTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordListLateArrivalTest.kt index 366693b062..e4325e9280 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordListLateArrivalTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordListLateArrivalTest.kt @@ -88,7 +88,7 @@ class ConcordListLateArrivalTest { override fun checkGetOrCreateNote(hexKey: HexKey): Note? = null - override fun getOrCreateAddressableNote(key: Address): AddressableNote = notes.getOrPut(key.toValue()) { AddressableNote(key) } + override fun getOrCreateAddressableNote(address: Address): AddressableNote = notes.getOrPut(address.toValue()) { AddressableNote(address) } override fun getEventStream(): ICacheEventStream = error("not used") diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/ui/note/ReplyContextTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/ui/note/ReplyContextTest.kt index e0d5791888..0a23769767 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/ui/note/ReplyContextTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/ui/note/ReplyContextTest.kt @@ -121,7 +121,7 @@ class ReplyContextTest { override fun checkGetOrCreateNote(hexKey: HexKey): Note? = notesById[hexKey] - override fun getOrCreateAddressableNote(key: Address): AddressableNote = error("not used by ReplyContext.from") + override fun getOrCreateAddressableNote(address: Address): AddressableNote = error("not used by ReplyContext.from") override fun getEventStream(): ICacheEventStream = error("not used by ReplyContext.from") diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/cache/DesktopLocalCache.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/cache/DesktopLocalCache.kt index af026c5746..b83d490a28 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/cache/DesktopLocalCache.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/cache/DesktopLocalCache.kt @@ -904,9 +904,9 @@ class DesktopLocalCache : ICacheProvider { Note(hexKey) } - override fun getOrCreateAddressableNote(key: Address): AddressableNote = - addressableNotes.getOrCreate(key.toValue()) { - AddressableNote(key) + override fun getOrCreateAddressableNote(address: Address): AddressableNote = + addressableNotes.getOrCreate(address.toValue()) { + AddressableNote(address) } // ----- Channel operations ----- diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/policies/LimitsPolicy.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/policies/LimitsPolicy.kt index 3aaa3e3f32..e543fa5a41 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/policies/LimitsPolicy.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/server/policies/LimitsPolicy.kt @@ -128,8 +128,8 @@ class LimitsPolicy( */ private fun capLimits(filters: List): List { val max = limits.maxLimit ?: return filters - if (filters.none { it.limit != null && it.limit!! > max }) return filters - return filters.map { if (it.limit != null && it.limit!! > max) it.copy(limit = max) else it } + if (filters.none { it.limit != null && it.limit > max }) return filters + return filters.map { if (it.limit != null && it.limit > max) it.copy(limit = max) else it } } private fun targetLimit(current: Int?): Int? = diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt index 1cfdfcea00..f0a6be04ca 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt @@ -327,9 +327,9 @@ class RelayProberFlowTest { } check.join() - val verdict = result!![fast]!! - assertEquals(true, verdict.writeAccepted, "the listed relay's OK must still be awaited and recorded") - assertNull(result!![foreign], "the foreign relay must not appear in the result") + val verdicts = result!! + assertEquals(true, verdicts[fast]!!.writeAccepted, "the listed relay's OK must still be awaited and recorded") + assertNull(verdicts[foreign], "the foreign relay must not appear in the result") } @Test diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/NegentropyStallRepro.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/NegentropyStallRepro.kt index c85e4fb80a..16ef708751 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/NegentropyStallRepro.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/NegentropyStallRepro.kt @@ -103,10 +103,10 @@ class NegentropyStallRepro { object : WebSocketListener { override fun onOpen( pingMillis: Int, - usingCompression: Boolean, + compression: Boolean, ) { - log(" [<-open] ${url.url} ping=${pingMillis}ms deflate=$usingCompression") - out.onOpen(pingMillis, usingCompression) + log(" [<-open] ${url.url} ping=${pingMillis}ms deflate=$compression") + out.onOpen(pingMillis, compression) } override suspend fun onMessage(text: String) { @@ -130,10 +130,10 @@ class NegentropyStallRepro { override fun onFailure( t: Throwable, code: Int?, - errorMessage: String?, + response: String?, ) { - log(" [<-failure] ${url.url} code=$code msg=$errorMessage err=${t.message}") - out.onFailure(t, code, errorMessage) + log(" [<-failure] ${url.url} code=$code msg=$response err=${t.message}") + out.onFailure(t, code, response) } } From c8e357381247f8f502fca679400d5e6d301d238e Mon Sep 17 00:00:00 2001 From: davotoula Date: Sat, 8 Aug 2026 08:23:50 +0200 Subject: [PATCH 066/132] feat(resourceusage): relay churn and traffic attribution counters The ledger could say how much relay data the app moved, but not why. It counted completed connections and a single undifferentiated byte total, so "1.65 GB/day across 6,600 connects" could not be broken down further, and relay.connfails was being read as a dial-failure count when it also fires for mid-session drops of successful connections. Adds, all as counters with no behaviour change: relay.dials / relay.disc real dial and disconnect counts relay.life. connection-lifetime histogram, bucketed to straddle STABLE_CONNECTION_IN_SECS relay.verb.up/down. the byte totals split by protocol verb relay.purpose.

.* REQ bytes, inbound bytes and frames by the SubPurpose that asked, read off the ExplainedFilter that already travels on the filter relay.subs.* REQs sent, closed, replayed after connect, and re-sent for an already-open subscription relay.events.* inbound EVENT frames and how many carried an event already delivered relay.notice. NOTICE frames by an allowlisted reason relay.hs / relay.gap the transport's own handshake timing, and everything before the request went out relay.trigger. which decision asked for a reconnect --- .../com/vitorpamplona/amethyst/AppModules.kt | 1 + .../relayClient/RelayProxyClientConnector.kt | 29 + .../resourceusage/RelayUsageListener.kt | 265 +++++- .../resourceusage/ResourceUsageAccountant.kt | 16 +- .../ResourceUsageReportAssembler.kt | 7 +- .../resourceusage/ResourceUsageStore.kt | 6 +- .../service/resourceusage/UsageKeys.kt | 610 +++++++++++++- .../loggedIn/buzz/AgentConsoleViewModel.kt | 4 +- .../loggedIn/buzz/BuzzDmListViewModel.kt | 4 +- .../screen/loggedIn/buzz/BuzzJoinReconnect.kt | 53 ++ .../loggedIn/buzz/BuzzRelayImportViewModel.kt | 10 +- .../resourceusage/ResourceUsageLedgerTest.kt | 774 ++++++++++++++++++ .../client/single/basic/BasicRelayClient.kt | 6 +- 13 files changed, 1744 insertions(+), 41 deletions(-) create mode 100644 amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/buzz/BuzzJoinReconnect.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/AppModules.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/AppModules.kt index 0c59012d38..0c3936c413 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/AppModules.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/AppModules.kt @@ -723,6 +723,7 @@ class AppModules( torManager.status, client, applicationIOScope, + onTrigger = { cause -> resourceUsage.add(UsageKeys.relayTrigger(cause), 1) }, ) // Verifies and inserts in the cache from all relays, all subscriptions diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/RelayProxyClientConnector.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/RelayProxyClientConnector.kt index 995870ce21..b7283b37d2 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/RelayProxyClientConnector.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/RelayProxyClientConnector.kt @@ -23,6 +23,7 @@ package com.vitorpamplona.amethyst.service.relayClient import com.vitorpamplona.amethyst.commons.tor.TorRelaySettings import com.vitorpamplona.amethyst.model.torState.TorRelayEvaluation import com.vitorpamplona.amethyst.service.connectivity.ConnectivityStatus +import com.vitorpamplona.amethyst.service.resourceusage.UsageKeys import com.vitorpamplona.amethyst.ui.tor.TorServiceStatus import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl @@ -49,6 +50,14 @@ class RelayProxyClientConnector( val torStatus: StateFlow, val client: INostrClient, val scope: CoroutineScope, + /** + * Called with the cause every time this connector *decides* to reconnect, so the + * usage ledger can attribute relay churn without this class knowing where the + * counters go. Causes are the `UsageKeys.TRIGGER_*` constants — see + * [UsageKeys.relayTrigger] for why these are an upper bound rather than a count + * of reconnects actually performed. + */ + val onTrigger: (String) -> Unit = {}, ) { data class RelayServiceInfra( val evaluator: TorRelayEvaluation, @@ -138,6 +147,9 @@ class RelayProxyClientConnector( infra.connectivity is ConnectivityStatus.Off -> { Log.d("ManageRelayServices") { "Connectivity Off: Pausing Relay Services ${infra.connectivity}" } if (client.isActive()) { + // Counted inside the guard: the upstream combine() re-emits Off + // repeatedly and only this branch does any work. + onTrigger(UsageKeys.TRIGGER_OFF) client.disconnect() } if (infra.torStatus is TorServiceStatus.Active) { @@ -156,6 +168,7 @@ class RelayProxyClientConnector( } // only calls this if the client is not active. Otherwise goes to the else below + onTrigger(UsageKeys.TRIGGER_COLD_START) client.connect() lastNetworkId = networkId lastTorSettings = torSettings @@ -211,10 +224,26 @@ class RelayProxyClientConnector( Log.d("ManageRelayServices") { "Network identity changed ($previousNetworkId -> $networkId), rebuilding every relay connection" } + // The expensive branch, and the one the churn investigation is + // aimed at: a full teardown re-dials the whole pool and replays + // every REQ. + // + // Only this cause is counted here, so a wifi<->cellular handoff — + // which mints a new network handle AND rebuilds the OkHttp clients + // off the metered bit — is booked as netid alone. relay.trigger.transport + // therefore undercounts exactly the case one would most want it for; + // read it as "transport changed WITHOUT the handle changing". + onTrigger(UsageKeys.TRIGGER_NETID) // Full teardown: disconnect() drops the dead sockets AND clears each // relay's backoff, so the new network starts from a clean slate. client.reconnect(onlyIfChanged = false, ignoreRetryDelays = true) } else { + // Non-exclusive: count each independently so the report shows + // which combination fired. + if (transportChanged) onTrigger(UsageKeys.TRIGGER_TRANSPORT) + if (torPolicyChanged) onTrigger(UsageKeys.TRIGGER_TOR_POLICY) + if (classificationChanged) onTrigger(UsageKeys.TRIGGER_CLASSIFICATION) + val freshStart = transportChanged || torPolicyChanged if (freshStart) { // The failures behind the current backoffs were measured against a diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/RelayUsageListener.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/RelayUsageListener.kt index 5d97ef56cb..79e4e9e9c6 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/RelayUsageListener.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/RelayUsageListener.kt @@ -20,10 +20,22 @@ */ package com.vitorpamplona.amethyst.service.resourceusage +import android.os.SystemClock +import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.ExplainedFilter import com.vitorpamplona.quartz.nip01Core.relay.client.listeners.RelayConnectionListener import com.vitorpamplona.quartz.nip01Core.relay.client.single.IRelayClient +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.ClosedMessage +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.CountMessage +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.EoseMessage +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.EventMessage import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.Message +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.NoticeMessage +import com.vitorpamplona.quartz.nip01Core.relay.commands.toRelay.CloseCmd import com.vitorpamplona.quartz.nip01Core.relay.commands.toRelay.Command +import com.vitorpamplona.quartz.nip01Core.relay.commands.toRelay.ReqCmd +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.utils.Log +import java.util.concurrent.ConcurrentHashMap /** * Counts relay websocket traffic into the usage ledger. Frame sizes are @@ -31,12 +43,89 @@ import com.vitorpamplona.quartz.nip01Core.relay.commands.toRelay.Command * (relay JSON is ASCII-dominant), consistent with how RelayStats counts. * Excludes WS framing/compression; good enough for "which subsystem is * eating my data plan" comparisons. + * + * Beyond the byte totals this also carries the relay-churn diagnostics: a + * per-verb split of those same bytes, a connection-lifetime histogram, and + * per-relay failure/short-session counts. See + * plans/2026-07-29-relay-churn-diagnostics.md for what each answers and how to + * read them together. + * + * The verb split takes its name straight from the wire label the command already + * knows (`Command.label()` / `Message.label()`), so `Σ verb == Σ msg` holds by + * construction and a new subtype needs no change here. */ class RelayUsageListener( private val accountant: ResourceUsageAccountant, private val isMobile: () -> Boolean, private val isForeground: () -> Boolean, + private val nowMs: () -> Long = { SystemClock.elapsedRealtime() }, ) : RelayConnectionListener { + /** + * Relay -> when its current session became ready. Touched from the per-relay + * OkHttp dispatcher threads, hence concurrent. Keyed by [NormalizedRelayUrl] to + * match the other per-relay caches (`RelayStats`, `RelayLimitsTracker`). + * + * Entries are consumed on disconnect. Three ways a session escapes the map + * unrecorded, all counted rather than prevented — see [UsageKeys.RELAY_LIFE_OVERWRITE], + * [UsageKeys.RELAY_LIFE_ORPHAN], and [UsageKeys.relayConnects] for process death. + */ + private val connectedSince = ConcurrentHashMap() + + /** + * Relay -> subscription ids currently open on this connection, so a REQ that + * replaces an in-flight subscription can be told apart from one that opens a new + * one. Cleared on disconnect, because the relay forgets them too — every REQ + * after a reconnect is legitimately new. + * + * Bounded by the live subscription count per relay (tens), not by session length. + */ + private val openSubs = ConcurrentHashMap>() + + /** + * Subscription id -> the purpose that opened it, for attributing inbound frames: + * an EVENT names only its subscription, never why the client asked for it. + * + * Deliberately **not** [openSubs]. Sharing one map conflated two different + * lifetimes and lost 41 % of the download to `unattributed` in the 2026-08-02 + * reading: a CLOSE removed the id, and a disconnect dropped the whole relay's + * map, while frames already in flight were still arriving. "Is this subscription + * open" and "what did this subscription belong to" answer different questions and + * expire at different times — the second stays true after the first turns false. + * + * Keyed by subscription id alone, without the relay. The same id is used across + * relays for one logical subscription, so the purpose is a property of the id; + * this also means a frame arriving after a reconnect still attributes. + * + * Bounded by [MAX_TRACKED_SUBS] with wholesale eviction rather than an LRU: this + * is a diagnostic on a hot path, ids are recycled steadily, and a rare reset that + * sends a few frames to `unattributed` is cheaper than per-frame bookkeeping. + * `unattributed` staying small is what says the bound is generous enough. + */ + private val subPurpose = ConcurrentHashMap() + + /** + * Event ids delivered recently, as the first 64 bits of the id. + * + * Held as a Long rather than the 64-char hex: at the window size below that is + * the difference between ~200 KB and several MB on a 512 MB-class device, for a + * counter that only has to spot repetition. 64 bits makes a collision between + * distinct ids negligible where a 32-bit hash would not be. + * + * The window only needs to span the fan-out, not the session: the same event + * arrives from every relay carrying it within seconds, so near-term memory + * catches the duplication this measures. Cleared wholesale at [MAX_TRACKED_EVENTS] + * for the same reason [subPurpose] is — the alternative is per-frame LRU + * bookkeeping on the hottest path in the app. A clear undercounts duplicates that + * straddle it, so the ratio is a floor. + */ + private val recentEventIds = ConcurrentHashMap.newKeySet() + + /** Relay -> when this dial was decided, so the pre-request cost can be separated from the handshake. */ + private val dialStartedAt = ConcurrentHashMap() + + /** Notice texts already logged, so one wording costs one line however often it arrives. */ + private val loggedNotices = ConcurrentHashMap.newKeySet() + override fun onSent( relay: IRelayClient, cmdStr: String, @@ -44,7 +133,44 @@ class RelayUsageListener( success: Boolean, ) { if (success) { - accountant.add(UsageKeys.relayMsg(isMobile(), isForeground(), received = false), cmdStr.length.toLong()) + val bytes = cmdStr.length.toLong() + val mobile = isMobile() + val fg = isForeground() + accountant.add(UsageKeys.relayMsg(mobile, fg, received = false), bytes) + accountant.add(UsageKeys.relayVerb(cmd.label(), received = false, mobile, fg), bytes) + + when (cmd.label()) { + ReqCmd.LABEL -> { + accountant.add(UsageKeys.relaySubsSent(mobile, fg), 1) + + val purpose = purposeOf(cmd) + accountant.add(UsageKeys.relayPurposeSent(purpose), 1) + accountant.add(UsageKeys.relayPurposeBytes(purpose), bytes) + + // Already open on this connection, so this REQ replaces a live + // subscription rather than starting one. + val subId = (cmd as ReqCmd).subId + if (subPurpose.size >= MAX_TRACKED_SUBS) subPurpose.clear() + subPurpose[subId] = purpose + + val known = openSubs.getOrPut(relay.url) { ConcurrentHashMap.newKeySet() } + if (!known.add(subId)) { + accountant.add(UsageKeys.relaySubsResent(mobile, fg), 1) + } + // Within the window after this relay's connect, so almost certainly + // part of syncState's replay rather than a user action. A time + // window because nothing on this side marks a frame as belonging to + // it; see UsageKeys.relaySubsReplay. + val since = connectedSince[relay.url] + if (since != null && nowMs() - since <= UsageKeys.REPLAY_WINDOW_MS) { + accountant.add(UsageKeys.relaySubsReplay(mobile, fg), 1) + } + } + CloseCmd.LABEL -> { + accountant.add(UsageKeys.relaySubsClosed(mobile, fg), 1) + openSubs[relay.url]?.remove((cmd as CloseCmd).subId) + } + } } } @@ -53,7 +179,63 @@ class RelayUsageListener( msgStr: String, msg: Message, ) { - accountant.add(UsageKeys.relayMsg(isMobile(), isForeground(), received = true), msgStr.length.toLong()) + val bytes = msgStr.length.toLong() + val mobile = isMobile() + val fg = isForeground() + accountant.add(UsageKeys.relayMsg(mobile, fg, received = true), bytes) + accountant.add(UsageKeys.relayVerb(msg.label(), received = true, mobile, fg), bytes) + + // Attribute the inbound side to whoever asked for it. Only frames that name a + // subscription can be attributed; NOTICE and OK are relay-wide and are left out + // rather than guessed at, which is why this does not reconcile to msg.rx. + // Resolved once: the duplicate check below needs the same answer, and an + // EVENT names only its subscription, never why the client asked for it. + val purpose = subIdOf(msg)?.let { subPurpose[it] ?: UsageKeys.PURPOSE_UNATTRIBUTED } + if (purpose != null) { + accountant.add(UsageKeys.relayPurposeDown(purpose), bytes) + accountant.add(UsageKeys.relayPurposeDownCount(purpose), 1) + } + + if (msg is EventMessage) { + accountant.add(UsageKeys.relayEventsSeen(mobile, fg), 1) + idPrefix(msg.event.id)?.let { key -> + if (recentEventIds.size >= MAX_TRACKED_EVENTS) recentEventIds.clear() + if (!recentEventIds.add(key)) { + accountant.add(UsageKeys.relayEventsDup(mobile, fg), 1) + accountant.add(UsageKeys.relayEventsDupBytes(mobile, fg), bytes) + if (purpose != null) accountant.add(UsageKeys.relayPurposeDupBytes(purpose), bytes) + } + } + } + + // A refused subscription arrives here and nowhere else: the NOTICE carries no + // subscription id, so RelayReqRefusals (wired to CLOSED) never sees it. + if (msg is NoticeMessage) { + val reason = UsageKeys.noticeReason(msg.message) + accountant.add(UsageKeys.relayNotice(reason), 1) + if (reason == UsageKeys.NOTICE_UNCLASSIFIED) { + // The counter alone cannot say whether an absent `toomanysubs` means no + // refusals or an allowlist that misses how this relay words them. + // + // INFO, not DEBUG: a debug build defaults to LogLevel.INFO + // (Amethyst.DEFAULT_LOG_LEVEL, with VERBOSE_LOGS off), so a DEBUG line + // here is dropped before it reaches the sink and this said nothing at + // all. Demote it once the allowlist stops needing evidence. + // + // One line per distinct wording rather than per frame: the ledger + // already has the count, what is missing is the variety. Truncated and + // capped because the text is server-controlled. + if (loggedNotices.size < MAX_DISTINCT_NOTICES && loggedNotices.add(msg.message)) { + Log.i(TAG) { "Unclassified NOTICE from ${relay.url.url}: ${msg.message.take(MAX_NOTICE_LOG)}" } + } + } + } + } + + /** Dial attempts. Unlike [onCannotConnect] this really is one per dial. */ + override fun onConnecting(relay: IRelayClient) { + accountant.add(UsageKeys.relayDials(isMobile(), isForeground()), 1) + dialStartedAt[relay.url] = nowMs() } // Every completed (re)connection paid a TCP+TLS handshake; high daily @@ -64,13 +246,90 @@ class RelayUsageListener( pingMillis: Int, compressed: Boolean, ) { - accountant.add(UsageKeys.relayConnects(isMobile(), isForeground()), 1) + val mobile = isMobile() + val fg = isForeground() + // Doubles as the lifetime histogram's denominator — one session begins here. + accountant.add(UsageKeys.relayConnects(mobile, fg), 1) + // pingMillis is the transport's own handshake timing; <= 0 means it could + // not measure it, and a fabricated 0 would be worse than no record. + if (pingMillis > 0) { + accountant.add(UsageKeys.relayHandshake(pingMillis.toLong(), mobile, fg), 1) + dialStartedAt.remove(relay.url)?.let { startedAt -> + val gap = nowMs() - startedAt - pingMillis + if (gap >= 0) accountant.add(UsageKeys.relayDialGap(gap, mobile, fg), 1) + } + } + + if (connectedSince.put(relay.url, nowMs()) != null) { + accountant.add(UsageKeys.RELAY_LIFE_OVERWRITE, 1) + } } + override fun onDisconnected(relay: IRelayClient) { + val mobile = isMobile() + val fg = isForeground() + accountant.add(UsageKeys.relayDisconnects(mobile, fg), 1) + + openSubs.remove(relay.url) + val startedAt = connectedSince.remove(relay.url) + if (startedAt == null) { + // A dial that never became ready, or a second disconnect for one session. + accountant.add(UsageKeys.RELAY_LIFE_ORPHAN, 1) + return + } + + val elapsed = (nowMs() - startedAt).coerceAtLeast(0) + accountant.add(UsageKeys.relayLife(elapsed, mobile, fg), 1) + } + + /** + * The [SubPurpose] behind a REQ, from the first filter that declares one. + * + * One subscription id carries one purpose in practice, so the first is the + * subscription's. A REQ whose filters are plain [com.vitorpamplona.quartz.nip01Core.relay.filters.Filter]s + * predates #3832's tagging and is counted separately rather than guessed at. + */ + private fun purposeOf(cmd: Command): String { + val filters = (cmd as? ReqCmd)?.filters ?: return UsageKeys.PURPOSE_UNEXPLAINED + val explained = + filters.firstOrNull { it is ExplainedFilter } as? ExplainedFilter + ?: return UsageKeys.PURPOSE_UNEXPLAINED + return UsageKeys.purposeKeyPart(explained.purpose) + } + + /** The first 64 bits of an event id, or null if it is not a well-formed id. */ + private fun idPrefix(id: String): Long? = if (id.length < 16) null else runCatching { id.substring(0, 16).toULong(16).toLong() }.getOrNull() + + /** The subscription a frame belongs to, when it names one. */ + private fun subIdOf(msg: Message): String? = + when (msg) { + is EventMessage -> msg.subId + is EoseMessage -> msg.subId + is ClosedMessage -> msg.subId + is CountMessage -> msg.queryId + else -> null + } + override fun onCannotConnect( relay: IRelayClient, errorMessage: String, ) { accountant.add(UsageKeys.relayConnectFails(isMobile(), isForeground()), 1) } + + companion object { + private const val TAG = "RelayUsage" + + /** NOTICE text is server-controlled; cap what reaches the log. */ + private const val MAX_NOTICE_LOG = 200 + + /** Ceiling on distinct wordings held in memory; relay prose is unbounded. */ + private const val MAX_DISTINCT_NOTICES = 200 + + /** Ceiling on remembered subscription-id purposes. See [subPurpose]. */ + private const val MAX_TRACKED_SUBS = 4_000 + + /** Recent-event-id window. See [recentEventIds]. */ + private const val MAX_TRACKED_EVENTS = 50_000 + } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageAccountant.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageAccountant.kt index eaa90e8d10..e7807eb02b 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageAccountant.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageAccountant.kt @@ -37,8 +37,15 @@ import java.util.concurrent.atomic.AtomicLong * AtomicLong (not LongAdder) because draining must be loss-free: getAndSet(0) * hands off the accumulated value atomically, whereas remove+sum on a * LongAdder can strand a racing increment on an orphaned cell. Entries stay - * in the map after a drain — the key space is small and fixed (dims x areas), - * so this costs a few hundred boxed zeros at most. + * in the map after a drain — the key space is small and *mostly* fixed + * (dims x areas), so this costs a few hundred boxed zeros at most. + * + * The exception is the per-relay churn counters (`relay.host..*`), whose + * cardinality follows the user's relay list rather than a compile-time set: + * two keys per relay, so a few hundred more entries for a large list. Still + * negligible in memory, but it does mean neither this map nor the persisted + * store has a fixed upper bound any more. Keep that in mind before adding + * another counter keyed on runtime data. * * Counters added from inside a pre-flush hook (the CPU sampler, the segment * integrators closing an open segment) never re-arm the debounce: they are @@ -74,7 +81,10 @@ class ResourceUsageAccountant( amount: Long, ) { if (amount <= 0) return - live.computeIfAbsent(key) { AtomicLong() }.addAndGet(amount) + // Plain get first: computeIfAbsent locks the bin head when the key is present + // but not the head node, and the churn counters roughly tripled the key count + // (so collisions) on a path that runs per relay frame. + (live[key] ?: live.computeIfAbsent(key) { AtomicLong() }).addAndGet(amount) if (inHookRun.get() == true) return if (flushScheduled.compareAndSet(false, true)) { scope.launch { diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageReportAssembler.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageReportAssembler.kt index 73c4f510f3..9b289e5ee7 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageReportAssembler.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageReportAssembler.kt @@ -29,8 +29,9 @@ import java.util.Locale * Assembles the Markdown resource-usage report the user can DM to the * developers via NIP-17 — same shape as the crash ReportAssembler: a device * header table, a human-readable summary, then the full per-day counter dump - * as the technical payload. Counters are sizes/durations/counts only; no - * URLs, relay names, or content. + * as the technical payload. Counters are sizes/durations/counts only, and never + * content. + * */ class ResourceUsageReportAssembler { fun buildReport( @@ -132,6 +133,8 @@ class ResourceUsageReportAssembler { /** Markdown table header/body separator row. */ private const val TABLE_SEPARATOR = "| --- | --- |\n" + /** Caps how many relay hosts a shared report can name. See the class doc. */ + fun formatBytes(bytes: Long): String = when { bytes >= 1024L * 1024L * 1024L -> String.format(Locale.US, "%.2f GB", bytes / (1024.0 * 1024.0 * 1024.0)) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageStore.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageStore.kt index 7e7824631b..c25bd7e3f5 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageStore.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageStore.kt @@ -34,7 +34,11 @@ import java.io.File * Jackson + Mutex + write-to-tmp-then-rename + version envelope. * * Day keys are UTC epoch-days (stringified for JSON). Buckets older than - * [keepDays] are pruned on every merge, so the file stays small (a few KB). + * [keepDays] are pruned on every merge, so the file stays small — a few KB, plus + * two keys per relay per day now that the churn counters are keyed on runtime + * data (see [ResourceUsageAccountant], which owns that caveat). The whole file is + * re-serialized on every flush (debounced to ~30s while traffic flows), so that + * growth is paid on each write, not just at rest. * Also carries the high-consumption alert state (last prompt time, opt-out) * so the whole feature has exactly one file. */ diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/UsageKeys.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/UsageKeys.kt index e43cde73a9..75271e3b7f 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/UsageKeys.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/UsageKeys.kt @@ -20,6 +20,11 @@ */ package com.vitorpamplona.amethyst.service.resourceusage +import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.SubPurpose +import com.vitorpamplona.quartz.nip01Core.relay.client.single.basic.BasicRelayClient +import com.vitorpamplona.quartz.nip66RelayMonitor.reachability.RelayObserver +import java.util.concurrent.ConcurrentHashMap + /** * Counter-key grammar for the resource-usage ledger. Keys are flat strings so * the on-disk store is schema-free — adding a counter never needs a migration. @@ -29,8 +34,28 @@ package com.vitorpamplona.amethyst.service.resourceusage * - visibility: `fg` (an activity is started) vs `bg` * - direction: `rx` (downloaded) vs `tx` (uploaded) * - * Counters are sizes, durations, and counts only — never URLs, relay names, or - * content. See plans/2026-07-12-resource-usage-ledger.md. + * Counters are sizes, durations, and counts only — never content, and never a + * full URL, and never a relay name. + * See plans/2026-07-12-resource-usage-ledger.md. + * + * ## Reserved segments — read before adding a counter + * + * [sumMatching] matches by dot-segment *membership*, not by prefix, and every + * headline figure in [UsageSummary] is built from it. A new key that happens to + * contain one of these segments silently joins that sum: + * + * rx tx msg connms connects connfails reqs bursts activems + * worker runs + every value in [HTTP_ROLES] + * + * Concretely: a key named `relay.rx.event.mobile.bg` would be counted by + * `traffic(MOBILE, BG)` *in addition to* `relay.msg.mobile.bg.rx`, doubling the + * reported data usage and halving the effective threshold of the + * background-mobile-data alert. That is why the relay verb split below uses + * `up`/`down` rather than `tx`/`rx`. + * + * `ResourceUsageLedgerTest.newKeysDoNotDisturbSummary` is the regression guard: + * it asserts [UsageSummary.from] is value-identical with and without every key + * this object can produce. */ object UsageKeys { const val MOBILE = "mobile" @@ -54,6 +79,39 @@ object UsageKeys { val HTTP_ROLES = listOf(ROLE_IMAGE, ROLE_VIDEO, ROLE_UPLOADS, ROLE_MONEY, ROLE_NIP05, ROLE_PREVIEW, ROLE_PUSH, ROLE_OTHER) + /** + * `mobile.bg` — the network x visibility pair every counter is split by. + * + * Table-backed rather than interpolated: this is evaluated on every relay frame + * and every HTTP response, and there are only four possible answers. Declared + * first because the key tables below are built from it at class-init. + */ + fun dim( + mobile: Boolean, + foreground: Boolean, + ): String = DIMS[dimIndex(mobile, foreground)] + + private val DIMS = arrayOf("$WIFI.$BG", "$WIFI.$FG", "$MOBILE.$BG", "$MOBILE.$FG") + + private fun dimIndex( + mobile: Boolean, + foreground: Boolean, + ): Int = (if (mobile) 2 else 0) or (if (foreground) 1 else 0) + + /** + * The four `.[.]` keys, indexed by [dimIndex]. + * + * Used for every `relay.*` key, because all of them are built from a relay + * callback — per frame for [relayMsg]/[relayVerb], per dial/connect/disconnect + * for the rest. The `net.*` builders below still interpolate: their key space is + * role x dim x metric and they are called once per HTTP response, so the table + * would be larger and buy less. If you add a `relay.*` counter, table it. + */ + private fun dimKeys( + prefix: String, + suffix: String? = null, + ): Array = Array(DIMS.size) { if (suffix == null) "$prefix.${DIMS[it]}" else "$prefix.${DIMS[it]}.$suffix" } + /** `net.image.mobile.bg.rx` — HTTP bytes for a subsystem. */ fun net( role: String, @@ -87,25 +145,526 @@ object UsageKeys { mobile: Boolean, foreground: Boolean, received: Boolean, - ): String = "relay.msg.${dim(mobile, foreground)}.${if (received) RX else TX}" + ): String = (if (received) RELAY_MSG_RX else RELAY_MSG_TX)[dimIndex(mobile, foreground)] + + private val RELAY_MSG_RX = dimKeys("relay.msg", RX) + private val RELAY_MSG_TX = dimKeys("relay.msg", TX) /** `relay.connms.mobile.bg` — Σ(open relay connections × elapsed ms). */ fun relayConnMs( mobile: Boolean, foreground: Boolean, - ): String = "relay.connms.${dim(mobile, foreground)}" + ): String = RELAY_CONNMS[dimIndex(mobile, foreground)] - /** `relay.connects.mobile.bg` — completed relay (re)connections: each one paid a TCP+TLS handshake. */ + private val RELAY_CONNMS = dimKeys("relay.connms") + + /** + * `relay.connects.mobile.bg` — completed relay (re)connections: each one paid a + * TCP+TLS handshake. + * + * Also the [relayLife] histogram's denominator — one session begins per + * `onConnected` — which is what makes the histogram's deficit measurable rather + * than assumed: + * + * connects − Σ life buckets − orphan = still open at report time + lost to process death + * + * Without that subtraction a leak, a still-open session and a session lost to a + * background kill are indistinguishable. + */ fun relayConnects( mobile: Boolean, foreground: Boolean, - ): String = "relay.connects.${dim(mobile, foreground)}" + ): String = RELAY_CONNECTS[dimIndex(mobile, foreground)] - /** `relay.connfails.mobile.bg` — dials that failed before the websocket opened. */ + private val RELAY_CONNECTS = dimKeys("relay.connects") + + /** + * `relay.connfails.mobile.bg` — every `onCannotConnect`. + * + * NOT a failed-dial count, despite the name. `BasicRelayClient.onFailure` + * raises `onCannotConnect` with no `isReady` test, so a connection that lived + * for ten minutes and then dropped increments both this and [relayConnects] + * from a single dial. Use [relayDials] for the actual number of dials. + */ fun relayConnectFails( mobile: Boolean, foreground: Boolean, - ): String = "relay.connfails.${dim(mobile, foreground)}" + ): String = RELAY_CONNFAILS[dimIndex(mobile, foreground)] + + private val RELAY_CONNFAILS = dimKeys("relay.connfails") + + /** + * `relay.dials.mobile.bg` — dial attempts, from `onConnecting`. + * + * Fires exactly once per dial, after the transport gate and the connect mutex + * and before the socket is built, so this is the honest denominator that + * [relayConnectFails] is not. + */ + fun relayDials( + mobile: Boolean, + foreground: Boolean, + ): String = RELAY_DIALS[dimIndex(mobile, foreground)] + + private val RELAY_DIALS = dimKeys("relay.dials") + + /** `relay.disc.mobile.bg` — every `onDisconnected`, whatever the cause. */ + fun relayDisconnects( + mobile: Boolean, + foreground: Boolean, + ): String = RELAY_DISC[dimIndex(mobile, foreground)] + + private val RELAY_DISC = dimKeys("relay.disc") + + /** + * `relay.subs.sent.mobile.bg` — REQ commands sent. + * + * A count to sit beside the `relay.verb.up.req` byte total: PR #3832 raised the + * number of live subscriptions per relay (background accounts now subscribe too) + * and measured refusals against nos.lol's cap of 20. Divided by [relayConnects] + * this is REQs per connection, which is what separates "we reconnect too often" + * from "each reconnect asks for too much" — different fixes. + */ + fun relaySubsSent( + mobile: Boolean, + foreground: Boolean, + ): String = RELAY_SUBS_SENT[dimIndex(mobile, foreground)] + + private val RELAY_SUBS_SENT = dimKeys("relay.subs.sent") + + /** `relay.subs.closed.mobile.bg` — CLOSE commands sent; sent minus closed is net subscription growth. */ + fun relaySubsClosed( + mobile: Boolean, + foreground: Boolean, + ): String = RELAY_SUBS_CLOSED[dimIndex(mobile, foreground)] + + private val RELAY_SUBS_CLOSED = dimKeys("relay.subs.closed") + + /** + * `relay.subs.replay.mobile.bg` — REQs sent within [REPLAY_WINDOW_MS] of that + * relay's connect, i.e. the post-connect resubscribe burst. + * + * An estimate, not an exact split. `PoolRequests.syncState` replays every desired + * filter from a coroutine launched at `onConnected`, but nothing on the listener + * side marks a frame as belonging to it, so this is a time window. It is the + * measurement behind the source report's inference that ~24 KB per connection + * "is exactly the size of a full REQ subscription replay" — which was arithmetic + * on a daily total, not an observation. + */ + fun relaySubsReplay( + mobile: Boolean, + foreground: Boolean, + ): String = RELAY_SUBS_REPLAY[dimIndex(mobile, foreground)] + + private val RELAY_SUBS_REPLAY = dimKeys("relay.subs.replay") + + /** How long after a connect a REQ still counts as part of the resubscribe burst. */ + const val REPLAY_WINDOW_MS = 2_000L + + /** + * `relay.notice.toomanysubs` — NOTICE frames by reason. + * + * Exists because a refused subscription is otherwise invisible. Per PR #3832's + * own known issue, `ERROR: too many concurrent REQs` arrives as a NOTICE, which + * carries no subscription id and so never reaches `RelayReqRefusals.onRefused` + * (wired to CLOSED only). The relay drops the REQ while the client still believes + * it is live — it never EOSEs, its `since` never advances, and `syncState` then + * re-requests its full backlog on every reconnect, forever. A non-trivial count + * here means subscription pressure is a *cause* of the download volume rather + * than a symptom of the reconnect count. + * + * The reason comes from a fixed allowlist, never from the relay's text. Relay + * prose is server-controlled and this key is persisted for 30 days; `RelayObserver` + * had to fix exactly this bug, where free-form CLOSED prose became its own tally + * key and cardinality grew with the number of distinct sentences relays wrote. + */ + fun relayNotice(reason: String): String = RELAY_NOTICE[reason] ?: RELAY_NOTICE.getValue(NOTICE_UNCLASSIFIED) + + const val NOTICE_TOO_MANY_SUBS = "toomanysubs" + const val NOTICE_RATE_LIMITED = "ratelimited" + const val NOTICE_AUTH_REQUIRED = "authrequired" + const val NOTICE_RESTRICTED = "restricted" + const val NOTICE_INVALID = "invalid" + const val NOTICE_BLOCKED = "blocked" + const val NOTICE_ERROR = "error" + const val NOTICE_UNSUPPORTED = "unsupported" + + /** The query itself was too expensive — too many kinds/steps/filters. Observed on nostr.land, relay.layer.systems. */ + const val NOTICE_QUERY_COST = "querycost" + + /** The relay refuses REQs outright. Observed on sendit.nosflare.com. */ + const val NOTICE_REQ_REFUSED = "reqrefused" + + /** Relay chatter that costs bytes but means nothing — keepalives, per-query PERF telemetry. */ + const val NOTICE_BENIGN = "benign" + + /** Deliberately not `other`: that is an [HTTP_ROLES] value and a reserved segment. */ + const val NOTICE_UNCLASSIFIED = "unclassified" + + val NOTICE_REASONS = + listOf( + NOTICE_TOO_MANY_SUBS, + NOTICE_RATE_LIMITED, + NOTICE_AUTH_REQUIRED, + NOTICE_RESTRICTED, + NOTICE_INVALID, + NOTICE_BLOCKED, + NOTICE_ERROR, + NOTICE_UNSUPPORTED, + NOTICE_QUERY_COST, + NOTICE_REQ_REFUSED, + NOTICE_BENIGN, + NOTICE_UNCLASSIFIED, + ) + + private val RELAY_NOTICE = NOTICE_REASONS.associateWith { "relay.notice.$it" } + + /** + * Classifies a NOTICE into one of [NOTICE_REASONS]. + * + * Matches on content markers rather than the NIP-01 machine-readable prefix + * alone, because the case this exists for does not have a useful one: strfry + * sends `ERROR: too many concurrent REQs`, whose prefix is just `error`. + * [RelayObserver.prefixOf] is consulted for the standard prefixes it does + * handle correctly. + */ + fun noticeReason(message: String): String { + val text = message.lowercase() + // Several relays prefix a NOTICE with the subscription id it concerns + // ("Kgo0HH: closed: too many steps"), which makes the *subscription id* the + // machine-readable prefix and hides the real one. Try the remainder too. + val prefix = RelayObserver.prefixOf(message) + val inner = RelayObserver.prefixOf(message.substringAfter(':', "")) + val prefixes = setOf(prefix, inner) + return when { + "too many" in text && ("req" in text || "subscription" in text || "concurrent" in text) -> NOTICE_TOO_MANY_SUBS + // A cost refusal is still a refusal: the REQ is dropped, so it never + // EOSEs and its `since` never advances. + "too many" in text || "too costly" in text || "too expensive" in text -> NOTICE_QUERY_COST + "does not accept" in text || "denied" in text || "not accepting" in text -> NOTICE_REQ_REFUSED + "keepalive" in text || "perf:" in text -> NOTICE_BENIGN + "rate-limited" in prefixes || ("rate" in text && "limit" in text) -> NOTICE_RATE_LIMITED + "auth-required" in prefixes || ("auth" in text && "required" in text) -> NOTICE_AUTH_REQUIRED + "restricted" in prefixes -> NOTICE_RESTRICTED + "invalid" in prefixes -> NOTICE_INVALID + "blocked" in prefixes -> NOTICE_BLOCKED + "unsupported" in prefixes -> NOTICE_UNSUPPORTED + "error" in prefixes -> NOTICE_ERROR + else -> NOTICE_UNCLASSIFIED + } + } + + /** + * The bar that decides reconnect behaviour, and so also what counts as a short + * session: below it a disconnect keeps the growing backoff, + * at or above it the backoff resets to 1s. (`NostrClient.KEEP_ALIVE_INTERVAL_MS` + * is the same 60s, but it is private, so this reads the one that is public.) + * + * Derived rather than copied: the plan retunes `STABLE_CONNECTION_IN_SECS` once + * the histogram is read, and a hand-written 60_000 here would silently stop + * meaning "session that kept the backoff growing" at that point. + * `UsageKeyHelpersTest.lifeBucketsAreHalfOpen` asserts the resulting bucket + * labels, so a retune surfaces as a test failure rather than as a histogram that + * quietly answers the wrong question. + */ + const val SHORT_SESSION_MS = BasicRelayClient.STABLE_CONNECTION_IN_SECS * 1_000L + + /** + * Half-open upper bounds, in ms, for the [relayLife] histogram. Deliberately + * straddles [SHORT_SESSION_MS]: a mean cannot tell a tight cluster sitting on + * that bar from a bimodal mix; this can. + */ + private val LIFE_BUCKET_BOUNDS_MS = + longArrayOf(5_000, 30_000, SHORT_SESSION_MS, 120_000, 300_000).also { + // [lifeBucketIndex] linear-scans for the first bound greater than the + // elapsed time, so the bounds must ascend. One of them is derived from + // BasicRelayClient.STABLE_CONNECTION_IN_SECS, and raising that to five + // minutes — exactly the retune commit 2 of the churn plan contemplates — + // would push it past the two bounds after it. The buckets between would + // become unreachable and the labels would start lying. Fail at class-init + // with the reason rather than as a puzzling boundary-test failure. + require(it.asList() == it.sorted()) { + "relay.life bounds must ascend, got ${it.toList()}. " + + "SHORT_SESSION_MS is ${SHORT_SESSION_MS}ms — reorder the bounds to match." + } + } + + /** Derived from the bounds so a bound change can never leave a label lying about it. */ + private val LIFE_BUCKET_NAMES = + Array(LIFE_BUCKET_BOUNDS_MS.size + 1) { i -> + if (i < LIFE_BUCKET_BOUNDS_MS.size) { + "lt${LIFE_BUCKET_BOUNDS_MS[i] / 1000}s" + } else { + "gte${LIFE_BUCKET_BOUNDS_MS.last() / 1000}s" + } + } + + private fun lifeBucketIndex(elapsedMs: Long): Int { + for (i in LIFE_BUCKET_BOUNDS_MS.indices) { + if (elapsedMs < LIFE_BUCKET_BOUNDS_MS[i]) return i + } + return LIFE_BUCKET_BOUNDS_MS.size + } + + fun lifeBucket(elapsedMs: Long): String = LIFE_BUCKET_NAMES[lifeBucketIndex(elapsedMs)] + + /** + * `relay.life.lt60s.mobile.bg` — connections that closed after living this long. + * [relayConnects] is the denominator; see its doc for the deficit equation. + */ + fun relayLife( + elapsedMs: Long, + mobile: Boolean, + foreground: Boolean, + ): String = RELAY_LIFE[lifeBucketIndex(elapsedMs)][dimIndex(mobile, foreground)] + + private val RELAY_LIFE = Array(LIFE_BUCKET_NAMES.size) { dimKeys("relay.life.${LIFE_BUCKET_NAMES[it]}") } + + /** + * `relay.life.overwrite` — a connect arrived for a relay that already had an + * unconsumed start stamp. + * + * Expected during a pool teardown: `NostrClient` runs `disconnect()` then + * `connect()` synchronously, so a stale failure callback for the old socket + * can land after the new socket is already open. The new stamp overwrites the + * old, and the stale disconnect then consumes the new one — booking a + * near-zero lifetime for a session that never ended. Bias runs toward `lt5s`, + * so read this before reading the histogram's short buckets. + */ + const val RELAY_LIFE_OVERWRITE = "relay.life.overwrite" + + /** `relay.life.orphan` — a disconnect with no matching start stamp. */ + const val RELAY_LIFE_ORPHAN = "relay.life.orphan" + + /** + * `relay.verb.up.req.mobile.bg` / `relay.verb.down.eose.mobile.bg` — the + * [relayMsg] bytes, split by wire verb. Parameterised on direction for the same + * reason [relayMsg] is: one memo strategy, not two. + * + * `verb` is the command's own `label()` (`REQ`, `EVENT`, ...) lowercased, so a + * new `Command`/`Message` subtype maps itself and nothing can land in a + * catch-all bucket unnoticed. Deliberately `up`/`down` rather than `tx`/`rx` — + * see the reserved-segment note above. + * + * Keys are memoized because this is on the per-frame path: the verb x dim space + * is a handful of entries, so steady state is a map lookup and an array index + * with no string building at all. + */ + fun relayVerb( + verb: String, + received: Boolean, + mobile: Boolean, + foreground: Boolean, + ): String = + (if (received) VERB_DOWN_KEYS else VERB_UP_KEYS) + .getOrPut(verb) { dimKeys("relay.verb.${if (received) DOWN else UP}.${verb.lowercase()}") }[dimIndex(mobile, foreground)] + + private const val UP = "up" + private const val DOWN = "down" + + private val VERB_UP_KEYS = ConcurrentHashMap>() + private val VERB_DOWN_KEYS = ConcurrentHashMap>() + + /** + * `relay.purpose.home_feed.sent` / `.bytes` — REQ frames and REQ bytes by the + * [SubPurpose] that asked for them. + * + * The counter that turns "REQ traffic is 64 % of upload" into an actionable + * name. Purpose travels on the filter itself (PR #3832's `ExplainedFilter`, + * which survives the `copy(since = …)` assemblers do after every EOSE), so this + * is a read, not a new registry. + * + * Cardinality is the enum, so it is bounded and stable. [PURPOSE_UNEXPLAINED] is + * its own bucket rather than folded into the enum's OTHER: a filter carrying no + * purpose at all means an assembler #3832 did not reach, which is a different + * fact from one that declared itself uncategorised — and if that bucket is large, + * the attribution below cannot be trusted. + */ + fun relayPurposeSent(purpose: String): String = "relay.purpose.$purpose.sent" + + fun relayPurposeBytes(purpose: String): String = "relay.purpose.$purpose.bytes" + + /** + * `relay.purpose.moderation.down` — bytes received on subscriptions opened for + * that purpose, resolved through the subscription id the frame carries. + * + * The upload counters answer "who is asking"; this answers "who is being + * answered", which is the larger number: inbound EVENT payload is ~74 % of relay + * traffic against ~26 % outbound. Without it, a fix to the REQ churn can only be + * credited with the upload it removes, when the interesting question is how much + * of the download it was causing — every re-subscription can make the relay + * re-send everything that matches. + */ + fun relayPurposeDown(purpose: String): String = "relay.purpose.$purpose.down" + + /** + * `relay.purpose.home_feed.downn` — inbound frames, alongside the bytes. + * + * Bytes alone cannot separate "many small events delivered repeatedly" from "few + * large ones", and those want opposite fixes. With a count, `down / downn` is the + * average frame size per purpose, and the total frame count set against + * `crypto.verify.count` — which the cache pays once per event it accepts — bounds + * how much of the download is the same events arriving from different relays + * under the outbox fan-out. + */ + fun relayPurposeDownCount(purpose: String): String = "relay.purpose.$purpose.downn" + + /** + * `relay.purpose.user_profile.dupbytes` — of that purpose's inbound bytes, how + * many carried an event already delivered. + * + * [relayEventsDupBytes] measures duplication across the whole client, which says + * how much is wasted but not where. The seventh reading needs exactly this split: + * `user_profile` was 57 % of download at ~17 KB per event, and whether that is + * mostly the same events arriving from many relays or mostly distinct large ones + * points at completely different fixes — suppress redundant delivery, or stop + * fetching the large thing per relay. + */ + fun relayPurposeDupBytes(purpose: String): String = "relay.purpose.$purpose.dupbytes" + + /** A frame whose subscription id we never saw opened — counters wiped mid-session, or a sub from before this connection. */ + const val PURPOSE_UNATTRIBUTED = "unattributed" + + /** A REQ whose filters carry no [ExplainedFilter] purpose. */ + const val PURPOSE_UNEXPLAINED = "unexplained" + + /** The enum's own OTHER, renamed: bare `other` is an [HTTP_ROLES] value and a reserved segment. */ + const val PURPOSE_OTHER = "otherpurpose" + + /** + * The key segment for a [SubPurpose]. The one place the enum is turned into a + * key, so the reserved-segment rename cannot drift between the producer and the + * test that guards it — which is exactly how it drifted the first time. + */ + fun purposeKeyPart(purpose: SubPurpose): String = if (purpose == SubPurpose.OTHER) PURPOSE_OTHER else purpose.name.lowercase() + + /** + * `relay.subs.resent.mobile.bg` — a REQ for a subscription id this relay already + * has open on the current connection. + * + * The distinction the churn question turns on. A REQ that opens a new + * subscription is work; a REQ that replaces one already in flight is the client + * changing its mind, and at ~1 KB each that is pure cost. Measured against + * [relaySubsSent] it says what fraction of the upload is re-subscription rather + * than subscription. + */ + fun relaySubsResent( + mobile: Boolean, + foreground: Boolean, + ): String = RELAY_SUBS_RESENT[dimIndex(mobile, foreground)] + + private val RELAY_SUBS_RESENT = dimKeys("relay.subs.resent") + + /** + * Half-open bounds, in ms, for the two connect-timing histograms below. + */ + private val CONNECT_BUCKET_BOUNDS_MS = longArrayOf(100, 500, 2_000, 10_000, 30_000) + private val CONNECT_BUCKET_NAMES = + Array(CONNECT_BUCKET_BOUNDS_MS.size + 1) { i -> + if (i < CONNECT_BUCKET_BOUNDS_MS.size) "lt${CONNECT_BUCKET_BOUNDS_MS[i]}ms" else "gte${CONNECT_BUCKET_BOUNDS_MS.last()}ms" + } + + private fun connectBucketIndex(ms: Long): Int { + for (i in CONNECT_BUCKET_BOUNDS_MS.indices) { + if (ms < CONNECT_BUCKET_BOUNDS_MS[i]) return i + } + return CONNECT_BUCKET_BOUNDS_MS.size + } + + fun connectBucket(ms: Long): String = CONNECT_BUCKET_NAMES[connectBucketIndex(ms)] + + /** + * `relay.hs.lt500ms.wifi.fg` — the websocket upgrade round-trip, as the + * transport measured it. + * + * This is `onConnected`'s `pingMillis`, which `BasicOkHttpWebSocket` computes as + * `receivedResponseAtMillis - sentRequestAtMillis` and which this listener + * previously discarded. PR #3843 is the cautionary tale: `RelayObserver` derived + * the same quantity from `onConnecting -> onConnected` instead, and on a large + * fan-out published a 33.5 s median that was the client's own backlog rather than + * relay latency. Those timestamps bracket the request itself, so everything + * before it — queueing, DNS, TCP, TLS — is excluded. Zero or negative means the + * transport could not time it, and nothing is recorded rather than a fabricated 0. + */ + fun relayHandshake( + ms: Long, + mobile: Boolean, + foreground: Boolean, + ): String = RELAY_HS[connectBucketIndex(ms)][dimIndex(mobile, foreground)] + + private val RELAY_HS = Array(CONNECT_BUCKET_NAMES.size) { dimKeys("relay.hs.${CONNECT_BUCKET_NAMES[it]}") } + + /** + * `relay.gap.lt2000ms.wifi.fg` — everything between deciding to dial and the + * upgrade request going out: dispatcher queueing, DNS, TCP, TLS. + * + * `(onConnected wall clock - onConnecting wall clock) - handshake`. This is the + * share of connect latency the app is responsible for rather than the relay, and + * it is what decides whether a high never-became-ready rate is relays being + * unreachable or ~500 simultaneous dials saturating name resolution and sockets. + * The dispatcher's own cap is not the constraint on a phone + * (`maxRequests = 1024`, `maxRequestsPerHost = 10`), so a large value here points + * at resolution and socket setup, not at a queue. + */ + fun relayDialGap( + ms: Long, + mobile: Boolean, + foreground: Boolean, + ): String = RELAY_GAP[connectBucketIndex(ms)][dimIndex(mobile, foreground)] + + private val RELAY_GAP = Array(CONNECT_BUCKET_NAMES.size) { dimKeys("relay.gap.${CONNECT_BUCKET_NAMES[it]}") } + + /** + * `relay.events.seen.wifi.fg` — inbound EVENT frames, and of those, how many + * carried an event id already delivered recently. + * + * The outbox model asks many relays for the same authors, so one event is + * delivered once per relay that carries it. Relay download is ~65 % of all data + * on the release build, so the duplication factor decides whether the largest + * number in the ledger is content or repetition — a question no other counter + * here can answer, and one [VERIFY_COUNT] only proxies (it counts what the cache + * accepted, not what arrived, and only while dedup-before-verify holds). + * + * [relayEventsDupBytes] is the number that matters: bytes that arrived and were + * already held. + */ + fun relayEventsSeen( + mobile: Boolean, + foreground: Boolean, + ): String = RELAY_EV_SEEN[dimIndex(mobile, foreground)] + + fun relayEventsDup( + mobile: Boolean, + foreground: Boolean, + ): String = RELAY_EV_DUP[dimIndex(mobile, foreground)] + + fun relayEventsDupBytes( + mobile: Boolean, + foreground: Boolean, + ): String = RELAY_EV_DUPB[dimIndex(mobile, foreground)] + + private val RELAY_EV_SEEN = dimKeys("relay.events.seen") + private val RELAY_EV_DUP = dimKeys("relay.events.dup") + private val RELAY_EV_DUPB = dimKeys("relay.events.dupbytes") + + /** + * `relay.trigger.netid` — reconnect *decisions*, by cause, not reconnects + * performed. + * + * Two reasons this is an upper bound, both of which matter when reading it: + * `NostrClient.reconnect` emits into a debounced flow that `subscribe` / + * `count` / `publish` / `onDisconnected` also feed very frequently, so a + * teardown can be coalesced away before it runs; and the flow's initial value + * fires one teardown per client construction with no trigger attributed. + */ + fun relayTrigger(cause: String): String = "relay.trigger.$cause" + + const val TRIGGER_NETID = "netid" + const val TRIGGER_TRANSPORT = "transport" + const val TRIGGER_TOR_POLICY = "torpolicy" + const val TRIGGER_CLASSIFICATION = "class" + const val TRIGGER_COLD_START = "coldstart" + const val TRIGGER_OFF = "off" + const val TRIGGER_BUZZ = "buzz" /** `worker.scheduledPost.runs` */ fun workerRuns(worker: String): String = "worker.$worker.runs" @@ -187,18 +746,35 @@ object UsageKeys { const val BATTERY_DRAIN_FG = "battery.drain.fg" const val BATTERY_DRAIN_BG = "battery.drain.bg" - fun dim( - mobile: Boolean, - foreground: Boolean, - ): String = "${if (mobile) MOBILE else WIFI}.${if (foreground) FG else BG}" - - /** Sums every counter whose key matches all the given dot-delimited parts. */ + /** + * Sums every counter whose key matches all the given dot-delimited parts. + * + * Scans segments in place rather than `key.split('.')`: [UsageSummary.from] makes + * ~54 of these passes over the whole day bucket, the usage screen builds 16 + * summaries per entry on the main thread, and the churn counters roughly tripled + * the key count — none of which can ever match (that is what + * `noNewKeyContainsAReservedSegment` guarantees), so every split was pure waste. + */ fun Map.sumMatching(vararg parts: String): Long { var total = 0L - for ((key, value) in this) { - val segments = key.split('.') - if (parts.all { it in segments }) total += value + outer@ for ((key, value) in this) { + for (part in parts) { + if (!key.hasSegment(part)) continue@outer + } + total += value } return total } + + /** True when `segment` is one of this key's whole dot-delimited segments. */ + private fun String.hasSegment(segment: String): Boolean { + var from = 0 + while (from <= length) { + var end = indexOf('.', from) + if (end < 0) end = length + if (end - from == segment.length && regionMatches(from, segment, 0, segment.length)) return true + from = end + 1 + } + return false + } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/buzz/AgentConsoleViewModel.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/buzz/AgentConsoleViewModel.kt index 98d19c7937..2433459063 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/buzz/AgentConsoleViewModel.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/buzz/AgentConsoleViewModel.kt @@ -109,9 +109,7 @@ class AgentConsoleViewModel : ViewModel() { relay?.let { val newlyJoined = BuzzWorkspaces.join(it) viewModelScope.launch { account.relayAuthLedger.setDecision(it.url, RelayAuthDecision.ALLOW) } - // A join makes the relay first-party; if the socket was already open its one-shot AUTH - // challenge was spent unauthenticated, so reconnect to re-challenge and authenticate. - if (newlyJoined) account.client.reconnect(onlyIfChanged = false, ignoreRetryDelays = true) + if (newlyJoined) reconnectPoolAfterJoin(account.client) } refresh() } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/buzz/BuzzDmListViewModel.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/buzz/BuzzDmListViewModel.kt index a619471e3f..dccf0cb851 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/buzz/BuzzDmListViewModel.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/buzz/BuzzDmListViewModel.kt @@ -130,9 +130,7 @@ class BuzzDmListViewModel : ViewModel() { val newlyJoined = BuzzWorkspaces.join(relay) viewModelScope.launch { account.relayAuthLedger.setDecision(relay.url, RelayAuthDecision.ALLOW) } - // A join makes the relay first-party; if the socket was already open its one-shot AUTH - // challenge was spent unauthenticated, so reconnect to re-challenge and authenticate. - if (newlyJoined) account.client.reconnect(onlyIfChanged = false, ignoreRetryDelays = true) + if (newlyJoined) reconnectPoolAfterJoin(account.client) // Paint from cache BEFORE any network work. [discoverMemberChannels] learns the channel ids // from a relay round-trip, so waiting on it left the Direct Messages section visibly empty diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/buzz/BuzzJoinReconnect.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/buzz/BuzzJoinReconnect.kt new file mode 100644 index 0000000000..9d419d535b --- /dev/null +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/buzz/BuzzJoinReconnect.kt @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.ui.screen.loggedIn.buzz + +import com.vitorpamplona.amethyst.Amethyst +import com.vitorpamplona.amethyst.service.resourceusage.UsageKeys +import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient + +/** + * Re-dials the whole relay pool after a Buzz workspace join. + * + * NIP-42 sends its AUTH challenge once, on connect. If the socket was already open + * before the join (the common case — the relay is in the user's lists and connected + * at startup), that challenge was spent while the relay was still NOT first-party, + * so the connection is unauthenticated and every `#p=me`-gated read on it is + * refused. Joining makes the relay first-party (see `AuthCoordinator.isFirstParty`); + * this forces the relay to re-challenge so the connection authenticates. + * + * Shared by the three join sites that need the re-challenge, both to keep the + * reconnect flags identical and to give the churn ledger one place to attribute from: + * without the counter, `Σ(relay.trigger.*)` would only account for the + * connectivity-driven teardowns `RelayProxyClientConnector` reports, and a full pool + * teardown is the most expensive thing either can do. + * + * `BuzzInviteScreen` is a fourth join+pre-approve site that deliberately does NOT + * reconnect — it hands off to the in-app browser rather than reading a `#p=me`-gated + * subscription — so `relay.trigger.buzz` undercounts joins, not re-challenges. + */ +internal fun reconnectPoolAfterJoin(client: INostrClient) { + // Guarded like every other ledger write that reaches the application singleton + // (MediaPlayTimeTracker, the workers): a diagnostics counter must never break a + // user-visible join, and `Amethyst.instance` is lateinit. + runCatching { Amethyst.instance.resourceUsage.add(UsageKeys.relayTrigger(UsageKeys.TRIGGER_BUZZ), 1) } + client.reconnect(onlyIfChanged = false, ignoreRetryDelays = true) +} diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/buzz/BuzzRelayImportViewModel.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/buzz/BuzzRelayImportViewModel.kt index 9cded7263f..a3976d203f 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/buzz/BuzzRelayImportViewModel.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/buzz/BuzzRelayImportViewModel.kt @@ -107,14 +107,8 @@ class BuzzRelayImportViewModel : ViewModel() { val newlyJoined = BuzzWorkspaces.join(normalized) viewModelScope.launch { account.relayAuthLedger.setDecision(normalized.url, RelayAuthDecision.ALLOW) } - // NIP-42 sends its AUTH challenge once, on connect. If the socket was already open before this - // join (the common case — the relay is in the user's lists and connected at startup), that - // challenge was spent while the relay was still NOT first-party, so the connection is - // unauthenticated and the persistent group-roster (39002) subscription is refused. Joining - // makes the relay first-party (see AuthCoordinator.isFirstParty); force a reconnect so the - // relay re-challenges and the connection authenticates — unlocking the roster (Join gate) and - // every other `#p=me`-gated read on the shared socket. - if (newlyJoined) account.client.reconnect(onlyIfChanged = false, ignoreRetryDelays = true) + // Unlocks the persistent group-roster (39002) subscription — see [reconnectPoolAfterJoin]. + if (newlyJoined) reconnectPoolAfterJoin(account.client) // Track "already added" against the live kind-10009 list, scoped to this relay, so the rows // follow every add/remove — from here, from the channel's top bar, or from another device. diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageLedgerTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageLedgerTest.kt index ac4c066788..c42b3bdada 100644 --- a/amethyst/src/test/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageLedgerTest.kt +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageLedgerTest.kt @@ -20,12 +20,33 @@ */ package com.vitorpamplona.amethyst.service.resourceusage +import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.ExplainedFilter +import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.SubPurpose import com.vitorpamplona.amethyst.service.playback.playerPool.MediaPlayTimeTracker import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.relay.client.single.IRelayClient +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.AuthMessage +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.ClosedMessage +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.CountMessage +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.EoseMessage +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.EventMessage +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.LimitsMessage +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.NoticeMessage +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.NotifyMessage +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.OkMessage +import com.vitorpamplona.quartz.nip01Core.relay.commands.toRelay.AuthCmd +import com.vitorpamplona.quartz.nip01Core.relay.commands.toRelay.CloseCmd +import com.vitorpamplona.quartz.nip01Core.relay.commands.toRelay.CountCmd +import com.vitorpamplona.quartz.nip01Core.relay.commands.toRelay.EventCmd +import com.vitorpamplona.quartz.nip01Core.relay.commands.toRelay.ReqCmd +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner import com.vitorpamplona.quartz.nip57Zaps.LnZapPrivateEvent import com.vitorpamplona.quartz.nip57Zaps.LnZapRequestEvent +import io.mockk.every +import io.mockk.mockk import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.flow.MutableStateFlow @@ -778,3 +799,756 @@ class ResourceUsageAlertsTest { assertFalse(ResourceUsageAlerts.shouldPrompt(lastAlertAtSec = 0, optOut = true, nowSec = now)) } } + +/** + * The guard that keeps the counter-key grammar honest — see the reserved-segment + * note on [UsageKeys] for the mechanism and what it would cost to get wrong. + */ +class UsageKeyGrammarTest { + /** Every diagnostic key shape, with values that would be obvious if they leaked into a sum. */ + private fun churnKeys(): Map { + val out = mutableMapOf() + var n = 1_000_000L + for (mobile in listOf(true, false)) { + for (fg in listOf(true, false)) { + out[UsageKeys.relayDials(mobile, fg)] = n++ + out[UsageKeys.relayDisconnects(mobile, fg)] = n++ + for (ms in listOf(1L, 10_000L, 45_000L, 90_000L, 200_000L, 900_000L)) { + out[UsageKeys.relayLife(ms, mobile, fg)] = n++ + } + for (verb in CMD_LABELS) { + out[UsageKeys.relayVerb(verb, received = false, mobile, fg)] = n++ + } + for (verb in MSG_LABELS) { + out[UsageKeys.relayVerb(verb, received = true, mobile, fg)] = n++ + } + } + } + out[UsageKeys.RELAY_LIFE_OVERWRITE] = n++ + out[UsageKeys.RELAY_LIFE_ORPHAN] = n++ + for (cause in listOf( + UsageKeys.TRIGGER_NETID, + UsageKeys.TRIGGER_TRANSPORT, + UsageKeys.TRIGGER_TOR_POLICY, + UsageKeys.TRIGGER_CLASSIFICATION, + UsageKeys.TRIGGER_COLD_START, + UsageKeys.TRIGGER_OFF, + UsageKeys.TRIGGER_BUZZ, + )) { + out[UsageKeys.relayTrigger(cause)] = n++ + } + for (mobile in listOf(true, false)) { + for (fg in listOf(true, false)) { + out[UsageKeys.relaySubsSent(mobile, fg)] = n++ + out[UsageKeys.relaySubsClosed(mobile, fg)] = n++ + out[UsageKeys.relaySubsReplay(mobile, fg)] = n++ + } + } + UsageKeys.NOTICE_REASONS.forEach { out[UsageKeys.relayNotice(it)] = n++ } + for (mobile in listOf(true, false)) { + for (fg in listOf(true, false)) { + out[UsageKeys.relaySubsResent(mobile, fg)] = n++ + out[UsageKeys.relayEventsSeen(mobile, fg)] = n++ + out[UsageKeys.relayEventsDup(mobile, fg)] = n++ + out[UsageKeys.relayEventsDupBytes(mobile, fg)] = n++ + for (ms in listOf(0L, 200L, 1_000L, 5_000L, 20_000L, 60_000L)) { + out[UsageKeys.relayHandshake(ms, mobile, fg)] = n++ + out[UsageKeys.relayDialGap(ms, mobile, fg)] = n++ + } + } + } + (SubPurpose.entries.map { UsageKeys.purposeKeyPart(it) } + UsageKeys.PURPOSE_UNEXPLAINED + UsageKeys.PURPOSE_UNATTRIBUTED).forEach { + out[UsageKeys.relayPurposeSent(it)] = n++ + out[UsageKeys.relayPurposeBytes(it)] = n++ + out[UsageKeys.relayPurposeDown(it)] = n++ + out[UsageKeys.relayPurposeDownCount(it)] = n++ + out[UsageKeys.relayPurposeDupBytes(it)] = n++ + } + return out + } + + /** A baseline of the pre-existing counters the summary is actually built from. */ + private fun baseline(): Map = + mapOf( + UsageKeys.relayMsg(mobile = true, foreground = false, received = true) to 500L, + UsageKeys.relayMsg(mobile = true, foreground = false, received = false) to 60L, + UsageKeys.relayMsg(mobile = false, foreground = true, received = true) to 900L, + UsageKeys.net(UsageKeys.ROLE_IMAGE, mobile = true, foreground = true, received = true) to 70L, + UsageKeys.netReqs(UsageKeys.ROLE_IMAGE, mobile = true, foreground = true) to 3L, + UsageKeys.netActiveMs(UsageKeys.ROLE_IMAGE, mobile = true, foreground = true) to 40L, + UsageKeys.radioBursts(mobile = true, foreground = true) to 2L, + UsageKeys.relayConnMs(mobile = true, foreground = false) to 1_234L, + UsageKeys.relayConnects(mobile = true, foreground = false) to 11L, + UsageKeys.relayConnectFails(mobile = true, foreground = false) to 22L, + UsageKeys.workerRuns("calendarReminder") to 1L, + ) + + @Test + fun newKeysDoNotDisturbSummary() { + val before = UsageSummary.from(baseline()) + val after = UsageSummary.from(baseline() + churnKeys()) + assertEquals(before, after) + } + + @Test + fun noNewKeyContainsAReservedSegment() { + val reserved = + setOf( + UsageKeys.RX, + UsageKeys.TX, + "msg", + "connms", + "connects", + "connfails", + "reqs", + "bursts", + "activems", + "worker", + "runs", + ) + UsageKeys.HTTP_ROLES + + churnKeys().keys.forEach { key -> + val clash = key.split('.').filter { it in reserved } + assertTrue("Key '$key' uses reserved segment(s) $clash", clash.isEmpty()) + } + } + + companion object { + /** + * The verb segments taken straight from quartz's own wire labels — the same + * source [RelayUsageListener] reads, so a new subtype cannot be tested against + * a stale hand-written list. + */ + val CMD_LABELS = listOf(ReqCmd.LABEL, EventCmd.LABEL, AuthCmd.LABEL, CloseCmd.LABEL, CountCmd.LABEL) + val MSG_LABELS = + listOf( + EventMessage.LABEL, + EoseMessage.LABEL, + OkMessage.LABEL, + NoticeMessage.LABEL, + AuthMessage.LABEL, + ClosedMessage.LABEL, + CountMessage.LABEL, + NotifyMessage.LABEL, + LimitsMessage.LABEL, + ) + } +} + +class UsageKeyHelpersTest { + @Test + fun lifeBucketsAreHalfOpen() { + assertEquals("lt5s", UsageKeys.lifeBucket(0)) + assertEquals("lt5s", UsageKeys.lifeBucket(4_999)) + assertEquals("lt30s", UsageKeys.lifeBucket(5_000)) + assertEquals("lt60s", UsageKeys.lifeBucket(59_999)) + // The one that matters: exactly the stability bar is NOT "under a minute". + assertEquals("lt120s", UsageKeys.lifeBucket(60_000)) + assertEquals("lt300s", UsageKeys.lifeBucket(299_999)) + assertEquals("gte300s", UsageKeys.lifeBucket(300_000)) + assertEquals("gte300s", UsageKeys.lifeBucket(Long.MAX_VALUE)) + } +} + +@OptIn(ExperimentalCoroutinesApi::class) +class RelayUsageListenerTest { + @get:Rule val tmp = TemporaryFolder() + + private var now = 0L + + private fun relay(url: String): IRelayClient { + val r = mockk(relaxed = true) + every { r.url } returns NormalizedRelayUrl(url) + return r + } + + private fun runLedger(block: suspend (ResourceUsageAccountant, RelayUsageListener) -> Unit) = + runTest { + val store = ResourceUsageStore(File(tmp.newFolder(), "usage.json")) + // backgroundScope, as everywhere else in this file: the accountant's + // debounced flush is auto-cancelled with the test instead of leaking a + // pending 30s delay into the test body. + val accountant = ResourceUsageAccountant(store, backgroundScope, epochDay = { 1L }) + val l = + RelayUsageListener( + accountant = accountant, + isMobile = { false }, + isForeground = { true }, + nowMs = { now }, + ) + block(accountant, l) + } + + private suspend fun counters(accountant: ResourceUsageAccountant): Map = accountant.allDaysIncludingLive()[1L].orEmpty() + + @Test + fun bucketsASessionByHowLongItLived() = + runLedger { accountant, l -> + val r = relay("wss://relay.damus.io/") + now = 1_000 + l.onConnected(r, 10, false) + now = 1_000 + 45_000 + l.onDisconnected(r) + + val c = counters(accountant) + assertEquals(1L, c[UsageKeys.relayLife(45_000, mobile = false, foreground = true)]) + assertEquals(1L, c[UsageKeys.relayConnects(mobile = false, foreground = true)]) + assertEquals(1L, c[UsageKeys.relayDisconnects(mobile = false, foreground = true)]) + assertNull(c[UsageKeys.RELAY_LIFE_ORPHAN]) + assertNull(c[UsageKeys.RELAY_LIFE_OVERWRITE]) + } + + @Test + fun aDialThatNeverConnectedProducesNoLifetime() = + runLedger { accountant, l -> + val r = relay("wss://nos.lol/") + l.onConnecting(r) + l.onCannotConnect(r, "WebSocket Failure: timeout (SocketTimeoutException)") + l.onDisconnected(r) + + val c = counters(accountant) + assertEquals(1L, c[UsageKeys.relayDials(mobile = false, foreground = true)]) + assertEquals(1L, c[UsageKeys.relayConnectFails(mobile = false, foreground = true)]) + // No start stamp to consume: counted as an orphan rather than a 0ms session, + // which would otherwise pile into lt5s and fake "instant failures". + assertEquals(1L, c[UsageKeys.RELAY_LIFE_ORPHAN]) + assertNull(c[UsageKeys.relayLife(0, mobile = false, foreground = true)]) + assertNull(c[UsageKeys.relayConnects(mobile = false, foreground = true)]) + } + + @Test + fun aSecondConnectBeforeDisconnectIsCountedAsAnOverwrite() = + runLedger { accountant, l -> + // The teardown race: disconnect(); connect() runs synchronously, so a stale + // failure callback for the old socket can land after the new one is open. + val r = relay("wss://relay.damus.io/") + now = 0 + l.onConnected(r, 10, false) + now = 500_000 + l.onConnected(r, 10, false) + now = 500_100 + l.onDisconnected(r) + + val c = counters(accountant) + assertEquals(1L, c[UsageKeys.RELAY_LIFE_OVERWRITE]) + assertEquals(2L, c[UsageKeys.relayConnects(mobile = false, foreground = true)]) + // The long session was lost; only the short one is recorded. `connects` is the + // denominator that makes that deficit visible instead of silent. + assertEquals(1L, c[UsageKeys.relayLife(100, mobile = false, foreground = true)]) + assertNull(c[UsageKeys.relayLife(500_000, mobile = false, foreground = true)]) + } + + @Test + fun twoRelaysDoNotShareASlot() = + runLedger { accountant, l -> + val a = relay("wss://relay.damus.io/") + val b = relay("wss://nos.lol/") + now = 0 + l.onConnected(a, 10, false) + l.onConnected(b, 10, false) + now = 90_000 + l.onDisconnected(a) + now = 200_000 + l.onDisconnected(b) + + val c = counters(accountant) + assertEquals(1L, c[UsageKeys.relayLife(90_000, mobile = false, foreground = true)]) + assertEquals(1L, c[UsageKeys.relayLife(200_000, mobile = false, foreground = true)]) + assertNull(c[UsageKeys.RELAY_LIFE_OVERWRITE]) + } + + @Test + fun sentVerbSplitSumsBackToTheByteTotal() = + runLedger { accountant, l -> + val r = relay("wss://relay.damus.io/") + val req = ReqCmd("sub1", listOf()) + val event = EventCmd(mockk(relaxed = true)) + l.onSent(r, "0123456789", req, success = true) + l.onSent(r, "01234", event, success = true) + // A failed send is not counted at all, matching relay.msg.*.tx. + l.onSent(r, "0123456789012345", req, success = false) + + val c = counters(accountant) + val total = c[UsageKeys.relayMsg(mobile = false, foreground = true, received = false)] + val split = + c.filterKeys { it.startsWith("relay.verb.up.") }.values.sum() + assertEquals(15L, total) + assertEquals(total, split) + assertEquals(10L, c[UsageKeys.relayVerb(ReqCmd.LABEL, received = false, mobile = false, foreground = true)]) + assertEquals(5L, c[UsageKeys.relayVerb(EventCmd.LABEL, received = false, mobile = false, foreground = true)]) + // The label is uppercase on the wire; the key segment is not. + assertEquals("relay.verb.up.req.wifi.fg", UsageKeys.relayVerb(ReqCmd.LABEL, received = false, mobile = false, foreground = true)) + } + + @Test + fun receivedVerbSplitSumsBackToTheByteTotal() = + runLedger { accountant, l -> + val r = relay("wss://relay.damus.io/") + l.onIncomingMessage(r, "0123456789", EoseMessage("sub1")) + l.onIncomingMessage(r, "012", NoticeMessage("hi")) + + val c = counters(accountant) + val total = c[UsageKeys.relayMsg(mobile = false, foreground = true, received = true)] + val split = c.filterKeys { it.startsWith("relay.verb.down.") }.values.sum() + assertEquals(13L, total) + assertEquals(total, split) + } + + @Test + fun reqsInsideTheConnectWindowCountAsReplay() = + runLedger { accountant, l -> + val r = relay("wss://relay.damus.io/") + now = 10_000 + l.onConnected(r, 10, false) + // syncState's burst: sent immediately after the socket is ready. + l.onSent(r, "[\"REQ\"]", ReqCmd("sub1", listOf()), success = true) + l.onSent(r, "[\"REQ\"]", ReqCmd("sub2", listOf()), success = true) + // Well past the window — a user opening a screen, not a replay. + now = 10_000 + UsageKeys.REPLAY_WINDOW_MS + 1 + l.onSent(r, "[\"REQ\"]", ReqCmd("sub3", listOf()), success = true) + l.onSent(r, "[\"CLOSE\"]", CloseCmd("sub1"), success = true) + + val c = counters(accountant) + assertEquals(3L, c[UsageKeys.relaySubsSent(mobile = false, foreground = true)]) + assertEquals(2L, c[UsageKeys.relaySubsReplay(mobile = false, foreground = true)]) + assertEquals(1L, c[UsageKeys.relaySubsClosed(mobile = false, foreground = true)]) + } + + @Test + fun aReqOnANeverConnectedRelayIsNotReplay() = + runLedger { accountant, l -> + // No onConnected, so no start stamp: must not be attributed to a burst. + val r = relay("wss://nos.lol/") + l.onSent(r, "[\"REQ\"]", ReqCmd("sub1", listOf()), success = true) + + val c = counters(accountant) + assertEquals(1L, c[UsageKeys.relaySubsSent(mobile = false, foreground = true)]) + assertNull(c[UsageKeys.relaySubsReplay(mobile = false, foreground = true)]) + } + + @Test + fun aFailedSendCountsNowhere() = + runLedger { accountant, l -> + val r = relay("wss://relay.damus.io/") + l.onSent(r, "[\"REQ\"]", ReqCmd("sub1", listOf()), success = false) + + val c = counters(accountant) + assertNull(c[UsageKeys.relaySubsSent(mobile = false, foreground = true)]) + } + + @Test + fun aRefusalNoticeIsCountedByReason() = + runLedger { accountant, l -> + val r = relay("wss://nos.lol/") + l.onIncomingMessage(r, "[\"NOTICE\",\"x\"]", NoticeMessage("ERROR: too many concurrent REQs")) + l.onIncomingMessage(r, "[\"NOTICE\",\"y\"]", NoticeMessage("hello")) + + val c = counters(accountant) + assertEquals(1L, c[UsageKeys.relayNotice(UsageKeys.NOTICE_TOO_MANY_SUBS)]) + assertEquals(1L, c[UsageKeys.relayNotice(UsageKeys.NOTICE_UNCLASSIFIED)]) + // Still part of the byte total, so the verb invariant is unaffected. + assertEquals( + c[UsageKeys.relayMsg(mobile = false, foreground = true, received = true)], + c.filterKeys { it.startsWith("relay.verb.down.") }.values.sum(), + ) + } + + @Test + fun aReqForAnAlreadyOpenSubscriptionCountsAsAResend() = + runLedger { accountant, l -> + val r = relay("wss://relay.damus.io/") + l.onConnected(r, 10, false) + l.onSent(r, "[\"REQ\"]", ReqCmd("sub1", listOf(Filter())), success = true) + // Same subId, still open: the assembler changed its mind. + l.onSent(r, "[\"REQ\"]", ReqCmd("sub1", listOf(Filter())), success = true) + l.onSent(r, "[\"REQ\"]", ReqCmd("sub2", listOf(Filter())), success = true) + + val c = counters(accountant) + assertEquals(3L, c[UsageKeys.relaySubsSent(mobile = false, foreground = true)]) + assertEquals(1L, c[UsageKeys.relaySubsResent(mobile = false, foreground = true)]) + } + + @Test + fun aClosedSubscriptionCanBeReopenedWithoutCountingAsAResend() = + runLedger { accountant, l -> + val r = relay("wss://relay.damus.io/") + l.onConnected(r, 10, false) + l.onSent(r, "[\"REQ\"]", ReqCmd("sub1", listOf(Filter())), success = true) + l.onSent(r, "[\"CLOSE\"]", CloseCmd("sub1"), success = true) + l.onSent(r, "[\"REQ\"]", ReqCmd("sub1", listOf(Filter())), success = true) + + assertNull(counters(accountant)[UsageKeys.relaySubsResent(mobile = false, foreground = true)]) + } + + @Test + fun aReconnectForgetsOpenSubscriptions() = + runLedger { accountant, l -> + // The relay forgets them too, so the post-reconnect replay is legitimately + // new work and must not be booked as the client changing its mind. + val r = relay("wss://relay.damus.io/") + l.onConnected(r, 10, false) + l.onSent(r, "[\"REQ\"]", ReqCmd("sub1", listOf(Filter())), success = true) + l.onDisconnected(r) + l.onConnected(r, 10, false) + l.onSent(r, "[\"REQ\"]", ReqCmd("sub1", listOf(Filter())), success = true) + + assertNull(counters(accountant)[UsageKeys.relaySubsResent(mobile = false, foreground = true)]) + } + + @Test + fun reqsAreAttributedToTheirSubscriptionPurpose() = + runLedger { accountant, l -> + val r = relay("wss://relay.damus.io/") + l.onConnected(r, 10, false) + l.onSent(r, "0123456789", ReqCmd("a", listOf(ExplainedFilter(purpose = SubPurpose.HOME_FEED))), success = true) + l.onSent(r, "01234", ReqCmd("b", listOf(ExplainedFilter(purpose = SubPurpose.OTHER))), success = true) + // An assembler #3832 never tagged: its own bucket, not a guess. + l.onSent(r, "012", ReqCmd("c", listOf(Filter())), success = true) + + val c = counters(accountant) + assertEquals(1L, c[UsageKeys.relayPurposeSent("home_feed")]) + assertEquals(10L, c[UsageKeys.relayPurposeBytes("home_feed")]) + assertEquals(1L, c[UsageKeys.relayPurposeSent(UsageKeys.PURPOSE_OTHER)]) + assertEquals(1L, c[UsageKeys.relayPurposeSent(UsageKeys.PURPOSE_UNEXPLAINED)]) + // Purpose bytes reconcile with the REQ verb total. + assertEquals( + c[UsageKeys.relayVerb("REQ", received = false, mobile = false, foreground = true)], + c.filterKeys { it.startsWith("relay.purpose.") && it.endsWith(".bytes") }.values.sum(), + ) + } + + @Test + fun handshakeAndDialGapSeparateTheRelayFromOurselves() = + runLedger { accountant, l -> + val r = relay("wss://relay.damus.io/") + now = 0 + l.onConnecting(r) + // 3s of wall clock to get connected, of which the transport says the + // upgrade round trip was 150ms — the other 2850ms is DNS/TCP/TLS/queueing. + now = 3_000 + l.onConnected(r, pingMillis = 150, compressed = false) + + val c = counters(accountant) + assertEquals(1L, c[UsageKeys.relayHandshake(150, mobile = false, foreground = true)]) + assertEquals(1L, c[UsageKeys.relayDialGap(2_850, mobile = false, foreground = true)]) + } + + @Test + fun anUntimeableHandshakeRecordsNothingRatherThanZero() = + runLedger { accountant, l -> + val r = relay("wss://relay.damus.io/") + l.onConnecting(r) + l.onConnected(r, pingMillis = 0, compressed = false) + + val c = counters(accountant) + assertNull(c[UsageKeys.relayHandshake(0, mobile = false, foreground = true)]) + assertNull(c[UsageKeys.relayDialGap(0, mobile = false, foreground = true)]) + // The connection itself is still counted. + assertEquals(1L, c[UsageKeys.relayConnects(mobile = false, foreground = true)]) + } + + @Test + fun inboundBytesAreAttributedToTheSubscriptionThatAskedForThem() = + runLedger { accountant, l -> + val r = relay("wss://relay.damus.io/") + l.onConnected(r, 10, false) + l.onSent(r, "[\"REQ\"]", ReqCmd("mod1", listOf(ExplainedFilter(purpose = SubPurpose.MODERATION))), success = true) + l.onSent(r, "[\"REQ\"]", ReqCmd("feed1", listOf(ExplainedFilter(purpose = SubPurpose.HOME_FEED))), success = true) + + l.onIncomingMessage(r, "0123456789", EventMessage("mod1", mockk(relaxed = true))) + l.onIncomingMessage(r, "01234", EventMessage("feed1", mockk(relaxed = true))) + l.onIncomingMessage(r, "012", EoseMessage("mod1")) + + val c = counters(accountant) + assertEquals(13L, c[UsageKeys.relayPurposeDown("moderation")]) + assertEquals(5L, c[UsageKeys.relayPurposeDown("home_feed")]) + // Frames, not just bytes: 13 bytes of moderation arrived as two frames, so + // the average frame size is recoverable per purpose. + assertEquals(2L, c[UsageKeys.relayPurposeDownCount("moderation")]) + assertEquals(1L, c[UsageKeys.relayPurposeDownCount("home_feed")]) + } + + @Test + fun framesStillInFlightAfterACloseAreStillAttributed() = + runLedger { accountant, l -> + // The bug this replaced: CLOSE removed the id, so events the relay had + // already queued landed in `unattributed`. 8,159 CLOSEs in one session + // sent 41% of the download there. + val r = relay("wss://relay.damus.io/") + l.onConnected(r, 10, false) + l.onSent(r, "[\"REQ\"]", ReqCmd("s1", listOf(ExplainedFilter(purpose = SubPurpose.HOME_FEED))), success = true) + l.onSent(r, "[\"CLOSE\"]", CloseCmd("s1"), success = true) + l.onIncomingMessage(r, "0123456789", EventMessage("s1", mockk(relaxed = true))) + + assertEquals(10L, counters(accountant)[UsageKeys.relayPurposeDown("home_feed")]) + } + + @Test + fun aFrameArrivingAfterAReconnectIsStillAttributed() = + runLedger { accountant, l -> + // Purpose is a property of the subscription id, not of the socket, so a + // disconnect must not forget it. + val r = relay("wss://relay.damus.io/") + l.onConnected(r, 10, false) + l.onSent(r, "[\"REQ\"]", ReqCmd("s1", listOf(ExplainedFilter(purpose = SubPurpose.ENGAGEMENT))), success = true) + l.onDisconnected(r) + l.onConnected(r, 10, false) + l.onIncomingMessage(r, "01234", EventMessage("s1", mockk(relaxed = true))) + + assertEquals(5L, counters(accountant)[UsageKeys.relayPurposeDown("engagement")]) + } + + @Test + fun aReconnectStillForgetsWhichSubscriptionsAreOpen() = + runLedger { accountant, l -> + // The other half of the split: the relay forgot them, so the replay is + // new work and must not read as the client changing its mind. + val r = relay("wss://relay.damus.io/") + l.onConnected(r, 10, false) + l.onSent(r, "[\"REQ\"]", ReqCmd("s1", listOf(ExplainedFilter(purpose = SubPurpose.HOME_FEED))), success = true) + l.onDisconnected(r) + l.onConnected(r, 10, false) + l.onSent(r, "[\"REQ\"]", ReqCmd("s1", listOf(ExplainedFilter(purpose = SubPurpose.HOME_FEED))), success = true) + + assertNull(counters(accountant)[UsageKeys.relaySubsResent(mobile = false, foreground = true)]) + } + + @Test + fun anInboundFrameForAnUnknownSubscriptionIsNotGuessedAt() = + runLedger { accountant, l -> + // Counters wiped mid-session, or a subscription opened before this + // connection: attributing it to a purpose would be an invention. + val r = relay("wss://relay.damus.io/") + l.onConnected(r, 10, false) + l.onIncomingMessage(r, "0123456789", EventMessage("ghost", mockk(relaxed = true))) + + val c = counters(accountant) + assertEquals(10L, c[UsageKeys.relayPurposeDown(UsageKeys.PURPOSE_UNATTRIBUTED)]) + assertEquals(1L, c[UsageKeys.relayPurposeDownCount(UsageKeys.PURPOSE_UNATTRIBUTED)]) + } + + @Test + fun relayWideFramesAreLeftOutRatherThanMisattributed() = + runLedger { accountant, l -> + // NOTICE and OK name no subscription, so nothing may be booked for them. + // This is why purpose.down deliberately does not reconcile to msg.rx. + val r = relay("wss://relay.damus.io/") + l.onConnected(r, 10, false) + l.onIncomingMessage(r, "0123456789", NoticeMessage("hello")) + l.onIncomingMessage(r, "01234", OkMessage("id", true, "")) + + val c = counters(accountant) + assertTrue(c.keys.none { it.startsWith("relay.purpose.") && it.endsWith(".down") }) + // Still counted in the byte totals and the verb split. + assertEquals(15L, c[UsageKeys.relayMsg(mobile = false, foreground = true, received = true)]) + } + + private fun eventWithId(id: String): EventMessage { + val ev = mockk(relaxed = true) + every { ev.id } returns id + val msg = mockk(relaxed = true) + every { msg.subId } returns "s1" + every { msg.event } returns ev + every { msg.label() } returns EventMessage.LABEL + return msg + } + + @Test + fun theSameEventFromTwoRelaysIsCountedOnceAsNewAndOnceAsDuplicate() = + runLedger { accountant, l -> + // The outbox fan-out asks many relays for the same authors, so one event + // arrives once per relay carrying it. That repetition is the measurement. + val a = relay("wss://relay.damus.io/") + val b = relay("wss://nos.lol/") + l.onConnected(a, 10, false) + l.onConnected(b, 10, false) + val id = "a".repeat(64) + l.onIncomingMessage(a, "0123456789", eventWithId(id)) + l.onIncomingMessage(b, "0123456789", eventWithId(id)) + + val c = counters(accountant) + assertEquals(2L, c[UsageKeys.relayEventsSeen(mobile = false, foreground = true)]) + assertEquals(1L, c[UsageKeys.relayEventsDup(mobile = false, foreground = true)]) + assertEquals(10L, c[UsageKeys.relayEventsDupBytes(mobile = false, foreground = true)]) + } + + @Test + fun duplicateBytesAreAttributedToThePurposeThatReceivedThem() = + runLedger { accountant, l -> + // Global duplication says how much is wasted; this says where, which is + // what separates "suppress redundant delivery" from "stop fetching it". + val a = relay("wss://relay.damus.io/") + val b = relay("wss://nos.lol/") + l.onConnected(a, 10, false) + l.onConnected(b, 10, false) + l.onSent(a, "[\"REQ\"]", ReqCmd("s1", listOf(ExplainedFilter(purpose = SubPurpose.USER_PROFILE))), success = true) + l.onSent(b, "[\"REQ\"]", ReqCmd("s1", listOf(ExplainedFilter(purpose = SubPurpose.USER_PROFILE))), success = true) + + val id = "b".repeat(64) + l.onIncomingMessage(a, "0123456789", eventWithId(id)) + l.onIncomingMessage(b, "0123456789", eventWithId(id)) + + val c = counters(accountant) + assertEquals(20L, c[UsageKeys.relayPurposeDown("user_profile")]) + // Only the second copy is waste. + assertEquals(10L, c[UsageKeys.relayPurposeDupBytes("user_profile")]) + assertEquals(10L, c[UsageKeys.relayEventsDupBytes(mobile = false, foreground = true)]) + } + + @Test + fun aFirstDeliveryIsNeverBookedAsDuplicate() = + runLedger { accountant, l -> + val r = relay("wss://relay.damus.io/") + l.onConnected(r, 10, false) + l.onSent(r, "[\"REQ\"]", ReqCmd("s1", listOf(ExplainedFilter(purpose = SubPurpose.HOME_FEED))), success = true) + l.onIncomingMessage(r, "0123456789", eventWithId("c".repeat(64))) + + val c = counters(accountant) + assertEquals(10L, c[UsageKeys.relayPurposeDown("home_feed")]) + assertNull(c[UsageKeys.relayPurposeDupBytes("home_feed")]) + } + + @Test + fun distinctEventsAreNotDuplicates() = + runLedger { accountant, l -> + val r = relay("wss://relay.damus.io/") + l.onConnected(r, 10, false) + // Differ only past the 64-bit prefix would be a collision; differ within it. + l.onIncomingMessage(r, "01234", eventWithId("1".repeat(64))) + l.onIncomingMessage(r, "01234", eventWithId("2".repeat(64))) + + val c = counters(accountant) + assertEquals(2L, c[UsageKeys.relayEventsSeen(mobile = false, foreground = true)]) + assertNull(c[UsageKeys.relayEventsDup(mobile = false, foreground = true)]) + } + + @Test + fun aMalformedEventIdIsCountedButNeverDeduplicated() = + runLedger { accountant, l -> + val r = relay("wss://relay.damus.io/") + l.onConnected(r, 10, false) + l.onIncomingMessage(r, "01234", eventWithId("short")) + l.onIncomingMessage(r, "01234", eventWithId("short")) + l.onIncomingMessage(r, "01234", eventWithId("zzzz".repeat(16))) + + val c = counters(accountant) + assertEquals(3L, c[UsageKeys.relayEventsSeen(mobile = false, foreground = true)]) + assertNull(c[UsageKeys.relayEventsDup(mobile = false, foreground = true)]) + } + + @Test + fun everyCommandAndMessageSubtypeHasItsOwnVerb() { + // Labels are read off *instances*, because `cmd.label()` is what + // RelayUsageListener calls — a subtype whose label() didn't return its own + // LABEL would be invisible to a constant-only check. Asserting these against + // UsageKeyGrammarTest's lists then keeps the two inventories from drifting, + // which they had already started to do. + val cmdVerbs = + listOf( + ReqCmd("s", listOf()), + CloseCmd("s"), + EventCmd(mockk(relaxed = true)), + AuthCmd(mockk(relaxed = true)), + CountCmd("s", listOf()), + ).map { it.label() } + val msgVerbs = + listOf( + EventMessage("s", mockk(relaxed = true)), + EoseMessage("s"), + OkMessage("id", true, ""), + NoticeMessage("m"), + AuthMessage("challenge"), + ClosedMessage("s", "m"), + CountMessage("s", mockk(relaxed = true)), + NotifyMessage("m"), + LimitsMessage(), + ).map { it.label() } + + assertEquals(UsageKeyGrammarTest.CMD_LABELS.toSet(), cmdVerbs.toSet()) + assertEquals(UsageKeyGrammarTest.MSG_LABELS.toSet(), msgVerbs.toSet()) + + // No two labels may collide within a direction, and none may be key-hostile + // (a dot would inject uncontrolled segments — see UsageKeys.sumMatching). + assertEquals(cmdVerbs.size, cmdVerbs.distinct().size) + assertEquals(msgVerbs.size, msgVerbs.distinct().size) + (cmdVerbs + msgVerbs).forEach { + assertFalse("Label '$it' is not usable as a counter key segment", it.isEmpty() || it.contains('.')) + } + } +} + +/** + * NOTICE classification. The reason must come from a fixed set: this key is + * persisted for 30 days and the text behind it is written by the relay. + */ +class NoticeReasonTest { + @Test + fun refusalsAreRecognisedWhateverTheRelayCallsThem() { + // strfry's, the one Hypothesis N is about. Its NIP-01 prefix is just + // "error", so prefix matching alone would not have caught it. + assertEquals(UsageKeys.NOTICE_TOO_MANY_SUBS, UsageKeys.noticeReason("ERROR: too many concurrent REQs")) + assertEquals(UsageKeys.NOTICE_TOO_MANY_SUBS, UsageKeys.noticeReason("too many concurrent NEG requests")) + assertEquals(UsageKeys.NOTICE_TOO_MANY_SUBS, UsageKeys.noticeReason("blocked: too many subscriptions")) + } + + @Test + fun standardPrefixesAreCategorised() { + assertEquals(UsageKeys.NOTICE_AUTH_REQUIRED, UsageKeys.noticeReason("auth-required: we need to know you")) + assertEquals(UsageKeys.NOTICE_RATE_LIMITED, UsageKeys.noticeReason("rate-limited: slow down")) + assertEquals(UsageKeys.NOTICE_RESTRICTED, UsageKeys.noticeReason("restricted: not on the allowlist")) + assertEquals(UsageKeys.NOTICE_INVALID, UsageKeys.noticeReason("invalid: bad filter")) + assertEquals(UsageKeys.NOTICE_BLOCKED, UsageKeys.noticeReason("blocked: you are banned")) + assertEquals(UsageKeys.NOTICE_ERROR, UsageKeys.noticeReason("error: something broke")) + } + + /** Verbatim from a device on 2026-08-02 — the wordings the allowlist was missing. */ + @Test + fun realWorldNoticesAreClassified() { + // Refusals. Each one drops a REQ that then never EOSEs, so its `since` never + // advances and syncState re-sends it on every reconnect. + assertEquals(UsageKeys.NOTICE_QUERY_COST, UsageKeys.noticeReason("Kgo0HH: closed: too many steps")) + assertEquals(UsageKeys.NOTICE_QUERY_COST, UsageKeys.noticeReason("too many kinds")) + assertEquals(UsageKeys.NOTICE_REQ_REFUSED, UsageKeys.noticeReason("Denied! This relay does not accept REQs.")) + + // Chatter that costs bytes and means nothing. + assertEquals(UsageKeys.NOTICE_BENIGN, UsageKeys.noticeReason("keepalive")) + assertEquals(UsageKeys.NOTICE_BENIGN, UsageKeys.noticeReason("as7rp4: PERF: [/!\\ LS] 1087 scan, 0 dedup, 500 match")) + + // A bare subscription id carries no meaning and must stay unclassified rather + // than being read as a machine-readable prefix. + assertEquals(UsageKeys.NOTICE_UNCLASSIFIED, UsageKeys.noticeReason("AccountFollowsLoaderSubAssemblerxE1r8A")) + assertEquals(UsageKeys.NOTICE_UNCLASSIFIED, UsageKeys.noticeReason("Kgo0HH")) + } + + @Test + fun aSubscriptionIdPrefixDoesNotHideTheRealOne() { + // These relays send ": : ", which makes the subId the + // prefix and buries the standard one behind it. + assertEquals(UsageKeys.NOTICE_AUTH_REQUIRED, UsageKeys.noticeReason("sub123: auth-required: need auth")) + assertEquals(UsageKeys.NOTICE_RATE_LIMITED, UsageKeys.noticeReason("sub123: rate-limited: slow down")) + // Still works without the prefix. + assertEquals(UsageKeys.NOTICE_AUTH_REQUIRED, UsageKeys.noticeReason("auth-required: need auth")) + } + + @Test + fun freeFormProseNeverBecomesAKey() { + // The bug RelayObserver had to fix: without a fixed output set, cardinality + // grows with the number of distinct sentences relays happen to write. + listOf( + "hello there", + "Please contact admin@example.com for access", + "", + "::::", + "a".repeat(5000), + ).forEach { + val reason = UsageKeys.noticeReason(it) + assertTrue("'$it' produced unlisted reason '$reason'", reason in UsageKeys.NOTICE_REASONS) + } + assertEquals(UsageKeys.NOTICE_UNCLASSIFIED, UsageKeys.noticeReason("hello there")) + } + + @Test + fun anUnlistedReasonCannotMintAKey() { + assertEquals(UsageKeys.relayNotice(UsageKeys.NOTICE_UNCLASSIFIED), UsageKeys.relayNotice("something-invented")) + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/basic/BasicRelayClient.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/basic/BasicRelayClient.kt index d0f1b7bc16..eec65748e9 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/basic/BasicRelayClient.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/single/basic/BasicRelayClient.kt @@ -206,7 +206,11 @@ open class BasicRelayClient( !msg.startsWith("failed to connect to /127.0.0.1") && msg != "Socket closed" && msg != "Socket is closed" && - msg != "Cancelled" + // OkHttp spells it with one L (RealCall throws + // IOException("Canceled")). "Cancelled" never matched, so + // client-initiated cancels were being reported as connection + // failures and inflating the relay.connfails counter. + msg != "Canceled" ) ) { if (code != null || response != null) { From 55c873858eefc3197424692b6973a0745a2b1cac Mon Sep 17 00:00:00 2001 From: davotoula Date: Sat, 8 Aug 2026 08:23:52 +0200 Subject: [PATCH 067/132] feat(resourceusage): copy and share the usage report The only way out of the Resource Usage screen was "Send report via DM", which builds the text into a draft message to a fixed pubkey. Reading your own report meant opening a composer and copying out of it. Adds Copy and Share beside it, handing over the same string for a bug report or a file. Reuses Clipboard.setText from ClipboardExt and the ACTION_SEND chooser pattern from ShareActions. --- .../loggedIn/settings/ResourceUsageScreen.kt | 69 +++++++++++++++---- amethyst/src/main/res/values/strings.xml | 4 +- 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/ResourceUsageScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/ResourceUsageScreen.kt index a916dea583..08bcca8f80 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/ResourceUsageScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/ResourceUsageScreen.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.settings +import android.content.Intent import androidx.annotation.StringRes import androidx.compose.foundation.background import androidx.compose.foundation.layout.Arrangement @@ -46,11 +47,13 @@ import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.produceState import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip import androidx.compose.ui.graphics.Color +import androidx.compose.ui.platform.LocalClipboard import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.res.pluralStringResource import androidx.compose.ui.text.font.FontWeight @@ -67,6 +70,7 @@ import com.vitorpamplona.amethyst.service.resourceusage.ResourceUsageReportAssem import com.vitorpamplona.amethyst.service.resourceusage.ResourceUsageReportAssembler.Companion.formatConnHours import com.vitorpamplona.amethyst.service.resourceusage.ResourceUsageReportAssembler.Companion.formatDurationMs import com.vitorpamplona.amethyst.service.resourceusage.UsageSummary +import com.vitorpamplona.amethyst.ui.components.util.setText import com.vitorpamplona.amethyst.ui.navigation.navs.INav import com.vitorpamplona.amethyst.ui.navigation.routes.Route import com.vitorpamplona.amethyst.ui.navigation.routes.routeToMessage @@ -77,6 +81,7 @@ import com.vitorpamplona.amethyst.ui.theme.allGoodColor import com.vitorpamplona.amethyst.ui.theme.warningColor import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay +import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import java.util.Locale @@ -579,6 +584,10 @@ private fun SendReportSection( today: Long, memory: MemorySnapshot?, ) { + val context = LocalContext.current + val clipboard = LocalClipboard.current + val scope = rememberCoroutineScope() + SettingsSection(R.string.resource_usage_send_section) { Column( modifier = Modifier.padding(16.dp), @@ -589,21 +598,55 @@ private fun SendReportSection( style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant, ) - Button( - onClick = { - val report = ResourceUsageReportAssembler().buildReport(days, today, memory) - nav.nav { - routeToMessage( - user = LocalCache.getOrCreateUser(DEV_REPORT_PUBKEY), - draftMessage = report, - accountViewModel = accountViewModel, - expiresDays = 30, - ) - } - }, + // The DM route needs a composer to exist before the text does, which makes + // it a poor fit for reading the report yourself — copying out of a draft + // message is the only way to get at it. Copy and Share hand over the same + // string directly, for pasting into an issue or saving to a file. + Row( modifier = Modifier.align(Alignment.End), + horizontalArrangement = Arrangement.spacedBy(8.dp), + verticalAlignment = Alignment.CenterVertically, ) { - Text(stringRes(R.string.resource_usage_send_button)) + TextButton( + onClick = { + val report = ResourceUsageReportAssembler().buildReport(days, today, memory) + scope.launch { clipboard.setText(report) } + }, + ) { + Text(stringRes(R.string.resource_usage_copy_button)) + } + TextButton( + onClick = { + val report = ResourceUsageReportAssembler().buildReport(days, today, memory) + val send = + Intent().apply { + action = Intent.ACTION_SEND + type = "text/plain" + putExtra(Intent.EXTRA_TEXT, report) + putExtra(Intent.EXTRA_TITLE, stringRes(context, R.string.resource_usage_send_section)) + } + context.startActivity( + Intent.createChooser(send, stringRes(context, R.string.resource_usage_share_button)), + ) + }, + ) { + Text(stringRes(R.string.resource_usage_share_button)) + } + Button( + onClick = { + val report = ResourceUsageReportAssembler().buildReport(days, today, memory) + nav.nav { + routeToMessage( + user = LocalCache.getOrCreateUser(DEV_REPORT_PUBKEY), + draftMessage = report, + accountViewModel = accountViewModel, + expiresDays = 30, + ) + } + }, + ) { + Text(stringRes(R.string.resource_usage_send_button)) + } } } } diff --git a/amethyst/src/main/res/values/strings.xml b/amethyst/src/main/res/values/strings.xml index da24418325..d8716bb5dd 100644 --- a/amethyst/src/main/res/values/strings.xml +++ b/amethyst/src/main/res/values/strings.xml @@ -4188,8 +4188,10 @@ Chatroom lists in memory Device memory class Share with the developers - If Amethyst seems to drain battery or data, you can send this report to the developers in an encrypted DM. It contains only the numbers on this screen and the technical counters behind them \u2014 no posts, contacts, or browsing details. Nothing is sent until you tap Send in the message screen. + If Amethyst seems to drain battery or data, you can send this report to the developers in an encrypted DM, or copy it and share it yourself. It contains only the numbers on this screen, the technical counters behind them, and the host names of the relays that reconnected most \u2014 no posts, contacts, or browsing details. The report leaves this screen only when you send, copy, or share it. Send report via DM + Copy + Share High resource usage detected Amethyst consumed more than expected recently: %1$s. Would you like to send a usage report to the developers in an encrypted DM? You will see the full report before anything is sent. %1$s of cellular data in the background in one day From 7f94cf5cd542918d937721b5276a8e52106a246d Mon Sep 17 00:00:00 2001 From: davotoula Date: Sat, 8 Aug 2026 09:04:50 +0200 Subject: [PATCH 068/132] Code review: - fix(resourceusage): atomic subscription map, and build the report off Main - fix(resourceusage): bound the technical dump so the report stays sendable - an orphan KDoc in ResourceUsageReportAssembler with no declaration under it, which Kotlin silently bound to formatBytes - paragraphs in ResourceUsageAccountant and ResourceUsageStore claiming the key space has no fixed upper bound; every remaining counter is compile-time bounded - the user-facing privacy string, which claimed the report contains the host names of the relays that reconnected most. It does not, and that file is Crowdin-bound, so the false claim would have reached translators. --- .../resourceusage/RelayUsageListener.kt | 128 +++++++++----- .../resourceusage/ResourceUsageAccountant.kt | 18 +- .../ResourceUsageReportAssembler.kt | 59 ++++++- .../resourceusage/ResourceUsageStore.kt | 9 +- .../service/resourceusage/UsageKeys.kt | 167 +++++++++++------- .../loggedIn/settings/ResourceUsageScreen.kt | 52 +++--- amethyst/src/main/res/values/strings.xml | 2 +- .../resourceusage/ResourceUsageLedgerTest.kt | 48 +++++ 8 files changed, 328 insertions(+), 155 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/RelayUsageListener.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/RelayUsageListener.kt index 79e4e9e9c6..5abc1998e0 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/RelayUsageListener.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/RelayUsageListener.kt @@ -21,7 +21,7 @@ package com.vitorpamplona.amethyst.service.resourceusage import android.os.SystemClock -import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.ExplainedFilter +import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.purposeOrNull import com.vitorpamplona.quartz.nip01Core.relay.client.listeners.RelayConnectionListener import com.vitorpamplona.quartz.nip01Core.relay.client.single.IRelayClient import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.ClosedMessage @@ -106,10 +106,13 @@ class RelayUsageListener( /** * Event ids delivered recently, as the first 64 bits of the id. * - * Held as a Long rather than the 64-char hex: at the window size below that is - * the difference between ~200 KB and several MB on a 512 MB-class device, for a - * counter that only has to spot repetition. 64 bits makes a collision between - * distinct ids negligible where a 32-bit hash would not be. + * Held as a Long rather than the 64-char hex, which saves the id strings + * themselves — but a boxed `Long` plus its map node still costs ~56 bytes an + * entry, so a full window is ~3 MB, not the few hundred KB the raw payload + * suggests. Worth knowing before raising [MAX_TRACKED_EVENTS] on a 512 MB-class + * device; an unboxed open-addressed `LongArray` would be ~400 KB if it ever needs + * to grow. 64 bits makes a collision between distinct ids negligible where a + * 32-bit hash would not be. * * The window only needs to span the fan-out, not the session: the same event * arrives from every relay carrying it within seconds, so near-term memory @@ -120,7 +123,11 @@ class RelayUsageListener( */ private val recentEventIds = ConcurrentHashMap.newKeySet() - /** Relay -> when this dial was decided, so the pre-request cost can be separated from the handshake. */ + /** + * Relay -> when this dial was decided, so the pre-request cost can be separated + * from the handshake. Consumed by whichever of `onConnected`/`onCannotConnect` + * ends the dial, so an entry never outlives the attempt that made it. + */ private val dialStartedAt = ConcurrentHashMap() /** Notice texts already logged, so one wording costs one line however often it arrives. */ @@ -132,45 +139,49 @@ class RelayUsageListener( cmd: Command, success: Boolean, ) { - if (success) { - val bytes = cmdStr.length.toLong() - val mobile = isMobile() - val fg = isForeground() - accountant.add(UsageKeys.relayMsg(mobile, fg, received = false), bytes) - accountant.add(UsageKeys.relayVerb(cmd.label(), received = false, mobile, fg), bytes) + if (!success) return - when (cmd.label()) { - ReqCmd.LABEL -> { - accountant.add(UsageKeys.relaySubsSent(mobile, fg), 1) + val bytes = cmdStr.length.toLong() + val mobile = isMobile() + val fg = isForeground() + accountant.add(UsageKeys.relayMsg(mobile, fg, received = false), bytes) + accountant.add(UsageKeys.relayVerb(cmd.label(), received = false, mobile, fg), bytes) - val purpose = purposeOf(cmd) - accountant.add(UsageKeys.relayPurposeSent(purpose), 1) - accountant.add(UsageKeys.relayPurposeBytes(purpose), bytes) + when (cmd) { + is ReqCmd -> { + accountant.add(UsageKeys.relaySubsSent(mobile, fg), 1) - // Already open on this connection, so this REQ replaces a live - // subscription rather than starting one. - val subId = (cmd as ReqCmd).subId - if (subPurpose.size >= MAX_TRACKED_SUBS) subPurpose.clear() - subPurpose[subId] = purpose + val purpose = purposeOf(cmd) + accountant.add(UsageKeys.relayPurposeSent(purpose), 1) + accountant.add(UsageKeys.relayPurposeBytes(purpose), bytes) - val known = openSubs.getOrPut(relay.url) { ConcurrentHashMap.newKeySet() } - if (!known.add(subId)) { - accountant.add(UsageKeys.relaySubsResent(mobile, fg), 1) - } - // Within the window after this relay's connect, so almost certainly - // part of syncState's replay rather than a user action. A time - // window because nothing on this side marks a frame as belonging to - // it; see UsageKeys.relaySubsReplay. - val since = connectedSince[relay.url] - if (since != null && nowMs() - since <= UsageKeys.REPLAY_WINDOW_MS) { - accountant.add(UsageKeys.relaySubsReplay(mobile, fg), 1) - } + if (subPurpose.size >= MAX_TRACKED_SUBS) subPurpose.clear() + subPurpose[cmd.subId] = purpose + + // Already open on this connection, so this REQ replaces a live + // subscription rather than starting one. + // computeIfAbsent, not getOrPut: the latter is get-then-put and two + // threads racing the first REQ after a (re)connect would each build a + // set, the losing put's subId vanishing with its orphaned set. + // `sendIfConnected` deliberately calls listeners outside PoolRequests' + // stripe lock, so same-relay concurrency here is by design. + val known = openSubs.computeIfAbsent(relay.url) { ConcurrentHashMap.newKeySet() } + if (!known.add(cmd.subId)) { + accountant.add(UsageKeys.relaySubsResent(mobile, fg), 1) } - CloseCmd.LABEL -> { - accountant.add(UsageKeys.relaySubsClosed(mobile, fg), 1) - openSubs[relay.url]?.remove((cmd as CloseCmd).subId) + // Within the window after this relay's connect, so almost certainly + // part of syncState's replay rather than a user action. A time + // window because nothing on this side marks a frame as belonging to + // it; see UsageKeys.relaySubsReplay. + val since = connectedSince[relay.url] + if (since != null && nowMs() - since <= UsageKeys.REPLAY_WINDOW_MS) { + accountant.add(UsageKeys.relaySubsReplay(mobile, fg), 1) } } + is CloseCmd -> { + accountant.add(UsageKeys.relaySubsClosed(mobile, fg), 1) + openSubs[relay.url]?.remove(cmd.subId) + } } } @@ -250,12 +261,15 @@ class RelayUsageListener( val fg = isForeground() // Doubles as the lifetime histogram's denominator — one session begins here. accountant.add(UsageKeys.relayConnects(mobile, fg), 1) + // Consumed unconditionally: the gap needs a handshake to subtract, but the + // stamp has to go either way or a dial that cannot be timed leaks its entry. + val dialedAt = dialStartedAt.remove(relay.url) // pingMillis is the transport's own handshake timing; <= 0 means it could // not measure it, and a fabricated 0 would be worse than no record. if (pingMillis > 0) { accountant.add(UsageKeys.relayHandshake(pingMillis.toLong(), mobile, fg), 1) - dialStartedAt.remove(relay.url)?.let { startedAt -> - val gap = nowMs() - startedAt - pingMillis + if (dialedAt != null) { + val gap = nowMs() - dialedAt - pingMillis if (gap >= 0) accountant.add(UsageKeys.relayDialGap(gap, mobile, fg), 1) } } @@ -289,16 +303,29 @@ class RelayUsageListener( * subscription's. A REQ whose filters are plain [com.vitorpamplona.quartz.nip01Core.relay.filters.Filter]s * predates #3832's tagging and is counted separately rather than guessed at. */ - private fun purposeOf(cmd: Command): String { - val filters = (cmd as? ReqCmd)?.filters ?: return UsageKeys.PURPOSE_UNEXPLAINED - val explained = - filters.firstOrNull { it is ExplainedFilter } as? ExplainedFilter - ?: return UsageKeys.PURPOSE_UNEXPLAINED - return UsageKeys.purposeKeyPart(explained.purpose) - } + private fun purposeOf(cmd: ReqCmd): String = + cmd.filters + .firstNotNullOfOrNull { it.purposeOrNull() } + ?.let { UsageKeys.purposeKeyPart(it) } + ?: UsageKeys.PURPOSE_UNEXPLAINED - /** The first 64 bits of an event id, or null if it is not a well-formed id. */ - private fun idPrefix(id: String): Long? = if (id.length < 16) null else runCatching { id.substring(0, 16).toULong(16).toLong() }.getOrNull() + /** + * The first 64 bits of an event id, or null if it is not a well-formed id. + * + * Parsed in place rather than `substring(0, 16).toULongOrNull(16)`: this runs on + * every inbound EVENT frame, and the substring would be a String plus its backing + * array per event, thrown away immediately. + */ + private fun idPrefix(id: String): Long? { + if (id.length < 16) return null + var acc = 0L + for (i in 0 until 16) { + val digit = Character.digit(id[i], 16) + if (digit < 0) return null + acc = (acc shl 4) or digit.toLong() + } + return acc + } /** The subscription a frame belongs to, when it names one. */ private fun subIdOf(msg: Message): String? = @@ -315,6 +342,9 @@ class RelayUsageListener( errorMessage: String, ) { accountant.add(UsageKeys.relayConnectFails(isMobile(), isForeground()), 1) + // A dial that ends here never reaches onConnected, so nothing else would + // ever consume its start stamp. + dialStartedAt.remove(relay.url) } companion object { diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageAccountant.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageAccountant.kt index e7807eb02b..3c4cde0ccf 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageAccountant.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageAccountant.kt @@ -37,15 +37,10 @@ import java.util.concurrent.atomic.AtomicLong * AtomicLong (not LongAdder) because draining must be loss-free: getAndSet(0) * hands off the accumulated value atomically, whereas remove+sum on a * LongAdder can strand a racing increment on an orphaned cell. Entries stay - * in the map after a drain — the key space is small and *mostly* fixed - * (dims x areas), so this costs a few hundred boxed zeros at most. - * - * The exception is the per-relay churn counters (`relay.host..*`), whose - * cardinality follows the user's relay list rather than a compile-time set: - * two keys per relay, so a few hundred more entries for a large list. Still - * negligible in memory, but it does mean neither this map nor the persisted - * store has a fixed upper bound any more. Keep that in mind before adding - * another counter keyed on runtime data. + * in the map after a drain — the key space is small and fixed (dims x areas), + * so this costs a few hundred boxed zeros at most. The churn counters roughly + * tripled it, but every one of them is still compile-time bounded: no counter + * is keyed on a relay url, a host, or any other runtime string. * * Counters added from inside a pre-flush hook (the CPU sampler, the segment * integrators closing an open segment) never re-arm the debounce: they are @@ -86,6 +81,11 @@ class ResourceUsageAccountant( // (so collisions) on a path that runs per relay frame. (live[key] ?: live.computeIfAbsent(key) { AtomicLong() }).addAndGet(amount) if (inHookRun.get() == true) return + // Plain read before the CAS: in steady state a flush is always already armed, + // and the churn counters made this 5-8 calls per relay frame — every one of + // which would otherwise be a read-modify-write on the same shared cache line + // from every relay's socket thread, only to fail. + if (flushScheduled.get()) return if (flushScheduled.compareAndSet(false, true)) { scope.launch { delay(flushDebounceMs) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageReportAssembler.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageReportAssembler.kt index 9b289e5ee7..4387cb2b40 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageReportAssembler.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageReportAssembler.kt @@ -29,9 +29,8 @@ import java.util.Locale * Assembles the Markdown resource-usage report the user can DM to the * developers via NIP-17 — same shape as the crash ReportAssembler: a device * header table, a human-readable summary, then the full per-day counter dump - * as the technical payload. Counters are sizes/durations/counts only, and never - * content. - * + * as the technical payload. Counters are sizes/durations/counts only; no + * URLs, relay names, or content. */ class ResourceUsageReportAssembler { fun buildReport( @@ -76,16 +75,49 @@ class ResourceUsageReportAssembler { sb.append("\nTechnical details (per epoch-day):\n") sb.append("```\n") - days.toSortedMap().forEach { (day, counters) -> + val sorted = days.toSortedMap() + val included = newestDaysWithin(sorted, MAX_DUMP_CHARS) + sorted.forEach { (day, counters) -> + if (day !in included) return@forEach sb.append("day $day (today=$today)\n") counters.toSortedMap().forEach { (key, value) -> sb.append(" $key = $value\n") } } + val omitted = sorted.size - included.size + if (omitted > 0) { + sb.append("($omitted earlier day(s) omitted to keep this report sendable; ") + sb.append("the summary tables above still cover them)\n") + } sb.append("```\n") return sb.toString() } + /** + * The most recent days whose dumps fit in [budget], newest first, always + * including at least the newest even if it alone exceeds it. + * + * Bounded by size rather than by a day count because the per-day size is not a + * constant: it tracks how many distinct counters the build emits, and that has + * grown by more than an order of magnitude. A fixed day count would have to be + * re-tuned every time a counter family is added, and would be wrong in the + * meantime. + */ + private fun newestDaysWithin( + days: Map>, + budget: Int, + ): Set { + val included = mutableSetOf() + var left = budget + for (day in days.keys.sortedDescending()) { + val size = days.getValue(day).entries.sumOf { it.key.length + DUMP_LINE_OVERHEAD } + if (included.isNotEmpty() && size > left) break + included.add(day) + left -= size + } + return included + } + private fun summaryTable(s: UsageSummary): String = buildString { append("| Metric | Value |\n") @@ -133,7 +165,24 @@ class ResourceUsageReportAssembler { /** Markdown table header/body separator row. */ private const val TABLE_SEPARATOR = "| --- | --- |\n" - /** Caps how many relay hosts a shared report can name. See the class doc. */ + /** + * Character budget for the raw per-day dump. + * + * This report exists to be sent to the developers as a NIP-17 DM, and relays + * commonly cap events between 64 and 256 KB — so an unbounded dump does not + * merely inconvenience, it makes the report unsendable by exactly the users + * whose ledgers are most worth seeing. It also travels through a ~1 MB Binder + * transaction when shared. + * + * The ledger keeps 30 days and a busy day now emits ~1,200 counters, which is + * ~52 KB of dump per day — so "every retained day" would be ~1.5 MB. This + * keeps the newest days and says how many it dropped; the summary tables + * above are unaffected and still cover the whole window. + */ + private const val MAX_DUMP_CHARS = 64 * 1024 + + /** ` ` + ` = ` + the value, per dumped line. */ + private const val DUMP_LINE_OVERHEAD = 24 fun formatBytes(bytes: Long): String = when { diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageStore.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageStore.kt index c25bd7e3f5..9d56c6b881 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageStore.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageStore.kt @@ -34,11 +34,10 @@ import java.io.File * Jackson + Mutex + write-to-tmp-then-rename + version envelope. * * Day keys are UTC epoch-days (stringified for JSON). Buckets older than - * [keepDays] are pruned on every merge, so the file stays small — a few KB, plus - * two keys per relay per day now that the churn counters are keyed on runtime - * data (see [ResourceUsageAccountant], which owns that caveat). The whole file is - * re-serialized on every flush (debounced to ~30s while traffic flows), so that - * growth is paid on each write, not just at rest. + * [keepDays] are pruned on every merge, so the file stays small (a few KB, on a + * key space the churn counters tripled but left compile-time bounded). The whole + * file is re-serialized on every flush (debounced to ~30s while traffic flows), + * so that size is paid on each write, not just at rest. * Also carries the high-consumption alert state (last prompt time, opt-out) * so the whole feature has exactly one file. */ diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/UsageKeys.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/UsageKeys.kt index 75271e3b7f..fd222cf993 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/UsageKeys.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/UsageKeys.kt @@ -22,6 +22,13 @@ package com.vitorpamplona.amethyst.service.resourceusage import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.SubPurpose import com.vitorpamplona.quartz.nip01Core.relay.client.single.basic.BasicRelayClient +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.MachineReadablePrefix.AUTH_REQUIRED +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.MachineReadablePrefix.BLOCKED +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.MachineReadablePrefix.ERROR +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.MachineReadablePrefix.INVALID +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.MachineReadablePrefix.RATE_LIMITED +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.MachineReadablePrefix.RESTRICTED +import com.vitorpamplona.quartz.nip01Core.relay.commands.toClient.MachineReadablePrefix.UNSUPPORTED import com.vitorpamplona.quartz.nip66RelayMonitor.reachability.RelayObserver import java.util.concurrent.ConcurrentHashMap @@ -110,7 +117,10 @@ object UsageKeys { private fun dimKeys( prefix: String, suffix: String? = null, - ): Array = Array(DIMS.size) { if (suffix == null) "$prefix.${DIMS[it]}" else "$prefix.${DIMS[it]}.$suffix" } + ): Array { + val tail = if (suffix == null) "" else ".$suffix" + return Array(DIMS.size) { "$prefix.${DIMS[it]}$tail" } + } /** `net.image.mobile.bg.rx` — HTTP bytes for a subsystem. */ fun net( @@ -342,13 +352,15 @@ object UsageKeys { "too many" in text || "too costly" in text || "too expensive" in text -> NOTICE_QUERY_COST "does not accept" in text || "denied" in text || "not accepting" in text -> NOTICE_REQ_REFUSED "keepalive" in text || "perf:" in text -> NOTICE_BENIGN - "rate-limited" in prefixes || ("rate" in text && "limit" in text) -> NOTICE_RATE_LIMITED - "auth-required" in prefixes || ("auth" in text && "required" in text) -> NOTICE_AUTH_REQUIRED - "restricted" in prefixes -> NOTICE_RESTRICTED - "invalid" in prefixes -> NOTICE_INVALID - "blocked" in prefixes -> NOTICE_BLOCKED - "unsupported" in prefixes -> NOTICE_UNSUPPORTED - "error" in prefixes -> NOTICE_ERROR + // Wire codes come from the enum rather than being hand-written a third + // time (after the enum itself and the NOTICE_* key segments). + RATE_LIMITED.code in prefixes || ("rate" in text && "limit" in text) -> NOTICE_RATE_LIMITED + AUTH_REQUIRED.code in prefixes || ("auth" in text && "required" in text) -> NOTICE_AUTH_REQUIRED + RESTRICTED.code in prefixes -> NOTICE_RESTRICTED + INVALID.code in prefixes -> NOTICE_INVALID + BLOCKED.code in prefixes -> NOTICE_BLOCKED + UNSUPPORTED.code in prefixes -> NOTICE_UNSUPPORTED + ERROR.code in prefixes -> NOTICE_ERROR else -> NOTICE_UNCLASSIFIED } } @@ -369,43 +381,61 @@ object UsageKeys { const val SHORT_SESSION_MS = BasicRelayClient.STABLE_CONNECTION_IN_SECS * 1_000L /** - * Half-open upper bounds, in ms, for the [relayLife] histogram. Deliberately - * straddles [SHORT_SESSION_MS]: a mean cannot tell a tight cluster sitting on - * that bar from a bimodal mix; this can. + * A half-open millisecond histogram: ascending upper [bounds], the derived + * bucket labels, and the `..` key table they index. + * + * Shared by all three histograms below (`relay.life`, `relay.hs`, `relay.gap`), + * which differ only in their bounds and in whether the label reads in seconds or + * milliseconds. Deriving the names from the bounds is what keeps a bound change + * from leaving a label lying about it. */ - private val LIFE_BUCKET_BOUNDS_MS = - longArrayOf(5_000, 30_000, SHORT_SESSION_MS, 120_000, 300_000).also { - // [lifeBucketIndex] linear-scans for the first bound greater than the - // elapsed time, so the bounds must ascend. One of them is derived from + private class MsHistogram( + prefix: String, + private val bounds: LongArray, + label: (Long) -> String, + ) { + init { + // [indexOf] linear-scans for the first bound greater than the elapsed + // time, so the bounds must ascend. One of relay.life's is derived from // BasicRelayClient.STABLE_CONNECTION_IN_SECS, and raising that to five // minutes — exactly the retune commit 2 of the churn plan contemplates — // would push it past the two bounds after it. The buckets between would // become unreachable and the labels would start lying. Fail at class-init // with the reason rather than as a puzzling boundary-test failure. - require(it.asList() == it.sorted()) { - "relay.life bounds must ascend, got ${it.toList()}. " + + require(bounds.asList() == bounds.sorted()) { + "$prefix bounds must ascend, got ${bounds.toList()}. " + "SHORT_SESSION_MS is ${SHORT_SESSION_MS}ms — reorder the bounds to match." } } - /** Derived from the bounds so a bound change can never leave a label lying about it. */ - private val LIFE_BUCKET_NAMES = - Array(LIFE_BUCKET_BOUNDS_MS.size + 1) { i -> - if (i < LIFE_BUCKET_BOUNDS_MS.size) { - "lt${LIFE_BUCKET_BOUNDS_MS[i] / 1000}s" - } else { - "gte${LIFE_BUCKET_BOUNDS_MS.last() / 1000}s" + val names = Array(bounds.size + 1) { i -> if (i < bounds.size) "lt${label(bounds[i])}" else "gte${label(bounds.last())}" } + + private val keys = Array(names.size) { dimKeys("$prefix.${names[it]}") } + + fun indexOf(ms: Long): Int { + for (i in bounds.indices) { + if (ms < bounds[i]) return i } + return bounds.size } - private fun lifeBucketIndex(elapsedMs: Long): Int { - for (i in LIFE_BUCKET_BOUNDS_MS.indices) { - if (elapsedMs < LIFE_BUCKET_BOUNDS_MS[i]) return i - } - return LIFE_BUCKET_BOUNDS_MS.size + fun nameOf(ms: Long): String = names[indexOf(ms)] + + fun key( + ms: Long, + mobile: Boolean, + foreground: Boolean, + ): String = keys[indexOf(ms)][dimIndex(mobile, foreground)] } - fun lifeBucket(elapsedMs: Long): String = LIFE_BUCKET_NAMES[lifeBucketIndex(elapsedMs)] + /** + * Bounds for the [relayLife] histogram deliberately straddle [SHORT_SESSION_MS]: + * a mean cannot tell a tight cluster sitting on that bar from a bimodal mix; + * this can. + */ + private val LIFE = MsHistogram("relay.life", longArrayOf(5_000, 30_000, SHORT_SESSION_MS, 120_000, 300_000)) { "${it / 1000}s" } + + fun lifeBucket(elapsedMs: Long): String = LIFE.nameOf(elapsedMs) /** * `relay.life.lt60s.mobile.bg` — connections that closed after living this long. @@ -415,9 +445,7 @@ object UsageKeys { elapsedMs: Long, mobile: Boolean, foreground: Boolean, - ): String = RELAY_LIFE[lifeBucketIndex(elapsedMs)][dimIndex(mobile, foreground)] - - private val RELAY_LIFE = Array(LIFE_BUCKET_NAMES.size) { dimKeys("relay.life.${LIFE_BUCKET_NAMES[it]}") } + ): String = LIFE.key(elapsedMs, mobile, foreground) /** * `relay.life.overwrite` — a connect arrived for a relay that already had an @@ -478,10 +506,15 @@ object UsageKeys { * purpose at all means an assembler #3832 did not reach, which is a different * fact from one that declared itself uncategorised — and if that bucket is large, * the attribution below cannot be trusted. + * + * Memoized for the same reason [relayVerb] is, and more urgently: [relayPurposeDown] + * and [relayPurposeDownCount] both run on every inbound frame that names a + * subscription, so interpolating would build two strings per frame on the hottest + * path in the app. The purpose space is the enum plus the three fallbacks below. */ - fun relayPurposeSent(purpose: String): String = "relay.purpose.$purpose.sent" + fun relayPurposeSent(purpose: String): String = purposeKeys(purpose)[P_SENT] - fun relayPurposeBytes(purpose: String): String = "relay.purpose.$purpose.bytes" + fun relayPurposeBytes(purpose: String): String = purposeKeys(purpose)[P_BYTES] /** * `relay.purpose.moderation.down` — bytes received on subscriptions opened for @@ -494,7 +527,7 @@ object UsageKeys { * of the download it was causing — every re-subscription can make the relay * re-send everything that matches. */ - fun relayPurposeDown(purpose: String): String = "relay.purpose.$purpose.down" + fun relayPurposeDown(purpose: String): String = purposeKeys(purpose)[P_DOWN] /** * `relay.purpose.home_feed.downn` — inbound frames, alongside the bytes. @@ -506,7 +539,7 @@ object UsageKeys { * how much of the download is the same events arriving from different relays * under the outbox fan-out. */ - fun relayPurposeDownCount(purpose: String): String = "relay.purpose.$purpose.downn" + fun relayPurposeDownCount(purpose: String): String = purposeKeys(purpose)[P_DOWNN] /** * `relay.purpose.user_profile.dupbytes` — of that purpose's inbound bytes, how @@ -519,7 +552,20 @@ object UsageKeys { * points at completely different fixes — suppress redundant delivery, or stop * fetching the large thing per relay. */ - fun relayPurposeDupBytes(purpose: String): String = "relay.purpose.$purpose.dupbytes" + fun relayPurposeDupBytes(purpose: String): String = purposeKeys(purpose)[P_DUPBYTES] + + private const val P_SENT = 0 + private const val P_BYTES = 1 + private const val P_DOWN = 2 + private const val P_DOWNN = 3 + private const val P_DUPBYTES = 4 + + private val PURPOSE_SUFFIXES = arrayOf("sent", "bytes", "down", "downn", "dupbytes") + + private val PURPOSE_KEYS = ConcurrentHashMap>() + + /** The five `relay.purpose..*` keys, indexed by the `P_` constants above. */ + private fun purposeKeys(purpose: String): Array = PURPOSE_KEYS.getOrPut(purpose) { Array(PURPOSE_SUFFIXES.size) { "relay.purpose.$purpose.${PURPOSE_SUFFIXES[it]}" } } /** A frame whose subscription id we never saw opened — counters wiped mid-session, or a sub from before this connection. */ const val PURPOSE_UNATTRIBUTED = "unattributed" @@ -534,8 +580,17 @@ object UsageKeys { * The key segment for a [SubPurpose]. The one place the enum is turned into a * key, so the reserved-segment rename cannot drift between the producer and the * test that guards it — which is exactly how it drifted the first time. + * + * Tabled by ordinal: this runs per REQ, and `name.lowercase()` would allocate a + * fresh String each time for one of a dozen fixed answers. */ - fun purposeKeyPart(purpose: SubPurpose): String = if (purpose == SubPurpose.OTHER) PURPOSE_OTHER else purpose.name.lowercase() + fun purposeKeyPart(purpose: SubPurpose): String = PURPOSE_PARTS[purpose.ordinal] + + private val PURPOSE_PARTS = + Array(SubPurpose.entries.size) { + val purpose = SubPurpose.entries[it] + if (purpose == SubPurpose.OTHER) PURPOSE_OTHER else purpose.name.lowercase() + } /** * `relay.subs.resent.mobile.bg` — a REQ for a subscription id this relay already @@ -554,23 +609,8 @@ object UsageKeys { private val RELAY_SUBS_RESENT = dimKeys("relay.subs.resent") - /** - * Half-open bounds, in ms, for the two connect-timing histograms below. - */ - private val CONNECT_BUCKET_BOUNDS_MS = longArrayOf(100, 500, 2_000, 10_000, 30_000) - private val CONNECT_BUCKET_NAMES = - Array(CONNECT_BUCKET_BOUNDS_MS.size + 1) { i -> - if (i < CONNECT_BUCKET_BOUNDS_MS.size) "lt${CONNECT_BUCKET_BOUNDS_MS[i]}ms" else "gte${CONNECT_BUCKET_BOUNDS_MS.last()}ms" - } - - private fun connectBucketIndex(ms: Long): Int { - for (i in CONNECT_BUCKET_BOUNDS_MS.indices) { - if (ms < CONNECT_BUCKET_BOUNDS_MS[i]) return i - } - return CONNECT_BUCKET_BOUNDS_MS.size - } - - fun connectBucket(ms: Long): String = CONNECT_BUCKET_NAMES[connectBucketIndex(ms)] + /** Half-open bounds, in ms, shared by the two connect-timing histograms below. */ + private val CONNECT_BOUNDS_MS = longArrayOf(100, 500, 2_000, 10_000, 30_000) /** * `relay.hs.lt500ms.wifi.fg` — the websocket upgrade round-trip, as the @@ -589,9 +629,9 @@ object UsageKeys { ms: Long, mobile: Boolean, foreground: Boolean, - ): String = RELAY_HS[connectBucketIndex(ms)][dimIndex(mobile, foreground)] + ): String = HANDSHAKE.key(ms, mobile, foreground) - private val RELAY_HS = Array(CONNECT_BUCKET_NAMES.size) { dimKeys("relay.hs.${CONNECT_BUCKET_NAMES[it]}") } + private val HANDSHAKE = MsHistogram("relay.hs", CONNECT_BOUNDS_MS) { "${it}ms" } /** * `relay.gap.lt2000ms.wifi.fg` — everything between deciding to dial and the @@ -609,9 +649,9 @@ object UsageKeys { ms: Long, mobile: Boolean, foreground: Boolean, - ): String = RELAY_GAP[connectBucketIndex(ms)][dimIndex(mobile, foreground)] + ): String = DIAL_GAP.key(ms, mobile, foreground) - private val RELAY_GAP = Array(CONNECT_BUCKET_NAMES.size) { dimKeys("relay.gap.${CONNECT_BUCKET_NAMES[it]}") } + private val DIAL_GAP = MsHistogram("relay.gap", CONNECT_BOUNDS_MS) { "${it}ms" } /** * `relay.events.seen.wifi.fg` — inbound EVENT frames, and of those, how many @@ -757,11 +797,8 @@ object UsageKeys { */ fun Map.sumMatching(vararg parts: String): Long { var total = 0L - outer@ for ((key, value) in this) { - for (part in parts) { - if (!key.hasSegment(part)) continue@outer - } - total += value + for ((key, value) in this) { + if (parts.all { key.hasSegment(it) }) total += value } return total } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/ResourceUsageScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/ResourceUsageScreen.kt index 08bcca8f80..beeb879295 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/ResourceUsageScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/settings/ResourceUsageScreen.kt @@ -588,6 +588,14 @@ private fun SendReportSection( val clipboard = LocalClipboard.current val scope = rememberCoroutineScope() + // Suspending, and off Main: assembling the report walks every counter of every + // retained day (UsageSummary makes ~54 passes per summary) over a key space the + // churn counters grew by an order of magnitude. `scope` is Main.immediate, so a + // bare `scope.launch { }` would still build it on the UI thread — the + // withContext is what actually moves it. Only the handover (clipboard, chooser, + // navigation) stays on Main. + suspend fun buildReport(): String = withContext(Dispatchers.Default) { ResourceUsageReportAssembler().buildReport(days, today, memory) } + SettingsSection(R.string.resource_usage_send_section) { Column( modifier = Modifier.padding(16.dp), @@ -609,39 +617,41 @@ private fun SendReportSection( ) { TextButton( onClick = { - val report = ResourceUsageReportAssembler().buildReport(days, today, memory) - scope.launch { clipboard.setText(report) } + scope.launch { clipboard.setText(buildReport()) } }, ) { Text(stringRes(R.string.resource_usage_copy_button)) } TextButton( onClick = { - val report = ResourceUsageReportAssembler().buildReport(days, today, memory) - val send = - Intent().apply { - action = Intent.ACTION_SEND - type = "text/plain" - putExtra(Intent.EXTRA_TEXT, report) - putExtra(Intent.EXTRA_TITLE, stringRes(context, R.string.resource_usage_send_section)) - } - context.startActivity( - Intent.createChooser(send, stringRes(context, R.string.resource_usage_share_button)), - ) + scope.launch { + val send = + Intent().apply { + action = Intent.ACTION_SEND + type = "text/plain" + putExtra(Intent.EXTRA_TEXT, buildReport()) + putExtra(Intent.EXTRA_TITLE, stringRes(context, R.string.resource_usage_send_section)) + } + context.startActivity( + Intent.createChooser(send, stringRes(context, R.string.resource_usage_share_button)), + ) + } }, ) { Text(stringRes(R.string.resource_usage_share_button)) } Button( onClick = { - val report = ResourceUsageReportAssembler().buildReport(days, today, memory) - nav.nav { - routeToMessage( - user = LocalCache.getOrCreateUser(DEV_REPORT_PUBKEY), - draftMessage = report, - accountViewModel = accountViewModel, - expiresDays = 30, - ) + scope.launch { + val report = buildReport() + nav.nav { + routeToMessage( + user = LocalCache.getOrCreateUser(DEV_REPORT_PUBKEY), + draftMessage = report, + accountViewModel = accountViewModel, + expiresDays = 30, + ) + } } }, ) { diff --git a/amethyst/src/main/res/values/strings.xml b/amethyst/src/main/res/values/strings.xml index d8716bb5dd..f3e96dffdf 100644 --- a/amethyst/src/main/res/values/strings.xml +++ b/amethyst/src/main/res/values/strings.xml @@ -4188,7 +4188,7 @@ Chatroom lists in memory Device memory class Share with the developers - If Amethyst seems to drain battery or data, you can send this report to the developers in an encrypted DM, or copy it and share it yourself. It contains only the numbers on this screen, the technical counters behind them, and the host names of the relays that reconnected most \u2014 no posts, contacts, or browsing details. The report leaves this screen only when you send, copy, or share it. + If Amethyst seems to drain battery or data, you can send this report to the developers in an encrypted DM, or copy it and share it yourself. It contains only the numbers on this screen and the technical counters behind them \u2014 no posts, contacts, relay names, or browsing details. The report leaves this screen only when you send, copy, or share it. Send report via DM Copy Share diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageLedgerTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageLedgerTest.kt index c42b3bdada..21dd106f48 100644 --- a/amethyst/src/test/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageLedgerTest.kt +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageLedgerTest.kt @@ -1552,3 +1552,51 @@ class NoticeReasonTest { assertEquals(UsageKeys.relayNotice(UsageKeys.NOTICE_UNCLASSIFIED), UsageKeys.relayNotice("something-invented")) } } + +/** + * The report is sent as a NIP-17 DM, so its size is a correctness property, not a + * cosmetic one: relays cap events and an oversized report is simply never delivered. + */ +class ReportSizeTest { + private fun day(keys: Int) = (0 until keys).associate { "relay.purpose.p$it.bytes" to 123_456L } + + @Test + fun aFullLedgerStaysSendable() { + // 30 retained days at the density a busy day now produces. + val days = (1L..30L).associateWith { day(1_200) } + val report = ResourceUsageReportAssembler().buildReport(days, today = 30L) + + assertTrue("report was ${report.length} chars", report.length < 100_000) + assertTrue("nothing was said about the omission", report.contains("omitted to keep this report sendable")) + } + + @Test + fun aSmallLedgerIsNotTruncatedAndSaysNothingAboutOmission() { + val days = (1L..3L).associateWith { day(20) } + val report = ResourceUsageReportAssembler().buildReport(days, today = 3L) + + assertTrue(report.contains("day 1 ")) + assertTrue(report.contains("day 3 ")) + assertFalse(report.contains("omitted")) + } + + @Test + fun theNewestDayIsKeptEvenIfItAloneExceedsTheBudget() { + // Dropping everything would leave a report that says nothing at all. + val days = mapOf(1L to day(50), 2L to day(20_000)) + val report = ResourceUsageReportAssembler().buildReport(days, today = 2L) + + assertTrue("newest day missing", report.contains("day 2 ")) + assertTrue(report.contains("1 earlier day(s) omitted")) + } + + @Test + fun omittedDaysStillCountTowardTheSummaryTables() { + // The tables are built from every day; only the raw dump is bounded. + val days = (1L..30L).associateWith { mapOf(UsageKeys.relayConnects(mobile = false, foreground = true) to 10L) } + val report = ResourceUsageReportAssembler().buildReport(days, today = 30L) + + // 7 days x 10 connections in the week table. + assertTrue(report.contains("| Relay reconnections | 70 ")) + } +} From a251a69b9315a349c92acb05b01ccd6c2829732a Mon Sep 17 00:00:00 2001 From: davotoula Date: Sat, 8 Aug 2026 12:47:12 +0200 Subject: [PATCH 069/132] perf(resourceusage): keep 7 days of ledger, not 30 ResourceUsageStore.persist() rewrites the whole file on every merge, and merges fire on the accountant's 30s flush debounce while traffic flows. Only today's bucket ever changes, so retention is a write-amplification setting as much as a history setting. Nothing reads past 7 days. --- .../service/resourceusage/ResourceUsageStore.kt | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageStore.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageStore.kt index 9d56c6b881..fc63089343 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageStore.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/resourceusage/ResourceUsageStore.kt @@ -43,7 +43,22 @@ import java.io.File */ class ResourceUsageStore( private val storageFile: File, - private val keepDays: Long = 30, + /** + * Retention, in days. + * + * Seven because that is the widest window anything reads: the summary tables and + * the trend chart both span `today - 6 .. today`, the alert evaluator looks at + * two days, and the report's raw dump is byte-bounded well below a week. At 30 — + * the previous value — twenty-three days were rewritten on every flush and read + * by nothing. + * + * That is not free: [persist] rewrites the whole file on every merge, and the + * relay-churn counters took a day's bucket from ~57 keys to ~241. Retention is + * therefore a write-amplification setting as much as a history setting — cutting + * it to a week is what keeps those counters at ~18% over the previous file size + * rather than ~5x. + */ + private val keepDays: Long = 7, ) { data class UsageFile( val version: Int = 1, From ac06d4c4358dbcfb88cb54394d131305fcc1499c Mon Sep 17 00:00:00 2001 From: vitorpamplona <532031+vitorpamplona@users.noreply.github.com> Date: Sat, 8 Aug 2026 14:12:34 +0000 Subject: [PATCH 070/132] chore: sync Crowdin translations and seed translator npub placeholders --- docs/changelog/translators.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog/translators.json b/docs/changelog/translators.json index 78f466826a..0697190820 100644 --- a/docs/changelog/translators.json +++ b/docs/changelog/translators.json @@ -93,12 +93,15 @@ { "user": "vitorpamplona", "languages": [ + "Chinese Simplified", "Czech", + "Dutch", "German", "Hindi", "Hungarian", "Polish", "Portuguese, Brazilian", + "Slovenian", "Swedish" ] }, From c84de87361e11f1deefbdb2316cfe2a8dcc66093 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Sat, 8 Aug 2026 12:01:55 -0400 Subject: [PATCH 071/132] fix(concord): adopt a mid-session control_root without rebuilding the session MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A staff-making Grant delivers the `control_root` inside the fold itself (CORD-04 §3), so it lands on an entry whose session was built as a read-only member long before. `ConcordSessionRegistry.sync` only rebuilt a session when `root`/`rootEpoch` changed, and `ConcordCommunitySession` derived `controlKeys` once at construction — adoption changes neither, so the live session kept `signer = null` and `canWrite == false` for the rest of the process. The promoted staffer saw their new role badge appear (that half reads the folded `state` flow) while every write affordance stayed hidden and `controlKeysForWrite` refused, until the app was restarted. Rebuilding the session on the change is not the fix: the new session starts with no buffered Control Plane wraps, so the community folds to "No channels yet" until every wrap happens to be re-delivered. Instead refresh the key material in place. Nothing about the plane moves — adoption is gated on the secret deriving to exactly the `control_pk` already held (CORD-02 §5) — so the address, read key, buffered wraps and subscription set are all invariant, and only the signer appears. `adoptControlMaterial` fails closed on a different community/root/epoch or an address change, leaving those to a rebuild. Verified on device (SM-T220, Android 14) against a loopback geode relay: an account promoted to staff while sitting on the community screen gains the edit/create affordances with no restart, keeps its folded channel list, and its next Control edition lands on the wire signed by `control_pk`. Co-Authored-By: Claude Opus 5 (1M context) --- .../model/concord/ConcordCommunitySession.kt | 52 ++++++++++++++- .../model/concord/ConcordSessionRegistry.kt | 7 +++ .../concord/ConcordSessionRegistryTest.kt | 63 ++++++++++++++++++- 3 files changed, 117 insertions(+), 5 deletions(-) diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt index 32869407e1..7670ff69c6 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt @@ -40,6 +40,7 @@ import com.vitorpamplona.quartz.utils.TimeUtils import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.update +import kotlin.concurrent.Volatile /** * A validated inner chat rumor emitted by a session: its parent [communityId] and @@ -97,10 +98,23 @@ enum class ConcordIngestOutcome { * [ConcordActions]/[ConcordPlaneRegistry] helpers. */ class ConcordCommunitySession( - val entry: ConcordCommunityListEntry, + entry: ConcordCommunityListEntry, val myPubKey: HexKey, private val onRumor: ConcordRumorSink = { _, _, _, _ -> }, ) { + /** + * The joined-list entry this session projects. Replaced in place — only ever by + * [adoptControlMaterial], and only within one epoch — because the Control Plane + * write key can arrive long after the session was built (CORD-04 §3). A change + * that moves the *planes* (a Refounding) rebuilds the session instead. + * + * Volatile because the ingest path, the UI and the adopting drain are different + * threads: the write happens under [lock], but readers take it unsynchronized. + */ + @Volatile + var entry: ConcordCommunityListEntry = entry + private set + private val root = entry.root.hexToByteArray() private val communityIdBytes = entry.id.hexToByteArray() @@ -109,7 +123,8 @@ class ConcordCommunitySession( * carries a `control_pk` (plus the write key when this account is staff and * holds the `control_root`), legacy single-key otherwise. */ - private val controlKeys: ControlPlaneKeys = ConcordActions.controlPlaneKeysFor(entry) + @Volatile + private var controlKeys: ControlPlaneKeys = ConcordActions.controlPlaneKeysFor(entry) /** The Guestbook Plane at this epoch — where member join/leave motions ride (CORD-02 §5). */ private val guestbookKey: GroupKey = ConcordActions.guestbookPlane(root, communityIdBytes, entry.rootEpoch) @@ -329,7 +344,38 @@ class ConcordCommunitySession( * editions. [ControlPlaneKeys.canWrite] is false for a regular member on a split * epoch (CORD-02 §2) — the caller must not attempt to publish an edition then. */ - fun controlPlaneKeys(): ControlPlaneKeys = controlKeys + fun controlPlaneKeys(): ControlPlaneKeys = lock.withLock { controlKeys } + + /** + * Adopt Control Plane key material that arrived *after* this session was built, at + * the same epoch: the `control_root` a staff-making Grant delivers (CORD-04 §3), or + * a `control_pk` filled in by a same-epoch Community List merge (CORD-02 §8). + * + * Done in place rather than by rebuilding the session, because a rebuild would drop + * the buffered Control Plane wraps and leave the community folded empty until every + * wrap happened to be re-delivered. Nothing about the *plane* moves here: adoption is + * gated on the secret deriving to exactly the `control_pk` already held (CORD-02 §5), + * so the address, the read key, the buffered wraps and the subscription set are all + * invariant — only [ControlPlaneKeys.signer] appears, flipping + * [ControlPlaneKeys.canWrite] and adding the stream key to [streamKeys]. + * + * Fails closed and returns false when [newEntry] is not the same community at the + * same root and epoch, or when the material it carries would move the plane's + * address — a caller must rebuild the session for that, never mutate it. Returns + * false too when nothing changed, so the caller can skip a needless revision bump. + */ + fun adoptControlMaterial(newEntry: ConcordCommunityListEntry): Boolean = + lock.withLock { + if (newEntry.id != entry.id || newEntry.root != entry.root || newEntry.rootEpoch != entry.rootEpoch) return@withLock false + if (newEntry.controlPk == entry.controlPk && newEntry.controlRoot == entry.controlRoot) return@withLock false + val newKeys = ConcordActions.controlPlaneKeysFor(newEntry) + // The plane is where the buffered wraps already are. If the new material points + // somewhere else, this is not an adoption — refuse and let the caller rebuild. + if (newKeys.address != controlKeys.address) return@withLock false + entry = newEntry + controlKeys = newKeys + true + } /** This account's standing, from the current fold. */ fun membership(): ConcordMembership { diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionRegistry.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionRegistry.kt index cb366ba6fa..09be414805 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionRegistry.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionRegistry.kt @@ -78,6 +78,13 @@ class ConcordSessionRegistry( if (existing == null || existing.entry.root != entry.root || existing.entry.rootEpoch != entry.rootEpoch) { sessions[id] = ConcordCommunitySession(entry, myPubKey, onRumor) created += id + } else { + // Same epoch, but the Control Plane write key may have just arrived — a + // staff-making Grant delivers it inside the fold itself (CORD-04 §3), long + // after this session was built. Adopt it in place: rebuilding would drop the + // buffered wraps and fold the community empty, and the plane's address is + // invariant under adoption anyway (CORD-02 §5). + existing.adoptControlMaterial(entry) } } created diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionRegistryTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionRegistryTest.kt index a8bedadc73..81519fd100 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionRegistryTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordSessionRegistryTest.kt @@ -30,8 +30,10 @@ import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal import kotlinx.coroutines.test.runTest import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertFalse import kotlin.test.assertNotNull import kotlin.test.assertNull +import kotlin.test.assertSame import kotlin.test.assertTrue class ConcordSessionRegistryTest { @@ -40,14 +42,16 @@ class ConcordSessionRegistryTest { private fun entryFor( community: NewConcordCommunity, name: String, + controlRoot: String? = community.controlRoot.toHexKey(), + rootEpoch: Long = community.rootEpoch, ) = ConcordCommunityListEntry( id = community.communityIdHex, owner = community.ownerPubKey, ownerSalt = community.ownerSalt.toHexKey(), root = community.communityRoot.toHexKey(), - rootEpoch = community.rootEpoch, + rootEpoch = rootEpoch, controlPk = community.controlPkHex, - controlRoot = community.controlRoot.toHexKey(), + controlRoot = controlRoot, relays = listOf("wss://r.example"), name = name, ) @@ -104,4 +108,59 @@ class ConcordSessionRegistryTest { val gamma = ConcordCommunityFactory.create(owner, "Gamma", createdAt = 1L, relays = listOf("wss://r.example")) assertEquals(ConcordIngestOutcome.NOT_MINE, registry.ingest(gamma.genesisWraps.first())) } + + /** + * A promotion to staff delivers the `control_root` inside the fold itself (CORD-04 §3), + * so it lands on an entry whose session was built as a read-only member long before. The + * session must pick the key up **without** being rebuilt: a rebuild would drop the + * buffered Control Plane wraps and fold the community empty, which is exactly what the + * user would see instead of their new moderation powers. + */ + @Test + fun adoptsAControlRootDeliveredMidSessionWithoutLosingTheFold() = + runTest { + val community = ConcordCommunityFactory.create(owner, "Alpha", createdAt = 1L, relays = listOf("wss://r.example")) + val registry = ConcordSessionRegistry() + + // Joined as a plain member: the address is held, the write secret is not. + val asMember = entryFor(community, "Alpha", controlRoot = null) + registry.sync(listOf(asMember), owner.pubKey) + val session = registry.sessionFor(community.communityIdHex)!! + community.genesisWraps.forEach { registry.ingest(it) } + + assertEquals( + "Alpha", + session.state.value + ?.metadata + ?.name, + ) + assertFalse(session.controlPlaneKeys().canWrite, "a member holds no control_root") + + // The staff-making Grant lands and the drain writes the secret onto the entry. + val asStaff = entryFor(community, "Alpha") + val createdOnAdopt = registry.sync(listOf(asStaff), owner.pubKey) + + // Adopted in place: same session object, no rebuild. + assertTrue(createdOnAdopt.isEmpty(), "adopting a control_root must not rebuild the session") + assertSame(session, registry.sessionFor(community.communityIdHex)) + + // The write key is live... + assertTrue(session.controlPlaneKeys().canWrite, "the delivered control_root must flip canWrite") + assertEquals(community.controlRoot.toHexKey(), session.entry.controlRoot, "the entry carries it onward for the next Grant") + assertTrue(session.streamKeys().any { it.publicKeyHex == community.controlPkHex }, "staff now AUTHs as the plane") + + // ...and the address and the fold are untouched — the bug this guards. + assertEquals(community.controlPlane.address, session.controlPlaneAddress) + assertEquals( + "Alpha", + session.state.value + ?.metadata + ?.name, + "adoption must not discard the buffered wraps", + ) + + // A real rotation still rebuilds rather than adopting in place. + val nextEpoch = entryFor(community, "Alpha", rootEpoch = community.rootEpoch + 1) + assertEquals(setOf(community.communityIdHex), registry.sync(listOf(nextEpoch), owner.pubKey)) + } } From 9ce0fb9559339295ac0e421b1e99ad85f7be3a8a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 17:43:11 +0000 Subject: [PATCH 072/132] fix(quartz): update NIP-66 relay records instead of rebuilding them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A kind:30166 is addressable, so RelayReachabilityStore keeps exactly one record per (monitor, relay) — but it is not necessarily the only thing writing per-relay knowledge under that identity. Both write paths built the record from their own tags and inserted it, so every update deleted whatever else was in that slot. Observed while adding a "this url is an alias of that one" tag alongside the monitor: `[d, n, rtt-open]` became `[d, redirect]` on our write, and the monitor's next observation turned it back into `[d, n, rtt-open]`. Nothing looks wrong at any point — the event still signs, still parses, still reads as a valid NIP-66 record. It just says less than it did, and the reader downstream cannot tell. Writing is now an edit: read this monitor's current record, carry across every tag the writer does not own — including tags this version of quartz has never heard of — and replace only what it measured. `n` and the three `rtt-*` types are owned by both paths, so a dead update still clears a stale rtt and liveness keeps meaning what it meant. `R` is owned only by the observation path, which is the one that learns whether a relay challenged us; writeOne leaves it alone rather than deleting what it cannot re-measure. Only OUR records are merged. Folding another monitor's tags into a document signed with this key would republish their claims as ours. The timestamp is now `max(now, current + 1)` rather than `now`. A store enforcing replaceable semantics REJECTS a record that is not strictly newer than the one it replaces, and two writers inside the same second — or a peer whose clock runs ahead — are ordinary. That is not theoretical: it silently swallowed a repair pass in the caller that found this bug, which reported success having written nothing. The reads are batched per call rather than per relay, so a flush over N relays costs one extra query, not N. Test plan: ./gradlew :quartz:jvmTest — 4,071 tests, all passing, including four new cases in RelayReachabilityStoreTest covering a foreign tag surviving an update, an update against a record stamped an hour ahead, a dead update clearing its rtt, and another monitor's record not being merged. ./gradlew :quartz:spotlessApply clean. --- .../reachability/RelayReachabilityStore.kt | 104 +++++++++++++++-- .../RelayReachabilityStoreTest.kt | 106 ++++++++++++++++++ 2 files changed, 201 insertions(+), 9 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStore.kt index fe28f94954..772fd3524a 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStore.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.quartz.nip66RelayMonitor.reachability +import com.vitorpamplona.quartz.nip01Core.core.TagArrayBuilder import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer @@ -30,6 +31,8 @@ import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.networkType import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.requirement import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.rtt import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.NetworkType +import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.NetworkTypeTag +import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.RequirementTag import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.RttType import com.vitorpamplona.quartz.utils.TimeUtils @@ -139,8 +142,9 @@ class RelayReachabilityStore( now: Long = TimeUtils.now(), rttOpenMs: Long = 0, ) { - for (relay in reachable) writeOne(relay, up = true, now, rttOpenMs) - for (relay in dead) if (relay !in reachable) writeOne(relay, up = false, now, rttOpenMs) + val current = currentRecords(reachable + dead) + for (relay in reachable) writeOne(relay, up = true, now, rttOpenMs, current[relay]) + for (relay in dead) if (relay !in reachable) writeOne(relay, up = false, now, rttOpenMs, current[relay]) } /** @@ -154,8 +158,9 @@ class RelayReachabilityStore( dead: Set, now: Long = TimeUtils.now(), ) { - for ((relay, rtt) in reachableRttMs) writeOne(relay, up = true, now, rtt.coerceAtLeast(0)) - for (relay in dead) if (relay !in reachableRttMs) writeOne(relay, up = false, now, 0) + val current = currentRecords(reachableRttMs.keys + dead) + for ((relay, rtt) in reachableRttMs) writeOne(relay, up = true, now, rtt.coerceAtLeast(0), current[relay]) + for (relay in dead) if (relay !in reachableRttMs) writeOne(relay, up = false, now, 0, current[relay]) } /** @@ -171,10 +176,11 @@ class RelayReachabilityStore( observations: Collection, now: Long = TimeUtils.now(), ): Int { + val reported = observations.filter { it.reachable || it.error != null } + val current = currentRecords(reported.map { it.url }) var written = 0 - for (o in observations) { - if (!o.reachable && o.error == null) continue - writeObserved(o, now) + for (o in reported) { + writeObserved(o, now, current[o.url]) written++ } return written @@ -183,9 +189,14 @@ class RelayReachabilityStore( private suspend fun writeObserved( o: RelayObserver.Observation, now: Long, + current: RelayDiscoveryEvent?, ) { + // `R` is included in the owned set here and NOT in [writeOne]: an + // observation knows whether this relay challenged us, so it may clear a + // requirement that no longer holds. writeOne never learns that, so it + // leaves the tag alone rather than deleting what it cannot re-measure. val template = - RelayDiscoveryEvent.build(o.url, createdAt = now) { + edit(o.url, now, current, OWNED_LIVENESS + RequirementTag.TAG_NAME) { networkType(networkTypeOf(o.url)) if (o.reachable) { // Liveness is the presence of rtt-open, per NIP-66. A relay we @@ -210,16 +221,91 @@ class RelayReachabilityStore( up: Boolean, now: Long, rttOpenMs: Long, + current: RelayDiscoveryEvent?, ) { val template = - RelayDiscoveryEvent.build(relay, createdAt = now) { + edit(relay, now, current, OWNED_LIVENESS) { networkType(networkTypeOf(relay)) if (up) rtt(RttType.OPEN, rttOpenMs) } store.insert(signer.sign(template)) } + /** + * Build this monitor's next record for [relay] as an EDIT of [current] + * rather than a fresh document. + * + * A 30166 is addressable, so a relay has exactly one record per monitor — + * and this class is not necessarily its only writer. Anything else keeping + * per-relay knowledge under the same identity (an operator marking a relay + * as a mirror of another, a crawler recording which kinds it served) writes + * into this same slot, and a build-from-scratch silently deletes it. The + * result still signs, still parses, and still reads as a valid NIP-66 + * record — it just says less than it did, and the reader downstream has no + * way to know something was lost. + * + * [owned] is what this writer measured and may therefore replace. + * Everything else is carried across untouched, including tags this version + * of quartz has never heard of. + * + * The timestamp is `max(now, current + 1)`, not `now`: a store enforcing + * replaceable semantics REJECTS a record that is not strictly newer than + * the one it replaces, and two writers inside the same second — or a peer + * whose clock runs ahead of ours — are ordinary. An update lost that way is + * indistinguishable from one that had nothing to say. + */ + private fun edit( + relay: NormalizedRelayUrl, + now: Long, + current: RelayDiscoveryEvent?, + owned: Set, + measured: TagArrayBuilder.() -> Unit, + ) = RelayDiscoveryEvent.build( + relay, + current?.content ?: "", + createdAt = maxOf(now, (current?.createdAt ?: 0L) + 1), + ) { + current?.tags?.forEach { tag -> + if (tag.firstOrNull() != "d" && tag.firstOrNull() !in owned) add(tag) + } + measured() + } + + /** + * This monitor's own current record for each relay, in one query. + * + * Only OUR records: merging another monitor's tags into a document signed + * with this key would republish their claims as ours. + */ + private suspend fun currentRecords(relays: Collection): Map { + if (relays.isEmpty()) return emptyMap() + val held = + store.query( + Filter( + kinds = listOf(RelayDiscoveryEvent.KIND), + authors = listOf(signer.pubKey), + tags = mapOf("d" to relays.map { it.url }.distinct()), + ), + ) + val out = HashMap(held.size) + for (ev in held) { + val relay = ev.relay() ?: continue + val seen = out[relay] + if (seen == null || ev.createdAt > seen.createdAt) out[relay] = ev + } + return out + } + companion object { + /** + * The tags this class measures on every write, and may therefore + * replace. A dead record must be able to CLEAR a stale rtt — liveness + * is the presence of `rtt-open` — so all three rtt types are owned even + * though only `rtt-open` is written by every path. + */ + private val OWNED_LIVENESS = + setOf(NetworkTypeTag.TAG_NAME, RttType.OPEN.tagName, RttType.READ.tagName, RttType.WRITE.tagName) + /** Default freshness window: a relay's status is trusted for a day, then re-probed. */ const val DEFAULT_TTL_SECONDS = 24L * 60 * 60 diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStoreTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStoreTest.kt index 2117163f78..8b338bd72e 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStoreTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStoreTest.kt @@ -21,11 +21,15 @@ package com.vitorpamplona.quartz.nip66RelayMonitor.reachability import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore +import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.RelayDiscoveryEvent import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.NetworkType +import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.RttType import kotlinx.coroutines.runBlocking import kotlin.test.Test import kotlin.test.assertEquals @@ -110,4 +114,106 @@ class RelayReachabilityStoreTest { // A host that merely contains ".onion" as a substring is clearnet, not Tor. assertEquals(NetworkType.CLEARNET, RelayReachabilityStore.networkTypeOf(fakeOnion)) } + // ---- one address, more than one writer --------------------------------- + + private suspend fun tagsOf( + store: EventStore, + signer: NostrSignerInternal, + relay: NormalizedRelayUrl, + ): List> = + store + .query( + Filter(kinds = listOf(RelayDiscoveryEvent.KIND), authors = listOf(signer.pubKey), tags = mapOf("d" to listOf(relay.url))), + ).maxByOrNull { it.createdAt } + ?.tags + ?.toList() + .orEmpty() + + private fun names(tags: List>) = tags.mapNotNull { it.firstOrNull() }.toSet() + + /** + * A 30166 is addressable, so this monitor has one record per relay — and it + * is not necessarily the only thing writing per-relay knowledge under that + * identity. A record rebuilt from this writer's own tags deletes the rest, + * and the loss is invisible: the event still signs and still parses. + */ + @Test + fun `an update keeps tags this writer does not own`() = + runBlocking { + val store = store() + val signer = NostrSignerInternal(KeyPair()) + val cache = RelayReachabilityStore(store, signer, ttlSeconds = 3600) + + cache.recordProbed(mapOf(live1 to 120L), emptySet(), now = 1_700_000_000) + // Something else records what it knows about the same relay, + // keeping what the monitor already put there. + val existing = tagsOf(store, signer, live1) + val withExtra = + RelayDiscoveryEvent.build(live1, "", createdAt = 1_700_000_001) { + existing.forEach { if (it.firstOrNull() != "d") add(it) } + add(arrayOf("redirect", "wss://canonical.example.com/")) + } + store.insert(signer.sign(withExtra)) + + cache.recordProbed(mapOf(live1 to 131L), emptySet(), now = 1_700_000_002) + + val after = tagsOf(store, signer, live1) + assertTrue("redirect" in names(after), "the update erased another writer's tag: ${names(after)}") + // ...and still replaced what it does own. + assertEquals("131", after.first { it[0] == RttType.OPEN.tagName }[1]) + } + + /** + * A store enforcing replaceable semantics rejects a record that is not + * strictly newer than the one it replaces. Two writers inside one second, + * or a peer whose clock runs ahead, are ordinary — and an update lost that + * way looks exactly like one that had nothing to say. + */ + @Test + fun `an update lands even when the record it replaces is newer than the clock`() = + runBlocking { + val store = store() + val signer = NostrSignerInternal(KeyPair()) + val cache = RelayReachabilityStore(store, signer, ttlSeconds = 3600) + + cache.recordProbed(mapOf(live1 to 120L), emptySet(), now = 1_700_003_600) + cache.recordProbed(mapOf(live1 to 131L), emptySet(), now = 1_700_000_000) + + assertEquals("131", tagsOf(store, signer, live1).first { it[0] == RttType.OPEN.tagName }[1]) + } + + /** A relay that went down must lose its rtt, or it still reads as live. */ + @Test + fun `a dead update clears the rtt it replaces`() = + runBlocking { + val store = store() + val signer = NostrSignerInternal(KeyPair()) + val cache = RelayReachabilityStore(store, signer, ttlSeconds = 3600) + + cache.recordProbed(mapOf(live1 to 120L), emptySet(), now = 1_700_000_000) + cache.record(reachable = emptySet(), dead = setOf(live1), now = 1_700_000_100) + + assertTrue(RttType.OPEN.tagName !in names(tagsOf(store, signer, live1))) + assertTrue(cache.snapshot(now = 1_700_000_200).isKnownDead(live1)) + } + + /** Only OUR records merge: republishing another monitor's tags under this key would launder their claims. */ + @Test + fun `another monitor's record is not merged into ours`() = + runBlocking { + val store = store() + val signer = NostrSignerInternal(KeyPair()) + val other = NostrSignerInternal(KeyPair()) + val cache = RelayReachabilityStore(store, signer, ttlSeconds = 3600) + + val theirs = + RelayDiscoveryEvent.build(live1, "", createdAt = 1_700_000_000) { + add(arrayOf("redirect", "wss://not-ours.example.com/")) + } + store.insert(other.sign(theirs)) + + cache.recordProbed(mapOf(live1 to 120L), emptySet(), now = 1_700_000_100) + + assertTrue("redirect" !in names(tagsOf(store, signer, live1))) + } } From a81c43e1d4c32ae206077f3b14051ae6da49e54a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 18:04:15 +0000 Subject: [PATCH 073/132] fix(quartz): address code-review findings on the record merge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five issues from a review pass on the previous commit, all in the new merge path. currentRecords() bound one SQL host parameter per relay with no chunking. Callers pass the whole relay universe — RelayProber's own measurement puts that at 16,507 — and a bundled SQLite refuses past 32,766 variables. The throw lands BEFORE anything is written, so an entire probe run's records are lost rather than one relay's. Chunked at 500, in the same range as the author chunking elsewhere. The created_at bump had no ceiling, so a stamp that once landed in the future was sticky: every later edit derived from the bad value and never re-anchored to now. Such a record never ages out of snapshot()'s TTL window (an isKnownDead verdict that can never expire) and relays enforcing future-timestamp limits reject every publish for it. Capped at 60s past now — a pathological record now costs the updates made while the clock catches up, and heals itself. writeOne owned all three rtt names but only ever measures rtt-open, so the reachable path deleted rtt-read/rtt-write taken by an observation — the exact silent loss this change exists to stop. It now owns rtt-open alone; only the dead path clears them all, which liveness semantics require. writeObserved owned the whole R tag name but can only prove `auth`, so it erased `R pow` and friends written by RelayProber. Ownership is now per VALUE, which is why edit() takes a predicate rather than a set of names. The read-modify-write spans a store round trip and IEventStore exposes no read inside a transaction, so a concurrent writer to the same address can still win the race and get our stale insert rejected. That cannot be closed at this layer; it is now isolated per relay so one loser does not end the loop and silently drop every relay after it. Test plan: ./gradlew :quartz:jvmTest — 4,075 tests, all passing. Four new cases, one per fixable finding: a flush wider than one chunk writing every relay, a far-future record not being pushed further ahead, a reachable update keeping latencies it never measured, and an observation clearing only `auth`. --- .../reachability/RelayReachabilityStore.kt | 138 +++++++++++++----- .../RelayReachabilityStoreTest.kt | 92 +++++++++++- 2 files changed, 189 insertions(+), 41 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStore.kt index 772fd3524a..0c993461b1 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStore.kt @@ -143,8 +143,8 @@ class RelayReachabilityStore( rttOpenMs: Long = 0, ) { val current = currentRecords(reachable + dead) - for (relay in reachable) writeOne(relay, up = true, now, rttOpenMs, current[relay]) - for (relay in dead) if (relay !in reachable) writeOne(relay, up = false, now, rttOpenMs, current[relay]) + for (relay in reachable) writeSafely { writeOne(relay, up = true, now, rttOpenMs, current[relay]) } + for (relay in dead) if (relay !in reachable) writeSafely { writeOne(relay, up = false, now, rttOpenMs, current[relay]) } } /** @@ -159,8 +159,8 @@ class RelayReachabilityStore( now: Long = TimeUtils.now(), ) { val current = currentRecords(reachableRttMs.keys + dead) - for ((relay, rtt) in reachableRttMs) writeOne(relay, up = true, now, rtt.coerceAtLeast(0), current[relay]) - for (relay in dead) if (relay !in reachableRttMs) writeOne(relay, up = false, now, 0, current[relay]) + for ((relay, rtt) in reachableRttMs) writeSafely { writeOne(relay, up = true, now, rtt.coerceAtLeast(0), current[relay]) } + for (relay in dead) if (relay !in reachableRttMs) writeSafely { writeOne(relay, up = false, now, 0, current[relay]) } } /** @@ -180,23 +180,44 @@ class RelayReachabilityStore( val current = currentRecords(reported.map { it.url }) var written = 0 for (o in reported) { - writeObserved(o, now, current[o.url]) - written++ + if (writeSafely { writeObserved(o, now, current[o.url]) }) written++ } return written } + /** + * Run one relay's write, keeping its failure to that relay. + * + * The read-modify-write below spans a store round trip and [IEventStore] + * offers no read inside a transaction, so a concurrent writer to the same + * address can still win the race — and on a store enforcing replaceable + * semantics our now-stale insert is REJECTED. Unisolated, that one throw + * ends the loop and drops every relay after it; the run reports fewer + * records than it measured and nothing says why. + */ + private inline fun writeSafely(write: () -> Unit): Boolean = + try { + write() + true + } catch (e: Exception) { + false + } + private suspend fun writeObserved( o: RelayObserver.Observation, now: Long, current: RelayDiscoveryEvent?, ) { - // `R` is included in the owned set here and NOT in [writeOne]: an - // observation knows whether this relay challenged us, so it may clear a - // requirement that no longer holds. writeOne never learns that, so it - // leaves the tag alone rather than deleting what it cannot re-measure. + // Owns the `auth` REQUIREMENT VALUE, not the whole `R` tag name: an + // observation learns whether this relay challenged us and may clear + // that, but `R payment`, `R pow` and the negated forms — written by + // RelayProber among others — are somebody else's measurement. val template = - edit(o.url, now, current, OWNED_LIVENESS + RequirementTag.TAG_NAME) { + edit(o.url, now, current, { tag -> + tag.firstOrNull() == NetworkTypeTag.TAG_NAME || + tag.firstOrNull() in ALL_RTT || + (tag.firstOrNull() == RequirementTag.TAG_NAME && tag.getOrNull(1) == AUTH_REQUIREMENT) + }) { networkType(networkTypeOf(o.url)) if (o.reachable) { // Liveness is the presence of rtt-open, per NIP-66. A relay we @@ -211,7 +232,7 @@ class RelayReachabilityStore( // Observed, not read off NIP-11: this relay actually challenged // us. A relay advertising open reads and then demanding AUTH is // exactly what a monitor exists to catch. - if (o.authRequired) requirement("auth") + if (o.authRequired) requirement(AUTH_REQUIREMENT) } store.insert(signer.sign(template)) } @@ -223,8 +244,19 @@ class RelayReachabilityStore( rttOpenMs: Long, current: RelayDiscoveryEvent?, ) { + // Owns every rtt only when writing a DEAD record: liveness is the + // presence of rtt-open, so a relay that went down must lose all of + // them. On the reachable path it owns rtt-open alone — deleting a + // `rtt-read` this call never measured is the same silent loss this + // whole change exists to stop. + val owned = + if (up) { + { tag: Array -> tag.firstOrNull() == NetworkTypeTag.TAG_NAME || tag.firstOrNull() == RttType.OPEN.tagName } + } else { + { tag: Array -> tag.firstOrNull() == NetworkTypeTag.TAG_NAME || tag.firstOrNull() in ALL_RTT } + } val template = - edit(relay, now, current, OWNED_LIVENESS) { + edit(relay, now, current, owned) { networkType(networkTypeOf(relay)) if (up) rtt(RttType.OPEN, rttOpenMs) } @@ -244,29 +276,40 @@ class RelayReachabilityStore( * record — it just says less than it did, and the reader downstream has no * way to know something was lost. * - * [owned] is what this writer measured and may therefore replace. - * Everything else is carried across untouched, including tags this version - * of quartz has never heard of. + * [owns] decides what this writer measured and may therefore replace — a + * predicate rather than a set of names, because ownership is sometimes per + * VALUE: an observation may clear `R auth` without touching the `R pow` + * another writer measured. Everything it does not claim is carried across + * untouched, including tags this version of quartz has never heard of. * * The timestamp is `max(now, current + 1)`, not `now`: a store enforcing * replaceable semantics REJECTS a record that is not strictly newer than * the one it replaces, and two writers inside the same second — or a peer - * whose clock runs ahead of ours — are ordinary. An update lost that way is - * indistinguishable from one that had nothing to say. + * whose clock runs slightly ahead — are ordinary. An update lost that way + * is indistinguishable from one that had nothing to say. + * + * That bump is CAPPED at [MAX_FUTURE_SKEW_SECONDS] past `now`. Without a + * ceiling a `created_at` that once landed in the future is sticky: every + * later edit derives from the bad value and never re-anchors, so the record + * never ages out of [snapshot]'s TTL window (a stale `isKnownDead` that can + * never expire) and relays enforcing future-timestamp limits reject + * everything this monitor publishes for that relay. Capped, a pathological + * record costs the updates made while `now` catches up — bounded, and it + * heals itself — instead of poisoning the slot permanently. */ private fun edit( relay: NormalizedRelayUrl, now: Long, current: RelayDiscoveryEvent?, - owned: Set, + owns: (Array) -> Boolean, measured: TagArrayBuilder.() -> Unit, ) = RelayDiscoveryEvent.build( relay, current?.content ?: "", - createdAt = maxOf(now, (current?.createdAt ?: 0L) + 1), + createdAt = minOf(maxOf(now, (current?.createdAt ?: 0L) + 1), now + MAX_FUTURE_SKEW_SECONDS), ) { current?.tags?.forEach { tag -> - if (tag.firstOrNull() != "d" && tag.firstOrNull() !in owned) add(tag) + if (tag.firstOrNull() != "d" && !owns(tag)) add(tag) } measured() } @@ -279,32 +322,47 @@ class RelayReachabilityStore( */ private suspend fun currentRecords(relays: Collection): Map { if (relays.isEmpty()) return emptyMap() - val held = - store.query( - Filter( - kinds = listOf(RelayDiscoveryEvent.KIND), - authors = listOf(signer.pubKey), - tags = mapOf("d" to relays.map { it.url }.distinct()), - ), - ) - val out = HashMap(held.size) - for (ev in held) { - val relay = ev.relay() ?: continue - val seen = out[relay] - if (seen == null || ev.createdAt > seen.createdAt) out[relay] = ev + val out = HashMap() + // CHUNKED: a `d` filter binds one host parameter per url, and callers + // pass the whole relay universe — RelayProber's own measurement puts + // that at 16,507. A bundled SQLite refuses past 32,766 variables, and + // the throw would land BEFORE anything was written, losing an entire + // probe run's records rather than one relay's. + for (chunk in relays.map { it.url }.distinct().chunked(RELAYS_PER_QUERY)) { + val held = + store.query( + Filter(kinds = listOf(RelayDiscoveryEvent.KIND), authors = listOf(signer.pubKey), tags = mapOf("d" to chunk)), + ) + for (ev in held) { + val relay = ev.relay() ?: continue + val seen = out[relay] + if (seen == null || ev.createdAt > seen.createdAt) out[relay] = ev + } } return out } companion object { + /** Every rtt tag name. A DEAD record must clear all of them: liveness is the presence of `rtt-open`. */ + private val ALL_RTT = setOf(RttType.OPEN.tagName, RttType.READ.tagName, RttType.WRITE.tagName) + + /** The one NIP-66 requirement an observation can prove: the relay challenged us. */ + const val AUTH_REQUIREMENT = "auth" + /** - * The tags this class measures on every write, and may therefore - * replace. A dead record must be able to CLEAR a stale rtt — liveness - * is the presence of `rtt-open` — so all three rtt types are owned even - * though only `rtt-open` is written by every path. + * How far past `now` an edit may stamp itself to clear a record that + * is already ahead of the clock. Enough to cover ordinary skew between + * two writers; small enough that a pathological record heals in + * minutes rather than never. See [edit]. */ - private val OWNED_LIVENESS = - setOf(NetworkTypeTag.TAG_NAME, RttType.OPEN.tagName, RttType.READ.tagName, RttType.WRITE.tagName) + const val MAX_FUTURE_SKEW_SECONDS = 60L + + /** + * Urls per `d` lookup. Well under a bundled SQLite's 32,766-variable + * ceiling, and in the same range as the author chunking elsewhere in + * this codebase. + */ + const val RELAYS_PER_QUERY = 500 /** Default freshness window: a relay's status is trusted for a day, then re-probed. */ const val DEFAULT_TTL_SECONDS = 24L * 60 * 60 diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStoreTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStoreTest.kt index 8b338bd72e..3d5422230a 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStoreTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStoreTest.kt @@ -29,6 +29,7 @@ import com.vitorpamplona.quartz.nip01Core.store.sqlite.DefaultIndexingStrategy import com.vitorpamplona.quartz.nip01Core.store.sqlite.EventStore import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.RelayDiscoveryEvent import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.NetworkType +import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.RequirementTag import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.RttType import kotlinx.coroutines.runBlocking import kotlin.test.Test @@ -176,12 +177,101 @@ class RelayReachabilityStoreTest { val signer = NostrSignerInternal(KeyPair()) val cache = RelayReachabilityStore(store, signer, ttlSeconds = 3600) - cache.recordProbed(mapOf(live1 to 120L), emptySet(), now = 1_700_003_600) + // Ordinary skew: the record ahead of our clock by seconds, which is + // what two writers or a slightly fast peer produce. + cache.recordProbed(mapOf(live1 to 120L), emptySet(), now = 1_700_000_010) cache.recordProbed(mapOf(live1 to 131L), emptySet(), now = 1_700_000_000) assertEquals("131", tagsOf(store, signer, live1).first { it[0] == RttType.OPEN.tagName }[1]) } + /** + * Without a ceiling the bump is sticky: a record that once landed in the + * future is derived from forever, so it never ages out of the TTL window + * and relays enforcing future-timestamp limits reject every publish. + */ + @Test + fun `a record already far in the future is never pushed further ahead`() = + runBlocking { + val store = store() + val signer = NostrSignerInternal(KeyPair()) + val cache = RelayReachabilityStore(store, signer, ttlSeconds = 3600) + val now = 1_700_000_000L + + cache.recordProbed(mapOf(live1 to 120L), emptySet(), now = now + 86_400) + cache.recordProbed(mapOf(live1 to 131L), emptySet(), now = now) + + val held = + store + .query( + Filter(kinds = listOf(RelayDiscoveryEvent.KIND), authors = listOf(signer.pubKey), tags = mapOf("d" to listOf(live1.url))), + ).maxByOrNull { it.createdAt } + assertEquals(now + 86_400, held?.createdAt, "the pathological stamp was carried forward instead of capped") + } + + /** writeOne measures rtt-open only; deleting a read/write latency it never took is the loss this guards. */ + @Test + fun `a reachable update keeps latencies it did not measure`() = + runBlocking { + val store = store() + val signer = NostrSignerInternal(KeyPair()) + val cache = RelayReachabilityStore(store, signer, ttlSeconds = 3600) + + val observed = RelayObserver() + observed.record(live1, true, 100L, null) + observed.observationOf(live1)?.rttReadMs = 55L + cache.record(observed.collectUnreported(), now = 1_700_000_000) + assertTrue(RttType.READ.tagName in names(tagsOf(store, signer, live1))) + + cache.recordProbed(mapOf(live1 to 120L), emptySet(), now = 1_700_000_100) + + val after = tagsOf(store, signer, live1) + assertEquals("55", after.first { it[0] == RttType.READ.tagName }[1], "rtt-read was deleted by a writer that never measured it") + assertEquals("120", after.first { it[0] == RttType.OPEN.tagName }[1]) + } + + /** An observation proves `R auth` and nothing else; other requirements belong to whoever measured them. */ + @Test + fun `an observation clears only the auth requirement`() = + runBlocking { + val store = store() + val signer = NostrSignerInternal(KeyPair()) + val cache = RelayReachabilityStore(store, signer, ttlSeconds = 3600) + + cache.recordProbed(mapOf(live1 to 120L), emptySet(), now = 1_700_000_000) + val existing = tagsOf(store, signer, live1) + val withReqs = + RelayDiscoveryEvent.build(live1, "", createdAt = 1_700_000_001) { + existing.forEach { if (it.firstOrNull() != "d") add(it) } + add(arrayOf(RequirementTag.TAG_NAME, "pow")) + add(arrayOf(RequirementTag.TAG_NAME, RelayReachabilityStore.AUTH_REQUIREMENT)) + } + store.insert(signer.sign(withReqs)) + + // Reached without a challenge: auth no longer holds, pow was never ours. + val observed = RelayObserver() + observed.record(live1, true, 100L, null) + cache.record(observed.collectUnreported(), now = 1_700_000_002) + + val after = tagsOf(store, signer, live1).filter { it[0] == RequirementTag.TAG_NAME }.map { it[1] } + assertTrue("pow" in after, "another writer's requirement was erased: " + after) + assertTrue(RelayReachabilityStore.AUTH_REQUIREMENT !in after, "auth should have been cleared: " + after) + } + + /** One `d` filter binds one host parameter per url, and callers pass the whole relay universe. */ + @Test + fun `a flush wider than one query chunk still writes every relay`() = + runBlocking { + val store = store() + val signer = NostrSignerInternal(KeyPair()) + val cache = RelayReachabilityStore(store, signer, ttlSeconds = 3600) + val many = (0 until RelayReachabilityStore.RELAYS_PER_QUERY * 2 + 7).map { RelayUrlNormalizer.normalize("wss://r" + it + ".example.com") } + + cache.record(reachable = many.toSet(), dead = emptySet(), now = 1_700_000_000) + + assertEquals(many.size, cache.snapshot(now = 1_700_000_100).live.size) + } + /** A relay that went down must lose its rtt, or it still reads as live. */ @Test fun `a dead update clears the rtt it replaces`() = From 56eec420e43d43e269007709204fe638a7b315ae Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 18:35:57 +0000 Subject: [PATCH 074/132] fix(quartz): correct the merge's ownership, guard and timestamp rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second review pass on the merge itself. Three of these reverse choices made in the previous commit; the reasoning there was wrong. The created_at cap is gone. Capping the bump to a window past `now` looked prudent and was worse: a record already further ahead than the cap can then never be replaced, because every stamp we are willing to write is older than what is stored, so the relay's live/dead verdict freezes until the wall clock catches up — 24h in the test that shipped asserting that behaviour as correct. It did not even buy the freshness it claimed: snapshot() selects on `since` alone, so a future-stamped record sits inside the window either way. A record ahead of the clock is a defect in whatever produced it; this class's job is to keep updating it. Ownership is now the full liveness set on every write — `n`, all three rtt types, and both polarities of `R auth` — rather than the narrower per-path sets. A 30166 carries ONE created_at, so a tag carried across is re-dated as a current measurement: keeping a rtt-read from an earlier observation beside a fresh rtt-open republishes a stale latency as today's, which aggregators rank on, and RelayObserver documents exactly how wrong a queued rtt can be. Carrying `R auth` forward was worse still — only an observation can clear it and that needs the connection the flag discourages, so it became permanent, a regression against the rebuild this PR replaced. Owning only the positive auth form also let `R !auth` survive while `requirement("auth")` appended the opposite, publishing a record asserting both. The per-relay guard no longer swallows. It caught Exception, which includes CancellationException, so a shutdown flush wrapped in withTimeout — the pattern RelayMonitor.close() prescribes — could not abort and would grind through every remaining relay. And a caught failure went nowhere: collectUnreported() has already cleared the observation flags by then, so the measurement is lost for good while the run reports success. Cancellation now propagates, every relay is still attempted, and the first real failure is rethrown once the loop finishes. Also corrected a comment: the 16,507-relay figure is measured in RelayObserver, not RelayProber, and the SQLite ceiling is verified here rather than quoted — 32,765 `d` values pass, 32,766 fails. Test plan: ./gradlew :quartz:jvmTest — 4,078 tests, all passing. Four new cases: a future-stamped record still updatable, a stale rtt-read not re-dated, an auth wall not outliving its observation, and a run whose writes all fail reporting failure instead of success. --- .../reachability/RelayReachabilityStore.kt | 155 +++++++++++------- .../RelayReachabilityStoreTest.kt | 101 ++++++++++-- 2 files changed, 180 insertions(+), 76 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStore.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStore.kt index 0c993461b1..dace6321f6 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStore.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStore.kt @@ -35,6 +35,7 @@ import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.NetworkTypeTag import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.RequirementTag import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.RttType import com.vitorpamplona.quartz.utils.TimeUtils +import kotlinx.coroutines.CancellationException /** * A durable, shareable relay-reachability cache backed by an [IEventStore] as @@ -143,8 +144,10 @@ class RelayReachabilityStore( rttOpenMs: Long = 0, ) { val current = currentRecords(reachable + dead) - for (relay in reachable) writeSafely { writeOne(relay, up = true, now, rttOpenMs, current[relay]) } - for (relay in dead) if (relay !in reachable) writeSafely { writeOne(relay, up = false, now, rttOpenMs, current[relay]) } + val up = reachable.toList() + val down = dead.filterNot { it in reachable } + eachRelay(up.size) { i -> writeOne(up[i], up = true, now, rttOpenMs, current[up[i]]) } + eachRelay(down.size) { i -> writeOne(down[i], up = false, now, rttOpenMs, current[down[i]]) } } /** @@ -159,8 +162,10 @@ class RelayReachabilityStore( now: Long = TimeUtils.now(), ) { val current = currentRecords(reachableRttMs.keys + dead) - for ((relay, rtt) in reachableRttMs) writeSafely { writeOne(relay, up = true, now, rtt.coerceAtLeast(0), current[relay]) } - for (relay in dead) if (relay !in reachableRttMs) writeSafely { writeOne(relay, up = false, now, 0, current[relay]) } + val up = reachableRttMs.toList() + val down = dead.filterNot { it in reachableRttMs } + eachRelay(up.size) { i -> writeOne(up[i].first, up = true, now, up[i].second.coerceAtLeast(0), current[up[i].first]) } + eachRelay(down.size) { i -> writeOne(down[i], up = false, now, 0, current[down[i]]) } } /** @@ -178,46 +183,53 @@ class RelayReachabilityStore( ): Int { val reported = observations.filter { it.reachable || it.error != null } val current = currentRecords(reported.map { it.url }) - var written = 0 - for (o in reported) { - if (writeSafely { writeObserved(o, now, current[o.url]) }) written++ - } - return written + return eachRelay(reported.size) { i -> writeObserved(reported[i], now, current[reported[i].url]) } } /** - * Run one relay's write, keeping its failure to that relay. + * Run every relay's write, then fail if any of them did. * - * The read-modify-write below spans a store round trip and [IEventStore] - * offers no read inside a transaction, so a concurrent writer to the same - * address can still win the race — and on a store enforcing replaceable - * semantics our now-stale insert is REJECTED. Unisolated, that one throw - * ends the loop and drops every relay after it; the run reports fewer - * records than it measured and nothing says why. + * The read-modify-write spans a store round trip and [IEventStore] offers + * no read inside a transaction, so a concurrent writer to the same address + * can win the race and our now-stale insert is REJECTED. One such throw + * must not end the loop and drop every relay after it — but it must not + * vanish either: [RelayObserver.collectUnreported] has already cleared the + * flags by the time this runs, so a swallowed failure loses the + * measurement for good and the caller cannot tell an empty run from a + * failed one. So: attempt all, remember the first failure, rethrow it. + * + * Cancellation is never caught. A shutdown flush wrapped in `withTimeout` + * — the pattern [RelayMonitor.close] prescribes — would otherwise be + * unabortable, grinding through every remaining relay with each write + * throwing and being swallowed. */ - private inline fun writeSafely(write: () -> Unit): Boolean = - try { - write() - true - } catch (e: Exception) { - false + private suspend inline fun eachRelay( + count: Int, + write: (Int) -> Unit, + ): Int { + var first: Exception? = null + var written = 0 + for (i in 0 until count) { + try { + write(i) + written++ + } catch (e: CancellationException) { + throw e + } catch (e: Exception) { + if (first == null) first = e + } } + first?.let { throw it } + return written + } private suspend fun writeObserved( o: RelayObserver.Observation, now: Long, current: RelayDiscoveryEvent?, ) { - // Owns the `auth` REQUIREMENT VALUE, not the whole `R` tag name: an - // observation learns whether this relay challenged us and may clear - // that, but `R payment`, `R pow` and the negated forms — written by - // RelayProber among others — are somebody else's measurement. val template = - edit(o.url, now, current, { tag -> - tag.firstOrNull() == NetworkTypeTag.TAG_NAME || - tag.firstOrNull() in ALL_RTT || - (tag.firstOrNull() == RequirementTag.TAG_NAME && tag.getOrNull(1) == AUTH_REQUIREMENT) - }) { + edit(o.url, now, current, ::ownsLiveness) { networkType(networkTypeOf(o.url)) if (o.reachable) { // Liveness is the presence of rtt-open, per NIP-66. A relay we @@ -244,19 +256,8 @@ class RelayReachabilityStore( rttOpenMs: Long, current: RelayDiscoveryEvent?, ) { - // Owns every rtt only when writing a DEAD record: liveness is the - // presence of rtt-open, so a relay that went down must lose all of - // them. On the reachable path it owns rtt-open alone — deleting a - // `rtt-read` this call never measured is the same silent loss this - // whole change exists to stop. - val owned = - if (up) { - { tag: Array -> tag.firstOrNull() == NetworkTypeTag.TAG_NAME || tag.firstOrNull() == RttType.OPEN.tagName } - } else { - { tag: Array -> tag.firstOrNull() == NetworkTypeTag.TAG_NAME || tag.firstOrNull() in ALL_RTT } - } val template = - edit(relay, now, current, owned) { + edit(relay, now, current, ::ownsLiveness) { networkType(networkTypeOf(relay)) if (up) rtt(RttType.OPEN, rttOpenMs) } @@ -288,14 +289,15 @@ class RelayReachabilityStore( * whose clock runs slightly ahead — are ordinary. An update lost that way * is indistinguishable from one that had nothing to say. * - * That bump is CAPPED at [MAX_FUTURE_SKEW_SECONDS] past `now`. Without a - * ceiling a `created_at` that once landed in the future is sticky: every - * later edit derives from the bad value and never re-anchors, so the record - * never ages out of [snapshot]'s TTL window (a stale `isKnownDead` that can - * never expire) and relays enforcing future-timestamp limits reject - * everything this monitor publishes for that relay. Capped, a pathological - * record costs the updates made while `now` catches up — bounded, and it - * heals itself — instead of poisoning the slot permanently. + * The bump is deliberately NOT capped to some window past `now`. Capping + * it looks prudent and is worse: a record already further ahead than the + * cap can then never be replaced at all, because every stamp we are willing + * to write is older than what is stored, so the relay's live/dead verdict + * freezes until the wall clock catches up. It does not even buy the thing + * it appears to — [snapshot] selects on `since` alone, so a future-stamped + * record sits inside the freshness window either way. A record stamped + * ahead of the clock is a defect in whatever produced it; this class's job + * is to keep updating it, not to freeze it. */ private fun edit( relay: NormalizedRelayUrl, @@ -306,7 +308,7 @@ class RelayReachabilityStore( ) = RelayDiscoveryEvent.build( relay, current?.content ?: "", - createdAt = minOf(maxOf(now, (current?.createdAt ?: 0L) + 1), now + MAX_FUTURE_SKEW_SECONDS), + createdAt = maxOf(now, (current?.createdAt ?: 0L) + 1), ) { current?.tags?.forEach { tag -> if (tag.firstOrNull() != "d" && !owns(tag)) add(tag) @@ -314,6 +316,35 @@ class RelayReachabilityStore( measured() } + /** + * The tags this class measures, and may therefore replace. + * + * Everything here expires together with the record: a 30166 carries ONE + * `created_at` for the whole document, so a tag carried across is re-dated + * as a current measurement. Keeping a `rtt-read` from an earlier + * observation beside a fresh `rtt-open` would republish a stale latency as + * today's — and [RelayObserver] documents exactly how wrong a queued rtt + * can be. So this class's own liveness facts are rewritten wholesale on + * every write, including clearing `R auth` when nothing re-asserts it: a + * permanent auth flag is worse than a missing one, because it discourages + * the very connection that could clear it. + * + * Both polarities of the auth requirement are owned. Owning only the + * positive form let `R !auth` survive while `requirement("auth")` appended + * the opposite, publishing a record that asserted both at once. + * + * Everything NOT matched here — `R pow`, `R payment`, annotations another + * writer keeps on this address, tags this version has never heard of — is + * somebody else's measurement and is carried across untouched. + */ + private fun ownsLiveness(tag: Array): Boolean = + when (tag.firstOrNull()) { + NetworkTypeTag.TAG_NAME -> true + in ALL_RTT -> true + RequirementTag.TAG_NAME -> tag.getOrNull(1) in AUTH_REQUIREMENT_FORMS + else -> false + } + /** * This monitor's own current record for each relay, in one query. * @@ -324,10 +355,13 @@ class RelayReachabilityStore( if (relays.isEmpty()) return emptyMap() val out = HashMap() // CHUNKED: a `d` filter binds one host parameter per url, and callers - // pass the whole relay universe — RelayProber's own measurement puts - // that at 16,507. A bundled SQLite refuses past 32,766 variables, and - // the throw would land BEFORE anything was written, losing an entire - // probe run's records rather than one relay's. + // pass the whole relay universe — the fan-out this module measures + // itself against is 16,507 relays (see RelayObserver). Measured on + // BundledSQLiteDriver, 32,765 `d` values pass and 32,766 fails with + // "too many SQL variables"; the throw lands BEFORE anything is + // written, so an entire probe run's records are lost rather than one + // relay's. The headroom here is deliberate — the ceiling is a property + // of the driver, not of this query. for (chunk in relays.map { it.url }.distinct().chunked(RELAYS_PER_QUERY)) { val held = store.query( @@ -349,13 +383,8 @@ class RelayReachabilityStore( /** The one NIP-66 requirement an observation can prove: the relay challenged us. */ const val AUTH_REQUIREMENT = "auth" - /** - * How far past `now` an edit may stamp itself to clear a record that - * is already ahead of the clock. Enough to cover ordinary skew between - * two writers; small enough that a pathological record heals in - * minutes rather than never. See [edit]. - */ - const val MAX_FUTURE_SKEW_SECONDS = 60L + /** Both polarities, so an update cannot leave the record asserting `auth` and `!auth` at once. */ + private val AUTH_REQUIREMENT_FORMS = setOf(AUTH_REQUIREMENT, "!$AUTH_REQUIREMENT") /** * Urls per `d` lookup. Well under a bundled SQLite's 32,766-variable diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStoreTest.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStoreTest.kt index 3d5422230a..073f961f14 100644 --- a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStoreTest.kt +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayReachabilityStoreTest.kt @@ -186,12 +186,16 @@ class RelayReachabilityStoreTest { } /** - * Without a ceiling the bump is sticky: a record that once landed in the - * future is derived from forever, so it never ages out of the TTL window - * and relays enforcing future-timestamp limits reject every publish. + * A record stamped ahead of the clock is a defect in whatever produced it. + * Capping our stamp to some window past `now` looks prudent and is worse: + * every stamp we would write is then older than what is stored, so the + * insert is rejected and the relay's live/dead verdict freezes until the + * wall clock catches up. It does not even buy freshness — snapshot() + * selects on `since` alone, so the future record is inside the window + * either way. */ @Test - fun `a record already far in the future is never pushed further ahead`() = + fun `a record stamped ahead of the clock can still be updated`() = runBlocking { val store = store() val signer = NostrSignerInternal(KeyPair()) @@ -201,17 +205,17 @@ class RelayReachabilityStoreTest { cache.recordProbed(mapOf(live1 to 120L), emptySet(), now = now + 86_400) cache.recordProbed(mapOf(live1 to 131L), emptySet(), now = now) - val held = - store - .query( - Filter(kinds = listOf(RelayDiscoveryEvent.KIND), authors = listOf(signer.pubKey), tags = mapOf("d" to listOf(live1.url))), - ).maxByOrNull { it.createdAt } - assertEquals(now + 86_400, held?.createdAt, "the pathological stamp was carried forward instead of capped") + assertEquals("131", tagsOf(store, signer, live1).first { it[0] == RttType.OPEN.tagName }[1]) } - /** writeOne measures rtt-open only; deleting a read/write latency it never took is the loss this guards. */ + /** + * A 30166 carries ONE created_at, so a carried tag is re-dated as a current + * measurement. This class's own liveness facts must therefore be rewritten + * wholesale, or a stale rtt-read is republished as today's number — which + * aggregators rank on. + */ @Test - fun `a reachable update keeps latencies it did not measure`() = + fun `a later observation does not re-date an older latency`() = runBlocking { val store = store() val signer = NostrSignerInternal(KeyPair()) @@ -226,10 +230,81 @@ class RelayReachabilityStoreTest { cache.recordProbed(mapOf(live1 to 120L), emptySet(), now = 1_700_000_100) val after = tagsOf(store, signer, live1) - assertEquals("55", after.first { it[0] == RttType.READ.tagName }[1], "rtt-read was deleted by a writer that never measured it") + assertTrue(RttType.READ.tagName !in names(after), "a stale rtt-read was carried onto a fresh record") assertEquals("120", after.first { it[0] == RttType.OPEN.tagName }[1]) } + /** + * Only [writeObserved] can clear `auth`, and it needs a connection the flag + * discourages — so carrying it forward would make it permanent. It expires + * with the rest of this class's liveness facts. + */ + @Test + fun `an auth requirement does not outlive the observation that set it`() = + runBlocking { + val store = store() + val signer = NostrSignerInternal(KeyPair()) + val cache = RelayReachabilityStore(store, signer, ttlSeconds = 3600) + + val walled = RelayObserver() + walled.record(live1, true, 100L, null) + walled.observationOf(live1)?.authRequired = true + cache.record(walled.collectUnreported(), now = 1_700_000_000) + assertTrue(RequirementTag.TAG_NAME in names(tagsOf(store, signer, live1))) + + cache.recordProbed(mapOf(live1 to 120L), emptySet(), now = 1_700_000_100) + + assertTrue(RequirementTag.TAG_NAME !in names(tagsOf(store, signer, live1)), "the auth wall became permanent") + } + + /** Owning only the positive form left `!auth` in place while appending `auth`. */ + @Test + fun `an update never leaves the record asserting both auth polarities`() = + runBlocking { + val store = store() + val signer = NostrSignerInternal(KeyPair()) + val cache = RelayReachabilityStore(store, signer, ttlSeconds = 3600) + + cache.recordProbed(mapOf(live1 to 120L), emptySet(), now = 1_700_000_000) + val existing = tagsOf(store, signer, live1) + val negated = + RelayDiscoveryEvent.build(live1, "", createdAt = 1_700_000_001) { + existing.forEach { if (it.firstOrNull() != "d") add(it) } + add(arrayOf(RequirementTag.TAG_NAME, "!auth")) + } + store.insert(signer.sign(negated)) + + val walled = RelayObserver() + walled.record(live1, true, 100L, null) + walled.observationOf(live1)?.authRequired = true + cache.record(walled.collectUnreported(), now = 1_700_000_002) + + val reqs = tagsOf(store, signer, live1).filter { it[0] == RequirementTag.TAG_NAME }.map { it[1] } + assertEquals(listOf(RelayReachabilityStore.AUTH_REQUIREMENT), reqs, "record asserts contradictory requirements: " + reqs) + } + + /** + * collectUnreported() has already cleared the flags by the time a write + * runs, so a swallowed failure loses the measurement for good and an empty + * run is indistinguishable from a failed one. + */ + @Test + fun `a failing write is reported, not swallowed`() = + runBlocking { + val store = store() + val signer = NostrSignerInternal(KeyPair()) + val cache = RelayReachabilityStore(store, signer, ttlSeconds = 3600) + store.close() + + var threw = false + try { + cache.recordProbed(mapOf(live1 to 120L, live2 to 130L), emptySet(), now = 1_700_000_000) + } catch (e: Exception) { + threw = true + } + assertTrue(threw, "every write failed and the run reported success") + } + /** An observation proves `R auth` and nothing else; other requirements belong to whoever measured them. */ @Test fun `an observation clears only the auth requirement`() = From 4b49032e52ee4357d95afbfbad6f20de30a0a567 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 18:59:17 +0000 Subject: [PATCH 075/132] fix(quartz): merge probe verdicts into the record they replace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #3882, which made RelayReachabilityStore edit a relay's kind:30166 rather than rebuild it. toDiscoveryEventTemplate was the remaining co-writer: it builds from the verdict alone, so a consumer following its own KDoc — sign with the monitor key, insert — wipes whatever else is on that address, undoing the merge for exactly the writer #3882 set out to protect. It now takes the current record and carries across every tag the verdict did not measure, on the same rules: - Ownership is per writer, and this one measures more than the store does. A write probe determines `pow` from the OK message, so `R pow` is its own finding and must not be re-dated from an older record. Without a ReadWriteVerdict it never exercised the write path, so the same tag is somebody else's and is carried across untouched — hence the hasReadWrite flag rather than a fixed set. - Both polarities of each requirement are owned, so an update cannot leave the record asserting `pow` and `!pow` at once. - created_at is max(requested, current + 1): a store enforcing replaceable semantics rejects anything not strictly newer, and the probe would be lost with nothing to show for the round trip. The parameter defaults to null, so every existing caller keeps today's behaviour and the change is additive. Test plan: ./gradlew :quartz:jvmTest — 4,081 tests, all passing. Three new cases in RelayProberFlowTest: a foreign tag and an unmeasured `R pow` surviving a probe without a write verdict, a stale `R pow` being replaced when the write path DID run, and the stamp landing past the record it replaces. --- .../reachability/RelayProber.kt | 49 ++++++++++++++++- .../reachability/RelayProberFlowTest.kt | 55 +++++++++++++++++++ 2 files changed, 101 insertions(+), 3 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt index 711c38551a..9c56c43c99 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProber.kt @@ -39,6 +39,8 @@ import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.RelayDiscoveryEvent import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.networkType import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.requirement import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.rtt +import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.NetworkTypeTag +import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.RequirementTag import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.tags.RttType import com.vitorpamplona.quartz.utils.TimeUtils import com.vitorpamplona.quartz.utils.concurrent.ConcurrentMap @@ -430,8 +432,18 @@ class RelayProber( fun RelayProber.Verdict.toDiscoveryEventTemplate( createdAt: Long = TimeUtils.now(), readWrite: RelayProber.ReadWriteVerdict? = null, + current: RelayDiscoveryEvent? = null, ): EventTemplate = - RelayDiscoveryEvent.build(relay, createdAt = createdAt) { + RelayDiscoveryEvent.build( + relay, + current?.content ?: "", + // Strictly newer than what it replaces, or a store enforcing + // replaceable semantics rejects it and the probe is lost silently. + createdAt = maxOf(createdAt, (current?.createdAt ?: 0L) + 1), + ) { + current?.tags?.forEach { tag -> + if (tag.firstOrNull() != "d" && !probeOwns(tag, readWrite != null)) add(tag) + } networkType(RelayReachabilityStore.networkTypeOf(relay)) if (reachable) rtt(RttType.OPEN, rttOpenMs.coerceAtLeast(0)) if (readWrite != null) { @@ -441,6 +453,37 @@ fun RelayProber.Verdict.toDiscoveryEventTemplate( val authWalled = error?.startsWith("closed:auth-required") == true || readWrite?.writeMessage?.startsWith("auth-required") == true - if (authWalled) requirement("auth") - if (readWrite?.writeMessage?.startsWith("pow:") == true) requirement("pow") + if (authWalled) requirement(RelayReachabilityStore.AUTH_REQUIREMENT) + if (readWrite?.writeMessage?.startsWith("pow:") == true) requirement(POW_REQUIREMENT) + } + +/** The NIP-66 requirement a write probe can prove, alongside `auth`. */ +private const val POW_REQUIREMENT = "pow" + +/** + * What a probe verdict measured, and may therefore replace in [current]. + * + * A 30166 carries ONE `created_at`, so any tag carried across is re-dated as a + * current measurement — this verdict's own facts must be rewritten wholesale or + * a stale latency is republished as today's number. + * + * [hasReadWrite] narrows it: without a [RelayProber.ReadWriteVerdict] this probe + * never exercised the write path, so `R pow` is somebody else's finding and is + * carried across rather than deleted. Both polarities of each requirement are + * owned, so an update cannot leave the record asserting `pow` and `!pow` at once. + */ +private fun probeOwns( + tag: Array, + hasReadWrite: Boolean, +): Boolean = + when (tag.firstOrNull()) { + NetworkTypeTag.TAG_NAME -> true + RttType.OPEN.tagName, RttType.READ.tagName, RttType.WRITE.tagName -> true + RequirementTag.TAG_NAME -> + when (tag.getOrNull(1)) { + RelayReachabilityStore.AUTH_REQUIREMENT, "!" + RelayReachabilityStore.AUTH_REQUIREMENT -> true + POW_REQUIREMENT, "!" + POW_REQUIREMENT -> hasReadWrite + else -> false + } + else -> false } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt index f0a6be04ca..05bc37fc08 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip66RelayMonitor/reachability/RelayProberFlowTest.kt @@ -34,6 +34,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer import com.vitorpamplona.quartz.nip01Core.signers.EventTemplate import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal +import com.vitorpamplona.quartz.nip66RelayMonitor.discovery.RelayDiscoveryEvent import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.delay import kotlinx.coroutines.launch @@ -472,4 +473,58 @@ class RelayProberFlowTest { assertTrue(listOf("n", "tor") in tagsOf(template)) } + // ---- merging into an existing record ---------------------------------- + + /** + * A 30166 is addressable, so a consumer that follows this function's KDoc — + * sign with the monitor key, insert — replaces whatever else is on that + * address. Built from the verdict alone it deletes it. + */ + @Test + fun probeTemplateKeepsTagsItDidNotMeasure() { + val verdict = RelayProber.Verdict(fast, reachable = true, rttOpenMs = 120, rttEoseMs = 200, error = null) + val existing = + RelayDiscoveryEvent( + "id", + "pubkey", + 1_000, + arrayOf( + arrayOf("d", fast.url), + arrayOf("R", "pow"), + arrayOf("redirect", "wss://canonical.example.com/"), + ), + "", + "sig", + ) + + val tags = tagsOf(verdict.toDiscoveryEventTemplate(createdAt = 2_000, current = existing)) + + assertTrue(listOf("redirect", "wss://canonical.example.com/") in tags, "a foreign tag was deleted: $tags") + // No write probe ran, so `R pow` is somebody else's finding. + assertTrue(listOf("R", "pow") in tags, "an unmeasured requirement was deleted: $tags") + } + + /** With a write verdict the probe DOES measure pow, so a stale one must not be re-dated. */ + @Test + fun probeTemplateReplacesRequirementsItDidMeasure() { + val verdict = RelayProber.Verdict(fast, reachable = true, rttOpenMs = 120, rttEoseMs = 200, error = null) + val existing = + RelayDiscoveryEvent("id", "pubkey", 1_000, arrayOf(arrayOf("d", fast.url), arrayOf("R", "pow")), "", "sig") + val clean = RelayProber.ReadWriteVerdict(fast, rttReadMs = 10, rttWriteMs = 20, writeAccepted = true, writeMessage = null) + + val tags = tagsOf(verdict.toDiscoveryEventTemplate(createdAt = 2_000, readWrite = clean, current = existing)) + + assertTrue(listOf("R", "pow") !in tags, "a stale requirement was carried onto a fresh measurement: $tags") + } + + /** Replaceable ordering: an update not strictly newer is rejected and lost. */ + @Test + fun probeTemplateStampsPastTheRecordItReplaces() { + val verdict = RelayProber.Verdict(fast, reachable = true, rttOpenMs = 120, rttEoseMs = 200, error = null) + val existing = RelayDiscoveryEvent("id", "pubkey", 9_000, arrayOf(arrayOf("d", fast.url)), "", "sig") + + val template = verdict.toDiscoveryEventTemplate(createdAt = 2_000, current = existing) + + assertTrue(template.createdAt > 9_000, "stamped ${template.createdAt}, which cannot replace 9000") + } } From f2160f626425d5edaf6edb2a62add8644826ef45 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 21:02:37 +0000 Subject: [PATCH 076/132] test(quartz): pin what a soft-banned Concord staffer can still do MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CORD-04 §4 row 3 of docs/concord-banlist-rank-conformance.md was left open as "a genuine fixpoint-ordering question". This reproduces what that gap costs. ConcordCommunityState.fold gates METADATA/CHANNEL/INVITE through authority.hasPermission (`!isBanned && ..`), but ROLE, GRANT and BANLIST are gated inside AuthorityResolver.resolve by holdsManageRoles / bitsOf / effectivePermissionsOf, none of which consult the banlist — and none of which can, as written, since the roles/grants fixpoint settles before `banned` is computed. So half the Control Plane honors a ban and half is blind to it. A banned member who still holds control_root therefore keeps the roster: they revoke the surviving moderators, retire the roles beneath them, ban everyone they outrank, and — since a role edition they author is honored — mint a fresh, unbanned npub at the next position down. That npub passes every ban-aware gate, so it tombstones the channels (terminal ids), rewrites the metadata, and, being a non-banned BAN holder, is accepted as a rotator by drainConcordRekeys. The tests assert the CURRENT, VULNERABLE behaviour so it cannot regress silently; each ESCALATION assertion is to be inverted, not deleted, when the ordering rule lands. Two companions pin what the fix must preserve: self-unban and puppet-unban both stay refused, closed already by the delta rank rule. Also records why a chain-local fix is insufficient — forking the banlist at genesis dodges any "was the author banned by this edition's parent" rule, and §4's re-heal union carries the rogue bans in anyway. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DrJhpFhhLjuDJQNkGvYMGj --- .../cord04Roles/BannedStaffEscalationTest.kt | 276 ++++++++++++++++++ 1 file changed, 276 insertions(+) create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/BannedStaffEscalationTest.kt diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/BannedStaffEscalationTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/BannedStaffEscalationTest.kt new file mode 100644 index 0000000000..8ad22aba6b --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/BannedStaffEscalationTest.kt @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.concord.cord04Roles + +import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityState +import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +/** + * What a **soft-banned staffer** can still do to a community — the reproduction behind + * `docs/concord-banlist-rank-conformance.md` §4 row 3, which the report left open as "a genuine + * fixpoint-ordering question, not a plain oversight". + * + * The asymmetry these tests pin: [ConcordCommunityState.fold] gates METADATA / CHANNEL / INVITE + * through `authority.hasPermission`, which is `!isBanned && …`, but ROLE, GRANT and BANLIST are + * gated *inside* [AuthorityResolver.resolve] by `holdsManageRoles` / `bitsOf` / + * `effectivePermissionsOf` — none of which consult the banlist. Nor could they as written: the + * roles/grants fixpoint runs before `banned` is computed at all. So half the Control Plane honors + * a ban and half is structurally blind to it, and a banned member who still holds `control_root` + * keeps full authority over the roster. + * + * **These tests assert the CURRENT, VULNERABLE behaviour**, so the escalation cannot regress + * silently or be "fixed" by accident without someone noticing. Every `ESCALATION:` assertion here + * must be INVERTED — not deleted — when the ordering rule lands. [selfUnbanIsStillRefused] and + * [aJuniorPuppetCannotLiftASeniorsBan] are the opposite: they pin behaviour the fix must preserve. + * + * Note for whoever writes that fix: a chain-local rule ("the author must not be banned by the state + * their edition chains from") is NOT sufficient — see + * [aBannedAdminForksTheBanlistAtGenesisRatherThanChainingOntoTheirOwnBan]. The rule has to bind + * CORD-04 §4's re-heal union too. + */ +class BannedStaffEscalationTest { + private val owner = "0f".repeat(32) + private val alice = "a1".repeat(32) // Admin, position 1 — the member who gets banned + private val bob = "b2".repeat(32) // Mod, position 5 + private val carol = "c3".repeat(32) // plain member, no role + private val puppet = "e5".repeat(32) // a fresh npub alice controls + + private val adminRole = "11".repeat(32) + private val modRole = "22".repeat(32) + private val puppetRole = "33".repeat(32) + + private val banlistEntity = "44".repeat(32) + private val channelEntity = "55".repeat(32) + private val metadataEntity = "66".repeat(32) + private val bobGrantEntity = "32".repeat(32) + private val puppetGrantEntity = "35".repeat(32) + + // MANAGE_ROLES|MANAGE_CHANNELS|MANAGE_METADATA|KICK|BAN|CREATE_INVITE = 1+2+4+8+16+64 + private val adminJson = """{"name":"Admin","position":1,"permissions":"95"}""" + private val modJson = """{"name":"Mod","position":5,"permissions":"24"}""" // KICK|BAN + + private fun edition( + kind: ControlEntityKind, + entity: String, + version: Long, + prev: ByteArray?, + content: String, + author: String, + rumorId: String, + ) = ControlEdition(kind, entity.hexToByteArray(), version, prev, null, content, author, rumorId, 0) + + private fun role( + id: String, + json: String, + author: String = owner, + version: Long = 0, + prev: ByteArray? = null, + ) = edition(ControlEntityKind.ROLE, id, version, prev, json, author, "role-$id-$version-$author") + + private fun grant( + coordinate: String, + member: String, + roleIds: List, + author: String, + version: Long = 0, + prev: ByteArray? = null, + ) = edition( + ControlEntityKind.GRANT, + coordinate, + version, + prev, + """{"member":"$member","role_ids":[${roleIds.joinToString(",") { "\"$it\"" }}]}""", + author, + "grant-$coordinate-$version-$author", + ) + + private fun banlist( + author: String, + version: Long, + prev: ByteArray?, + vararg banned: String, + ) = edition( + ControlEntityKind.BANLIST, + banlistEntity, + version, + prev, + "[${banned.joinToString(",") { "\"$it\"" }}]", + author, + "ban-$version-$author", + ) + + private fun channel( + json: String, + author: String, + version: Long, + prev: ByteArray?, + ) = edition(ControlEntityKind.CHANNEL, channelEntity, version, prev, json, author, "chan-$version-$author") + + private fun metadata( + json: String, + author: String, + version: Long, + prev: ByteArray?, + ) = edition(ControlEntityKind.METADATA, metadataEntity, version, prev, json, author, "meta-$version-$author") + + private val channelV0 = channel("""{"name":"general"}""", owner, 0, null) + private val metadataV0 = metadata("""{"name":"My Community"}""", owner, 0, null) + private val bobGrantV0 = grant(bobGrantEntity, bob, listOf(modRole), owner) + private val modRoleV0 = role(modRole, modJson) + + /** The owner-authored community every test starts from: two roles, two grants, a channel, metadata. */ + private fun community() = + mutableListOf( + role(adminRole, adminJson), + modRoleV0, + grant("31".repeat(32), alice, listOf(adminRole), owner), + bobGrantV0, + channelV0, + metadataV0, + ) + + /** The owner bans alice. Genesis of the banlist, so every test can fork or chain off it. */ + private val ownerBansAlice = banlist(owner, 0, null, alice) + + /** Alice, already banned, mints a role just below herself and hands it to a fresh npub. */ + private fun aliceMintsAPuppet() = + listOf( + role(puppetRole, """{"name":"Puppet","position":2,"permissions":"95"}""", author = alice), + grant(puppetGrantEntity, puppet, listOf(puppetRole), author = alice), + ) + + @Test + fun aBanStripsTheAuthorityCheckedByFoldButNotTheOneCheckedByTheResolver() { + val r = AuthorityResolver.resolve(community() + ownerBansAlice, owner) + + assertTrue(r.isBanned(alice), "the owner's ban lands") + assertFalse(r.hasPermission(alice, ConcordPermissions.MANAGE_ROLES), "the ban-aware check refuses her") + // ...but this is the one every ROLE/GRANT/BANLIST gate inside resolve() actually consults. + assertTrue( + r.effectivePermissions(alice).has(ConcordPermissions.MANAGE_ROLES), + "ESCALATION: a banned staffer keeps the permissions the resolver's own gates read", + ) + } + + @Test + fun aBannedAdminPromotesAFreshSockpuppetToAdmin() { + val r = AuthorityResolver.resolve(community() + ownerBansAlice + aliceMintsAPuppet(), owner) + + assertEquals(2, r.rank(puppet), "ESCALATION: the banned admin's role edition is honored") + assertFalse(r.isBanned(puppet), "the puppet is a clean npub — nothing to filter it on") + assertTrue( + r.hasPermission(puppet, ConcordPermissions.MANAGE_CHANNELS), + "ESCALATION: a banned member minted a live admin with the ban-aware check passing", + ) + } + + @Test + fun theSockpuppetDeletesEveryChannelAndRewritesTheMetadata() { + val editions = + community() + ownerBansAlice + aliceMintsAPuppet() + + // A channel tombstone is terminal — CORD-03: the id is never reused. + channel("""{"name":"general","deleted":true}""", puppet, 1, channelV0.hash) + + metadata("""{"name":"Owned by the guy you banned"}""", puppet, 1, metadataV0.hash) + + val state = ConcordCommunityState.fold(editions, owner) + + assertEquals(0, state.channels.size, "ESCALATION: the community's channels are irrecoverably tombstoned") + assertEquals("Owned by the guy you banned", state.metadata?.name, "ESCALATION: and its identity rewritten") + } + + @Test + fun theSockpuppetBansEveryMemberBeneathIt() { + val editions = community() + ownerBansAlice + aliceMintsAPuppet() + banlist(puppet, 1, ownerBansAlice.hash, alice, bob, carol) + + val r = AuthorityResolver.resolve(editions, owner) + + assertTrue(r.isBanned(bob), "ESCALATION: the surviving moderator is silenced, losing all authority with it") + assertTrue(r.isBanned(carol), "ESCALATION: and the plain members with them") + } + + @Test + fun aBannedAdminBansEveryoneBeneathThemWithoutNeedingAPuppetAtAll() { + val editions = community() + ownerBansAlice + banlist(alice, 1, ownerBansAlice.hash, alice, bob, carol) + + val r = AuthorityResolver.resolve(editions, owner) + + assertTrue(r.isBanned(bob), "ESCALATION: banGate reads effectivePermissionsOf, which ignores her own ban") + assertTrue(r.isBanned(carol), "ESCALATION: same") + } + + @Test + fun aBannedAdminForksTheBanlistAtGenesisRatherThanChainingOntoTheirOwnBan() { + // The same attack as above, except her edition does NOT chain onto the edition that banned + // her — it forks at genesis. So a rule that only asks "was the author banned by this + // edition's parent?" never sees her ban, and CORD-04 §4's re-heal union carries her bans in + // regardless. Any fix has to bind the union, not just the chain. + val editions = community() + ownerBansAlice + banlist(alice, 0, null, bob, carol) + + val r = AuthorityResolver.resolve(editions, owner) + + assertTrue(r.isBanned(alice), "the owner's ban survives the fork — the union is down-only") + assertTrue(r.isBanned(bob), "ESCALATION: and so does the banned admin's, healed in as a concurrent ban") + assertTrue(r.isBanned(carol), "ESCALATION: same") + } + + @Test + fun aBannedAdminRevokesTheSurvivingModerators() { + val editions = community() + ownerBansAlice + grant(bobGrantEntity, bob, emptyList(), author = alice, version = 1, prev = bobGrantV0.hash) + + val r = AuthorityResolver.resolve(editions, owner) + + assertEquals(null, r.rank(bob), "ESCALATION: a banned admin stripped a live moderator's roles") + assertFalse(r.hasPermission(bob, ConcordPermissions.BAN), "ESCALATION: leaving nobody but the owner able to act") + } + + @Test + fun aBannedAdminDeletesEveryRoleBeneathThem() { + val tombstone = role(modRole, """{"name":"Mod","position":5,"permissions":"24","deleted":true}""", author = alice, version = 1, prev = modRoleV0.hash) + + val r = AuthorityResolver.resolve(community() + ownerBansAlice + tombstone, owner) + + assertEquals(null, r.roles()[modRole], "ESCALATION: a banned admin retired a role beneath them") + assertEquals(null, r.rank(bob), "ESCALATION: every holder of it silently loses their standing") + } + + @Test + fun selfUnbanIsStillRefused() { + // docs/concord-banlist-rank-conformance.md §4 row 3, the half that IS closed: the delta rule + // gates removals too, and strict outranking means nobody outranks themselves. + val editions = community() + ownerBansAlice + banlist(alice, 1, ownerBansAlice.hash) + + assertTrue(AuthorityResolver.resolve(editions, owner).isBanned(alice), "a banned member may not lift their own ban") + } + + @Test + fun aJuniorPuppetCannotLiftASeniorsBan() { + // The puppet sits at position 2 and alice at 1, and no edition may claim a position at or + // above its own signer — so her delegation chain can only ever descend. Nothing she mints + // can outrank her, and so nothing she mints can unban her. + val editions = community() + ownerBansAlice + aliceMintsAPuppet() + banlist(puppet, 1, ownerBansAlice.hash) + + assertTrue(AuthorityResolver.resolve(editions, owner).isBanned(alice), "the puppet does not outrank its creator") + } +} From fb7c710a88a6940ff783dbbd92025e4d326e312b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 21:11:54 +0000 Subject: [PATCH 077/132] test(geode): pin that a Concord plane key cannot delete the channel MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A soft ban leaves the community_root in the ex-member's hands, so they keep deriving the channel's stream key. CORD-01 signs every wrap with that shared key rather than with the author, so on the wire a Concord channel looks like a single author publishing everything — and NIP-09/NIP-62 authorize on the outer pubkey. Read naively that hands any ex-member a one-event wipe of the whole community's history, and geode's own Nip09DeletionTest guarantee ("a kind-5 from pubkey X cannot delete pubkey Y's events") would be vacuous inside a plane. It is refused, but only because of a rule written for something else: Event.owner() gives a kind-1059 to its p-tag RECIPIENT rather than its signer, and ConcordStreamEnvelope stamps a freshly random p-tag on every wrap. Each wrap is therefore owned by a one-time key nobody holds, attacker included. Neither half was written with this attack in mind and either one silently re-opens it, so both are pinned: two tests fail if ownership ever moves back to the signer, and a counterfactual (a wrap addressed to a real key IS deletable by its holder) fails the moment that p-tag becomes anything a member holds. Scope: this is our relay's rule, not the protocol's. A third-party relay that authorizes deletion by matching pubkey still hands every ex-member a wipe button, and a Refounding only protects the future. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DrJhpFhhLjuDJQNkGvYMGj --- .../geode/ConcordPlaneKeyDeletionTest.kt | 242 ++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 geode/src/test/kotlin/com/vitorpamplona/geode/ConcordPlaneKeyDeletionTest.kt diff --git a/geode/src/test/kotlin/com/vitorpamplona/geode/ConcordPlaneKeyDeletionTest.kt b/geode/src/test/kotlin/com/vitorpamplona/geode/ConcordPlaneKeyDeletionTest.kt new file mode 100644 index 0000000000..15c5fcd7a1 --- /dev/null +++ b/geode/src/test/kotlin/com/vitorpamplona/geode/ConcordPlaneKeyDeletionTest.kt @@ -0,0 +1,242 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.geode + +import com.vitorpamplona.quartz.concord.cord03Channels.ChannelChat +import com.vitorpamplona.quartz.concord.cord03Channels.ConcordChannelKeys +import com.vitorpamplona.quartz.concord.envelope.ConcordStreamEnvelope +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.toHexKey +import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair +import com.vitorpamplona.quartz.nip01Core.relay.client.NostrClient +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.publishAndConfirm +import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerSync +import com.vitorpamplona.quartz.nip09Deletions.DeletionEvent +import com.vitorpamplona.quartz.nip62RequestToVanish.RequestToVanishEvent +import com.vitorpamplona.quartz.utils.RandomInstance +import com.vitorpamplona.quartz.utils.TimeUtils +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeout +import kotlin.test.AfterTest +import kotlin.test.BeforeTest +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Why a soft-banned member cannot delete a Concord community's history from the relay — and what + * keeps it that way. + * + * The worry is real. CORD-01 inverts NIP-59: every wrap on a plane is signed by the *shared stream + * key*, not by its author, and the true author only exists inside the encrypted seal. A member + * banned yesterday still derives `group_key("concord/channel", community_root, channel_id, epoch)` + * from the root they kept, so they can still sign events *as the channel itself*. If NIP-09 and + * NIP-62 authorized on the outer `pubkey`, [Nip09DeletionTest]'s "a kind-5 from pubkey X cannot + * delete pubkey Y's events" would be vacuous inside a plane: one event from any ex-member would + * erase the whole community's history. + * + * What stops it is [com.vitorpamplona.quartz.nip01Core.store.owner]: a kind-1059 gift wrap is + * controlled by its **p-tag recipient**, not its signer. Concord stamps a *freshly random* p-tag on + * every wrap ([ConcordStreamEnvelope.wrapSeal]), so each wrap is owned by a one-time key that + * nobody — attacker, author, or owner — ever holds. The channel is undeletable by construction. + * + * Both halves of that are load-bearing and neither was written for this reason, so both are pinned + * here: [theEphemeralPTagIsWhatMakesTheChannelUndeletable] fails the moment the p-tag becomes a + * real key, and the first two tests fail the moment ownership goes back to the signer. + * + * **Scope.** This is our relay's rule, not the protocol's. A community publishes wherever its + * metadata points, and a third-party relay that reads NIP-09 the naive way — deletion authorized by + * matching `pubkey` — hands every ex-member a wipe button for the whole channel. The protocol-level + * fix is the same one that already exists for everything else: a CORD-06 Refounding rotates the + * plane address, which protects the future but cannot restore what a relay already dropped. + */ +class ConcordPlaneKeyDeletionTest { + private lateinit var hub: InProcessRelays + private lateinit var scope: CoroutineScope + private lateinit var client: NostrClient + private val relayUrl: NormalizedRelayUrl = RelayUrlNormalizer.normalize("ws://127.0.0.1:7770/") + + @BeforeTest + fun setup() { + hub = InProcessRelays() + scope = CoroutineScope(Dispatchers.Default + SupervisorJob()) + client = NostrClient(hub, scope) + } + + @AfterTest + fun teardown() { + client.disconnect() + scope.cancel() + hub.close() + } + + private suspend fun query(filter: Filter): List { + val ch = Channel(Channel.UNLIMITED) + val subId = "sub-${System.nanoTime()}" + client.subscribe( + subId, + mapOf(relayUrl to listOf(filter)), + object : SubscriptionListener { + override suspend fun onEvent( + event: Event, + isLive: Boolean, + relay: NormalizedRelayUrl, + forFilters: List?, + ) { + ch.trySend(Msg.Ev(event)) + } + + override fun onEose( + relay: NormalizedRelayUrl, + forFilters: List?, + ) { + ch.trySend(Msg.Eose) + } + }, + ) + val events = mutableListOf() + withTimeout(5000) { + while (true) { + when (val msg = ch.receive()) { + is Msg.Ev -> events += msg.event + Msg.Eose -> return@withTimeout + } + } + } + client.unsubscribe(subId) + return events + } + + private sealed interface Msg { + data class Ev( + val event: Event, + ) : Msg + + object Eose : Msg + } + + /** A public channel plane: derived from the community root, so every member holds its secret. */ + private val communityRoot = RandomInstance.bytes(32) + private val channelId = RandomInstance.bytes(32) + private val plane = ConcordChannelKeys.publicChannel(communityRoot, channelId, rootEpoch = 0) + + /** The plane's own signer — what the banned member reconstructs from the root they kept. */ + private fun planeSigner() = NostrSignerSync(KeyPair(privKey = plane.secretKey)) + + private suspend fun postAs( + author: NostrSignerInternal, + text: String, + createdAt: Long, + ): Event { + val rumor = ChannelChat.message(author.pubKey, channelId.toHexKey(), epoch = 0, text = text, createdAt = createdAt) + return ConcordStreamEnvelope.wrap(rumor, plane, author, encrypted = true, createdAt = createdAt) + } + + @Test + fun aPlaneKeyHolderCannotDeleteTheChannelsHistory() = + runBlocking { + val now = TimeUtils.now() + val bob = NostrSignerInternal(KeyPair()) + val carol = NostrSignerInternal(KeyPair()) + + val history = + listOf( + postAs(bob, "hello", now), + postAs(carol, "hi bob", now + 1), + postAs(bob, "how's the project going?", now + 2), + ) + history.forEach { assertEquals(true, client.publishAndConfirm(it, setOf(relayUrl)), "seed the channel history") } + assertEquals(3, query(Filter(authors = listOf(plane.publicKeyHex))).size, "three messages on the plane") + + // The banned member still derives `plane`, and every wrap above IS authored by it — so + // this kind-5 satisfies a same-author check. It must still be refused. + val deletion = planeSigner().sign(DeletionEvent.build(history, createdAt = now + 10)) + assertEquals(true, client.publishAndConfirm(deletion, setOf(relayUrl)), "the relay accepts the event itself") + + assertEquals( + 3, + query(Filter(authors = listOf(plane.publicKeyHex), kinds = listOf(ConcordStreamEnvelope.KIND_WRAP))).size, + "signing as the plane must NOT delete the community's messages", + ) + } + + @Test + fun aPlaneKeyHolderCannotVanishTheChannelPlane() = + runBlocking { + val now = TimeUtils.now() + val bob = NostrSignerInternal(KeyPair()) + + val history = listOf(postAs(bob, "one", now), postAs(bob, "two", now + 1)) + history.forEach { client.publishAndConfirm(it, setOf(relayUrl)) } + assertEquals(2, query(Filter(authors = listOf(plane.publicKeyHex))).size) + + // NIP-62 needs no per-event targeting: one event, and everything that pubkey published + // is gone. The sharpest version of the attack, and the same rule has to stop it. + val vanish = planeSigner().sign(RequestToVanishEvent.build(relayUrl, "", createdAt = now + 10)) + assertEquals(true, client.publishAndConfirm(vanish, setOf(relayUrl))) + + assertEquals( + 2, + query(Filter(authors = listOf(plane.publicKeyHex), kinds = listOf(ConcordStreamEnvelope.KIND_WRAP))).size, + "a kind-62 signed as the plane must not wipe the channel", + ) + } + + @Test + fun theEphemeralPTagIsWhatMakesTheChannelUndeletableSoDoNotMakeItMeaningful() = + runBlocking { + // The counterfactual, so the invariant is visible rather than incidental: ownership of a + // 1059 follows the p-tag, so a wrap addressed to a REAL key is deletable by whoever holds + // that key. Concord is safe only because `wrapSeal` stamps a fresh throwaway pubkey there. + // If that p-tag ever becomes something a member holds — a recipient, a channel id, a + // community id — every ex-holder of it can delete the plane's history. + val now = TimeUtils.now() + val mallory = NostrSignerInternal(KeyPair()) + + val addressedWrap = + planeSigner().signNormal( + now, + ConcordStreamEnvelope.KIND_WRAP, + arrayOf(arrayOf("p", mallory.pubKey)), + "not-a-real-seal", + ) + assertEquals(true, client.publishAndConfirm(addressedWrap, setOf(relayUrl))) + assertEquals(1, query(Filter(ids = listOf(addressedWrap.id))).size) + + val deletion = mallory.sign(DeletionEvent.build(listOf(addressedWrap), createdAt = now + 1)) + assertEquals(true, client.publishAndConfirm(deletion, setOf(relayUrl))) + + assertEquals( + 0, + query(Filter(ids = listOf(addressedWrap.id))).size, + "the p-tag recipient owns a 1059 — which is exactly why Concord's p-tag must stay random", + ) + } +} From 41a035034ba514f9964b8e1e1d1f6731715c789e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 21:30:14 +0000 Subject: [PATCH 078/132] test(quartz): pin the hand-crafted routes out of a Concord ban MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ban/unban verb is not the threat model — a malicious client writes editions directly, so what matters is which routes the FOLD refuses. Three more, all of them ones the UI would never author. Two are refused, and it is worth pinning why, because neither is refused by the rule you would expect. Removing yourself from the banlist is caught by the delta rule's strict outranking (nobody outranks themselves), so the sharper attempt does not remove anything: it forks the banlist at genesis, or builds a private chain, that simply never mentions him, at a version high enough to win the head fold. There is then nothing to remove and the rank rule never fires. What catches it is CORD-04 §4's re-heal — the owner's edition is not on the forged head's back-chain, so it is unioned back in as a concurrent ban. The union is load-bearing security here, not just convergence. The third works. A §3 compaction re-wraps one edition per entity and the ROTATOR picks it, so a rotator can decline to carry the banlist forward; every edition it serves is genuine and no signature check can see the omission. A banned member cannot rotate — drainConcordRekeys gates the rotator on the ban-aware hasPermission — but the puppet from the previous commit is not banned and can. EntityFloor is the entire defense, so the community splits: clients that already folded the ban refuse the rollback, fresh joiners have no floor and see no ban. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DrJhpFhhLjuDJQNkGvYMGj --- .../cord04Roles/BannedStaffEscalationTest.kt | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/BannedStaffEscalationTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/BannedStaffEscalationTest.kt index 8ad22aba6b..aedadd885a 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/BannedStaffEscalationTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/BannedStaffEscalationTest.kt @@ -273,4 +273,54 @@ class BannedStaffEscalationTest { assertTrue(AuthorityResolver.resolve(editions, owner).isBanned(alice), "the puppet does not outrank its creator") } + + @Test + fun aForkedBanlistThatOmitsHimCannotLaunderTheBanAway() { + // A malicious client is not limited to what the ban/unban verb will author. The sharpest + // hand-crafted route does not try to REMOVE his ban — removal is what the strict-outrank-self + // rule guards — it forks at genesis and simply never mentions him, at a version high enough + // to win the head fold. The head's own effective list then never carried his ban, so there is + // nothing to remove and the rank rule never fires. + // + // §4's re-heal is what closes it: the owner's edition is authorized and is NOT on the forked + // head's back-chain, so it is unioned back in as a concurrent ban. + val editions = community() + ownerBansAlice + banlist(alice, 99, null, carol) + + assertTrue(AuthorityResolver.resolve(editions, owner).isBanned(alice), "the re-heal union must put the owner's ban back") + } + + @Test + fun aPrivateBanlistChainOfHisOwnCannotLaunderTheBanAway() { + // The same idea two editions deep, so the winning head has a clean ancestry entirely of his + // own making. Ancestry is walked over the full pool, so the owner's ban is still recognised + // as a concurrent fork rather than a superseded ancestor. + val mine = banlist(alice, 50, null) + val editions = community() + ownerBansAlice + mine + banlist(alice, 51, mine.hash) + + assertTrue(AuthorityResolver.resolve(editions, owner).isBanned(alice), "a self-authored chain must not launder the ban away") + } + + @Test + fun aRogueRotatorCompactsTheBanAwayForEveryClientWithoutAFloor() { + // The route that does work, and the one no signature check can catch. A CORD-06 §3 compaction + // re-wraps ONE edition per entity and the ROTATOR picks it, so a rotator can simply not carry + // the banlist forward. Every edition it serves is genuine; the ban is erased by omission. + // + // A banned member cannot rotate (drainConcordRekeys gates the rotator on hasPermission, which + // is ban-aware) — but the puppet minted above is not banned, and it can. EntityFloor is the + // whole defense, so this splits the community in two: clients that already folded the ban + // refuse the rollback, while fresh joiners have no floor to refuse with and see no ban at all. + val editions = community() + ownerBansAlice + val floors = ConcordCommunityState.authorizedHeads(editions, owner) + val compacted = editions.filter { it.entityKind != ControlEntityKind.BANLIST } + + assertFalse( + ConcordCommunityState.fold(compacted, owner).authority.isBanned(alice), + "ESCALATION: a fresh joiner holds no floor, so the omitted ban simply never existed", + ) + assertTrue( + ConcordCommunityState.fold(compacted, owner, floors).authority.isBanned(alice), + "a client that already folded the ban must refuse the rollback", + ) + } } From 1e6cda712dc965575b39ea0553dac7887f9af3da Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Aug 2026 23:06:54 +0000 Subject: [PATCH 079/132] docs(concord): audit the soft-ban and Control Plane attack surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collects the findings from this branch into docs/concord-soft-ban-audit.md, each marked Verified (a test reproduces it, named) or Read (follows from the code, untested), with a suggested order of attack. Adds the reproduction for the one finding that was still unverified, and it did not hold up the way it was first described. Version inflation does not poison the anti-rollback floor through the chain walk — that walk advances only to head.version + 1 citing the head's hash, so a fresh joiner is untouched. It goes through the COMPACTION ARM: once a client holds a floor and the entity is in the epoch snapshot, the head comes from bootstrapHead, which is highest-version at or above the floor with no prev, no hash and no contiguity. Version is then the whole contest and Long.MAX_VALUE wins it permanently — the floor rises to MAX_VALUE, no honest edition can exceed it, and a Refounding that drops the poison falls back to EntityFloor.known, which is the poison. That makes it the worst item on the list: unrecoverable, and authored in the tests by a current, legitimately granted moderator — no ban, no sockpuppet, one ordinary permission bit. compactControlPlane picks per entity by raw max version too, so honest rotators carry it into every future epoch. The banlist escapes only because AuthorityResolver folds it on a floor-less chain walk and re-heals the union, so an honest ban still lands. That accident is all that separates this from a permanently unmoderatable community, so it is pinned by its own test. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DrJhpFhhLjuDJQNkGvYMGj --- docs/concord-soft-ban-audit.md | 230 ++++++++++++++++++ .../ControlPlaneVersionExhaustionTest.kt | 167 +++++++++++++ 2 files changed, 397 insertions(+) create mode 100644 docs/concord-soft-ban-audit.md create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlPlaneVersionExhaustionTest.kt diff --git a/docs/concord-soft-ban-audit.md b/docs/concord-soft-ban-audit.md new file mode 100644 index 0000000000..239258ff40 --- /dev/null +++ b/docs/concord-soft-ban-audit.md @@ -0,0 +1,230 @@ +# Concord: soft-ban and Control Plane audit + +**Scope:** what a removed member — or a moderator who turns — can still do to a Concord community, +assuming a **malicious client** (no client-side rule binds them; only cryptography, the fold, and +the relay do). +**Date:** 2026-08-08. **Status:** findings only, nothing fixed yet. +**Companion:** `docs/concord-banlist-rank-conformance.md` (the rank half of CORD-04 §4, already +reported to Armada and fixed here). + +Each finding says how it was established. **Verified** means a test in this repo reproduces it; +**Read** means it follows from the code but no test was written. Every "Verified" line names the +test. + +--- + +## Summary + +| # | Finding | Severity | Needs a ban? | Recoverable? | +|---|---------|----------|--------------|--------------| +| [V1](#v1) | One edition at `version = Long.MAX_VALUE` pins an entity forever | **Critical** | No — any bit-holder | **No** | +| [V2](#v2) | A banned staffer keeps Role/Grant/Banlist authority | **Critical** | Yes | Yes (Refounding) | +| [V3](#v3) | A rogue rotator compacts the banlist away | High | Via V2 | Partly | +| [V4](#v4) | The Refounding recipient set is attacker-inflatable | High | No | Yes | +| [V5](#v5) | The ban is a per-pubkey display rule; the channel key is not revoked | High | Yes | Yes (Refounding) | +| [V6](#v6) | Channel history is deletable on a naive third-party relay | High | Yes | **No** (history) | +| [V7](#v7) | Banlist rank rule diverges from Armada | Medium | — | — | +| [V8](#v8) | A soft ban revokes no read access and no live invite | Medium | Yes | Yes (Refounding) | +| [V9](#v9) | The base-rekey plane is writable by every member | Low | Yes | Yes | + +The two structural causes worth naming up front, because most of the list collapses into them: + +- **Authority is checked in two places that disagree.** `ConcordCommunityState.fold` gates + METADATA/CHANNEL/INVITE through `authority.hasPermission` (`!isBanned && …`), while ROLE, GRANT + and BANLIST are gated *inside* `AuthorityResolver.resolve` by `holdsManageRoles` / `bitsOf` / + `effectivePermissionsOf`, none of which consult the banlist. That is V2, and V3 follows from it. +- **A ban removes standing, never keys.** Everything a member holds — `community_root`, channel + keys, `control_root` if staff, live invite links — survives it. Only a CORD-06 Refounding rotates + those, which is why V4 (making Refounding expensive) is worth more to an attacker than it looks. + +--- + +## V1 — One edition at `Long.MAX_VALUE` pins an entity forever + +**Critical. Does not require a banned user, a sockpuppet, or the owner's absence. Unrecoverable.** + +*Verified:* `quartz/…/cord04Roles/ControlPlaneVersionExhaustionTest.kt` (3 tests). + +Any current holder of an entity's permission bit publishes one edition at `version = +Long.MAX_VALUE`. For every client that holds an `EntityFloor` for that entity, that edition becomes +the permanent head: + +1. it wins, so the entity shows the attacker's content; +2. `authorizedHeads` raises the entity's floor to `Long.MAX_VALUE`; +3. no honest edition can ever exceed that floor, so the entity can never be repaired; +4. a Refounding that drops the poison does not help — nothing is offered at or above the floor, the + fold reports a gap, and falls back to `EntityFloor.known`, which *is* the poison. + +The chain walk is not the weakness (it advances only to `head.version + 1` citing the head's hash, +so a fresh joiner is unaffected). The weakness is the **compaction arm** of `EditionFold.foldEntity`: +once a floor exists and the entity is in the epoch snapshot — which `fold` always builds from the +editions handed to it — the head comes from `bootstrapHead`, i.e. *highest version at or above the +floor*, with no `prev`, no hash, no contiguity. Version becomes the whole contest. + +Concretely: a moderator with `MANAGE_CHANNELS` deletes `#general` permanently for everyone; one +with `MANAGE_METADATA` renames the community permanently. Demoting or banning them afterwards +changes nothing — the damage is in every client's floor. `ConcordRefounding.compactControlPlane` +also selects the head per entity by raw highest version, ungated, so an honest rotator carries the +poison into every future epoch, where fresh joiners then anchor on it as their baseline. + +The banlist survives, by accident: `AuthorityResolver` folds it on its own floor-less chain walk and +re-heals the union across authorized editions, so an honest ban still lands. That accident is the +only thing separating this from a permanently unmoderatable community, and it is now pinned by +`aPoisonedBanlistStillAcceptsTheOwnersBan`. + +**Fix direction.** The compaction arm needs a bound, since it is the arm that trades contiguity for +cross-epoch tolerance. Options, roughly in order of preference: + +- Cap the version delta the arm will accept in one step (a compacted head is legitimately ahead of + the floor, but by a chain's worth, not by 2^63). Anything above the cap is a gap, not a head. +- Make `bootstrapHead` prefer the highest version *reachable by a chain* among the offered editions, + falling back to raw version only when no chain connects. +- Have `compactControlPlane` select the authority-gated fold head rather than raw max version, so a + poison is at least not propagated by honest rotators. + +The first is the smallest change and closes the unrecoverability; the third should happen regardless. + +## V2 — A banned staffer keeps Role, Grant and Banlist authority + +**Critical.** *Verified:* `quartz/…/cord04Roles/BannedStaffEscalationTest.kt` (13 tests). + +`hasPermission` is ban-aware; the resolver's internal gates are not, and structurally cannot be as +written — the roles/grants fixpoint settles before `banned` is computed. So a banned member who +still holds `control_root` keeps the roster. In the reproduction they: + +- ban every member they outrank, directly, with no puppet; +- revoke the surviving moderators' grants and retire the roles beneath them; +- **mint a fresh, unbanned npub** at the next position down, which then passes every ban-aware gate: + deletes every channel, rewrites the metadata, bans the rest of the community, creates invites; +- and, because `drainConcordRekeys` authorizes a rotator by `hasPermission(rotator, BAN)`, that + puppet can publish a Refounding omitting the owner — every honest client follows it and the owner + is stranded on a dead root. + +Self-unban is *not* reachable and neither is a puppet-unban: the delta rule gates removals and +strict outranking means nobody outranks themselves, while no edition may claim a position at or +above its signer, so the delegation chain only descends. Two hand-crafted attempts that avoid +removal entirely — forking the banlist at genesis, and building a private chain — are also refused, +by CORD-04 §4's re-heal union rather than by the rank rule. **The union is load-bearing security +here, not just convergence.** + +**Fix direction.** Make the resolver's gates ban-aware. The ordering problem is real (you cannot +know who is banned before folding the banlist, nor who may write it before knowing who is banned), +so resolve it as a bounded two-pass where authority only ever *shrinks*: pass A settles the roster +as today and computes the banlist; pass B re-resolves roles/grants dropping editions whose author is +banned in pass A; then recompute the banlist under pass B's roster, keeping only bans still +authorized. Deterministic, terminates, no oscillation on mutual bans. **Consensus-affecting**: until +Armada ships the same rule, we will drop editions they honor. + +## V3 — A rogue rotator compacts the banlist away + +**High.** *Verified:* `aRogueRotatorCompactsTheBanAwayForEveryClientWithoutAFloor`. + +A CORD-06 §3 compaction re-wraps one edition per entity and the *rotator* picks it, so a rotator can +decline to carry the banlist forward. Every edition it serves is genuine, so no signature check sees +the omission — `EntityFloor`'s own KDoc names this case ("clearing a banlist"). A banned member +cannot rotate, but the V2 puppet can. + +The result is not a clean unban but a **split community**: clients that already folded the ban +refuse the rollback and still see it, fresh joiners have no floor and see no ban at all. Two +populations permanently disagreeing about who is a member, with no event either side can call +forged. Closing V2 removes the puppet and takes this with it; floors alone do not, since they only +protect people who were already there. + +## V4 — The Refounding recipient set is attacker-inflatable + +**High.** *Read:* `ConcordCommunitySession.allMembers()` / `emitChannelRumors`; +`AccountConcordActions.refoundConcordCommunity` step 2; `ConcordRefounding.buildBaseRekeyWraps`. + +`allMembers()` = Guestbook joins ∪ `observedAuthors` ∪ roster ∪ owner, and it *is* the Refounding +recipient set. Both contributing sets are unbounded and both are attacker-writable: Guestbook joins +are self-signed (any key, no authority), and every author we decrypt is folded into +`observedAuthors` by design (CORD-02 §5, "observably present"). + +So each throwaway npub an attacker posts from, or announces, is one more mandatory NIP-44 blob in +the next Refounding, chunked 120 per event. 100k identities ⇒ ~100k encryptions and ~830 published +events — while they keep posting. **The attack inflates the cost of its own remedy**, and the remedy +is the only hard removal Concord has. + +This is the cheapest thing on the list to fix and the only one that is not consensus-affecting: cap +the recipient set, prefer recent/attested members when over the cap, and surface what was dropped +(a silent truncation strands real members). Worth doing first. + +## V5 — The ban is a per-pubkey display rule and the channel key is not revoked + +**High.** *Read:* `Account.consumeConcordRumorGated` (`isBanned(rumor.pubKey)`), `Account.isAcceptable`. + +Writing to a channel needs the channel key, which the ban does not take away; the seal author is +whatever key the client feels like using. A malicious client therefore posts every message from a +fresh npub and `isBanned` never matches — moderation is whack-a-mole against an infinite identity +supply. Each message also costs every member two NIP-44 decrypts and two signature verifications +*before* the banlist check runs, and each fresh author inflates V4. + +There is no client-side answer; only a Refounding rotates the key out from under them. That is the +correct design, which is why V4 matters so much. + +## V6 — Channel history is deletable on a naive third-party relay + +**High, external.** *Verified (that we are safe):* +`geode/…/ConcordPlaneKeyDeletionTest.kt` (3 tests). + +CORD-01 signs every wrap with the shared stream key, so on the wire a Concord channel is one author +publishing everything — and every member holds that author's secret. NIP-09 and NIP-62 authorize on +the outer `pubkey`. Read the obvious way, that hands any ex-member a one-event wipe of the whole +community's history, and geode's own guarantee ("a kind-5 from pubkey X cannot delete pubkey Y's +events") is vacuous inside a plane. + +**On our relay it is refused, but only because of a rule written for something else:** +`Event.owner()` gives a kind-1059 to its *p-tag recipient* rather than its signer, and +`ConcordStreamEnvelope` stamps a freshly random p-tag on every wrap, so each wrap is owned by a +one-time key nobody holds. Both halves are load-bearing, neither was written for this, and either +one silently re-opens the hole — all three are now pinned, including a counterfactual showing a wrap +addressed to a *real* key is deletable by its holder. + +A community publishes wherever its metadata points. Any relay that authorizes deletion by matching +`pubkey` still hands every ex-member the wipe button, and a Refounding protects only the future. +Worth a note in the CORD-01 spec and a line in the relay-selection guidance. + +## V7 — Banlist rank rule diverges from Armada + +**Medium, known, deliberate.** See `docs/concord-banlist-rank-conformance.md`, already reported. + +We enforce §3's rank half on the Banlist and Armada does not, so the two clients can show different +banlists. Shipped knowingly. Row 3 of that report ("a banned `BAN` holder unbans themselves") was +left open as a fixpoint-ordering question — V2 is the general form of it, and the fix proposed there +resolves both. + +## V8 — A soft ban revokes no read access and no live invite + +**Medium, inherent.** *Read:* CORD-02/05. + +Until a Refounding, a banned member decrypts everything published — the ban only stops honest +clients from *showing* their posts, not from delivering the group's posts to them. They also keep +any invite links they created while privileged; those still resolve to bundles carrying the current +root. Publishing the root, or one live link, invites an unbanned crowd that each has to be banned +individually (and see V5). + +Not a bug so much as the definition of a soft ban, but it belongs on the list because the UI should +say so: "Ban" and "Remove from community" are very different promises and users will read the first +as the second. + +## V9 — The base-rekey plane is writable by every member + +**Low.** *Read:* `ConcordKeyDerivation.baseRekeyAddress`, `AccountConcordActions.drainConcordRekeys`. + +The base-rekey address derives from `community_root`, so any member — banned included — can mint +valid wraps there. Authorization happens after the blobs are scanned, so a flood costs every member +a locator scan per blob on every revision tick. Bounded work per wrap and no correctness impact; +listed for completeness. + +--- + +## Suggested order + +1. **V4** — cheapest, not consensus-affecting, and it protects the remedy every other fix depends on. +2. **V1** — worst blast radius and the only unrecoverable one; does not need an attacker to be + banned or privileged beyond a single ordinary bit. +3. **V2** (+V3, +V7 row 3) — one two-pass change closes all three. Coordinate with Armada first; + this one splits consensus. +4. **V6** — spec note + relay guidance; our own behaviour is already correct and now pinned. +5. **V5 / V8** — UI honesty about what a ban does, and a "Remove from community" affordance that + Refounds rather than bans. diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlPlaneVersionExhaustionTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlPlaneVersionExhaustionTest.kt new file mode 100644 index 0000000000..bbba1dc8d3 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlPlaneVersionExhaustionTest.kt @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.concord.cord04Roles + +import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityState +import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * **V1 in `docs/concord-soft-ban-audit.md` — reproduction.** A single Control Plane edition at + * `version = Long.MAX_VALUE` pins its entity to the author's content permanently, for every client + * that holds a floor for it. + * + * The chain walk is not the weakness — it advances only to `head.version + 1` citing the head's + * hash, so an inflated version is unreachable and a fresh joiner is unaffected. The weakness is the + * **compaction arm** of [EditionFold.foldEntity]: once a client holds a floor for an entity and that + * entity appears in the epoch snapshot (which `ConcordCommunityState.fold` always builds from the + * editions handed to it), the head is chosen by [EditionFold.bootstrapHead] — *highest version at or + * above the floor*, with no `prev`, no hash, and no contiguity. Version is then the whole contest, + * and `Long.MAX_VALUE` wins it forever: + * + * 1. the poison becomes the head, so the entity shows the attacker's content; + * 2. `authorizedHeads` raises the entity's floor to `Long.MAX_VALUE`; + * 3. no honest edition can ever exceed that floor, so the entity can never be repaired; + * 4. a Refounding that drops the poison does not help either — nothing is offered at or above the + * floor, so the fold reports a gap and falls back to [EntityFloor.known], which *is* the poison. + * + * Note who the attacker is. Every test here is authored by **bob, a current and legitimately granted + * moderator** — not a banned member, not a sockpuppet. Any holder of the entity's permission bit can + * do this at any time, and demoting or banning them afterwards changes nothing, because the damage + * is already in every client's floor. It is also carried into every future epoch by + * `ConcordRefounding.compactControlPlane`, which selects the head per entity by raw highest version. + * + * The banlist is the one entity that survives, and by accident: `AuthorityResolver` folds it with + * its own floor-less chain walk and then re-heals the union across authorized editions, so an + * honest ban lands even when the head is poisoned. [aPoisonedBanlistStillAcceptsTheOwnersBan] pins + * that, because it is the only thing standing between this bug and a permanently unmoderatable + * community. + */ +class ControlPlaneVersionExhaustionTest { + private val owner = "0f".repeat(32) + private val bob = "b2".repeat(32) + + private val modRole = "22".repeat(32) + private val metadataEntity = "66".repeat(32) + private val channelEntity = "55".repeat(32) + private val banlistEntity = "44".repeat(32) + + private fun edition( + kind: ControlEntityKind, + entity: String, + version: Long, + prev: ByteArray?, + content: String, + author: String, + rumorId: String, + ) = ControlEdition(kind, entity.hexToByteArray(), version, prev, null, content, author, rumorId, 0) + + /** bob holds exactly one bit, granted by the owner, entirely legitimately. */ + private fun communityWhereBobHolds( + permissions: String, + vararg rest: ControlEdition, + ) = listOf( + edition(ControlEntityKind.ROLE, modRole, 0, null, """{"name":"Mod","position":5,"permissions":"$permissions"}""", owner, "role-mod"), + edition(ControlEntityKind.GRANT, "32".repeat(32), 0, null, """{"member":"$bob","role_ids":["$modRole"]}""", owner, "grant-bob"), + ) + rest + + @Test + fun oneEditionAtMaxVersionPinsTheMetadataForever() { + val metadataV0 = edition(ControlEntityKind.METADATA, metadataEntity, 0, null, """{"name":"My Community"}""", owner, "meta-0") + val community = communityWhereBobHolds(ConcordPermissions.of(ConcordPermissions.MANAGE_METADATA).toWire(), metadataV0) + + // A client that has folded this community once holds a floor for the metadata entity. + val floorsBefore = ConcordCommunityState.authorizedHeads(community, owner) + assertEquals(0, floorsBefore[metadataEntity]?.version, "an ordinary floor at the genesis edition") + + val poison = edition(ControlEntityKind.METADATA, metadataEntity, Long.MAX_VALUE, metadataV0.hash, """{"name":"PWNED"}""", bob, "meta-poison") + val floorsAfter = ConcordCommunityState.authorizedHeads(community + poison, owner, floorsBefore) + assertEquals(Long.MAX_VALUE, floorsAfter[metadataEntity]?.version, "VULNERABLE: the floor is now at the top of the version space") + + // The owner tries to repair it, chaining honestly onto their own genesis. + val repair = edition(ControlEntityKind.METADATA, metadataEntity, 1, metadataV0.hash, """{"name":"My Community"}""", owner, "meta-1") + val pool = community + poison + repair + + assertEquals( + "My Community", + ConcordCommunityState.fold(pool, owner).metadata?.name, + "a fresh joiner walks the chain and is unaffected", + ) + assertEquals( + "PWNED", + ConcordCommunityState.fold(pool, owner, floorsAfter).metadata?.name, + "VULNERABLE: every client holding a floor is pinned to the attacker's content", + ) + assertEquals( + "PWNED", + ConcordCommunityState.fold(community + repair, owner, floorsAfter).metadata?.name, + "VULNERABLE: even a Refounding that drops the poison falls back to it as EntityFloor.known", + ) + } + + @Test + fun oneEditionAtMaxVersionDeletesAChannelForever() { + val channelV0 = edition(ControlEntityKind.CHANNEL, channelEntity, 0, null, """{"name":"general"}""", owner, "chan-0") + val community = communityWhereBobHolds(ConcordPermissions.of(ConcordPermissions.MANAGE_CHANNELS).toWire(), channelV0) + + val floorsBefore = ConcordCommunityState.authorizedHeads(community, owner) + val poison = edition(ControlEntityKind.CHANNEL, channelEntity, Long.MAX_VALUE, channelV0.hash, """{"name":"general","deleted":true}""", bob, "chan-poison") + val floorsAfter = ConcordCommunityState.authorizedHeads(community + poison, owner, floorsBefore) + + val repair = edition(ControlEntityKind.CHANNEL, channelEntity, 1, channelV0.hash, """{"name":"general"}""", owner, "chan-1") + val pool = community + poison + repair + + assertEquals(1, ConcordCommunityState.fold(pool, owner).channels.size, "a fresh joiner still sees the channel") + assertEquals(0, ConcordCommunityState.fold(pool, owner, floorsAfter).channels.size, "VULNERABLE: the channel is gone and cannot be restored") + assertEquals( + 0, + ConcordCommunityState.fold(community + repair, owner, floorsAfter).channels.size, + "VULNERABLE: dropping the poison does not bring the channel back", + ) + } + + @Test + fun aPoisonedBanlistStillAcceptsTheOwnersBan() { + // The saving grace, and the reason this bug is "unmoderatable community" rather than + // "community with a broken name". AuthorityResolver folds the banlist on its own floor-less + // chain walk and re-heals the union across every authorized edition, so the owner's ban lands + // even while the banlist's own floor sits at Long.MAX_VALUE. Do not "unify" the banlist onto + // the floored fold without replacing this protection. + val banlistV0 = edition(ControlEntityKind.BANLIST, banlistEntity, 0, null, "[]", owner, "ban-0") + val community = communityWhereBobHolds(ConcordPermissions.of(ConcordPermissions.BAN).toWire(), banlistV0) + + val floorsBefore = ConcordCommunityState.authorizedHeads(community, owner) + val poison = edition(ControlEntityKind.BANLIST, banlistEntity, Long.MAX_VALUE, banlistV0.hash, "[]", bob, "ban-poison") + val floorsAfter = ConcordCommunityState.authorizedHeads(community + poison, owner, floorsBefore) + assertEquals(Long.MAX_VALUE, floorsAfter[banlistEntity]?.version, "the banlist floor is poisoned like any other") + + val ownerBansBob = edition(ControlEntityKind.BANLIST, banlistEntity, 1, banlistV0.hash, """["$bob"]""", owner, "ban-1") + val pool = community + poison + ownerBansBob + + assertTrue(ConcordCommunityState.fold(pool, owner).authority.isBanned(bob), "a fresh joiner honors the ban") + assertTrue( + ConcordCommunityState.fold(pool, owner, floorsAfter).authority.isBanned(bob), + "the re-heal union must keep the banlist working even with a poisoned floor", + ) + } +} From d37e183a575ed9266f58a1404c953b4e89b3c068 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 01:28:22 +0000 Subject: [PATCH 080/132] docs(concord): audit the surfaces the first pass never opened MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first pass was bounded by the Control Plane, the fold and the relay. Three more findings from the surfaces it skipped, plus an explicit list of what is still unexamined so the next reader knows where the edges are. V10 is the serious one, and it forks. ConcordStrandedRecovery.isStranded takes only (entry, bundle): no banlist check, no check that we were legitimately re-keyed. The whole test is "the bundle at my stored invite_ref sits at a higher epoch than I do", and the unlock token lives in the link fragment an ex-member keeps forever. So whether a removed member walks back in depends only on whether anything re-mints at that coordinate. Amethyst mints a fresh link signer per invite and the Refounding neither re-mints nor revokes, so today nothing does — which means stranded recovery never fires for anyone, and the cure that drainConcordRekeys' KDoc points to for "a BAN-holder can evict anyone, the owner included, by omission" does not actually exist. If any client does re-mint at a stable coordinate, as CORD-05's design describes, then every removed member auto-recovers the new root on the 15-minute sweep and re-announces a Guestbook join. Either the safety net is missing or the only hard removal is undone; which one it is needs a spec answer, not a patch. V11: voice rooms authenticate with the channel's derived voice signer key against a stateless SFU that holds no community secret and cannot know a banlist exists, so a banned member keeps talking until a Refounding. V12: ingestTyping filters on binding and self only, so they keep showing as "typing". Checked and sound, recorded so they are not re-audited: the envelope pins rumor.pubKey == seal.pubKey (no author impersonation), and Note.latestConcordEdit is author-gated, so a member cannot rewrite someone else's message. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DrJhpFhhLjuDJQNkGvYMGj --- docs/concord-soft-ban-audit.md | 81 ++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/docs/concord-soft-ban-audit.md b/docs/concord-soft-ban-audit.md index 239258ff40..8f284085ff 100644 --- a/docs/concord-soft-ban-audit.md +++ b/docs/concord-soft-ban-audit.md @@ -26,6 +26,9 @@ test. | [V7](#v7) | Banlist rank rule diverges from Armada | Medium | — | — | | [V8](#v8) | A soft ban revokes no read access and no live invite | Medium | Yes | Yes (Refounding) | | [V9](#v9) | The base-rekey plane is writable by every member | Low | Yes | Yes | +| [V10](#v10) | Stranded recovery: either broken, or a removal bypass | **Critical** | Yes | — | +| [V11](#v11) | Voice rooms are key-gated, not roster-gated | High | Yes | Yes (Refounding) | +| [V12](#v12) | Typing indicators are not ban-filtered | Low | Yes | Yes | The two structural causes worth naming up front, because most of the list collapses into them: @@ -216,8 +219,86 @@ valid wraps there. Authorization happens after the blobs are scanned, so a flood a locator scan per blob on every revision tick. Bounded work per wrap and no correctness impact; listed for completeness. +## V10 — Stranded recovery: either broken, or a removal bypass + +**Critical, and it forks — one of the two halves is true and both are bad.** +*Read:* `ConcordStrandedRecovery`, `AccountConcordActions.recoverStrandedConcordCommunities`, +`AccountConcordActions.mintConcordInvite`. + +`ConcordStrandedRecovery.isStranded` / `mergeForward` take only `(entry, bundle)`. There is **no +banlist check and no check that we were legitimately re-keyed** — the entire test is "the bundle at +my stored `inviteRef` sits at a higher epoch than I do". The unlock token lives in the link +fragment, which an ex-member keeps forever. So whether a removed member walks back in with the new +root depends *only* on whether the bundle at that coordinate ever advances an epoch. + +In Amethyst it never does: `mintConcordInvite` mints a **fresh link signer per mint**, so nothing +re-publishes at an existing coordinate, and `refoundConcordCommunity` does not re-mint or revoke +anything. Two consequences, and they are the fork: + +- **If nothing re-mints** — today's behaviour — then stranded recovery never fires *for anyone*. + That makes it dead code, and the cure `drainConcordRekeys`' own KDoc points to for "a BAN-holder + can evict anyone (the owner included) by omission" does not exist. An owner evicted by a rogue + admin has no way back. +- **If anything re-mints at a stable coordinate** — which is what CORD-05's design describes ("the + community keeps publishing its bundle at that same addressable coordinate, re-minted at the + current epoch"), so plausibly Armada in a cross-client community — then every removed member who + joined through a still-live link auto-recovers the new root on the 15-minute sweep, and + re-announces a Guestbook join so they look current again. **Refounding, the only hard removal, + is silently undone.** + +Note also that `refoundConcordCommunity` never revokes the invite links the removed member created +or joined through, even though `ControlEntityKind.INVITE_REVOKED` exists and `classifyInvite` +already honors it. + +**Fix direction.** Decide the intended semantics first — this needs a spec answer, not a patch. +Then: gate `mergeForward` on not being banned in the epoch we are merging *from*, have the +Refounding revoke the removed members' links, and either implement re-minting (so legitimate +recovery works) or drop the mechanism and give evicted owners a different route. + +## V11 — Voice rooms are key-gated, not roster-gated + +**High.** *Read:* `ConcordBrokerToken`, CORD-07 §2. + +A member proves voice-room membership by signing a NIP-98 kind-27235 request with the channel's +**derived voice signer key**, whose pubkey is the SFU room name. The broker is stateless and holds +no community secret, so it cannot consult the Control Plane and has no idea a banlist exists. A +banned member keeps that key until a Refounding, so they can join the voice room and stay in it. +Nothing on the client side can evict them — kicking them from the UI does not kick them from the SFU. + +This is the one place where a ban fails *audibly*, in real time, in front of everyone. Worth ranking +above its technical severity for that reason alone. + +## V12 — Typing indicators are not ban-filtered + +**Low.** *Read:* `ConcordCommunitySession.ingestTyping`. + +`ingestTyping` checks the rumor is a typing heartbeat, is bound to the channel/epoch, and is not our +own — and nothing else. A banned member (or any fresh npub holding the channel key, see V5) shows +in the "… is typing" row indefinitely. Cheap to fix and user-visible: the promise a ban makes is +that the member disappears, and here they do not. + --- +## What was NOT examined + +This audit is bounded by what was opened. Checked and found sound: the wrap/seal envelope (no author +impersonation — `rumor.pubKey == seal.pubKey` and `rumor.verifyId()`), Concord chat edits +(`Note.latestConcordEdit` is author-gated, so a member cannot rewrite someone else's message), and +self-unban (V2). + +Not looked at at all: + +- **Private channels** (CORD-03 derived keys) — key delivery on grant, and channel-scoped rekey. + Note that no channel-scoped rekey *receive* path appears to exist: `drainConcordRekeys` handles + `ROOT_SCOPE` only, and `entry.privateChannels` is carried forward but never populated by a + delivery path. If that is right, the only removal Amethyst can perform is a full-community + Refounding — which is exactly what V4 makes expensive. +- **In-plane reactions and deletes** — the edit path is author-gated; the delete path was not read. +- **Guestbook kicks** (kind 3309) — the builder documents a KICK-bit + rank rule; the receive side + was not verified against it. +- Unread counts and notification triggers, media/upload references from messages, the NIP-53 nests + overlap, and the desktop client's Concord paths. + ## Suggested order 1. **V4** — cheapest, not consensus-affecting, and it protects the remedy every other fix depends on. From f52a8b0432d3c4abd2ca10fa5b2b9fdfc51b5929 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 01:41:52 +0000 Subject: [PATCH 081/132] docs(concord): split the audit by what the attacker needs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-reviewed every finding against the shipping app rather than against the protocol, and split the list in two: what a banned user can do with stock Amethyst (our bugs) versus what needs a hand-written client (fix in the fold, or defend against). Several items moved, and the review turned up a new one that belongs at the top. A1 is new and is the realistic attack. mintConcordInvite checks only that the account is writeable and that we hold the community — no CREATE_INVITE, no banlist — and unlike the Edit and channel buttons next to it, the invite IconButton carries no guard at all. A banned user stays in the app, taps person-add, and shares a working link to the community. The mint publishes a fresh link signer, so revoking the links they were given does not touch the ones they make; and because the bundle is a standalone kind-33301 outside the Control Plane, the CREATE_INVITE bit the fold enforces on INVITE_* entities never applies to the actual invite mechanism. A3 is the general form: every moderation verb checks isWriteable() and the Control write key and nothing else, so authority lives in the composable that draws the button — and those gates use effectivePermissions, which is ban-blind. Ban and Remove survive only because a second, unrelated condition routes through the ban-aware canActOn. refoundConcordCommunity guards itself with effectivePermissions outright, so a banned BAN-holder can launch a Refounding from the shipping app; honest receivers refuse it, but that is a race against banlist propagation, not a check. A2 moves to Part A because our own client is what performs it: the recovery sweep runs every 15 minutes with no banlist check. C2 (voice) is downgraded from High — ConcordBrokerToken and VoicePresence are referenced nowhere outside quartz, so there is no shipping path to attack. It is a note for whoever wires one up. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DrJhpFhhLjuDJQNkGvYMGj --- docs/concord-soft-ban-audit.md | 347 ++++++++++++++++++++++----------- 1 file changed, 233 insertions(+), 114 deletions(-) diff --git a/docs/concord-soft-ban-audit.md b/docs/concord-soft-ban-audit.md index 8f284085ff..58429394ca 100644 --- a/docs/concord-soft-ban-audit.md +++ b/docs/concord-soft-ban-audit.md @@ -1,48 +1,205 @@ # Concord: soft-ban and Control Plane audit -**Scope:** what a removed member — or a moderator who turns — can still do to a Concord community, -assuming a **malicious client** (no client-side rule binds them; only cryptography, the fold, and -the relay do). -**Date:** 2026-08-08. **Status:** findings only, nothing fixed yet. +**Scope:** what a removed member — or a moderator who turns — can still do to a Concord community. +**Date:** 2026-08-09. **Status:** findings only, nothing fixed yet. **Companion:** `docs/concord-banlist-rank-conformance.md` (the rank half of CORD-04 §4, already reported to Armada and fixed here). Each finding says how it was established. **Verified** means a test in this repo reproduces it; -**Read** means it follows from the code but no test was written. Every "Verified" line names the -test. +**Read** means it follows from the code but no test was written. Every "Verified" line names the test. + +--- + +## How to read this list + +Findings are split by **what the attacker needs**, because that decides who owns the fix and how +urgent it is: + +- **[Part A — reachable from stock Amethyst](#part-a).** A banned user opens the shipping app and + taps a button, or our own client does it for them on a timer. These are straightforwardly *our + bugs*, they need no attacker sophistication at all, and every one of them is fixable in this repo + without touching the protocol or coordinating with anyone. +- **[Part B — requires a malicious client](#part-b).** The attacker writes their own events, so no + client-side rule binds them. We cannot stop them from *authoring* anything; we can only refuse to + *honor* it. Fixes live in the fold, the store, or the spec. +- **[Part C — interop and not-yet-shipped surfaces](#part-c).** + +The distinction is not academic. Part A is where the realistic attacker is: an irritated user who +just got banned has the app already installed and is not going to write a Nostr client. Part B is +where the *damage ceiling* is. Fix Part A first because it is cheap and it is what will actually +happen; fix Part B because it is what ends communities. + +Two structural causes account for most of both halves: + +- **Authority is checked in several places that disagree.** `ConcordCommunityState.fold` gates + METADATA/CHANNEL/INVITE through the ban-aware `authority.hasPermission`. `AuthorityResolver` + gates ROLE/GRANT/BANLIST internally through `holdsManageRoles` / `bitsOf` / + `effectivePermissionsOf`, which are ban-blind. The **UI** gates through `effectivePermissions`, + also ban-blind. The **action layer** mostly does not gate at all. Same question, four answers. +- **A ban removes standing, never keys.** `community_root`, channel keys, `control_root` if staff, + and live invite links all survive it. Only a CORD-06 Refounding rotates those — which is why + anything that makes Refounding expensive (B4) or reversible (A2) is worth more to an attacker + than it first looks. --- ## Summary -| # | Finding | Severity | Needs a ban? | Recoverable? | -|---|---------|----------|--------------|--------------| -| [V1](#v1) | One edition at `version = Long.MAX_VALUE` pins an entity forever | **Critical** | No — any bit-holder | **No** | -| [V2](#v2) | A banned staffer keeps Role/Grant/Banlist authority | **Critical** | Yes | Yes (Refounding) | -| [V3](#v3) | A rogue rotator compacts the banlist away | High | Via V2 | Partly | -| [V4](#v4) | The Refounding recipient set is attacker-inflatable | High | No | Yes | -| [V5](#v5) | The ban is a per-pubkey display rule; the channel key is not revoked | High | Yes | Yes (Refounding) | -| [V6](#v6) | Channel history is deletable on a naive third-party relay | High | Yes | **No** (history) | -| [V7](#v7) | Banlist rank rule diverges from Armada | Medium | — | — | -| [V8](#v8) | A soft ban revokes no read access and no live invite | Medium | Yes | Yes (Refounding) | -| [V9](#v9) | The base-rekey plane is writable by every member | Low | Yes | Yes | -| [V10](#v10) | Stranded recovery: either broken, or a removal bypass | **Critical** | Yes | — | -| [V11](#v11) | Voice rooms are key-gated, not roster-gated | High | Yes | Yes (Refounding) | -| [V12](#v12) | Typing indicators are not ban-filtered | Low | Yes | Yes | +### Part A — reachable from stock Amethyst (our bugs) -The two structural causes worth naming up front, because most of the list collapses into them: +| # | Finding | Severity | Was | +|---|---------|----------|-----| +| [A1](#a1) | Any member — banned included — mints a working invite in one tap | **Critical** | new | +| [A2](#a2) | Stranded recovery runs on a timer and never checks the banlist | **Critical** | V10 | +| [A3](#a3) | The action layer has no permission checks; the UI's are ban-blind | High | new | +| [A4](#a4) | A banned member keeps broadcasting "typing", and we keep showing it | Low | V12 | +| [A5](#a5) | A banned member's own client keeps reading and rendering everything | Medium | V8 | -- **Authority is checked in two places that disagree.** `ConcordCommunityState.fold` gates - METADATA/CHANNEL/INVITE through `authority.hasPermission` (`!isBanned && …`), while ROLE, GRANT - and BANLIST are gated *inside* `AuthorityResolver.resolve` by `holdsManageRoles` / `bitsOf` / - `effectivePermissionsOf`, none of which consult the banlist. That is V2, and V3 follows from it. -- **A ban removes standing, never keys.** Everything a member holds — `community_root`, channel - keys, `control_root` if staff, live invite links — survives it. Only a CORD-06 Refounding rotates - those, which is why V4 (making Refounding expensive) is worth more to an attacker than it looks. +### Part B — requires a malicious client + +| # | Finding | Severity | Needs a ban? | Recoverable? | Was | +|---|---------|----------|--------------|--------------|-----| +| [B1](#b1) | One edition at `version = Long.MAX_VALUE` pins an entity forever | **Critical** | No — any bit-holder | **No** | V1 | +| [B2](#b2) | A banned staffer keeps Role/Grant/Banlist authority | **Critical** | Yes | Yes (Refounding) | V2 | +| [B3](#b3) | A rogue rotator compacts the banlist away | High | Via B2 | Partly | V3 | +| [B4](#b4) | The Refounding recipient set is attacker-inflatable | High | No | Yes | V4 | +| [B5](#b5) | The ban is per-pubkey; the channel key is not revoked | High | Yes | Yes (Refounding) | V5 | +| [B6](#b6) | Channel history is deletable on a naive third-party relay | High | Yes | **No** (history) | V6 | +| [B7](#b7) | The base-rekey plane is writable by every member | Low | Yes | Yes | V9 | + +### Part C — interop and not-yet-shipped + +| # | Finding | Severity | Was | +|---|---------|----------|-----| +| [C1](#c1) | Banlist rank rule diverges from Armada | Medium | V7 | +| [C2](#c2) | CORD-07 voice rooms are key-gated, not roster-gated | Design | V11 | --- -## V1 — One edition at `Long.MAX_VALUE` pins an entity forever +# Part A — reachable from stock Amethyst + +No custom tooling. A banned user with the shipping app, or our own background sweep. + +## A1 — Any member, banned included, mints a working invite in one tap + +**Critical. The single most likely thing an irritated banned user actually does.** +*Read:* `AccountConcordActions.mintConcordInvite`, `ConcordChannelListScreen` (the `PersonAdd` +`IconButton`). + +`mintConcordInvite` checks exactly two things: that the account is writeable, and that we have the +community in our joined list. **No `CREATE_INVITE` check. No banlist check.** And unlike the Edit +and channel-management buttons beside it, the invite `IconButton` is rendered with no `canEdit` +guard at all — it is always there, for everyone. + +So the flow is: get banned, stay in the app, tap the person-add icon, share the link. The minted +bundle carries the community root we still hold, so anyone who opens it joins for real. Every +invited account is a fresh unbanned npub that moderators then have to ban one at a time. + +Two aggravating details. The mint publishes a **fresh link signer per invite**, so it is a brand-new +coordinate — revoking the links the banned member was given does not touch the ones they mint. +And `CREATE_INVITE` is a real permission bit that the fold enforces on `INVITE_*` Control entities, +but the actual invite mechanism is a standalone kind-33301 addressable event published *outside* the +Control Plane, so that gate never applies to it. The permission is, in practice, unenforced. + +**Fix.** Gate `mintConcordInvite` on `hasPermission(me, CREATE_INVITE) || isOwner(me)`, and gate the +button on the same. This is contained, uncontroversial, and closes the realistic attack. Do it first. + +## A2 — Stranded recovery runs on a timer and never checks the banlist + +**Critical, and it forks.** *Read:* `ConcordStrandedRecovery`, +`AccountConcordActions.recoverStrandedConcordCommunities`, `AccountConcordActions.mintConcordInvite`. + +This is in Part A because **our own client performs it, unprompted**: the recovery sweep runs on the +revision tick for every joined community holding an `inviteRef`, every 15 minutes. The banned user +does nothing but leave the app installed. + +`ConcordStrandedRecovery.isStranded` / `mergeForward` take only `(entry, bundle)` — no banlist +check, no check that we were legitimately re-keyed. The whole test is "the bundle at my stored +`inviteRef` sits at a higher epoch than I do", and the unlock token lives in the link fragment an +ex-member keeps forever. So whether a removed member walks back in depends *only* on whether +anything re-mints at that coordinate: + +- **If nothing re-mints** — today, since Amethyst mints a fresh link signer per invite and the + Refounding neither re-mints nor revokes — stranded recovery never fires for anyone. It is dead + code, and the cure `drainConcordRekeys`' own KDoc points to for "a BAN-holder can evict anyone + (the owner included) by omission" does not exist. An owner evicted by a rogue admin has no way back. +- **If anything re-mints at a stable coordinate** — which is what CORD-05's design describes, so + plausibly Armada in a cross-client community — every removed member auto-recovers the new root and + re-announces a Guestbook join, looking current again. **The only hard removal is silently undone.** + +Note also that `refoundConcordCommunity` never revokes the links the removed member created or +joined through, though `ControlEntityKind.INVITE_REVOKED` exists and `classifyInvite` honors it. + +**Fix.** Decide the intended semantics first — this needs a spec answer. Then gate `mergeForward` on +not being banned in the epoch we merge *from*, have the Refounding revoke the removed members' +links, and either implement re-minting so legitimate recovery works, or drop the mechanism and give +evicted owners another route. + +## A3 — The action layer has no permission checks; the UI's are ban-blind + +**High (defense in depth).** *Read:* `AccountConcordActions` (`banConcordMember`, +`unbanConcordMember`, `editConcordMetadata`, `deleteConcordChannel`, `refoundConcordCommunity`), +`ConcordMembersScreen`, `ConcordChannelListScreen`. + +Every moderation verb checks `isWriteable()` and the Control write key, and **nothing else** — no +permission bit, no banlist. Authority lives entirely in the composable that draws the button. Two +consequences: + +1. **The UI's own gates are ban-blind.** `iCanBan`, `canEdit` (metadata) and `canManageChannels` all + use `effectivePermissions`, which ignores the banlist. A banned admin still sees the Edit and + channel-management controls. Those particular editions are dropped by every client's fold + (METADATA/CHANNEL are `hasPermission`-gated), so the result is a **silently no-op control** — + which this codebase elsewhere explicitly calls out as worse than no control at all. +2. **Ban/Remove survive only because of a second, unrelated gate.** `canBan` is + `viewerCanBan && canBanTarget`, and `canBanTarget` routes through `canActOn`, which *is* + ban-aware. Remove the second condition and a banned admin gets a working Ban button. That is a + thin margin for a Critical-severity outcome (B2). + +`refoundConcordCommunity` is the sharpest instance: its own guard is +`isOwner || effectivePermissions(me).has(BAN)` — deliberately ban-blind — so a banned BAN-holder can +launch a full community Refounding from the shipping app. Honest receivers refuse it +(`drainConcordRekeys` checks the ban-aware `hasPermission`), so the blast radius today is noise plus +self-stranding — but it is a race against banlist propagation, and a fresh joiner who has not folded +the ban yet has no reason to refuse. + +**Fix.** Move the authority check into the action layer where it cannot be bypassed by a new caller +(desktop, CLI, a future screen), and switch every `effectivePermissions` used as an authorization +test to `hasPermission`. Keep `effectivePermissions` only where the question really is "what do +their roles say", independent of standing. + +## A4 — A banned member keeps broadcasting "typing", and we keep showing it + +**Low, both halves ours.** *Read:* `AccountConcordActions.sendConcordTyping`, +`ConcordCommunitySession.ingestTyping`. + +The send side checks `isWriteable()` and nothing else, so a banned member's stock app keeps emitting +kind-23311 heartbeats. The receive side checks that the rumor is a typing heartbeat, is bound to the +channel/epoch, and is not our own — and nothing else. So a banned member sits in the "… is typing" +row indefinitely, in a channel where every message they send is hidden. Cheap to fix on both ends, +and it directly contradicts what a ban promises the user. + +## A5 — A banned member's own client keeps reading and rendering everything + +**Medium, partly inherent.** *Read:* CORD-02/05, `ConcordCommunitySession`. + +Until a Refounding, a ban stops honest clients from *showing* the banned member's posts; it does not +stop delivering the community's posts *to* them. Their stock app keeps subscribing, decrypting and +rendering the whole community in real time. They also keep any invite links they hold (and can mint +more — A1). + +The cryptography here is inherent to a soft ban, but the **product** side is ours: "Ban" and "Remove +from community" are very different promises and the UI presents them as neighbours in one menu. +Worth making the difference explicit at the point of choice, and worth defaulting destructive +moderation to the Refounding path. + +--- + +# Part B — requires a malicious client + +The attacker writes their own events, so nothing client-side binds them. We can only refuse to honor +what they publish. + +## B1 — One edition at `Long.MAX_VALUE` pins an entity forever **Critical. Does not require a banned user, a sockpuppet, or the owner's absence. Unrecoverable.** @@ -87,7 +244,7 @@ cross-epoch tolerance. Options, roughly in order of preference: The first is the smallest change and closes the unrecoverability; the third should happen regardless. -## V2 — A banned staffer keeps Role, Grant and Banlist authority +## B2 — A banned staffer keeps Role, Grant and Banlist authority **Critical.** *Verified:* `quartz/…/cord04Roles/BannedStaffEscalationTest.kt` (13 tests). @@ -118,22 +275,22 @@ banned in pass A; then recompute the banlist under pass B's roster, keeping only authorized. Deterministic, terminates, no oscillation on mutual bans. **Consensus-affecting**: until Armada ships the same rule, we will drop editions they honor. -## V3 — A rogue rotator compacts the banlist away +## B3 — A rogue rotator compacts the banlist away **High.** *Verified:* `aRogueRotatorCompactsTheBanAwayForEveryClientWithoutAFloor`. A CORD-06 §3 compaction re-wraps one edition per entity and the *rotator* picks it, so a rotator can decline to carry the banlist forward. Every edition it serves is genuine, so no signature check sees the omission — `EntityFloor`'s own KDoc names this case ("clearing a banlist"). A banned member -cannot rotate, but the V2 puppet can. +cannot rotate, but the B2 puppet can. The result is not a clean unban but a **split community**: clients that already folded the ban refuse the rollback and still see it, fresh joiners have no floor and see no ban at all. Two populations permanently disagreeing about who is a member, with no event either side can call -forged. Closing V2 removes the puppet and takes this with it; floors alone do not, since they only +forged. Closing B2 removes the puppet and takes this with it; floors alone do not, since they only protect people who were already there. -## V4 — The Refounding recipient set is attacker-inflatable +## B4 — The Refounding recipient set is attacker-inflatable **High.** *Read:* `ConcordCommunitySession.allMembers()` / `emitChannelRumors`; `AccountConcordActions.refoundConcordCommunity` step 2; `ConcordRefounding.buildBaseRekeyWraps`. @@ -152,7 +309,7 @@ This is the cheapest thing on the list to fix and the only one that is not conse the recipient set, prefer recent/attested members when over the cap, and surface what was dropped (a silent truncation strands real members). Worth doing first. -## V5 — The ban is a per-pubkey display rule and the channel key is not revoked +## B5 — The ban is a per-pubkey display rule and the channel key is not revoked **High.** *Read:* `Account.consumeConcordRumorGated` (`isBanned(rumor.pubKey)`), `Account.isAcceptable`. @@ -160,12 +317,12 @@ Writing to a channel needs the channel key, which the ban does not take away; th whatever key the client feels like using. A malicious client therefore posts every message from a fresh npub and `isBanned` never matches — moderation is whack-a-mole against an infinite identity supply. Each message also costs every member two NIP-44 decrypts and two signature verifications -*before* the banlist check runs, and each fresh author inflates V4. +*before* the banlist check runs, and each fresh author inflates B4. There is no client-side answer; only a Refounding rotates the key out from under them. That is the -correct design, which is why V4 matters so much. +correct design, which is why B4 matters so much. -## V6 — Channel history is deletable on a naive third-party relay +## B6 — Channel history is deletable on a naive third-party relay **High, external.** *Verified (that we are safe):* `geode/…/ConcordPlaneKeyDeletionTest.kt` (3 tests). @@ -187,30 +344,7 @@ A community publishes wherever its metadata points. Any relay that authorizes de `pubkey` still hands every ex-member the wipe button, and a Refounding protects only the future. Worth a note in the CORD-01 spec and a line in the relay-selection guidance. -## V7 — Banlist rank rule diverges from Armada - -**Medium, known, deliberate.** See `docs/concord-banlist-rank-conformance.md`, already reported. - -We enforce §3's rank half on the Banlist and Armada does not, so the two clients can show different -banlists. Shipped knowingly. Row 3 of that report ("a banned `BAN` holder unbans themselves") was -left open as a fixpoint-ordering question — V2 is the general form of it, and the fix proposed there -resolves both. - -## V8 — A soft ban revokes no read access and no live invite - -**Medium, inherent.** *Read:* CORD-02/05. - -Until a Refounding, a banned member decrypts everything published — the ban only stops honest -clients from *showing* their posts, not from delivering the group's posts to them. They also keep -any invite links they created while privileged; those still resolve to bundles carrying the current -root. Publishing the root, or one live link, invites an unbanned crowd that each has to be banned -individually (and see V5). - -Not a bug so much as the definition of a soft ban, but it belongs on the list because the UI should -say so: "Ban" and "Remove from community" are very different promises and users will read the first -as the second. - -## V9 — The base-rekey plane is writable by every member +## B7 — The base-rekey plane is writable by every member **Low.** *Read:* `ConcordKeyDerivation.baseRekeyAddress`, `AccountConcordActions.drainConcordRekeys`. @@ -219,63 +353,36 @@ valid wraps there. Authorization happens after the blobs are scanned, so a flood a locator scan per blob on every revision tick. Bounded work per wrap and no correctness impact; listed for completeness. -## V10 — Stranded recovery: either broken, or a removal bypass -**Critical, and it forks — one of the two halves is true and both are bad.** -*Read:* `ConcordStrandedRecovery`, `AccountConcordActions.recoverStrandedConcordCommunities`, -`AccountConcordActions.mintConcordInvite`. +--- -`ConcordStrandedRecovery.isStranded` / `mergeForward` take only `(entry, bundle)`. There is **no -banlist check and no check that we were legitimately re-keyed** — the entire test is "the bundle at -my stored `inviteRef` sits at a higher epoch than I do". The unlock token lives in the link -fragment, which an ex-member keeps forever. So whether a removed member walks back in with the new -root depends *only* on whether the bundle at that coordinate ever advances an epoch. +# Part C — interop and not-yet-shipped -In Amethyst it never does: `mintConcordInvite` mints a **fresh link signer per mint**, so nothing -re-publishes at an existing coordinate, and `refoundConcordCommunity` does not re-mint or revoke -anything. Two consequences, and they are the fork: +## C1 — Banlist rank rule diverges from Armada -- **If nothing re-mints** — today's behaviour — then stranded recovery never fires *for anyone*. - That makes it dead code, and the cure `drainConcordRekeys`' own KDoc points to for "a BAN-holder - can evict anyone (the owner included) by omission" does not exist. An owner evicted by a rogue - admin has no way back. -- **If anything re-mints at a stable coordinate** — which is what CORD-05's design describes ("the - community keeps publishing its bundle at that same addressable coordinate, re-minted at the - current epoch"), so plausibly Armada in a cross-client community — then every removed member who - joined through a still-live link auto-recovers the new root on the 15-minute sweep, and - re-announces a Guestbook join so they look current again. **Refounding, the only hard removal, - is silently undone.** +**Medium, known, deliberate.** See `docs/concord-banlist-rank-conformance.md`, already reported. -Note also that `refoundConcordCommunity` never revokes the invite links the removed member created -or joined through, even though `ControlEntityKind.INVITE_REVOKED` exists and `classifyInvite` -already honors it. +We enforce §3's rank half on the Banlist and Armada does not, so the two clients can show different +banlists. Shipped knowingly. Row 3 of that report ("a banned `BAN` holder unbans themselves") was +left open as a fixpoint-ordering question — B2 is the general form of it, and the fix proposed there +resolves both. -**Fix direction.** Decide the intended semantics first — this needs a spec answer, not a patch. -Then: gate `mergeForward` on not being banned in the epoch we are merging *from*, have the -Refounding revoke the removed members' links, and either implement re-minting (so legitimate -recovery works) or drop the mechanism and give evicted owners a different route. +## C2 — Voice rooms are key-gated, not roster-gated -## V11 — Voice rooms are key-gated, not roster-gated +**Design-level; not currently reachable.** *Read:* `ConcordBrokerToken`, CORD-07 §2. -**High.** *Read:* `ConcordBrokerToken`, CORD-07 §2. +Downgraded from High on review: `ConcordBrokerToken` and `VoicePresence` are referenced nowhere +outside `quartz`, so Amethyst ships no Concord voice path yet. This is a note for whoever wires +one up, not a live hole. A member proves voice-room membership by signing a NIP-98 kind-27235 request with the channel's **derived voice signer key**, whose pubkey is the SFU room name. The broker is stateless and holds no community secret, so it cannot consult the Control Plane and has no idea a banlist exists. A banned member keeps that key until a Refounding, so they can join the voice room and stay in it. Nothing on the client side can evict them — kicking them from the UI does not kick them from the SFU. +It would be the one place where a ban fails *audibly*, in real time, in front of everyone, so it is +worth designing the roster check in before shipping rather than after. -This is the one place where a ban fails *audibly*, in real time, in front of everyone. Worth ranking -above its technical severity for that reason alone. - -## V12 — Typing indicators are not ban-filtered - -**Low.** *Read:* `ConcordCommunitySession.ingestTyping`. - -`ingestTyping` checks the rumor is a typing heartbeat, is bound to the channel/epoch, and is not our -own — and nothing else. A banned member (or any fresh npub holding the channel key, see V5) shows -in the "… is typing" row indefinitely. Cheap to fix and user-visible: the promise a ban makes is -that the member disappears, and here they do not. --- @@ -284,7 +391,7 @@ that the member disappears, and here they do not. This audit is bounded by what was opened. Checked and found sound: the wrap/seal envelope (no author impersonation — `rumor.pubKey == seal.pubKey` and `rumor.verifyId()`), Concord chat edits (`Note.latestConcordEdit` is author-gated, so a member cannot rewrite someone else's message), and -self-unban (V2). +self-unban (B2). Not looked at at all: @@ -292,7 +399,7 @@ Not looked at at all: Note that no channel-scoped rekey *receive* path appears to exist: `drainConcordRekeys` handles `ROOT_SCOPE` only, and `entry.privateChannels` is carried forward but never populated by a delivery path. If that is right, the only removal Amethyst can perform is a full-community - Refounding — which is exactly what V4 makes expensive. + Refounding — which is exactly what B4 makes expensive. - **In-plane reactions and deletes** — the edit path is author-gated; the delete path was not read. - **Guestbook kicks** (kind 3309) — the builder documents a KICK-bit + rank rule; the receive side was not verified against it. @@ -301,11 +408,23 @@ Not looked at at all: ## Suggested order -1. **V4** — cheapest, not consensus-affecting, and it protects the remedy every other fix depends on. -2. **V1** — worst blast radius and the only unrecoverable one; does not need an attacker to be - banned or privileged beyond a single ordinary bit. -3. **V2** (+V3, +V7 row 3) — one two-pass change closes all three. Coordinate with Armada first; - this one splits consensus. -4. **V6** — spec note + relay guidance; our own behaviour is already correct and now pinned. -5. **V5 / V8** — UI honesty about what a ban does, and a "Remove from community" affordance that - Refounds rather than bans. +**Part A first.** It is the whole of the realistic threat — a banned user with the app already +installed — and none of it needs coordination with anyone. + +1. **A1** — one guard on `mintConcordInvite` plus one on its button. Smallest fix on the list and it + closes the attack a banned user will actually reach for. +2. **A3** — move authority into the action layer and replace `effectivePermissions` with + `hasPermission` everywhere it is used as an authorization test. This is also the cheapest partial + mitigation for B2: it shrinks what a banned staffer can do *without* writing their own client. +3. **A2** — needs the semantics decided before any code. Raise it with the spec. +4. **A4 / A5** — small, user-visible, and they make the product honest about what a ban is. + +**Then Part B**, hardest first because the ceiling is highest: + +5. **B4** — cheap, not consensus-affecting, and it protects the remedy every other fix depends on. +6. **B1** — worst blast radius, the only unrecoverable one, and the bar is a single ordinary + permission bit. +7. **B2 (+B3, +C1's open row)** — one two-pass change closes all three. Coordinate with Armada + first; this one splits consensus. +8. **B6** — spec note plus relay-selection guidance; our own behaviour is already correct and pinned. +9. **B5 / B7** — accept, or bound. From 37d715830b68639392f6098ba6d1b1a7f87a7622 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Sun, 9 Aug 2026 04:24:35 +0000 Subject: [PATCH 082/132] Let a drained paged walk close the leg below it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A paged band records the events it SAW, never the range it asked for, so `legs()` can only ever say "walked this far" and keeps re-asking the leg below the floor. Against a relay whose corpus for one kind simply starts later than the others' that leg is unclosable: it comes back empty every cycle, an empty fetch earns no band, so the floor never moves. Measured on a live mirror of five NIP-65 indexers, three were in that state — kind 10002 re-walked from the beginning of time to Feb 2023 forever, because relay lists did not exist before then. The missing fact is why a page ended. `fetchAllPages` treated all three terminal signals as one bare `Unit`, so an empty page could not be told apart from silence or a CLOSED. It now carries a PageEnd, and reports `onDrained` only for the one ending that proves absence: an EOSE on a page that returned nothing, with no filter capped by its `limit` and no `search` filter in play (both stop the walk short of the corpus). An idle timeout is silence, not an answer, and recording it would durably claim coverage the relay never served. A callback rather than a richer return type: ~25 call sites across quartz, geode and downstream use the `Int`, and none should have to change to learn a fact they do not want. It follows `onNewPage`'s shape. `SyncCoverage.record` takes `drained` and marks the kinds that produced evidence complete — which required completeness to move from Band onto Span. It could not stay on the band: once kinds diverge, `legs()` hands each group its own ask, so a walk that drained `kinds: [10002]` proves nothing about kind 0, and a band-level flag set from that leg would claim both. That is the same over-claim per-kind spans exist to prevent, one level up. `Band.complete` stays as a DERIVED all-kinds-complete, so both state files keep writing the flag a pre-per-kind reader expects, and read it back as every span's default. A kind the walk never saw at all still earns nothing: there is no interval to anchor a claim to, and inventing one would be the over-claim again. Tests: five in NostrClientFetchAllPagesDrainTest pinning EOSE-empty vs silence vs CLOSED vs cannot-connect vs a fulfilled limit, and five in SyncCoverageTest for per-kind completeness, widening, and the deeper-floor escape hatch that a drain must not defeat. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016TNy5BsU9NErXYa3UNGTeJ --- .../geode/mirror/SyncCoverageFile.kt | 20 +- .../NostrClientFetchAllPagesExt.kt | 77 +++++- .../relay/client/accessories/SyncCoverage.kt | 78 ++++-- .../client/accessories/SyncCoverageTest.kt | 120 ++++++++- .../NostrClientFetchAllPagesDrainTest.kt | 230 ++++++++++++++++++ 5 files changed, 495 insertions(+), 30 deletions(-) create mode 100644 quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesDrainTest.kt diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt index ea5f9aad21..a13fa57420 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/SyncCoverageFile.kt @@ -108,7 +108,6 @@ class SyncCoverageFile( key to SyncCoverage.Band( spansOf(o), - o["complete"]?.jsonPrimitive?.boolean ?: false, o["fullAt"]?.jsonPrimitive?.long ?: 0L, ) }.toMap(), @@ -128,17 +127,31 @@ class SyncCoverageFile( * discarded, and the first paged walk that reports per kind replaces it. * Dropping it instead would re-download every upstream's corpus once on * upgrade, which is the cost bands exist to avoid. + * + * Completeness is read the same way, one level down: a span written before + * it was per kind has no `complete` of its own, so it inherits the band's — + * which is precisely what that flag used to mean for every kind at once. */ private fun spansOf(o: JsonObject): Map { + val bandComplete = o["complete"]?.jsonPrimitive?.boolean ?: false o["spans"]?.jsonObject?.let { spans -> return spans.entries.associate { (kind, v) -> val span = v.jsonObject - kind.toInt() to SyncCoverage.Span(span.getValue("min").jsonPrimitive.long, span.getValue("max").jsonPrimitive.long) + kind.toInt() to + SyncCoverage.Span( + span.getValue("min").jsonPrimitive.long, + span.getValue("max").jsonPrimitive.long, + span["complete"]?.jsonPrimitive?.boolean ?: bandComplete, + ) } } return mapOf( SyncCoverage.ALL_KINDS to - SyncCoverage.Span(o.getValue("min").jsonPrimitive.long, o.getValue("max").jsonPrimitive.long), + SyncCoverage.Span( + o.getValue("min").jsonPrimitive.long, + o.getValue("max").jsonPrimitive.long, + bandComplete, + ), ) } @@ -170,6 +183,7 @@ class SyncCoverageFile( buildJsonObject { put("min", span.min) put("max", span.max) + put("complete", span.complete) }, ) } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesExt.kt index f7b4834515..e08d432ee1 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesExt.kt @@ -32,6 +32,24 @@ import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.ensureActive import kotlin.coroutines.coroutineContext +/** + * Why one page stopped. The three terminal signals used to be indistinguishable — + * all of them put a bare `Unit` on the page's channel — and [fetchAllPages] only + * ever asked *whether* a page ended, never *how*. [onDrained] needs the + * difference: an empty page is proof the relay has nothing older only when the + * relay actually said so. + */ +private enum class PageEnd { + /** The relay finished serving its stored events for this REQ. */ + EOSE, + + /** The relay ended the subscription itself — auth required, rate limited, policy. */ + CLOSED, + + /** Never got to ask. */ + CANNOT_CONNECT, +} + /** * Downloads all pages of events matching [filters] from a single [relay] using * paginated `until` cursors. @@ -87,6 +105,19 @@ import kotlin.coroutines.coroutineContext * bounds this walk is a [Filter.limit] (the documented way to cap a download) or * cancelling the caller, which the [ensureActive] at the top of each page honors. * @param onEvent Called once for every distinct event delivered, in page order. + * @param onDrained Called at most once, just before returning, when the walk ended + * because the relay served everything [filters] match at-or-below the starting + * `until` — an empty page confirmed by an EOSE. That is the difference between + * "the relay has nothing older" and "the relay stopped answering", which the + * `Int` return cannot express: a caller recording sync coverage may treat the + * range below the oldest event it saw as verified-empty ONLY in the first case. + * Every other ending — an idle timeout, a CLOSED, a failed connect, a fulfilled + * [Filter.limit] — leaves it unfired, because none of them prove absence. + * + * A callback rather than a richer return type on purpose: this function has + * ~25 call sites across quartz, geode and downstream repos that use the `Int`, + * and none of them should have to change to learn a fact they do not want. It + * follows the shape [onNewPage] already set. * @return Total number of distinct events delivered across all pages. */ suspend fun INostrClient.fetchAllPages( @@ -94,10 +125,12 @@ suspend fun INostrClient.fetchAllPages( filters: List, idleTimeoutMs: Long = 30_000L, onNewPage: ((Long) -> Unit)? = null, + onDrained: (() -> Unit)? = null, onEvent: suspend (Event) -> Unit, ): Int { var until: Long? = null var totalEvents = 0 + var drained = false // Track how many matching events each filter has received so far. val matchCountPerFilter = IntArray(filters.size) @@ -155,7 +188,7 @@ suspend fun INostrClient.fetchAllPages( // REQ, so firing this earlier would report a page that never happens. if (until != null) onNewPage?.invoke(until) - val doneChannel = Channel(Channel.CONFLATED) + val doneChannel = Channel(Channel.CONFLATED) // Idle watchdog for this page: every arriving event bumps it, so the page's // timeout measures silence since the relay's most recent message (the same @@ -169,6 +202,12 @@ suspend fun INostrClient.fetchAllPages( var pageMinTs = Long.MAX_VALUE val idsAtPageMin = HashSet() + // How this page ended, read after the wait: null for an idle timeout, which + // [receiveWithinIdle] reports by returning null. Only an EOSE can support a + // drain claim below — silence is not an answer, and a CLOSED is the relay + // declining to give one. + var pageEnd: PageEnd? = null + try { val listener = object : SubscriptionListener { @@ -241,7 +280,7 @@ suspend fun INostrClient.fetchAllPages( relay: NormalizedRelayUrl, forFilters: List?, ) { - doneChannel.trySend(Unit) + doneChannel.trySend(PageEnd.EOSE) } override fun onClosed( @@ -249,7 +288,7 @@ suspend fun INostrClient.fetchAllPages( relay: NormalizedRelayUrl, forFilters: List?, ) { - doneChannel.trySend(Unit) + doneChannel.trySend(PageEnd.CLOSED) } override fun onCannotConnect( @@ -257,7 +296,7 @@ suspend fun INostrClient.fetchAllPages( message: String, forFilters: List?, ) { - doneChannel.trySend(Unit) + doneChannel.trySend(PageEnd.CANNOT_CONNECT) } } @@ -266,7 +305,7 @@ suspend fun INostrClient.fetchAllPages( // Wait for the page's terminal signal (EOSE / CLOSED / cannot-connect), // giving up only after [idleTimeoutMs] of silence — the wait resets on every // arriving event, so an actively streaming page is never cut mid-delivery. - doneChannel.receiveWithinIdle(clock, idleTimeoutMs) + pageEnd = doneChannel.receiveWithinIdle(clock, idleTimeoutMs) unsubscribe(subId) doneChannel.close() @@ -277,8 +316,26 @@ suspend fun INostrClient.fetchAllPages( totalEvents += delivered - // The relay sent nothing at-or-below `until` → the whole set is drained. - if (received == 0) break + // The relay sent nothing at-or-below `until`. Whether that DRAINS the set + // depends on why the page ended and on what was asked: + // + // - only an EOSE proves absence. An idle timeout (`pageEnd == null`) is + // silence and a CLOSED is the relay declining to answer; reading either + // as "nothing older exists" would durably record coverage the relay + // never served, which is the one error a coverage claim must not make. + // - a filter that reached its [Filter.limit] stopped early on the caller's + // own instruction, so nothing below its last event was ever asked for. + // - a `search` filter runs on the first page only, so every page after it + // dropped out never carried it and cannot speak for it. + if (received == 0) { + val cappedByLimit = + filters.indices.any { i -> + val limit = filters[i].limit + limit != null && matchCountPerFilter[i] >= limit + } + drained = pageEnd == PageEnd.EOSE && !cappedByLimit && filters.none { it.search != null } + break + } if (delivered == 0) { // Every event this page was a boundary-second duplicate; nothing older @@ -312,6 +369,10 @@ suspend fun INostrClient.fetchAllPages( until = nextUntil } + // After the loop, not at the break: every other exit above leaves `drained` + // false, and firing from one place keeps "at most once" true by construction. + if (drained) onDrained?.invoke() + return totalEvents } @@ -320,6 +381,7 @@ suspend fun INostrClient.fetchAllPages( filters: List, idleTimeoutMs: Long = 30_000L, onNewPage: ((Long) -> Unit)? = null, + onDrained: (() -> Unit)? = null, onEvent: suspend (Event) -> Unit, ): Int = fetchAllPages( @@ -327,5 +389,6 @@ suspend fun INostrClient.fetchAllPages( filters = filters, idleTimeoutMs = idleTimeoutMs, onNewPage = onNewPage, + onDrained = onDrained, onEvent = onEvent, ) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt index c8e2e3aed8..03a0ec6d40 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverage.kt @@ -102,12 +102,27 @@ class SyncCoverage( } } - /** A covered `created_at` interval, inclusive at both ends. */ + /** + * A covered `created_at` interval, inclusive at both ends. + * + * [complete] is the difference between "we walked this interval" and "there + * is nothing below it". A paged fetch earns the first by seeing events; it + * earns the second only when the relay EOSEs on an empty page, which is the + * one answer that distinguishes an exhausted corpus from a relay that capped + * us or went quiet (see `fetchAllPages`'s `onDrained`). A finished negentropy + * reconcile earns it too, by comparing the whole range at once. + * + * It lives HERE rather than on [Band] because a paged leg does not have to + * cover every kind: once kinds diverge, [legs] hands each group its own ask, + * so a walk that drained `kinds: [10002]` says nothing about kind 0. A + * band-level flag set from such a leg would claim both. + */ data class Span( val min: Long, val max: Long, + val complete: Boolean = false, ) { - fun widen(other: Span) = Span(minOf(min, other.min), maxOf(max, other.max)) + fun widen(other: Span) = Span(minOf(min, other.min), maxOf(max, other.max), complete || other.complete) } /** @@ -125,30 +140,36 @@ class SyncCoverage( * span under [ALL_KINDS] — the same claim as before, correctly scoped to * the case where it is the only claim available. * - * [complete] is the difference between "we walked this span" (a paged - * fetch) and "we are in sync below this point" (a finished negentropy - * reconcile, which compared the whole range). Only a complete band may - * skip its older leg. It is a property of the BAND rather than of a span: - * a reconcile compares the filter's whole id set at once, so it either - * covers every kind in it or none. + * Completeness is per kind, on [Span] — see there for why a paged leg + * cannot speak for kinds it did not ask about. * * [fullAt] is when the last pass that started from nothing finished — the * clock for the periodic re-walk. */ data class Band( val spans: Map, - val complete: Boolean = false, val fullAt: Long = 0, ) { /** The outer edges across every kind — for logging and for the file's compatibility fields. */ val minCreatedAt: Long get() = spans.values.minOfOrNull { it.min } ?: 0 val maxCreatedAt: Long get() = spans.values.maxOfOrNull { it.max } ?: 0 + /** + * The band-level claim, DERIVED: true only when every kind is complete. + * + * Kept for the state files, which still write a `complete` beside + * `min`/`max` so a build from before per-kind completeness reads them and + * behaves as it always did. `all` rather than `any` is the safe direction + * for that reader: it cannot see the per-kind detail, so it must be told + * the weakest true thing, not the strongest. + */ + val complete: Boolean get() = spans.isNotEmpty() && spans.values.all { it.complete } + /** Widen each kind by its counterpart, keeping kinds only one side knows. */ fun widen(other: Band): Band { val merged = spans.toMutableMap() for ((kind, span) in other.spans) merged[kind] = merged[kind]?.widen(span) ?: span - return Band(merged, complete || other.complete, fullAt) + return Band(merged, fullAt) } } @@ -189,7 +210,8 @@ class SyncCoverage( val kinds = filter.kinds if (kinds.isNullOrEmpty()) { // Nothing to split by. One span, exactly as before. - return windows(filter, band.spans[ALL_KINDS], band.complete, floor) + val span = band.spans[ALL_KINDS] + return windows(filter, span, span?.complete == true, floor) .map { (since, until) -> filter.copy(since = since, until = until) } } @@ -205,7 +227,12 @@ class SyncCoverage( // wider claim for every kind — the behaviour this replaces — and // self-corrects on the first paged walk that reports per kind. val span = band.spans[kind] ?: band.spans[ALL_KINDS] - byWindows.getOrPut(windows(filter, span, band.complete, floor)) { mutableListOf() }.add(kind) + // Completeness comes from the SPAN, so a leg that drained one kind + // drops only that kind's older leg. Before this it came from the + // band, and a reconcile was the only thing that could set it — which + // is why a drained paged walk over `kinds: [10002]` used to leave an + // older leg that no future walk could ever close. + byWindows.getOrPut(windows(filter, span, span?.complete == true, floor)) { mutableListOf() }.add(kind) } return byWindows.flatMap { (windows, group) -> // toList(): `group` is the mutable accumulator above, and handing @@ -266,6 +293,19 @@ class SyncCoverage( * the sync STARTED — recorded against that instant rather than the newest * event seen, because "the relay had nothing newer" and "we never asked" * must not look alike. + * + * [drained] is the paged equivalent, and the only way a paged walk can ever + * claim its older leg is finished. Pass it from `fetchAllPages`'s + * `onDrained` — the relay EOSEd on an empty page, so there is nothing below + * what was seen. Without it a paged band only ever says "walked this far", + * and the leg below it is re-asked every cycle forever: against a relay + * whose corpus for a kind simply starts later than the others', that leg + * returns nothing, records nothing, and so can never close itself. + * + * It marks only the kinds that produced evidence. A kind the walk never saw + * at all has no interval to anchor a claim to, and inventing one would be + * the over-claim [Span] exists to prevent — so it keeps no band and is + * asked again, which is the safe direction. */ fun record( url: NormalizedRelayUrl, @@ -275,13 +315,14 @@ class SyncCoverage( paged: Boolean, reconciledThrough: Long? = null, observedByKind: Map? = null, + drained: Boolean = false, ) { if (reconciledThrough != null) { // A reconcile compares the filter's whole id set in one pass, so // the span it earns is the same for every kind the filter names — // no per-kind evidence needed or possible. - val span = Span(observedMin ?: reconciledThrough, reconciledThrough) - put(url, filter, kindsOf(filter).associateWith { span }, complete = true) + val span = Span(observedMin ?: reconciledThrough, reconciledThrough, complete = true) + put(url, filter, kindsOf(filter).associateWith { span }) return } if (!paged) return @@ -298,7 +339,7 @@ class SyncCoverage( } if (plausible.isEmpty()) return val named = filter.kinds - val spans = + val spans: Map = if (named.isNullOrEmpty()) { // A filter naming no kinds cannot be split, so [legs] reads // ALL_KINDS and nothing else. Storing what the walk saw per @@ -321,7 +362,7 @@ class SyncCoverage( plausible.filterKeys { it in named } } if (spans.isEmpty()) return - put(url, filter, spans, complete = false) + put(url, filter, if (drained) spans.mapValues { it.value.copy(complete = true) } else spans) return } @@ -349,7 +390,7 @@ class SyncCoverage( // range nothing can be in, forever. if (observedMin == null || observedMax == null) return if (!isPlausible(observedMin, at) || !isPlausible(observedMax, at)) return - put(url, filter, kinds.associateWith { Span(observedMin, observedMax) }, complete = false) + put(url, filter, kinds.associateWith { Span(observedMin, observedMax, complete = drained) }) } /** The kinds a band is keyed by: the filter's, or [ALL_KINDS] when it names none. */ @@ -364,9 +405,8 @@ class SyncCoverage( url: NormalizedRelayUrl, filter: Filter, spans: Map, - complete: Boolean, ) { - val fresh = Band(spans, complete, now()) + val fresh = Band(spans, now()) bands.merge(key(url, filter), fresh) { old, new -> if (isStale(old)) new else old.widen(new) } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt index abf1113f02..3ead83ba3c 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/SyncCoverageTest.kt @@ -25,6 +25,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer import com.vitorpamplona.quartz.utils.TimeUtils import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertFalse import kotlin.test.assertNull import kotlin.test.assertSame import kotlin.test.assertTrue @@ -511,7 +512,6 @@ class SyncCoverageTest { key to SyncCoverage.Band( mapOf(SyncCoverage.ALL_KINDS to SyncCoverage.Span(1_690_000_000L, 1_700_000_000L)), - complete = false, fullAt = now(), ), ), @@ -579,4 +579,122 @@ class SyncCoverageTest { assertEquals(2, c.legs(relay, anyKind).size) assertEquals(1_690_000_000L, c.legs(relay, anyKind)[0].until) } + + // ---- draining: the leg a paged walk is finally allowed to close --------- + + @Test + fun `a drained paged walk stops asking about the past`() { + // THE OTHER HALF OF THE BUG ABOVE. Per-kind spans stopped kind 0 from + // vouching for 30382 — but they left 30382 an older leg that nothing + // could ever close. The relay's corpus for it simply starts later, so + // that leg comes back empty every cycle, records nothing (an empty + // fetch earns no band), and the floor never moves. Forever. + // + // A drain is the missing evidence: the relay EOSEd on an empty page, so + // there IS nothing below what we saw, and the leg is done. + val walked = SyncCoverage() + walked.record(relay, profiles, 1_690_000_000L, 1_700_000_000L, paged = true) + assertEquals(2, walked.legs(relay, profiles).size, "not drained: still asks below the floor") + + val drained = SyncCoverage() + drained.record(relay, profiles, 1_690_000_000L, 1_700_000_000L, paged = true, drained = true) + val legs = drained.legs(relay, profiles) + assertEquals(1, legs.size, "drained: only the newer leg is left") + assertEquals(1_700_000_000L, legs[0].since) + assertNull(legs[0].until) + } + + @Test + fun `a drained leg closes its own kind and not the others`() { + // Why completeness had to move from the band onto the span. After the + // kinds diverge, legs() hands each group its own ask — so a walk that + // drained `kinds: [30382]` proves nothing whatever about kind 0. A + // band-level flag set from that leg would have claimed both, which is + // the same over-claim per-kind spans exist to prevent, just one level up. + val c = SyncCoverage() + c.record( + relay, + mixed, + null, + null, + paged = true, + observedByKind = mapOf(30382 to SyncCoverage.Span(1_690_000_000L, 1_700_000_000L)), + drained = true, + ) + // Kind 0 has no evidence at all here, so it keeps asking for everything. + c.record( + relay, + mixed, + null, + null, + paged = true, + observedByKind = mapOf(0 to SyncCoverage.Span(1_600_000_000L, 1_700_000_000L)), + ) + + val legs = c.legs(relay, mixed) + assertTrue(!reaches(legs, 30382, 1_650_000_000L), "30382 drained — its past is settled") + assertTrue(reaches(legs, 0, 1_500_000_000L), "kind 0 did not drain, so its older leg stands") + } + + @Test + fun `a band is complete only when every kind is`() { + // The band-level flag is derived now, and the state files still write it + // for a reader that predates per-kind completeness. That reader cannot + // see the detail, so it has to be told the weakest true thing. + val c = SyncCoverage() + c.record( + relay, + mixed, + null, + null, + paged = true, + observedByKind = mapOf(0 to SyncCoverage.Span(1_690_000_000L, 1_700_000_000L)), + drained = true, + ) + assertTrue(c.band(relay, mixed)!!.complete, "the only kind with a span drained") + + c.record( + relay, + mixed, + null, + null, + paged = true, + observedByKind = mapOf(30382 to SyncCoverage.Span(1_695_000_000L, 1_700_000_000L)), + ) + assertFalse(c.band(relay, mixed)!!.complete, "…and now one of the two has not") + } + + @Test + fun `widening carries a drain forward rather than losing it`() { + // Two legs of one walk against the same relay land in the same span. + // Widening must not let the second, undrained one erase what the first + // proved: the past below the merged floor really was checked. + val c = SyncCoverage() + c.record(relay, profiles, 1_690_000_000L, 1_695_000_000L, paged = true, drained = true) + c.record(relay, profiles, 1_696_000_000L, 1_700_000_000L, paged = true) + + assertTrue( + c + .band(relay, profiles)!! + .spans + .getValue(0) + .complete, + ) + assertEquals(1, c.legs(relay, profiles).size, "still done with the past") + } + + @Test + fun `a deeper floor still re-opens history below a drained band`() { + // A drain says "nothing below what this walk asked for", not "nothing + // below, ever". A caller that now reaches deeper than the band's floor + // gets its older leg back — the same escape hatch a finished reconcile + // has, and the reason `since` is consulted at all. + val c = SyncCoverage() + c.record(relay, profiles, 1_690_000_000L, 1_700_000_000L, paged = true, drained = true) + + assertEquals(1, c.legs(relay, profiles).size) + val deeper = c.legs(relay, profiles, floor = 1_600_000_000L) + assertEquals(2, deeper.size, "the caller's floor dropped below the band, so the past re-opens") + assertEquals(1_690_000_000L, deeper[0].until) + } } diff --git a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesDrainTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesDrainTest.kt new file mode 100644 index 0000000000..e8797a8d0f --- /dev/null +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesDrainTest.kt @@ -0,0 +1,230 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay + +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.relay.client.EmptyNostrClient +import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.fetchAllPages +import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +/** + * Pins `onDrained` — the one signal that tells a caller the relay served + * *everything* below where the walk stopped, rather than merely stopping there. + * + * The distinction is invisible in the `Int` return and matters enormously to + * anything recording sync coverage: without it, "the relay has nothing older" + * and "the relay capped us / went quiet / hung up" look identical, so a coverage + * band can never close its oldest leg and re-asks a range that will always come + * back empty, every cycle, forever. + * + * Real-clock ([runBlocking]) for the same reason the idle-timeout suite is: the + * page watchdog is a monotonic clock bumped from the socket reader thread. + */ +class NostrClientFetchAllPagesDrainTest { + /** Captures the subscription listener so the test can play a relay, page by page. */ + private class ScriptedClient : INostrClient by EmptyNostrClient() { + @Volatile + var listener: SubscriptionListener? = null + + @Volatile + var subscribeCount = 0 + + override fun subscribe( + subId: String, + filters: Map>, + listener: SubscriptionListener?, + ) { + subscribeCount++ + this.listener = listener + } + + /** Block until the walk has opened its [n]th page, so a script can answer it. */ + suspend fun awaitPage(n: Int) { + while (subscribeCount < n) delay(2) + } + } + + private val relay = RelayUrlNormalizer.normalize("wss://drain.example.com") + + private fun event(createdAt: Long) = + Event( + id = createdAt.toString(16).padStart(64, '0'), + pubKey = "f".repeat(64), + createdAt = createdAt, + kind = 1, + tags = emptyArray(), + content = "e$createdAt", + sig = "0".repeat(128), + ) + + @Test + fun anEmptyPageConfirmedByEoseDrains() = + runBlocking { + val client = ScriptedClient() + val feeder = + launch { + client.awaitPage(1) + client.listener!!.onEvent(event(2000), false, relay, null) + client.listener!!.onEvent(event(1000), false, relay, null) + client.listener!!.onEose(relay, null) + + // The second page asks below 1000 and the relay says, with an + // EOSE, that it has nothing. THAT is a drain. + client.awaitPage(2) + client.listener!!.onEose(relay, null) + } + + var drained = false + val total = + client.fetchAllPages( + relay = relay, + filters = listOf(Filter(kinds = listOf(1))), + idleTimeoutMs = 2_000, + onDrained = { drained = true }, + ) { } + feeder.join() + + assertEquals(2, total) + assertTrue(drained, "an empty page the relay EOSEd is proof there is nothing older") + } + + @Test + fun aSilentPageDoesNotDrain() = + runBlocking { + val client = ScriptedClient() + val feeder = + launch { + client.awaitPage(1) + client.listener!!.onEvent(event(2000), false, relay, null) + client.listener!!.onEvent(event(1000), false, relay, null) + client.listener!!.onEose(relay, null) + // Page two: the relay simply stops answering. Silence is not an + // answer — reading it as "nothing older exists" would durably + // record coverage that was never served. + client.awaitPage(2) + } + + var drained = false + val total = + client.fetchAllPages( + relay = relay, + filters = listOf(Filter(kinds = listOf(1))), + idleTimeoutMs = 200, + onDrained = { drained = true }, + ) { } + feeder.join() + + assertEquals(2, total, "the events already delivered are still kept") + assertFalse(drained, "an idle timeout says nothing about what the relay holds") + } + + @Test + fun aClosedSubscriptionDoesNotDrain() = + runBlocking { + val client = ScriptedClient() + val feeder = + launch { + client.awaitPage(1) + client.listener!!.onEvent(event(2000), false, relay, null) + client.listener!!.onEvent(event(1000), false, relay, null) + client.listener!!.onEose(relay, null) + // Page two: the relay ends the subscription instead of serving + // it — auth-required, rate limit, policy. It declined to answer, + // which is not the same as answering "nothing". + client.awaitPage(2) + client.listener!!.onClosed("auth-required: we don't serve that", relay, null) + } + + var drained = false + client.fetchAllPages( + relay = relay, + filters = listOf(Filter(kinds = listOf(1))), + idleTimeoutMs = 2_000, + onDrained = { drained = true }, + ) { } + feeder.join() + + assertFalse(drained, "a CLOSED is the relay declining, not an empty corpus") + } + + @Test + fun aRelayItCannotReachDoesNotDrain() = + runBlocking { + val client = ScriptedClient() + val feeder = + launch { + client.awaitPage(1) + client.listener!!.onCannotConnect(relay, "connection refused", null) + } + + var drained = false + client.fetchAllPages( + relay = relay, + filters = listOf(Filter(kinds = listOf(1))), + idleTimeoutMs = 2_000, + onDrained = { drained = true }, + ) { } + feeder.join() + + assertFalse(drained, "never got to ask") + } + + @Test + fun aFulfilledLimitDoesNotDrain() = + runBlocking { + // The caller bounded the download itself, so the walk stopped on its + // own instruction rather than at the end of the relay's corpus. + // Nothing below the last event was ever asked for. + val client = ScriptedClient() + val feeder = + launch { + client.awaitPage(1) + client.listener!!.onEvent(event(2000), false, relay, null) + client.listener!!.onEvent(event(1000), false, relay, null) + client.listener!!.onEose(relay, null) + } + + var drained = false + val total = + client.fetchAllPages( + relay = relay, + filters = listOf(Filter(kinds = listOf(1), limit = 2)), + idleTimeoutMs = 2_000, + onDrained = { drained = true }, + ) { } + feeder.join() + + assertEquals(2, total) + assertEquals(1, client.subscribeCount, "the limit was met, so there was no second page") + assertFalse(drained, "a fulfilled limit is the caller stopping, not the corpus ending") + } +} From 5136a97b16c737382a54bab04d6b57b0f1479cda Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Sun, 9 Aug 2026 05:25:06 +0000 Subject: [PATCH 083/132] Return the walk's outcome instead of signalling a drain by callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `onDrained` was the wrong shape. It reported the one ending a coverage caller happens to need and threw the rest away, so a CLOSED and an idle timeout still arrived indistinguishable from a clean finish — the very conflation this branch set out to remove, just moved one step along. `fetchAllPages` now returns `PagedFetchResult(downloaded, end)`, where `end` names every way the loop can stop: DRAINED, LIMIT_REACHED, IDLE, CLOSED, CANNOT_CONNECT, UNPAGEABLE. `drained` stays as a shorthand on the result so the meaning lives in one place. A caller can no longer ignore the reason by accident, and the two failure endings are now reportable rather than silently swallowed. I argued for the callback on the grounds that ~25 call sites use the `Int`. That was overstated: most call it as a statement and never touch the return. Six needed a `.downloaded`, all mechanical. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016TNy5BsU9NErXYa3UNGTeJ --- .../geode/mirror/MirrorWorker.kt | 2 +- .../NostrClientFetchAllPagesExt.kt | 139 +++++++++++++----- .../NostrClientFetchAllPagesPoolExt.kt | 4 +- .../NostrClientFetchAllPagesDrainTest.kt | 73 +++++---- ...NostrClientFetchAllPagesIdleTimeoutTest.kt | 4 +- .../NostrClientReqBypassingRelayLimitsTest.kt | 6 +- 6 files changed, 145 insertions(+), 83 deletions(-) diff --git a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt index e10f2b14bf..b78c0077a9 100644 --- a/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt +++ b/geode/src/main/kotlin/com/vitorpamplona/geode/mirror/MirrorWorker.kt @@ -548,7 +548,7 @@ class MirrorWorker( // The watchdog matches negentropySync's default rather // than fetchAllPages' shorter one: a paged catch-up // sits behind the same slow upstreams. - downloaded += client.fetchAllPages(up.url, listOf(leg), idleTimeoutMs = 120_000L) { observe(it) } + downloaded += client.fetchAllPages(up.url, listOf(leg), idleTimeoutMs = 120_000L) { observe(it) }.downloaded true } paged = paged || legPaged diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesExt.kt index e08d432ee1..203311e8e4 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesExt.kt @@ -33,13 +33,13 @@ import kotlinx.coroutines.ensureActive import kotlin.coroutines.coroutineContext /** - * Why one page stopped. The three terminal signals used to be indistinguishable — + * Why ONE page stopped. The three terminal signals used to be indistinguishable — * all of them put a bare `Unit` on the page's channel — and [fetchAllPages] only - * ever asked *whether* a page ended, never *how*. [onDrained] needs the + * ever asked *whether* a page ended, never *how*. [PagedFetchResult] needs the * difference: an empty page is proof the relay has nothing older only when the * relay actually said so. */ -private enum class PageEnd { +private enum class PageSignal { /** The relay finished serving its stored events for this REQ. */ EOSE, @@ -50,6 +50,62 @@ private enum class PageEnd { CANNOT_CONNECT, } +/** + * What a [fetchAllPages] walk delivered, and why it stopped. + * + * The count alone cannot answer the question that matters to anything recording + * coverage: is there nothing older, or did the relay simply stop giving us more? + * Those look identical from `downloaded`, and a caller that guesses wrong either + * re-walks a corpus forever or claims history it never read. + */ +data class PagedFetchResult( + /** Total number of distinct events delivered across all pages. */ + val downloaded: Int, + val end: End, +) { + enum class End { + /** + * A page came back empty and the relay EOSEd it: there is nothing at or + * below the last cursor. The only ending that proves ABSENCE, and so the + * only one a coverage claim may be built on. + */ + DRAINED, + + /** + * A [Filter.limit] was fulfilled. The caller bounded the download itself, + * so the walk stopped on its own instruction, not at the end of the + * corpus — nothing below the last event was ever asked for. + */ + LIMIT_REACHED, + + /** + * The relay went quiet for `idleTimeoutMs` without ending the page. + * Silence is not an answer; everything delivered so far is still good. + */ + IDLE, + + /** The relay ended the subscription — auth required, rate limited, policy. */ + CLOSED, + + /** Never got to ask. */ + CANNOT_CONNECT, + + /** + * The walk cannot advance its cursor: only `search` hits came back (NIP-50 + * results are relevance-ranked, so they never page), or a first page + * delivered nothing any active filter matched. + */ + UNPAGEABLE, + } + + /** + * Shorthand for the one ending that licenses skipping work later. Read this + * rather than comparing to [End.DRAINED] by hand, so the meaning stays in one + * place if the enum grows. + */ + val drained: Boolean get() = end == End.DRAINED +} + /** * Downloads all pages of events matching [filters] from a single [relay] using * paginated `until` cursors. @@ -105,32 +161,25 @@ private enum class PageEnd { * bounds this walk is a [Filter.limit] (the documented way to cap a download) or * cancelling the caller, which the [ensureActive] at the top of each page honors. * @param onEvent Called once for every distinct event delivered, in page order. - * @param onDrained Called at most once, just before returning, when the walk ended - * because the relay served everything [filters] match at-or-below the starting - * `until` — an empty page confirmed by an EOSE. That is the difference between - * "the relay has nothing older" and "the relay stopped answering", which the - * `Int` return cannot express: a caller recording sync coverage may treat the - * range below the oldest event it saw as verified-empty ONLY in the first case. - * Every other ending — an idle timeout, a CLOSED, a failed connect, a fulfilled - * [Filter.limit] — leaves it unfired, because none of them prove absence. - * - * A callback rather than a richer return type on purpose: this function has - * ~25 call sites across quartz, geode and downstream repos that use the `Int`, - * and none of them should have to change to learn a fact they do not want. It - * follows the shape [onNewPage] already set. - * @return Total number of distinct events delivered across all pages. + * @return What was delivered and WHY the walk stopped — see [PagedFetchResult]. + * The reason is part of the answer, not a detail: `downloaded` cannot tell + * "the relay has nothing older" from "the relay stopped answering", and a + * caller recording sync coverage may only treat the range below the oldest + * event it saw as verified-empty in the first case. */ suspend fun INostrClient.fetchAllPages( relay: NormalizedRelayUrl, filters: List, idleTimeoutMs: Long = 30_000L, onNewPage: ((Long) -> Unit)? = null, - onDrained: (() -> Unit)? = null, onEvent: suspend (Event) -> Unit, -): Int { +): PagedFetchResult { var until: Long? = null var totalEvents = 0 - var drained = false + // Overwritten by whichever break ends the loop. UNPAGEABLE is the honest + // default: the two breaks that leave it alone are both "the cursor cannot + // advance", and it is the reading that licenses the least. + var end = PagedFetchResult.End.UNPAGEABLE // Track how many matching events each filter has received so far. val matchCountPerFilter = IntArray(filters.size) @@ -181,14 +230,25 @@ suspend fun INostrClient.fetchAllPages( stillNeedsMore && pageableThisPage } - if (activeFilters.isEmpty()) break + if (activeFilters.isEmpty()) { + // Every filter either met its limit or is a search that has had its + // one page. The first is the caller stopping the walk; the second + // cannot page at all. Neither is the corpus ending. + end = + if (filters.any { it.limit != null }) { + PagedFetchResult.End.LIMIT_REACHED + } else { + PagedFetchResult.End.UNPAGEABLE + } + break + } // Announce the page only now that we know it will actually be fetched: a // search-only filter drops out of activeFilters above and breaks with no // REQ, so firing this earlier would report a page that never happens. if (until != null) onNewPage?.invoke(until) - val doneChannel = Channel(Channel.CONFLATED) + val doneChannel = Channel(Channel.CONFLATED) // Idle watchdog for this page: every arriving event bumps it, so the page's // timeout measures silence since the relay's most recent message (the same @@ -206,7 +266,7 @@ suspend fun INostrClient.fetchAllPages( // [receiveWithinIdle] reports by returning null. Only an EOSE can support a // drain claim below — silence is not an answer, and a CLOSED is the relay // declining to give one. - var pageEnd: PageEnd? = null + var pageEnd: PageSignal? = null try { val listener = @@ -280,7 +340,7 @@ suspend fun INostrClient.fetchAllPages( relay: NormalizedRelayUrl, forFilters: List?, ) { - doneChannel.trySend(PageEnd.EOSE) + doneChannel.trySend(PageSignal.EOSE) } override fun onClosed( @@ -288,7 +348,7 @@ suspend fun INostrClient.fetchAllPages( relay: NormalizedRelayUrl, forFilters: List?, ) { - doneChannel.trySend(PageEnd.CLOSED) + doneChannel.trySend(PageSignal.CLOSED) } override fun onCannotConnect( @@ -296,7 +356,7 @@ suspend fun INostrClient.fetchAllPages( message: String, forFilters: List?, ) { - doneChannel.trySend(PageEnd.CANNOT_CONNECT) + doneChannel.trySend(PageSignal.CANNOT_CONNECT) } } @@ -333,7 +393,15 @@ suspend fun INostrClient.fetchAllPages( val limit = filters[i].limit limit != null && matchCountPerFilter[i] >= limit } - drained = pageEnd == PageEnd.EOSE && !cappedByLimit && filters.none { it.search != null } + end = + when { + pageEnd == PageSignal.CLOSED -> PagedFetchResult.End.CLOSED + pageEnd == PageSignal.CANNOT_CONNECT -> PagedFetchResult.End.CANNOT_CONNECT + pageEnd == null -> PagedFetchResult.End.IDLE + cappedByLimit -> PagedFetchResult.End.LIMIT_REACHED + filters.any { it.search != null } -> PagedFetchResult.End.UNPAGEABLE + else -> PagedFetchResult.End.DRAINED + } break } @@ -345,14 +413,17 @@ suspend fun INostrClient.fetchAllPages( // recovers progress, dropping only the second's unreachable tail). Both // are resolved by stepping strictly past it. `boundary` is null only on // the first page, which has no dedup and so can't be all-duplicate. - val step = boundary ?: break + val step = boundary ?: break // first page, all-duplicate: impossible, and `end` stays UNPAGEABLE until = step - 1 seenAtBoundary = HashSet() continue } // Only search hits advanced nothing pageable → can't page further. - if (pageMinTs == Long.MAX_VALUE) break + if (pageMinTs == Long.MAX_VALUE) { + end = PagedFetchResult.End.UNPAGEABLE + break + } // Advance inclusively to the oldest second seen, carrying its dedup set: // still the same boundary → accumulate; a genuinely older one → replace. @@ -369,11 +440,7 @@ suspend fun INostrClient.fetchAllPages( until = nextUntil } - // After the loop, not at the break: every other exit above leaves `drained` - // false, and firing from one place keeps "at most once" true by construction. - if (drained) onDrained?.invoke() - - return totalEvents + return PagedFetchResult(totalEvents, end) } suspend fun INostrClient.fetchAllPages( @@ -381,14 +448,12 @@ suspend fun INostrClient.fetchAllPages( filters: List, idleTimeoutMs: Long = 30_000L, onNewPage: ((Long) -> Unit)? = null, - onDrained: (() -> Unit)? = null, onEvent: suspend (Event) -> Unit, -): Int = +): PagedFetchResult = fetchAllPages( relay = RelayUrlNormalizer.normalize(relay), filters = filters, idleTimeoutMs = idleTimeoutMs, onNewPage = onNewPage, - onDrained = onDrained, onEvent = onEvent, ) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesPoolExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesPoolExt.kt index 7de482b0a4..e59fcc34ed 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesPoolExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesPoolExt.kt @@ -79,14 +79,14 @@ suspend fun INostrClient.fetchAllPagesFromPool( launch { try { onRelayStart?.invoke(relay) - val total = + val result = fetchAllPages( relay = relay, filters = filtersForRelay, idleTimeoutMs = idleTimeoutMs, onNewPage = onNewPage?.let { cb -> { until -> cb(until, relay) } }, ) { event -> onEvent(event, relay) } - onRelayComplete?.invoke(relay, total) + onRelayComplete?.invoke(relay, result.downloaded) } finally { semaphore.release() } diff --git a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesDrainTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesDrainTest.kt index e8797a8d0f..f2de98b505 100644 --- a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesDrainTest.kt +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesDrainTest.kt @@ -23,6 +23,7 @@ package com.vitorpamplona.quartz.nip01Core.relay import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.client.EmptyNostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.PagedFetchResult import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.fetchAllPages import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter @@ -37,14 +38,13 @@ import kotlin.test.assertFalse import kotlin.test.assertTrue /** - * Pins `onDrained` — the one signal that tells a caller the relay served - * *everything* below where the walk stopped, rather than merely stopping there. + * Pins [PagedFetchResult.End] — WHY a walk stopped, which is the half of the + * answer `downloaded` cannot carry. * - * The distinction is invisible in the `Int` return and matters enormously to - * anything recording sync coverage: without it, "the relay has nothing older" - * and "the relay capped us / went quiet / hung up" look identical, so a coverage - * band can never close its oldest leg and re-asks a range that will always come - * back empty, every cycle, forever. + * It matters enormously to anything recording sync coverage: without it, "the + * relay has nothing older" and "the relay capped us / went quiet / hung up" look + * identical, so a coverage band can never close its oldest leg and re-asks a + * range that will always come back empty, every cycle, forever. * * Real-clock ([runBlocking]) for the same reason the idle-timeout suite is: the * page watchdog is a monotonic clock bumped from the socket reader thread. @@ -103,18 +103,17 @@ class NostrClientFetchAllPagesDrainTest { client.listener!!.onEose(relay, null) } - var drained = false - val total = + val result = client.fetchAllPages( relay = relay, filters = listOf(Filter(kinds = listOf(1))), idleTimeoutMs = 2_000, - onDrained = { drained = true }, ) { } feeder.join() - assertEquals(2, total) - assertTrue(drained, "an empty page the relay EOSEd is proof there is nothing older") + assertEquals(2, result.downloaded) + assertEquals(PagedFetchResult.End.DRAINED, result.end, "an empty page the relay EOSEd is proof there is nothing older") + assertTrue(result.drained) } @Test @@ -133,18 +132,17 @@ class NostrClientFetchAllPagesDrainTest { client.awaitPage(2) } - var drained = false - val total = + val result = client.fetchAllPages( relay = relay, filters = listOf(Filter(kinds = listOf(1))), idleTimeoutMs = 200, - onDrained = { drained = true }, ) { } feeder.join() - assertEquals(2, total, "the events already delivered are still kept") - assertFalse(drained, "an idle timeout says nothing about what the relay holds") + assertEquals(2, result.downloaded, "the events already delivered are still kept") + assertEquals(PagedFetchResult.End.IDLE, result.end) + assertFalse(result.drained, "an idle timeout says nothing about what the relay holds") } @Test @@ -164,16 +162,16 @@ class NostrClientFetchAllPagesDrainTest { client.listener!!.onClosed("auth-required: we don't serve that", relay, null) } - var drained = false - client.fetchAllPages( - relay = relay, - filters = listOf(Filter(kinds = listOf(1))), - idleTimeoutMs = 2_000, - onDrained = { drained = true }, - ) { } + val result = + client.fetchAllPages( + relay = relay, + filters = listOf(Filter(kinds = listOf(1))), + idleTimeoutMs = 2_000, + ) { } feeder.join() - assertFalse(drained, "a CLOSED is the relay declining, not an empty corpus") + assertEquals(PagedFetchResult.End.CLOSED, result.end, "a CLOSED is the relay declining, not an empty corpus") + assertFalse(result.drained) } @Test @@ -186,16 +184,16 @@ class NostrClientFetchAllPagesDrainTest { client.listener!!.onCannotConnect(relay, "connection refused", null) } - var drained = false - client.fetchAllPages( - relay = relay, - filters = listOf(Filter(kinds = listOf(1))), - idleTimeoutMs = 2_000, - onDrained = { drained = true }, - ) { } + val result = + client.fetchAllPages( + relay = relay, + filters = listOf(Filter(kinds = listOf(1))), + idleTimeoutMs = 2_000, + ) { } feeder.join() - assertFalse(drained, "never got to ask") + assertEquals(PagedFetchResult.End.CANNOT_CONNECT, result.end, "never got to ask") + assertFalse(result.drained) } @Test @@ -213,18 +211,17 @@ class NostrClientFetchAllPagesDrainTest { client.listener!!.onEose(relay, null) } - var drained = false - val total = + val result = client.fetchAllPages( relay = relay, filters = listOf(Filter(kinds = listOf(1), limit = 2)), idleTimeoutMs = 2_000, - onDrained = { drained = true }, ) { } feeder.join() - assertEquals(2, total) + assertEquals(2, result.downloaded) assertEquals(1, client.subscribeCount, "the limit was met, so there was no second page") - assertFalse(drained, "a fulfilled limit is the caller stopping, not the corpus ending") + assertEquals(PagedFetchResult.End.LIMIT_REACHED, result.end, "a fulfilled limit is the caller stopping, not the corpus ending") + assertFalse(result.drained) } } diff --git a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesIdleTimeoutTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesIdleTimeoutTest.kt index fb9855a7a6..534f995cdb 100644 --- a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesIdleTimeoutTest.kt +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesIdleTimeoutTest.kt @@ -111,7 +111,7 @@ class NostrClientFetchAllPagesIdleTimeoutTest { ) { got.add(it) } feeder.join() - assertEquals(6, total, "a slowly-but-actively streaming page must never be cropped") + assertEquals(6, total.downloaded, "a slowly-but-actively streaming page must never be cropped") assertEquals(6, got.size) assertEquals(1, client.subscribeCount, "the whole stream must arrive in ONE page — a hard deadline would truncate and re-subscribe") assertEquals(0, pages, "no pagination should be needed") @@ -145,7 +145,7 @@ class NostrClientFetchAllPagesIdleTimeoutTest { feeder.join() val elapsedMs = start.elapsedNow().inWholeMilliseconds - assertEquals(2, total, "events delivered before the stall are kept") + assertEquals(2, total.downloaded, "events delivered before the stall are kept") assertTrue(elapsedMs >= 300, "must wait out at least one idle window, took ${elapsedMs}ms") assertTrue(elapsedMs < 5_000, "a stalled page must end promptly after the idle window, took ${elapsedMs}ms") } diff --git a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientReqBypassingRelayLimitsTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientReqBypassingRelayLimitsTest.kt index 844ff29282..96971d4e0c 100644 --- a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientReqBypassingRelayLimitsTest.kt +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientReqBypassingRelayLimitsTest.kt @@ -67,7 +67,7 @@ class NostrClientReqBypassingRelayLimitsTest : RelayClientTest() { events.add(event) } - assertEquals(1000, totalFound) + assertEquals(1000, totalFound.downloaded) assertEquals(1000, events.size) events.forEach { event -> assertEquals(MetadataEvent.KIND, event.kind) @@ -115,7 +115,7 @@ class NostrClientReqBypassingRelayLimitsTest : RelayClientTest() { } } - assertEquals(2500, totalFound) + assertEquals(2500, totalFound.downloaded) assertEquals(1000, metadataEvents.size) assertEquals(1500, contactListEvents.size) } @@ -165,7 +165,7 @@ class NostrClientReqBypassingRelayLimitsTest : RelayClientTest() { filters = listOf(Filter(search = "kotlin")), onNewPage = { searchPages++ }, ) { searchEvents.add(it) } - assertEquals(2, searchTotal, "a search filter must be fetched as a single page (the relay's cap)") + assertEquals(2, searchTotal.downloaded, "a search filter must be fetched as a single page (the relay's cap)") assertEquals(2, searchEvents.size) assertEquals(0, searchPages, "a search filter must never advance the until cursor") } finally { From 94b679fe7c32bdc76f4f62b1642cc42d2e878e60 Mon Sep 17 00:00:00 2001 From: sandwich <299465+dskvr@users.noreply.github.com> Date: Sun, 9 Aug 2026 15:19:15 +0100 Subject: [PATCH 084/132] Address NIP-5D review comments --- .../amethyst/napplet/NappletBrokerService.kt | 2 +- .../amethyst/napplet/NappletIdentityWatch.kt | 22 +++--- .../amethyst/napplet/NappletLauncher.kt | 8 +-- .../gateways/NappletResourceFetcher.kt | 37 +++++++++- .../amethyst/napplet/NappletLauncherTest.kt | 68 +++++++++++++++++++ .../amethyst/commons/napplet/NappletBroker.kt | 7 +- .../commons/napplet/NappletRequestRouter.kt | 10 ++- .../napplet/protocol/NappletProtocolJson.kt | 3 + .../napplet/NappletRequestRouterTest.kt | 14 ++++ 9 files changed, 149 insertions(+), 22 deletions(-) create mode 100644 amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/NappletLauncherTest.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletBrokerService.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletBrokerService.kt index 58453f479b..f49b497538 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletBrokerService.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletBrokerService.kt @@ -320,7 +320,7 @@ class NappletBrokerService : Service() { // NAP-IDENTITY has no watch/unwatch request. Once the consent-gated startup // snapshot succeeds, the runtime owns identity.changed delivery for this // trusted launch token until the broker service closes. - if (requestType == "identity.getPublicKey" && outcome.payload.contains("\"ok\":true") && launchToken != null) { + if (requestType == "identity.getPublicKey" && outcome.response is NappletResponse.PublicKey && launchToken != null) { identityWatch.start(launchToken, session.accountPubKey) { push(replyTo, it) } } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletIdentityWatch.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletIdentityWatch.kt index bbb6a3aaa2..819c55b4a2 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletIdentityWatch.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletIdentityWatch.kt @@ -27,6 +27,7 @@ import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.distinctUntilChanged import kotlinx.coroutines.flow.drop import kotlinx.coroutines.launch +import java.util.concurrent.ConcurrentHashMap /** * Streams `identity.changed` pushes to an applet that registered `napplet.identity.onChanged`. It @@ -41,21 +42,24 @@ class NappletIdentityWatch( private val scope: CoroutineScope, private val pubKey: (boundPubKey: String) -> Flow, ) { - private val jobs = mutableMapOf() + private val jobs = ConcurrentHashMap() fun start( watchId: String, boundPubKey: String, push: (String) -> Unit, ) { - if (jobs.containsKey(watchId)) return - jobs[watchId] = - scope.launch { - pubKey(boundPubKey) - .distinctUntilChanged() - .drop(1) - .collect { push(NappletProtocolJson.encodeIdentityChanged(it)) } - } + jobs.computeIfAbsent(watchId) { id -> + scope + .launch { + pubKey(boundPubKey) + .distinctUntilChanged() + .drop(1) + .collect { push(NappletProtocolJson.encodeIdentityChanged(it)) } + }.also { job -> + job.invokeOnCompletion { jobs.remove(id, job) } + } + } } fun stopAll() { diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLauncher.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLauncher.kt index 29c0a2f091..c923b799d7 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLauncher.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/NappletLauncher.kt @@ -24,7 +24,6 @@ import android.content.Context import android.content.Intent import android.content.res.Configuration import android.os.Bundle -import android.util.Log import com.vitorpamplona.amethyst.Amethyst import com.vitorpamplona.amethyst.commons.napplet.NappletArtifactPolicy import com.vitorpamplona.amethyst.commons.napplet.NappletIdentity @@ -39,6 +38,7 @@ import com.vitorpamplona.quartz.nip01Core.crypto.verify import com.vitorpamplona.quartz.nip5aStaticWebsites.tags.PathTag import com.vitorpamplona.quartz.nip5dNapplets.NappletManifest import com.vitorpamplona.quartz.nipB7Blossom.BlossomServersEvent +import com.vitorpamplona.quartz.utils.Log /** * Opens a napplet/nsite in the sandboxed [NappletHostActivity] (the `:napplet` process). Only @@ -56,7 +56,7 @@ object NappletLauncher { ) { val event = manifest as? Event if (event?.verify() != true || event.pubKey != authorPubKey) { - Log.w(TAG, "Refusing NIP-5D manifest that failed signature/author verification") + Log.w(TAG) { "Refusing NIP-5D manifest that failed signature/author verification" } return } buildLaunchParams(context, manifest, authorPubKey, identifier)?.let { openHost(context, it) } @@ -80,7 +80,7 @@ object NappletLauncher { profile: HostProfile, ) { if (profile != HostProfile.WEBSITE) { - Log.w(TAG, "Refusing raw NIP-5D launch without a verified manifest") + Log.w(TAG) { "Refusing raw NIP-5D launch without a verified manifest" } return } val params = @@ -219,7 +219,7 @@ object NappletLauncher { ): Bundle? { val event = manifest as? Event if (event?.verify() != true || event.pubKey != authorPubKey) { - Log.w(TAG, "Refusing embedded NIP-5D manifest that failed signature/author verification") + Log.w(TAG) { "Refusing embedded NIP-5D manifest that failed signature/author verification" } return null } return runCatching { diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/gateways/NappletResourceFetcher.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/gateways/NappletResourceFetcher.kt index 9acdf7b690..24bf6e655c 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/gateways/NappletResourceFetcher.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/napplet/gateways/NappletResourceFetcher.kt @@ -39,16 +39,21 @@ import com.vitorpamplona.quartz.nip19Bech32.entities.NPub import com.vitorpamplona.quartz.nip5aStaticWebsites.resolver.StaticSiteResolver import com.vitorpamplona.quartz.nip5aStaticWebsites.resolver.sniffContentType import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.suspendCancellableCoroutine import kotlinx.coroutines.withContext import kotlinx.serialization.json.Json import okhttp3.Authenticator +import okhttp3.Call +import okhttp3.Callback import okhttp3.CookieJar import okhttp3.Dns import okhttp3.HttpUrl import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import okhttp3.OkHttpClient import okhttp3.Request +import okhttp3.Response import java.io.ByteArrayOutputStream +import java.io.IOException import java.io.InterruptedIOException import java.net.InetAddress import java.net.URLDecoder @@ -127,7 +132,7 @@ class NappletResourceFetcher( ) }.build() - private fun fetchHttps( + private suspend fun fetchHttps( url: String, client: OkHttpClient, ): NappletResourceResult { @@ -141,7 +146,7 @@ class NappletResourceFetcher( .url(current) .get() .build(), - ).execute() + ).await() .use { response -> if (response.isRedirect) { if (hop >= MAX_REDIRECTS) return failure(ERROR_BLOCKED, "Redirect limit exceeded.") @@ -227,7 +232,7 @@ class NappletResourceFetcher( * wrong server can never substitute the blob. Returns null for a malformed hash or if no server * serves it. */ - private fun fetchBlossom( + private suspend fun fetchBlossom( url: String, client: OkHttpClient, ): NappletResourceResult { @@ -257,6 +262,32 @@ class NappletResourceFetcher( return failure(ERROR_NOT_FOUND, "No Blossom server returned the verified blob.") } + private suspend fun Call.await(): Response = + suspendCancellableCoroutine { continuation -> + continuation.invokeOnCancellation { cancel() } + enqueue( + object : Callback { + override fun onFailure( + call: Call, + e: IOException, + ) { + if (continuation.isActive) continuation.resumeWith(Result.failure(e)) + } + + override fun onResponse( + call: Call, + response: Response, + ) { + if (continuation.isActive) { + continuation.resumeWith(Result.success(response)) + } else { + response.close() + } + } + }, + ) + } + /** Parses a `data:[][;base64],` URL into bytes + content type. */ private fun decodeDataUrl(url: String): NappletResourceResult { val comma = url.indexOf(',') diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/NappletLauncherTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/NappletLauncherTest.kt new file mode 100644 index 0000000000..6145418d88 --- /dev/null +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/napplet/NappletLauncherTest.kt @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.napplet + +import android.content.Context +import android.content.Intent +import com.vitorpamplona.amethyst.napplethost.HostProfile +import com.vitorpamplona.quartz.nip5dNapplets.NappletManifest +import io.mockk.every +import io.mockk.mockk +import io.mockk.verify +import org.junit.Assert.assertNull +import org.junit.Test + +class NappletLauncherTest { + private val context = mockk(relaxed = true) + private val manifest = mockk() + private val author = "aa".repeat(32) + + @Test + fun manifestLaunchRejectsNonEventManifest() { + NappletLauncher.launch(context, manifest, author, "demo") + + verify(exactly = 0) { context.startActivity(any()) } + } + + @Test + fun embeddedBuildLaunchParamsRejectsNonEventManifest() { + assertNull(NappletLauncher.buildLaunchParams(context, manifest, author, "demo")) + } + + @Test + fun rawLaunchRejectsNappletProfile() { + every { context.startActivity(any()) } returns Unit + + NappletLauncher.launch( + context = context, + paths = emptyList(), + servers = emptyList(), + authorPubKey = author, + identifier = "demo", + aggregateHash = null, + title = "Demo", + requires = emptyList(), + profile = HostProfile.NAPPLET, + ) + + verify(exactly = 0) { context.startActivity(any()) } + } +} diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletBroker.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletBroker.kt index 0a9a923848..b7f9adf73d 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletBroker.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletBroker.kt @@ -33,6 +33,7 @@ import com.vitorpamplona.amethyst.commons.napplet.permissions.NappletPermissionL import com.vitorpamplona.amethyst.commons.napplet.permissions.PermissionDecision import com.vitorpamplona.amethyst.commons.napplet.protocol.NappletRequest import com.vitorpamplona.amethyst.commons.napplet.protocol.NappletResponse +import com.vitorpamplona.amethyst.commons.napplet.protocol.NappletStorageScope import com.vitorpamplona.amethyst.commons.napplet.protocol.toSignerOp import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner @@ -376,11 +377,11 @@ class NappletBroker( private fun storageCoordinate( identity: NappletIdentity, - scope: com.vitorpamplona.amethyst.commons.napplet.protocol.NappletStorageScope, + scope: NappletStorageScope, ): String = when (scope) { - com.vitorpamplona.amethyst.commons.napplet.protocol.NappletStorageScope.SHARED -> identity.storageCoordinate - com.vitorpamplona.amethyst.commons.napplet.protocol.NappletStorageScope.INSTANCE -> identity.instanceStorageCoordinate + NappletStorageScope.SHARED -> identity.storageCoordinate + NappletStorageScope.INSTANCE -> identity.instanceStorageCoordinate } /** diff --git a/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletRequestRouter.kt b/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletRequestRouter.kt index b1e7d9440f..fb27dd611e 100644 --- a/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletRequestRouter.kt +++ b/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletRequestRouter.kt @@ -42,6 +42,7 @@ object NappletRequestRouter { /** Send this `.result` payload back, correlated to the request's id. */ data class Reply( val payload: String, + val response: NappletResponse? = null, ) : Outcome /** Open a live relay subscription; the host streams `relay.event`/`relay.eose`/`relay.closed` by [subId]. */ @@ -114,7 +115,12 @@ object NappletRequestRouter { val request = runCatching { NappletProtocolJson.decodeRequest(payload) }.getOrNull() - ?: return Outcome.Ignore + ?: return if (runCatching { NappletProtocolJson.readId(payload) }.getOrNull() != null) { + val failure = NappletResponse.Failed("Malformed or unsupported request.") + Outcome.Reply(NappletProtocolJson.encodeResponse(requestType, failure), failure) + } else { + Outcome.Ignore + } val response = broker.handle(identity, request, declared) @@ -131,6 +137,6 @@ object NappletRequestRouter { } } - return Outcome.Reply(NappletProtocolJson.encodeResponse(requestType, response)) + return Outcome.Reply(NappletProtocolJson.encodeResponse(requestType, response), response) } } diff --git a/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/napplet/protocol/NappletProtocolJson.kt b/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/napplet/protocol/NappletProtocolJson.kt index 1df933a9e7..7cfb2a95e0 100644 --- a/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/napplet/protocol/NappletProtocolJson.kt +++ b/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/napplet/protocol/NappletProtocolJson.kt @@ -61,6 +61,9 @@ object NappletProtocolJson { /** The `type` discriminant of a request envelope, used to build the matching `.result` type. */ fun readType(envelopeJson: String): String? = json.parseToJsonElement(envelopeJson).jsonObject.str("type") + /** The request `id`, when the envelope expects a correlated reply. */ + fun readId(envelopeJson: String): String? = json.parseToJsonElement(envelopeJson).jsonObject.str("id") + /** The `subId` of a subscription request, used to key the `relay.event`/`relay.eose` pushes back to it. */ fun readSubId(envelopeJson: String): String? = json.parseToJsonElement(envelopeJson).jsonObject.str("subId") diff --git a/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletRequestRouterTest.kt b/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletRequestRouterTest.kt index eca9000437..6269b971ea 100644 --- a/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletRequestRouterTest.kt +++ b/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/napplet/NappletRequestRouterTest.kt @@ -125,6 +125,20 @@ class NappletRequestRouterTest { assertEquals(NappletRequestRouter.Outcome.Ignore, route("""{"type":"relay.publish"}""")) } + @Test + fun keyedUnknownAndMalformedRequestsReplyWithFailure() = + runTest { + val unknown = route("""{"type":"totally.unknown","id":"r1"}""") + assertIs(unknown) + assertTrue(unknown.payload.contains("totally.unknown.result")) + assertTrue(unknown.payload.contains("Malformed or unsupported request.")) + + val malformed = route("""{"type":"relay.publish","id":"r2"}""") + assertIs(malformed) + assertTrue(malformed.payload.contains("relay.publish.result")) + assertTrue(malformed.payload.contains("Malformed or unsupported request.")) + } + @Test fun queryRepliesWithItsResult() = runTest { From 395e821da2831420e2a2d5116971645ec03cce52 Mon Sep 17 00:00:00 2001 From: vitorpamplona <532031+vitorpamplona@users.noreply.github.com> Date: Sun, 9 Aug 2026 14:43:52 +0000 Subject: [PATCH 085/132] chore: sync Crowdin translations and seed translator npub placeholders --- amethyst/src/main/res/values-hi-rIN/strings.xml | 4 +++- amethyst/src/main/res/values-hu-rHU/strings.xml | 4 +++- amethyst/src/main/res/values-pl-rPL/strings.xml | 2 ++ 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/amethyst/src/main/res/values-hi-rIN/strings.xml b/amethyst/src/main/res/values-hi-rIN/strings.xml index 6faae86163..c26533c1f8 100644 --- a/amethyst/src/main/res/values-hi-rIN/strings.xml +++ b/amethyst/src/main/res/values-hi-rIN/strings.xml @@ -3840,8 +3840,10 @@ चर्चाशाला सूचियाँ स्मृति में यन्त्र स्मृति वर्ग क्रमलेखकों के साथ बाँटें - यदि अमेथिस्ट द्वारा विद्युतकोष अथवा जानकारी यातायात का व्यय हो रहा है तो आप इस वृत्तान्त को क्रमलेखकों को भेज सकते हैं एक रहस्यीकृत सीधासन्देश में। इसमें केवल इस पटल पर दृश्यमान संख्याएँ हैं तथा इनके पीछे के तन्त्रविषयक संकलन \u2014 कोई पत्र सम्पर्क अथवा वीक्षण विवरण नहीं। कुछ भी नहीं भेजा जाएगा आपके द्वारा सन्देश पटल पर भेजें दबाने के पूर्व। + यदि अमेथिस्ट द्वारा विद्युतकोष अथवा जानकारी यातायात का व्यय हो रहा है तो आप इस वृत्तान्त को क्रमलेखकों को भेज सकते हैं एक रहस्यीकृत सीधासन्देश में। अथवा इसकी अनुकृति करके स्वयम बाँटें। इसमें केवल इस पटल पर दृश्यमान संख्याएँ हैं तथा इनके पीछे के तन्त्रविषयक संकलनसूचक \u2014 कोई पत्र अथवा सम्पर्क अथवा पुनःप्रसारक नाम अथवा जालभ्रमण विवरण नहीं। वृत्तान्त इस पटल से केवल तब जाएगा जब आप इसे भेजेंगे अथवा इसकी अनुकृति बाँटेंगे। सीधासन्देश द्वारा वृत्तान्त भेजें + अनुकृति + बाँटें उच्च संसाधन उपयोग का बोध अमेथिस्ट द्वारा अपेक्षित से अधिक उपभोग हुआ निकटकाल में : %1$s। क्या आप एक उपयोग वृत्तान्त भेजना चाहते क्रमलेखकों को एक रहस्यीकृत सीधासन्देश में। आप सम्पूर्ण वृत्तान्त देखेंगे कुछ भी भेजने के पूर्व। चलनशील जानकारी %1$s पृष्ठभूत एक दिन में diff --git a/amethyst/src/main/res/values-hu-rHU/strings.xml b/amethyst/src/main/res/values-hu-rHU/strings.xml index ec099bd65c..b586914a22 100644 --- a/amethyst/src/main/res/values-hu-rHU/strings.xml +++ b/amethyst/src/main/res/values-hu-rHU/strings.xml @@ -3841,8 +3841,10 @@ Csevegőszoba-listák a memóriában Eszköz memóriájának osztálya Megosztás a fejlesztőkkel - Ha az Amethyst túl sok akkumulátort vagy adatot használna, elküldheti ezt a jelentést a fejlesztőknek egy titkosított közvetlen üzenetben (DM). Kizárólag a képernyőn látható számokat és az azok mögött álló technikai számlálókat tartalmazza \u2014 bejegyzéseket, névjegyeket vagy böngészési adatokat nem. Semmi sem kerül elküldésre addig, amíg az üzenetküldő képernyőn rá nem koppint a Küldés gombra. + Ha úgy tűnik, hogy az Amethyst túlzottan meríti az akkumulátort vagy fogyasztja az adatkeretet, elküldheti ezt a jelentést a fejlesztőknek egy titkosított közvetlen üzenetben vagy lemásolhatja és megoszthatja saját maga. A jelentés kizárólag az ezen a képernyőn látható számokat és a mögöttük lévő technikai számlálókat tartalmazza – bejegyzéseket, névjegyeket, átjátszóneveket vagy böngészési adatokat nem. A jelentés csak akkor hagyja el ezt a képernyőt, ha Ön elküldi, lemásolja vagy megosztja azt. Jelentés elküldése közvetlen üzenetben + Másolás + Megosztás Magas erőforrás-használat észlelhető Az Amethyst a közelmúltban a vártnál többet fogyasztott: %1$s. Szeretne használati jelentést küldeni a fejlesztőknek egy titkosított közvetlen üzenetben? A küldés előtt megtekintheti a teljes jelentést. %1$s mobiladat-forgalom a háttérben egyetlen nap alatt diff --git a/amethyst/src/main/res/values-pl-rPL/strings.xml b/amethyst/src/main/res/values-pl-rPL/strings.xml index 4d613a32e3..469d6f9e0f 100644 --- a/amethyst/src/main/res/values-pl-rPL/strings.xml +++ b/amethyst/src/main/res/values-pl-rPL/strings.xml @@ -3979,6 +3979,8 @@ Zaplanowane posty z innych kont nie zostaną opublikowane, dopóki to konto jest Udostępnij deweloperom Jeśli zauważysz, że aplikacja Amethyst nadmiernie zużywa baterię lub transfer danych, możesz przesłać ten raport do twórców w zaszyfrowanej wiadomości prywatnej. Zawiera on wyłącznie liczby widoczne na tym ekranie oraz związane z nimi liczniki techniczne \u2014 nie zawiera żadnych postów, kontaktów ani szczegółów dotyczących przeglądania stron. Żadne dane nie zostaną wysłane, dopóki nie klikniesz przycisku „Wyślij” na ekranie wiadomości. Wyślij raport za pośrednictwem DM + Kopiuj + Udostępnij Wykryto wysokie użycie zasobów W ostatnim czasie Amethyst zużył więcej zasobów, niż oczekiwano: %1$s. Czy chcesz wysłać raport dotyczący zużycia do twórców w zaszyfrowanej wiadomości prywatnej? Przed wysłaniem zobaczysz pełną treść raportu. %1$s danych komórkowych w tle w ciągu jednego dnia From 3a53292993741375352ecd307025dae0672770a2 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 15:02:43 +0000 Subject: [PATCH 086/132] fix(concord): close the soft-ban holes reachable from the shipping app MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part A of docs/concord-soft-ban-audit.md — the ones a banned user reaches by tapping a button, no custom tooling involved. A1. mintConcordInvite checked that the account was writeable and that we held the community, and nothing else, while its button was the one control on the screen with no gate at all. A member banned a minute ago could hand out a working link to the community they were removed from, and every account they invited arrived as a fresh un-banned npub. Now gated on CREATE_INVITE, both in the verb and on the button. Worth noting the bit was not enforced anywhere else: the fold gates the INVITE_* Control entities on it, but a link's bundle is a standalone kind-33301 published outside the Control Plane, so this check is the only one that exists. A3. Every moderation verb checked isWriteable() plus the Control write key — which is a spam gate, never authority (CORD-02 §5) — and left the real decision to whichever composable drew the button. Those gates then tested effectivePermissions, which ignores the banlist, so a banned staffer kept seeing the controls; the editions were dropped by everyone's fold, making them silently no-op, which this codebase elsewhere calls out as worse than absent. Ban and Remove survived only because a second, unrelated condition happened to route through the ban-aware canActOn. Authority now lives in the action layer behind isAuthorizedFor(), so a caller from desktop, amy or a future screen inherits it, and every authorization test uses hasPermission. refoundConcordCommunity's own guard was ban-blind outright and now rank-checks each removed member too. A2. The recovery sweep merges us onto any higher-epoch bundle found at our stored invite_ref, and an ex-member keeps that link's unlock token forever — so our own background timer walked a removed member back into the epoch a Refounding had rotated them out of. isStranded/mergeForward now take bannedAtCurrentEpoch as a required argument rather than leaving it to callers, because a caller that forgets it inverts the mechanism. The liveness half of that finding (nothing re-mints at a stable coordinate, so legitimate recovery never fires either) needs a spec answer and is untouched here. A4. Typing heartbeats are filtered on both ends, so a banned member stops announcing that they are typing messages nobody will see. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DrJhpFhhLjuDJQNkGvYMGj --- .../amethyst/model/AccountConcordActions.kt | 110 ++++++++++++++++-- .../concord/ConcordChannelListScreen.kt | 49 +++++--- .../concord/ConcordMembersScreen.kt | 5 +- .../commons/actions/ConcordActions.kt | 3 +- .../model/concord/ConcordCommunitySession.kt | 3 + .../cord05Invites/ConcordStrandedRecovery.kt | 17 ++- .../ConcordStrandedRecoveryTest.kt | 26 +++-- 7 files changed, 173 insertions(+), 40 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt index fba41f5e01..c403e08986 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt @@ -171,6 +171,16 @@ class AccountConcordActions( val entry = account.concordChannelList.liveCommunities.value .firstOrNull { it.id == communityId } ?: return null + // CREATE_INVITE, and not while banned. This used to check only that we held the community, + // which made minting the one moderation-free action in the app: a member the owner had just + // banned could tap the invite button and hand out a working link to the community they were + // removed from, and every account they invited arrived as a fresh un-banned npub. + // + // Note the bit is not otherwise enforced anywhere. The fold gates the INVITE_* Control + // entities on CREATE_INVITE, but a link's bundle is a standalone kind-33301 published + // OUTSIDE the Control Plane, so no fold ever sees it. This check is the only one there is. + val session = account.concordSessions.sessionFor(communityId) ?: return null + if (!isAuthorizedFor(session, ConcordPermissions.CREATE_INVITE)) return null val invite = ConcordActions.inviteFor( communityIdHex = entry.id, @@ -430,7 +440,17 @@ class AccountConcordActions( channelIdHex: String, ) { if (!account.isWriteable()) return - val entry = account.concordSessions.sessionFor(communityId)?.entry ?: return + val session = account.concordSessions.sessionFor(communityId) ?: return + // A ban hides every message we send, so continuing to announce that we are typing them is + // both noise and a contradiction of what the ban told the room. Filtered on the receive side + // too (ConcordCommunitySession.ingestTyping) — a malicious client would keep sending. + if (session.state.value + ?.authority + ?.isBanned(account.signer.pubKey) == true + ) { + return + } + val entry = session.entry val channelKey = ConcordActions.publicChannel(entry.root.hexToByteArray(), channelIdHex.hexToByteArray(), entry.rootEpoch) val wrap = ConcordActions.buildChannelTyping(account.signer, channelKey, channelIdHex, entry.rootEpoch, TimeUtils.now()) val relays = entry.relays.mapNotNullTo(mutableSetOf()) { RelayUrlNormalizer.normalizeOrNull(it) } @@ -487,6 +507,49 @@ class AccountConcordActions( return cp } + /** + * Whether this account may take the action guarded by [bit] in [session] — and, when [target] is + * given, take it *against that member* (CORD-04 §3's rank rule, "equal cannot act on equal"). + * + * Every moderation verb below funnels through this. It used to live only in the composables that + * drew the buttons, which failed three ways: the screens tested `effectivePermissions`, which + * ignores the banlist, so a banned staffer still saw the controls; a verb reached from anywhere + * else (desktop, `amy`, a new screen) inherited no check at all; and holding `control_root` — + * a spam gate, never authority (CORD-02 §5) — was the only thing actually being enforced. + * + * Fails **closed**, with one deliberate exception: the owner is read from [ConcordCommunityListEntry] + * rather than from the fold, because the community id proves them (CORD-02) and they must stay able + * to moderate before their Control Plane has finished folding — or through a fold a rogue has + * damaged. Everyone else needs a resolved roster, so an unfolded community grants nobody else + * anything. + */ + private fun isAuthorizedFor( + session: ConcordCommunitySession, + bit: Int, + target: HexKey? = null, + ): Boolean { + val me = account.signer.pubKey + if (session.entry.owner.equals(me, ignoreCase = true)) return true + val authority = session.state.value?.authority ?: return false + // hasPermission, never effectivePermissions: the latter reads the roles alone and would let a + // banned staffer keep acting for as long as they hold the key. + val allowed = if (target == null) authority.hasPermission(me, bit) else authority.canActOn(me, target, bit) + if (!allowed) { + Log.w("Concord") { "Refusing a Concord action in ${session.entry.id}: not authorized for bit $bit${target?.let { " on $it" } ?: ""} (CORD-04 §3)" } + } + return allowed + } + + /** [controlKeysForWrite] gated by [isAuthorizedFor] — the standing check and the key check together. */ + private fun controlKeysForAction( + session: ConcordCommunitySession, + bit: Int, + target: HexKey? = null, + ): ControlPlaneKeys? { + if (!isAuthorizedFor(session, bit, target)) return null + return controlKeysForWrite(session) + } + /** Grant [member] exactly [roleIds] (empty list revokes their roles). */ suspend fun grantConcordRole( communityId: String, @@ -495,7 +558,7 @@ class AccountConcordActions( ): Boolean { val session = account.concordSessions.sessionFor(communityId) ?: return false if (!account.isWriteable()) return false - val cp = controlKeysForWrite(session) ?: return false + val cp = controlKeysForAction(session, ConcordPermissions.MANAGE_ROLES, member) ?: return false // A Grant that first makes its member staff must deliver the control_root in the same // edition (CORD-04 §3) — grantWithStaffDelivery attaches the pairwise wrap when the // roles carry a Control-writing bit and we hold the secret to hand over. @@ -567,7 +630,7 @@ class AccountConcordActions( ): Boolean { val session = account.concordSessions.sessionFor(communityId) ?: return false if (!account.isWriteable()) return false - val cp = controlKeysForWrite(session) ?: return false + val cp = controlKeysForAction(session, ConcordPermissions.MANAGE_ROLES, member) ?: return false val existing = session.state.value @@ -609,7 +672,7 @@ class AccountConcordActions( ): Boolean { val session = account.concordSessions.sessionFor(communityId) ?: return false if (!account.isWriteable()) return false - val cp = controlKeysForWrite(session) ?: return false + val cp = controlKeysForAction(session, ConcordPermissions.MANAGE_ROLES, member) ?: return false val grantWrap = ConcordModeration.grant(account.signer, cp, communityId.hexToByteArray(), member, emptyList(), session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) publishConcordWrap(session.entry, grantWrap) return true @@ -657,7 +720,7 @@ class AccountConcordActions( ): Boolean { val session = account.concordSessions.sessionFor(communityId) ?: return false if (!account.isWriteable()) return false - val cp = controlKeysForWrite(session) ?: return false + val cp = controlKeysForAction(session, ConcordPermissions.BAN, member) ?: return false val wrap = ConcordModeration.ban(account.signer, cp, communityId.hexToByteArray(), member, session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) publishConcordWrap(session.entry, wrap) return true @@ -670,7 +733,7 @@ class AccountConcordActions( ): Boolean { val session = account.concordSessions.sessionFor(communityId) ?: return false if (!account.isWriteable()) return false - val cp = controlKeysForWrite(session) ?: return false + val cp = controlKeysForAction(session, ConcordPermissions.BAN, member) ?: return false val wrap = ConcordModeration.unban(account.signer, cp, communityId.hexToByteArray(), member, session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) publishConcordWrap(session.entry, wrap) return true @@ -701,10 +764,22 @@ class AccountConcordActions( val session = account.concordSessions.sessionFor(communityId) ?: return false val state = session.state.value ?: return false val authority = state.authority - val iCanBan = authority.isOwner(account.signer.pubKey) || authority.effectivePermissions(account.signer.pubKey).has(ConcordPermissions.BAN) + // hasPermission, not effectivePermissions: a Refounding is the hardest action in the protocol + // and this guard used to ignore the banlist, so a banned BAN-holder could launch one from the + // shipping app. Honest receivers refuse such a rotation (drainConcordRekeys checks the same + // ban-aware predicate), but that is a race against banlist propagation, not a check. + val iCanBan = authority.isOwner(account.signer.pubKey) || authority.hasPermission(account.signer.pubKey, ConcordPermissions.BAN) if (!iCanBan) return false val removedLower = removed.mapTo(HashSet()) { it.lowercase() } if (removedLower.isEmpty() || removedLower.any { authority.isOwner(it) }) return false + // Removal is the hardest form of a ban, so it takes the same rank rule (CORD-04 §3): an admin + // cannot Refound a peer admin out of the community any more than they could ban one. The owner + // short-circuits, as everywhere else, because canActOn starts at hasPermission. + if (!authority.isOwner(account.signer.pubKey) && + removedLower.any { !authority.canActOn(account.signer.pubKey, it, ConcordPermissions.BAN) } + ) { + return false + } // A Refounding writes the current plane (the pre-rotation bans) and the new one (the // compaction), so on a split epoch it takes the current control_root (CORD-02 §2). A // rank-qualified refounder whose secret hasn't arrived yet must wait for re-delivery. @@ -986,7 +1061,18 @@ class AccountConcordActions( // Only a live bundle recovers: an expired/revoked link is not a rotation we missed. val bundle = (ConcordActions.classifyInvite(wraps, parsed.fragment.token) as? InviteBundleStatus.Live)?.invite ?: continue - val merged = ConcordActions.recoverStranded(entry, bundle) ?: continue + // A removed member holds the link's unlock token forever, so without this the sweep + // walks them straight back into the epoch they were rotated out of — see A2 in + // docs/concord-soft-ban-audit.md. Read off the epoch we are LEAVING, which is the last + // one whose Control Plane we can still fold. + val bannedHere = + account.concordSessions + .sessionFor(entry.id) + ?.state + ?.value + ?.authority + ?.isBanned(account.signer.pubKey) == true + val merged = ConcordActions.recoverStranded(entry, bundle, bannedHere) ?: continue if (!adoptedConcordRotations.add("${entry.id}:${merged.rootEpoch}")) continue Log.i("Concord", "Stranded recovery: ${entry.id} ${entry.rootEpoch} -> ${merged.rootEpoch}") account.sendMyPublicAndPrivateOutbox(account.concordChannelList.follow(merged)) @@ -1009,7 +1095,7 @@ class AccountConcordActions( ): Boolean { val session = account.concordSessions.sessionFor(communityId) ?: return false if (!account.isWriteable()) return false - val cp = controlKeysForWrite(session) ?: return false + val cp = controlKeysForAction(session, ConcordPermissions.MANAGE_METADATA) ?: return false val metadata = MetadataEntity(name = name, icon = icon, banner = banner, description = description, relays = relays) val wrap = ConcordModeration.editMetadata(account.signer, cp, communityId.hexToByteArray(), metadata, session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) publishConcordWrap(session.entry, wrap) @@ -1027,7 +1113,7 @@ class AccountConcordActions( ): Boolean { val session = account.concordSessions.sessionFor(communityId) ?: return false if (!account.isWriteable()) return false - val cp = controlKeysForWrite(session) ?: return false + val cp = controlKeysForAction(session, ConcordPermissions.MANAGE_CHANNELS) ?: return false val channelId = RandomInstance.bytes(32) val channel = ChannelEntity(name = name.trim()) val wrap = ConcordModeration.defineChannel(account.signer, cp, channelId, channel, session.controlEditions(), TimeUtils.now(), owner = session.entry.owner) @@ -1043,7 +1129,7 @@ class AccountConcordActions( ): Boolean { val session = account.concordSessions.sessionFor(communityId) ?: return false if (!account.isWriteable()) return false - val cp = controlKeysForWrite(session) ?: return false + val cp = controlKeysForAction(session, ConcordPermissions.MANAGE_CHANNELS) ?: return false // Carry the standing definition forward and change only the name. A ChannelEntity built from // scratch defaults `private` and `voice` to false, so renaming a private channel used to // publish an edition declaring it PUBLIC — and a voice channel became a text channel. @@ -1066,7 +1152,7 @@ class AccountConcordActions( ): Boolean { val session = account.concordSessions.sessionFor(communityId) ?: return false if (!account.isWriteable()) return false - val cp = controlKeysForWrite(session) ?: return false + val cp = controlKeysForAction(session, ConcordPermissions.MANAGE_CHANNELS) ?: return false // Same as rename: preserve the standing flags so a tombstone does not also silently // reclassify the channel it retires. val standing = diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordChannelListScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordChannelListScreen.kt index d8c94f5ace..d33dfee5e2 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordChannelListScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordChannelListScreen.kt @@ -170,10 +170,14 @@ fun ConcordChannelListScreen( // Rank alone isn't enough on a split epoch: publishing any Control edition also takes the // control_root (CORD-02 §2), which a freshly promoted staffer may not hold yet (CORD-04 §3), // so the affordance waits for the key too. + // hasPermission, never effectivePermissions: the latter reads the roles alone, so a banned + // moderator kept seeing every control here. The editions they authored were dropped by everyone's + // fold, which made these buttons silently no-op — worse than absent, and the same trap this file + // already avoids for the Roles… menu. val canManageChannels = state?.authority?.let { it.isOwner(account.signer.pubKey) || - it.effectivePermissions(account.signer.pubKey).has(ConcordPermissions.MANAGE_CHANNELS) + it.hasPermission(account.signer.pubKey, ConcordPermissions.MANAGE_CHANNELS) } == true && session?.controlPlaneKeys()?.canWrite == true @@ -242,10 +246,19 @@ fun ConcordChannelListScreen( val canEdit = state?.authority?.let { it.isOwner(account.signer.pubKey) || - it.effectivePermissions(account.signer.pubKey).has(ConcordPermissions.MANAGE_METADATA) + it.hasPermission(account.signer.pubKey, ConcordPermissions.MANAGE_METADATA) } == true && session?.controlPlaneKeys()?.canWrite == true + // Minting an invite hands out a working key to the community, so it takes + // CREATE_INVITE like any other privileged action. This button used to be the one + // control on the screen with no gate at all. + val canInvite = + state?.authority?.let { + it.isOwner(account.signer.pubKey) || + it.hasPermission(account.signer.pubKey, ConcordPermissions.CREATE_INVITE) + } == true + IconButton(onClick = { nav.nav(Route.ConcordMembers(communityId)) }) { SymbolIcon(symbol = MaterialSymbols.Group, contentDescription = stringRes(com.vitorpamplona.amethyst.R.string.concord_members_title)) } @@ -254,22 +267,24 @@ fun ConcordChannelListScreen( SymbolIcon(symbol = MaterialSymbols.Edit, contentDescription = stringRes(com.vitorpamplona.amethyst.R.string.concord_edit_title)) } } - IconButton( - enabled = !minting, - onClick = { - minting = true - scope.launch { - try { - inviteLink = account.concord.mintConcordInvite(communityId) - } finally { - // Always clear the flag — a thrown mint would otherwise leave the - // button disabled until the screen is recreated. - minting = false + if (canInvite) { + IconButton( + enabled = !minting, + onClick = { + minting = true + scope.launch { + try { + inviteLink = account.concord.mintConcordInvite(communityId) + } finally { + // Always clear the flag — a thrown mint would otherwise leave the + // button disabled until the screen is recreated. + minting = false + } } - } - }, - ) { - SymbolIcon(symbol = MaterialSymbols.PersonAdd, contentDescription = stringRes(com.vitorpamplona.amethyst.R.string.concord_invite_action)) + }, + ) { + SymbolIcon(symbol = MaterialSymbols.PersonAdd, contentDescription = stringRes(com.vitorpamplona.amethyst.R.string.concord_invite_action)) + } } // Overflow, mirroring the NIP-29 relay-group top bar: destructive membership diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordMembersScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordMembersScreen.kt index 57b146653b..a9e166c46c 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordMembersScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordMembersScreen.kt @@ -133,7 +133,10 @@ fun ConcordMembersScreen( } val iAmOwner = state?.authority?.isOwner(myPubKey) == true - val iCanBan = state?.let { it.authority.isOwner(myPubKey) || it.authority.effectivePermissions(myPubKey).has(ConcordPermissions.BAN) } == true + // hasPermission, never effectivePermissions: a banned BAN-holder used to keep the whole Ban / + // Remove menu. It only stayed harmless because `canBanTarget` below routes through canActOn, + // which IS ban-aware — a thin margin for the escalation in docs/concord-soft-ban-audit.md. + val iCanBan = state?.let { it.authority.isOwner(myPubKey) || it.authority.hasPermission(myPubKey, ConcordPermissions.BAN) } == true val iCanManageRoles = state?.authority?.hasPermission(myPubKey, ConcordPermissions.MANAGE_ROLES) == true // The roles this viewer may actually hand out. The fold drops a grant whose granter does diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt index d99679263f..dff1143c45 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt @@ -454,7 +454,8 @@ object ConcordActions { fun recoverStranded( entry: ConcordCommunityListEntry, bundle: CommunityInvite, - ): ConcordCommunityListEntry? = ConcordStrandedRecovery.mergeForward(entry, bundle) + bannedAtCurrentEpoch: Boolean, + ): ConcordCommunityListEntry? = ConcordStrandedRecovery.mergeForward(entry, bundle, bannedAtCurrentEpoch) /** Decrypts + validates a fetched bundle event with the link token; null if invalid. */ fun openBundle( diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt index 7670ff69c6..58f6156b89 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordCommunitySession.kt @@ -486,6 +486,9 @@ class ConcordCommunitySession( if (!ChannelChat.isTyping(rumor) || !ChannelChat.isBoundTo(rumor, channelIdHex, epoch)) return val who = rumor.pubKey.lowercase() if (who == myPubKey.lowercase()) return // never show my own typing back to me + // A banned member's messages are dropped everywhere, so their typing heartbeat must be too — + // otherwise they sit in the "… is typing" row forever in a channel they cannot be heard in. + if (_state.value?.authority?.isBanned(who) == true) return val now = TimeUtils.now() // Update the map and publish inside the lock so a concurrent heartbeat on another // channel can't publish an older snapshot last and drop this channel's typers. diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordStrandedRecovery.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordStrandedRecovery.kt index 4f6e02d447..9869cadb91 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordStrandedRecovery.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordStrandedRecovery.kt @@ -46,12 +46,24 @@ object ConcordStrandedRecovery { * True when [bundle], resolved at [entry]'s stored invite link, proves we were * left behind: it must describe the same community and sit at a strictly higher * epoch. Same or lower is a no-op (we are current, or the bundle is stale). + * + * [bannedAtCurrentEpoch] is the caller's answer to "does the community, as I fold + * it right now, have me on its banlist?" — and a `true` refuses the recovery + * outright. It is a required argument rather than a caller-side `if` because + * getting it wrong turns this mechanism inside out: recovery exists so a member + * *wrongly* omitted from a rotation can catch up, but the test it performs (a + * higher epoch at a link whose unlock token an ex-member keeps forever) cannot + * tell that member apart from one the community deliberately removed. Without + * this, a Refounding — the only hard removal Concord has — is undone by our own + * background sweep a few minutes later. */ fun isStranded( entry: ConcordCommunityListEntry, bundle: CommunityInvite, + bannedAtCurrentEpoch: Boolean, ): Boolean = - entry.inviteRef != null && + !bannedAtCurrentEpoch && + entry.inviteRef != null && bundle.communityId.equals(entry.id, ignoreCase = true) && bundle.rootEpoch > entry.rootEpoch @@ -73,8 +85,9 @@ object ConcordStrandedRecovery { fun mergeForward( entry: ConcordCommunityListEntry, bundle: CommunityInvite, + bannedAtCurrentEpoch: Boolean, ): ConcordCommunityListEntry? { - if (!isStranded(entry, bundle)) return null + if (!isStranded(entry, bundle, bannedAtCurrentEpoch)) return null // Bank the epoch we are leaving with its control_pk, so its Control Plane // stays re-subscribable for the anti-rollback floor (a split epoch's address diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordStrandedRecoveryTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordStrandedRecoveryTest.kt index 6c0f9aa59c..96c1124e57 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordStrandedRecoveryTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordStrandedRecoveryTest.kt @@ -82,7 +82,7 @@ class ConcordStrandedRecoveryTest { val prior = HeldRoot(0L, "aa".repeat(32)) val stranded = entry(epoch = 1, heldRoots = listOf(prior)) - val merged = ConcordStrandedRecovery.mergeForward(stranded, bundle(epoch = 5)) + val merged = ConcordStrandedRecovery.mergeForward(stranded, bundle(epoch = 5), bannedAtCurrentEpoch = false) assertNotNull(merged, "a higher-epoch bundle at our own invite link means we were left behind") // adopted the new epoch's access root @@ -106,27 +106,27 @@ class ConcordStrandedRecoveryTest { @Test fun sameEpochBundleIsANoOp() { - assertNull(ConcordStrandedRecovery.mergeForward(entry(epoch = 5), bundle(epoch = 5))) - assertFalse(ConcordStrandedRecovery.isStranded(entry(epoch = 5), bundle(epoch = 5))) + assertNull(ConcordStrandedRecovery.mergeForward(entry(epoch = 5), bundle(epoch = 5), bannedAtCurrentEpoch = false)) + assertFalse(ConcordStrandedRecovery.isStranded(entry(epoch = 5), bundle(epoch = 5), bannedAtCurrentEpoch = false)) } @Test fun lowerEpochBundleIsANoOp() { // Epoch-monotonic: a stale bundle must never walk the membership backwards. - assertNull(ConcordStrandedRecovery.mergeForward(entry(epoch = 7), bundle(epoch = 3))) + assertNull(ConcordStrandedRecovery.mergeForward(entry(epoch = 7), bundle(epoch = 3), bannedAtCurrentEpoch = false)) } @Test fun entryWithoutInviteRefIsInert() { // Direct invites and legacy entries have no anchor — expected, not an error. val noAnchor = entry(epoch = 1, ref = null) - assertFalse(ConcordStrandedRecovery.isStranded(noAnchor, bundle(epoch = 9))) - assertNull(ConcordStrandedRecovery.mergeForward(noAnchor, bundle(epoch = 9))) + assertFalse(ConcordStrandedRecovery.isStranded(noAnchor, bundle(epoch = 9), bannedAtCurrentEpoch = false)) + assertNull(ConcordStrandedRecovery.mergeForward(noAnchor, bundle(epoch = 9), bannedAtCurrentEpoch = false)) } @Test fun bundleForAnotherCommunityIsIgnored() { - assertNull(ConcordStrandedRecovery.mergeForward(entry(epoch = 1), bundle(epoch = 9, id = "99".repeat(32)))) + assertNull(ConcordStrandedRecovery.mergeForward(entry(epoch = 1), bundle(epoch = 9, id = "99".repeat(32)), bannedAtCurrentEpoch = false)) } // ---- the bare `#` anchor form ---------------------------- @@ -252,4 +252,16 @@ class ConcordStrandedRecoveryTest { assertEquals(4L, other.rootEpoch) assertEquals(inviteRef, other.inviteRef) } + + @Test + fun aBannedMemberDoesNotRecoverIntoTheEpochTheyWereRemovedFrom() { + // The removal case the higher-epoch test cannot tell apart on its own: an ex-member keeps the + // link's unlock token forever, so without the ban gate the recovery sweep merges them into the + // very epoch a Refounding rotated them out of. See A2 in docs/concord-soft-ban-audit.md. + val stranded = entry(epoch = 1) + assertFalse(ConcordStrandedRecovery.isStranded(stranded, bundle(epoch = 5), bannedAtCurrentEpoch = true)) + assertNull(ConcordStrandedRecovery.mergeForward(stranded, bundle(epoch = 5), bannedAtCurrentEpoch = true)) + // ...and the legitimate case still works, so the gate is not just "recovery off". + assertNotNull(ConcordStrandedRecovery.mergeForward(stranded, bundle(epoch = 5), bannedAtCurrentEpoch = false)) + } } From 54c412da7aace3e93e6c82cffb85e634b5fd7638 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 15:12:41 +0000 Subject: [PATCH 087/132] fix(quartz): stop a stray edition from pinning a Control entity forever MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit B1 in docs/concord-soft-ban-audit.md, the worst item on the list: one edition at version = Long.MAX_VALUE permanently pinned its entity to the author's content, for every client holding a floor for it, with no way back. The floor rose to MAX_VALUE, no honest edition could exceed it, and a Refounding that dropped the poison fell back to EntityFloor.known — the poison. Authored in the tests by a current, legitimately granted moderator: no ban, no sockpuppet, one ordinary permission bit. The chain walk was never the weakness; it advances only to head.version + 1 citing the head's hash, so a fresh joiner was untouched. The compaction arm was: it trades contiguity for cross-epoch tolerance, which left VERSION as the only contest an edition had to win. Two changes. The arm now tries the floor-anchored chain first and falls back to the raw-version bootstrap only when nothing connects, so a stray never wins a fold where the honest chain is present. And the bootstrap will not follow a jump of more than MAX_COMPACTION_VERSION_JUMP above the floor — a compacted head is legitimately ahead by a chain's worth, not by 2^63 — so the version space cannot be exhausted in a step. A new test pins the tolerance the arm exists for, so the bound cannot later be tightened into breaking CORD-06 §3. compactControlPlane picked its per-entity head by raw highest version too, which made an honest rotator the delivery mechanism: a disconnected stray never joins the chain but won that comparison, and was re-wrapped into the new epoch as the entity's whole history, where fresh joiners anchor on it. It now picks the chain head, keeping foldEntity's fresh-joiner fallback for the dangling `prev` a prior compaction leaves behind. The three reproductions now assert the fixed behaviour. The banlist's escape hatch (a floor-less chain walk plus the re-heal union) is kept and still pinned. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DrJhpFhhLjuDJQNkGvYMGj --- .../quartz/concord/cord04Roles/EditionFold.kt | 66 ++++++++- .../concord/cord06Rekey/ConcordRefounding.kt | 26 +++- .../ControlPlaneVersionExhaustionTest.kt | 132 +++++++++++++----- 3 files changed, 176 insertions(+), 48 deletions(-) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/EditionFold.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/EditionFold.kt index 4e5b8fa72c..956a0ec461 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/EditionFold.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/EditionFold.kt @@ -150,9 +150,63 @@ object EditionFold { floorVersion: Long, ): ControlEdition? = editions - .filter { it.version >= floorVersion } + .filter { it.version >= floorVersion && it.version - floorVersion <= MAX_COMPACTION_VERSION_JUMP } .minWithOrNull(compareByDescending { it.version }.thenBy { it.rumorId }) + /** + * How far above the floor the compaction arm will follow an edition in one step. + * + * The arm trades contiguity for cross-epoch tolerance, which made VERSION the only contest an + * edition had to win — so a single authorized edition at `version = Long.MAX_VALUE` used to + * become an entity's permanent head: it won the arm, `authorizedHeads` raised the floor to + * `Long.MAX_VALUE`, and from there no honest edition could ever exceed the floor again. Even a + * Refounding that dropped the poison did not help, because nothing was then offered at or above + * the floor and the fold fell back to [EntityFloor.known] — the poison itself. See B1 in + * `docs/concord-soft-ban-audit.md`. + * + * A compacted head is legitimately ahead of the floor by however many editions the entity gained + * while we were away — a chain's worth, not 2^63. This bound is deliberately far above any real + * community (a channel renamed a thousand times a day for three years stays under it) and far + * below the point where the version space can be exhausted. Anything beyond it is not a + * compaction we missed; it is someone reaching for the ceiling, and it is treated as a gap. + */ + const val MAX_COMPACTION_VERSION_JUMP = 1_000_000L + + /** + * The floor-anchored chain head among [editions], or null when nothing connects to [floor]. + * + * The same anchor-then-walk the main path uses, factored out so the compaction arm can try it + * first: the anchor is the floor's own edition (same version AND hash) or its immediate + * successor citing that hash, then the walk climbs while each `version + 1` cites the current + * head. Reports no gap — a null here means "fall back", not "refuse". + */ + private fun chainHead( + editions: List, + floor: EntityFloor, + ): ControlEdition? { + val byVersion = HashMap>() + for (e in editions) byVersion.getOrPut(e.version) { ArrayList() }.add(e) + + val lowest = byVersion.keys.filter { it >= floor.version }.minOrNull() ?: return null + val winner = byVersion[lowest]?.minByOrNull { it.rumorId } ?: return null + var head = + when (lowest) { + floor.version -> winner.takeIf { it.hashHex == floor.hashHex } + floor.version + 1 -> winner.takeIf { it.prevHash != null && it.prevHash.toHexKey() == floor.hashHex } + else -> null + } ?: return null + + while (true) { + val next = + byVersion[head.version + 1] + ?.filter { it.prevHash != null && it.prevHash.toHexKey() == head.hashHex } + ?.minByOrNull { it.rumorId } + ?: break + head = next + } + return head + } + /** * Groups mixed [editions] by entity id and folds each to its head, honoring the * per-entity anti-rollback [floors] (keyed by [ControlEdition.entityIdHex]). @@ -210,10 +264,16 @@ object EditionFold { // to the compacted head. Presence of the entity in the snapshot selects the ARM; // version selects the HEAD, over every edition we hold and not just the subset. if (floor != null && snapshot != null && editions.any { it.rumorId in snapshot }) { + // Chain first, bootstrap only as the fallback. The arm exists for the case where the + // offered head genuinely cannot be connected — but when it CAN be, the connected head is + // strictly better evidence than "highest number wins", and preferring it denies a stray + // high-version edition its free win in every ordinary fold. The bootstrap keeps the + // cross-epoch case working, now bounded by MAX_COMPACTION_VERSION_JUMP. + chainHead(editions, floor)?.let { return it } return bootstrapHead(editions, floor.version) ?: run { - // Nothing at or above the floor was served: the head we already accepted - // vanished from the offered set — withheld, so fail closed. + // Nothing admissible at or above the floor was served: the head we already + // accepted vanished from the offered set — withheld, so fail closed. onGap(editions[0].entityIdHex, floor.version, editions.maxOf { it.version }) floor.known } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefounding.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefounding.kt index 18df7e3c46..7dc80de9cb 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefounding.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefounding.kt @@ -21,6 +21,7 @@ package com.vitorpamplona.quartz.concord.cord06Rekey import com.vitorpamplona.quartz.concord.cord04Roles.ControlEdition +import com.vitorpamplona.quartz.concord.cord04Roles.EditionFold import com.vitorpamplona.quartz.concord.crypto.ConcordKeyDerivation import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys import com.vitorpamplona.quartz.concord.crypto.GroupKey @@ -167,18 +168,29 @@ object ConcordRefounding { priorControlKeys: ControlPlaneKeys, newControlKeys: ControlPlaneKeys, ): List { - // entity coordinate -> (head edition, its verified seal) - val heads = HashMap>() + // entity coordinate -> every edition we can open, paired with its verified seal. + val byCoordinate = HashMap>>() for (wrap in priorWraps) { val opened = ConcordStreamEnvelope.openOrNull(wrap, priorControlKeys) ?: continue val edition = ControlEdition.fromRumor(opened.rumor) ?: continue val coord = edition.entityKind.wire + ":" + edition.entityIdHex - val current = heads[coord] - if (current == null || edition.version > current.first.version) { - heads[coord] = edition to opened.seal - } + byCoordinate.getOrPut(coord) { ArrayList() }.add(edition to opened.seal) } - return heads.values.map { (_, seal) -> ConcordStreamEnvelope.wrapSeal(seal, newControlKeys, createdAt = seal.createdAt) } + + // The head is the CHAIN head, not the highest version. Picking by raw version made an honest + // rotator the delivery mechanism for a disconnected stray: an edition minted at an arbitrary + // version never joins the chain, but it won this comparison and was then re-wrapped into the + // new epoch as that entity's whole history — where a fresh joiner, holding no floor, anchors + // on it as their baseline. See B1 in `docs/concord-soft-ban-audit.md`. foldEntity walks from + // genesis and keeps the fresh-joiner fallback for a head whose own `prev` dangles into an + // epoch this rotator no longer holds, which is the ordinary shape after a prior compaction. + val out = ArrayList(byCoordinate.size) + for ((_, entries) in byCoordinate) { + val head = EditionFold.foldEntity(entries.map { it.first }) ?: continue + val seal = entries.firstOrNull { it.first.rumorId == head.rumorId }?.second ?: continue + out.add(ConcordStreamEnvelope.wrapSeal(seal, newControlKeys, createdAt = seal.createdAt)) + } + return out } /** diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlPlaneVersionExhaustionTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlPlaneVersionExhaustionTest.kt index bbba1dc8d3..c97ed0dba9 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlPlaneVersionExhaustionTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/ControlPlaneVersionExhaustionTest.kt @@ -27,35 +27,43 @@ import kotlin.test.assertEquals import kotlin.test.assertTrue /** - * **V1 in `docs/concord-soft-ban-audit.md` — reproduction.** A single Control Plane edition at - * `version = Long.MAX_VALUE` pins its entity to the author's content permanently, for every client - * that holds a floor for it. + * **B1 in `docs/concord-soft-ban-audit.md` — regression guard.** A single Control Plane edition at + * `version = Long.MAX_VALUE` used to pin its entity to the author's content permanently, for every + * client that held a floor for it. These tests failed before the fix and pass after it. * - * The chain walk is not the weakness — it advances only to `head.version + 1` citing the head's - * hash, so an inflated version is unreachable and a fresh joiner is unaffected. The weakness is the - * **compaction arm** of [EditionFold.foldEntity]: once a client holds a floor for an entity and that - * entity appears in the epoch snapshot (which `ConcordCommunityState.fold` always builds from the - * editions handed to it), the head is chosen by [EditionFold.bootstrapHead] — *highest version at or - * above the floor*, with no `prev`, no hash, and no contiguity. Version is then the whole contest, - * and `Long.MAX_VALUE` wins it forever: + * The chain walk was never the weakness — it advances only to `head.version + 1` citing the head's + * hash, so an inflated version is unreachable and a fresh joiner was unaffected. The weakness was the + * **compaction arm** of `EditionFold.foldEntity`: once a client held a floor for an entity and that + * entity appeared in the epoch snapshot (which `ConcordCommunityState.fold` always builds from the + * editions handed to it), the head came from the raw-version bootstrap — *highest version at or above + * the floor*, with no `prev`, no hash, and no contiguity. Version was then the whole contest, and + * `Long.MAX_VALUE` won it forever: * - * 1. the poison becomes the head, so the entity shows the attacker's content; - * 2. `authorizedHeads` raises the entity's floor to `Long.MAX_VALUE`; - * 3. no honest edition can ever exceed that floor, so the entity can never be repaired; - * 4. a Refounding that drops the poison does not help either — nothing is offered at or above the - * floor, so the fold reports a gap and falls back to [EntityFloor.known], which *is* the poison. + * 1. the poison became the head, so the entity showed the attacker's content; + * 2. `authorizedHeads` raised the entity's floor to `Long.MAX_VALUE`; + * 3. no honest edition could ever exceed that floor, so the entity could never be repaired; + * 4. a Refounding that dropped the poison did not help either — nothing was then offered at or above + * the floor, so the fold reported a gap and fell back to `EntityFloor.known`, which *was* the poison. + * + * Two changes close it, and both are pinned below. The arm now tries the floor-anchored **chain** + * first and only falls back to the raw-version bootstrap when nothing connects, so a stray never wins + * a fold where the honest chain is present; and the bootstrap will not follow a jump larger than + * [EditionFold.MAX_COMPACTION_VERSION_JUMP], so the version space cannot be exhausted in one step. + * [aGenuineCompactionJumpIsStillFollowed] pins the tolerance the arm exists for, so the bound cannot + * be tightened into breaking CORD-06 §3. * * Note who the attacker is. Every test here is authored by **bob, a current and legitimately granted * moderator** — not a banned member, not a sockpuppet. Any holder of the entity's permission bit can - * do this at any time, and demoting or banning them afterwards changes nothing, because the damage - * is already in every client's floor. It is also carried into every future epoch by - * `ConcordRefounding.compactControlPlane`, which selects the head per entity by raw highest version. + * could do this at any time, and demoting or banning them afterwards changed nothing, because the + * damage was already in every client's floor. `ConcordRefounding.compactControlPlane` also selected + * the head per entity by raw highest version, which made an honest rotator the delivery mechanism — + * it now picks the chain head instead. * - * The banlist is the one entity that survives, and by accident: `AuthorityResolver` folds it with + * The banlist was the one entity that survived, and by accident: `AuthorityResolver` folds it with * its own floor-less chain walk and then re-heals the union across authorized editions, so an - * honest ban lands even when the head is poisoned. [aPoisonedBanlistStillAcceptsTheOwnersBan] pins - * that, because it is the only thing standing between this bug and a permanently unmoderatable - * community. + * honest ban landed even when the head was poisoned. [aPoisonedBanlistStillAcceptsTheOwnersBan] + * keeps pinning that, because it was the only thing standing between this bug and a permanently + * unmoderatable community. */ class ControlPlaneVersionExhaustionTest { private val owner = "0f".repeat(32) @@ -86,7 +94,7 @@ class ControlPlaneVersionExhaustionTest { ) + rest @Test - fun oneEditionAtMaxVersionPinsTheMetadataForever() { + fun oneEditionAtMaxVersionNoLongerPinsTheMetadata() { val metadataV0 = edition(ControlEntityKind.METADATA, metadataEntity, 0, null, """{"name":"My Community"}""", owner, "meta-0") val community = communityWhereBobHolds(ConcordPermissions.of(ConcordPermissions.MANAGE_METADATA).toWire(), metadataV0) @@ -96,7 +104,7 @@ class ControlPlaneVersionExhaustionTest { val poison = edition(ControlEntityKind.METADATA, metadataEntity, Long.MAX_VALUE, metadataV0.hash, """{"name":"PWNED"}""", bob, "meta-poison") val floorsAfter = ConcordCommunityState.authorizedHeads(community + poison, owner, floorsBefore) - assertEquals(Long.MAX_VALUE, floorsAfter[metadataEntity]?.version, "VULNERABLE: the floor is now at the top of the version space") + assertEquals(0, floorsAfter[metadataEntity]?.version, "the floor must not follow a stray to the top of the version space") // The owner tries to repair it, chaining honestly onto their own genesis. val repair = edition(ControlEntityKind.METADATA, metadataEntity, 1, metadataV0.hash, """{"name":"My Community"}""", owner, "meta-1") @@ -108,19 +116,19 @@ class ControlPlaneVersionExhaustionTest { "a fresh joiner walks the chain and is unaffected", ) assertEquals( - "PWNED", + "My Community", ConcordCommunityState.fold(pool, owner, floorsAfter).metadata?.name, - "VULNERABLE: every client holding a floor is pinned to the attacker's content", + "a client holding a floor follows the honest chain, not the stray", ) assertEquals( - "PWNED", + "My Community", ConcordCommunityState.fold(community + repair, owner, floorsAfter).metadata?.name, - "VULNERABLE: even a Refounding that drops the poison falls back to it as EntityFloor.known", + "and a Refounding that drops the poison stays repaired", ) } @Test - fun oneEditionAtMaxVersionDeletesAChannelForever() { + fun oneEditionAtMaxVersionNoLongerDeletesAChannel() { val channelV0 = edition(ControlEntityKind.CHANNEL, channelEntity, 0, null, """{"name":"general"}""", owner, "chan-0") val community = communityWhereBobHolds(ConcordPermissions.of(ConcordPermissions.MANAGE_CHANNELS).toWire(), channelV0) @@ -132,28 +140,29 @@ class ControlPlaneVersionExhaustionTest { val pool = community + poison + repair assertEquals(1, ConcordCommunityState.fold(pool, owner).channels.size, "a fresh joiner still sees the channel") - assertEquals(0, ConcordCommunityState.fold(pool, owner, floorsAfter).channels.size, "VULNERABLE: the channel is gone and cannot be restored") + assertEquals(1, ConcordCommunityState.fold(pool, owner, floorsAfter).channels.size, "and so does a client holding a floor") assertEquals( - 0, + 1, ConcordCommunityState.fold(community + repair, owner, floorsAfter).channels.size, - "VULNERABLE: dropping the poison does not bring the channel back", + "the channel survives a Refounding too", ) } @Test fun aPoisonedBanlistStillAcceptsTheOwnersBan() { - // The saving grace, and the reason this bug is "unmoderatable community" rather than - // "community with a broken name". AuthorityResolver folds the banlist on its own floor-less - // chain walk and re-heals the union across every authorized edition, so the owner's ban lands - // even while the banlist's own floor sits at Long.MAX_VALUE. Do not "unify" the banlist onto - // the floored fold without replacing this protection. + // This was the saving grace before the fix — the reason the bug was "community with a broken + // name" rather than "community nobody can moderate". AuthorityResolver folds the banlist on + // its own floor-less chain walk and re-heals the union across every authorized edition, so + // the owner's ban landed even while the banlist's floor sat at Long.MAX_VALUE. The floor can + // no longer be poisoned, but keep this: do not "unify" the banlist onto the floored fold + // without replacing the protection. val banlistV0 = edition(ControlEntityKind.BANLIST, banlistEntity, 0, null, "[]", owner, "ban-0") val community = communityWhereBobHolds(ConcordPermissions.of(ConcordPermissions.BAN).toWire(), banlistV0) val floorsBefore = ConcordCommunityState.authorizedHeads(community, owner) val poison = edition(ControlEntityKind.BANLIST, banlistEntity, Long.MAX_VALUE, banlistV0.hash, "[]", bob, "ban-poison") val floorsAfter = ConcordCommunityState.authorizedHeads(community + poison, owner, floorsBefore) - assertEquals(Long.MAX_VALUE, floorsAfter[banlistEntity]?.version, "the banlist floor is poisoned like any other") + assertEquals(0, floorsAfter[banlistEntity]?.version, "the banlist floor is no longer poisonable either") val ownerBansBob = edition(ControlEntityKind.BANLIST, banlistEntity, 1, banlistV0.hash, """["$bob"]""", owner, "ban-1") val pool = community + poison + ownerBansBob @@ -164,4 +173,51 @@ class ControlPlaneVersionExhaustionTest { "the re-heal union must keep the banlist working even with a poisoned floor", ) } + + @Test + fun aGenuineCompactionJumpIsStillFollowed() { + // The tolerance the compaction arm exists for, pinned so the bound above cannot be tightened + // into breaking CORD-06 §3. After a Refounding the compacted head carries the `prev` it had + // before compaction, citing an edition in the PRIOR epoch that this client no longer holds — + // so it connects to nothing, and its version is legitimately several ahead of our floor. + val metadataV0 = edition(ControlEntityKind.METADATA, metadataEntity, 0, null, """{"name":"My Community"}""", owner, "meta-0") + val community = communityWhereBobHolds(ConcordPermissions.of(ConcordPermissions.MANAGE_METADATA).toWire(), metadataV0) + val floors = ConcordCommunityState.authorizedHeads(community, owner) + + val danglingPrev = ByteArray(32) { 0x7f } + val compacted = edition(ControlEntityKind.METADATA, metadataEntity, 4, danglingPrev, """{"name":"Renamed While We Were Away"}""", owner, "meta-compacted") + + assertEquals( + "Renamed While We Were Away", + ConcordCommunityState.fold(community.filter { it.entityKind != ControlEntityKind.METADATA } + compacted, owner, floors).metadata?.name, + "a compacted head whose prev dangles by design must still be adopted", + ) + } + + @Test + fun aJumpBeyondTheCapIsRefusedAsAGap() { + // Same shape as the genuine compaction above, one version past the bound: not a compaction we + // missed, so the fold reports a gap and keeps what it already had rather than following it. + val metadataV0 = edition(ControlEntityKind.METADATA, metadataEntity, 0, null, """{"name":"My Community"}""", owner, "meta-0") + val community = communityWhereBobHolds(ConcordPermissions.of(ConcordPermissions.MANAGE_METADATA).toWire(), metadataV0) + val floors = ConcordCommunityState.authorizedHeads(community, owner) + + val danglingPrev = ByteArray(32) { 0x7f } + val tooFar = + edition( + ControlEntityKind.METADATA, + metadataEntity, + EditionFold.MAX_COMPACTION_VERSION_JUMP + 1, + danglingPrev, + """{"name":"PWNED"}""", + bob, + "meta-far", + ) + + assertEquals( + "My Community", + ConcordCommunityState.fold(community.filter { it.entityKind != ControlEntityKind.METADATA } + tooFar, owner, floors).metadata?.name, + "a jump past the bound is a gap, not a head", + ) + } } From 4e99aafb59f5f081f5fe6f13924abcc15ee38f8e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 15:47:41 +0000 Subject: [PATCH 088/132] fix(quartz): honor the banlist against the Control Plane itself MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit B2 in docs/concord-soft-ban-audit.md, plus B4's bound and the audit's status pass. hasPermission was ban-aware; the resolver's own ROLE/GRANT/BANLIST gates were not, and could not be as written — the roles/grants fixpoint settled before `banned` was computed at all. So half the Control Plane honored a ban and half was blind to it, and a banned staffer still holding control_root kept the whole roster: banning everyone beneath them, revoking the surviving moderators, retiring the roles under them, and minting a fresh un-banned npub that passed every ban-aware gate and finished the job. resolve() is now a bounded two-pass where authority only ever shrinks. Pass A resolves as before and yields a candidate banlist; pass B re-resolves with every author on it treated as holding no authority. Two passes always, so it terminates by construction, and mutual bans cannot oscillate because the rank rule makes them unreachable — only someone who strictly outranks you may ban you, and you cannot outrank them back. A chain-local rule would not have worked: forking the banlist at genesis means no parent ever mentions the ban and §4's re-heal union carries it in regardless, so the rule is a whole-pass mask rather than a per-edition check. This cascades, deliberately: every edition a banned member ever authored is dropped, grants included, so banning an admin also demotes everyone that admin promoted. That is the literal reading of CORD-04 §4 and it is what kills the sockpuppet, but a legitimate promotion by a later-banned admin vanishes with it and has to be re-issued. Both the cascade and its blast radius are pinned, and the trade-off is written up in the Armada report as the answer to its own open row 3 — which also widens the divergence recorded there: we now drop editions they honor wherever a privileged member was banned. B4: the Refounding recipient set is capped. allMembers() is the Guestbook ∪ observedAuthors ∪ the roster, and the first two are unbounded and attacker-writable, so each throwaway npub someone posts from became one more mandatory blob in the next Refounding — the attack inflating the cost of its own remedy. The owner-rooted roster is kept first and anything dropped is logged, never silently truncated, because a dropped member is stranded. The nine escalation reproductions now assert the fixed behaviour. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DrJhpFhhLjuDJQNkGvYMGj --- .../amethyst/model/AccountConcordActions.kt | 48 ++++++- docs/concord-banlist-rank-conformance.md | 35 ++++- docs/concord-soft-ban-audit.md | 134 ++++++++++++------ .../concord/cord04Roles/AuthorityResolver.kt | 52 ++++++- .../cord04Roles/BannedStaffEscalationTest.kt | 111 ++++++++++----- 5 files changed, 299 insertions(+), 81 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt index c403e08986..5b5c8fe0db 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt @@ -33,6 +33,7 @@ import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEven import com.vitorpamplona.quartz.concord.cord02Community.HeldRoot import com.vitorpamplona.quartz.concord.cord02Community.ImagePointer import com.vitorpamplona.quartz.concord.cord03Channels.ChannelChat +import com.vitorpamplona.quartz.concord.cord04Roles.AuthorityResolver import com.vitorpamplona.quartz.concord.cord04Roles.ChannelEntity import com.vitorpamplona.quartz.concord.cord04Roles.ConcordJson import com.vitorpamplona.quartz.concord.cord04Roles.ConcordPermissions @@ -76,6 +77,15 @@ private const val CONCORD_ADMIN_ROLE = "Admin" */ private const val RECOVERY_CHECK_INTERVAL_MS = 15 * 60 * 1000L +/** + * How many recipients one Refounding will re-key. See `AccountConcordActions.boundRecipients`. + * + * 120 blobs ride in each kind-3303 chunk, so this is ~42 published events and ~5k NIP-44 + * encryptions at the ceiling — heavy but survivable on a phone, and far above any real community. + * Raising it raises the cost of the attack it exists to bound, not the safety. + */ +private const val MAX_REFOUNDING_RECIPIENTS = 5_000 + /** * Concord (encrypted communities) orchestration for an [Account]: join/create/ * invite flows, channel messages/reactions/edits/typing, roles and moderation, @@ -810,7 +820,7 @@ class AccountConcordActions( .apply { removeAll(removedLower) removeAll(authority.bannedMembers()) - }.toList() + }.let { candidates -> boundRecipients(candidates, authority) } // 3. Build the refounding: new root, compacted Control Plane, per-recipient rekey blobs. val entry = session.entry @@ -853,6 +863,42 @@ class AccountConcordActions( return true } + /** + * Caps the Refounding recipient set, keeping the members whose standing we can actually vouch + * for when there are too many. + * + * `allMembers()` is the Guestbook ∪ `observedAuthors` ∪ the roster, and the first two are + * unbounded and attacker-writable: a Guestbook Join is self-signed by any key at all, and every + * author we decrypt is folded in by design (CORD-02 §5, "observably present"). So each throwaway + * npub someone posts from, or simply announces, becomes one more mandatory blob in the next + * Refounding — meaning the attack inflates the cost of its own remedy, and the remedy is the only + * hard removal Concord has. See B4 in `docs/concord-soft-ban-audit.md`. + * + * The roster and the owner are kept unconditionally: they are owner-rooted, so they cannot be + * padded from outside. The remainder fills the budget, and anything dropped is **logged rather + * than silently truncated** — a dropped member is stranded on the dead epoch and their only way + * back is a recovery path that needs to know it happened. + */ + private fun boundRecipients( + candidates: Set, + authority: AuthorityResolver, + ): List { + if (candidates.size <= MAX_REFOUNDING_RECIPIENTS) return candidates.toList() + + val vouched = authority.roleHolders() + authority.staffMembers() + val kept = LinkedHashSet(MAX_REFOUNDING_RECIPIENTS) + candidates.filterTo(kept) { it in vouched } + for (candidate in candidates) { + if (kept.size >= MAX_REFOUNDING_RECIPIENTS) break + kept.add(candidate) + } + Log.w("Concord") { + "Refounding recipient set capped at $MAX_REFOUNDING_RECIPIENTS of ${candidates.size}: " + + "${candidates.size - kept.size} member(s) will be stranded on the prior epoch" + } + return kept.toList() + } + // Rotations we've already adopted ("communityId:epoch"), so a base-rekey wrap still buffered // in the pre-rebuild window (the session rebuild off `liveCommunities` is async) is not // adopted — and re-published — twice on successive revision ticks. diff --git a/docs/concord-banlist-rank-conformance.md b/docs/concord-banlist-rank-conformance.md index afbe98fdbd..04e92c9de7 100644 --- a/docs/concord-banlist-rank-conformance.md +++ b/docs/concord-banlist-rank-conformance.md @@ -1,7 +1,8 @@ # Concord: the Banlist is not rank-gated in any implementation (CORD-04 conformance) **Status:** conformance bug. Reproduced in Amethyst and **fixed there** (see §6); present by -inspection in Armada. +inspection in Armada. Finding #3, left open in §4 as a fixpoint-ordering question, is now also +implemented in Amethyst — see the 2026-08-09 update before §Rollout status. **Severity:** privilege escalation. Any `BAN` holder can neutralise every authority above them, including the owner. **Reported by:** Amethyst (MIT), 2026-07-20. Findings verified by unit test; see "Evidence" below. @@ -190,6 +191,38 @@ We'd also suggest **§4 restating the rank half inline**, the way §2 does for G does for Kicks. Both independent implementations read §4 in isolation and both got it wrong the same way; that is strong evidence the section is the problem, not the readers. +### Update, 2026-08-09: we have now implemented #3 + +Amethyst now answers the ordering question rather than leaving it open, because #3 turned out to be +the doorway to a full community takeover and not merely an inconsistency — a banned staffer who kept +`control_root` kept the entire roster, and could mint a fresh un-banned npub that passed every +ban-aware gate. The write-up is `docs/concord-soft-ban-audit.md` (B2). + +The rule we shipped: **authority only ever shrinks, over two passes.** Pass A resolves exactly as +before and yields a candidate banlist; pass B re-resolves with every author on that list treated as +holding no authority at all, for roles, grants and the Banlist alike. Two passes, always, so it +terminates by construction. It cannot oscillate on mutual bans either, because the rank rule makes +them unreachable: only a member who strictly outranks you may ban you, and you cannot outrank them +back. + +Note what it costs, because it is not obvious and you would hit it too: this **cascades**. Every +edition a banned member ever authored is dropped, grants included, so banning an admin also demotes +everyone that admin promoted. We think that is the literal reading of §4 and it is what kills the +sockpuppet — but a legitimate promotion by a later-banned admin vanishes with it, and the owner has +to re-issue it. If you read §4 as scoping only to editions authored *after* the ban, say so; that is +implementable too, but it needs the spec to define an ordering between an edition and a Banlist +entry, which today it does not. + +A chain-local rule is not enough, and this is the trap worth flagging: "the author must not be banned +by the state their edition chains from" is bypassed by forking the Banlist at genesis, where no +parent ever mentions the ban and §4's re-heal union carries it in anyway. The rule has to bind the +union, which is why ours is a whole-pass mask rather than a per-edition check. + +This widens the divergence in §6: we now drop editions you honor in any community where a privileged +member was banned, not only where a signer failed to outrank their target. + +--- + Separately, please rule on **#3**: whether a banned npub's Banlist edition is honored. Our reading of §4 ("drops every event from a banned npub — message, reaction, edit, or authority action") is that it must not be, but the fixpoint ordering needs to be stated for that to be implementable consistently. diff --git a/docs/concord-soft-ban-audit.md b/docs/concord-soft-ban-audit.md index 58429394ca..1da43ed4d0 100644 --- a/docs/concord-soft-ban-audit.md +++ b/docs/concord-soft-ban-audit.md @@ -1,7 +1,9 @@ # Concord: soft-ban and Control Plane audit **Scope:** what a removed member — or a moderator who turns — can still do to a Concord community. -**Date:** 2026-08-09. **Status:** findings only, nothing fixed yet. +**Date:** 2026-08-09. **Status:** A1–A4, B1, B2 and B4 are **fixed** on this branch; the rest are +accepted, deferred to the spec, or belong to other people's relays. Each section carries its own +status line. **Companion:** `docs/concord-banlist-rank-conformance.md` (the rank half of CORD-04 §4, already reported to Armada and fixed here). @@ -47,32 +49,32 @@ Two structural causes account for most of both halves: ### Part A — reachable from stock Amethyst (our bugs) -| # | Finding | Severity | Was | -|---|---------|----------|-----| -| [A1](#a1) | Any member — banned included — mints a working invite in one tap | **Critical** | new | -| [A2](#a2) | Stranded recovery runs on a timer and never checks the banlist | **Critical** | V10 | -| [A3](#a3) | The action layer has no permission checks; the UI's are ban-blind | High | new | -| [A4](#a4) | A banned member keeps broadcasting "typing", and we keep showing it | Low | V12 | -| [A5](#a5) | A banned member's own client keeps reading and rendering everything | Medium | V8 | +| # | Finding | Severity | Status | +|---|---------|----------|--------| +| [A1](#a1) | Any member — banned included — mints a working invite in one tap | **Critical** | **Fixed** | +| [A2](#a2) | Stranded recovery runs on a timer and never checks the banlist | **Critical** | **Fixed** (security half; liveness half open) | +| [A3](#a3) | The action layer has no permission checks; the UI's are ban-blind | High | **Fixed** | +| [A4](#a4) | A banned member keeps broadcasting "typing", and we keep showing it | Low | **Fixed** | +| [A5](#a5) | A banned member's own client keeps reading and rendering everything | Medium | Inherent — product decision | ### Part B — requires a malicious client -| # | Finding | Severity | Needs a ban? | Recoverable? | Was | -|---|---------|----------|--------------|--------------|-----| -| [B1](#b1) | One edition at `version = Long.MAX_VALUE` pins an entity forever | **Critical** | No — any bit-holder | **No** | V1 | -| [B2](#b2) | A banned staffer keeps Role/Grant/Banlist authority | **Critical** | Yes | Yes (Refounding) | V2 | -| [B3](#b3) | A rogue rotator compacts the banlist away | High | Via B2 | Partly | V3 | -| [B4](#b4) | The Refounding recipient set is attacker-inflatable | High | No | Yes | V4 | -| [B5](#b5) | The ban is per-pubkey; the channel key is not revoked | High | Yes | Yes (Refounding) | V5 | -| [B6](#b6) | Channel history is deletable on a naive third-party relay | High | Yes | **No** (history) | V6 | -| [B7](#b7) | The base-rekey plane is writable by every member | Low | Yes | Yes | V9 | +| # | Finding | Severity | Needs a ban? | Status | +|---|---------|----------|--------------|--------| +| [B1](#b1) | One edition at `version = Long.MAX_VALUE` pins an entity forever | **Critical** | No — any bit-holder | **Fixed** | +| [B2](#b2) | A banned staffer keeps Role/Grant/Banlist authority | **Critical** | Yes | **Fixed** (consensus-affecting) | +| [B3](#b3) | A rogue rotator compacts the banlist away | High | Via B2 | **Mitigated** by B2 | +| [B4](#b4) | The Refounding recipient set is attacker-inflatable | High | No | **Fixed** (bounded) | +| [B5](#b5) | The ban is per-pubkey; the channel key is not revoked | High | Yes | Inherent — Refounding is the answer | +| [B6](#b6) | Channel history is deletable on a naive third-party relay | High | Yes | Correct here; external relays at risk | +| [B7](#b7) | The base-rekey plane is writable by every member | Low | Yes | Accepted | ### Part C — interop and not-yet-shipped -| # | Finding | Severity | Was | -|---|---------|----------|-----| -| [C1](#c1) | Banlist rank rule diverges from Armada | Medium | V7 | -| [C2](#c2) | CORD-07 voice rooms are key-gated, not roster-gated | Design | V11 | +| # | Finding | Severity | Status | +|---|---------|----------|--------| +| [C1](#c1) | Banlist rank rule diverges from Armada | Medium | Reported; B2 widens the divergence | +| [C2](#c2) | CORD-07 voice rooms are key-gated, not roster-gated | Design | Note for whoever ships voice | --- @@ -82,6 +84,9 @@ No custom tooling. A banned user with the shipping app, or our own background sw ## A1 — Any member, banned included, mints a working invite in one tap +**Status: fixed.** `mintConcordInvite` and its button now require `CREATE_INVITE` (or ownership). + + **Critical. The single most likely thing an irritated banned user actually does.** *Read:* `AccountConcordActions.mintConcordInvite`, `ConcordChannelListScreen` (the `PersonAdd` `IconButton`). @@ -106,6 +111,12 @@ button on the same. This is contained, uncontroversial, and closes the realistic ## A2 — Stranded recovery runs on a timer and never checks the banlist +**Status: security half fixed; liveness half open.** `isStranded` / `mergeForward` now take +`bannedAtCurrentEpoch` as a *required* argument, so a removed member is no longer walked back in. +Whether anything should re-mint at a stable coordinate — without which legitimate recovery never +fires for anyone — still needs a spec answer and is untouched. + + **Critical, and it forks.** *Read:* `ConcordStrandedRecovery`, `AccountConcordActions.recoverStrandedConcordCommunities`, `AccountConcordActions.mintConcordInvite`. @@ -137,6 +148,10 @@ evicted owners another route. ## A3 — The action layer has no permission checks; the UI's are ban-blind +**Status: fixed.** Authority now lives in `AccountConcordActions.isAuthorizedFor`, which every +moderation verb funnels through, and every authorization test uses the ban-aware `hasPermission`. + + **High (defense in depth).** *Read:* `AccountConcordActions` (`banConcordMember`, `unbanConcordMember`, `editConcordMetadata`, `deleteConcordChannel`, `refoundConcordCommunity`), `ConcordMembersScreen`, `ConcordChannelListScreen`. @@ -169,6 +184,9 @@ their roles say", independent of standing. ## A4 — A banned member keeps broadcasting "typing", and we keep showing it +**Status: fixed on both ends.** + + **Low, both halves ours.** *Read:* `AccountConcordActions.sendConcordTyping`, `ConcordCommunitySession.ingestTyping`. @@ -180,6 +198,11 @@ and it directly contradicts what a ban promises the user. ## A5 — A banned member's own client keeps reading and rendering everything +**Status: inherent; no code change.** The cryptography cannot be fixed without a Refounding, so what +is left is a product decision about how "Ban" and "Remove from community" are presented. Left for a +design pass rather than guessed at here. + + **Medium, partly inherent.** *Read:* CORD-02/05, `ConcordCommunitySession`. Until a Refounding, a ban stops honest clients from *showing* the banned member's posts; it does not @@ -201,6 +224,12 @@ what they publish. ## B1 — One edition at `Long.MAX_VALUE` pins an entity forever +**Status: fixed.** The compaction arm tries the floor-anchored chain first and bounds the bootstrap +jump at `EditionFold.MAX_COMPACTION_VERSION_JUMP`; `compactControlPlane` picks the chain head rather +than raw max version. The three reproductions now assert the fixed behaviour, and +`aGenuineCompactionJumpIsStillFollowed` pins the CORD-06 §3 tolerance the bound must not break. + + **Critical. Does not require a banned user, a sockpuppet, or the owner's absence. Unrecoverable.** *Verified:* `quartz/…/cord04Roles/ControlPlaneVersionExhaustionTest.kt` (3 tests). @@ -246,6 +275,14 @@ The first is the smallest change and closes the unrecoverability; the third shou ## B2 — A banned staffer keeps Role, Grant and Banlist authority +**Status: fixed — and consensus-affecting.** `AuthorityResolver.resolve` is now a bounded two-pass +where authority only shrinks. Note the deliberate cascade it brings: every edition a banned member +ever authored is dropped, so banning an admin also demotes everyone that admin promoted. That is the +literal reading of CORD-04 §4 and it is what kills the sockpuppet, but a legitimate promotion by a +later-banned admin vanishes with it and has to be re-issued. Until Armada ships the same rule the two +clients can disagree about any community where a privileged member was banned. + + **Critical.** *Verified:* `quartz/…/cord04Roles/BannedStaffEscalationTest.kt` (13 tests). `hasPermission` is ban-aware; the resolver's internal gates are not, and structurally cannot be as @@ -277,6 +314,11 @@ Armada ships the same rule, we will drop editions they honor. ## B3 — A rogue rotator compacts the banlist away +**Status: mitigated by B2.** The rotator this needed was the sockpuppet, which can no longer be +minted. A *legitimately* privileged rotator can still omit the banlist, and `EntityFloor` remains the +only defense for clients that already folded it — unchanged, and still worth a spec fix. + + **High.** *Verified:* `aRogueRotatorCompactsTheBanAwayForEveryClientWithoutAFloor`. A CORD-06 §3 compaction re-wraps one edition per entity and the *rotator* picks it, so a rotator can @@ -292,6 +334,10 @@ protect people who were already there. ## B4 — The Refounding recipient set is attacker-inflatable +**Status: fixed (bounded).** The recipient set is capped, the owner-rooted roster is kept first, and +anything dropped is logged rather than silently truncated. + + **High.** *Read:* `ConcordCommunitySession.allMembers()` / `emitChannelRumors`; `AccountConcordActions.refoundConcordCommunity` step 2; `ConcordRefounding.buildBaseRekeyWraps`. @@ -311,6 +357,9 @@ the recipient set, prefer recent/attested members when over the cap, and surface ## B5 — The ban is a per-pubkey display rule and the channel key is not revoked +**Status: inherent.** No client-side fix exists; a Refounding is the answer, which is why B4 mattered. + + **High.** *Read:* `Account.consumeConcordRumorGated` (`isBanned(rumor.pubKey)`), `Account.isAcceptable`. Writing to a channel needs the channel key, which the ban does not take away; the seal author is @@ -324,6 +373,10 @@ correct design, which is why B4 matters so much. ## B6 — Channel history is deletable on a naive third-party relay +**Status: correct on our relay and pinned; external relays remain exposed.** Needs a CORD-01 spec note +and relay-selection guidance, not code. + + **High, external.** *Verified (that we are safe):* `geode/…/ConcordPlaneKeyDeletionTest.kt` (3 tests). @@ -346,6 +399,9 @@ Worth a note in the CORD-01 spec and a line in the relay-selection guidance. ## B7 — The base-rekey plane is writable by every member +**Status: accepted.** Bounded work per wrap, no correctness impact. + + **Low.** *Read:* `ConcordKeyDerivation.baseRekeyAddress`, `AccountConcordActions.drainConcordRekeys`. The base-rekey address derives from `community_root`, so any member — banned included — can mint @@ -406,25 +462,19 @@ Not looked at at all: - Unread counts and notification triggers, media/upload references from messages, the NIP-53 nests overlap, and the desktop client's Concord paths. -## Suggested order +## What is left -**Part A first.** It is the whole of the realistic threat — a banned user with the app already -installed — and none of it needs coordination with anyone. - -1. **A1** — one guard on `mintConcordInvite` plus one on its button. Smallest fix on the list and it - closes the attack a banned user will actually reach for. -2. **A3** — move authority into the action layer and replace `effectivePermissions` with - `hasPermission` everywhere it is used as an authorization test. This is also the cheapest partial - mitigation for B2: it shrinks what a banned staffer can do *without* writing their own client. -3. **A2** — needs the semantics decided before any code. Raise it with the spec. -4. **A4 / A5** — small, user-visible, and they make the product honest about what a ban is. - -**Then Part B**, hardest first because the ceiling is highest: - -5. **B4** — cheap, not consensus-affecting, and it protects the remedy every other fix depends on. -6. **B1** — worst blast radius, the only unrecoverable one, and the bar is a single ordinary - permission bit. -7. **B2 (+B3, +C1's open row)** — one two-pass change closes all three. Coordinate with Armada - first; this one splits consensus. -8. **B6** — spec note plus relay-selection guidance; our own behaviour is already correct and pinned. -9. **B5 / B7** — accept, or bound. +1. **A2's liveness half** — decide whether a community re-mints its invite bundle at a stable + coordinate. Today nothing does, so stranded recovery never fires for anyone, and an owner evicted + by a rogue admin has no route back. Needs a spec answer before code. +2. **C1 / B2 interop** — tell Armada about the two-pass rule, as with the rank rule before it. The + divergence is now wider: we drop editions they honor whenever a privileged member is banned. +3. **B6** — a CORD-01 note that a plane's wraps must stay owned by a key nobody holds, plus guidance + that a relay authorizing NIP-09/62 by `pubkey` hands every ex-member a wipe button. +4. **B3's residue** — a legitimately privileged rotator can still omit an entity during compaction. + `EntityFloor` catches it for clients that were present; fresh joiners have nothing. +5. **A5** — a design pass on how "Ban" and "Remove from community" are presented, since they promise + very different things. +6. **The unexamined surfaces below**, particularly private channels — there appears to be no + channel-scoped rekey receive path at all, which would mean the full-community Refounding is the + only removal Amethyst can perform. diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt index 039a63e7ca..ff85147ea6 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt @@ -135,9 +135,54 @@ data class AuthorityResolver private constructor( /** The owner's rank — supreme and unremovable. No Role may claim it. */ const val OWNER_RANK = 0L + /** + * The owner-rooted authority state of a community, with the banlist honored **against the + * Control Plane itself** (CORD-04 §4: a reader "drops every event from a banned npub — + * message, reaction, edit, or authority action"). + * + * This is a bounded two-pass, because the rule is circular as stated: you cannot know who is + * banned until you fold the Banlist, and you cannot decide who may write the Banlist without + * knowing who is banned. `docs/concord-banlist-rank-conformance.md` §4 row 3 flagged that to + * the spec authors and left it open. We resolve it by making authority only ever **shrink**: + * + * - **Pass A** resolves exactly as before, ban-blind, and yields a candidate banlist. + * - **Pass B** re-resolves with every author in that banlist treated as unauthorized, for + * roles, grants and the banlist alike. + * + * Two passes, always, so it terminates by construction — pass B never feeds back. It cannot + * oscillate on mutual bans either, because the rank rule makes them unreachable: only a + * member who strictly outranks you may ban you, and you cannot outrank them back. + * + * **This cascades, deliberately.** Every edition a banned member ever authored is dropped, + * including grants they made while in good standing — so banning an admin also demotes + * everyone that admin promoted. That is the literal reading of §4, and it is the point: the + * escalation in `docs/concord-soft-ban-audit.md` B2 was a banned staffer minting a fresh, + * un-banned npub and acting through it, and dropping the grant is what kills the puppet. The + * cost is that a legitimate promotion by a later-banned admin vanishes too, and the owner has + * to re-issue it. + * + * **Consensus-affecting.** Armada gates the Control Plane on role-derived permissions alone, + * so until it ships the same rule the two clients can disagree about any community where a + * privileged member was banned. + */ fun resolve( editions: Collection, ownerPubKey: String, + ): AuthorityResolver { + val passA = resolveOnce(editions, ownerPubKey, bannedAuthors = emptySet()) + if (passA.banned.isEmpty()) return passA + return resolveOnce(editions, ownerPubKey, bannedAuthors = passA.banned) + } + + /** + * One resolution pass. [bannedAuthors] are treated as holding no authority at all — their + * role, grant and banlist editions are dropped rather than merely being unable to act on + * others. Empty on pass A; pass A's banlist on pass B. See [resolve]. + */ + private fun resolveOnce( + editions: Collection, + ownerPubKey: String, + bannedAuthors: Set, ): AuthorityResolver { val ownerLower = ownerPubKey.lowercase() @@ -191,6 +236,7 @@ data class AuthorityResolver private constructor( ): Boolean { val author = e.author.lowercase() if (author == ownerLower) return true + if (author in bannedAuthors) return false if (!holdsManageRoles(author)) return false val authorRank = rankOf(author) ?: return false val r = ConcordJson.decodeOrNull(e.content) ?: return false @@ -223,6 +269,7 @@ data class AuthorityResolver private constructor( fun grantGate(e: ControlEdition): Boolean { val granter = e.author.lowercase() if (granter == ownerLower) return true + if (granter in bannedAuthors) return false if (!holdsManageRoles(granter)) return false val granterRank = rankOf(granter) ?: return false val g = ConcordJson.decodeOrNull(e.content) ?: return false @@ -269,7 +316,9 @@ data class AuthorityResolver private constructor( // a concurrent ban is never lost, while an on-chain unban still takes effect. val allBanlist = editions.filter { it.entityKind == ControlEntityKind.BANLIST } - fun banGate(e: ControlEdition): Boolean = e.author.lowercase() == ownerLower || effectivePermissionsOf(e.author.lowercase()).has(ConcordPermissions.BAN) + fun banGate(e: ControlEdition): Boolean = + e.author.lowercase() == ownerLower || + (e.author.lowercase() !in bannedAuthors && effectivePermissionsOf(e.author.lowercase()).has(ConcordPermissions.BAN)) val authorizedBanlist = allBanlist.filter(::banGate) // CORD-04 §3's rank rule binds "every action", and it names banning as its example ("an @@ -292,6 +341,7 @@ data class AuthorityResolver private constructor( // owner is never a valid target — not even for themselves. if (target == ownerLower) return false if (author == ownerLower) return true + if (author in bannedAuthors) return false if (!effectivePermissionsOf(author).has(ConcordPermissions.BAN)) return false val authorRank = rankOf(author) ?: return false val targetRank = rankOf(target) ?: Long.MAX_VALUE // no roles ⇒ lowest authority diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/BannedStaffEscalationTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/BannedStaffEscalationTest.kt index aedadd885a..5b6c0fc64d 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/BannedStaffEscalationTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/BannedStaffEscalationTest.kt @@ -28,27 +28,31 @@ import kotlin.test.assertFalse import kotlin.test.assertTrue /** - * What a **soft-banned staffer** can still do to a community — the reproduction behind - * `docs/concord-banlist-rank-conformance.md` §4 row 3, which the report left open as "a genuine - * fixpoint-ordering question, not a plain oversight". + * **B2 in `docs/concord-soft-ban-audit.md` — regression guard.** What a soft-banned staffer used to + * be able to do to a community, and can no longer. Every test here failed before the two-pass rule + * in [AuthorityResolver.resolve] and passes after it. * - * The asymmetry these tests pin: [ConcordCommunityState.fold] gates METADATA / CHANNEL / INVITE - * through `authority.hasPermission`, which is `!isBanned && …`, but ROLE, GRANT and BANLIST are - * gated *inside* [AuthorityResolver.resolve] by `holdsManageRoles` / `bitsOf` / - * `effectivePermissionsOf` — none of which consult the banlist. Nor could they as written: the - * roles/grants fixpoint runs before `banned` is computed at all. So half the Control Plane honors - * a ban and half is structurally blind to it, and a banned member who still holds `control_root` - * keeps full authority over the roster. + * The asymmetry they were written to pin: [ConcordCommunityState.fold] gates METADATA / CHANNEL / + * INVITE through `authority.hasPermission`, which is `!isBanned && …`, but ROLE, GRANT and BANLIST + * were gated *inside* [AuthorityResolver.resolve] by `holdsManageRoles` / `bitsOf` / + * `effectivePermissionsOf` — none of which consulted the banlist, nor could they as written, since + * the roles/grants fixpoint ran before `banned` was computed at all. Half the Control Plane honored + * a ban and half was structurally blind to it, so a banned member still holding `control_root` kept + * full authority over the roster: they banned everyone beneath them, revoked the surviving + * moderators, retired the roles under them, and minted a fresh un-banned npub that passed every + * ban-aware gate and finished the job. * - * **These tests assert the CURRENT, VULNERABLE behaviour**, so the escalation cannot regress - * silently or be "fixed" by accident without someone noticing. Every `ESCALATION:` assertion here - * must be INVERTED — not deleted — when the ordering rule lands. [selfUnbanIsStillRefused] and - * [aJuniorPuppetCannotLiftASeniorsBan] are the opposite: they pin behaviour the fix must preserve. + * The fix resolves the ordering by making authority only ever shrink across two passes — see + * [AuthorityResolver.resolve]. Note that a chain-local rule ("the author must not be banned by the + * state their edition chains from") would NOT have been enough: + * [aBannedAdminForksTheBanlistAtGenesisRatherThanChainingOntoTheirOwnBan] forks at genesis so no + * parent ever mentions the ban, and CORD-04 §4's re-heal union would carry it in anyway. The rule + * had to bind the union too, which is why it is expressed as a whole-pass mask. * - * Note for whoever writes that fix: a chain-local rule ("the author must not be banned by the state - * their edition chains from") is NOT sufficient — see - * [aBannedAdminForksTheBanlistAtGenesisRatherThanChainingOntoTheirOwnBan]. The rule has to bind - * CORD-04 §4's re-heal union too. + * Three tests pin behaviour the fix had to *preserve* rather than change: + * [selfUnbanIsStillRefused], [aJuniorPuppetCannotLiftASeniorsBan], and + * [aBanByOneAdminDoesNotDropTheGrantsOfAnother]. One pins the cost it deliberately accepts: + * [banningAnAdminAlsoDemotesEveryoneThatAdminPromoted]. */ class BannedStaffEscalationTest { private val owner = "0f".repeat(32) @@ -167,10 +171,12 @@ class BannedStaffEscalationTest { assertTrue(r.isBanned(alice), "the owner's ban lands") assertFalse(r.hasPermission(alice, ConcordPermissions.MANAGE_ROLES), "the ban-aware check refuses her") - // ...but this is the one every ROLE/GRANT/BANLIST gate inside resolve() actually consults. + // effectivePermissions still reports what her ROLES say — that is its job, and the members + // screen reads it to label her. What changed is that the resolver's own ROLE/GRANT/BANLIST + // gates no longer consult it for a banned author; they drop the edition outright. assertTrue( r.effectivePermissions(alice).has(ConcordPermissions.MANAGE_ROLES), - "ESCALATION: a banned staffer keeps the permissions the resolver's own gates read", + "the role-derived view is unchanged — only what it authorizes is", ) } @@ -178,11 +184,11 @@ class BannedStaffEscalationTest { fun aBannedAdminPromotesAFreshSockpuppetToAdmin() { val r = AuthorityResolver.resolve(community() + ownerBansAlice + aliceMintsAPuppet(), owner) - assertEquals(2, r.rank(puppet), "ESCALATION: the banned admin's role edition is honored") - assertFalse(r.isBanned(puppet), "the puppet is a clean npub — nothing to filter it on") - assertTrue( + assertEquals(null, r.rank(puppet), "the banned admin's role and grant editions are both dropped") + assertFalse(r.isBanned(puppet), "the puppet itself is a clean npub — it is never banned, just powerless") + assertFalse( r.hasPermission(puppet, ConcordPermissions.MANAGE_CHANNELS), - "ESCALATION: a banned member minted a live admin with the ban-aware check passing", + "a banned member cannot mint authority it no longer has to give", ) } @@ -196,8 +202,8 @@ class BannedStaffEscalationTest { val state = ConcordCommunityState.fold(editions, owner) - assertEquals(0, state.channels.size, "ESCALATION: the community's channels are irrecoverably tombstoned") - assertEquals("Owned by the guy you banned", state.metadata?.name, "ESCALATION: and its identity rewritten") + assertEquals(1, state.channels.size, "the puppet holds nothing, so its tombstone is inert") + assertEquals("My Community", state.metadata?.name, "and the community keeps its identity") } @Test @@ -206,8 +212,8 @@ class BannedStaffEscalationTest { val r = AuthorityResolver.resolve(editions, owner) - assertTrue(r.isBanned(bob), "ESCALATION: the surviving moderator is silenced, losing all authority with it") - assertTrue(r.isBanned(carol), "ESCALATION: and the plain members with them") + assertFalse(r.isBanned(bob), "the puppet's banlist edition is unauthorized, so the moderator stands") + assertFalse(r.isBanned(carol), "and so do the plain members") } @Test @@ -216,8 +222,9 @@ class BannedStaffEscalationTest { val r = AuthorityResolver.resolve(editions, owner) - assertTrue(r.isBanned(bob), "ESCALATION: banGate reads effectivePermissionsOf, which ignores her own ban") - assertTrue(r.isBanned(carol), "ESCALATION: same") + assertTrue(r.isBanned(alice), "her own ban stands — it was the owner's") + assertFalse(r.isBanned(bob), "banGate now drops a banned author's edition outright") + assertFalse(r.isBanned(carol), "same") } @Test @@ -231,8 +238,8 @@ class BannedStaffEscalationTest { val r = AuthorityResolver.resolve(editions, owner) assertTrue(r.isBanned(alice), "the owner's ban survives the fork — the union is down-only") - assertTrue(r.isBanned(bob), "ESCALATION: and so does the banned admin's, healed in as a concurrent ban") - assertTrue(r.isBanned(carol), "ESCALATION: same") + assertFalse(r.isBanned(bob), "the fix binds the UNION too: her fork is dropped before it can be healed in") + assertFalse(r.isBanned(carol), "same") } @Test @@ -241,8 +248,8 @@ class BannedStaffEscalationTest { val r = AuthorityResolver.resolve(editions, owner) - assertEquals(null, r.rank(bob), "ESCALATION: a banned admin stripped a live moderator's roles") - assertFalse(r.hasPermission(bob, ConcordPermissions.BAN), "ESCALATION: leaving nobody but the owner able to act") + assertEquals(5, r.rank(bob), "a banned admin's revoke is dropped, so the moderator keeps their role") + assertTrue(r.hasPermission(bob, ConcordPermissions.BAN), "and keeps the authority that comes with it") } @Test @@ -251,8 +258,8 @@ class BannedStaffEscalationTest { val r = AuthorityResolver.resolve(community() + ownerBansAlice + tombstone, owner) - assertEquals(null, r.roles()[modRole], "ESCALATION: a banned admin retired a role beneath them") - assertEquals(null, r.rank(bob), "ESCALATION: every holder of it silently loses their standing") + assertEquals(5, r.roles()[modRole]?.position, "a banned admin's tombstone is dropped, so the role survives") + assertEquals(5, r.rank(bob), "and its holders keep their standing") } @Test @@ -323,4 +330,36 @@ class BannedStaffEscalationTest { "a client that already folded the ban must refuse the rollback", ) } + + @Test + fun banningAnAdminAlsoDemotesEveryoneThatAdminPromoted() { + // The deliberate cascade, pinned because it is surprising and because it is the whole point. + // CORD-04 §4 drops every event from a banned npub, authority actions included, so a grant + // they made while in good standing goes too. That is what kills a sockpuppet minted moments + // before the ban — and the same rule costs the owner a legitimate promotion, which they have + // to re-issue. See B2 in docs/concord-soft-ban-audit.md. + val promoted = grant("36".repeat(32), carol, listOf(modRole), author = alice) + + val before = AuthorityResolver.resolve(community() + promoted, owner) + assertEquals(5, before.rank(carol), "while alice is in good standing, her grant stands") + + val after = AuthorityResolver.resolve(community() + promoted + ownerBansAlice, owner) + assertEquals(null, after.rank(carol), "banning alice retroactively drops the grant she authored") + } + + @Test + fun aBanByOneAdminDoesNotDropTheGrantsOfAnother() { + // The cascade must follow the banned author, not spread. Bob is untouched by alice's ban, so + // everything he authored keeps standing. + val carolByBob = grant("37".repeat(32), carol, listOf(modRole), author = bob) + // bob is a Mod at position 5 and the role he hands out is that same position, so the grant is + // only honored when authored by someone who outranks it — the owner does, bob does not. + val carolByOwner = grant("38".repeat(32), carol, listOf(modRole), author = owner) + + val r = AuthorityResolver.resolve(community() + ownerBansAlice + carolByBob + carolByOwner, owner) + + assertTrue(r.isBanned(alice), "alice is the only one banned") + assertEquals(5, r.rank(bob), "bob is untouched") + assertEquals(5, r.rank(carol), "and the owner's grant of carol stands") + } } From 6147f72c8112e8a0e9652064a970695634b09743 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 15:54:55 +0000 Subject: [PATCH 089/132] docs(concord): check the audit against Armada, and correct two conclusions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Read gitlab.com/soapbox-pub/armada src/concord-v2/ against every finding. Two conclusions change. B2 is NOT consensus-affecting, and the warning in the last commit was wrong. Armada's foldControlState already runs the same bounded two-pass — fold once, take the banlist, re-fold with banned authors' editions excluded — arrived at independently, same shape, same CORD-04 §4 justification in the comment. This change brings us into line rather than out of it. One narrower divergence remains: they keep pass 1's banlist as final, we recompute it in pass 2, so a banned admin's mass-ban still stands for them and is dropped by us. Both defensible; ours closes an attack theirs leaves open, and the self-erasure they guard against is unreachable under the rank rule. A2's fork is resolved, in favour of the fix having been necessary. useLinkRefreshWatch2 re-posts every invite bundle on each epoch change, so the "if anything re-mints at a stable coordinate" branch is what actually happens — in any cross-client community a removed member's Amethyst client would have pulled the new root within fifteen minutes. Their catch-up is push instead: a privileged member sends a direct invite carrying the fresher root, so a human authorizes each re-admission, and useBanSelfRemove2 has a banned member's own client silently drop the community. The liveness half stands and now has two concrete options rather than an open question. Also recorded: B1 is present in Armada unfixed, in exactly the same place (bootstrapHead is unbounded, headCandidates uses it, pickHead raises the floor) — the second bug both clients share by reading one section the same way, so it goes to them in writing like the rank rule did. A1 was ours alone; they gate invite creation on CREATE_INVITE in both the hook and the page. C1 is unchanged on their side. A4 is a shared gap. And a divergence in the other direction: their banlist takes only the head's content, with no §4 re-heal union, so we honor concurrent bans they drop. B4 is marked unchecked rather than guessed at — I could not locate their recipient-set construction with confidence. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DrJhpFhhLjuDJQNkGvYMGj --- docs/concord-banlist-rank-conformance.md | 23 +++++++ docs/concord-soft-ban-audit.md | 80 +++++++++++++++++++++--- 2 files changed, 94 insertions(+), 9 deletions(-) diff --git a/docs/concord-banlist-rank-conformance.md b/docs/concord-banlist-rank-conformance.md index 04e92c9de7..d48623c113 100644 --- a/docs/concord-banlist-rank-conformance.md +++ b/docs/concord-banlist-rank-conformance.md @@ -213,6 +213,29 @@ to re-issue it. If you read §4 as scoping only to editions authored *after* the implementable too, but it needs the spec to define an ordering between an edition and a Banlist entry, which today it does not. +We checked your implementation before writing this, and you got there first: `foldControlState` +already runs the same two-pass, with the same §4 justification in the comment. So this is us catching +up, not diverging — with one narrower difference. You keep **pass 1's** Banlist as the final word; +we recompute it in pass 2. So a banned admin's mass-ban of everyone beneath them still stands for you +and is dropped by us. Your stated reason is to stop the anti-roster erasing itself; ours is that an +edition should not outlive its author's removal, and the self-erasure case is unreachable under the +rank rule anyway, since only a member who strictly outranks you can ban you. We would rather converge +than be right — tell us which way and we will move. + +Two more things that fell out of reading `src/concord-v2/` side by side, both worth their own look: + +- **`bootstrapHead` has no bound** (`lib/version.ts`), and `headCandidates` uses it for the + compaction arm while `pickHead` then raises the stored floor to whatever won. One authorized + edition at `version = 2^63 - 1` therefore becomes an entity's permanent head: the floor rises to + match, nothing honest can exceed it, and even a Refounding that drops the edition falls back to the + remembered head — which is that edition. It needs no ban and no sockpuppet, just one ordinary + permission bit. This is the second bug both implementations share by reading the same section the + same way; ours is described in `docs/concord-soft-ban-audit.md` (B1), and we bounded the jump the + arm will follow. +- **Your Banlist takes only the gated head's content**, with no §4 re-heal union. We union in every + authorized non-ancestor edition, which is what defeats an attempt to launder a ban away by forking + the list at genesis. So we honor concurrent bans you drop. Which is normative? + A chain-local rule is not enough, and this is the trap worth flagging: "the author must not be banned by the state their edition chains from" is bypassed by forking the Banlist at genesis, where no parent ever mentions the ban and §4's re-heal union carries it in anyway. The rule has to bind the diff --git a/docs/concord-soft-ban-audit.md b/docs/concord-soft-ban-audit.md index 1da43ed4d0..6040d8ee7f 100644 --- a/docs/concord-soft-ban-audit.md +++ b/docs/concord-soft-ban-audit.md @@ -62,7 +62,7 @@ Two structural causes account for most of both halves: | # | Finding | Severity | Needs a ban? | Status | |---|---------|----------|--------------|--------| | [B1](#b1) | One edition at `version = Long.MAX_VALUE` pins an entity forever | **Critical** | No — any bit-holder | **Fixed** | -| [B2](#b2) | A banned staffer keeps Role/Grant/Banlist authority | **Critical** | Yes | **Fixed** (consensus-affecting) | +| [B2](#b2) | A banned staffer keeps Role/Grant/Banlist authority | **Critical** | Yes | **Fixed** (matches Armada) | | [B3](#b3) | A rogue rotator compacts the banlist away | High | Via B2 | **Mitigated** by B2 | | [B4](#b4) | The Refounding recipient set is attacker-inflatable | High | No | **Fixed** (bounded) | | [B5](#b5) | The ban is per-pubkey; the channel key is not revoked | High | Yes | Inherent — Refounding is the answer | @@ -111,10 +111,13 @@ button on the same. This is contained, uncontroversial, and closes the realistic ## A2 — Stranded recovery runs on a timer and never checks the banlist -**Status: security half fixed; liveness half open.** `isStranded` / `mergeForward` now take -`bannedAtCurrentEpoch` as a *required* argument, so a removed member is no longer walked back in. -Whether anything should re-mint at a stable coordinate — without which legitimate recovery never -fires for anyone — still needs a spec answer and is untouched. +**Status: security half fixed; liveness half open — and the fork is now resolved.** `isStranded` / +`mergeForward` take `bannedAtCurrentEpoch` as a *required* argument, so a removed member is no longer +walked back in. The open question was whether anything re-mints at a stable coordinate. **Armada +does** — `useLinkRefreshWatch2` re-posts every bundle on each epoch change — so in any cross-client +community this was a *live* removal bypass, not a hypothetical, and the fix was load-bearing. The +liveness half stands: Amethyst re-mints nothing, so legitimate recovery never fires for an +Amethyst-only community. See [the Armada comparison](#armada) for the two ways out. **Critical, and it forks.** *Read:* `ConcordStrandedRecovery`, @@ -275,13 +278,13 @@ The first is the smallest change and closes the unrecoverability; the third shou ## B2 — A banned staffer keeps Role, Grant and Banlist authority -**Status: fixed — and consensus-affecting.** `AuthorityResolver.resolve` is now a bounded two-pass +**Status: fixed. Not consensus-affecting after all** — see [Armada comparison](#armada). Armada +already implements the same two-pass, so this brings us *into* line rather than out of it. One +narrower divergence remains, described there. `AuthorityResolver.resolve` is now a bounded two-pass where authority only shrinks. Note the deliberate cascade it brings: every edition a banned member ever authored is dropped, so banning an admin also demotes everyone that admin promoted. That is the literal reading of CORD-04 §4 and it is what kills the sockpuppet, but a legitimate promotion by a -later-banned admin vanishes with it and has to be re-issued. Until Armada ships the same rule the two -clients can disagree about any community where a privileged member was banned. - +later-banned admin vanishes with it and has to be re-issued. **Critical.** *Verified:* `quartz/…/cord04Roles/BannedStaffEscalationTest.kt` (13 tests). @@ -440,6 +443,65 @@ It would be the one place where a ban fails *audibly*, in real time, in front of worth designing the roster check in before shipping rather than after. + +--- + +## Armada comparison (checked 2026-08-09) + +Read against `gitlab.com/soapbox-pub/armada` at `src/concord-v2/`. Worth doing before shipping any of +this, and it changed two conclusions. + +**B2 — they already do it, and we had it backwards.** `foldControlState` (`lib/control.ts`) runs the +same bounded two-pass: fold once, take the banlist, and if any edition was authored by someone on it, +re-fold with those editions excluded. Independently arrived at, same shape, same CORD-04 §4 +justification in the comment. So this change brings us *into* line with Armada rather than out of it, +and the consensus warning in the earlier revision of this doc was wrong. + +One real divergence remains, and it is ours to defend: Armada keeps **pass 1's** banlist as the final +word ("the first pass's Banlist stays the final word"), while we recompute the banlist in pass 2. So +a banned admin's mass-ban of everyone beneath them still stands in Armada and is dropped by us — the +`aBannedAdminBansEveryoneBeneathThemWithoutNeedingAPuppetAtAll` case. Their stated reason is to stop +the anti-roster erasing itself; ours is that an edition from a banned author should not survive its +own author's removal. Both are defensible; ours closes an attack theirs leaves open, and the +self-erasure they worry about is unreachable for us because the rank rule makes mutual bans +impossible (only someone who strictly outranks you can ban you). Worth raising with them. + +**B1 — the same bug, unfixed, in exactly the same place.** `bootstrapHead` (`lib/version.ts:155`) +takes the highest version at or above the floor with no bound; `headCandidates` uses it for the +compaction arm; `pickHead` then raises the stored floor to whatever won. That is the whole +version-exhaustion chain. This is now the second bug both implementations share because both read +the same section the same way, and it deserves the same treatment as the rank rule: a written report. + +**A1 — ours alone.** Armada gates invite creation on `CREATE_INVITE` in both the hook +(`useInvites2.ts`) and the page (`canCreateInvite`). We were the only client handing a banned member +a working invite button. + +**A2 — different architecture, and it is better.** Armada's catch-up is **push**, not pull: a +privileged member sends a stranded member a direct invite carrying the fresher root +(`useDirectInvites2`, `catchUp`), so a human authorizes each re-admission. `useRekeyWatch2` merely +reports `{ stranded: boolean }` for the UI. They also ship `useBanSelfRemove2`: a banned member's own +client silently drops the community from their private list — network-silent, deliberately narrower +than rekey-exclusion, because "a rotation can be a mistake; a ban is a judgment". Our pull-from-my-own- +old-link design is what made the bypass possible, and their per-epoch bundle refresh is what would +have supplied the higher epoch to pull. Two ways forward: adopt a refresh of our own (restores +liveness, keeps the pull design and its risk), or move to their push model (safer, and it is what the +one existing implementation does). + +**C1 — still open on their side.** `banlistGate` remains a bare `isAuthorized(roster, author, owner, +BAN)`: no rank check, no delta rule. The divergence from +`docs/concord-banlist-rank-conformance.md` is unchanged. + +**A divergence in the other direction.** Armada's banlist takes only the gated head's content — +there is no §4 re-heal union. Ours unions in every authorized non-ancestor edition, which is what +defeats the genesis-fork laundering attempt in `BannedStaffEscalationTest`. So we honor concurrent +bans they drop. Worth a spec question about which is normative. + +**A4 — shared gap.** No ban filter on typing there either. + +**B4 — not established.** I could not locate a recipient-set bound in their rekey path, but I also +could not locate the recipient-set construction itself with confidence, so treat this as unchecked +rather than as a finding either way. + --- ## What was NOT examined From 54d3bfc84a6f78d5adb3f108c8f3c23ba3813ee5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 16:09:53 +0000 Subject: [PATCH 090/132] perf(quartz): skip the resolver's second pass when it cannot change anything MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two-pass ban-aware fold doubles resolve(), which runs once per held epoch in controlFloorsLocked plus once in fold, on a client that re-folds the whole buffer from scratch on every Control Plane change. So the second pass is now skipped unless a banned member actually authored a Control edition — not merely when the banlist is empty. Bans overwhelmingly land on plain members who hold no role and write nothing, and for those pass B is provably identical to pass A. Armada's fold checks the same condition. Measured over ConcordCommunityState.fold (throwaway benchmark, not committed; 226 and 2059 editions, 200 reps after warmup). Pass A is byte-for-byte the old algorithm, so the single-pass rows are the before-numbers: 226 eds, no bans 1457 us 226 eds, 20 bans, none authored 994 us 226 eds, 20 bans, one authored -> pass B 1881 us 2059 eds, no bans 2194 us 2059 eds, 50 bans, none authored 2001 us 2059 eds, 50 bans, one authored -> pass B 5697 us 2059 eds, with floors (B1's arm) 2015 us So the common case is free, and B1's chain-first compaction arm is not measurable — the floored fold matches the unfloored one. A banned staffer costs ~2-3x, which is the price of the fix and is paid only under the attack. The audit records this, plus the standing opportunity it surfaced: we have no fold memoization where Armada does, which predates this work and would absorb the pass-B cost too. Not done here — that is a change to make on its own merits. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DrJhpFhhLjuDJQNkGvYMGj --- docs/concord-soft-ban-audit.md | 40 +++++++++++++++++++ .../concord/cord04Roles/AuthorityResolver.kt | 5 +++ 2 files changed, 45 insertions(+) diff --git a/docs/concord-soft-ban-audit.md b/docs/concord-soft-ban-audit.md index 6040d8ee7f..8e766c8b46 100644 --- a/docs/concord-soft-ban-audit.md +++ b/docs/concord-soft-ban-audit.md @@ -502,6 +502,46 @@ bans they drop. Worth a spec question about which is normative. could not locate the recipient-set construction itself with confidence, so treat this as unchecked rather than as a finding either way. + +--- + +## Performance of the fixes + +Measured on the JVM with a synthetic Control Plane (throwaway benchmark, not committed — +`ConcordCommunityState.fold` over 226 and 2059 editions, 200 reps after warmup). Pass A of the +two-pass resolver is byte-for-byte the old algorithm, so the single-pass rows below *are* the +before-numbers. + +| Case | µs / fold | +|---|---| +| 226 editions, no bans | 1457 | +| 226 editions, 20 bans, none of them authors | 994 | +| 226 editions, 20 bans, one an author → pass B runs | 1881 | +| 2059 editions, no bans | 2194 | +| 2059 editions, 50 bans, none of them authors | 2001 | +| 2059 editions, 50 bans, one an author → pass B runs | 5697 | +| 2059 editions, with floors (B1's compaction arm) | 2015 | + +Two things to take from it. + +**B1 costs nothing measurable.** Trying the floor-anchored chain before the raw-version bootstrap +adds a per-entity version index on the compaction arm, but an entity carries a handful of editions, +and the floored fold measures the same as the unfloored one. + +**B2 costs a second fold, but only when it can change the answer.** `resolve` skips pass B when +nobody is banned *or* when nobody banned ever authored a Control edition — the overwhelmingly common +shape, since bans land on plain members who hold no role and write nothing. Those rows show no +regression. When a banned member *did* author editions — a banned staffer, exactly the case B2 exists +for — the fold costs ~2–3× more. That is the price of the fix and it is paid only by communities +under the attack. + +**Worth knowing, unrelated to this work:** Amethyst re-folds the whole buffer from scratch on every +Control Plane change, and `resolve` runs once per held epoch inside `controlFloorsLocked` plus once +in `fold`, so a refresh is already several folds. Armada memoizes the fold by +`(community, owner, floors, snapshot, edition ids)`; we do not. That is the real optimization here, +it predates these fixes, and it would also absorb the pass-B cost. Left alone deliberately — it is a +change to make on its own merits, with its own measurements. + --- ## What was NOT examined diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt index ff85147ea6..ffc637bd2c 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt @@ -170,7 +170,12 @@ data class AuthorityResolver private constructor( ownerPubKey: String, ): AuthorityResolver { val passA = resolveOnce(editions, ownerPubKey, bannedAuthors = emptySet()) + // Pass B costs a whole second fold, so skip it unless it could change something. Nobody + // banned, or nobody banned who ever wrote to the Control Plane — the overwhelmingly common + // shape, since most bans land on plain members who hold no role and author no editions — + // and pass B is provably identical to pass A. This is also what Armada's fold checks. if (passA.banned.isEmpty()) return passA + if (editions.none { it.author.lowercase() in passA.banned }) return passA return resolveOnce(editions, ownerPubKey, bannedAuthors = passA.banned) } From 9cc19c60ca6f9d83b548b741488a95b7003e6904 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 16:38:35 +0000 Subject: [PATCH 091/132] fix(concord): three defects found auditing this branch's own changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-review of the diff before merge. One of these is a real correctness bug in the B2 fix as shipped. The resolver stopped after two passes, which left the mask a pass resolved UNDER disagreeing with the banlist that pass produced — and the disagreement is not cosmetic. A moderator whose only ban came from an admin the owner banned concurrently is released by pass 2, correctly; but pass 2 had already dropped her editions, because she was on pass 1's list. The fold then reported her as a moderator in good standing whose promotions had silently vanished, and did so deterministically, so she never got them back. resolve() now iterates until the mask and the resulting banlist agree. The mask cannot simply be assumed to shrink, which is why this is bounded rather than proven monotone: masking an author can strip a THIRD member's role, which drops their rank to roleless, which lets a junior BAN holder who previously could not reach them ban them after all. The loop keeps its last pass if it does not settle within the cap — still better than the two-pass answer, and it always terminates. Real communities settle on the first or second pass, and the skip-if-no-banned-author guard means most never enter the loop at all. boundRecipients could exceed its own budget while reporting that it had capped at it, because the roster was added with filterTo before the budget loop ran. The roster now goes in whole deliberately — it is owner-rooted and cannot be padded from outside, and dropping an admin to make room for a stranger inverts the point — and the log reports what was actually kept and dropped. mintConcordInvite started requiring a session, which the owner's own invite button would not have on a cold start, since sessions are built asynchronously off the joined list. The owner is proven by the community id, so they are read off the entry; everyone else still needs the folded roster. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DrJhpFhhLjuDJQNkGvYMGj --- .../amethyst/model/AccountConcordActions.kt | 24 +++++++++---- docs/concord-soft-ban-audit.md | 7 ++-- .../concord/cord04Roles/AuthorityResolver.kt | 35 +++++++++++++++++-- .../cord04Roles/BannedStaffEscalationTest.kt | 32 +++++++++++++++++ 4 files changed, 86 insertions(+), 12 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt index 5b5c8fe0db..fc812f1bb8 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt @@ -189,8 +189,13 @@ class AccountConcordActions( // Note the bit is not otherwise enforced anywhere. The fold gates the INVITE_* Control // entities on CREATE_INVITE, but a link's bundle is a standalone kind-33301 published // OUTSIDE the Control Plane, so no fold ever sees it. This check is the only one there is. - val session = account.concordSessions.sessionFor(communityId) ?: return null - if (!isAuthorizedFor(session, ConcordPermissions.CREATE_INVITE)) return null + // The owner is proven by the community id (CORD-02), so they are read off the entry and can + // mint before the session exists — the session is built asynchronously off the joined list, + // and requiring it here would have made the owner's own invite button fail on a cold start. + // Everyone else needs the folded roster, so no session means no invite. + val session = account.concordSessions.sessionFor(communityId) + val amOwner = entry.owner.equals(account.signer.pubKey, ignoreCase = true) + if (!amOwner && (session == null || !isAuthorizedFor(session, ConcordPermissions.CREATE_INVITE))) return null val invite = ConcordActions.inviteFor( communityIdHex = entry.id, @@ -885,16 +890,23 @@ class AccountConcordActions( ): List { if (candidates.size <= MAX_REFOUNDING_RECIPIENTS) return candidates.toList() + // The roster goes in whole even if it alone exceeds the budget: it is owner-rooted, so it + // cannot be padded from outside, and dropping an admin to make room for a stranger inverts + // the point of the cap. val vouched = authority.roleHolders() + authority.staffMembers() - val kept = LinkedHashSet(MAX_REFOUNDING_RECIPIENTS) + val kept = LinkedHashSet() candidates.filterTo(kept) { it in vouched } for (candidate in candidates) { if (kept.size >= MAX_REFOUNDING_RECIPIENTS) break kept.add(candidate) } - Log.w("Concord") { - "Refounding recipient set capped at $MAX_REFOUNDING_RECIPIENTS of ${candidates.size}: " + - "${candidates.size - kept.size} member(s) will be stranded on the prior epoch" + val dropped = candidates.size - kept.size + if (dropped > 0) { + Log.w("Concord") { + "Refounding recipient set trimmed to ${kept.size} of ${candidates.size} " + + "(budget $MAX_REFOUNDING_RECIPIENTS, roster kept whole): $dropped member(s) will be " + + "stranded on the prior epoch" + } } return kept.toList() } diff --git a/docs/concord-soft-ban-audit.md b/docs/concord-soft-ban-audit.md index 8e766c8b46..43848a1302 100644 --- a/docs/concord-soft-ban-audit.md +++ b/docs/concord-soft-ban-audit.md @@ -528,12 +528,13 @@ Two things to take from it. adds a per-entity version index on the compaction arm, but an entity carries a handful of editions, and the floored fold measures the same as the unfloored one. -**B2 costs a second fold, but only when it can change the answer.** `resolve` skips pass B when +**B2 costs a further fold, but only when it can change the answer.** `resolve` skips pass B when nobody is banned *or* when nobody banned ever authored a Control edition — the overwhelmingly common shape, since bans land on plain members who hold no role and write nothing. Those rows show no regression. When a banned member *did* author editions — a banned staffer, exactly the case B2 exists -for — the fold costs ~2–3× more. That is the price of the fix and it is paid only by communities -under the attack. +for — the fold costs ~2–3× more, and one more pass again in the rare case where a banned member had +themselves authored a ban. That is the price of the fix and it is paid only by communities under the +attack. **Worth knowing, unrelated to this work:** Amethyst re-folds the whole buffer from scratch on every Control Plane change, and `resolve` runs once per held epoch inside `controlFloorsLocked` plus once diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt index ffc637bd2c..126eaade9b 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt @@ -135,6 +135,14 @@ data class AuthorityResolver private constructor( /** The owner's rank — supreme and unremovable. No Role may claim it. */ const val OWNER_RANK = 0L + /** + * How many times [resolve] will re-fold chasing a stable banlist. Real communities settle on + * the first or second — the mask only moves when a banned member authored a *ban*, and it + * stops moving as soon as those are gone. The cap is a termination backstop for an + * adversarial edition set, not a tuning knob. + */ + private const val MAX_BAN_RESOLUTION_PASSES = 4 + /** * The owner-rooted authority state of a community, with the banlist honored **against the * Control Plane itself** (CORD-04 §4: a reader "drops every event from a banned npub — @@ -170,13 +178,34 @@ data class AuthorityResolver private constructor( ownerPubKey: String, ): AuthorityResolver { val passA = resolveOnce(editions, ownerPubKey, bannedAuthors = emptySet()) - // Pass B costs a whole second fold, so skip it unless it could change something. Nobody + // A further pass costs a whole fold, so skip it unless it could change something. Nobody // banned, or nobody banned who ever wrote to the Control Plane — the overwhelmingly common // shape, since most bans land on plain members who hold no role and author no editions — - // and pass B is provably identical to pass A. This is also what Armada's fold checks. + // and the next pass is provably identical to this one. Armada's fold checks the same. if (passA.banned.isEmpty()) return passA if (editions.none { it.author.lowercase() in passA.banned }) return passA - return resolveOnce(editions, ownerPubKey, bannedAuthors = passA.banned) + + // Iterate to a fixpoint where the mask a pass was resolved UNDER equals the banlist that + // pass produced. Stopping at two passes leaves those two disagreeing, and the disagreement + // is not cosmetic: a moderator whose only ban came from an admin the owner banned + // concurrently is released by pass 2 — correctly — but pass 2 dropped her editions too, + // because she was on pass 1's list. The fold then reports her as a moderator in good + // standing whose promotions have silently vanished, and it does so deterministically, so + // she never gets them back. + // + // The mask cannot simply be assumed to shrink: masking an author can strip a THIRD + // member's role, dropping their rank to "roleless", which lets a junior BAN holder who + // could not previously reach them ban them after all. So this is bounded rather than + // proven monotone, and it keeps the last pass it computed if it somehow does not settle — + // still strictly better than the two-pass answer, and it always terminates. + var mask = passA.banned + var result = passA + repeat(MAX_BAN_RESOLUTION_PASSES) { + result = resolveOnce(editions, ownerPubKey, bannedAuthors = mask) + if (result.banned == mask) return result + mask = result.banned + } + return result } /** diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/BannedStaffEscalationTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/BannedStaffEscalationTest.kt index 5b6c0fc64d..3bed1c8575 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/BannedStaffEscalationTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/BannedStaffEscalationTest.kt @@ -362,4 +362,36 @@ class BannedStaffEscalationTest { assertEquals(5, r.rank(bob), "bob is untouched") assertEquals(5, r.rank(carol), "and the owner's grant of carol stands") } + + @Test + fun aMemberReleasedByTheSecondPassKeepsTheEditionsTheyAuthored() { + // The mask a pass resolves UNDER has to equal the banlist that pass produces, or the fold + // reports a state that contradicts itself. Concretely: the rogue admin bans a moderator while + // the owner concurrently bans the rogue. The moderator is correctly released — the only ban on + // her came from someone who turned out to be banned — but a fold that stops after two passes + // has already dropped her editions, because she was on the FIRST pass's list. She then reads + // as a moderator in good standing whose promotions silently vanished, deterministically and + // forever. resolve() iterates until the two agree. + val juniorRole = "23".repeat(32) + val seniorRole = "24".repeat(32) + val editions = + community() + + // the baseline Mod role carries no MANAGE_ROLES, so give bob one that can grant + role(seniorRole, """{"name":"Senior","position":5,"permissions":"95"}""") + + grant(bobGrantEntity, bob, listOf(seniorRole), author = owner, version = 1, prev = bobGrantV0.hash) + + role(juniorRole, """{"name":"Junior","position":9,"permissions":"8"}""") + + // bob promotes carol himself, while in good standing + grant("39".repeat(32), carol, listOf(juniorRole), author = bob) + + // the rogue admin bans bob... + banlist(alice, 0, null, bob) + + // ...while the owner concurrently bans the rogue, never naming bob + ownerBansAlice + + val r = AuthorityResolver.resolve(editions, owner) + + assertTrue(r.isBanned(alice), "the owner's ban of the rogue stands") + assertFalse(r.isBanned(bob), "and the rogue's ban of the moderator falls with them") + assertEquals(5, r.rank(bob), "the released moderator keeps their own role") + assertEquals(9, r.rank(carol), "and the promotion they authored survives with them") + } } From d15ee295d7bc8d1ce2932566fbd1c1f081a79253 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 16:45:12 +0000 Subject: [PATCH 092/132] fix: stop reposts from being bundled with the always-on service notification Android 16 force-groups notifications the app leaves loose. A group child whose summary is missing counts as ungrouped (GroupHelper. isGroupChildWithoutSummary), and with config_autoGroupAtCount at 2 it takes one such child plus one other ungrouped notification in the same shade section to form an aggregate bundle. We produced both halves. sendGroupSummary only posted the summary once two children of a group were live, so a lone repost or reaction sat there as a summary-less child; and the always-on relay service posts an ongoing, IMPORTANCE_LOW notification, which shares the Silent section with those two IMPORTANCE_LOW kinds. The system's aggregate summary inherits FLAG_ONGOING_EVENT from any child that has it, so the resulting bundle could not be swiped away without dismissing the service notification. Post our own group summary from the first child on, which keeps the group app-owned and off the system's list. It changes nothing visually: the shade hides any group with fewer than two children and renders the child on its own. That promotion is also why the summary now needs cleaning up: a promoted child is no longer "the only child in its group", so dismissing it leaves the summary behind, and a childless summary is both shown standalone by the shade and force-grouped by the system. Children now carry a delete intent that prunes it, the mark-read and inline-reply paths prune through cancelAndPrune, and the pruning ignores an id cancelled moments ago (cancel and notify are asynchronous, so activeNotifications can still list it) and leaves the platform's own aggregate summaries alone. Summaries also gain GROUP_ALERT_CHILDREN so they stay silent now that they go up alongside the first child. --- .../notifications/NotificationRelayService.kt | 10 ++ .../NotificationReplyReceiver.kt | 12 ++- .../notifications/NotificationUtils.kt | 102 ++++++++++++++++-- 3 files changed, 114 insertions(+), 10 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationRelayService.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationRelayService.kt index 3411379e4c..d90c812658 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationRelayService.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationRelayService.kt @@ -373,6 +373,16 @@ class NotificationRelayService : Service() { if (fresh.isNotEmpty()) lastBreakdown = fresh val breakdown = fresh.ifEmpty { lastBreakdown }.takeIf { it.isNotEmpty() } + // Deliberately left ungrouped. This notification is ongoing and IMPORTANCE_LOW, so it + // sits in the shade's Silent section next to the low-importance content kinds + // (reactions, reposts) — and Android 16 sweeps everything ungrouped in a section into + // one aggregate bundle whose summary inherits FLAG_ONGOING_EVENT from any child that + // has it, making the whole bundle un-swipeable. Giving this one a group of its own + // would not help: a group with a summary but no children, or a child with no summary, + // is force-grouped just the same. What keeps content notifications out of that bundle + // is that they always post their own group summary (see NotificationUtils), which + // leaves this the only ungrouped silent notification we post — one is below the + // threshold, so no bundle is formed and nothing gets stapled to it. return NotificationCompat .Builder(this, CHANNEL_ID) .setContentTitle(getString(R.string.always_on_notif_title)) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationReplyReceiver.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationReplyReceiver.kt index ddb16c72cb..635ce0a930 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationReplyReceiver.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationReplyReceiver.kt @@ -30,6 +30,8 @@ import com.vitorpamplona.amethyst.Amethyst import com.vitorpamplona.amethyst.LocalPreferences import com.vitorpamplona.amethyst.model.LocalCache import com.vitorpamplona.amethyst.model.accountsCache.AccountCacheState +import com.vitorpamplona.amethyst.service.notifications.NotificationUtils.cancelAndPrune +import com.vitorpamplona.amethyst.service.notifications.NotificationUtils.cancelChildlessGroupSummaries import com.vitorpamplona.amethyst.ui.actions.NewMessageTagger import com.vitorpamplona.quartz.nip01Core.hints.EventHintBundle import com.vitorpamplona.quartz.nip01Core.tags.people.PTag @@ -60,7 +62,13 @@ class NotificationReplyReceiver : BroadcastReceiver() { when (intent.action) { NotificationUtils.MARK_READ_ACTION -> { - notificationManager.cancel(notificationId) + notificationManager.cancelAndPrune(notificationId) + } + + // The user swiped the notification away. It is already gone; all that is left + // is to take its group summary with it when it was the last child. + NotificationUtils.DISMISS_ACTION -> { + notificationManager.cancelChildlessGroupSummaries(alreadyGone = notificationId) } NotificationUtils.REPLY_ACTION -> { @@ -138,7 +146,7 @@ class NotificationReplyReceiver : BroadcastReceiver() { try { block() - notificationManager.cancel(notificationId) + notificationManager.cancelAndPrune(notificationId) } catch (e: Exception) { if (e is CancellationException) throw e Log.e("NotificationReply") { "Failed to send reply: ${e.message}" } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationUtils.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationUtils.kt index a1879a70f4..d7191cf468 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationUtils.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationUtils.kt @@ -70,6 +70,7 @@ object NotificationUtils { const val PUBLIC_REPLY_ACTION = "com.vitorpamplona.amethyst.PUBLIC_REPLY_ACTION" const val MARMOT_REPLY_ACTION = "com.vitorpamplona.amethyst.MARMOT_REPLY_ACTION" const val MARK_READ_ACTION = "com.vitorpamplona.amethyst.MARK_READ_ACTION" + const val DISMISS_ACTION = "com.vitorpamplona.amethyst.DISMISS_ACTION" const val KEY_REPLY_TEXT = "key_reply_text" const val KEY_NOTIFICATION_ID = "key_notification_id" const val KEY_ACCOUNT_NPUB = "key_account_npub" @@ -82,6 +83,14 @@ object NotificationUtils { const val REPLY_GROUP_KEY_PREFIX = "com.vitorpamplona.amethyst.REPLY_NOTIFICATION" private const val REPLY_SUMMARY_ID_BASE = 0x50000 + /** + * Every group key this object posts under starts with this. Used to tell our own + * summaries apart from the ones the system creates when it force-groups us (those + * live under `userId|pkg|g:Aggregate_…`), so the cleanup below never fights the + * platform over a bundle it owns. + */ + private const val OWN_GROUP_PREFIX = "com.vitorpamplona.amethyst." + // Event ids the user has just read/dismissed in-app. The enrichment path // re-posts a notification as metadata arrives; without this guard a // notification the user already dismissed would be resurrected seconds later @@ -218,6 +227,7 @@ object NotificationUtils { .setPriority(category.priority()) .setCategory(NotificationCompat.CATEGORY_SOCIAL) .setGroup(groupKey) + .setDeleteIntent(dismissIntent(applicationContext, notId)) .setAutoCancel(true) .setOnlyAlertOnce(true) .setWhen(time * 1000) @@ -335,6 +345,7 @@ object NotificationUtils { .setPriority(category.priority()) .setCategory(NotificationCompat.CATEGORY_MESSAGE) .setGroup(groupKey) + .setDeleteIntent(dismissIntent(applicationContext, notId)) .setAutoCancel(true) .setOnlyAlertOnce(true) .setWhen(time * 1000) @@ -370,6 +381,36 @@ object NotificationUtils { ) } + /** + * Fires when the user swipes this notification away (or hits "Clear all"), so the + * group summary can follow its last child out. + * + * We can't rely on the shade to take the summary with it: SystemUI hides a group + * with a single child and renders that child at the top level + * (`ShadeListBuilder.MIN_CHILDREN_FOR_GROUP`), and once promoted the child no + * longer counts as "the only child in its group", so dismissing it leaves our + * summary behind. A childless summary is not harmless — SystemUI promotes it into + * the shade on its own, and the system force-groups it + * (`GroupHelper.isGroupSummaryWithoutChildren`) into the same aggregate bundle we + * post summaries to stay out of. + */ + private fun dismissIntent( + applicationContext: Context, + notId: Int, + ): PendingIntent { + val intent = + Intent(applicationContext, NotificationReplyReceiver::class.java).apply { + action = DISMISS_ACTION + putExtra(KEY_NOTIFICATION_ID, notId) + } + return PendingIntent.getBroadcast( + applicationContext, + notId + 2, + intent, + PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_UPDATE_CURRENT, + ) + } + private fun replyRemoteInput(applicationContext: Context): RemoteInput = RemoteInput .Builder(KEY_REPLY_TEXT) @@ -549,16 +590,32 @@ object NotificationUtils { // Group summaries, dedup, dismissal // --------------------------------------------------------------------- + /** + * Posts (or refreshes) our own summary for [groupKey]. + * + * The summary goes up with the **first** child, not once a second one shows up. + * Android 16 counts a group child whose summary is missing as ungrouped + * (`GroupHelper.isGroupChildWithoutSummary`) and force-groups it into the + * package's per-section aggregate bundle, next to every other ungrouped + * notification in the same shade section. The bar is low: `config_autoGroupAtCount` + * is 2, so a single summary-less child plus one other ungrouped notification is a + * bundle. The always-on relay service is exactly that other notification — ongoing + * and IMPORTANCE_LOW, it shares the Silent section with our two IMPORTANCE_LOW + * kinds (reactions and reposts), so one lone repost would end up bundled with it. + * The bundle then refuses to swipe away, because the system's aggregate summary + * inherits FLAG_ONGOING_EVENT from any child carrying it — and the service + * notification always does. + * + * Providing the summary from the start keeps the group ours and the system leaves + * it alone. It costs nothing visually: the shade hides any group with fewer than + * two children and shows the child on its own. + */ private fun NotificationManager.sendGroupSummary( category: NotificationCategory, groupKey: String, summaryId: Int, applicationContext: Context, ) { - val activeCount = activeNotifications.count { it.notification.group == groupKey && it.id != summaryId } - - if (activeCount < 2) return - val summaryBuilder = NotificationCompat .Builder(applicationContext, category.channelId(applicationContext)) @@ -566,6 +623,9 @@ object NotificationUtils { .setColor(category.color) .setGroup(groupKey) .setGroupSummary(true) + // The children do the alerting. Without this the summary would buzz on + // its own the moment it starts going up alongside the first child. + .setGroupAlertBehavior(NotificationCompat.GROUP_ALERT_CHILDREN) .setAutoCancel(true) .setOnlyAlertOnce(true) .setStyle( @@ -604,16 +664,42 @@ object NotificationUtils { // items), so bail out before touching anything when nothing is posted for it. if (activeNotifications.none { it.id == notId }) return - cancel(notId) - cancelChildlessGroupSummaries() + cancelAndPrune(notId) } - private fun NotificationManager.cancelChildlessGroupSummaries() { + /** + * Cancels [notId] and drops the group summary it leaves behind, if it was the last + * child. Use this instead of a bare [NotificationManager.cancel] for anything we + * posted through [postStandard] / [postConversation] — every one of those is a + * group child with a summary above it. + */ + fun NotificationManager.cancelAndPrune(notId: Int) { + cancel(notId) + cancelChildlessGroupSummaries(alreadyGone = notId) + } + + /** + * Drops our summaries that no longer have any children. + * + * [alreadyGone] is the id of a notification cancelled moments ago: both + * [NotificationManager.cancel] and [NotificationManager.notify] are asynchronous, + * so [NotificationManager.activeNotifications] can still be listing it and would + * otherwise keep its summary alive forever. + * + * Only summaries under [OWN_GROUP_PREFIX] are touched. The system's own aggregate + * summaries also carry FLAG_GROUP_SUMMARY and show up in this list; cancelling one + * only makes the platform rebuild it. + */ + fun NotificationManager.cancelChildlessGroupSummaries(alreadyGone: Int? = null) { val active: Array = activeNotifications for (summary in active) { if (summary.notification.flags and Notification.FLAG_GROUP_SUMMARY == 0) continue val group = summary.notification.group ?: continue - val hasChildren = active.any { it.id != summary.id && it.notification.group == group } + if (!group.startsWith(OWN_GROUP_PREFIX)) continue + val hasChildren = + active.any { + it.id != summary.id && it.id != alreadyGone && it.notification.group == group + } if (!hasChildren) cancel(summary.id) } } From f1241e891bb46ae9bbcdbfb48a3a1a23f653f894 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 17:11:45 +0000 Subject: [PATCH 093/132] fix(quartz): compaction must carry the authority-gated head, not the chain head MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Findings from an independent review of this branch (a different model, per CONTRIBUTING-WITH-AI.md). Two were real, and the first is a regression this branch introduced. compactControlPlane picked its per-entity head with a bare structural chain walk, which is worse than the raw max-version it replaced. With no floor, foldEntity anchors at the lowest-version edition carrying no `prev` — and after a PRIOR compaction the genuine head's `prev` dangles into a trimmed epoch by design. So a forged `version = 1, prev = null` decoy outranks a real v50→v52 chain, and because nothing in this path checks a signature it became the entity's entire carried-forward state. A forged empty banlist would have erased every ban at the next Refounding. Reproduced, then fixed by selecting the owner-rooted authority-gated head — the same edition ConcordCommunityState.fold would seat, so the new epoch starts where the old one left off, and an unprivileged author cannot influence the choice at all. recoverStrandedConcordCommunities derived its new ban gate with `?.isBanned(..) == true`, which reads "not banned" when the session does not exist yet or its first fold has not landed. The sweep runs on the revision tick, so a banned member's own client would have hit that window on cold start and recovered itself — the exact bypass the gate exists to stop. It now fails closed and retries on the next sweep. Also from the review: resolve() now warns when the ban fixpoint exhausts its pass cap without settling, instead of silently returning a roster folded under a mask that no longer matches its banlist; and banGate stops lowercasing the same author three times. Two review findings are accepted rather than fixed, and recorded on the PR: the anchor tie-break picks the lowest rumor id before testing whether that candidate connects (pre-existing, and changing it is consensus-affecting), and non-owner moderators now need a resolved roster before a verb succeeds, which is the intended fail-closed trade. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DrJhpFhhLjuDJQNkGvYMGj --- .../amethyst/model/AccountConcordActions.kt | 16 +++- .../commons/actions/ConcordActions.kt | 2 + .../commons/actions/ConcordActionsTest.kt | 1 + .../model/concord/ConcordRollbackFloorTest.kt | 6 +- .../concord/cord04Roles/AuthorityResolver.kt | 18 ++++- .../concord/cord06Rekey/ConcordRefounding.kt | 38 ++++++---- .../cord06Rekey/ConcordRefoundingTest.kt | 76 +++++++++++++++++++ .../cord06Rekey/ControlRootRotationTest.kt | 2 + 8 files changed, 138 insertions(+), 21 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt index fc812f1bb8..f659e38564 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt @@ -852,6 +852,7 @@ class AccountConcordActions( recipientsXOnly = recipients, staffXOnly = staff, createdAt = TimeUtils.now(), + ownerPubKey = entry.owner, ) // 4. Publish the compacted Control Plane (the new epoch's state) then the rekey blobs @@ -1123,13 +1124,24 @@ class AccountConcordActions( // walks them straight back into the epoch they were rotated out of — see A2 in // docs/concord-soft-ban-audit.md. Read off the epoch we are LEAVING, which is the last // one whose Control Plane we can still fold. - val bannedHere = + // + // Fails CLOSED. `?.isBanned(..) == true` reads "not banned" for a session that does not + // exist yet or whose first fold has not landed, and this sweep runs on the revision tick + // — so a banned member's own client would have hit that window on cold start and + // recovered itself, which is precisely the bypass this gate exists to stop. No verdict + // means no recovery; the next sweep retries once the roster is known. + val authority = account.concordSessions .sessionFor(entry.id) ?.state ?.value ?.authority - ?.isBanned(account.signer.pubKey) == true + if (authority == null) { + Log.i("Concord") { "Stranded-recovery check deferred for ${entry.id}: control plane not folded yet" } + lastConcordRecoveryCheck.remove(entry.id) + continue + } + val bannedHere = authority.isBanned(account.signer.pubKey) val merged = ConcordActions.recoverStranded(entry, bundle, bannedHere) ?: continue if (!adoptedConcordRotations.add("${entry.id}:${merged.rootEpoch}")) continue Log.i("Concord", "Stranded recovery: ${entry.id} ${entry.rootEpoch} -> ${merged.rootEpoch}") diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt index dff1143c45..e70fe18364 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt @@ -545,6 +545,7 @@ object ConcordActions { recipientsXOnly: List, staffXOnly: Set, createdAt: Long, + ownerPubKey: HexKey, ): RefoundingBuild = ConcordRefounding.build( rotatorSigner = rotatorSigner, @@ -558,6 +559,7 @@ object ConcordActions { recipientsXOnly = recipientsXOnly, staffXOnly = staffXOnly, createdAt = createdAt, + ownerPubKey = ownerPubKey, ) /** diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActionsTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActionsTest.kt index a5d8ff772d..82169a7708 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActionsTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActionsTest.kt @@ -123,6 +123,7 @@ class ConcordActionsTest { // Only the owner is staff, so only the owner's blob carries the secret. staffXOnly = setOf(owner.pubKey), createdAt = 5L, + ownerPubKey = owner.pubKey, ) val baseRekey = ConcordActions.nextBaseRekeyPlane(community.communityRoot, community.communityId, community.rootEpoch) diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordRollbackFloorTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordRollbackFloorTest.kt index d3c1a50a75..ae06a84891 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordRollbackFloorTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordRollbackFloorTest.kt @@ -78,7 +78,7 @@ class ConcordRollbackFloorTest { // new epoch's plane is split and addressed by the derived signer, not the root. val newControlRoot = ByteArray(32) { 0x44 } val newControl = ControlPlaneKeys.forStaff(newRoot, community.communityId, newEpoch, newControlRoot) - val rolledBack = ConcordRefounding.compactControlPlane(community.genesisWraps, community.controlPlane, newControl) + val rolledBack = ConcordRefounding.compactControlPlane(community.genesisWraps, community.controlPlane, newControl, community.ownerPubKey) val entry = ConcordCommunityListEntry( @@ -145,7 +145,7 @@ class ConcordRollbackFloorTest { val newControlRoot = ByteArray(32) { 0x44 } val newControl = ControlPlaneKeys.forStaff(newRoot, community.communityId, newEpoch, newControlRoot) // Honest: compacted from the FULL prior plane, so each entity's head (metadata v1) survives. - val honest = ConcordRefounding.compactControlPlane(epoch0Wraps, community.controlPlane, newControl) + val honest = ConcordRefounding.compactControlPlane(epoch0Wraps, community.controlPlane, newControl, community.ownerPubKey) val entry = ConcordCommunityListEntry( @@ -186,7 +186,7 @@ class ConcordRollbackFloorTest { // new epoch's plane is split and addressed by the derived signer, not the root. val newControlRoot = ByteArray(32) { 0x44 } val newControl = ControlPlaneKeys.forStaff(newRoot, community.communityId, newEpoch, newControlRoot) - val compacted = ConcordRefounding.compactControlPlane(community.genesisWraps, community.controlPlane, newControl) + val compacted = ConcordRefounding.compactControlPlane(community.genesisWraps, community.controlPlane, newControl, community.ownerPubKey) val entry = ConcordCommunityListEntry( diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt index 126eaade9b..5677e56244 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord04Roles/AuthorityResolver.kt @@ -21,6 +21,7 @@ package com.vitorpamplona.quartz.concord.cord04Roles import com.vitorpamplona.quartz.nip01Core.core.toHexKey +import com.vitorpamplona.quartz.utils.Log /** * Resolves the owner-rooted authority state of a Concord community from its @@ -132,6 +133,8 @@ data class AuthorityResolver private constructor( } companion object { + private const val TAG = "ConcordAuthorityResolver" + /** The owner's rank — supreme and unremovable. No Role may claim it. */ const val OWNER_RANK = 0L @@ -205,6 +208,14 @@ data class AuthorityResolver private constructor( if (result.banned == mask) return result mask = result.banned } + // Exhausted the cap without settling. The returned roster was folded under a mask that is + // no longer the banlist beside it, so this is reported rather than swallowed — the same + // reasoning as EditionFold.LOG_GAP: an unsettled fold is either an adversarial edition set + // or a rule of ours that does not converge, and both are things a reader wants to know. + Log.w(TAG) { + "Banlist resolution did not settle in $MAX_BAN_RESOLUTION_PASSES passes for owner $ownerPubKey " + + "(${editions.size} editions, ${result.banned.size} banned): keeping the last pass" + } return result } @@ -350,9 +361,10 @@ data class AuthorityResolver private constructor( // a concurrent ban is never lost, while an on-chain unban still takes effect. val allBanlist = editions.filter { it.entityKind == ControlEntityKind.BANLIST } - fun banGate(e: ControlEdition): Boolean = - e.author.lowercase() == ownerLower || - (e.author.lowercase() !in bannedAuthors && effectivePermissionsOf(e.author.lowercase()).has(ConcordPermissions.BAN)) + fun banGate(e: ControlEdition): Boolean { + val author = e.author.lowercase() + return author == ownerLower || (author !in bannedAuthors && effectivePermissionsOf(author).has(ConcordPermissions.BAN)) + } val authorizedBanlist = allBanlist.filter(::banGate) // CORD-04 §3's rank rule binds "every action", and it names banning as its example ("an diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefounding.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefounding.kt index 7dc80de9cb..74fb73b594 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefounding.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefounding.kt @@ -20,8 +20,8 @@ */ package com.vitorpamplona.quartz.concord.cord06Rekey +import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityState import com.vitorpamplona.quartz.concord.cord04Roles.ControlEdition -import com.vitorpamplona.quartz.concord.cord04Roles.EditionFold import com.vitorpamplona.quartz.concord.crypto.ConcordKeyDerivation import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys import com.vitorpamplona.quartz.concord.crypto.GroupKey @@ -124,11 +124,12 @@ object ConcordRefounding { recipientsXOnly: List, staffXOnly: Set, createdAt: Long, + ownerPubKey: HexKey, ): RefoundingBuild { val newEpoch = rootEpoch + 1 val newControlKeys = ControlPlaneKeys.forStaff(newRoot, communityId, newEpoch, newControlRoot) - val controlWraps = compactControlPlane(priorControlWraps, priorControlKeys, newControlKeys) + val controlWraps = compactControlPlane(priorControlWraps, priorControlKeys, newControlKeys, ownerPubKey) val baseRekeyKey = ConcordKeyDerivation.baseRekeyAddress(priorRoot, communityId, newEpoch) val prevCommit = ConcordKeyDerivation.epochKeyCommitment(rootEpoch, priorRoot).toHexKey() @@ -167,6 +168,7 @@ object ConcordRefounding { priorWraps: List, priorControlKeys: ControlPlaneKeys, newControlKeys: ControlPlaneKeys, + ownerPubKey: HexKey, ): List { // entity coordinate -> every edition we can open, paired with its verified seal. val byCoordinate = HashMap>>() @@ -177,17 +179,27 @@ object ConcordRefounding { byCoordinate.getOrPut(coord) { ArrayList() }.add(edition to opened.seal) } - // The head is the CHAIN head, not the highest version. Picking by raw version made an honest - // rotator the delivery mechanism for a disconnected stray: an edition minted at an arbitrary - // version never joins the chain, but it won this comparison and was then re-wrapped into the - // new epoch as that entity's whole history — where a fresh joiner, holding no floor, anchors - // on it as their baseline. See B1 in `docs/concord-soft-ban-audit.md`. foldEntity walks from - // genesis and keeps the fresh-joiner fallback for a head whose own `prev` dangles into an - // epoch this rotator no longer holds, which is the ordinary shape after a prior compaction. - val out = ArrayList(byCoordinate.size) - for ((_, entries) in byCoordinate) { - val head = EditionFold.foldEntity(entries.map { it.first }) ?: continue - val seal = entries.firstOrNull { it.first.rumorId == head.rumorId }?.second ?: continue + // The head to carry forward is the one every READER honors — the authority-gated head — not + // the highest version and not the bare structural chain head. + // + // Raw highest version made an honest rotator the delivery mechanism for a disconnected stray: + // an edition minted at an arbitrary version never joins the chain, but it won that comparison + // and was re-wrapped into the new epoch as the entity's whole history (B1 in + // `docs/concord-soft-ban-audit.md`). The bare chain walk is *worse*, and this is the trap: + // with no floor it anchors at the lowest-version edition carrying no `prev`, and after a prior + // compaction the real head's `prev` dangles by design — so a forged `version = 1, prev = null` + // decoy outranks a genuine v50→v52 chain and, because nothing here checks signatures, becomes + // the entity's entire carried-forward state. A forged empty banlist would erase every ban. + // + // Gating on the owner-rooted roster is the only selection that cannot be gamed by an + // unprivileged author, and it is exactly what ConcordCommunityState.fold would seat, so the + // compacted epoch starts where the previous one left off. + val editions = byCoordinate.values.flatten() + val honored = ConcordCommunityState.authorizedHeads(editions.map { it.first }, ownerPubKey) + val out = ArrayList(honored.size) + for ((_, floor) in honored) { + val head = floor.known ?: continue + val seal = editions.firstOrNull { it.first.rumorId == head.rumorId }?.second ?: continue out.add(ConcordStreamEnvelope.wrapSeal(seal, newControlKeys, createdAt = seal.createdAt)) } return out diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefoundingTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefoundingTest.kt index 06778c4c71..dbc6e9c81d 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefoundingTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ConcordRefoundingTest.kt @@ -74,6 +74,7 @@ class ConcordRefoundingTest { recipientsXOnly = listOf(alice.pubKey, bob.pubKey), staffXOnly = setOf(owner.pubKey), createdAt = now, + ownerPubKey = owner.pubKey, ) assertEquals(community.rootEpoch + 1, build.newEpoch) @@ -113,6 +114,7 @@ class ConcordRefoundingTest { recipientsXOnly = listOf(alice.pubKey), staffXOnly = setOf(owner.pubKey), createdAt = now, + ownerPubKey = owner.pubKey, ) val newControl = build.newControlKeys @@ -184,6 +186,7 @@ class ConcordRefoundingTest { recipientsXOnly = listOf(alice.pubKey), staffXOnly = setOf(owner.pubKey), createdAt = now, + ownerPubKey = owner.pubKey, ) val newControl = build.newControlKeys @@ -223,6 +226,7 @@ class ConcordRefoundingTest { recipientsXOnly = listOf(alice.pubKey), staffXOnly = setOf(owner.pubKey), createdAt = now, + ownerPubKey = owner.pubKey, ) val baseRekeyKey = ConcordKeyDerivation.baseRekeyAddress(community.communityRoot, community.communityId, build.newEpoch) @@ -230,4 +234,76 @@ class ConcordRefoundingTest { val wrongRoot = ByteArray(32) { 0x11 } assertNull(ConcordRefounding.findNewRoot(build.rekeyWraps, baseRekeyKey, alice, community.communityId, wrongRoot, community.rootEpoch)) } + + @Test + fun compactionRefusesAForgedGenesisAndCarriesTheAuthorizedHead() = + runTest { + // A compaction re-wraps ONE edition per entity and nothing downstream re-checks the + // choice, so how that edition is picked is a security decision, not a detail. + // + // Raw highest-version lets a stray at an arbitrary version through. But the bare + // structural chain walk is worse: with no floor it anchors at the lowest-version edition + // carrying no `prev`, and after a PRIOR compaction the real head's `prev` dangles into a + // trimmed epoch by design — so a forged `version = 1, prev = null` decoy outranks a + // genuine v50→v52 chain, and becomes the entity's entire carried-forward state. A forged + // empty banlist would erase every ban that way. Only the owner-rooted gate is safe. + val community = ConcordCommunityFactory.create(owner, "Test", now) + val communityId = community.communityId + val control = community.controlPlane + + // The metadata entity, already compacted once: its head chains from an epoch we no longer hold. + val danglingPrev = ByteArray(32) { 0x7F } + val realHead = + ConcordStreamEnvelope.wrap( + ControlEditionBuilder.rumor( + owner.pubKey, + ControlEntityKind.METADATA, + communityId, + 50, + danglingPrev, + ConcordJson.instance.encodeToString(MetadataEntity.serializer(), MetadataEntity(name = "Real")), + now, + null, + ), + control, + owner, + encrypted = false, + createdAt = now, + ) + + // carol holds nothing at all and mints a genesis-shaped decoy at version 1. + val forged = + ConcordStreamEnvelope.wrap( + ControlEditionBuilder.rumor( + carol.pubKey, + ControlEntityKind.METADATA, + communityId, + 1, + null, + ConcordJson.instance.encodeToString(MetadataEntity.serializer(), MetadataEntity(name = "PWNED")), + now, + null, + ), + control, + carol, + encrypted = false, + createdAt = now, + ) + + val newEpoch = community.rootEpoch + 1 + val newControl = + com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys + .forStaff(newRoot, communityId, newEpoch, newControlRoot) + val compacted = ConcordRefounding.compactControlPlane(listOf(realHead, forged), control, newControl, owner.pubKey) + + val carried = + compacted + .mapNotNull { ConcordStreamEnvelope.openOrNull(it, newControl) } + .mapNotNull { ControlEdition.fromRumor(it.rumor) } + .filter { it.entityKind == ControlEntityKind.METADATA } + + assertEquals(1, carried.size, "one metadata edition carried forward") + assertEquals(50, carried.single().version, "the owner's real head, not the forged genesis") + assertEquals("Real", ConcordJson.decodeOrNull(carried.single().content)?.name) + } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ControlRootRotationTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ControlRootRotationTest.kt index bcd5eec1e5..7a4440e1a0 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ControlRootRotationTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord06Rekey/ControlRootRotationTest.kt @@ -109,6 +109,7 @@ class ControlRootRotationTest { recipientsXOnly = listOf(owner.pubKey, moderator.pubKey, member.pubKey), staffXOnly = setOf(owner.pubKey, moderator.pubKey), createdAt = now, + ownerPubKey = owner.pubKey, ) val baseRekey = ConcordKeyDerivation.baseRekeyAddress(community.communityRoot, community.communityId, build.newEpoch) @@ -149,6 +150,7 @@ class ControlRootRotationTest { recipientsXOnly = listOf(owner.pubKey, member.pubKey), staffXOnly = setOf(owner.pubKey), createdAt = now, + ownerPubKey = owner.pubKey, ) // The rotator's own view writes; a member's view of the same epoch only reads. From 9bfec38697533cd1e02183227d24be7dd2fb5a24 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 17:24:46 +0000 Subject: [PATCH 094/132] fix: bring the embed keyboard back on a tap in an already-focused field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Returning to a warm embedded tab left the page's field focused but with no way to get the keyboard back: the surface just moves off-screen, so the page never fires another focus event, and `ime.focus` was the host's ONLY keyboard-raising signal. A tap on the still-focused field only produced `ime.carettap` (which re-shows the insertion handle), so the keyboard stayed down until the user tapped away and tapped back. Same dead end after dismissing the keyboard with back, without leaving the tab at all. Keep the page focus — blurring it on tab-away would fire the page's own blur handlers (validation, autocomplete dismissal, submit-on-blur) for a switch the user never made inside the page, and lose the caret the user came back to — and give the host the two signals it was missing: - `ime.refocus` (page -> host), the focus payload for a field that is already focused, sent on every tap inside it. The host re-takes the field and raises the keyboard; when it is still the field's mirror it only re-raises, so a live composing region survives. - `ime.resync` (host -> page), sent when a tab becomes active again, answered with the same payload. The keyboard comes back only for a tab that was left with it up, the way Android restores a window's IME state; a tab whose keyboard the user had dismissed comes back with the caret in place and the page unobstructed. Also folds the two identical private `parseImeEvent` copies in the browser and napplet controllers into one shared parser next to the bridge. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01AC3ambee9KFcvHCS6HRqhS --- .../browser/EmbeddedWebAppController.kt | 36 +---------- .../loggedIn/embed/EmbeddedImeBridge.kt | 59 +++++++++++++++++++ .../screen/loggedIn/embed/EmbeddedTabHost.kt | 21 +++++++ .../screen/loggedIn/embed/EmbeddedTabLayer.kt | 27 +++++++++ .../ui/screen/loggedIn/embed/RemoteImeView.kt | 40 +++++++++++-- .../favorites/EmbeddedNostrAppController.kt | 36 +---------- .../composeResources/files/napplet/shim.js | 23 +++++++- 7 files changed, 167 insertions(+), 75 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/browser/EmbeddedWebAppController.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/browser/EmbeddedWebAppController.kt index 2e615a33d2..82df6800dd 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/browser/EmbeddedWebAppController.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/browser/EmbeddedWebAppController.kt @@ -49,8 +49,7 @@ import com.vitorpamplona.amethyst.ui.screen.loggedIn.embed.EmbeddedMagnifierProb import com.vitorpamplona.amethyst.ui.screen.loggedIn.embed.EmbeddedSurfaceController import com.vitorpamplona.amethyst.ui.screen.loggedIn.embed.ImeEvent import com.vitorpamplona.amethyst.ui.screen.loggedIn.embed.MagnifierFrame -import com.vitorpamplona.amethyst.ui.screen.loggedIn.embed.parseSelectionGeometry -import org.json.JSONObject +import com.vitorpamplona.amethyst.ui.screen.loggedIn.embed.parseImeEvent import java.util.concurrent.atomic.AtomicLong /** @@ -337,39 +336,6 @@ class EmbeddedWebAppController( putLong(NappletBrowserContract.KEY_MAG_REQ_T, SystemClock.elapsedRealtimeNanos()) } - private fun parseImeEvent(payload: String): ImeEvent? { - val o = runCatching { JSONObject(payload) }.getOrNull() ?: return null - return when (o.optString("type")) { - "ime.focus" -> - ImeEvent.Focus( - inputType = o.optString("inputType", "text"), - enterKeyHint = o.optString("enterKeyHint", ""), - multiline = o.optBoolean("multiline", false), - text = o.optString("text", ""), - selStart = o.optInt("selStart", 0), - selEnd = o.optInt("selEnd", 0), - geometry = parseSelectionGeometry(o.optJSONObject("geom")), - ) - "ime.blur" -> ImeEvent.Blur - "ime.state" -> - ImeEvent.State( - text = o.optString("text", ""), - selStart = o.optInt("selStart", 0), - selEnd = o.optInt("selEnd", 0), - geometry = parseSelectionGeometry(o.optJSONObject("geom")), - ) - "ime.pagesel" -> - ImeEvent.PageSelection( - active = o.optBoolean("active", false), - text = o.optString("text", ""), - geometry = parseSelectionGeometry(o.optJSONObject("geom")), - ) - "ime.scroll" -> ImeEvent.Scroll(active = o.optBoolean("active", false)) - "ime.carettap" -> ImeEvent.CaretTap(geometry = parseSelectionGeometry(o.optJSONObject("geom"))) - else -> null - } - } - private inline fun send( what: Int, crossinline block: Bundle.() -> Unit, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedImeBridge.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedImeBridge.kt index d0b55dc108..469d01ae87 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedImeBridge.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedImeBridge.kt @@ -56,6 +56,54 @@ interface EmbeddedImeBridge { fun sendImeOp(json: String) } +/** + * Asks the page to re-announce its focused field (as an [ImeEvent.ReFocus]) — sent when a tab becomes the + * active one again. A warm tab keeps the page's DOM focus while it sits off-screen, so returning to it fires + * no `focusin` and the host would otherwise never learn there is a field to put the keyboard back on. + */ +fun EmbeddedImeBridge.requestImeResync() = sendImeOp(JSONObject().put("type", "ime.resync").toString()) + +/** Parses one page → host `ime.*` envelope into an [ImeEvent], or null for anything unrecognized. */ +fun parseImeEvent(payload: String): ImeEvent? { + val o = runCatching { JSONObject(payload) }.getOrNull() ?: return null + return when (o.optString("type")) { + "ime.focus" -> parseFocus(o) + "ime.refocus" -> + ImeEvent.ReFocus( + focus = parseFocus(o), + userAsked = o.optBoolean("raise", false), + ) + "ime.blur" -> ImeEvent.Blur + "ime.state" -> + ImeEvent.State( + text = o.optString("text", ""), + selStart = o.optInt("selStart", 0), + selEnd = o.optInt("selEnd", 0), + geometry = parseSelectionGeometry(o.optJSONObject("geom")), + ) + "ime.pagesel" -> + ImeEvent.PageSelection( + active = o.optBoolean("active", false), + text = o.optString("text", ""), + geometry = parseSelectionGeometry(o.optJSONObject("geom")), + ) + "ime.scroll" -> ImeEvent.Scroll(active = o.optBoolean("active", false)) + "ime.carettap" -> ImeEvent.CaretTap(geometry = parseSelectionGeometry(o.optJSONObject("geom"))) + else -> null + } +} + +private fun parseFocus(o: JSONObject) = + ImeEvent.Focus( + inputType = o.optString("inputType", "text"), + enterKeyHint = o.optString("enterKeyHint", ""), + multiline = o.optBoolean("multiline", false), + text = o.optString("text", ""), + selStart = o.optInt("selStart", 0), + selEnd = o.optInt("selEnd", 0), + geometry = parseSelectionGeometry(o.optJSONObject("geom")), + ) + /** What the focused page field reports up to the host keyboard. */ sealed interface ImeEvent { /** A field took focus; carries enough to configure the keyboard and seed the editing buffer. */ @@ -69,6 +117,17 @@ sealed interface ImeEvent { val geometry: SelectionGeometry? = null, ) : ImeEvent + /** + * A field that is **already** focused re-announced itself: the user tapped inside it ([userAsked]), or the + * tab was re-activated and answered the host's resync. Distinct from [Focus] because page focus never + * changed — the host must not restart the input on a field it is already hosting (that would kill a live + * composing region mid-word); it only re-takes the keyboard when it isn't hosting the field anymore. + */ + data class ReFocus( + val focus: Focus, + val userAsked: Boolean, + ) : ImeEvent + /** The field lost focus — dismiss the keyboard. */ data object Blur : ImeEvent diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabHost.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabHost.kt index 7246b19577..9f4bf544d5 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabHost.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabHost.kt @@ -149,9 +149,27 @@ object EmbeddedTabHost { contentBounds = bounds } + // Tabs whose soft keyboard was up at the moment the user left them. A warm tab keeps its page focus while + // it sits off-screen, so coming back is a *resume*: the keyboard returns only for the tabs that had one, + // the way Android restores a window's IME state. A tab the user deliberately typed on and then dismissed + // the keyboard for comes back with the caret still in the field but the page unobstructed. + private val keyboardUpOnLeave = mutableSetOf() + + /** Records whether the soft keyboard was up as [id] stops being the active tab. */ + fun noteKeyboardOnLeave( + id: String, + wasUp: Boolean, + ) { + if (wasUp) keyboardUpOnLeave.add(id) else keyboardUpOnLeave.remove(id) + } + + /** Consumes the "restore the keyboard" mark for [id] (a restore happens at most once per leave). */ + fun takeKeyboardRestore(id: String): Boolean = keyboardUpOnLeave.remove(id) + fun evict(id: String) { val w = warm.firstOrNull { it.id == id } ?: return if (activeId == id) activeId = null + keyboardUpOnLeave.remove(id) warm.remove(w) w.controller.teardown() } @@ -165,6 +183,7 @@ object EmbeddedTabHost { fun evictAll() { activeId = null + keyboardUpOnLeave.clear() val copy = warm.toList() warm.clear() copy.forEach { it.controller.teardown() } @@ -177,6 +196,8 @@ object EmbeddedTabHost { * its screen re-acquires — the user just sees the current tab reload, not a blanked-out surface. */ fun rebuildAll() { + // Every page is about to be rebuilt from scratch, so no field survives to restore a keyboard onto. + keyboardUpOnLeave.clear() val copy = warm.toList() warm.clear() copy.forEach { it.controller.teardown() } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabLayer.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabLayer.kt index 77021fa742..93ee6131a1 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabLayer.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabLayer.kt @@ -348,7 +348,16 @@ fun EmbeddedTabLayer(barFavoriteIds: List) { // the selection). All the show/hide state lives in one [SelectionUiState] so the rules — toolbar hides // while dragging or scrolling, handles hide while scrolling — are expressed in one place. val sel = remember { SelectionUiState() } + // Read at dispose time to record whether the keyboard was up as the user left this tab (see + // [EmbeddedTabHost.noteKeyboardOnLeave]). The dispose runs before anything hides the IME — our own + // onPageBlur below is what takes it down — so this still reads the pre-switch state. + val keyboardUp = rememberUpdatedState(imeBottomPx > 0) DisposableEffect(imeBridge) { + val boundId = activeId + // A warm tab keeps its page focus while parked, so returning to it fires no focus event: ask the + // page to re-announce whatever field is still focused. Restores the mirror (so a tap can raise the + // keyboard) and, when the user left mid-typing, brings the keyboard straight back. + var pendingRestore = boundId != null && EmbeddedTabHost.takeKeyboardRestore(boundId) imeView.bind(imeBridge) imeView.onRangeSelectionChanged = { sel.onFieldRangeToggle(it) } imeView.onEdited = { sel.onEdited() } @@ -364,6 +373,15 @@ fun EmbeddedTabLayer(barFavoriteIds: List) { sel.scrolling = false sel.onFieldGeometry(event.geometry, event.text.isNotEmpty()) } + is ImeEvent.ReFocus -> { + // Raise the keyboard when the user asked for it (a tap in the field) or when this tab + // was left with the keyboard up; either way re-take the field so typing works again. + imeView.onPageReFocus(event.focus, event.userAsked || pendingRestore) + pendingRestore = false + // Only a tap re-arms the caret handle. A silent tab-return resync must not pop one + // up on its own — native shows nothing until the user touches the field. + if (event.userAsked) sel.onFieldGeometry(event.focus.geometry, event.focus.text.isNotEmpty()) + } ImeEvent.Blur -> { imeView.onPageBlur() sel.onBlur() @@ -377,11 +395,20 @@ fun EmbeddedTabLayer(barFavoriteIds: List) { is ImeEvent.CaretTap -> sel.onCaretTap(event.geometry) } } + // Posted, not sent inline: the session's own `active` effect resumes a paused (parked) WebView in + // this same effect pass, and a message delivered to a still-paused page can be lost. One turn of + // the main looper later, the resume is already on its way over the same (FIFO) channel. + imeView.post { imeBridge?.requestImeResync() } onDispose { imeBridge?.onImeEvent = null imeView.onRangeSelectionChanged = null imeView.onEdited = null sel.reset() + // Remember whether the keyboard was up BEFORE onPageBlur takes it down, so returning to this + // tab resumes exactly the state it was left in. The page keeps its focus (and caret) either + // way: blurring it here would fire the page's own blur handlers — validation, autocomplete + // dismissal, submit-on-blur — for a switch the user never made inside the page. + if (boundId != null) EmbeddedTabHost.noteKeyboardOnLeave(boundId, keyboardUp.value) imeView.onPageBlur() imeView.bind(null) } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/RemoteImeView.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/RemoteImeView.kt index 2feb4901e8..6efed253c5 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/RemoteImeView.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/RemoteImeView.kt @@ -152,9 +152,15 @@ class RemoteImeView( fun selectAllText(): Boolean = onTextContextMenuItem(android.R.id.selectAll) - /** A page field focused: configure the keyboard, seed the buffer, and raise the IME. */ - @Suppress("DEPRECATION") // InputMethodManager.SHOW_IMPLICIT is deprecated; no equivalent flag on the newer API. - fun onPageFocus(focus: ImeEvent.Focus) { + /** + * A page field focused: configure the keyboard, seed the buffer, and — unless [raiseKeyboard] is false — + * raise the IME. A restored tab passes false: its field is focused again in this mirror (so a tap can put + * the keyboard straight back) without a keyboard the user had dismissed popping up over the page. + */ + fun onPageFocus( + focus: ImeEvent.Focus, + raiseKeyboard: Boolean = true, + ) { configureFor(focus) // Focus the EditText BEFORE seeding text/selection. An EditText jumps its caret to the end when it // gains focus; if we seed first, that end-position then overrides the seed and gets shipped to the @@ -164,7 +170,33 @@ class RemoteImeView( requestFocus() imm.restartInput(this) applyRemote(focus.text, focus.selStart, focus.selEnd) - // Post the show so it runs after focus/attachment has settled (showSoftInput can no-op otherwise). + if (raiseKeyboard) showKeyboard() + } + + /** + * The page re-announced a field that is **already** focused there: the user tapped inside it, or the tab + * came back on screen and answered our resync. + * + * Page focus never moved, so when this view is still the field's mirror there is nothing to re-seed — + * restarting the input would drop a live composing region mid-word — and the only thing left to do is put + * the keyboard back if it was dismissed. When the mirror was released (a tab switch clears it, see + * [onPageBlur]) this is the ONLY way back: the page will never fire another focus event for a field it + * never blurred, so re-take it here from the re-announced state. + */ + fun onPageReFocus( + focus: ImeEvent.Focus, + raiseKeyboard: Boolean, + ) { + if (hasFocus()) { + if (raiseKeyboard) showKeyboard() + } else { + onPageFocus(focus, raiseKeyboard) + } + } + + /** Post the show so it runs after focus/attachment has settled (showSoftInput can no-op otherwise). */ + @Suppress("DEPRECATION") // InputMethodManager.SHOW_IMPLICIT is deprecated; no equivalent flag on the newer API. + private fun showKeyboard() { post { if (hasFocus()) imm.showSoftInput(this, InputMethodManager.SHOW_IMPLICIT) } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/favorites/EmbeddedNostrAppController.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/favorites/EmbeddedNostrAppController.kt index 9e6ea3531c..ec509755da 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/favorites/EmbeddedNostrAppController.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/favorites/EmbeddedNostrAppController.kt @@ -47,8 +47,7 @@ import com.vitorpamplona.amethyst.ui.screen.loggedIn.embed.EmbeddedMagnifierProb import com.vitorpamplona.amethyst.ui.screen.loggedIn.embed.EmbeddedSurfaceController import com.vitorpamplona.amethyst.ui.screen.loggedIn.embed.ImeEvent import com.vitorpamplona.amethyst.ui.screen.loggedIn.embed.MagnifierFrame -import com.vitorpamplona.amethyst.ui.screen.loggedIn.embed.parseSelectionGeometry -import org.json.JSONObject +import com.vitorpamplona.amethyst.ui.screen.loggedIn.embed.parseImeEvent import java.util.concurrent.atomic.AtomicLong /** @@ -282,39 +281,6 @@ class EmbeddedNostrAppController( putLong(NappletEmbedContract.KEY_MAG_REQ_T, SystemClock.elapsedRealtimeNanos()) } - private fun parseImeEvent(payload: String): ImeEvent? { - val o = runCatching { JSONObject(payload) }.getOrNull() ?: return null - return when (o.optString("type")) { - "ime.focus" -> - ImeEvent.Focus( - inputType = o.optString("inputType", "text"), - enterKeyHint = o.optString("enterKeyHint", ""), - multiline = o.optBoolean("multiline", false), - text = o.optString("text", ""), - selStart = o.optInt("selStart", 0), - selEnd = o.optInt("selEnd", 0), - geometry = parseSelectionGeometry(o.optJSONObject("geom")), - ) - "ime.blur" -> ImeEvent.Blur - "ime.state" -> - ImeEvent.State( - text = o.optString("text", ""), - selStart = o.optInt("selStart", 0), - selEnd = o.optInt("selEnd", 0), - geometry = parseSelectionGeometry(o.optJSONObject("geom")), - ) - "ime.pagesel" -> - ImeEvent.PageSelection( - active = o.optBoolean("active", false), - text = o.optString("text", ""), - geometry = parseSelectionGeometry(o.optJSONObject("geom")), - ) - "ime.scroll" -> ImeEvent.Scroll(active = o.optBoolean("active", false)) - "ime.carettap" -> ImeEvent.CaretTap(geometry = parseSelectionGeometry(o.optJSONObject("geom"))) - else -> null - } - } - fun back() = send(NappletEmbedContract.MSG_BACK) fun reload() = send(NappletEmbedContract.MSG_RELOAD) diff --git a/commons/src/commonMain/composeResources/files/napplet/shim.js b/commons/src/commonMain/composeResources/files/napplet/shim.js index 89e256a2f4..8e52213e68 100644 --- a/commons/src/commonMain/composeResources/files/napplet/shim.js +++ b/commons/src/commonMain/composeResources/files/napplet/shim.js @@ -378,6 +378,15 @@ return { type:'ime.focus', inputType: inputType, enterKeyHint: (n.enterKeyHint || ''), multiline: multiline, text: valOf(n), selStart: sel[0], selEnd: sel[1], geom: fieldGeom(n) }; } + // Same payload as `ime.focus`, but for a field that is ALREADY focused — the host uses it to (re-)take the + // keyboard without treating it as a fresh focus. `raise` marks the cases the user explicitly asked for the + // keyboard (a tap in the field); a resync reply leaves the decision to the host. + function refocusInfo(n, raise){ + var info = focusInfo(n); + info.type = 'ime.refocus'; + info.raise = !!raise; + return info; + } // Last selection we either applied (applyState) or already reported, so the asynchronous // selectionchange our own setSel triggers doesn't echo back to the host as a fresh edit. var lastSel = null; @@ -464,7 +473,16 @@ if (s[0] !== s[1] && !sameSel(s, lastSel)) reportState(); // report the word only if not already sent }, true); document.addEventListener('click', function(e){ - if (!el || isCE(el) || e.target !== el) return; + if (!el) return; + // A tap inside the ALREADY-focused editable fires no `focusin`, so the host — whose only keyboard-raising + // signal used to be `ime.focus` — never learned that the user wants the keyboard back after dismissing it + // (or after a tab switch dropped it). Re-announce the field on every such tap: the host raises the + // keyboard and, if it isn't hosting this field anymore, re-seeds its mirror from this payload. + // Sent on the focusing tap too (which the host answers with a no-op, since it is already hosting the + // field): telling the two apart here would take a focusin-to-click timing guess, and a guess that comes + // in late swallows a real "give me the keyboard back" tap. + if (e.target === el || (el.contains && el.contains(e.target))) send(refocusInfo(el, true)); + if (isCE(el) || e.target !== el) return; var sel = selOf(el); if (sel[0] !== sel[1]) { // Tap landed on a selection. Defer the collapse: if a dblclick follows (within the tap window) it @@ -708,6 +726,9 @@ document.addEventListener('scroll', onAnyScroll, true); // capture: any scroller, not just the document window.__nappletImeHandle = function(msg){ + // The tab came back on screen. Focus never left the page (the surface just moved off-screen), so no + // `focusin` will fire — re-announce the still-focused field so the host can re-take the keyboard. + if (msg.type === 'ime.resync') { if (el) send(refocusInfo(el, false)); return; } if (msg.type === 'ime.set') applyState(msg); else if (msg.type === 'ime.action') enter(el); else if (msg.type === 'ime.pageextend') pageExtend(msg.edge, msg.x, msg.y); From 59994a7847091d354f80d7a2003709587dc065d8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 17:32:42 +0000 Subject: [PATCH 095/132] fix: mark a notification dismissed on swipe, mark-read and inline reply MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 25s enrichment window re-posts a notification every time metadata for it lands, and postStandard/postConversation only skip that when NotificationUtils.wasDismissed says the user is done with the event. Only one path ever recorded that: reading the note in-app. Swiping the notification away, hitting "mark as read", or replying from the tray all just cancelled the notification id, so the enricher happily put it back seconds later — and kept a relay subscription and a wakelock open for it until the window elapsed. Every notification already carries a delete intent and its actions target the same receiver, so thread the event id through them and mark it dismissed there. Replies mark it only once the send succeeds, leaving a failed send free to enrich and retry. Also, in the same area: - Pin the group summary's timestamp to the child's event time. It defaulted to "now", and the summary is re-posted on every enrichment re-render, so the group kept re-sorting in the shade while the user was reading it. - Make the childless-summary scan a single pass. Now that every child ships with a summary the active list is about twice as long and the pairwise scan grew four-fold. Deciding what is a child by the summary flag instead of by comparing ids also fixes the case where a child's id equals the summary's. --- .../NotificationReplyReceiver.kt | 24 +++++- .../notifications/NotificationUtils.kt | 78 ++++++++++++++----- 2 files changed, 78 insertions(+), 24 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationReplyReceiver.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationReplyReceiver.kt index 635ce0a930..26b2930f02 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationReplyReceiver.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationReplyReceiver.kt @@ -56,6 +56,22 @@ class NotificationReplyReceiver : BroadcastReceiver() { intent: Intent, ) { val notificationId = intent.getIntExtra(NotificationUtils.KEY_NOTIFICATION_ID, 0) + + // Whatever the action, the user is done with this notification, so record it before + // doing anything else. An enrichment window may still be open on the event (up to + // 25s from the first post), and it re-posts the notification every time metadata + // lands — without this the notification the user just dealt with comes back, and the + // enricher keeps a relay subscription and a wakelock alive for it until the window + // elapses. Replies mark it after the send succeeds instead, so a failure leaves the + // notification to enrich and retry. + val eventId = intent.getStringExtra(NotificationUtils.KEY_EVENT_ID) + if (intent.action != NotificationUtils.REPLY_ACTION && + intent.action != NotificationUtils.PUBLIC_REPLY_ACTION && + intent.action != NotificationUtils.MARMOT_REPLY_ACTION + ) { + eventId?.let { NotificationUtils.markDismissed(it) } + } + val notificationManager = ContextCompat.getSystemService(context, NotificationManager::class.java) as NotificationManager @@ -86,7 +102,7 @@ class NotificationReplyReceiver : BroadcastReceiver() { if (members.isEmpty()) return - runOnRelay(notificationManager, notificationId) { + runOnRelay(notificationManager, notificationId, eventId) { sendReply(accountNpub, members, replyText) } } @@ -103,7 +119,7 @@ class NotificationReplyReceiver : BroadcastReceiver() { val accountNpub = intent.getStringExtra(NotificationUtils.KEY_ACCOUNT_NPUB) ?: return val targetEventId = intent.getStringExtra(NotificationUtils.KEY_TARGET_EVENT_ID) ?: return - runOnRelay(notificationManager, notificationId) { + runOnRelay(notificationManager, notificationId, eventId) { sendPublicReply(accountNpub, targetEventId, replyText) } } @@ -122,7 +138,7 @@ class NotificationReplyReceiver : BroadcastReceiver() { val replyToInnerId = intent.getStringExtra(NotificationUtils.KEY_MARMOT_REPLY_TO_INNER_ID) val replyToInnerAuthor = intent.getStringExtra(NotificationUtils.KEY_MARMOT_REPLY_TO_INNER_AUTHOR) - runOnRelay(notificationManager, notificationId) { + runOnRelay(notificationManager, notificationId, eventId) { sendMarmotReply(accountNpub, nostrGroupId, replyToInnerId, replyToInnerAuthor, replyText) } } @@ -132,6 +148,7 @@ class NotificationReplyReceiver : BroadcastReceiver() { private fun runOnRelay( notificationManager: NotificationManager, notificationId: Int, + eventId: String?, block: suspend () -> Unit, ) { val pendingResult = goAsync() @@ -146,6 +163,7 @@ class NotificationReplyReceiver : BroadcastReceiver() { try { block() + eventId?.let { NotificationUtils.markDismissed(it) } notificationManager.cancelAndPrune(notificationId) } catch (e: Exception) { if (e is CancellationException) throw e diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationUtils.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationUtils.kt index d7191cf468..6d4791386c 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationUtils.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/notifications/NotificationUtils.kt @@ -73,6 +73,13 @@ object NotificationUtils { const val DISMISS_ACTION = "com.vitorpamplona.amethyst.DISMISS_ACTION" const val KEY_REPLY_TEXT = "key_reply_text" const val KEY_NOTIFICATION_ID = "key_notification_id" + + /** + * Hex id of the event this notification was posted for, carried on every action + * and on the delete intent so the receiver can mark it dismissed. Distinct from + * [KEY_TARGET_EVENT_ID], which is the note an inline reply is addressed to. + */ + const val KEY_EVENT_ID = "key_event_id" const val KEY_ACCOUNT_NPUB = "key_account_npub" const val KEY_CHATROOM_MEMBERS = "key_chatroom_members" const val KEY_TARGET_EVENT_ID = "key_target_event_id" @@ -91,16 +98,19 @@ object NotificationUtils { */ private const val OWN_GROUP_PREFIX = "com.vitorpamplona.amethyst." - // Event ids the user has just read/dismissed in-app. The enrichment path - // re-posts a notification as metadata arrives; without this guard a - // notification the user already dismissed would be resurrected seconds later - // when its author's kind:0 lands. Keyed by the event id string (not the - // hashCode) so distinct events can't collide. Entries self-expire after a - // window comfortably longer than the 25s enrichment window. + // Event ids the user is done with. The enrichment path re-posts a notification + // as metadata arrives; without this guard a notification the user already got + // rid of would be resurrected seconds later when its author's kind:0 lands, and + // the enricher would go on holding a relay window and a wakelock open for it. + // Every way a user can be done with a notification has to record here — reading + // the event in-app, swiping the notification away, "mark as read", and replying + // inline — or that path leaks the resurrection. Keyed by the event id string + // (not the hashCode) so distinct events can't collide. Entries self-expire after + // a window comfortably longer than the 25s enrichment window. private const val DISMISS_GUARD_MS = 90_000L private val recentlyDismissed = ConcurrentHashMap() - private fun markDismissed(eventId: String) { + fun markDismissed(eventId: String) { val now = SystemClock.elapsedRealtime() recentlyDismissed[eventId] = now + DISMISS_GUARD_MS if (recentlyDismissed.size > 256) { @@ -227,7 +237,7 @@ object NotificationUtils { .setPriority(category.priority()) .setCategory(NotificationCompat.CATEGORY_SOCIAL) .setGroup(groupKey) - .setDeleteIntent(dismissIntent(applicationContext, notId)) + .setDeleteIntent(dismissIntent(applicationContext, notId, id)) .setAutoCancel(true) .setOnlyAlertOnce(true) .setWhen(time * 1000) @@ -246,11 +256,11 @@ object NotificationUtils { } if (inlineReply != null) { - builder.addAction(publicReplyAction(applicationContext, notId, inlineReply)) + builder.addAction(publicReplyAction(applicationContext, notId, id, inlineReply)) } notify(notId, builder.build()) - sendGroupSummary(category, groupKey, summaryId, applicationContext) + sendGroupSummary(category, groupKey, summaryId, time, applicationContext) } // --------------------------------------------------------------------- @@ -345,21 +355,21 @@ object NotificationUtils { .setPriority(category.priority()) .setCategory(NotificationCompat.CATEGORY_MESSAGE) .setGroup(groupKey) - .setDeleteIntent(dismissIntent(applicationContext, notId)) + .setDeleteIntent(dismissIntent(applicationContext, notId, id)) .setAutoCancel(true) .setOnlyAlertOnce(true) .setWhen(time * 1000) when (replyAction) { - is ReplyAction.Dm -> builder.addAction(dmReplyAction(applicationContext, notId, replyAction)) - is ReplyAction.Marmot -> builder.addAction(marmotReplyAction(applicationContext, notId, replyAction)) - null -> publicInlineReply?.let { builder.addAction(publicReplyAction(applicationContext, notId, it)) } + is ReplyAction.Dm -> builder.addAction(dmReplyAction(applicationContext, notId, id, replyAction)) + is ReplyAction.Marmot -> builder.addAction(marmotReplyAction(applicationContext, notId, id, replyAction)) + null -> publicInlineReply?.let { builder.addAction(publicReplyAction(applicationContext, notId, id, it)) } } - if (addMarkRead) builder.addAction(markReadAction(applicationContext, notId)) + if (addMarkRead) builder.addAction(markReadAction(applicationContext, notId, id)) notify(notId, builder.build()) - sendGroupSummary(category, groupKey, summaryId, applicationContext) + sendGroupSummary(category, groupKey, summaryId, time, applicationContext) } // --------------------------------------------------------------------- @@ -397,11 +407,13 @@ object NotificationUtils { private fun dismissIntent( applicationContext: Context, notId: Int, + eventId: String, ): PendingIntent { val intent = Intent(applicationContext, NotificationReplyReceiver::class.java).apply { action = DISMISS_ACTION putExtra(KEY_NOTIFICATION_ID, notId) + putExtra(KEY_EVENT_ID, eventId) } return PendingIntent.getBroadcast( applicationContext, @@ -440,12 +452,14 @@ object NotificationUtils { private fun dmReplyAction( applicationContext: Context, notId: Int, + eventId: String, action: ReplyAction.Dm, ): NotificationCompat.Action { val intent = Intent(applicationContext, NotificationReplyReceiver::class.java).apply { this.action = REPLY_ACTION putExtra(KEY_NOTIFICATION_ID, notId) + putExtra(KEY_EVENT_ID, eventId) putExtra(KEY_ACCOUNT_NPUB, action.accountNpub) putExtra(KEY_CHATROOM_MEMBERS, action.chatroomMembers) } @@ -455,12 +469,14 @@ object NotificationUtils { private fun marmotReplyAction( applicationContext: Context, notId: Int, + eventId: String, action: ReplyAction.Marmot, ): NotificationCompat.Action { val intent = Intent(applicationContext, NotificationReplyReceiver::class.java).apply { this.action = MARMOT_REPLY_ACTION putExtra(KEY_NOTIFICATION_ID, notId) + putExtra(KEY_EVENT_ID, eventId) putExtra(KEY_ACCOUNT_NPUB, action.accountNpub) putExtra(KEY_MARMOT_GROUP_ID, action.nostrGroupId) action.replyToInnerEventId?.let { putExtra(KEY_MARMOT_REPLY_TO_INNER_ID, it) } @@ -472,12 +488,14 @@ object NotificationUtils { private fun publicReplyAction( applicationContext: Context, notId: Int, + eventId: String, target: InlineReplyTarget, ): NotificationCompat.Action { val intent = Intent(applicationContext, NotificationReplyReceiver::class.java).apply { action = PUBLIC_REPLY_ACTION putExtra(KEY_NOTIFICATION_ID, notId) + putExtra(KEY_EVENT_ID, eventId) putExtra(KEY_ACCOUNT_NPUB, target.accountNpub) putExtra(KEY_TARGET_EVENT_ID, target.targetEventId) } @@ -487,11 +505,13 @@ object NotificationUtils { private fun markReadAction( applicationContext: Context, notId: Int, + eventId: String, ): NotificationCompat.Action { val markReadIntent = Intent(applicationContext, NotificationReplyReceiver::class.java).apply { action = MARK_READ_ACTION putExtra(KEY_NOTIFICATION_ID, notId) + putExtra(KEY_EVENT_ID, eventId) } val markReadPendingIntent = PendingIntent.getBroadcast( @@ -614,6 +634,7 @@ object NotificationUtils { category: NotificationCategory, groupKey: String, summaryId: Int, + time: Long, applicationContext: Context, ) { val summaryBuilder = @@ -628,6 +649,11 @@ object NotificationUtils { .setGroupAlertBehavior(NotificationCompat.GROUP_ALERT_CHILDREN) .setAutoCancel(true) .setOnlyAlertOnce(true) + // Pinned to the child's event time rather than left to default to "now". + // The summary is re-posted on every one of the enrichment path's re-renders, + // and a fresh timestamp each time would keep re-sorting the group in the + // shade while the user is looking at it. + .setWhen(time * 1000) .setStyle( NotificationCompat .InboxStyle() @@ -692,15 +718,25 @@ object NotificationUtils { */ fun NotificationManager.cancelChildlessGroupSummaries(alreadyGone: Int? = null) { val active: Array = activeNotifications + + // Collect the groups that still have a child in one pass, then cancel the + // summaries not in that set. Every child now ships with a summary, so this list + // is about twice as long as it used to be and the pairwise scan it replaces grew + // four-fold. Membership is decided by the summary flag rather than by comparing + // ids, which is also what makes it correct when a child's id happens to equal the + // summary's. + val groupsWithChildren = HashSet(active.size) + for (child in active) { + if (child.notification.flags and Notification.FLAG_GROUP_SUMMARY != 0) continue + if (child.id == alreadyGone) continue + child.notification.group?.let { groupsWithChildren.add(it) } + } + for (summary in active) { if (summary.notification.flags and Notification.FLAG_GROUP_SUMMARY == 0) continue val group = summary.notification.group ?: continue if (!group.startsWith(OWN_GROUP_PREFIX)) continue - val hasChildren = - active.any { - it.id != summary.id && it.id != alreadyGone && it.notification.group == group - } - if (!hasChildren) cancel(summary.id) + if (group !in groupsWithChildren) cancel(summary.id) } } From a0f4328f2a34aa17164f2c6d3604de7a3f02d067 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 19:38:50 +0000 Subject: [PATCH 096/132] fix: keep the embed re-focus ping payload-free and re-seed via resync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Audit of the previous commit found three problems in it. The `ime.refocus` sent on every tap carried the field's full editing state, so a tap in a long textarea put 40KB on the wire per tap (145B before), and its geometry made the page mirror the whole field into a hidden div and force a synchronous layout — measured at ~3.5ms per tap on a 40k-char textarea, doubling the cost of every tap in a field. Split the message in two: taps ring a payload-free `ime.wantkb` doorbell, and the host answers it with the `ime.resync` it already had — but only when it no longer mirrors the field, so the common "keyboard was dismissed, tap to get it back" case is one small message and no round trip. Per-tap payload is now constant (~180B) and the per-tap CPU cost is back at parity with before the fix. The "am I already hosting this field" check read `hasFocus()` alone, so a focus that lingers past `clearFocus()` (a lone focusable in the hierarchy can take it straight back) would have skipped the re-seed and shipped the previous tab's text to the page on the first keystroke. Track mirroring explicitly. The keyboard-restore mark was armed by any keyboard up at tab-switch time, including one belonging to the browser's own address bar; require that the mirror actually holds it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01AC3ambee9KFcvHCS6HRqhS --- .../loggedIn/embed/EmbeddedImeBridge.kt | 35 ++++++++++------ .../screen/loggedIn/embed/EmbeddedTabLayer.kt | 35 ++++++++++++---- .../ui/screen/loggedIn/embed/RemoteImeView.kt | 35 ++++++++++++---- .../composeResources/files/napplet/shim.js | 42 +++++++++++-------- 4 files changed, 100 insertions(+), 47 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedImeBridge.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedImeBridge.kt index 469d01ae87..c49a87b2ab 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedImeBridge.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedImeBridge.kt @@ -57,9 +57,10 @@ interface EmbeddedImeBridge { } /** - * Asks the page to re-announce its focused field (as an [ImeEvent.ReFocus]) — sent when a tab becomes the - * active one again. A warm tab keeps the page's DOM focus while it sits off-screen, so returning to it fires - * no `focusin` and the host would otherwise never learn there is a field to put the keyboard back on. + * Asks the page to re-announce its focused field (as an [ImeEvent.ReFocus]). A warm tab keeps the page's DOM + * focus while it sits off-screen, so no `focusin` ever fires for it again and this is the only way the host + * learns there is a field to put the keyboard back on. Sent when a tab becomes the active one again, and when + * an [ImeEvent.WantKeyboard] tap arrives for a field this host no longer mirrors. */ fun EmbeddedImeBridge.requestImeResync() = sendImeOp(JSONObject().put("type", "ime.resync").toString()) @@ -68,11 +69,8 @@ fun parseImeEvent(payload: String): ImeEvent? { val o = runCatching { JSONObject(payload) }.getOrNull() ?: return null return when (o.optString("type")) { "ime.focus" -> parseFocus(o) - "ime.refocus" -> - ImeEvent.ReFocus( - focus = parseFocus(o), - userAsked = o.optBoolean("raise", false), - ) + "ime.wantkb" -> ImeEvent.WantKeyboard + "ime.refocus" -> ImeEvent.ReFocus(parseFocus(o)) "ime.blur" -> ImeEvent.Blur "ime.state" -> ImeEvent.State( @@ -118,14 +116,25 @@ sealed interface ImeEvent { ) : ImeEvent /** - * A field that is **already** focused re-announced itself: the user tapped inside it ([userAsked]), or the - * tab was re-activated and answered the host's resync. Distinct from [Focus] because page focus never - * changed — the host must not restart the input on a field it is already hosting (that would kill a live - * composing region mid-word); it only re-takes the keyboard when it isn't hosting the field anymore. + * The user tapped inside a field that is **already** focused in the page: put the keyboard back up. No + * focus event follows a tap on a field that never blurred, so this is the only signal that the user wants + * the keyboard back after dismissing it — or after a tab switch released the host's mirror. + * + * Deliberately payload-free (it fires on every tap in a field): a host that no longer mirrors the field + * answers it with an [requestImeResync] and takes the state from the [ReFocus] that comes back. + */ + data object WantKeyboard : ImeEvent + + /** + * The page's answer to [requestImeResync]: the editing state of a field that is already focused there. + * Distinct from [Focus] because page focus never changed — the host must not restart the input on a field + * it is already mirroring (that would kill a live composing region mid-word). + * + * Carries no geometry: measuring a caret rect forces a synchronous layout in the page, and the host has no + * use for it here (the `ime.carettap` that rides along with a tap carries fresh geometry). */ data class ReFocus( val focus: Focus, - val userAsked: Boolean, ) : ImeEvent /** The field lost focus — dismiss the keyboard. */ diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabLayer.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabLayer.kt index 93ee6131a1..50bc57161f 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabLayer.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabLayer.kt @@ -357,7 +357,9 @@ fun EmbeddedTabLayer(barFavoriteIds: List) { // A warm tab keeps its page focus while parked, so returning to it fires no focus event: ask the // page to re-announce whatever field is still focused. Restores the mirror (so a tap can raise the // keyboard) and, when the user left mid-typing, brings the keyboard straight back. - var pendingRestore = boundId != null && EmbeddedTabHost.takeKeyboardRestore(boundId) + // Consumed only when there is a bridge to ask (the mark is one-shot, so spending it on a bind that + // can't send the resync would drop the restore this tab was owed). + var pendingRestore = imeBridge != null && boundId != null && EmbeddedTabHost.takeKeyboardRestore(boundId) imeView.bind(imeBridge) imeView.onRangeSelectionChanged = { sel.onFieldRangeToggle(it) } imeView.onEdited = { sel.onEdited() } @@ -373,14 +375,26 @@ fun EmbeddedTabLayer(barFavoriteIds: List) { sel.scrolling = false sel.onFieldGeometry(event.geometry, event.text.isNotEmpty()) } + ImeEvent.WantKeyboard -> { + // Still mirroring this field (the keyboard was merely dismissed) → just put it back. + // Otherwise the mirror was released by a tab switch and holds another tab's buffer: + // ask for the state first and raise once it lands. + if (imeView.isMirroringPageField()) { + imeView.raiseKeyboard() + } else { + pendingRestore = true + imeBridge.requestImeResync() + } + } is ImeEvent.ReFocus -> { - // Raise the keyboard when the user asked for it (a tap in the field) or when this tab - // was left with the keyboard up; either way re-take the field so typing works again. - imeView.onPageReFocus(event.focus, event.userAsked || pendingRestore) + // Raise only if something asked for it — a tap that rang the doorbell above, or a tab + // left with the keyboard up. A plain tab return re-takes the field silently. + imeView.onPageReFocus(event.focus, pendingRestore) pendingRestore = false - // Only a tap re-arms the caret handle. A silent tab-return resync must not pop one - // up on its own — native shows nothing until the user touches the field. - if (event.userAsked) sel.onFieldGeometry(event.focus.geometry, event.focus.text.isNotEmpty()) + // Re-arm "the field has text" so a tap can show the insertion handle again (a tab + // switch resets it). Geometry stays null here, so this can't pop a handle up on its + // own — native shows nothing until the user touches the field. + sel.onFieldGeometry(null, event.focus.text.isNotEmpty()) } ImeEvent.Blur -> { imeView.onPageBlur() @@ -408,7 +422,12 @@ fun EmbeddedTabLayer(barFavoriteIds: List) { // tab resumes exactly the state it was left in. The page keeps its focus (and caret) either // way: blurring it here would fire the page's own blur handlers — validation, autocomplete // dismissal, submit-on-blur — for a switch the user never made inside the page. - if (boundId != null) EmbeddedTabHost.noteKeyboardOnLeave(boundId, keyboardUp.value) + // + // Only a keyboard THIS mirror holds counts: the tab's own chrome has host-side fields (the + // browser's address bar), and typing in one of those must not arm a restore for a page field. + if (boundId != null) { + EmbeddedTabHost.noteKeyboardOnLeave(boundId, keyboardUp.value && imeView.isMirroringPageField()) + } imeView.onPageBlur() imeView.bind(null) } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/RemoteImeView.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/RemoteImeView.kt index 6efed253c5..bdb55e1aa0 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/RemoteImeView.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/RemoteImeView.kt @@ -65,6 +65,13 @@ class RemoteImeView( // The last state we sent, so we never ship a no-op (avoids feedback churn with the page). private var lastSent: String? = null + // True while this view mirrors a live page field, i.e. between a page focus and the blur that releases it. + // Distinct from [hasFocus]: clearing focus on a View can hand it straight back (a lone focusable in the + // hierarchy re-takes it), and window focus comes and goes on its own. Only this flag says the buffer still + // belongs to the page's field — without it, a lingering focus would let a re-focus skip the re-seed and + // ship the PREVIOUS tab's text to the page on the first keystroke. + private var mirroring = false + private val imm get() = context.getSystemService(Context.INPUT_METHOD_SERVICE) as InputMethodManager private val flush = Runnable { flushState() } @@ -153,15 +160,16 @@ class RemoteImeView( fun selectAllText(): Boolean = onTextContextMenuItem(android.R.id.selectAll) /** - * A page field focused: configure the keyboard, seed the buffer, and — unless [raiseKeyboard] is false — + * A page field focused: configure the keyboard, seed the buffer, and — unless [withKeyboard] is false — * raise the IME. A restored tab passes false: its field is focused again in this mirror (so a tap can put * the keyboard straight back) without a keyboard the user had dismissed popping up over the page. */ fun onPageFocus( focus: ImeEvent.Focus, - raiseKeyboard: Boolean = true, + withKeyboard: Boolean = true, ) { configureFor(focus) + mirroring = true // Focus the EditText BEFORE seeding text/selection. An EditText jumps its caret to the end when it // gains focus; if we seed first, that end-position then overrides the seed and gets shipped to the // page — so a tap mid-text lands the caret at the end of the field. Seeding AFTER focus makes the @@ -170,7 +178,7 @@ class RemoteImeView( requestFocus() imm.restartInput(this) applyRemote(focus.text, focus.selStart, focus.selEnd) - if (raiseKeyboard) showKeyboard() + if (withKeyboard) raiseKeyboard() } /** @@ -185,18 +193,26 @@ class RemoteImeView( */ fun onPageReFocus( focus: ImeEvent.Focus, - raiseKeyboard: Boolean, + withKeyboard: Boolean, ) { - if (hasFocus()) { - if (raiseKeyboard) showKeyboard() + if (isMirroringPageField()) { + if (withKeyboard) raiseKeyboard() } else { - onPageFocus(focus, raiseKeyboard) + onPageFocus(focus, withKeyboard) } } - /** Post the show so it runs after focus/attachment has settled (showSoftInput can no-op otherwise). */ + /** True while the keyboard this view holds belongs to a page field — see [mirroring]. */ + fun isMirroringPageField() = mirroring && hasFocus() + + /** + * Put the keyboard back on the field this view already mirrors — the user tapped it after dismissing the + * keyboard, which leaves the page's focus (and this mirror) untouched, so there is nothing to re-seed. + * + * Post the show so it runs after focus/attachment has settled (showSoftInput can no-op otherwise). + */ @Suppress("DEPRECATION") // InputMethodManager.SHOW_IMPLICIT is deprecated; no equivalent flag on the newer API. - private fun showKeyboard() { + fun raiseKeyboard() { post { if (hasFocus()) imm.showSoftInput(this, InputMethodManager.SHOW_IMPLICIT) } @@ -239,6 +255,7 @@ class RemoteImeView( /** The page field blurred: drop the keyboard. */ fun onPageBlur() { + mirroring = false removeCallbacks(reportRangeLost) if (hadRange) { hadRange = false diff --git a/commons/src/commonMain/composeResources/files/napplet/shim.js b/commons/src/commonMain/composeResources/files/napplet/shim.js index 8e52213e68..6afbca5e04 100644 --- a/commons/src/commonMain/composeResources/files/napplet/shim.js +++ b/commons/src/commonMain/composeResources/files/napplet/shim.js @@ -378,14 +378,19 @@ return { type:'ime.focus', inputType: inputType, enterKeyHint: (n.enterKeyHint || ''), multiline: multiline, text: valOf(n), selStart: sel[0], selEnd: sel[1], geom: fieldGeom(n) }; } - // Same payload as `ime.focus`, but for a field that is ALREADY focused — the host uses it to (re-)take the - // keyboard without treating it as a fresh focus. `raise` marks the cases the user explicitly asked for the - // keyboard (a tap in the field); a resync reply leaves the decision to the host. - function refocusInfo(n, raise){ - var info = focusInfo(n); - info.type = 'ime.refocus'; - info.raise = !!raise; - return info; + // Like `ime.focus`, but for a field that is ALREADY focused: the answer to the host's `ime.resync`, which + // it asks for when it needs to (re-)take a field whose focus never moved in the page. + // + // Carries NO geometry, unlike focusInfo: fieldGeom's caret measurement mirrors the whole field into a + // hidden div and forces a synchronous layout (~3.5ms on a 40k-char textarea). The host only needs the + // editing state here; the `ime.carettap` that rides along with a tap carries fresh geometry. + function refocusInfo(n){ + var t = (n.tagName || '').toUpperCase(); + var sel = selOf(n); + return { type:'ime.refocus', + inputType: isCE(n) ? 'text' : (t === 'TEXTAREA' ? 'textarea' : (n.type || 'text').toLowerCase()), + enterKeyHint: (n.enterKeyHint || ''), multiline: isCE(n) || t === 'TEXTAREA', + text: valOf(n), selStart: sel[0], selEnd: sel[1] }; } // Last selection we either applied (applyState) or already reported, so the asynchronous // selectionchange our own setSel triggers doesn't echo back to the host as a fresh edit. @@ -476,12 +481,14 @@ if (!el) return; // A tap inside the ALREADY-focused editable fires no `focusin`, so the host — whose only keyboard-raising // signal used to be `ime.focus` — never learned that the user wants the keyboard back after dismissing it - // (or after a tab switch dropped it). Re-announce the field on every such tap: the host raises the - // keyboard and, if it isn't hosting this field anymore, re-seeds its mirror from this payload. - // Sent on the focusing tap too (which the host answers with a no-op, since it is already hosting the - // field): telling the two apart here would take a focusin-to-click timing guess, and a guess that comes - // in late swallows a real "give me the keyboard back" tap. - if (e.target === el || (el.contains && el.contains(e.target))) send(refocusInfo(el, true)); + // (or after a tab switch dropped it). Ring the doorbell on every such tap; the host puts the keyboard + // back, and asks for a resync first if it no longer mirrors this field. + // + // Deliberately payload-free: this fires on every tap in a field, including the tap that focused it (the + // host answers that one with a no-op — telling the two apart here would take a focusin-to-click timing + // guess, and a guess that lands late swallows a real "give me the keyboard back" tap). Attaching the + // editing state would put the field's whole text — 40KB for a long textarea — on the wire per tap. + if (e.target === el || (el.contains && el.contains(e.target))) send({ type:'ime.wantkb' }); if (isCE(el) || e.target !== el) return; var sel = selOf(el); if (sel[0] !== sel[1]) { @@ -726,9 +733,10 @@ document.addEventListener('scroll', onAnyScroll, true); // capture: any scroller, not just the document window.__nappletImeHandle = function(msg){ - // The tab came back on screen. Focus never left the page (the surface just moved off-screen), so no - // `focusin` will fire — re-announce the still-focused field so the host can re-take the keyboard. - if (msg.type === 'ime.resync') { if (el) send(refocusInfo(el, false)); return; } + // The host needs to (re-)take a field whose focus never left the page — the tab came back on screen, or + // a tap rang the doorbell above and the host has no mirror for it. No `focusin` will fire for a field + // that never blurred, so hand it the editing state here. + if (msg.type === 'ime.resync') { if (el) send(refocusInfo(el)); return; } if (msg.type === 'ime.set') applyState(msg); else if (msg.type === 'ime.action') enter(el); else if (msg.type === 'ime.pageextend') pageExtend(msg.edge, msg.x, msg.y); From 05a331068f81f053908d84b36d79bd841feba657 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Sun, 9 Aug 2026 17:51:39 -0400 Subject: [PATCH 097/132] test(concord): pin that a banned member's typing heartbeat is dropped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The soft-ban audit's A4 shipped with a send-side guard and a receive-side filter, and the receive-side filter — the only one that binds a modified client — had no test. Bans first, so the assertion exercises the filter rather than an entry that was seated before the ban, and checks the filter is targeted rather than a blanket mute. Mutation-tested: removing the isBanned check in ConcordCommunitySession.ingestTyping fails it. Co-Authored-By: Claude Opus 5 (1M context) --- .../model/concord/ConcordBannedTypingTest.kt | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordBannedTypingTest.kt diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordBannedTypingTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordBannedTypingTest.kt new file mode 100644 index 0000000000..a89e26789e --- /dev/null +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordBannedTypingTest.kt @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.model.concord + +import com.vitorpamplona.amethyst.commons.actions.ConcordActions +import com.vitorpamplona.amethyst.commons.actions.ConcordModeration +import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityFactory +import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEntry +import com.vitorpamplona.quartz.concord.cord02Community.NewConcordCommunity +import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray +import com.vitorpamplona.quartz.nip01Core.core.toHexKey +import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal +import com.vitorpamplona.quartz.utils.TimeUtils +import kotlinx.coroutines.test.runTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * The receive half of the soft-ban audit's A4: a banned member's typing heartbeat must not reach + * the "… is typing" row. The send half is a guard in the app's own action layer, which a malicious + * or modified client simply won't run — so this filter, on the receive side, is the only one that + * actually protects the room. It shipped without a test; this is it. + */ +class ConcordBannedTypingTest { + private val owner = NostrSignerInternal(KeyPair()) + private val troll = NostrSignerInternal(KeyPair()) + private val regular = NostrSignerInternal(KeyPair()) + + private fun entryFor(community: NewConcordCommunity) = + ConcordCommunityListEntry( + id = community.communityIdHex, + owner = community.ownerPubKey, + ownerSalt = community.ownerSalt.toHexKey(), + root = community.communityRoot.toHexKey(), + rootEpoch = community.rootEpoch, + controlPk = community.controlPkHex, + controlRoot = community.controlRoot.toHexKey(), + relays = listOf("wss://r.example"), + name = "Nostrichs", + ) + + @Test + fun dropsABannedMembersTypingHeartbeatAndKeepsEveryoneElses() = + runTest { + val community = ConcordCommunityFactory.create(owner, "Nostrichs", createdAt = 1L, relays = listOf("wss://r.example")) + val session = ConcordCommunitySession(entryFor(community), owner.pubKey) + community.genesisWraps.forEach { session.ingest(it) } + + val channelId = community.generalChannelIdHex + val plane = ConcordActions.publicChannel(community.communityRoot, community.generalChannelId, community.rootEpoch) + val now = TimeUtils.now() + + // Ban first, so what follows tests the filter rather than an entry seated before the ban. + // The banlist edition folds through the Control Plane exactly as it would on the wire. + session.ingest( + ConcordModeration.ban( + actor = owner, + controlPlane = session.controlPlaneKeys(), + communityId = community.communityIdHex.hexToByteArray(), + member = troll.pubKey, + current = session.controlEditions(), + createdAt = now, + owner = community.ownerPubKey, + ), + ) + assertTrue( + session.state.value + ?.authority + ?.isBanned(troll.pubKey) == true, + "the ban must have folded before the heartbeats are judged", + ) + + // The banned member keeps broadcasting — a modified client ignores the send-side guard. + session.ingest(ConcordActions.buildChannelTyping(troll, plane, channelId, community.rootEpoch, now)) + assertEquals( + null, + session.typing.value[channelId]?.get(troll.pubKey.lowercase()), + "a banned member must never be seated in the typing row", + ) + + // The filter is targeted, not a blanket mute: an ordinary member still types normally. + session.ingest(ConcordActions.buildChannelTyping(regular, plane, channelId, community.rootEpoch, now)) + assertTrue( + session.typing.value[channelId]?.containsKey(regular.pubKey.lowercase()) == true, + "an unbanned member's typing heartbeat must still show", + ) + assertEquals( + null, + session.typing.value[channelId]?.get(troll.pubKey.lowercase()), + "seating one member must not drag the banned one in", + ) + } +} From 3153942bac5a007e65d95d7f4d28517f9102a2a6 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Sun, 9 Aug 2026 18:03:28 -0400 Subject: [PATCH 098/132] feat(cli): let amy adopt the Control Plane write key a Grant delivers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A promotion to staff delivers the `control_root` inside the Grant edition itself (CORD-04 §3). Amethyst drains that on its Concord revision tick, but that logic lived only in `AccountConcordActions`, so `amy` could hold a rank it could never write under: the fold seated it as staff and every moderation verb still refused with `forbidden`. That is also why #3873's delivery path shipped without a CLI test — the harness could not accept a promotion. Extracts the decision into `commons` as `ConcordReceive`, pure and shared: - `deliveredControlRoot` — the whole fail-closed check (are we staff by our OWN fold, does a Grant carry a wrap, does it open under the pairwise key, name this epoch, and derive to the `control_pk` we already hold). - `withAdoptedRoot` — the entry rewrite a base rotation produces, banking the leaving epoch's address for the anti-rollback floor. - `isAuthorizedRotator` — the ban-aware rotator check. Amethyst now calls the shared versions (no behaviour change; its persist + publish and Guestbook re-announce stay put). amy adopts during the Control Plane drain every moderation command already performs, since it has no tick of its own, and returns the refreshed record so a freshly promoted staffer can pass the secret on in its own Grant. Adoption is local to amy's store on purpose: Amethyst republishes the kind-13302 list so a user's other devices follow, and doing that here would mean rebuilding and signing the whole list from the CLI. Verified end to end against a loopback geode: bob is refused before the promotion, alice promotes him, bob's stored `control_root` is blank, his next command adopts and persists it, and his BAN lands and is honored by alice's independent fold. A role edition he lacks MANAGE_ROLES for is still dropped on fold — possession remains a spam gate, never authority. Still Android-only: `recoverStrandedConcordCommunities`, which needs invite re-resolution over the network. Co-Authored-By: Claude Opus 5 (1M context) --- .../amethyst/model/AccountConcordActions.kt | 73 ++------- .../amethyst/cli/commands/ConcordCommands.kt | 42 +++++ .../cli/commands/ConcordModCommands.kt | 46 +++++- .../commons/actions/ConcordReceive.kt | 145 ++++++++++++++++++ 4 files changed, 237 insertions(+), 69 deletions(-) create mode 100644 commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordReceive.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt index f659e38564..4caf607709 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt @@ -22,6 +22,7 @@ package com.vitorpamplona.amethyst.model import com.vitorpamplona.amethyst.commons.actions.ConcordActions import com.vitorpamplona.amethyst.commons.actions.ConcordModeration +import com.vitorpamplona.amethyst.commons.actions.ConcordReceive import com.vitorpamplona.amethyst.commons.actions.ConcordSubscriptionPlanner import com.vitorpamplona.amethyst.commons.model.concord.ConcordChannel import com.vitorpamplona.amethyst.commons.model.concord.ConcordCommunitySession @@ -35,17 +36,12 @@ import com.vitorpamplona.quartz.concord.cord02Community.ImagePointer import com.vitorpamplona.quartz.concord.cord03Channels.ChannelChat import com.vitorpamplona.quartz.concord.cord04Roles.AuthorityResolver import com.vitorpamplona.quartz.concord.cord04Roles.ChannelEntity -import com.vitorpamplona.quartz.concord.cord04Roles.ConcordJson import com.vitorpamplona.quartz.concord.cord04Roles.ConcordPermissions -import com.vitorpamplona.quartz.concord.cord04Roles.ControlEntityKind -import com.vitorpamplona.quartz.concord.cord04Roles.ControlRootWrap -import com.vitorpamplona.quartz.concord.cord04Roles.GrantEntity import com.vitorpamplona.quartz.concord.cord04Roles.MetadataEntity import com.vitorpamplona.quartz.concord.cord04Roles.RoleEntity import com.vitorpamplona.quartz.concord.cord05Invites.CommunityInvite import com.vitorpamplona.quartz.concord.cord05Invites.InviteBundleStatus import com.vitorpamplona.quartz.concord.cord05Invites.InviteRelayDictionary -import com.vitorpamplona.quartz.concord.crypto.ConcordKeyDerivation import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys import com.vitorpamplona.quartz.concord.crypto.GroupKey import com.vitorpamplona.quartz.concord.envelope.ConcordStreamEnvelope @@ -932,35 +928,11 @@ class AccountConcordActions( newControlRoot: ByteArray? = null, ) { if (!adoptedConcordRotations.add("${entry.id}:$newEpoch")) return - // The epoch we're leaving is banked with the address it was folded at, so its Control - // Plane stays subscribable for the anti-rollback floor (a split epoch's address can - // never be re-derived, only remembered — CORD-02 §2). - val held = (entry.heldRoots + HeldRoot(entry.rootEpoch, entry.root, entry.controlPk, entry.controlRoot)).distinctBy { it.epoch } - val next = - ConcordCommunityListEntry( - id = entry.id, - owner = entry.owner, - ownerSalt = entry.ownerSalt, - root = newRoot.toHexKey(), - rootEpoch = newEpoch, - // A rotation that delivered no control material is a legacy, pre-split one - // (CORD-06 §3): the new epoch keeps folding at the legacy address, and the - // stale prior-epoch values must NOT be carried into it. - controlPk = newControlPk?.toHexKey(), - controlRoot = newControlRoot?.toHexKey(), - heldRoots = held, - privateChannels = entry.privateChannels, - relays = entry.relays, - name = entry.name, - addedAt = entry.addedAt, - // The invite_ref anchor must survive a rotation, or the *next* Refounding we're left - // out of would be unrecoverable. - inviteRef = entry.inviteRef, - excludedAtEpoch = entry.excludedAtEpoch, - // Unknown keys another client wrote (Armada's list is `[k: string]: unknown`) - // must survive our rotation write, or we delete their data on every rekey. - residue = entry.residue, - ) + // The rewrite itself — banking the leaving epoch's address for the anti-rollback floor, + // dropping stale control material on a legacy rotation, preserving invite_ref and residue — + // is shared with `amy` in [ConcordReceive.withAdoptedRoot]. Only the persist + publish and + // the Guestbook re-announce below are Android's. + val next = ConcordReceive.withAdoptedRoot(entry, newRoot, newEpoch, newControlPk, newControlRoot) account.sendMyPublicAndPrivateOutbox(account.concordChannelList.follow(next)) announceConcordGuestbookJoin(next, inviteCreator = null, inviteLabel = null) } @@ -1030,39 +1002,16 @@ class AccountConcordActions( */ internal suspend fun drainConcordStaffGrants() { if (!account.isWriteable()) return - val me = account.signer.pubKey.lowercase() for (session in account.concordSessions.sessions()) { val entry = session.entry - // Already staff at this epoch, or a legacy community with no split to join. - val heldControlPk = entry.controlPk - if (entry.controlRoot != null || heldControlPk == null) continue val state = session.state.value ?: continue - // Only a Grant our fold honors can deliver: an unauthorized edition hands us nothing. - if (!state.authority.isStaff(me)) continue - - val myGrantCoordinate = - ConcordKeyDerivation - .grantCoordinate(entry.id.hexToByteArray(), me.hexToByteArray()) - .toHexKey() - val delivered = - session - .controlEditions() - .filter { it.entityKind == ControlEntityKind.GRANT && it.entityIdHex == myGrantCoordinate } - // Newest first: a re-issued Grant (a lost key, a head superseded before we - // fetched it) carries the fresher wrap. - .sortedByDescending { it.version } - .firstNotNullOfOrNull { edition -> - val wrap = ConcordJson.decodeOrNull(edition.content)?.controlWrap ?: return@firstNotNullOfOrNull null - val opened = ControlRootWrap.openOrNull(wrap, account.signer, edition.author) ?: return@firstNotNullOfOrNull null - if (opened.epoch != entry.rootEpoch) return@firstNotNullOfOrNull null - // Fails closed: a secret that doesn't derive to the pk we hold is dropped, - // never adopted — we will not split ourselves off from the plane's readers. - if (!ControlRootWrap.derivesTo(opened.controlRoot, entry.id.hexToByteArray(), entry.rootEpoch, heldControlPk)) return@firstNotNullOfOrNull null - opened.controlRoot - } ?: continue + // The whole decision — are we staff, does a Grant carry a wrap, does it open, name our + // epoch, and derive to the control_pk we hold — is shared with `amy` in + // [ConcordReceive.deliveredControlRoot]. Only the persist + publish below is Android's. + val delivered = ConcordReceive.deliveredControlRoot(entry, session.controlEditions(), state.authority, account.signer) ?: continue account.sendMyPublicAndPrivateOutbox( - account.concordChannelList.follow(entry.withControlRoot(delivered.toHexKey())), + account.concordChannelList.follow(entry.withControlRoot(delivered)), ) } } diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt index ff151176ef..689b0f18ee 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt @@ -28,7 +28,12 @@ import com.vitorpamplona.amethyst.cli.stores.ConcordStore import com.vitorpamplona.amethyst.cli.stores.StoredCommunity import com.vitorpamplona.amethyst.cli.stores.StoredHeldRoot import com.vitorpamplona.amethyst.commons.actions.ConcordActions +import com.vitorpamplona.amethyst.commons.actions.ConcordReceive +import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEntry import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEvent +import com.vitorpamplona.quartz.concord.cord04Roles.AuthorityResolver +import com.vitorpamplona.quartz.concord.cord04Roles.ControlEdition +import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray import com.vitorpamplona.quartz.nip01Core.core.toHexKey import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter @@ -316,6 +321,43 @@ object ConcordCommands { controlRoot = sc.controlRoot.ifBlank { null }, ) + /** + * Adopts the `control_root` a staff-making Grant delivered to us (CORD-04 §3), persisting it to + * the local store and returning the now-writable keys — or null when nothing was delivered. + * + * The decision itself is [ConcordReceive.deliveredControlRoot], shared with Amethyst: it fails + * closed unless our own fold seats us as staff, the wrap opens under the granter↔member pairwise + * key, it names this epoch, and the secret derives to exactly the `control_pk` we already hold. + * + * Local-only on purpose: Amethyst republishes the kind-13302 list on adoption so a user's other + * devices follow, and doing that here would need amy to rebuild and sign the whole list. A CLI + * adoption therefore unblocks *this* account's writes; other devices adopt from their own fold. + */ + suspend fun adoptDeliveredControlRoot( + ctx: Context, + dataDir: DataDir, + sc: StoredCommunity, + editions: List, + ): Pair? { + val entry = + ConcordCommunityListEntry( + id = sc.communityId, + owner = sc.owner, + ownerSalt = sc.ownerSalt, + root = sc.root, + rootEpoch = sc.rootEpoch, + controlPk = sc.controlPk.ifBlank { null }, + controlRoot = sc.controlRoot.ifBlank { null }, + relays = sc.relays, + name = sc.name, + ) + val authority = AuthorityResolver.resolve(editions, sc.owner) + val delivered = ConcordReceive.deliveredControlRoot(entry, editions, authority, ctx.signer) ?: return null + val updated = sc.copy(controlRoot = delivered) + ConcordStore(dataDir.concordFile).upsert(updated) + return updated to controlPlaneKeysFor(updated) + } + fun notFound(handle: String): Int { Output.error("not_found", "no joined community matching '$handle' — run `amy concord list`") return 1 diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt index 3633b24dbe..a2d364e8ec 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt @@ -51,7 +51,7 @@ object ConcordModCommands { val sc = ConcordStore(dataDir.concordFile).find(handle) ?: return ConcordCommands.notFound(handle) Context.open(dataDir).use { ctx -> ctx.prepare() - val (_, editions) = load(ctx, sc) + val (_, editions) = load(ctx, sc, dataDir) val state = ConcordCommunityState.fold(editions, sc.owner) Output.emit( mapOf( @@ -92,7 +92,7 @@ object ConcordModCommands { Context.open(dataDir).use { ctx -> ctx.prepare() - val (cp, editions) = load(ctx, sc) + val (cp, editions) = load(ctx, sc, dataDir) writeGuard(cp)?.let { return it } val roleId = RandomInstance.bytes(32) val role = RoleEntity(name = name, position = position, permissions = ConcordPermissions.of(*permBits.toIntArray()).toWire()) @@ -119,7 +119,8 @@ object ConcordModCommands { Context.open(dataDir).use { ctx -> ctx.prepare() val member = ctx.requireUserHex(userRef) - val (cp, editions) = load(ctx, sc) + val loaded = load(ctx, sc, dataDir) + val (cp, editions) = loaded writeGuard(cp)?.let { return it } // A Grant that first makes its member staff must carry the write secret in the same // edition (CORD-04 §3); ConcordModeration wraps it pairwise when the granted roles @@ -134,7 +135,10 @@ object ConcordModCommands { current = editions, createdAt = TimeUtils.now(), owner = sc.owner, - controlRoot = sc.controlRoot.ifBlank { null }?.hexToByteArray(), + controlRoot = + loaded.community.controlRoot + .ifBlank { null } + ?.hexToByteArray(), epoch = sc.rootEpoch, ) val ack = ctx.publish(wrap, ConcordCommands.relaysFor(ctx, sc)) @@ -170,7 +174,7 @@ object ConcordModCommands { Context.open(dataDir).use { ctx -> ctx.prepare() val member = ctx.requireUserHex(userRef) - val (cp, editions) = load(ctx, sc) + val (cp, editions) = load(ctx, sc, dataDir) writeGuard(cp)?.let { return it } val cid = sc.communityId.hexToByteArray() val wrap = @@ -186,11 +190,28 @@ object ConcordModCommands { } } + /** + * The drained Control Plane: the community as stored *after* any adoption, its keys, and the + * editions to chain onto. [community] matters because adopting a delivered `control_root` + * rewrites the stored record — a caller that kept the pre-load copy would then fail to pass the + * secret on in its own Grant (CORD-04 §3). + */ + private class LoadedControl( + val community: StoredCommunity, + val keys: ControlPlaneKeys, + val editions: List, + ) { + operator fun component1() = keys + + operator fun component2() = editions + } + /** Drain the control plane and return its keys + current editions to chain onto. */ private suspend fun load( ctx: Context, sc: StoredCommunity, - ): Pair> { + dataDir: DataDir? = null, + ): LoadedControl { val cp = ConcordCommands.controlPlaneKeysFor(sc) val relays = ConcordCommands.relaysFor(ctx, sc) // Concord relays serve the plane's kind-1059 only to a connection AUTHed as the stream @@ -198,7 +219,18 @@ object ConcordModCommands { // that secret is staff-only (CORD-02 §2), and a member simply has nothing to register. ctx.registerConcordStreamKeys(relays, listOfNotNull(cp.signer?.secretKey)) val wraps = ctx.drain(relays.associateWith { listOf(ConcordActions.planeFilter(cp.address)) }, pendingOnAuthRequired = true).map { it.second } - return cp to ConcordActions.controlEditions(wraps, cp) + val editions = ConcordActions.controlEditions(wraps, cp) + + // A promotion to staff delivers the Control Plane write key inside the Grant itself + // (CORD-04 §3), so the fold that seats the role is also when the key arrives. Amethyst + // drains this on its revision tick; amy has no tick, so the fold a command already does is + // the moment to adopt — otherwise a CLI-promoted staffer holds a rank it can never write + // under. Same shared, fail-closed check both clients use. + if (dataDir != null && !cp.canWrite) { + val adopted = ConcordCommands.adoptDeliveredControlRoot(ctx, dataDir, sc, editions) + if (adopted != null) return LoadedControl(adopted.first, adopted.second, ConcordActions.controlEditions(wraps, adopted.second)) + } + return LoadedControl(sc, cp, editions) } /** diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordReceive.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordReceive.kt new file mode 100644 index 0000000000..b807ae24de --- /dev/null +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordReceive.kt @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.actions + +import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEntry +import com.vitorpamplona.quartz.concord.cord02Community.HeldRoot +import com.vitorpamplona.quartz.concord.cord04Roles.AuthorityResolver +import com.vitorpamplona.quartz.concord.cord04Roles.ConcordJson +import com.vitorpamplona.quartz.concord.cord04Roles.ConcordPermissions +import com.vitorpamplona.quartz.concord.cord04Roles.ControlEdition +import com.vitorpamplona.quartz.concord.cord04Roles.ControlEntityKind +import com.vitorpamplona.quartz.concord.cord04Roles.ControlRootWrap +import com.vitorpamplona.quartz.concord.cord04Roles.GrantEntity +import com.vitorpamplona.quartz.concord.crypto.ConcordKeyDerivation +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray +import com.vitorpamplona.quartz.nip01Core.core.toHexKey +import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner + +/** + * The **receive** half of Concord's key lifecycle, as pure functions: what a client must adopt + * when a Grant hands it the Control Plane write key (CORD-04 §3), and how an entry is rewritten + * when a base rotation moves the community to a new epoch (CORD-06). + * + * These lived only in Amethyst's `AccountConcordActions`, which meant a headless client (`amy`) + * could hold a rank it could never write under, and could not follow a Refounding at all. The + * logic is platform-agnostic — the only Android-shaped parts were the persistence and publish, + * which stay with the caller. Every function here decides *what* to adopt and returns it; the + * caller owns storing it and republishing the kind-13302 list. + * + * Everything fails closed: an undecryptable, mis-epoched or non-deriving delivery yields null, + * never a partially-adopted entry. + */ +object ConcordReceive { + /** + * The `control_root` a staff-making Grant delivered to the account behind [recipientSigner], + * or null when there is nothing to adopt (CORD-04 §3). + * + * Gated three ways, each of which fails closed: + * - only a Grant **our own fold honors** can deliver, so [authority] must already seat us as + * staff — a rogue cannot feed us a key by minting an edition nobody accepts; + * - the wrap must open under the granter↔member pairwise key, and name [entry]'s epoch, + * because compaction re-wraps a Grant head verbatim across Refoundings and a folded head + * can legitimately carry a wrap minted for a prior epoch; + * - the secret must derive to exactly the `control_pk` we already hold, or adopting it would + * split us off from the plane's readers. + * + * Returns null (not an error) when the entry already holds the secret, holds no `control_pk` + * to check against (a legacy pre-split community), or when we are not staff. + */ + suspend fun deliveredControlRoot( + entry: ConcordCommunityListEntry, + editions: List, + authority: AuthorityResolver, + recipientSigner: NostrSigner, + ): HexKey? { + val heldControlPk = entry.controlPk + if (entry.controlRoot != null || heldControlPk == null) return null + val me = recipientSigner.pubKey.lowercase() + if (!authority.isStaff(me)) return null + + val myGrantCoordinate = + ConcordKeyDerivation + .grantCoordinate(entry.id.hexToByteArray(), me.hexToByteArray()) + .toHexKey() + + return editions + .filter { it.entityKind == ControlEntityKind.GRANT && it.entityIdHex == myGrantCoordinate } + // Newest first: a re-issued Grant (a lost key, a head superseded before we fetched it) + // carries the fresher wrap. + .sortedByDescending { it.version } + .firstNotNullOfOrNull { edition -> + val wrap = ConcordJson.decodeOrNull(edition.content)?.controlWrap ?: return@firstNotNullOfOrNull null + val opened = ControlRootWrap.openOrNull(wrap, recipientSigner, edition.author) ?: return@firstNotNullOfOrNull null + if (opened.epoch != entry.rootEpoch) return@firstNotNullOfOrNull null + if (!ControlRootWrap.derivesTo(opened.controlRoot, entry.id.hexToByteArray(), entry.rootEpoch, heldControlPk)) return@firstNotNullOfOrNull null + opened.controlRoot.toHexKey() + } + } + + /** + * Whether [rotator] was allowed to launch the base rotation that [entry] is being moved by + * (CORD-06). `hasPermission`, never `effectivePermissions`: the latter ignores the banlist, so + * a banned BAN-holder could rotate the whole community out from under it. + */ + fun isAuthorizedRotator( + authority: AuthorityResolver, + rotator: HexKey, + ): Boolean = authority.isOwner(rotator) || authority.hasPermission(rotator, ConcordPermissions.BAN) + + /** + * The entry that results from adopting a base rotation to [newEpoch] — a pure rewrite, so the + * caller can diff, persist and publish it however its platform does. + * + * The epoch being left is banked in `heldRoots` **with the address it was folded at**, because + * a split epoch's Control address can never be re-derived, only remembered (CORD-02 §2) — that + * banked address is what keeps the anti-rollback floor rebuildable. A rotation that delivered + * no control material is a legacy pre-split one (CORD-06 §3): the new epoch folds at the legacy + * address, and the stale prior-epoch values must NOT be carried into it. `inviteRef` survives, + * or the *next* Refounding we are left out of becomes unrecoverable; `residue` survives, or we + * delete another client's unknown keys on every rekey. + */ + fun withAdoptedRoot( + entry: ConcordCommunityListEntry, + newRoot: ByteArray, + newEpoch: Long, + newControlPk: ByteArray? = null, + newControlRoot: ByteArray? = null, + ): ConcordCommunityListEntry = + ConcordCommunityListEntry( + id = entry.id, + owner = entry.owner, + ownerSalt = entry.ownerSalt, + root = newRoot.toHexKey(), + rootEpoch = newEpoch, + controlPk = newControlPk?.toHexKey(), + controlRoot = newControlRoot?.toHexKey(), + heldRoots = (entry.heldRoots + HeldRoot(entry.rootEpoch, entry.root, entry.controlPk, entry.controlRoot)).distinctBy { it.epoch }, + privateChannels = entry.privateChannels, + relays = entry.relays, + name = entry.name, + addedAt = entry.addedAt, + inviteRef = entry.inviteRef, + excludedAtEpoch = entry.excludedAtEpoch, + residue = entry.residue, + ) +} From 4022a6a5da0fc08203c12bdc270e82fb49a33e65 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Sun, 9 Aug 2026 18:37:05 -0400 Subject: [PATCH 099/132] feat(cli): add `amy concord recover` for stranded-recovery (CORD-05/06) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the CLI's Concord receive path. A Refounding carries only `(newRoot, newEpoch, rotator)` and no recipient list, so a member simply left out of the rekey receives nothing and sits on the dead epoch forever while everyone else moves on. There is no message to miss, which is why the rekey drain cannot help: the only way back is the invite link the membership was joined through, since the community keeps re-minting its bundle at the same addressable coordinate. amy never stored that anchor, so recovery was impossible in principle. Adds `inviteRef` to the stored record, populated on `join` (bare, domain-agnostic) and carried through `import` — backstopped by what we already held, because a list entry without one must not clear ours or the NEXT exclusion becomes unrecoverable. Recovery is an explicit verb rather than Amethyst's timer sweep, so it stays deterministic and scriptable. Each community reports why it did or didn't move: `no_invite_ref`, `bad_invite_ref`, `no_live_bundle`, `banned`, `already_current`, `control_plane_not_folded`, or the epoch it advanced to. The ban gate is the part that matters (A2 in docs/concord-soft-ban-audit.md): a removed member keeps the link's unlock token forever, so without it this walks them straight back into the epoch they were rotated out of. It reads the banlist of the epoch being LEFT — the last plane we can still fold — and fails closed: a plane that will not fold yields no verdict and is skipped, never recovered. Verified against a loopback geode: a current member gets `already_current`, a community with no anchor gets `no_invite_ref`, and a member banned at the current epoch is refused with `banned`. The merge-forward itself is quartz's `ConcordStrandedRecovery` (already unit-tested); it is not exercised live here because amy cannot perform a Refounding to strand anyone with. Co-Authored-By: Claude Opus 5 (1M context) --- .../amethyst/cli/commands/ConcordCommands.kt | 154 ++++++++++++++++-- .../amethyst/cli/stores/ConcordStore.kt | 5 + 2 files changed, 146 insertions(+), 13 deletions(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt index 689b0f18ee..457be94472 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt @@ -31,8 +31,10 @@ import com.vitorpamplona.amethyst.commons.actions.ConcordActions import com.vitorpamplona.amethyst.commons.actions.ConcordReceive import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEntry import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEvent +import com.vitorpamplona.quartz.concord.cord02Community.HeldRoot import com.vitorpamplona.quartz.concord.cord04Roles.AuthorityResolver import com.vitorpamplona.quartz.concord.cord04Roles.ControlEdition +import com.vitorpamplona.quartz.concord.cord05Invites.InviteBundleStatus import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray import com.vitorpamplona.quartz.nip01Core.core.toHexKey @@ -61,6 +63,9 @@ object ConcordCommands { | [--epoch N] [--root HEX] --epoch/--root read a prior epoch's plane | concord invite COMMUNITY [--base URL] mint + publish a shareable invite link | concord join URL redeem an invite link and save the community + | concord recover [COMMUNITY] re-resolve the joined-through invite link and + | follow a Refounding we were left out of + | (CORD-06); refuses if that epoch banned us | concord roles COMMUNITY list live roles + current banlist (CORD-04) | concord role COMMUNITY NAME POSITION PERM… define a role (perms by name, e.g. BAN KICK) | concord grant COMMUNITY USER ROLE-ID grant a role to a member @@ -75,7 +80,7 @@ object ConcordCommands { route( "concord", tail, - "concord ", + "concord ", help = USAGE, routes = mapOf( @@ -87,6 +92,7 @@ object ConcordCommands { "read" to { rest -> ConcordChannelCommands.read(dataDir, rest) }, "invite" to { rest -> invite(dataDir, rest) }, "join" to { rest -> join(dataDir, rest) }, + "recover" to { rest -> recover(dataDir, rest) }, "roles" to { rest -> ConcordModCommands.roles(dataDir, rest) }, "role" to { rest -> ConcordModCommands.defineRole(dataDir, rest) }, "grant" to { rest -> ConcordModCommands.grant(dataDir, rest) }, @@ -214,6 +220,9 @@ object ConcordCommands { generalChannelId = prior?.generalChannelId ?: "", relays = e.relays, heldRoots = e.heldRoots.map { StoredHeldRoot(it.epoch, it.key, it.controlPk ?: "") }, + // Survives every merge: losing the anchor makes the NEXT exclusion + // unrecoverable, so a list entry without one must not clear ours. + inviteRef = e.inviteRef ?: prior?.inviteRef ?: "", ), ) mapOf( @@ -289,6 +298,9 @@ object ConcordCommands { // community is still pre-split and folds at the legacy address. controlPk = bundle.controlPk ?: "", relays = bundle.relays, + // The stranded-recovery anchor: if a later Refounding leaves us out, re-resolving + // this link is the only way back (CORD-05/06). Stored bare, domain-agnostic. + inviteRef = ConcordActions.bareInviteRef(url) ?: "", ), ) Output.emit(mapOf("community_id" to bundle.communityId, "name" to bundle.name, "relays" to bundle.relays)) @@ -339,18 +351,7 @@ object ConcordCommands { sc: StoredCommunity, editions: List, ): Pair? { - val entry = - ConcordCommunityListEntry( - id = sc.communityId, - owner = sc.owner, - ownerSalt = sc.ownerSalt, - root = sc.root, - rootEpoch = sc.rootEpoch, - controlPk = sc.controlPk.ifBlank { null }, - controlRoot = sc.controlRoot.ifBlank { null }, - relays = sc.relays, - name = sc.name, - ) + val entry = entryFor(sc) val authority = AuthorityResolver.resolve(editions, sc.owner) val delivered = ConcordReceive.deliveredControlRoot(entry, editions, authority, ctx.signer) ?: return null val updated = sc.copy(controlRoot = delivered) @@ -358,6 +359,133 @@ object ConcordCommands { return updated to controlPlaneKeysFor(updated) } + /** The quartz list entry a [StoredCommunity] describes — the shape every commons helper takes. */ + fun entryFor(sc: StoredCommunity) = + ConcordCommunityListEntry( + id = sc.communityId, + owner = sc.owner, + ownerSalt = sc.ownerSalt, + root = sc.root, + rootEpoch = sc.rootEpoch, + controlPk = sc.controlPk.ifBlank { null }, + controlRoot = sc.controlRoot.ifBlank { null }, + heldRoots = sc.heldRoots.map { HeldRoot(it.epoch, it.root, it.controlPk.ifBlank { null }) }, + relays = sc.relays, + name = sc.name, + inviteRef = sc.inviteRef.ifBlank { null }, + ) + + /** Folds [entry] back into the stored shape after a rotation is adopted. */ + fun storedFrom( + sc: StoredCommunity, + entry: ConcordCommunityListEntry, + ) = sc.copy( + root = entry.root, + rootEpoch = entry.rootEpoch, + controlPk = entry.controlPk ?: "", + controlRoot = entry.controlRoot ?: "", + heldRoots = entry.heldRoots.map { StoredHeldRoot(it.epoch, it.key, it.controlPk ?: "") }, + relays = entry.relays, + name = entry.name.ifBlank { sc.name }, + inviteRef = entry.inviteRef ?: sc.inviteRef, + ) + + /** + * `concord recover [COMMUNITY]` — the stranded-recovery receive path (CORD-05/06 A2). + * + * A Refounding carries only `(newRoot, newEpoch, rotator)` and **no recipient list**, so a + * member simply left out of the rekey receives nothing and sits on the dead epoch forever while + * everyone else moves on. There is no message to miss, which is why the rekey drain cannot help. + * The way back is the invite link the membership was joined through: the community keeps + * re-minting its bundle at the same addressable coordinate, so a live bundle at a **strictly + * higher** epoch than ours proves we were left behind — and carries the new root. + * + * Amethyst sweeps this on a timer; amy makes it an explicit verb, so it stays deterministic and + * scriptable rather than a background loop. + * + * The ban gate is the point of care. A removed member keeps the link's unlock token forever, so + * without it this walks them straight back into the epoch they were rotated out of. It reads the + * banlist of the epoch we are **leaving** (the last Control Plane we can still fold) and **fails + * closed**: a community whose plane will not fold yields no verdict and is skipped, never + * recovered. + */ + private suspend fun recover( + dataDir: DataDir, + rest: Array, + ): Int { + val args = Args(rest) + val handle = args.positionalOrNull(0) + args.rejectUnknown() + val store = ConcordStore(dataDir.concordFile) + val targets = + if (handle != null) { + listOf(store.find(handle) ?: return notFound(handle)) + } else { + store.load() + } + + Context.open(dataDir).use { ctx -> + ctx.prepare() + val results = mutableListOf>() + for (sc in targets) { + val inviteRef = sc.inviteRef.ifBlank { null } + if (inviteRef == null) { + results += mapOf("community_id" to sc.communityId, "name" to sc.name, "recovered" to false, "reason" to "no_invite_ref") + continue + } + val parsed = ConcordActions.parseInviteLink(inviteRef) + if (parsed == null) { + results += mapOf("community_id" to sc.communityId, "name" to sc.name, "recovered" to false, "reason" to "bad_invite_ref") + continue + } + val relays = (normalize(parsed.fragment.relays) + normalize(sc.relays)).ifEmpty { ctx.outboxRelays() } + val wraps = ctx.drain(relays.associateWith { listOf(ConcordActions.bundleFilter(parsed.linkSignerPubKey)) }).map { it.second } + // Only a LIVE bundle recovers: an expired or revoked link is not a rotation we missed. + val bundle = (ConcordActions.classifyInvite(wraps, parsed.fragment.token) as? InviteBundleStatus.Live)?.invite + if (bundle == null) { + results += mapOf("community_id" to sc.communityId, "name" to sc.name, "recovered" to false, "reason" to "no_live_bundle") + continue + } + + // Fold the epoch we are leaving to learn whether it banned us. No fold, no verdict, + // no recovery — the gate fails closed rather than assuming "not banned". + val cp = controlPlaneKeysFor(sc) + ctx.registerConcordStreamKeys(relays, listOfNotNull(cp.signer?.secretKey)) + val controlWraps = ctx.drain(relays.associateWith { listOf(ConcordActions.planeFilter(cp.address)) }, pendingOnAuthRequired = true).map { it.second } + val editions = ConcordActions.controlEditions(controlWraps, cp) + if (editions.isEmpty()) { + results += mapOf("community_id" to sc.communityId, "name" to sc.name, "recovered" to false, "reason" to "control_plane_not_folded") + continue + } + val bannedHere = AuthorityResolver.resolve(editions, sc.owner).isBanned(ctx.signer.pubKey) + + val merged = ConcordActions.recoverStranded(entryFor(sc), bundle, bannedHere) + if (merged == null) { + results += + mapOf( + "community_id" to sc.communityId, + "name" to sc.name, + "recovered" to false, + "reason" to if (bannedHere) "banned" else "already_current", + "root_epoch" to sc.rootEpoch, + ) + continue + } + store.upsert(storedFrom(sc, merged)) + results += + mapOf( + "community_id" to sc.communityId, + "name" to sc.name, + "recovered" to true, + "from_epoch" to sc.rootEpoch, + "root_epoch" to merged.rootEpoch, + ) + } + Output.emit(mapOf("communities" to results)) + return 0 + } + } + fun notFound(handle: String): Int { Output.error("not_found", "no joined community matching '$handle' — run `amy concord list`") return 1 diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt index 7ae731a877..d2de85a258 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt @@ -48,6 +48,11 @@ data class StoredCommunity( // Past access roots kept per epoch (CORD-06 Refounding rotates the root). Lets `read --epoch ` // re-derive a prior epoch's Chat Plane to reach pre-refounding history. Populated by `import`. val heldRoots: List = emptyList(), + // The bare `#` invite this membership was joined through — the stranded-recovery + // anchor (CORD-05/06). A Refounding carries no recipient list, so a member simply left out of the + // rekey has no message to miss: re-resolving this link is the only way back. Blank for a direct + // invite or a community joined before amy stored it. + val inviteRef: String = "", ) /** A past community_root for a specific epoch, mirroring quartz `HeldRoot`. */ From b7f55d86971f0d74a8865e45f8712cbd0248bd04 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 22:44:46 +0000 Subject: [PATCH 100/132] chore: add a runtime perf probe for the embedded vs full-screen WebView MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The embedded tab and the full-screen browser are the same WebView in the same `:napplet` process with byte-identical WebSettings, so a site whose JS feels slower in the embed is being slowed by the host, not by its configuration. `perf.html` measures which host effect it is: page visibility (a page Chromium treats as hidden gets ~1Hz timers and no rAF), raw CPU throughput (the renderer inherits its scheduling class from whichever process hosts the WebView — the embed's is a plain bound service, the full-screen one is top-app), forced-layout cost, rAF rate, long tasks, and input-delivery latency measured from the platform's own event timestamp. Open the same URL in both hosts and compare the summary line; the README says what each divergence points at. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01AC3ambee9KFcvHCS6HRqhS --- tools/ime-test/README.md | 42 ++++++++ tools/ime-test/perf.html | 206 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 248 insertions(+) create mode 100644 tools/ime-test/perf.html diff --git a/tools/ime-test/README.md b/tools/ime-test/README.md index 816bd030aa..8eb2fb00fe 100644 --- a/tools/ime-test/README.md +++ b/tools/ime-test/README.md @@ -52,3 +52,45 @@ live only under `tools/`. low now that the surface no longer resizes on IME show). - `MAINTHREAD BLOCKED` / `LONGTASK` = something is stalling the WebView thread. - `HEARTBEAT` lines changing while idle = spontaneous focus/selection drift. + +## `perf.html` — why does the embed feel slower than the full-screen browser? + +`index.html` profiles the IME relay. `perf.html` answers a different question: +the embedded tab and the full-screen browser are the **same WebView in the same +`:napplet` process** with byte-identical `WebSettings`, so when a site's JS feels +slower in the embed, the cause is host-induced — and this page measures which +host effect it is. + +Serve the directory (above) and open **the same URL in both hosts**, then compare +the summary line at the bottom of the page: + +- **`vis=hidden`** — decisive. Chromium considers the embedded page hidden, so it + clamps timers to ~1Hz and suspends `requestAnimationFrame`. Everything the site + schedules lands late; it reads as "the JS got slow". Confirmed by + `timer50` (a 50ms interval firing at 500-1000ms) and `raf` (0 fps). +- **`vis=visible` but `cpu` is 2-4× the full-screen number** — the process is + running on the little cores. The site's JS runs in the WebView *renderer* + process, whose scheduling class is inherited from its host: `:napplet` is + `top-app` when it fronts the full-screen activity, but only a bound service + (`BIND_AUTO_CREATE`, no `BIND_IMPORTANT`) when it serves the embed. Cross-check + off-device with: + + ```bash + adb shell dumpsys activity processes | grep -E 'napplet|sandboxed' + adb shell "cat /proc/$(adb shell pidof com.vitorpamplona.amethyst:napplet)/cgroup" + ``` + + Expect `/top-app` with the full-screen browser open and `/foreground` (or lower) + with an embed tab open. +- **`cpu` matches but `inputDelivery` is much higher** — the gap is input routing + into the embedded window, not compute. `inputDelivery` is the time between the + platform stamping the touch and JS receiving it. +- **`layout` much higher in the embed** — layout/paint is the bottleneck (check + logcat for WebView software-rendering warnings; a non-hardware-accelerated + `SurfaceControlViewHost` window would put Chromium on the software path). +- **`focus=false` in the embed is expected** and is not itself a throttle: the + host window owns the keyboard, which is the whole reason `RemoteImeView` exists. + +`longtasks` counts main-thread blocks over 50ms while the page was measuring — +high counts in the embed with a matching `cpu` number point at something else in +the process competing (e.g. parked warm tabs that are never paused). diff --git a/tools/ime-test/perf.html b/tools/ime-test/perf.html new file mode 100644 index 0000000000..b1efbb4d78 --- /dev/null +++ b/tools/ime-test/perf.html @@ -0,0 +1,206 @@ + + + + + +Embed vs direct — runtime perf probe + + + +

Runtime perf probe

+

Open this same URL twice — once as an embedded tab, once in the full-screen browser — and compare. Both are the same WebView in the same process; any gap is host-induced.

+ + + +
+ + + + + + + + From 0aa3adfc07d26daa9047ec13cde57685a0ca4dbd Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Sun, 9 Aug 2026 18:48:28 -0400 Subject: [PATCH 101/132] =?UTF-8?q?feat(cli):=20add=20`amy=20concord=20ref?= =?UTF-8?q?ound`=20+=20`rekey`=20=E2=80=94=20the=20rotation=20half=20of=20?= =?UTF-8?q?CORD-06?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `refound` is the hard removal a ban cannot give: a ban only strips standing, while the removed member keeps every key they ever held. Rotating the `community_root` — and, since CORD-02 §2, a fresh `control_root` beside it so a demoted staffer's retained secret dies with the epoch — is what actually closes the room. The compacted Control Plane is re-sealed at the new epoch and each retained member gets a rekey blob. Authority mirrors Amethyst exactly: `hasPermission`, never `effectivePermissions`, so a banned BAN-holder cannot launch one; the owner is never a valid target; and removal takes the same rank rule as a ban (CORD-04 §3) — an admin cannot Refound a peer admin out. The recipient set reaches past the roster to the Guestbook AND the authors of every channel message we can decrypt, because a member who only ever posted holds no role and files no Guestbook motion — building the set without them silently expels them. It is still a floor, not a census. `rekey` is the receive half, and without it `refound` was actively harmful from the CLI: a retained member's blob sat on the relay unopened, so a Refounding launched from amy stranded every other amy member. It authorizes the rotator against the roster of the epoch being LEFT and fails closed. Verified end to end against a loopback geode — the full cycle, which was not previously expressible from the CLI at all: alice refound --remove → epoch 0 → 1, recipients=2 (bob is kept because he POSTED, holding no role — the author harvest) bob rekey → epoch 0 → 1, same root + control_pk bob sends, alice reads it at the new epoch alice roles → the removed member is banned Co-Authored-By: Claude Opus 5 (1M context) --- .../amethyst/cli/commands/ConcordCommands.kt | 71 +++++++- .../cli/commands/ConcordModCommands.kt | 171 ++++++++++++++++++ 2 files changed, 241 insertions(+), 1 deletion(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt index 457be94472..1e299b4191 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt @@ -63,6 +63,8 @@ object ConcordCommands { | [--epoch N] [--root HEX] --epoch/--root read a prior epoch's plane | concord invite COMMUNITY [--base URL] mint + publish a shareable invite link | concord join URL redeem an invite link and save the community + | concord rekey [COMMUNITY] follow a Refounding we were re-keyed for: + | open our blob and adopt the new epoch | concord recover [COMMUNITY] re-resolve the joined-through invite link and | follow a Refounding we were left out of | (CORD-06); refuses if that epoch banned us @@ -71,6 +73,9 @@ object ConcordCommands { | concord grant COMMUNITY USER ROLE-ID grant a role to a member | concord ban COMMUNITY USER ban a member | concord unban COMMUNITY USER unban a member + | concord refound COMMUNITY --remove U[,U] CORD-06 Refounding: rotate the root (and the + | control_root) so removed members lose every + | key — the hard removal a ban cannot give """.trimMargin() suspend fun dispatch( @@ -80,7 +85,7 @@ object ConcordCommands { route( "concord", tail, - "concord ", + "concord ", help = USAGE, routes = mapOf( @@ -93,11 +98,13 @@ object ConcordCommands { "invite" to { rest -> invite(dataDir, rest) }, "join" to { rest -> join(dataDir, rest) }, "recover" to { rest -> recover(dataDir, rest) }, + "rekey" to { rest -> rekey(dataDir, rest) }, "roles" to { rest -> ConcordModCommands.roles(dataDir, rest) }, "role" to { rest -> ConcordModCommands.defineRole(dataDir, rest) }, "grant" to { rest -> ConcordModCommands.grant(dataDir, rest) }, "ban" to { rest -> ConcordModCommands.ban(dataDir, rest) }, "unban" to { rest -> ConcordModCommands.unban(dataDir, rest) }, + "refound" to { rest -> ConcordModCommands.refound(dataDir, rest) }, ), ) @@ -486,6 +493,68 @@ object ConcordCommands { } } + /** + * `concord rekey [COMMUNITY]` — follow a Refounding we WERE re-keyed for (CORD-06). + * + * The normal counterpart to [recover]: a retained member gets a per-recipient blob on the next + * epoch's base-rekey plane, and opening it yields the new root. Amethyst drains this on its + * revision tick; amy has no tick, so it is a verb. Without it a Refounding launched from the CLI + * strands every other CLI member even though their blob is sitting on the relay. + * + * The rotator is authorized against the roster of the epoch being **left** — `hasPermission`, + * never `effectivePermissions`, so a banned BAN-holder cannot rotate us (CORD-06). Fails closed: + * a plane that will not fold yields no verdict and the community is skipped. + */ + private suspend fun rekey( + dataDir: DataDir, + rest: Array, + ): Int { + val args = Args(rest) + val handle = args.positionalOrNull(0) + args.rejectUnknown() + val store = ConcordStore(dataDir.concordFile) + val targets = if (handle != null) listOf(store.find(handle) ?: return notFound(handle)) else store.load() + + Context.open(dataDir).use { ctx -> + ctx.prepare() + val results = mutableListOf>() + for (sc in targets) { + val relays = relaysFor(ctx, sc) + val baseRekey = ConcordActions.nextBaseRekeyPlane(sc.root.hexToByteArray(), sc.communityId.hexToByteArray(), sc.rootEpoch) + ctx.registerConcordStreamKeys(relays, listOf(baseRekey.secretKey)) + val wraps = ctx.drain(relays.associateWith { listOf(ConcordActions.planeFilter(baseRekey.publicKeyHex)) }, pendingOnAuthRequired = true).map { it.second } + val received = + ConcordActions.openBaseRekey(wraps, baseRekey, ctx.signer, sc.communityId, sc.root.hexToByteArray(), sc.rootEpoch) + if (received == null) { + results += mapOf("community_id" to sc.communityId, "name" to sc.name, "rekeyed" to false, "reason" to "no_blob_for_us", "root_epoch" to sc.rootEpoch) + continue + } + if (received.newEpoch <= sc.rootEpoch) { + results += mapOf("community_id" to sc.communityId, "name" to sc.name, "rekeyed" to false, "reason" to "already_current", "root_epoch" to sc.rootEpoch) + continue + } + // Authorize the rotator against the epoch we are LEAVING — the last plane we can fold. + val cp = controlPlaneKeysFor(sc) + ctx.registerConcordStreamKeys(relays, listOfNotNull(cp.signer?.secretKey)) + val controlWraps = ctx.drain(relays.associateWith { listOf(ConcordActions.planeFilter(cp.address)) }, pendingOnAuthRequired = true).map { it.second } + val editions = ConcordActions.controlEditions(controlWraps, cp) + if (editions.isEmpty()) { + results += mapOf("community_id" to sc.communityId, "name" to sc.name, "rekeyed" to false, "reason" to "control_plane_not_folded") + continue + } + if (!ConcordReceive.isAuthorizedRotator(AuthorityResolver.resolve(editions, sc.owner), received.rotator)) { + results += mapOf("community_id" to sc.communityId, "name" to sc.name, "rekeyed" to false, "reason" to "unauthorized_rotator", "rotator" to received.rotator) + continue + } + val adopted = ConcordReceive.withAdoptedRoot(entryFor(sc), received.newRoot, received.newEpoch, received.newControlPk, received.newControlRoot) + store.upsert(storedFrom(sc, adopted)) + results += mapOf("community_id" to sc.communityId, "name" to sc.name, "rekeyed" to true, "from_epoch" to sc.rootEpoch, "root_epoch" to received.newEpoch, "rotator" to received.rotator) + } + Output.emit(mapOf("communities" to results)) + return 0 + } + } + fun notFound(handle: String): Int { Output.error("not_found", "no joined community matching '$handle' — run `amy concord list`") return 1 diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt index a2d364e8ec..a9e8b0d2d8 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt @@ -28,6 +28,7 @@ import com.vitorpamplona.amethyst.cli.stores.ConcordStore import com.vitorpamplona.amethyst.cli.stores.StoredCommunity import com.vitorpamplona.amethyst.commons.actions.ConcordActions import com.vitorpamplona.amethyst.commons.actions.ConcordModeration +import com.vitorpamplona.amethyst.commons.actions.ConcordReceive import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityState import com.vitorpamplona.quartz.concord.cord04Roles.ConcordPermissions import com.vitorpamplona.quartz.concord.cord04Roles.ControlEdition @@ -206,6 +207,176 @@ object ConcordModCommands { operator fun component2() = editions } + /** + * `concord refound COMMUNITY --remove USER[,USER…]` — a CORD-06 Refounding: the hard removal. + * + * A ban only strips standing; the removed member keeps every key they ever held, so the room is + * only truly closed to them by rotating the `community_root` (and, since CORD-02 §2, a fresh + * `control_root` beside it, so a demoted staffer's retained secret dies with the epoch). The + * compacted Control Plane is re-sealed at the new epoch and each retained member gets a rekey + * blob; nobody else can follow. + * + * Authority mirrors Amethyst exactly: `hasPermission`, never `effectivePermissions`, so a banned + * BAN-holder cannot launch one; the owner is never a valid target; and removal takes the same + * rank rule as a ban (CORD-04 §3) — an admin cannot Refound a peer admin out. + * + * **The recipient set is a floor, not a census.** It is the roster ∪ Guestbook ∪ the authors of + * every channel message we can decrypt ∪ ourselves, minus the removed and already-banned — the + * same union Amethyst builds, because a member who only ever posted holds no role and leaves no + * Guestbook motion, and omitting them silently expels them. A member with no trace at all still + * cannot be re-keyed; `concord recover` is how they get back. + */ + suspend fun refound( + dataDir: DataDir, + rest: Array, + ): Int { + val args = Args(rest) + val handle = args.positional(0, "community") + val removeArg = args.flag("remove") ?: return Output.error("bad_args", "refound --remove USER[,USER…]").let { 2 } + args.rejectUnknown() + val sc = ConcordStore(dataDir.concordFile).find(handle) ?: return ConcordCommands.notFound(handle) + + Context.open(dataDir).use { ctx -> + ctx.prepare() + val removed = + removeArg + .split(',') + .map { it.trim() } + .filter { it.isNotEmpty() } + .map { ctx.requireUserHex(it).lowercase() } + .toSet() + if (removed.isEmpty()) return Output.error("bad_args", "--remove needs at least one user") + + val loaded = load(ctx, sc, dataDir) + val (cp, editions) = loaded + val state = ConcordCommunityState.fold(editions, sc.owner) + val authority = state.authority + val me = ctx.signer.pubKey + + if (!ConcordReceive.isAuthorizedRotator(authority, me)) { + return Output.error("forbidden", "this account cannot refound: a Refounding takes BAN (or ownership), and a banned holder is refused (CORD-06)") + } + if (removed.any { authority.isOwner(it) }) { + return Output.error("forbidden", "the owner is never a valid removal target (CORD-04 §3)") + } + // An admin cannot Refound a peer admin out any more than they could ban one. + if (!authority.isOwner(me) && removed.any { !authority.canActOn(me, it, ConcordPermissions.BAN) }) { + return Output.error("forbidden", "you do not outrank every member you are removing (CORD-04 §3, equal cannot act on equal)") + } + // A Refounding writes the current plane (the pre-rotation bans) and the new one, so on a + // split epoch it takes the current control_root (CORD-02 §2). + writeGuard(cp)?.let { return it } + + val relays = ConcordCommands.relaysFor(ctx, sc) + + // 1. Ban the removed on the CURRENT plane, so the compacted snapshot — and therefore the + // new epoch — carries the ban. Each edition chains onto the updated banlist head. + var chain = editions + for (target in removed) { + val banWrap = ConcordModeration.ban(ctx.signer, cp, sc.communityId.hexToByteArray(), target, chain, TimeUtils.now(), owner = sc.owner) + ctx.publish(banWrap, relays) + chain = chain + (ConcordActions.controlEditions(listOf(banWrap), cp)) + } + + // 2. Everyone we are keeping. See the note above on why this reaches past the roster. + val recipients = + (rosterOf(authority) + guestbookMembersOf(ctx, sc) + channelAuthorsOf(ctx, sc, state) + me) + .mapTo(HashSet()) { it.lowercase() } + .apply { + removeAll(removed) + removeAll(authority.bannedMembers().map { it.lowercase() }.toSet()) + }.toList() + + // 3. Build: new root + fresh control_root, compacted plane, per-recipient blobs (staff + // get the 136-byte form carrying the secret, everyone else the 104-byte pubkey one). + val newRoot = RandomInstance.bytes(32) + val newControlRoot = RandomInstance.bytes(32) + val controlWraps = ctx.drain(relays.associateWith { listOf(ConcordActions.planeFilter(cp.address)) }, pendingOnAuthRequired = true).map { it.second } + val build = + ConcordActions.buildRefounding( + rotatorSigner = ctx.signer, + communityId = sc.communityId, + priorRoot = sc.root.hexToByteArray(), + newRoot = newRoot, + newControlRoot = newControlRoot, + rootEpoch = sc.rootEpoch, + priorControlWraps = controlWraps, + priorControlKeys = cp, + recipientsXOnly = recipients, + staffXOnly = authority.staffMembers(), + createdAt = TimeUtils.now(), + ownerPubKey = sc.owner, + ) + + // 4. The compacted plane (the new epoch's state) then the blobs (the key that opens it). + build.controlWraps.forEach { ctx.publish(it, relays) } + build.rekeyWraps.forEach { ctx.publish(it, relays) } + + // 5. Adopt the new epoch ourselves — the same pure rewrite Amethyst uses, banking the + // epoch we are leaving for the anti-rollback floor. + val adopted = + ConcordReceive.withAdoptedRoot( + ConcordCommands.entryFor(loaded.community), + newRoot, + build.newEpoch, + build.newControlKeys.address.hexToByteArray(), + newControlRoot, + ) + ConcordStore(dataDir.concordFile).upsert(ConcordCommands.storedFrom(loaded.community, adopted)) + + Output.emit( + mapOf( + "community_id" to sc.communityId, + "removed" to removed.toList(), + "from_epoch" to sc.rootEpoch, + "root_epoch" to build.newEpoch, + "recipients" to recipients.size, + "control_wraps" to build.controlWraps.size, + "rekey_wraps" to build.rekeyWraps.size, + ), + ) + return 0 + } + } + + /** Owner + everyone holding a role — owner-rooted, so it cannot be padded from outside. */ + private fun rosterOf(authority: com.vitorpamplona.quartz.concord.cord04Roles.AuthorityResolver): Set = (authority.roleHolders() + authority.staffMembers()).mapTo(HashSet()) { it.lowercase() } + + /** Live Guestbook membership at this epoch (joins minus later leaves, CORD-02 §5). */ + private suspend fun guestbookMembersOf( + ctx: Context, + sc: StoredCommunity, + ): Set = + runCatching { + val gb = ConcordActions.guestbookPlane(sc.root.hexToByteArray(), sc.communityId.hexToByteArray(), sc.rootEpoch) + val relays = ConcordCommands.relaysFor(ctx, sc) + ctx.registerConcordStreamKeys(relays, listOf(gb.secretKey)) + val wraps = ctx.drain(relays.associateWith { listOf(ConcordActions.planeFilter(gb.publicKeyHex)) }, pendingOnAuthRequired = true).map { it.second } + ConcordActions.guestbookMembers(wraps, gb).mapTo(HashSet()) { it.lowercase() } + }.getOrDefault(emptySet()) + + /** + * Authors of every channel message we can decrypt. Most members never send a Guestbook motion, + * so without this a Refounding silently expels everyone who had only ever posted. + */ + private suspend fun channelAuthorsOf( + ctx: Context, + sc: StoredCommunity, + state: ConcordCommunityState, + ): Set { + val out = HashSet() + val relays = ConcordCommands.relaysFor(ctx, sc) + for ((channelIdHex, _) in state.channels) { + runCatching { + val key = ConcordActions.publicChannel(sc.root.hexToByteArray(), channelIdHex.hexToByteArray(), sc.rootEpoch) + ctx.registerConcordStreamKeys(relays, listOf(key.secretKey)) + val wraps = ctx.drain(relays.associateWith { listOf(ConcordActions.planeFilter(key.publicKeyHex)) }, pendingOnAuthRequired = true).map { it.second } + ConcordActions.channelMessages(wraps, key, channelIdHex, sc.rootEpoch).mapTo(out) { it.author.lowercase() } + } + } + return out + } + /** Drain the control plane and return its keys + current editions to chain onto. */ private suspend fun load( ctx: Context, From 409339b375d34666be96a8d215f413efe52decd3 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Sun, 9 Aug 2026 19:06:39 -0400 Subject: [PATCH 102/132] feat(concord): re-mint invite links on Refounding so stranded recovery fires MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the liveness half of A2. Recovery's whole premise is that the community keeps re-minting its bundle at the SAME addressable coordinate, so the link a stranded member already holds starts pointing at the new epoch. Nothing did: `ConcordInviteBundle.mintLink` generates a fresh KeyPair and token per call, and no client persisted `linkSignerPrivKey`. Every mint was a new coordinate, so `recover` could only ever return `already_current` — the mechanism was dead code, and an owner evicted by a rogue admin had no way back. The kind-33301 bundle is addressable and authored by the link signer, so re-signing at that coordinate with the same token replaces what is there and every holder of that link keeps working. Exposes that as `ConcordActions.remintBundleAt`, persists the link signer + token in amy's store at mint time, and has `concord refound` refresh every link it minted for the new epoch. Re-minting every live link is safe precisely because the security half is already in: `refound` bans the removed members on the way out, and `recover` reads the banlist of the epoch being LEFT, so a removed member's own recovery is refused even though their link now resolves. That gate stops being belt-and-braces here and becomes load-bearing — which is what the audit predicted for any client that re-mints (Armada does). Verified end to end against a loopback geode, both directions: bob joins by link, holds no role, never posts (unfindable by a rotation) alice refound --remove → recipients=1, invites_refreshed=1 bob rekey → no_blob_for_us (genuinely stranded) bob recover → recovered, epoch 0 → 1 ← first time this has ever fired bob reads the community at the new epoch alice refound --remove bob → epoch 1 → 2, invites_refreshed=1 bob recover → refused, reason "banned", still at epoch 1 Still open for the shipping client: Amethyst persists no link signer, so A2 liveness remains open on Android. Doing it there means deciding where the secret lives in the kind-13302 list, which Armada also reads — a wire-schema call, not a code one. Co-Authored-By: Claude Opus 5 (1M context) --- .../amethyst/cli/commands/ConcordCommands.kt | 15 +++++++ .../cli/commands/ConcordModCommands.kt | 40 ++++++++++++++++++- .../amethyst/cli/stores/ConcordStore.kt | 16 ++++++++ .../commons/actions/ConcordActions.kt | 23 +++++++++++ 4 files changed, 93 insertions(+), 1 deletion(-) diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt index 1e299b4191..3e06aa5b72 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt @@ -27,6 +27,7 @@ import com.vitorpamplona.amethyst.cli.Output import com.vitorpamplona.amethyst.cli.stores.ConcordStore import com.vitorpamplona.amethyst.cli.stores.StoredCommunity import com.vitorpamplona.amethyst.cli.stores.StoredHeldRoot +import com.vitorpamplona.amethyst.cli.stores.StoredMintedInvite import com.vitorpamplona.amethyst.commons.actions.ConcordActions import com.vitorpamplona.amethyst.commons.actions.ConcordReceive import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEntry @@ -265,6 +266,20 @@ object ConcordCommands { val ack = ctx.publish(minted.bundleEvent, relaysFor(ctx, sc)) RawEventSupport.publishGuard(ack, minted.bundleEvent.id)?.let { return it } + // Keep the link signer + token so a later Refounding can refresh THIS coordinate rather + // than orphaning the link at a dead epoch — the liveness half of stranded recovery (A2). + ConcordStore(dataDir.concordFile).upsert( + sc.copy( + mintedInvites = + sc.mintedInvites + + StoredMintedInvite( + linkSignerPrivKey = minted.linkSignerPrivKey.toHexKey(), + token = minted.token.toHexKey(), + createdAt = TimeUtils.now(), + ), + ), + ) + Output.emit( mapOf( "url" to minted.url, diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt index a9e8b0d2d8..342ae2501d 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt @@ -322,7 +322,44 @@ object ConcordModCommands { build.newControlKeys.address.hexToByteArray(), newControlRoot, ) - ConcordStore(dataDir.concordFile).upsert(ConcordCommands.storedFrom(loaded.community, adopted)) + val stored = ConcordCommands.storedFrom(loaded.community, adopted) + ConcordStore(dataDir.concordFile).upsert(stored) + + // 6. Refresh every link we minted, at its OWN coordinate, so it now resolves to the new + // epoch. This is the liveness half of stranded recovery (A2): a member this Refounding + // left out has no rekey blob and no message to miss, so re-resolving their link is the + // only way back — and it only works if the bundle moves with the community instead of + // being orphaned at a dead epoch. Minting a fresh link would not help them; the link + // they hold is the one that must move. + // + // Safe for every link because recovery is ban-gated at the epoch being left, and step 1 + // banned everyone being removed — so a removed member's own `recover` is refused even + // though their link now resolves. + val refreshedInvite = + ConcordActions.inviteFor( + stored.communityId, + stored.owner, + stored.ownerSalt, + stored.root, + stored.rootEpoch, + stored.name, + stored.relays, + stored.controlPk.ifBlank { null }, + ) + var refreshed = 0 + for (link in stored.mintedInvites) { + runCatching { + val event = + ConcordActions.remintBundleAt( + linkSignerPrivKey = link.linkSignerPrivKey.hexToByteArray(), + token = link.token.hexToByteArray(), + invite = refreshedInvite, + createdAt = TimeUtils.now(), + ) + ctx.publish(event, relays) + refreshed++ + } + } Output.emit( mapOf( @@ -333,6 +370,7 @@ object ConcordModCommands { "recipients" to recipients.size, "control_wraps" to build.controlWraps.size, "rekey_wraps" to build.rekeyWraps.size, + "invites_refreshed" to refreshed, ), ) return 0 diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt index d2de85a258..e08bfcc8f8 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt @@ -53,6 +53,22 @@ data class StoredCommunity( // rekey has no message to miss: re-resolving this link is the only way back. Blank for a direct // invite or a community joined before amy stored it. val inviteRef: String = "", + // Invite links WE minted for this community, kept so a Refounding can re-publish each bundle at + // its own coordinate for the new epoch. Without this the link a member joined through points at + // a dead epoch forever and stranded recovery can never fire (A2). Holds link-signer secrets, so + // it sits beside `root`/`controlRoot` in the same already-secret file. + val mintedInvites: List = emptyList(), +) + +/** + * One invite link this account minted: enough to re-sign at its addressable coordinate later. The + * coordinate is the link signer's pubkey, so keeping the private key is what lets a Refounding + * refresh the link (and, in future, revoke it) instead of orphaning it. + */ +data class StoredMintedInvite( + val linkSignerPrivKey: String = "", + val token: String = "", + val createdAt: Long = 0, ) /** A past community_root for a specific epoch, mirroring quartz `HeldRoot`. */ diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt index e70fe18364..6263c65add 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt @@ -436,6 +436,29 @@ object ConcordActions { relays: List? = null, ): MintedInviteLink = ConcordInviteBundle.mintLink(base, invite, createdAt, relays) + /** + * Re-publishes a bundle at an **existing** link's coordinate, carrying [invite] refreshed for the + * current epoch (CORD-05 §1). The kind-33301 bundle is addressable and authored by the link + * signer, so re-signing with the same [linkSignerPrivKey] and re-encrypting under the same + * [token] replaces what is there — every holder of that link keeps working, now pointing at the + * new root. + * + * This is what makes stranded recovery live: a member a Refounding left out has no rekey blob and + * no message to miss, and re-resolving their link is the only way back — which requires the + * community to re-mint at the *same* coordinate rather than issuing a fresh link. Minting a new + * link leaves the old one pointing at a dead epoch forever. + * + * Safe to call for every live link because recovery is ban-gated at the epoch being left + * (CORD-06, A2): a member the Refounding removed was banned on the way out, so their own + * `recover` is refused even though their link now resolves. + */ + fun remintBundleAt( + linkSignerPrivKey: ByteArray, + token: ByteArray, + invite: CommunityInvite, + createdAt: Long, + ): Event = ConcordInviteBundle.build(linkSignerPrivKey, token, invite, createdAt) + /** Parses a shareable invite URL into its pointer + private fragment. */ fun parseInviteLink(url: String): ParsedInviteLink? = ConcordInviteLink.parseUrl(url) From 3b7ffcd06c13ee7405ae3f1552e419775f702b0e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 23:38:18 +0000 Subject: [PATCH 103/132] Make a paged walk terminate: floor the cursor at 0, stop when a relay ignores it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `fetchAllPages` could not end against a relay that does not honour `until`. Found in production on purplepag.es, which holds twelve `kind 10002` events stamped `created_at = 0` and treats `until <= 0` as *no* `until` — so the page below them comes back with its five hundred NEWEST events. None of those matches the filter's own `until`, so the page delivers nothing, which read as "the boundary second is too dense to page", stepped one second lower, and asked the identical unanswerable question again. Measured against the live relay: ~5.5 pages a second, 500 events fetched and discarded on each, an EOSE on every single page, `until` marching one second further negative every time, for as long as the process ran. A cold walk pulled 1,490,010 real events in ~10.8 minutes and then never returned, so the caller's coverage was never recorded and the next boot re-walked all of it. Not a rate limit: ~1,000 consecutive pages drew no NOTICE, no CLOSED and no throttling. Two guards, both at the points where the cursor moves: - The cursor floors at zero. `created_at` is unsigned, so nothing can exist below epoch 0: a cursor that would step under it has reached the bottom of the time axis and the walk is DRAINED. `until = 0` is still asked — it is a legal query and the boundary re-fetch for epoch-stamped events — only going BELOW it ends the walk. This also keeps a negative `until` off the wire, which relays disagree violently about: measured across five, one CLOSEs the subscription with a parse error, three answer a NOTICE and then never EOSE, and one drops the bound and serves its newest events. The floor is applied on the advance path too, not just the step: `pageMinTs` is an event's own `created_at`, so one relay serving a negative timestamp is enough to drive the cursor under zero, and clamping rather than stopping would not help — such an event never equals the boundary, so it dodges the dedup and returns on every page. - A relay that ignores the cursor is UNPAGEABLE. When a page delivers nothing and every event it received was NEWER than the `until` it asked for, the relay is not paging at all and stepping one second lower just repeats the question. `aboveBoundary == received` is what tells this apart from a genuinely dense boundary second, whose events are AT the boundary rather than above it. UNPAGEABLE is deliberate and conservative: it proves nothing about what the relay holds, so no coverage claim can be built on a page the relay never really answered. This second guard is the structural fix. Giving the filter a `since` does not substitute for it: the step path decrements `until` without regard to `since`, so a cursor-ignoring relay still walks from the window floor down to 0 — up to ~1.5 billion pages. A `since` only helps when the relay honours it, and then only because the empty page arrives as a drain. Three scripted tests cover both guards, and CursorTerminationProbe dials the five relays that found this (opt-in, `-PprodRelayBench=1`, asserts nothing). Against the live relays after the change: purplepag.es ends UNPAGEABLE in one page and 2.4s with the cursor never going under 0, where it previously ran 244 pages to `until = -243` without ending; the other four still DRAIN unchanged. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HPSzniNdvJxkhRsCe1QcyT --- .../NostrClientFetchAllPagesExt.kt | 73 ++++++++ .../NostrClientFetchAllPagesDrainTest.kt | 111 ++++++++++++ .../relay/prodbench/CursorTerminationProbe.kt | 167 ++++++++++++++++++ 3 files changed, 351 insertions(+) create mode 100644 quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/CursorTerminationProbe.kt diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesExt.kt index 203311e8e4..e5e98116f8 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllPagesExt.kt @@ -134,6 +134,28 @@ data class PagedFetchResult( * cap, so a larger value is clamped to the same page). Stepping past at least keeps * the download progressing to older events instead of stalling forever. * + * **Two guards keep that step from becoming a walk that never ends**, both learned + * from a relay in production rather than from reasoning: + * + * - **The cursor floors at zero.** `created_at` is an unsigned timestamp, so nothing + * can exist below epoch 0: a cursor that would step under it has reached the bottom + * of the time axis and the walk is [PagedFetchResult.End.DRAINED]. `until = 0` + * itself is still asked — it is a legal query, and the boundary re-fetch for events + * stamped at the epoch — it is only going *below* it that ends the walk. This also + * keeps a negative `until` off the wire, which relays disagree violently about: + * measured across five, one CLOSEs the subscription with a parse error, three + * answer a `NOTICE` and then never EOSE, and one drops the bound and serves its + * NEWEST events. + * - **A relay that ignores the cursor is [PagedFetchResult.End.UNPAGEABLE].** If a + * page delivered nothing and every event it received was NEWER than the `until` it + * asked for, the relay is not paging at all, and stepping one second lower just + * asks the same unanswered question again. That is exactly how the first guard's + * relay behaves — it treats `until <= 0` as no `until` — and without this the walk + * ran ~5.5 pages a second, 500 events fetched and discarded on each, EOSE on every + * one, for as long as the process lived. UNPAGEABLE is deliberate and conservative: + * it proves nothing about what the relay holds, so no coverage claim can be built + * on a page the relay never really answered. + * * A `search` ([Filter.search]) filter is the exception: NIP-50 results are ranked by * relevance, not `created_at`, so paging one by a `until` cursor is meaningless — it * would silently turn a top-N search into a time-walk, and never terminate against a @@ -259,6 +281,14 @@ suspend fun INostrClient.fetchAllPages( val boundary = until var received = 0 var delivered = 0 + + /** + * Events that came back NEWER than the `until` this page asked for — which an + * honest relay never sends. Counted because it is the only way to tell a relay + * that ignored the cursor apart from a boundary second too dense to page: both + * deliver nothing, and only one of them can be fixed by stepping past. + */ + var aboveBoundary = 0 var pageMinTs = Long.MAX_VALUE val idsAtPageMin = HashSet() @@ -289,6 +319,9 @@ suspend fun INostrClient.fetchAllPages( // early) or an unsafely published `idsAtPageMin`. try { received++ + // Before the dedup return, so it is counted for every event + // the page received, not just the ones that reach the match. + if (boundary != null && event.createdAt > boundary) aboveBoundary++ // Drop a boundary-second event we already delivered on an // earlier page (the inclusive re-fetch returns it again). if (boundary != null && event.createdAt == boundary && event.id in seenAtBoundary) return @@ -414,6 +447,35 @@ suspend fun INostrClient.fetchAllPages( // are resolved by stepping strictly past it. `boundary` is null only on // the first page, which has no dedup and so can't be all-duplicate. val step = boundary ?: break // first page, all-duplicate: impossible, and `end` stays UNPAGEABLE + + // The relay is not honouring `until`: every event it sent was NEWER than + // the cursor this page asked for. Stepping past cannot help — the next + // page repeats the same ask one second lower and gets the same answer, + // forever. Measured on a live relay (purplepag.es, which treats + // `until <= 0` as no `until` and answers with its newest page): ~5.5 + // pages a second, 500 events fetched and discarded on each, `until` + // marching one second further negative every time, an EOSE on every + // single page, for as long as the process ran. This is the ONE reading + // that ends it, and it is safely conservative — UNPAGEABLE proves + // nothing about what the relay holds, so no coverage claim is built on + // a page the relay never actually answered. + if (aboveBoundary == received) { + end = PagedFetchResult.End.UNPAGEABLE + break + } + + // Below the boundary there is nothing left to ask for: `created_at` is an + // unsigned timestamp, so no event can exist under epoch 0 and a cursor + // stepping past it has reached the bottom of the time axis. Ending here + // rather than sending `until = -1` also keeps a value off the wire that + // relays disagree violently about — measured across five: one CLOSEs the + // subscription with a parse error, three answer a NOTICE and then never + // EOSE (so every page burns a whole idle timeout), one drops the bound + // and serves its newest events. + if (step <= 0L) { + end = PagedFetchResult.End.DRAINED + break + } until = step - 1 seenAtBoundary = HashSet() continue @@ -432,6 +494,17 @@ suspend fun INostrClient.fetchAllPages( // termination both rely on `until` never increasing. Honest relays only // return events at-or-below `until`, so this is a no-op for them. val nextUntil = if (boundary != null) minOf(pageMinTs, boundary) else pageMinTs + + // The same floor as the step above, on the other way the cursor moves. It is + // reachable here too, and not only through a bug: `pageMinTs` is an event's + // own `created_at`, so one relay serving a negative timestamp is enough to + // put the cursor under zero. Clamping to 0 instead of stopping would not + // help — such an event never equals the boundary, so it dodges the dedup and + // comes back on every page, pinning the walk there for good. + if (nextUntil < 0L) { + end = PagedFetchResult.End.DRAINED + break + } if (boundary != null && nextUntil == boundary) { seenAtBoundary.addAll(idsAtPageMin) } else { diff --git a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesDrainTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesDrainTest.kt index f2de98b505..bc949a4826 100644 --- a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesDrainTest.kt +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesDrainTest.kt @@ -224,4 +224,115 @@ class NostrClientFetchAllPagesDrainTest { assertEquals(PagedFetchResult.End.LIMIT_REACHED, result.end, "a fulfilled limit is the caller stopping, not the corpus ending") assertFalse(result.drained) } + + // ---- termination: the walk must END, whatever the relay does ------------- + + @Test + fun aRelayThatIgnoresTheCursorEndsTheWalkInsteadOfSteppingForever() = + runBlocking { + // The production bug, scripted. purplepag.es holds events stamped + // `created_at = 0` and treats `until <= 0` as NO `until`, so the page + // below them comes back with its NEWEST events instead. None of those + // matches the filter's own `until`, so the page delivers nothing — + // which used to read as "the boundary second is too dense", step one + // second lower, and ask the identical unanswerable question again. + // Measured against the live relay: ~5.5 pages a second, 500 events + // discarded on each, an EOSE on every one, for as long as the process + // ran. `aboveBoundary == received` is what tells the two apart. + val client = ScriptedClient() + val feeder = + launch { + client.awaitPage(1) + client.listener!!.onEvent(event(2000), false, relay, null) + client.listener!!.onEvent(event(1000), false, relay, null) + client.listener!!.onEose(relay, null) + + // Page two asks for `until = 1000` and gets events from the top + // of the corpus — the answer to a query nobody made. + client.awaitPage(2) + client.listener!!.onEvent(event(9000), false, relay, null) + client.listener!!.onEvent(event(8000), false, relay, null) + client.listener!!.onEose(relay, null) + } + + val result = + client.fetchAllPages( + relay = relay, + filters = listOf(Filter(kinds = listOf(1))), + idleTimeoutMs = 2_000, + ) { } + feeder.join() + + assertEquals(2, result.downloaded, "only the two events the relay actually answered for") + assertEquals(2, client.subscribeCount, "and it stops on the FIRST page the relay refused to page") + assertEquals(PagedFetchResult.End.UNPAGEABLE, result.end, "a relay ignoring `until` is not paging, and cannot be stepped past") + assertFalse(result.drained, "which proves nothing about what it holds, so no coverage may be claimed") + } + + @Test + fun aCursorSteppingUnderTheEpochDrainsInsteadOfGoingNegative() = + runBlocking { + // `created_at` is unsigned, so nothing exists below epoch 0. A boundary + // second AT the epoch that only ever returns duplicates has reached the + // bottom of the time axis: the walk is done, and `until = -1` must never + // reach a relay — one of the five indexers CLOSEs the subscription over + // it, three answer a NOTICE and then never EOSE. + val client = ScriptedClient() + val feeder = + launch { + client.awaitPage(1) + client.listener!!.onEvent(event(0), false, relay, null) + client.listener!!.onEose(relay, null) + + // Page two re-asks the boundary inclusively and gets back only + // the event page one already delivered: nothing new, and nowhere + // left below to step to. + client.awaitPage(2) + client.listener!!.onEvent(event(0), false, relay, null) + client.listener!!.onEose(relay, null) + } + + val result = + client.fetchAllPages( + relay = relay, + filters = listOf(Filter(kinds = listOf(1))), + idleTimeoutMs = 2_000, + ) { } + feeder.join() + + assertEquals(1, result.downloaded, "the epoch event, delivered once") + assertEquals(2, client.subscribeCount, "no third page: there is nothing under zero to ask for") + assertEquals(PagedFetchResult.End.DRAINED, result.end, "the bottom of the time axis is an end, not a stall") + assertTrue(result.drained) + } + + @Test + fun anEventStampedBeforeTheEpochCannotPinTheWalk() = + runBlocking { + // `pageMinTs` is an event's own `created_at`, so one relay serving a + // negative timestamp drives the cursor under zero on the ADVANCE path + // rather than the step path. Clamping to 0 would not save it: such an + // event never equals the boundary, so it dodges the dedup and comes + // back on every page, pinning the walk at 0 for good. + val client = ScriptedClient() + val feeder = + launch { + client.awaitPage(1) + client.listener!!.onEvent(event(2000), false, relay, null) + client.listener!!.onEvent(event(-5), false, relay, null) + client.listener!!.onEose(relay, null) + } + + val result = + client.fetchAllPages( + relay = relay, + filters = listOf(Filter(kinds = listOf(1))), + idleTimeoutMs = 2_000, + ) { } + feeder.join() + + assertEquals(2, result.downloaded, "both events are still delivered — they were received") + assertEquals(1, client.subscribeCount, "but there is no second page to ask") + assertEquals(PagedFetchResult.End.DRAINED, result.end, "below the epoch there is nothing left to walk") + } } diff --git a/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/CursorTerminationProbe.kt b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/CursorTerminationProbe.kt new file mode 100644 index 0000000000..a0efae418e --- /dev/null +++ b/quartz/src/jvmTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/prodbench/CursorTerminationProbe.kt @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay.prodbench + +import com.vitorpamplona.quartz.nip01Core.relay.client.NostrClient +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.fetchAllPages +import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer +import com.vitorpamplona.quartz.nip01Core.relay.sockets.okhttp.BasicOkHttpWebSocket +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withTimeoutOrNull +import okhttp3.OkHttpClient +import java.time.Duration +import kotlin.test.Test + +/** + * The live half of the paging termination guards, against the relay that found + * them. [NostrClientFetchAllPagesDrainTest] scripts this behaviour, so it pins our + * INTERPRETATION of a relay; only dialling one can say whether the interpretation + * matches anything real. + * + * ## What it walks, and why that relay + * + * purplepag.es holds twelve `kind 10002` events stamped `created_at = 0` and treats + * `until <= 0` as *no* `until`, answering with its five hundred NEWEST events. A + * cursor walk therefore reaches zero, gets a page it never asked for, delivers none + * of it, and — before the guards — stepped one second lower and asked again. Measured + * against the live relay: ~5.5 pages a second, 500 events fetched and discarded on + * each, an EOSE on *every* page, `until` marching one second further negative every + * time, for as long as the process ran. A cold walk pulled 1,490,010 real events in + * ~10.8 minutes and then never returned. + * + * The ceiling is set just above the epoch-stamped events rather than `now` on + * purpose: this relay serves ~2,300 kind 0/10002 events a second and holds years of + * them, so starting at the top would spend a quarter of an hour on history that is + * not what this measures. One page from [TRAP_CEILING] already carries them. + * + * ## Reading it + * + * `lowest until` is the whole tell. A walk that ends leaves it at a real timestamp; a + * walk that cannot end leaves it below zero. With the guards in place the expected + * report is `UNPAGEABLE` with the cursor never going under `0`. + * + * OFF by default and not a gate: it dials the public internet, so it is neither + * hermetic nor reproducible, and a relay being down is not a code regression. It + * asserts nothing for that reason — it REPORTS, and a human reads it. + * + * ``` + * ./gradlew :quartz:jvmTest --tests "*.CursorTerminationProbe" -PprodRelayBench=1 -i + * ``` + */ +class CursorTerminationProbe { + @Test + fun reportWhetherAPagedWalkTerminates() { + if (System.getenv("PROD_RELAY_BENCH") == null && System.getProperty("prodRelayBench") == null) { + println("reportWhetherAPagedWalkTerminates skipped. Run with -PprodRelayBench=1 to enable.") + return + } + val okhttp = + OkHttpClient + .Builder() + .connectTimeout(Duration.ofSeconds(20)) + .pingInterval(Duration.ofSeconds(120)) + .build() + val scope = CoroutineScope(SupervisorJob()) + val client = NostrClient(BasicOkHttpWebSocket.Builder { okhttp }, scope) + + println("=".repeat(78)) + println("Does a paged walk TERMINATE? kinds [0, 10002], from $TRAP_CEILING down") + println("=".repeat(78)) + try { + for (url in RELAYS) { + val relay = RelayUrlNormalizer.normalize(url) + var events = 0 + var pages = 0 + var lowest = Long.MAX_VALUE + val startedAt = System.currentTimeMillis() + val outcome = + runCatching { + runBlocking { + // A hard ceiling, which `fetchAllPages` deliberately does + // not have: its own doc says a walk is bounded by a + // `limit` or by cancelling the caller, and this is the + // caller cancelling. Without it a relay with no guard + // hangs the probe — which is exactly what it is here to + // detect, so it must be detected rather than suffered. + withTimeoutOrNull(TERMINATION_MS) { + client.fetchAllPages( + relay, + listOf(Filter(kinds = listOf(0, 10002), until = TRAP_CEILING)), + idleTimeoutMs = 20_000L, + onNewPage = { until -> + pages++ + if (until < lowest) lowest = until + }, + ) { events++ } + } + } + } + val took = System.currentTimeMillis() - startedAt + val verdict = + outcome.fold( + onSuccess = { r -> + when (r) { + null -> "NEVER ENDED in ${TERMINATION_MS / 1000}s — THE GUARD IS NOT WORKING" + else -> "${r.end} (${r.downloaded} event(s), drained=${r.drained})" + } + }, + onFailure = { "threw ${it::class.simpleName}: ${it.message}" }, + ) + val reached = if (lowest == Long.MAX_VALUE) "no page after the first" else "$lowest" + println(" %-26s %-52s".format(url.removePrefix("wss://"), verdict)) + println(" %-26s %d page(s), %d event(s), %dms, lowest until=%s".format("", pages, events, took, reached)) + } + } finally { + runCatching { client.disconnect() } + scope.cancel() + } + println("=".repeat(78)) + } + + companion object { + /** + * purplepag.es is the one that found this. The other four are controls: they + * hold nothing at all below `1.5e9`, so they drain in a single page and prove + * the guards did not change an ordinary walk. + */ + private val RELAYS = + listOf( + "wss://purplepag.es", + "wss://user.kindpag.es", + "wss://directory.yabu.me", + "wss://profiles.nostr1.com", + "wss://indexer.coracle.social", + ) + + /** Just above the `created_at = 0` events, so one page reaches the cursor that matters. */ + private const val TRAP_CEILING = 1_600_000_000L + + /** + * Not an idle timeout — the relay answers, with an EOSE, the entire time. + * This is how long a walk gets to prove it can END. + */ + private const val TERMINATION_MS = 45_000L + } +} From 7ca4fbab337357f09afbcc0c8acdb9315f7dccb6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 9 Aug 2026 23:48:56 +0000 Subject: [PATCH 104/132] Cover the dense-second step-past the new guard sits in front of MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The step-past path had no test, and it is the one thing the ignored-cursor guard could plausibly break: both cases reach the same `delivered == 0` branch. They are told apart by WHERE the events landed — a dense boundary second returns them AT the boundary, so `aboveBoundary` stays 0 while `received` is 1 and the guard holds its fire; only a relay answering ABOVE the boundary is not paging at all. Scripted end to end: a second the relay's page cap can only ever return the head of, the step strictly past it, and the empty EOSEd page below. Also proves the documented cost is still paid rather than silently changed — the unreachable tail of that second is lost, and `downloaded` says so. `event()` grows a nonce so two events can share one `created_at`; the id was derived from the timestamp alone, which collapsed them into one event and made a dense second impossible to script. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HPSzniNdvJxkhRsCe1QcyT --- .../NostrClientFetchAllPagesDrainTest.kt | 71 ++++++++++++++++--- 1 file changed, 61 insertions(+), 10 deletions(-) diff --git a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesDrainTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesDrainTest.kt index bc949a4826..69b088dc40 100644 --- a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesDrainTest.kt +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientFetchAllPagesDrainTest.kt @@ -75,16 +75,23 @@ class NostrClientFetchAllPagesDrainTest { private val relay = RelayUrlNormalizer.normalize("wss://drain.example.com") - private fun event(createdAt: Long) = - Event( - id = createdAt.toString(16).padStart(64, '0'), - pubKey = "f".repeat(64), - createdAt = createdAt, - kind = 1, - tags = emptyArray(), - content = "e$createdAt", - sig = "0".repeat(128), - ) + /** + * [nonce] distinguishes two events sharing one `created_at`, which the id would + * otherwise collapse into the same event — and a boundary second holding more + * than one is the whole subject of the dense-second test below. + */ + private fun event( + createdAt: Long, + nonce: String = "", + ) = Event( + id = (createdAt.toString(16) + nonce).padStart(64, '0'), + pubKey = "f".repeat(64), + createdAt = createdAt, + kind = 1, + tags = emptyArray(), + content = "e$createdAt$nonce", + sig = "0".repeat(128), + ) @Test fun anEmptyPageConfirmedByEoseDrains() = @@ -227,6 +234,50 @@ class NostrClientFetchAllPagesDrainTest { // ---- termination: the walk must END, whatever the relay does ------------- + @Test + fun aBoundarySecondDenserThanAPageIsStillSteppedPast() = + runBlocking { + // The step-past path itself, which had no test and which the + // ignored-cursor guard now sits in front of. The two look identical + // from `delivered == 0` and must NOT be treated alike: a dense second + // returns events AT the boundary, so `aboveBoundary` stays 0 while + // `received` is 1, the guard holds its fire, and the walk steps past + // exactly as before. Only a relay answering ABOVE the boundary — which + // is not paging at all — trips it. + val client = ScriptedClient() + val feeder = + launch { + client.awaitPage(1) + client.listener!!.onEvent(event(2000), false, relay, null) + client.listener!!.onEvent(event(1000, "a"), false, relay, null) + client.listener!!.onEose(relay, null) + + // Page two re-asks second 1000 inclusively. The relay's page cap + // hands back the same head of that second — event "b" living + // there too can never be reached. Nothing new: stuck. + client.awaitPage(2) + client.listener!!.onEvent(event(1000, "a"), false, relay, null) + client.listener!!.onEose(relay, null) + + // So the walk steps strictly past to 999 and finds the corpus + // ends there. + client.awaitPage(3) + client.listener!!.onEose(relay, null) + } + + val result = + client.fetchAllPages( + relay = relay, + filters = listOf(Filter(kinds = listOf(1))), + idleTimeoutMs = 2_000, + ) { } + feeder.join() + + assertEquals(2, result.downloaded, "the duplicate is dropped, the dense second's tail is the documented loss") + assertEquals(3, client.subscribeCount, "it stepped past the stuck second instead of stopping on it") + assertEquals(PagedFetchResult.End.DRAINED, result.end, "and reached a genuinely empty, EOSEd page below it") + } + @Test fun aRelayThatIgnoresTheCursorEndsTheWalkInsteadOfSteppingForever() = runBlocking { From 293ffd0bc57daa134cea5ddac1efe0c788a62b45 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Sun, 9 Aug 2026 19:50:56 -0400 Subject: [PATCH 105/132] feat(concord): implement the CORD-05 Invite List (kind 13303), wire-compatible with Armada MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit kept link secrets in amy's local store, which made link refresh work but only for that one client. The spec already defines where they belong, and Armada implements it, so this replaces the local field with the real cross-client document. Kind 13303, replaceable, NIP-44-encrypted to self — the creator's private bookkeeping: { "entries": [ { "token", "signer_sk", "community_id", "url", "label?", "created_at", "expires_at?" } ], "tombstones": [ { "token", "community_id" } ] } `token` is both the link's unlock secret and the merge key; `signer_sk` is what lets any of the creator's clients re-sign at that link's addressable coordinate. Armada types both the entry and the tombstone as `[k: string]: unknown`, so unknown keys are contract: the codec preserves entry-, tombstone- and document-level residue, and re-encoding never deletes another client's data. Merge is by token, read-merge-write rather than overwrite — the list is replaceable and per-creator, so two devices minting concurrently would otherwise destroy each other's `signer_sk`, which is unrecoverable. A token tombstoned on either side stays dropped, so a stale device cannot resurrect a retired link. Registers 13303 in EventFactory (without it the kind deserializes as a plain Event and every typed read fails), and points amy's mint and Refounding refresh at the list instead of its own store. Semantics were taken from the spec and confirmed against Armada's observable behaviour — read for semantics only, never copied: Armada is AGPLv3 and Amethyst is MIT. Verified against a loopback geode: `concord invite` publishes an encrypted, untagged 13303; a Refounding reads it back, refreshes the live links, and a member with no role who never posted — unfindable by any rotation — recovers epoch 0 → 1 through the link he already held. Co-Authored-By: Claude Opus 5 (1M context) --- .../amethyst/cli/commands/ConcordCommands.kt | 71 +++++- .../cli/commands/ConcordModCommands.kt | 11 +- .../amethyst/cli/stores/ConcordStore.kt | 16 -- .../cord05Invites/ConcordInviteList.kt | 209 ++++++++++++++++++ .../cord05Invites/ConcordInviteListEvent.kt | 80 +++++++ .../quartz/utils/EventFactory.kt | 2 + .../cord05Invites/ConcordInviteListTest.kt | 133 +++++++++++ 7 files changed, 491 insertions(+), 31 deletions(-) create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteList.kt create mode 100644 quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListEvent.kt create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListTest.kt diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt index 3e06aa5b72..76e7a8d59b 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt @@ -27,7 +27,6 @@ import com.vitorpamplona.amethyst.cli.Output import com.vitorpamplona.amethyst.cli.stores.ConcordStore import com.vitorpamplona.amethyst.cli.stores.StoredCommunity import com.vitorpamplona.amethyst.cli.stores.StoredHeldRoot -import com.vitorpamplona.amethyst.cli.stores.StoredMintedInvite import com.vitorpamplona.amethyst.commons.actions.ConcordActions import com.vitorpamplona.amethyst.commons.actions.ConcordReceive import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEntry @@ -35,6 +34,10 @@ import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEven import com.vitorpamplona.quartz.concord.cord02Community.HeldRoot import com.vitorpamplona.quartz.concord.cord04Roles.AuthorityResolver import com.vitorpamplona.quartz.concord.cord04Roles.ControlEdition +import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteList +import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteListDocument +import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteListEntry +import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteListEvent import com.vitorpamplona.quartz.concord.cord05Invites.InviteBundleStatus import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray @@ -266,18 +269,25 @@ object ConcordCommands { val ack = ctx.publish(minted.bundleEvent, relaysFor(ctx, sc)) RawEventSupport.publishGuard(ack, minted.bundleEvent.id)?.let { return it } - // Keep the link signer + token so a later Refounding can refresh THIS coordinate rather - // than orphaning the link at a dead epoch — the liveness half of stranded recovery (A2). - ConcordStore(dataDir.concordFile).upsert( - sc.copy( - mintedInvites = - sc.mintedInvites + - StoredMintedInvite( - linkSignerPrivKey = minted.linkSignerPrivKey.toHexKey(), - token = minted.token.toHexKey(), - createdAt = TimeUtils.now(), + // Record the link in the CORD-05 Invite List (kind 13303) so any of this creator's + // clients — Amethyst, Armada — can later refresh THIS coordinate instead of orphaning + // the link at a dead epoch. That list is the liveness half of stranded recovery (A2). + publishInviteList( + ctx, + extraRelays = relaysFor(ctx, sc), + patch = + ConcordInviteListDocument( + entries = + listOf( + ConcordInviteListEntry( + token = minted.token.toHexKey(), + signerSk = minted.linkSignerPrivKey.toHexKey(), + communityId = sc.communityId, + url = minted.url, + createdAt = TimeUtils.now(), + ), ), - ), + ), ) Output.emit( @@ -570,6 +580,43 @@ object ConcordCommands { } } + /** + * This account's CORD-05 Invite List (kind 13303) — the creator's private, self-encrypted record + * of every link they minted, so a rotation can refresh those links instead of orphaning them. + * Empty when none was ever published. + */ + suspend fun readInviteList( + ctx: Context, + extraRelays: Set = emptySet(), + ): ConcordInviteListDocument { + val relays = ctx.outboxRelays() + extraRelays + if (relays.isEmpty()) return ConcordInviteListDocument.EMPTY + val filter = Filter(kinds = listOf(ConcordInviteListEvent.KIND), authors = listOf(ctx.signer.pubKey)) + val newest = + ctx + .drain(relays.associateWith { listOf(filter) }) + .map { it.second } + .maxByOrNull { it.createdAt } + return (newest as? ConcordInviteListEvent)?.decrypt(ctx.signer) ?: ConcordInviteListDocument.EMPTY + } + + /** + * Merges [patch] into the published list and republishes it. Read-merge-write rather than + * overwrite: the list is replaceable and per-creator, so two devices minting concurrently would + * otherwise delete each other's links (and their `signer_sk`, which is unrecoverable). + */ + suspend fun publishInviteList( + ctx: Context, + patch: ConcordInviteListDocument, + extraRelays: Set = emptySet(), + ) { + val relays = ctx.outboxRelays() + extraRelays + if (relays.isEmpty()) return + val merged = ConcordInviteList.merge(readInviteList(ctx, extraRelays), patch) + val event = ConcordInviteListEvent.create(ctx.signer, merged, TimeUtils.now()) + ctx.publish(event, relays) + } + fun notFound(handle: String): Int { Output.error("not_found", "no joined community matching '$handle' — run `amy concord list`") return 1 diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt index 342ae2501d..23ec2a47ef 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt @@ -346,15 +346,20 @@ object ConcordModCommands { stored.relays, stored.controlPk.ifBlank { null }, ) + val now = TimeUtils.now() var refreshed = 0 - for (link in stored.mintedInvites) { + for (link in ConcordCommands.readInviteList(ctx, relays).entries) { + if (link.communityId != stored.communityId) continue + // An elapsed link can no longer be joined, so re-posting it would only resurrect a + // dead URL at a live epoch (CORD-05). + if (link.isExpired(now)) continue runCatching { val event = ConcordActions.remintBundleAt( - linkSignerPrivKey = link.linkSignerPrivKey.hexToByteArray(), + linkSignerPrivKey = link.signerSk.hexToByteArray(), token = link.token.hexToByteArray(), invite = refreshedInvite, - createdAt = TimeUtils.now(), + createdAt = now, ) ctx.publish(event, relays) refreshed++ diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt index e08bfcc8f8..d2de85a258 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt @@ -53,22 +53,6 @@ data class StoredCommunity( // rekey has no message to miss: re-resolving this link is the only way back. Blank for a direct // invite or a community joined before amy stored it. val inviteRef: String = "", - // Invite links WE minted for this community, kept so a Refounding can re-publish each bundle at - // its own coordinate for the new epoch. Without this the link a member joined through points at - // a dead epoch forever and stranded recovery can never fire (A2). Holds link-signer secrets, so - // it sits beside `root`/`controlRoot` in the same already-secret file. - val mintedInvites: List = emptyList(), -) - -/** - * One invite link this account minted: enough to re-sign at its addressable coordinate later. The - * coordinate is the link signer's pubkey, so keeping the private key is what lets a Refounding - * refresh the link (and, in future, revoke it) instead of orphaning it. - */ -data class StoredMintedInvite( - val linkSignerPrivKey: String = "", - val token: String = "", - val createdAt: Long = 0, ) /** A past community_root for a specific epoch, mirroring quartz `HeldRoot`. */ diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteList.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteList.kt new file mode 100644 index 0000000000..3eea2095c3 --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteList.kt @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.concord.cord05Invites + +import com.vitorpamplona.quartz.concord.cord04Roles.ConcordJson +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.KSerializer +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.descriptors.elementNames +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonTransformingSerializer +import kotlinx.serialization.json.jsonObject + +private val NoExtras: JsonObject = JsonObject(emptyMap()) + +/** + * One minted invite link, as the creator's own bookkeeping (CORD-05, kind 13303). + * + * [token] is both the link's unlock secret and the **merge key** across devices, and [signerSk] is + * the link signer's private key — which is what makes a link *refreshable*. The kind-33301 bundle is + * addressable and authored by that signer, so re-posting under it moves the link to the current + * epoch without changing the URL anyone already holds. Lose the secret and the link is orphaned at a + * dead epoch forever, which is what made stranded recovery unreachable in practice. + * + * [residue] carries wire keys this build does not model. Armada types both the entry and the + * tombstone as `[k: string]: unknown`, so unknown keys are part of the contract: dropping them on a + * re-encode deletes another client's data. + */ +class ConcordInviteListEntry( + val token: String, + val signerSk: String, + val communityId: String, + val url: String, + val label: String? = null, + val createdAt: Long = 0, + val expiresAt: Long? = null, + val residue: JsonObject = NoExtras, +) { + /** True when this link can no longer be joined, so it must not be refreshed (CORD-05). */ + fun isExpired(nowSecs: Long): Boolean = expiresAt != null && expiresAt <= nowSecs +} + +/** A retired link: the creator's record that [token] is gone, kept so a merge cannot resurrect it. */ +class ConcordInviteListTombstone( + val token: String, + val communityId: String, + val residue: JsonObject = NoExtras, +) + +/** The decoded kind-13303 document: live [entries], [tombstones], and document-level [residue]. */ +class ConcordInviteListDocument( + val entries: List = emptyList(), + val tombstones: List = emptyList(), + val residue: JsonObject = NoExtras, +) { + companion object { + val EMPTY = ConcordInviteListDocument() + } +} + +/** + * Codec + merge for the CORD-05 Invite List (kind 13303), the creator's private, NIP-44 self- + * encrypted bookkeeping of the links they minted. Wire-compatible with Armada's `invite.ts`: + * + * ```jsonc + * { "entries": [ { "token", "signer_sk", "community_id", "url", "label?", "created_at", "expires_at?" } ], + * "tombstones": [ { "token", "community_id" } ] } + * ``` + */ +object ConcordInviteList { + private const val EXTRAS = "__extras" + + /** Wraps a generated serializer so unknown keys survive a decode → modify → encode. */ + private open class ExtrasPreserving( + delegate: KSerializer, + ) : JsonTransformingSerializer(delegate) { + @OptIn(ExperimentalSerializationApi::class) + private val known = delegate.descriptor.elementNames.toSet() - EXTRAS + + override fun transformDeserialize(element: JsonElement): JsonElement { + val obj = element as? JsonObject ?: return element + val extras = obj.filterKeys { it !in known } + if (extras.isEmpty()) return obj + return JsonObject(obj.filterKeys { it in known } + (EXTRAS to JsonObject(extras))) + } + + override fun transformSerialize(element: JsonElement): JsonElement { + val obj = element as? JsonObject ?: return element + val extras = obj[EXTRAS]?.jsonObject ?: return obj + return JsonObject(extras + (obj - EXTRAS)) + } + } + + @Serializable + private class WireEntry( + val token: String = "", + @SerialName("signer_sk") val signerSk: String = "", + @SerialName("community_id") val communityId: String = "", + val url: String = "", + val label: String? = null, + @SerialName("created_at") val createdAt: Long = 0, + @SerialName("expires_at") val expiresAt: Long? = null, + @SerialName(EXTRAS) val extras: JsonObject = NoExtras, + ) + + @Serializable + private class WireTombstone( + val token: String = "", + @SerialName("community_id") val communityId: String = "", + @SerialName(EXTRAS) val extras: JsonObject = NoExtras, + ) + + private object WireEntrySerializer : ExtrasPreserving(WireEntry.serializer()) + + private object WireTombstoneSerializer : ExtrasPreserving(WireTombstone.serializer()) + + @Serializable + private class WireDocument( + val entries: List< + @Serializable(WireEntrySerializer::class) + WireEntry, + > = emptyList(), + val tombstones: List< + @Serializable(WireTombstoneSerializer::class) + WireTombstone, + > = emptyList(), + @SerialName(EXTRAS) val extras: JsonObject = NoExtras, + ) + + private object WireDocumentSerializer : ExtrasPreserving(WireDocument.serializer()) + + /** + * Decodes the plaintext document. A malformed document yields [ConcordInviteListDocument.EMPTY] + * rather than throwing — but note the sharp edge this shape shares with the community list: one + * unparseable entry aborts the whole array, so every field defaults instead of being required. + */ + fun decode(json: String): ConcordInviteListDocument = + try { + val doc = ConcordJson.instance.decodeFromString(WireDocumentSerializer, json) + ConcordInviteListDocument( + entries = + doc.entries.map { + ConcordInviteListEntry(it.token, it.signerSk, it.communityId, it.url, it.label, it.createdAt, it.expiresAt, it.extras) + }, + tombstones = doc.tombstones.map { ConcordInviteListTombstone(it.token, it.communityId, it.extras) }, + residue = doc.extras, + ) + } catch (_: Exception) { + ConcordInviteListDocument.EMPTY + } + + fun encode(doc: ConcordInviteListDocument): String = + ConcordJson.instance.encodeToString( + WireDocumentSerializer, + WireDocument( + entries = + doc.entries.map { + WireEntry(it.token, it.signerSk, it.communityId, it.url, it.label, it.createdAt, it.expiresAt, it.residue) + }, + tombstones = doc.tombstones.map { WireTombstone(it.token, it.communityId, it.residue) }, + extras = doc.residue, + ), + ) + + /** + * Merges [patch] onto [base], keyed by `token` — the spec's own merge key. A token present in + * either side's tombstones is dropped from the result and kept tombstoned, so a retired link + * cannot be resurrected by a device that still has it cached. [patch] wins field-by-field on a + * token both sides carry, which is what makes "read remote, apply my change, publish" converge. + */ + fun merge( + base: ConcordInviteListDocument, + patch: ConcordInviteListDocument, + ): ConcordInviteListDocument { + val tombstones = LinkedHashMap() + for (t in base.tombstones + patch.tombstones) tombstones[t.token] = t + + val entries = LinkedHashMap() + for (e in base.entries + patch.entries) { + if (e.token in tombstones) continue + entries[e.token] = e + } + return ConcordInviteListDocument( + entries = entries.values.toList(), + tombstones = tombstones.values.toList(), + residue = JsonObject(base.residue + patch.residue), + ) + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListEvent.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListEvent.kt new file mode 100644 index 0000000000..f4bbd80f1c --- /dev/null +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListEvent.kt @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.concord.cord05Invites + +import androidx.compose.runtime.Immutable +import com.vitorpamplona.quartz.nip01Core.core.Address +import com.vitorpamplona.quartz.nip01Core.core.BaseReplaceableEvent +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.signers.NostrSigner +import com.vitorpamplona.quartz.utils.TimeUtils + +/** + * The CORD-05 **Invite List** (kind 13303): the creator's private, NIP-44 self-encrypted record of + * every link they minted — `token` (the unlock secret and merge key) and `signer_sk` (the link + * signer's private key) per entry. + * + * It exists so a link can be *refreshed*: the kind-33301 bundle is addressable and authored by the + * link signer, so re-posting under it moves the link to the current epoch behind the same URL (e.g. + * after a Rekey). Without the list a client cannot re-sign at that coordinate, every rotation + * orphans every outstanding link, and stranded recovery — whose whole premise is re-resolving the + * link you joined through — can never fire. + * + * Replaceable and per-creator: the coordinate is (kind, creator pubkey, ""), so a creator's devices + * converge on one list. Merge by `token` ([ConcordInviteList.merge]) rather than overwriting, or two + * devices minting concurrently lose each other's links. + */ +@Immutable +class ConcordInviteListEvent( + id: HexKey, + pubKey: HexKey, + createdAt: Long, + tags: Array>, + content: String, + sig: HexKey, +) : BaseReplaceableEvent(id, pubKey, createdAt, KIND, tags, content, sig) { + /** + * Decrypts the whole document with [signer] — entries, tombstones and the document residue. + * Use this (never a partial read) whenever the result will be re-encoded, or another client's + * unknown keys are dropped on the next publish. + */ + suspend fun decrypt(signer: NostrSigner): ConcordInviteListDocument = + try { + ConcordInviteList.decode(signer.nip44Decrypt(content, signer.pubKey)) + } catch (_: Exception) { + ConcordInviteListDocument.EMPTY + } + + companion object { + const val KIND = 13303 + + fun createAddress(pubKey: HexKey) = Address(KIND, pubKey, "") + + suspend fun create( + signer: NostrSigner, + document: ConcordInviteListDocument, + createdAt: Long = TimeUtils.now(), + ): ConcordInviteListEvent { + val content = signer.nip44Encrypt(ConcordInviteList.encode(document), signer.pubKey) + return signer.sign(createdAt, KIND, emptyArray(), content) + } + } +} diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/EventFactory.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/EventFactory.kt index 622fa0b889..3955952fdd 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/EventFactory.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/utils/EventFactory.kt @@ -101,6 +101,7 @@ import com.vitorpamplona.quartz.buzz.wpWorkspaceProfile.SetWorkspaceProfileEvent import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityListEvent import com.vitorpamplona.quartz.concord.cord03Channels.ConcordChatEditEvent import com.vitorpamplona.quartz.concord.cord04Roles.control.ControlEditionEvent +import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteListEvent import com.vitorpamplona.quartz.concord.cord05Invites.bundle.ConcordInviteBundleEvent import com.vitorpamplona.quartz.experimental.agora.FundraiserEvent import com.vitorpamplona.quartz.experimental.attestations.attestation.AttestationEvent @@ -804,6 +805,7 @@ class EventFactory { RequestToVanishEvent.KIND -> RequestToVanishEvent(id, pubKey, createdAt, tags, content, sig) ConcordCommunityListEvent.KIND -> ConcordCommunityListEvent(id, pubKey, createdAt, tags, content, sig) ControlEditionEvent.KIND -> ControlEditionEvent(id, pubKey, createdAt, tags, content, sig) + ConcordInviteListEvent.KIND -> ConcordInviteListEvent(id, pubKey, createdAt, tags, content, sig) ConcordInviteBundleEvent.KIND -> ConcordInviteBundleEvent(id, pubKey, createdAt, tags, content, sig) SealedRumorEvent.KIND -> SealedRumorEvent(id, pubKey, createdAt, tags, content, sig) SearchRelayListEvent.KIND -> SearchRelayListEvent(id, pubKey, createdAt, tags, content, sig) diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListTest.kt new file mode 100644 index 0000000000..e488aeccb5 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListTest.kt @@ -0,0 +1,133 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.concord.cord05Invites + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +/** + * Wire conformance for the CORD-05 Invite List (kind 13303). The whole point of this document is + * cross-client: a link minted in Armada must be refreshable from Amethyst and back, so the field + * names and the merge key are contract, not preference. + */ +class ConcordInviteListTest { + // The spec's own example document, verbatim in shape. + private val specJson = + """ + { "entries": [ + { "token": "aa11", + "signer_sk": "bb22", + "community_id": "cc33", + "url": "https://vector.chat/invite/naddr1abc#frag", + "label": "Reddit", + "created_at": 1719800000, + "expires_at": 1722400000 } ], + "tombstones": [ { "token": "dd44", "community_id": "cc33" } ] } + """.trimIndent() + + @Test + fun readsTheSpecDocumentIntoTypedEntries() { + val doc = ConcordInviteList.decode(specJson) + + assertEquals(1, doc.entries.size) + val e = doc.entries.first() + assertEquals("aa11", e.token) + assertEquals("bb22", e.signerSk) + assertEquals("cc33", e.communityId) + assertEquals("https://vector.chat/invite/naddr1abc#frag", e.url) + assertEquals("Reddit", e.label) + assertEquals(1719800000L, e.createdAt) + assertEquals(1722400000L, e.expiresAt) + + assertEquals(1, doc.tombstones.size) + assertEquals("dd44", doc.tombstones.first().token) + assertEquals("cc33", doc.tombstones.first().communityId) + } + + @Test + fun emitsTheSnakeCaseKeysAnotherClientReads() { + val json = ConcordInviteList.encode(ConcordInviteList.decode(specJson)) + // Field names are the interop contract — a camelCase slip silently orphans every link. + for (key in listOf("\"token\"", "\"signer_sk\"", "\"community_id\"", "\"url\"", "\"created_at\"", "\"expires_at\"", "\"entries\"", "\"tombstones\"")) { + assertTrue(json.contains(key), "missing wire key $key") + } + } + + @Test + fun keepsUnknownKeysAcrossADecodeEncodeCycle() { + // Armada types the entry and tombstone as `[k: string]: unknown`, so dropping a key we do + // not model deletes another client's data on our next publish. + val withExtras = + """ + { "entries": [ { "token": "aa11", "signer_sk": "bb22", "community_id": "cc33", + "url": "u", "created_at": 1, "future_field": {"a":1} } ], + "tombstones": [ { "token": "dd44", "community_id": "cc33", "why": "revoked" } ], + "doc_level_unknown": 7 } + """.trimIndent() + + val round = ConcordInviteList.encode(ConcordInviteList.decode(withExtras)) + + assertTrue(round.contains("future_field"), "entry-level unknown key dropped") + assertTrue(round.contains("doc_level_unknown"), "document-level unknown key dropped") + assertTrue(round.contains("\"why\""), "tombstone unknown key dropped") + } + + @Test + fun mergesByTokenAndLetsTombstonesWin() { + val base = + ConcordInviteListDocument( + entries = + listOf( + ConcordInviteListEntry("t1", "sk1", "c", "url1", createdAt = 1), + ConcordInviteListEntry("t2", "sk2", "c", "url2", createdAt = 2), + ), + ) + // Another device minted t3 and retired t1. + val patch = + ConcordInviteListDocument( + entries = listOf(ConcordInviteListEntry("t3", "sk3", "c", "url3", createdAt = 3)), + tombstones = listOf(ConcordInviteListTombstone("t1", "c")), + ) + + val merged = ConcordInviteList.merge(base, patch) + val tokens = merged.entries.map { it.token }.toSet() + + assertEquals(setOf("t2", "t3"), tokens, "merge is keyed by token; a tombstoned link is dropped") + assertTrue(merged.tombstones.any { it.token == "t1" }, "the tombstone must persist or a stale device resurrects the link") + } + + @Test + fun aMalformedDocumentYieldsEmptyRatherThanThrowing() { + assertEquals(0, ConcordInviteList.decode("not json").entries.size) + assertEquals(0, ConcordInviteList.decode("{\"entries\":\"wrong type\"}").entries.size) + } + + @Test + fun anExpiredLinkIsNotRefreshable() { + val live = ConcordInviteListEntry("t", "sk", "c", "u", expiresAt = 100) + val forever = ConcordInviteListEntry("t", "sk", "c", "u", expiresAt = null) + + assertTrue(live.isExpired(nowSecs = 101), "an elapsed link can no longer be joined") + assertTrue(!live.isExpired(nowSecs = 99)) + assertTrue(!forever.isExpired(nowSecs = Long.MAX_VALUE), "no expiry means it never elapses") + } +} From 66d27772623c258911edada18760896d9a186875 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Sun, 9 Aug 2026 21:48:19 -0400 Subject: [PATCH 106/132] feat(concord): wire the Invite List into Amethyst; fix the QR quiet zone MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Android half of the CORD-05 Invite List (kind 13303). Minting records the link's `token` + `signer_sk` in the shared, self-encrypted list, and a Refounding re-posts every live link it finds there at that link's own coordinate, carrying the new epoch. Read-merge-write, never overwrite: two of the user's devices minting concurrently would otherwise delete each other's `signer_sk`, which is unrecoverable. Expired links are skipped — re-posting one would only resurrect a dead URL at a live epoch. Verified on a Galaxy Tab A7 Lite against a loopback geode, driving the real UI: - tapping Invite publishes a kind-13303 authored by the device - Remove member → the Refounding publishes 5 control wraps + 1 rekey blob; `amy` follows it (epoch 0 → 1), and the removed member gets `no_blob_for_us` and stays behind - with a device-minted link in the list, the next Refounding logs "refreshed 1 invite link(s) to epoch 2" and the bundle at that link's coordinate is REPLACED in place rather than orphaned Also fixes the invite QR rendering as a postage stamp. QrCodeDrawer's quiet zone was a fixed 100px per side, which does not scale: at the dialog's 220dp box that ate ~45% of the canvas, so a long payload drew tiny inside a large white card. Expressed as the QR spec's 4-module zone it stays proportional, and the code now fills whatever box it is given at every call site. Note: OpenCV cannot decode this drawer's stylized modules either before or after the change, so scannability was not machine-verified — the change only shrinks excess quiet zone to the spec minimum and enlarges the modules, but a camera check before release is worthwhile. Co-Authored-By: Claude Opus 5 (1M context) --- .../amethyst/model/AccountConcordActions.kt | 107 ++++++++++++++++++ .../ui/note/share/ShareNoteAsQrScreen.kt | 2 +- .../ui/screen/loggedIn/qrcode/QrCodeDrawer.kt | 32 ++++-- 3 files changed, 131 insertions(+), 10 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt index 4caf607709..543206fc94 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt @@ -40,6 +40,10 @@ import com.vitorpamplona.quartz.concord.cord04Roles.ConcordPermissions import com.vitorpamplona.quartz.concord.cord04Roles.MetadataEntity import com.vitorpamplona.quartz.concord.cord04Roles.RoleEntity import com.vitorpamplona.quartz.concord.cord05Invites.CommunityInvite +import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteList +import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteListDocument +import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteListEntry +import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteListEvent import com.vitorpamplona.quartz.concord.cord05Invites.InviteBundleStatus import com.vitorpamplona.quartz.concord.cord05Invites.InviteRelayDictionary import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys @@ -164,6 +168,83 @@ class AccountConcordActions( return community.communityIdHex } + // ---- CORD-05 Invite List (kind 13303) ------------------------------------- + + /** + * This account's Invite List: the creator's private, self-encrypted record of every link they + * minted (`token` + `signer_sk` per entry). Empty when none was ever published. + * + * Fetched rather than read from [LocalCache] because nothing subscribes to 13303 — it is + * bookkeeping the user never sees, needed only at mint and at rotation. + */ + private suspend fun readConcordInviteList(relays: Set): ConcordInviteListDocument { + if (relays.isEmpty()) return ConcordInviteListDocument.EMPTY + val filter = Filter(kinds = listOf(ConcordInviteListEvent.KIND), authors = listOf(account.signer.pubKey)) + val newest = + account.client + .fetchAll(filters = relays.associateWith { listOf(filter) }) + .maxByOrNull { it.createdAt } + return (newest as? ConcordInviteListEvent)?.decrypt(account.signer) ?: ConcordInviteListDocument.EMPTY + } + + /** + * Merges [patch] into the published Invite List and republishes it. Read-merge-write, never + * overwrite: the list is replaceable and per-creator, so two of the user's devices minting + * concurrently would otherwise delete each other's `signer_sk` — and that secret is + * unrecoverable, orphaning the link at whatever epoch it was last refreshed to. + */ + private suspend fun publishConcordInviteList( + patch: ConcordInviteListDocument, + relays: Set, + ) { + val publishTo = relays.ifEmpty { account.outboxRelays.flow.value } + if (publishTo.isEmpty()) return + val merged = ConcordInviteList.merge(readConcordInviteList(publishTo), patch) + account.client.publish(ConcordInviteListEvent.create(account.signer, merged, TimeUtils.now()), publishTo) + } + + /** + * Re-posts every live link this account minted for [entry]'s community at its own coordinate, + * carrying the CURRENT epoch (CORD-05). The kind-33301 bundle is addressable and authored by the + * link signer, so this moves the link behind the same URL instead of orphaning it at a dead + * epoch — which is the whole premise stranded recovery rests on. + * + * Safe to call for every live link: recovery is ban-gated at the epoch being left, and a + * Refounding bans the members it removes on the way out, so a removed member's own recovery is + * refused even though their link now resolves. + */ + private suspend fun refreshConcordInviteLinks(entry: ConcordCommunityListEntry): Int { + val relays = entry.relays.mapNotNullTo(mutableSetOf()) { RelayUrlNormalizer.normalizeOrNull(it) }.ifEmpty { account.outboxRelays.flow.value } + if (relays.isEmpty()) return 0 + val now = TimeUtils.now() + val refreshed = + ConcordActions.inviteFor( + communityIdHex = entry.id, + ownerPubKey = entry.owner, + ownerSaltHex = entry.ownerSalt, + communityRootHex = entry.root, + rootEpoch = entry.rootEpoch, + name = entry.name, + relays = entry.relays, + controlPk = entry.controlPk, + ) + var count = 0 + for (link in readConcordInviteList(relays).entries) { + if (link.communityId != entry.id) continue + // An elapsed link can no longer be joined, so re-posting it would only resurrect a dead + // URL at a live epoch. + if (link.isExpired(now)) continue + runCatching { + account.client.publish( + ConcordActions.remintBundleAt(link.signerSk.hexToByteArray(), link.token.hexToByteArray(), refreshed, now), + relays, + ) + count++ + }.onFailure { Log.w("Concord", "invite refresh failed for ${entry.id}", it) } + } + return count + } + /** * Mint a shareable invite link for a joined community and publish its * kind-33301 public bundle to the community relays. Returns the `…/invite/…` @@ -209,6 +290,24 @@ class AccountConcordActions( val publishTo = entry.relays.mapNotNullTo(mutableSetOf()) { RelayUrlNormalizer.normalizeOrNull(it) }.ifEmpty { account.outboxRelays.flow.value } if (publishTo.isNotEmpty()) account.client.publish(minted.bundleEvent, publishTo) + + // Record the link so a later Refounding can refresh THIS coordinate rather than orphaning it + // (CORD-05, kind 13303). Shared with amy and Armada, so any of the creator's clients can. + publishConcordInviteList( + ConcordInviteListDocument( + entries = + listOf( + ConcordInviteListEntry( + token = minted.token.toHexKey(), + signerSk = minted.linkSignerPrivKey.toHexKey(), + communityId = entry.id, + url = minted.url, + createdAt = TimeUtils.now(), + ), + ), + ), + publishTo, + ) return minted.url } @@ -862,6 +961,14 @@ class AccountConcordActions( // 5. Adopt the new epoch ourselves. This rebuilds our session under the new root and // re-folds the compacted Control Plane (with the ban), dropping the removed members. adoptConcordRoot(entry, newRoot, build.newEpoch, build.newControlKeys.address.hexToByteArray(), newControlRoot) + + // 6. Move every link we minted to the new epoch. Without this the Refounding orphans them, + // and a member it left out — no rekey blob, no message to miss — has no way back at all. + val moved = + account.concordChannelList.liveCommunities.value + .firstOrNull { it.id == communityId } + ?.let { refreshConcordInviteLinks(it) } ?: 0 + Log.i("Concord") { "Refounding ${entry.id}: refreshed $moved invite link(s) to epoch ${build.newEpoch}" } return true } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/share/ShareNoteAsQrScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/share/ShareNoteAsQrScreen.kt index 5bbf7bfdfa..03c78556de 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/share/ShareNoteAsQrScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/note/share/ShareNoteAsQrScreen.kt @@ -61,7 +61,7 @@ import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel import com.vitorpamplona.amethyst.ui.screen.loggedIn.qrcode.QrCodeDrawer import com.vitorpamplona.amethyst.ui.stringRes -// A cap, not a fixed size: QrCodeDrawer's own quiet zone (QR_MARGIN_PX in QrCodeDrawer.kt) is a +// A cap, not a fixed size: QrCodeDrawer's own quiet zone (QR_QUIET_ZONE_MODULES in QrCodeDrawer.kt) is a // fixed pixel count subtracted from raw size.width, so its share of the tile grows as density // falls. Hard-sizing this call to a small dp value starved long-form naddr payloads of scannable // resolution on low-density screens. Deriving the size from the available column width keeps diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/qrcode/QrCodeDrawer.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/qrcode/QrCodeDrawer.kt index d4e624d926..a31a83b21c 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/qrcode/QrCodeDrawer.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/qrcode/QrCodeDrawer.kt @@ -49,7 +49,15 @@ import com.google.zxing.qrcode.encoder.Encoder import com.google.zxing.qrcode.encoder.QRCode import com.vitorpamplona.amethyst.ui.theme.QuoteBorder -const val QR_MARGIN_PX = 100f +/** + * The quiet zone around the code, in **modules** — the QR spec's minimum of 4. + * + * It was a fixed 100px per side, which does not scale: at a small draw size those 200px ate most of + * the canvas, so a long payload (a Concord invite link, an nprofile) rendered as a postage stamp + * floating in white. Expressed in modules the zone stays proportional, so the code fills whatever + * box it is given at every size while remaining scannable. + */ +const val QR_QUIET_ZONE_MODULES = 4f @Preview @Composable @@ -78,13 +86,16 @@ fun QrCodeDrawer( ) { Canvas(modifier = Modifier.fillMaxSize()) { // Calculate the height and width of each column/row - val rowHeight = (size.width - QR_MARGIN_PX * 2f) / qrCode.matrix.height - val columnWidth = (size.width - QR_MARGIN_PX * 2f) / qrCode.matrix.width + // Solve for the module size with the quiet zone measured in modules, so the whole code + // (zone included) is exactly as wide as the canvas. + val rowHeight = size.height / (qrCode.matrix.height + QR_QUIET_ZONE_MODULES * 2f) + val columnWidth = size.width / (qrCode.matrix.width + QR_QUIET_ZONE_MODULES * 2f) val radius = CornerRadius(20f) // Draw all of the finder patterns required by the QR spec. Calculate the ratio // of the number of rows/columns to the width and height drawQrCodeFinders( + quietZonePx = columnWidth * QR_QUIET_ZONE_MODULES, sideLength = size.width, finderPatternSize = Size( @@ -97,6 +108,7 @@ fun QrCodeDrawer( // Draw data bits (encoded data part) drawAllQrCodeDataBits( + quietZonePx = columnWidth * QR_QUIET_ZONE_MODULES, bytes = qrCode.matrix, size = Size( @@ -119,7 +131,7 @@ private fun createQrCode(contents: String): QRCode { ErrorCorrectionLevel.Q, mapOf( EncodeHintType.CHARACTER_SET to "UTF-8", - EncodeHintType.MARGIN to QR_MARGIN_PX, + EncodeHintType.MARGIN to QR_QUIET_ZONE_MODULES, EncodeHintType.ERROR_CORRECTION to ErrorCorrectionLevel.Q, ), ) @@ -132,6 +144,7 @@ fun newPath(withPath: Path.() -> Unit) = } fun DrawScope.drawAllQrCodeDataBits( + quietZonePx: Float, bytes: ByteMatrix, size: Size, color: Color, @@ -182,8 +195,8 @@ fun DrawScope.drawAllQrCodeDataBits( Rect( offset = Offset( - x = QR_MARGIN_PX + x * size.width, - y = QR_MARGIN_PX + y * size.height, + x = quietZonePx + x * size.width, + y = quietZonePx + y * size.height, ), size = newSize, ), @@ -212,6 +225,7 @@ private const val INTERIOR_BACKGROUND_EXTERIOR_SHAPE_CORNER_RADIUS = 0.5f * @param finderPatternSize [Size] of each finder patten, based on the QR code spec */ internal fun DrawScope.drawQrCodeFinders( + quietZonePx: Float, sideLength: Float, finderPatternSize: Size, cornerRadius: CornerRadius, @@ -219,11 +233,11 @@ internal fun DrawScope.drawQrCodeFinders( ) { setOf( // Draw top left finder pattern. - Offset(x = QR_MARGIN_PX, y = QR_MARGIN_PX), + Offset(x = quietZonePx, y = quietZonePx), // Draw top right finder pattern. - Offset(x = sideLength - (QR_MARGIN_PX + finderPatternSize.width), y = QR_MARGIN_PX), + Offset(x = sideLength - (quietZonePx + finderPatternSize.width), y = quietZonePx), // Draw bottom finder pattern. - Offset(x = QR_MARGIN_PX, y = sideLength - (QR_MARGIN_PX + finderPatternSize.height)), + Offset(x = quietZonePx, y = sideLength - (quietZonePx + finderPatternSize.height)), ).forEach { offset -> drawQrCodeFinder( topLeft = offset, From eb8c812690ebb5cc528df593415040aac7d7943e Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Sun, 9 Aug 2026 23:39:54 -0400 Subject: [PATCH 107/132] fix(concord): close ten review findings in the CORD-05 invite path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A high-effort review of the invite work found ten correctness bugs, nine confirmed and one plausible. All are fixed here; the on-device pass on a tablet proved the three that are observable through the UI. The load-bearing one: the kind-13303 Invite List is replaceable, and both clients merged a patch onto a base that silently degraded to EMPTY whenever the read failed — an unanswered relay or a bunker signer declining one decrypt was enough. Republishing that destroys every `signer_sk` it could not read, and those secrets cannot be regenerated, so every outstanding link is orphaned at a dead epoch. `decode` is now `decodeOrNull` and `decrypt` returns null, so "I could not read it" is distinguishable from "it is empty", and the write aborts rather than overwriting. The rest: - `join` is now ban-gated on both clients. A Refounding re-mints every outstanding link onto the new root, and an ex-member keeps the URL and its token forever, so the rotation meant to expel them handed them the new keys instead. Fails closed on an unreadable plane. - Android's Refounding re-read the entry from `liveCommunities` straight after adopting the new root, but that flow decrypts asynchronously, so every link was re-minted onto the epoch just left. `adoptConcordRoot` now returns the entry it wrote. - amy's refound folded the fresh bans locally and then never used them, re-draining from relays instead; a relay slow to echo them back would produce a new epoch whose roster never banned anyone. - Link refresh rebuilt the bundle from scratch, stripping expiry, channel grants, icon and label; it now moves the link's own current bundle and changes only the epoch's key material. - Refresh also re-posted over revocation tombstones, silently un-revoking a retired link. - The 13303 coordinate is (13303, me, "") — one list per account — but was read and written on per-community relays, forking it into divergent versions that newest-wins then collapsed. Now account-outbox only. - Minting returned the URL even when recording the link failed, handing out a link that could never be refreshed. It now fails closed. - amy's refound had no equivalent of Amethyst's recipient cap, leaving the attacker-writable half of the union unbounded. - amy's store dropped the banked epoch's `controlRoot` on the round-trip, losing the staff write key that rebuilds the anti-rollback floor. Co-Authored-By: Claude Opus 5 (1M context) --- .../amethyst/model/AccountConcordActions.kt | 186 +++++++++++------- .../amethyst/model/ConcordInviteResult.kt | 9 + .../concord/ConcordInviteScreen.kt | 2 + amethyst/src/main/res/values/strings.xml | 1 + .../amethyst/cli/commands/ConcordCommands.kt | 99 +++++++--- .../cli/commands/ConcordModCommands.kt | 96 ++++++--- .../amethyst/cli/stores/ConcordStore.kt | 6 + .../concord/cord05Invites/CommunityInvite.kt | 2 +- .../cord05Invites/ConcordInviteList.kt | 24 ++- .../cord05Invites/ConcordInviteListEvent.kt | 16 +- .../cord05Invites/ConcordInviteListTest.kt | 39 +++- 11 files changed, 339 insertions(+), 141 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt index 543206fc94..daad3aa9c7 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt @@ -171,74 +171,97 @@ class AccountConcordActions( // ---- CORD-05 Invite List (kind 13303) ------------------------------------- /** - * This account's Invite List: the creator's private, self-encrypted record of every link they - * minted (`token` + `signer_sk` per entry). Empty when none was ever published. + * This account's Invite List (kind 13303): the creator's private, self-encrypted record of every + * link they minted (`token` + `signer_sk` per entry). * - * Fetched rather than read from [LocalCache] because nothing subscribes to 13303 — it is + * Returns **null** when the list could not be read — no relay answered, or the signer refused + * the decrypt — and an empty document only when the account genuinely has no list yet. Callers + * must not conflate the two: republishing an "empty" list over this replaceable coordinate + * destroys every `signer_sk` it failed to read, and those secrets cannot be regenerated. + * + * Read on the account's OUTBOX relays, never a community's: the coordinate is + * (13303, me, "") — one list for the whole account — so scoping it per community would fork it + * into divergent versions that the newest-wins rule then silently collapses. + * + * Fetched rather than read from [LocalCache] because nothing subscribes to 13303: it is * bookkeeping the user never sees, needed only at mint and at rotation. */ - private suspend fun readConcordInviteList(relays: Set): ConcordInviteListDocument { - if (relays.isEmpty()) return ConcordInviteListDocument.EMPTY + private suspend fun readConcordInviteList(): ConcordInviteListDocument? { + val relays = account.outboxRelays.flow.value + if (relays.isEmpty()) return null val filter = Filter(kinds = listOf(ConcordInviteListEvent.KIND), authors = listOf(account.signer.pubKey)) val newest = account.client .fetchAll(filters = relays.associateWith { listOf(filter) }) .maxByOrNull { it.createdAt } - return (newest as? ConcordInviteListEvent)?.decrypt(account.signer) ?: ConcordInviteListDocument.EMPTY + ?: return ConcordInviteListDocument.EMPTY // nothing published yet — safe to start one + return (newest as? ConcordInviteListEvent)?.decrypt(account.signer) } /** - * Merges [patch] into the published Invite List and republishes it. Read-merge-write, never - * overwrite: the list is replaceable and per-creator, so two of the user's devices minting - * concurrently would otherwise delete each other's `signer_sk` — and that secret is - * unrecoverable, orphaning the link at whatever epoch it was last refreshed to. + * Merges [patch] into the published Invite List and republishes it, returning whether it landed. + * + * Read-merge-write, and **aborts rather than overwriting** when the read fails: the list is + * replaceable, so publishing a patch-only document over an unread list deletes every other + * link's `signer_sk` — unrecoverable, and it strands every holder of those links at the next + * rotation. A momentarily unreachable relay or a bunker signer that declines one decrypt is + * enough to trigger that, which is exactly how the kind-13302 community list was once emptied. */ - private suspend fun publishConcordInviteList( - patch: ConcordInviteListDocument, - relays: Set, - ) { - val publishTo = relays.ifEmpty { account.outboxRelays.flow.value } - if (publishTo.isEmpty()) return - val merged = ConcordInviteList.merge(readConcordInviteList(publishTo), patch) - account.client.publish(ConcordInviteListEvent.create(account.signer, merged, TimeUtils.now()), publishTo) + private suspend fun publishConcordInviteList(patch: ConcordInviteListDocument): Boolean { + val publishTo = account.outboxRelays.flow.value + if (publishTo.isEmpty()) return false + val base = + readConcordInviteList() ?: run { + Log.w("Concord") { "Refusing to write the invite list: could not read the current one (would drop other links' signer_sk)" } + return false + } + return runCatching { + account.client.publish(ConcordInviteListEvent.create(account.signer, ConcordInviteList.merge(base, patch), TimeUtils.now()), publishTo) + true + }.onFailure { Log.w("Concord", "invite list publish failed", it) }.getOrDefault(false) } /** * Re-posts every live link this account minted for [entry]'s community at its own coordinate, - * carrying the CURRENT epoch (CORD-05). The kind-33301 bundle is addressable and authored by the + * carrying [entry]'s epoch (CORD-05). The kind-33301 bundle is addressable and authored by the * link signer, so this moves the link behind the same URL instead of orphaning it at a dead * epoch — which is the whole premise stranded recovery rests on. * - * Safe to call for every live link: recovery is ban-gated at the epoch being left, and a - * Refounding bans the members it removes on the way out, so a removed member's own recovery is - * refused even though their link now resolves. + * [entry] MUST be the post-rotation entry, passed in rather than re-read: the joined-list flow + * decrypts asynchronously, so reading it straight after adopting a new root yields the OLD + * epoch and would re-mint every link onto the epoch we just left. + * + * Each link is refreshed from its own CURRENT bundle, not rebuilt from scratch, so per-link + * fields the bundle carries — expiry, channel grants, icon, label — survive the rotation. A + * coordinate whose newest event is a revocation tombstone is left alone: re-posting a live + * bundle over it would silently un-revoke the link. */ private suspend fun refreshConcordInviteLinks(entry: ConcordCommunityListEntry): Int { val relays = entry.relays.mapNotNullTo(mutableSetOf()) { RelayUrlNormalizer.normalizeOrNull(it) }.ifEmpty { account.outboxRelays.flow.value } if (relays.isEmpty()) return 0 + val list = readConcordInviteList() ?: return 0 + val tombstoned = list.tombstones.mapTo(HashSet()) { it.token } val now = TimeUtils.now() - val refreshed = - ConcordActions.inviteFor( - communityIdHex = entry.id, - ownerPubKey = entry.owner, - ownerSaltHex = entry.ownerSalt, - communityRootHex = entry.root, - rootEpoch = entry.rootEpoch, - name = entry.name, - relays = entry.relays, - controlPk = entry.controlPk, - ) var count = 0 - for (link in readConcordInviteList(relays).entries) { + for (link in list.entries) { if (link.communityId != entry.id) continue - // An elapsed link can no longer be joined, so re-posting it would only resurrect a dead - // URL at a live epoch. - if (link.isExpired(now)) continue + // An elapsed or retired link can no longer be joined; re-posting it would only resurrect + // a dead URL at a live epoch. + if (link.isExpired(now) || link.token in tombstoned) continue runCatching { - account.client.publish( - ConcordActions.remintBundleAt(link.signerSk.hexToByteArray(), link.token.hexToByteArray(), refreshed, now), - relays, - ) + val token = link.token.hexToByteArray() + val wraps = account.client.fetchAll(filters = relays.associateWith { listOf(ConcordActions.bundleFilter(link.signerPubKeyHex())) }) + // Honour a revocation published at this coordinate, and carry the live bundle's own + // fields forward — only the epoch's key material changes. + val current = ConcordActions.classifyInvite(wraps, token) as? InviteBundleStatus.Live ?: return@runCatching + val moved = + current.invite.copy( + communityRoot = entry.root, + rootEpoch = entry.rootEpoch, + controlPk = entry.controlPk, + relays = entry.relays, + ) + account.client.publish(ConcordActions.remintBundleAt(link.signerSk.hexToByteArray(), token, moved, now), relays) count++ }.onFailure { Log.w("Concord", "invite refresh failed for ${entry.id}", it) } } @@ -289,25 +312,30 @@ class AccountConcordActions( val minted = ConcordActions.mintInviteLink(base, invite, TimeUtils.now(), entry.relays) val publishTo = entry.relays.mapNotNullTo(mutableSetOf()) { RelayUrlNormalizer.normalizeOrNull(it) }.ifEmpty { account.outboxRelays.flow.value } - if (publishTo.isNotEmpty()) account.client.publish(minted.bundleEvent, publishTo) - - // Record the link so a later Refounding can refresh THIS coordinate rather than orphaning it - // (CORD-05, kind 13303). Shared with amy and Armada, so any of the creator's clients can. - publishConcordInviteList( - ConcordInviteListDocument( - entries = - listOf( - ConcordInviteListEntry( - token = minted.token.toHexKey(), - signerSk = minted.linkSignerPrivKey.toHexKey(), - communityId = entry.id, - url = minted.url, - createdAt = TimeUtils.now(), + // Record the link BEFORE handing the URL out (CORD-05, kind 13303). A link whose `signer_sk` + // was never stored can never be refreshed, so the next Refounding orphans it and everyone + // holding it is stranded — with nothing to have warned them. Failing the mint is the honest + // outcome; a stored entry for a link nobody received is harmless by comparison. + if (!publishConcordInviteList( + ConcordInviteListDocument( + entries = + listOf( + ConcordInviteListEntry( + token = minted.token.toHexKey(), + signerSk = minted.linkSignerPrivKey.toHexKey(), + communityId = entry.id, + url = minted.url, + createdAt = TimeUtils.now(), + ), ), - ), - ), - publishTo, - ) + ), + ) + ) { + Log.w("Concord") { "Invite not minted for ${entry.id}: its link signer could not be recorded, so the link could never be refreshed" } + return null + } + + if (publishTo.isNotEmpty()) account.client.publish(minted.bundleEvent, publishTo) return minted.url } @@ -373,6 +401,32 @@ class AccountConcordActions( return ConcordInviteResult.Joined(bundle.communityId) } + // Refuse a link that readmits us after we were removed. A Refounding re-mints every + // outstanding link onto the new root (CORD-05), and an ex-member keeps the URL and its + // unlock token forever — so without this the rotation meant to expel them hands them the new + // keys instead. `recoverStrandedConcordCommunities` has always been ban-gated; this is the + // other door into the same room. + // + // Fails CLOSED on an unreadable plane: the banlist is only knowable once the bundle yields + // the root, and no verdict means no join. + val joinKeys = + ConcordActions.controlPlaneKeys( + communityRoot = bundle.communityRoot.hexToByteArray(), + communityId = bundle.communityId.hexToByteArray(), + rootEpoch = bundle.rootEpoch, + controlPk = bundle.controlPk, + ) + val joinRelays = bundle.relays.mapNotNullTo(mutableSetOf()) { RelayUrlNormalizer.normalizeOrNull(it) }.ifEmpty { relays } + val joinEditions = + ConcordActions.controlEditions( + account.client.fetchAll(filters = joinRelays.associateWith { listOf(ConcordActions.planeFilter(joinKeys.address)) }), + joinKeys, + ) + if (joinEditions.isEmpty()) return ConcordInviteResult.NotReachable + if (AuthorityResolver.resolve(joinEditions, bundle.owner).isBanned(account.signer.pubKey)) { + return ConcordInviteResult.Banned + } + val entry = ConcordCommunityListEntry( id = bundle.communityId, @@ -960,14 +1014,13 @@ class AccountConcordActions( // 5. Adopt the new epoch ourselves. This rebuilds our session under the new root and // re-folds the compacted Control Plane (with the ban), dropping the removed members. - adoptConcordRoot(entry, newRoot, build.newEpoch, build.newControlKeys.address.hexToByteArray(), newControlRoot) + val adopted = adoptConcordRoot(entry, newRoot, build.newEpoch, build.newControlKeys.address.hexToByteArray(), newControlRoot) // 6. Move every link we minted to the new epoch. Without this the Refounding orphans them, // and a member it left out — no rekey blob, no message to miss — has no way back at all. - val moved = - account.concordChannelList.liveCommunities.value - .firstOrNull { it.id == communityId } - ?.let { refreshConcordInviteLinks(it) } ?: 0 + // Uses the entry adoption just wrote: `liveCommunities` decrypts asynchronously, so + // reading it here would hand us the epoch we just left and re-mint every link onto it. + val moved = adopted?.let { refreshConcordInviteLinks(it) } ?: 0 Log.i("Concord") { "Refounding ${entry.id}: refreshed $moved invite link(s) to epoch ${build.newEpoch}" } return true } @@ -1033,8 +1086,8 @@ class AccountConcordActions( newEpoch: Long, newControlPk: ByteArray? = null, newControlRoot: ByteArray? = null, - ) { - if (!adoptedConcordRotations.add("${entry.id}:$newEpoch")) return + ): ConcordCommunityListEntry? { + if (!adoptedConcordRotations.add("${entry.id}:$newEpoch")) return null // The rewrite itself — banking the leaving epoch's address for the anti-rollback floor, // dropping stale control material on a legacy rotation, preserving invite_ref and residue — // is shared with `amy` in [ConcordReceive.withAdoptedRoot]. Only the persist + publish and @@ -1042,6 +1095,7 @@ class AccountConcordActions( val next = ConcordReceive.withAdoptedRoot(entry, newRoot, newEpoch, newControlPk, newControlRoot) account.sendMyPublicAndPrivateOutbox(account.concordChannelList.follow(next)) announceConcordGuestbookJoin(next, inviteCreator = null, inviteLabel = null) + return next } /** diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/ConcordInviteResult.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/ConcordInviteResult.kt index 6b4d2599cb..8f502e358b 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/ConcordInviteResult.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/ConcordInviteResult.kt @@ -54,6 +54,15 @@ sealed interface ConcordInviteResult { */ data object Expired : ConcordInviteResult + /** + * The link opens, but this community's roster has banned us (CORD-04). + * + * A Refounding re-mints every outstanding link onto the new root, and a removed member keeps the + * URL and its unlock token forever — so honouring the link alone would hand the new keys to the + * very account the rotation expelled. + */ + data object Banned : ConcordInviteResult + /** * The bundle event was found but could not be opened with the link's token — * typically because it was minted by a newer/incompatible Concord client whose diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordInviteScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordInviteScreen.kt index 2788dee7ee..d56c800791 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordInviteScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordInviteScreen.kt @@ -124,6 +124,8 @@ fun ConcordInviteScreen( RedeemState.Failed(R.string.concord_invite_failed_incompatible, canRetry = false) is ConcordInviteResult.Revoked -> RedeemState.Failed(R.string.concord_invite_failed_revoked, canRetry = false) + is ConcordInviteResult.Banned -> + RedeemState.Failed(R.string.concord_invite_failed_banned, canRetry = false) is ConcordInviteResult.Expired -> RedeemState.Failed(R.string.concord_invite_failed_expired, canRetry = false) is ConcordInviteResult.NotReachable -> diff --git a/amethyst/src/main/res/values/strings.xml b/amethyst/src/main/res/values/strings.xml index f3e96dffdf..20e32ea0b7 100644 --- a/amethyst/src/main/res/values/strings.xml +++ b/amethyst/src/main/res/values/strings.xml @@ -321,6 +321,7 @@ This invite link is invalid or can\'t be opened with this account. This invite link can\'t be opened. It may be outdated or already replaced by a newer one, or created with a newer version of the app. Ask for a fresh invite link. This invite link has been revoked and can no longer be used. Ask for a new one. + This community has removed you. The link still works, but its member list does not admit you. This invite link has expired and can no longer be used. Ask for a fresh link. Community name is only revealed after you join Joining connects to this invite\'s relays, publishes a join announcement signed by your account, and adds the community to your list. Nothing is sent until you tap Join. diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt index 76e7a8d59b..f2890e7775 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt @@ -230,7 +230,7 @@ object ConcordCommands { controlRoot = e.controlRoot ?: priorSameEpoch?.controlRoot ?: "", generalChannelId = prior?.generalChannelId ?: "", relays = e.relays, - heldRoots = e.heldRoots.map { StoredHeldRoot(it.epoch, it.key, it.controlPk ?: "") }, + heldRoots = e.heldRoots.map { StoredHeldRoot(it.epoch, it.key, it.controlPk ?: "", it.controlRoot ?: "") }, // Survives every merge: losing the anchor makes the NEXT exclusion // unrecoverable, so a list entry without one must not clear ours. inviteRef = e.inviteRef ?: prior?.inviteRef ?: "", @@ -266,16 +266,13 @@ object ConcordCommands { // (CORD-05 §1); omitted for a legacy community, which has none to carry. val invite = ConcordActions.inviteFor(sc.communityId, sc.owner, sc.ownerSalt, sc.root, sc.rootEpoch, sc.name, sc.relays, sc.controlPk.ifBlank { null }) val minted = ConcordActions.mintInviteLink(base, invite, TimeUtils.now(), sc.relays) - val ack = ctx.publish(minted.bundleEvent, relaysFor(ctx, sc)) - RawEventSupport.publishGuard(ack, minted.bundleEvent.id)?.let { return it } - - // Record the link in the CORD-05 Invite List (kind 13303) so any of this creator's - // clients — Amethyst, Armada — can later refresh THIS coordinate instead of orphaning - // the link at a dead epoch. That list is the liveness half of stranded recovery (A2). - publishInviteList( - ctx, - extraRelays = relaysFor(ctx, sc), - patch = + // Record the link BEFORE publishing the bundle (CORD-05, kind 13303): a link whose + // `signer_sk` was never stored can never be refreshed, so the next Refounding orphans + // it and every holder is stranded. Better to mint nothing than to hand out a link that + // is already doomed. + val recorded = + publishInviteList( + ctx, ConcordInviteListDocument( entries = listOf( @@ -288,7 +285,16 @@ object ConcordCommands { ), ), ), - ) + ) + if (!recorded) { + return Output.error( + "invite_unrecordable", + "could not record the link signer in your invite list (kind 13303), so this link could never be refreshed after a Refounding — not minting it", + ) + } + + val ack = ctx.publish(minted.bundleEvent, relaysFor(ctx, sc)) + RawEventSupport.publishGuard(ack, minted.bundleEvent.id)?.let { return it } Output.emit( mapOf( @@ -318,6 +324,34 @@ object ConcordCommands { wraps.firstNotNullOfOrNull { ConcordActions.openBundle(it, parsed.fragment.token) } ?: return Output.error("not_found", "no valid bundle for this link").let { 1 } + // Refuse a link that readmits us after we were removed. A Refounding re-mints every + // outstanding link onto the new root (CORD-05), and an ex-member keeps the URL and its + // unlock token forever — so without this check the rotation that was supposed to expel + // them hands them the new keys instead. `recover` has always been ban-gated; `join` is + // the other door into the same room. + // + // Fails CLOSED on an unreadable plane: no verdict, no join. The banlist is only knowable + // after the bundle yields the root, which is why the check lives here rather than before. + val joinKeys = + ConcordActions.controlPlaneKeys( + communityRoot = bundle.communityRoot.hexToByteArray(), + communityId = bundle.communityId.hexToByteArray(), + rootEpoch = bundle.rootEpoch, + controlPk = bundle.controlPk, + ) + val joinRelays = normalize(bundle.relays).ifEmpty { relays } + val joinEditions = + ConcordActions.controlEditions( + ctx.drain(joinRelays.associateWith { listOf(ConcordActions.planeFilter(joinKeys.address)) }, pendingOnAuthRequired = true).map { it.second }, + joinKeys, + ) + if (joinEditions.isEmpty()) { + return Output.error("control_plane_unreadable", "could not fold this community's Control Plane, so whether it has banned you is unknown — refusing to join") + } + if (AuthorityResolver.resolve(joinEditions, bundle.owner).isBanned(ctx.signer.pubKey)) { + return Output.error("banned", "this community has banned this account; the link works but the roster does not admit you (CORD-04)") + } + ConcordStore(dataDir.concordFile).upsert( StoredCommunity( name = bundle.name, @@ -401,7 +435,7 @@ object ConcordCommands { rootEpoch = sc.rootEpoch, controlPk = sc.controlPk.ifBlank { null }, controlRoot = sc.controlRoot.ifBlank { null }, - heldRoots = sc.heldRoots.map { HeldRoot(it.epoch, it.root, it.controlPk.ifBlank { null }) }, + heldRoots = sc.heldRoots.map { HeldRoot(it.epoch, it.root, it.controlPk.ifBlank { null }, it.controlRoot.ifBlank { null }) }, relays = sc.relays, name = sc.name, inviteRef = sc.inviteRef.ifBlank { null }, @@ -416,7 +450,7 @@ object ConcordCommands { rootEpoch = entry.rootEpoch, controlPk = entry.controlPk ?: "", controlRoot = entry.controlRoot ?: "", - heldRoots = entry.heldRoots.map { StoredHeldRoot(it.epoch, it.key, it.controlPk ?: "") }, + heldRoots = entry.heldRoots.map { StoredHeldRoot(it.epoch, it.key, it.controlPk ?: "", it.controlRoot ?: "") }, relays = entry.relays, name = entry.name.ifBlank { sc.name }, inviteRef = entry.inviteRef ?: sc.inviteRef, @@ -585,36 +619,39 @@ object ConcordCommands { * of every link they minted, so a rotation can refresh those links instead of orphaning them. * Empty when none was ever published. */ - suspend fun readInviteList( - ctx: Context, - extraRelays: Set = emptySet(), - ): ConcordInviteListDocument { - val relays = ctx.outboxRelays() + extraRelays - if (relays.isEmpty()) return ConcordInviteListDocument.EMPTY + suspend fun readInviteList(ctx: Context): ConcordInviteListDocument? { + val relays = ctx.outboxRelays() + if (relays.isEmpty()) return null val filter = Filter(kinds = listOf(ConcordInviteListEvent.KIND), authors = listOf(ctx.signer.pubKey)) val newest = ctx .drain(relays.associateWith { listOf(filter) }) .map { it.second } .maxByOrNull { it.createdAt } - return (newest as? ConcordInviteListEvent)?.decrypt(ctx.signer) ?: ConcordInviteListDocument.EMPTY + ?: return ConcordInviteListDocument.EMPTY // nothing published yet — safe to start one + return (newest as? ConcordInviteListEvent)?.decrypt(ctx.signer) } /** - * Merges [patch] into the published list and republishes it. Read-merge-write rather than - * overwrite: the list is replaceable and per-creator, so two devices minting concurrently would - * otherwise delete each other's links (and their `signer_sk`, which is unrecoverable). + * Merges [patch] into the published list and republishes it, returning whether it landed. + * + * Read-merge-write, and **aborts rather than overwriting** when the read fails: kind 13303 is + * replaceable, so writing a patch-only document over a list we could not read deletes every + * other link's `signer_sk`. Those secrets cannot be regenerated, and losing one orphans its + * link at the next rotation, stranding everyone holding that URL. + * + * Account-scoped, like the coordinate itself — (13303, me, "") is one list for every community, + * so reading or writing it on a single community's relays would fork it. */ suspend fun publishInviteList( ctx: Context, patch: ConcordInviteListDocument, - extraRelays: Set = emptySet(), - ) { - val relays = ctx.outboxRelays() + extraRelays - if (relays.isEmpty()) return - val merged = ConcordInviteList.merge(readInviteList(ctx, extraRelays), patch) - val event = ConcordInviteListEvent.create(ctx.signer, merged, TimeUtils.now()) - ctx.publish(event, relays) + ): Boolean { + val relays = ctx.outboxRelays() + if (relays.isEmpty()) return false + val base = readInviteList(ctx) ?: return false + val event = ConcordInviteListEvent.create(ctx.signer, ConcordInviteList.merge(base, patch), TimeUtils.now()) + return ctx.publish(event, relays).values.any { it.accepted } } fun notFound(handle: String): Int { diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt index 23ec2a47ef..9b52786df6 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordModCommands.kt @@ -33,7 +33,9 @@ import com.vitorpamplona.quartz.concord.cord02Community.ConcordCommunityState import com.vitorpamplona.quartz.concord.cord04Roles.ConcordPermissions import com.vitorpamplona.quartz.concord.cord04Roles.ControlEdition import com.vitorpamplona.quartz.concord.cord04Roles.RoleEntity +import com.vitorpamplona.quartz.concord.cord05Invites.InviteBundleStatus import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys +import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray import com.vitorpamplona.quartz.nip01Core.core.toHexKey import com.vitorpamplona.quartz.utils.RandomInstance @@ -272,26 +274,37 @@ object ConcordModCommands { // 1. Ban the removed on the CURRENT plane, so the compacted snapshot — and therefore the // new epoch — carries the ban. Each edition chains onto the updated banlist head. var chain = editions + val banWraps = mutableListOf() for (target in removed) { val banWrap = ConcordModeration.ban(ctx.signer, cp, sc.communityId.hexToByteArray(), target, chain, TimeUtils.now(), owner = sc.owner) - ctx.publish(banWrap, relays) + val ack = ctx.publish(banWrap, relays) + if (ack.values.none { it.accepted }) { + return Output.error("ban_not_published", "the pre-rotation ban for $target was not accepted by any relay; refusing to refound with a banlist that would not survive") + } + banWraps += banWrap chain = chain + (ConcordActions.controlEditions(listOf(banWrap), cp)) } // 2. Everyone we are keeping. See the note above on why this reaches past the roster. - val recipients = + val candidates = (rosterOf(authority) + guestbookMembersOf(ctx, sc) + channelAuthorsOf(ctx, sc, state) + me) .mapTo(HashSet()) { it.lowercase() } .apply { removeAll(removed) removeAll(authority.bannedMembers().map { it.lowercase() }.toSet()) - }.toList() + } + val recipients = boundRecipients(candidates, authority) // 3. Build: new root + fresh control_root, compacted plane, per-recipient blobs (staff // get the 136-byte form carrying the secret, everyone else the 104-byte pubkey one). val newRoot = RandomInstance.bytes(32) val newControlRoot = RandomInstance.bytes(32) - val controlWraps = ctx.drain(relays.associateWith { listOf(ConcordActions.planeFilter(cp.address)) }, pendingOnAuthRequired = true).map { it.second } + // Compact from what we KNOW the plane holds: the wraps we drained plus the bans we just + // published. Re-draining alone would race the relay's indexing, and a relay that has not + // yet echoed the ban back (or that ACKed and stored nothing) would produce a new epoch + // whose roster never banned the member we are removing. + val drained = ctx.drain(relays.associateWith { listOf(ConcordActions.planeFilter(cp.address)) }, pendingOnAuthRequired = true).map { it.second } + val controlWraps = (drained + banWraps).distinctBy { it.id } val build = ConcordActions.buildRefounding( rotatorSigner = ctx.signer, @@ -335,33 +348,30 @@ object ConcordModCommands { // Safe for every link because recovery is ban-gated at the epoch being left, and step 1 // banned everyone being removed — so a removed member's own `recover` is refused even // though their link now resolves. - val refreshedInvite = - ConcordActions.inviteFor( - stored.communityId, - stored.owner, - stored.ownerSalt, - stored.root, - stored.rootEpoch, - stored.name, - stored.relays, - stored.controlPk.ifBlank { null }, - ) val now = TimeUtils.now() var refreshed = 0 - for (link in ConcordCommands.readInviteList(ctx, relays).entries) { + val list = ConcordCommands.readInviteList(ctx) + val tombstoned = list?.tombstones?.mapTo(HashSet()) { it.token } ?: emptySet() + for (link in list?.entries.orEmpty()) { if (link.communityId != stored.communityId) continue - // An elapsed link can no longer be joined, so re-posting it would only resurrect a - // dead URL at a live epoch (CORD-05). - if (link.isExpired(now)) continue + // An elapsed or retired link can no longer be joined, so re-posting it would only + // resurrect a dead URL at a live epoch (CORD-05). + if (link.isExpired(now) || link.token in tombstoned) continue runCatching { - val event = - ConcordActions.remintBundleAt( - linkSignerPrivKey = link.signerSk.hexToByteArray(), - token = link.token.hexToByteArray(), - invite = refreshedInvite, - createdAt = now, + val token = link.token.hexToByteArray() + // Refresh from the link's CURRENT bundle so its own fields — expiry, channel + // grants, icon, label — survive the rotation, and so a coordinate whose newest + // event is a revocation tombstone is left revoked instead of being re-opened. + val wraps = ctx.drain(relays.associateWith { listOf(ConcordActions.bundleFilter(link.signerPubKeyHex())) }).map { it.second } + val live = ConcordActions.classifyInvite(wraps, token) as? InviteBundleStatus.Live ?: return@runCatching + val moved = + live.invite.copy( + communityRoot = stored.root, + rootEpoch = stored.rootEpoch, + controlPk = stored.controlPk.ifBlank { null }, + relays = stored.relays, ) - ctx.publish(event, relays) + ctx.publish(ConcordActions.remintBundleAt(link.signerSk.hexToByteArray(), token, moved, now), relays) refreshed++ } } @@ -382,6 +392,40 @@ object ConcordModCommands { } } + /** + * How many recipients one Refounding will re-key, mirroring Amethyst's own cap. + * + * Two thirds of the recipient union — Guestbook joins and observed channel authors — are + * attacker-writable: any key can announce a join or post once. Without a bound, padding those + * sets inflates the cost of the only hard removal Concord has until rotating becomes + * impractical, so the attack raises the price of its own remedy (B4 in the soft-ban audit). + */ + private const val MAX_REFOUNDING_RECIPIENTS = 5_000 + + /** + * Caps [candidates], keeping the members whose standing is owner-rooted and therefore cannot be + * padded from outside. Anything dropped is reported rather than silently truncated — a dropped + * member is stranded on the dead epoch and their only way back is `concord recover`. + */ + private fun boundRecipients( + candidates: Set, + authority: com.vitorpamplona.quartz.concord.cord04Roles.AuthorityResolver, + ): List { + if (candidates.size <= MAX_REFOUNDING_RECIPIENTS) return candidates.toList() + val vouched = (authority.roleHolders() + authority.staffMembers()).mapTo(HashSet()) { it.lowercase() } + val kept = LinkedHashSet() + candidates.filterTo(kept) { it in vouched } + for (candidate in candidates) { + if (kept.size >= MAX_REFOUNDING_RECIPIENTS) break + kept.add(candidate) + } + val dropped = candidates.size - kept.size + if (dropped > 0) { + System.err.println("[concord] refounding recipient set trimmed to ${kept.size} of ${candidates.size}: $dropped member(s) will be stranded on the prior epoch") + } + return kept.toList() + } + /** Owner + everyone holding a role — owner-rooted, so it cannot be padded from outside. */ private fun rosterOf(authority: com.vitorpamplona.quartz.concord.cord04Roles.AuthorityResolver): Set = (authority.roleHolders() + authority.staffMembers()).mapTo(HashSet()) { it.lowercase() } diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt index d2de85a258..c36b1ac915 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/stores/ConcordStore.kt @@ -61,6 +61,12 @@ data class StoredHeldRoot( val root: String = "", /** That epoch's Control Plane address; blank for a legacy, pre-split epoch (CORD-02 §5). */ val controlPk: String = "", + /** + * That epoch's staff write key, banked only if we held it. A relay that gates the prior epoch's + * Control Plane on NIP-42 AUTH as the stream key will not serve those wraps without it — and + * those wraps are what rebuild the anti-rollback floor. + */ + val controlRoot: String = "", ) /** diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/CommunityInvite.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/CommunityInvite.kt index d652c70143..632a7f9e03 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/CommunityInvite.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/CommunityInvite.kt @@ -52,7 +52,7 @@ class InviteChannel( * into a kind-33301 bundle (link invites) or a NIP-59 giftwrap (direct invites). */ @Serializable -class CommunityInvite( +data class CommunityInvite( @SerialName("community_id") val communityId: String, val owner: String, @SerialName("owner_salt") val ownerSalt: String, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteList.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteList.kt index 3eea2095c3..5b930eb261 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteList.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteList.kt @@ -21,6 +21,10 @@ package com.vitorpamplona.quartz.concord.cord05Invites import com.vitorpamplona.quartz.concord.cord04Roles.ConcordJson +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray +import com.vitorpamplona.quartz.nip01Core.core.toHexKey +import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.KSerializer import kotlinx.serialization.SerialName @@ -58,6 +62,12 @@ class ConcordInviteListEntry( ) { /** True when this link can no longer be joined, so it must not be refreshed (CORD-05). */ fun isExpired(nowSecs: Long): Boolean = expiresAt != null && expiresAt <= nowSecs + + /** + * The link signer's pubkey — the addressable coordinate the bundle lives at, derived from the + * secret we kept. Refreshing or retiring a link means writing at exactly this author. + */ + fun signerPubKeyHex(): HexKey = KeyPair(privKey = signerSk.hexToByteArray()).pubKey.toHexKey() } /** A retired link: the creator's record that [token] is gone, kept so a merge cannot resurrect it. */ @@ -150,11 +160,15 @@ object ConcordInviteList { private object WireDocumentSerializer : ExtrasPreserving(WireDocument.serializer()) /** - * Decodes the plaintext document. A malformed document yields [ConcordInviteListDocument.EMPTY] - * rather than throwing — but note the sharp edge this shape shares with the community list: one - * unparseable entry aborts the whole array, so every field defaults instead of being required. + * Decodes the plaintext document, or **null** when it cannot be parsed. + * + * Null rather than an empty document on purpose: this list is replaceable, so a caller that + * treats "I could not read it" as "it is empty" and republishes destroys every `signer_sk` it + * did not manage to read — secrets that cannot be regenerated, orphaning every outstanding + * invite at a dead epoch. Callers MUST distinguish the two (see [ConcordInviteList.merge]'s + * callers). Each field still defaults, so one odd entry does not abort the whole array. */ - fun decode(json: String): ConcordInviteListDocument = + fun decodeOrNull(json: String): ConcordInviteListDocument? = try { val doc = ConcordJson.instance.decodeFromString(WireDocumentSerializer, json) ConcordInviteListDocument( @@ -166,7 +180,7 @@ object ConcordInviteList { residue = doc.extras, ) } catch (_: Exception) { - ConcordInviteListDocument.EMPTY + null } fun encode(doc: ConcordInviteListDocument): String = diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListEvent.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListEvent.kt index f4bbd80f1c..3816f60a6f 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListEvent.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListEvent.kt @@ -52,15 +52,19 @@ class ConcordInviteListEvent( sig: HexKey, ) : BaseReplaceableEvent(id, pubKey, createdAt, KIND, tags, content, sig) { /** - * Decrypts the whole document with [signer] — entries, tombstones and the document residue. - * Use this (never a partial read) whenever the result will be re-encoded, or another client's - * unknown keys are dropped on the next publish. + * Decrypts the whole document with [signer] — entries, tombstones and the document residue — or + * **null** if it cannot be decrypted or parsed. + * + * Null, never empty: a caller that reads a decrypt failure as "no links yet" and republishes + * wipes every `signer_sk` on this replaceable coordinate. A bunker signer that momentarily + * refuses is enough to trigger it. Use this (never a partial read) whenever the result will be + * re-encoded, or another client's unknown keys are dropped on the next publish. */ - suspend fun decrypt(signer: NostrSigner): ConcordInviteListDocument = + suspend fun decrypt(signer: NostrSigner): ConcordInviteListDocument? = try { - ConcordInviteList.decode(signer.nip44Decrypt(content, signer.pubKey)) + ConcordInviteList.decodeOrNull(signer.nip44Decrypt(content, signer.pubKey)) } catch (_: Exception) { - ConcordInviteListDocument.EMPTY + null } companion object { diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListTest.kt index e488aeccb5..7a3904bdbc 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListTest.kt @@ -46,7 +46,7 @@ class ConcordInviteListTest { @Test fun readsTheSpecDocumentIntoTypedEntries() { - val doc = ConcordInviteList.decode(specJson) + val doc = ConcordInviteList.decodeOrNull(specJson)!! assertEquals(1, doc.entries.size) val e = doc.entries.first() @@ -65,7 +65,7 @@ class ConcordInviteListTest { @Test fun emitsTheSnakeCaseKeysAnotherClientReads() { - val json = ConcordInviteList.encode(ConcordInviteList.decode(specJson)) + val json = ConcordInviteList.encode(ConcordInviteList.decodeOrNull(specJson)!!) // Field names are the interop contract — a camelCase slip silently orphans every link. for (key in listOf("\"token\"", "\"signer_sk\"", "\"community_id\"", "\"url\"", "\"created_at\"", "\"expires_at\"", "\"entries\"", "\"tombstones\"")) { assertTrue(json.contains(key), "missing wire key $key") @@ -84,7 +84,7 @@ class ConcordInviteListTest { "doc_level_unknown": 7 } """.trimIndent() - val round = ConcordInviteList.encode(ConcordInviteList.decode(withExtras)) + val round = ConcordInviteList.encode(ConcordInviteList.decodeOrNull(withExtras)!!) assertTrue(round.contains("future_field"), "entry-level unknown key dropped") assertTrue(round.contains("doc_level_unknown"), "document-level unknown key dropped") @@ -116,9 +116,36 @@ class ConcordInviteListTest { } @Test - fun aMalformedDocumentYieldsEmptyRatherThanThrowing() { - assertEquals(0, ConcordInviteList.decode("not json").entries.size) - assertEquals(0, ConcordInviteList.decode("{\"entries\":\"wrong type\"}").entries.size) + fun aMalformedDocumentYieldsNullSoCallersCannotOverwriteWithIt() { + // Null, not empty: a caller that republishes an "empty" list over this replaceable + // coordinate destroys every signer_sk it failed to read. + assertEquals(null, ConcordInviteList.decodeOrNull("not json")) + assertEquals(null, ConcordInviteList.decodeOrNull("{\"entries\":\"wrong type\"}")) + } + + @Test + fun anUnreadableListIsDistinguishableFromAnEmptyOne() { + // The whole point of the null: a caller must be able to tell "I could not read it" from + // "there is nothing in it". Publishing a merge onto the latter is fine; onto the former it + // destroys every signer_sk on this replaceable coordinate. + assertEquals(null, ConcordInviteList.decodeOrNull("")) + + val empty = ConcordInviteList.decodeOrNull("""{"entries":[],"tombstones":[]}""") + assertEquals(0, empty!!.entries.size, "a genuinely empty list decodes, it does not fail") + + // And a merge onto an empty base keeps the patch, so starting a first list still works. + val patch = ConcordInviteListDocument(entries = listOf(ConcordInviteListEntry("t", "sk", "c", "u"))) + assertEquals(listOf("t"), ConcordInviteList.merge(empty, patch).entries.map { it.token }) + } + + @Test + fun theSignerPubKeyIsTheCoordinateTheBundleLivesAt() { + // Refreshing or revoking a link means writing at exactly this author, so it must derive from + // the secret we kept rather than being stored (and drifting) separately. + val sk = "11".repeat(32) + val entry = ConcordInviteListEntry("t", sk, "c", "u") + assertEquals(64, entry.signerPubKeyHex().length) + assertEquals(entry.signerPubKeyHex(), ConcordInviteListEntry("t2", sk, "c", "u2").signerPubKeyHex()) } @Test From a1f980babd3d19f2447f12af521e422fab16e2e2 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Sun, 9 Aug 2026 23:40:30 -0400 Subject: [PATCH 108/132] fix(concord): make the invite failure messages visible in the dark theme MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found while proving the new ban gate on a tablet: the refusal reached the view hierarchy but rendered as black pixels on the black background, so the screen looked blank and the user was told nothing. `ConcordInviteScreen`'s Column sits on the bare window background with no Surface above it, so `LocalContentColor` is still Material 3's default black. This predates the invite work and silently affects every state the screen can end in — invalid, incompatible, revoked, expired, unreachable — plus the "Redeeming invite…" progress label, which is why only the spinner was ever visible while a join was in flight. Verified on device: the message now renders. Co-Authored-By: Claude Opus 5 (1M context) --- .../chats/publicChannels/concord/ConcordInviteScreen.kt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordInviteScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordInviteScreen.kt index d56c800791..7b19436713 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordInviteScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordInviteScreen.kt @@ -163,6 +163,10 @@ fun ConcordInviteScreen( Text( stringRes(R.string.concord_redeeming_invite), modifier = Modifier.padding(top = 16.dp), + // Explicit: this Column sits on the bare window background with no Surface + // above it, so LocalContentColor is still the M3 default black — which renders + // every one of these labels invisible in the dark theme. + color = MaterialTheme.colorScheme.onBackground, textAlign = TextAlign.Center, ) } @@ -172,6 +176,7 @@ fun ConcordInviteScreen( Text( stringRes(failed.messageRes), style = MaterialTheme.typography.bodyLarge, + color = MaterialTheme.colorScheme.onBackground, textAlign = TextAlign.Center, ) if (failed.canRetry) { From fc3f181184f4a034db604762fa7921f62d03d079 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Mon, 10 Aug 2026 00:30:09 -0400 Subject: [PATCH 109/132] =?UTF-8?q?feat(cli):=20add=20`amy=20concord=20rev?= =?UTF-8?q?oke`=20=E2=80=94=20retire=20an=20invite=20link=20(CORD-05)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reading half of revocation already existed: `classify` has always resolved a `vsk=9` tombstone to `Revoked`, and Amethyst's join honours it. Nothing anywhere could *produce* one, so a leaked link could only be outrun by a Refounding — rotating the whole community to retire one URL. `ConcordInviteBundle.buildRevocation` emits the grave the spec describes and Armada's `buildRevocationEvent` already publishes: kind 33301 at the link's own `["d",""]` coordinate, empty content, `["vsk","9"]`, signed by the `link_signer` secret. Empty content is the interop contract, not an omission — there is nothing to encrypt when the point is that no bundle key opens anything. `amy concord revoke COMMUNITY TOKEN|URL` takes either the shareable URL a creator actually has to hand or the bare token. It publishes the wire tombstone FIRST and records the kind-13303 tombstone second, which is the inverse of minting and deliberate: the list entry holds the only copy of the `signer_sk` the publish needs, and a merge drops a tombstoned token's entry terminally. Recording first and then failing to publish would leave the link live with its signer gone and no way left to retire it. A failed list write is recoverable by comparison and is reported rather than swallowed. Also fixes a revocation bypass in amy's own `join`, found while testing this: it opened the first wrap that decrypted instead of classifying the coordinate, so a relay still serving a stale copy alongside the grave would have handed out a revoked link. It now resolves per CORD-05 §2 like Amethyst does, and can say which of revoked/expired/unreadable/absent it hit instead of reporting everything as `not_found`. Verified end to end against a local relay: revoking flips the coordinate to vsk=9 with empty content, the link is refused from that moment on, a second revoke reports `already_revoked`, and a Refounding afterwards moves the surviving link while leaving the grave alone — the first real proof of the tombstone-skip in the refresh path, which until now had only unit coverage. Co-Authored-By: Claude Opus 5 (1M context) --- cli/README.md | 1 + cli/ROADMAP.md | 2 +- .../com/vitorpamplona/amethyst/cli/Main.kt | 1 + .../amethyst/cli/commands/ConcordCommands.kt | 107 +++++++++++++++++- .../commons/actions/ConcordActions.kt | 15 +++ .../cord05Invites/ConcordInviteBundle.kt | 16 +++ .../bundle/ConcordInviteBundleEvent.kt | 17 +++ .../ConcordInviteClassifyTest.kt | 35 ++++++ 8 files changed, 190 insertions(+), 4 deletions(-) diff --git a/cli/README.md b/cli/README.md index d3f0591f78..76ec8f2844 100644 --- a/cli/README.md +++ b/cli/README.md @@ -669,6 +669,7 @@ also carried on-relay as an encrypted kind:13302. | `amy concord send COMMUNITY CHANNEL TEXT` | Post a message (CHANNEL = `general`\|name\|id). | | `amy concord read COMMUNITY CHANNEL [--limit N] [--epoch N] [--root HEX]` | Read a channel's messages (default 50); `--epoch`/`--root` read a prior epoch's plane. | | `amy concord invite COMMUNITY [--base URL]` | Mint + publish a shareable invite link. | +| `amy concord revoke COMMUNITY TOKEN\|URL` | Retire a link you minted: publishes a `vsk=9` tombstone at its coordinate, then records it in your Invite List. | | `amy concord join URL` | Redeem an invite link and save the community. | | `amy concord roles COMMUNITY` | List live roles + the current banlist (CORD-04). | | `amy concord role COMMUNITY NAME POSITION PERM…` | Define a role (perms by name, e.g. `BAN KICK`). | diff --git a/cli/ROADMAP.md b/cli/ROADMAP.md index add1d87d06..590964f6d4 100644 --- a/cli/ROADMAP.md +++ b/cli/ROADMAP.md @@ -67,7 +67,7 @@ Status legend: ✅ shipped · 📦 logic lives in `commons/`, needs a command · | NIP-65 outbox model queries | ✅ | `OutboxCommand` — `amy outbox USER [--refresh]`, cache-first. | | CLINK offers + debits (`amy offer` / `amy debit`) | ✅ | `OfferCommands` + `DebitCommands` — pointer decode, NIP-05 discover, kind:21001/21002 round-trips, `offer pay --with NDEBIT` end-to-end settlement. `--timeout` is SECONDS. | | Geochat (Bitchat geohash, ephemeral kind:20000) | ✅ | `GeochatCommands` — listen/send/keys with per-geohash throwaway identity + geo-nearest relay routing; doubles as the Bitchat interop harness. | -| Concord Channels (encrypted communities) | ✅ | `ConcordCommands` — 13 sub-verbs (create/list/import/channels/send/read/invite/join/roles/role/grant/ban/unban) over shared `commons` `ConcordActions`; secrets in `concord.json`. | +| Concord Channels (encrypted communities) | ✅ | `ConcordCommands` — 17 sub-verbs (create/list/import/channels/send/read/invite/revoke/join/recover/rekey/roles/role/grant/ban/unban/refound) over shared `commons` `ConcordActions`; secrets in `concord.json`. | | NIP-5A nsites + NIP-5D napplets | ✅ | `NsiteCommands` + `NappletCommands` — fetch/publish/serve/list with sha256 + aggregate-hash verification and `requires` capability reporting. | | Podcasting 2.0 / podstr (`amy podcast20`) | ✅ | `Podcast20Commands` — kind:30078 metadata, 30054 episodes, 30055 trailers, list. | | Follows-of-follows (`amy fof get/list/sync`) | ✅ | `FofCommand` — single-hop social proof from the local store (`wot` kept as deprecation alias). | diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt index 5b14b8da61..0ddc3f5762 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Main.kt @@ -869,6 +869,7 @@ private fun printUsage() { | concord send COMMUNITY CHANNEL TEXT post a message (CHANNEL = general|name|id) | concord read COMMUNITY CHANNEL [--limit N] read a channel's messages | concord invite COMMUNITY [--base URL] mint + publish a shareable invite link + | concord revoke COMMUNITY TOKEN|URL retire a link you minted (vsk=9 tombstone) | concord join URL redeem an invite link and save the community | |Local event store (shared, under `/shared/`): diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt index f2890e7775..77047c4e5b 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt @@ -38,6 +38,7 @@ import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteList import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteListDocument import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteListEntry import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteListEvent +import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteListTombstone import com.vitorpamplona.quartz.concord.cord05Invites.InviteBundleStatus import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray @@ -66,6 +67,9 @@ object ConcordCommands { | concord read COMMUNITY CHANNEL [--limit N] read a channel's messages (default 50); | [--epoch N] [--root HEX] --epoch/--root read a prior epoch's plane | concord invite COMMUNITY [--base URL] mint + publish a shareable invite link + | concord revoke COMMUNITY TOKEN|URL retire a link you minted: publishes a vsk=9 + | tombstone at its coordinate, then tombstones + | it in your invite list so it stays retired | concord join URL redeem an invite link and save the community | concord rekey [COMMUNITY] follow a Refounding we were re-keyed for: | open our blob and adopt the new epoch @@ -89,7 +93,7 @@ object ConcordCommands { route( "concord", tail, - "concord ", + "concord ", help = USAGE, routes = mapOf( @@ -100,6 +104,7 @@ object ConcordCommands { "send" to { rest -> ConcordChannelCommands.send(dataDir, rest) }, "read" to { rest -> ConcordChannelCommands.read(dataDir, rest) }, "invite" to { rest -> invite(dataDir, rest) }, + "revoke" to { rest -> revoke(dataDir, rest) }, "join" to { rest -> join(dataDir, rest) }, "recover" to { rest -> recover(dataDir, rest) }, "rekey" to { rest -> rekey(dataDir, rest) }, @@ -307,6 +312,92 @@ object ConcordCommands { } } + /** + * `amy concord revoke ` — retires one link this account minted. + * + * Two records have to agree for a link to be gone, and they fail differently, so the order is + * deliberate. The wire tombstone (`vsk=9` at the link's own coordinate) is what actually stops + * a join, and publishing it needs the `signer_sk` that only the kind-13303 Invite List holds. + * The list tombstone is bookkeeping: it stops a later Refounding from re-minting the link. + * + * So the wire goes first and the list second. The reverse order would delete the entry — a + * merge drops a tombstoned token's entry terminally — and if the publish then failed, the link + * would stay live with its `signer_sk` gone and no way left to retire it. A failed list write + * is recoverable by comparison: the link is already dead on the wire, and the refresh path + * re-mints only a coordinate that still resolves Live, so it will not resurrect this one. + */ + private suspend fun revoke( + dataDir: DataDir, + rest: Array, + ): Int { + val args = Args(rest) + val handle = args.positional(0, "community") + val link = args.positional(1, "token|url") + args.rejectUnknown() + + // Accept either the shareable URL (what a creator actually has to hand) or the bare token. + val token = + ConcordActions + .parseInviteLink(link) + ?.fragment + ?.token + ?.toHexKey() ?: link.lowercase() + if (!TOKEN_HEX.matches(token)) { + return Output.error("bad_args", "expected an invite URL or a 32-hex-character link token, got '$link'").let { 2 } + } + + val sc = ConcordStore(dataDir.concordFile).find(handle) ?: return notFound(handle) + Context.open(dataDir).use { ctx -> + ctx.prepare() + + val list = + readInviteList(ctx) + ?: return Output.error("invite_list_unreadable", "could not read your invite list (kind 13303), so the link signer needed to revoke is unknown — refusing to guess") + + val entry = list.entries.firstOrNull { it.token == token } + if (entry == null) { + return if (list.tombstones.any { it.token == token }) { + Output.error("already_revoked", "this link was already revoked; its signer_sk is gone from the list, so there is nothing left to re-publish") + } else { + Output.error("not_found", "no link with token $token in your invite list — only the account that minted a link can revoke it") + } + } + if (entry.communityId != sc.communityId) { + return Output.error("wrong_community", "that link belongs to community ${entry.communityId}, not '$handle' (${sc.communityId})") + } + + val tombstone = ConcordActions.revokeBundleAt(entry.signerSk.hexToByteArray(), TimeUtils.now()) + val ack = ctx.publish(tombstone, relaysFor(ctx, sc)) + RawEventSupport.publishGuard(ack, tombstone.id)?.let { return it } + + val recorded = + publishInviteList( + ctx, + ConcordInviteListDocument(tombstones = listOf(ConcordInviteListTombstone(token = token, communityId = sc.communityId))), + ) + if (!recorded) { + System.err.println( + "[concord] the link is revoked on the wire but the tombstone could not be recorded in your invite list (kind 13303); re-run this command once your outbox relays are reachable", + ) + } + + Output.emit( + mapOf( + "revoked" to true, + "token" to token, + "community_id" to sc.communityId, + "link_signer" to entry.signerPubKeyHex(), + "tombstone_event_id" to tombstone.id, + "tombstoned_in_list" to recorded, + ) + RawEventSupport.ackFields(ack), + ) + return 0 + } + } + + /** A link token is 16 bytes on the wire, so 32 hex characters once stored in the list. */ + private val TOKEN_HEX = Regex("^[0-9a-f]{32}$") + private suspend fun join( dataDir: DataDir, rest: Array, @@ -320,9 +411,19 @@ object ConcordCommands { ctx.prepare() val relays = (normalize(parsed.fragment.relays) + ctx.bootstrapRelays()) val wraps = ctx.drain(relays.associateWith { listOf(ConcordActions.bundleFilter(parsed.linkSignerPubKey)) }).map { it.second } + // Resolve the coordinate per CORD-05 §2 rather than opening whatever happens to decrypt: + // the newest event wins, so a vsk=9 tombstone retires the link even when a stale but + // still-openable copy is also present. Opening the first wrap that decrypts would let a + // relay that kept the old version hand out a link its creator revoked — and it cannot + // tell the user which of "revoked", "expired" or "gone" they are looking at. val bundle = - wraps.firstNotNullOfOrNull { ConcordActions.openBundle(it, parsed.fragment.token) } - ?: return Output.error("not_found", "no valid bundle for this link").let { 1 } + when (val status = ConcordActions.classifyInvite(wraps, parsed.fragment.token)) { + is InviteBundleStatus.Live -> status.invite + is InviteBundleStatus.Expired -> return Output.error("expired", "this invite link has expired and can no longer be joined") + InviteBundleStatus.Revoked -> return Output.error("revoked", "this invite link was revoked by its creator") + InviteBundleStatus.Unreadable -> return Output.error("incompatible", "something is published at this link's coordinate, but it is not a bundle this client can open") + InviteBundleStatus.Absent -> return Output.error("not_found", "no bundle for this link on any of its relays") + } // Refuse a link that readmits us after we were removed. A Refounding re-mints every // outstanding link onto the new root (CORD-05), and an ex-member keeps the URL and its diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt index 6263c65add..dfb51935d7 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt @@ -459,6 +459,21 @@ object ConcordActions { createdAt: Long, ): Event = ConcordInviteBundle.build(linkSignerPrivKey, token, invite, createdAt) + /** + * Retires an existing link by publishing a `vsk=9` revocation tombstone at its coordinate + * (CORD-05 §2). Once this lands, every client resolving that URL gets + * [com.vitorpamplona.quartz.concord.cord05Invites.InviteBundleStatus.Revoked] instead of keys. + * + * Publish this *before* recording the tombstone in the kind-13303 Invite List — the list entry + * carries the only copy of the `signer_sk` this call needs, and the list merge drops a + * tombstoned token's entry for good. Recording first and failing to publish would leave the link + * live on the wire with no way left to retire it. + */ + fun revokeBundleAt( + linkSignerPrivKey: ByteArray, + createdAt: Long, + ): Event = ConcordInviteBundle.buildRevocation(linkSignerPrivKey, createdAt) + /** Parses a shareable invite URL into its pointer + private fragment. */ fun parseInviteLink(url: String): ParsedInviteLink? = ConcordInviteLink.parseUrl(url) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteBundle.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteBundle.kt index ee528efb5b..4f36ea409d 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteBundle.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteBundle.kt @@ -109,6 +109,22 @@ object ConcordInviteBundle { return signer.sign(ConcordInviteBundleEvent.build(content, createdAt)) } + /** + * Builds the kind-33301 revocation tombstone that retires the link owned by [linkSignerPrivKey] + * (CORD-05 §2). It re-posts the link's own coordinate with empty content and `vsk=9`, so the + * newest event there is a grave rather than keys and [classify] resolves the link + * [InviteBundleStatus.Revoked] for everyone who resolves it afterwards. + * + * Only the creator can do this: the coordinate is addressable and authored by the link signer, + * so retiring a link requires the `link_signer` secret — which lives in the creator's kind-13303 + * Invite List and nowhere else. Losing that secret makes a link permanently un-revokable, which + * is why the list is written before a link is ever handed out. + */ + fun buildRevocation( + linkSignerPrivKey: ByteArray, + createdAt: Long, + ): Event = NostrSignerSync(KeyPair(privKey = linkSignerPrivKey)).sign(ConcordInviteBundleEvent.buildRevocation(createdAt)) + /** Decrypts a kind-33301 bundle [event] with the link [token], or null if it isn't a valid bundle. */ fun parse( event: Event, diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/bundle/ConcordInviteBundleEvent.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/bundle/ConcordInviteBundleEvent.kt index 8e17c755fb..b2302cdf68 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/bundle/ConcordInviteBundleEvent.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/bundle/ConcordInviteBundleEvent.kt @@ -68,5 +68,22 @@ class ConcordInviteBundleEvent( addUnique(VskTag.assemble(ControlEntityKind.INVITE_LIVE)) initializer() } + + /** + * Builds the revocation tombstone that retires a link: the **same** `["d",""]` coordinate, + * empty content, and `["vsk","9"]` ([ControlEntityKind.INVITE_REVOKED]). + * + * Empty content is the interop contract, not an omission — the spec's "a fetcher finds the + * grave instead of keys", and byte-for-byte what Armada's `buildRevocationEvent` emits. + * There is nothing to encrypt: the point is that no bundle key opens anything here. + */ + fun buildRevocation( + createdAt: Long = TimeUtils.now(), + initializer: TagArrayBuilder.() -> Unit = {}, + ) = eventTemplate(KIND, "", createdAt) { + dTag("") + addUnique(VskTag.assemble(ControlEntityKind.INVITE_REVOKED)) + initializer() + } } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteClassifyTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteClassifyTest.kt index 95329c3b98..b0ac0bf1a9 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteClassifyTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteClassifyTest.kt @@ -29,6 +29,7 @@ import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray import com.vitorpamplona.quartz.nip01Core.core.toHexKey import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair +import com.vitorpamplona.quartz.nip01Core.crypto.verify import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal import kotlinx.coroutines.test.runTest import kotlin.test.Test @@ -152,6 +153,40 @@ class ConcordInviteClassifyTest { assertEquals(InviteBundleStatus.Absent, ConcordInviteBundle.classify(emptyList(), ByteArray(16))) } + @Test + fun buildRevocationEmitsTheWireShapeArmadaEmits() = + runTest { + val community = ConcordCommunityFactory.create(owner, "Nostrichs", createdAt = 1L, relays = listOf("wss://relay.example")) + val minted = ConcordInviteBundle.mintLink("https://vector.chat", inviteFor(community), createdAt = 1L, relays = listOf("wss://relay.example")) + + val grave = ConcordInviteBundle.buildRevocation(minted.linkSignerPrivKey, createdAt = 2L) + + // The interop contract, byte for byte: kind 33301 at the SAME addressable coordinate + // (same author, same empty d tag), empty content, vsk=9. Anything else here and a + // non-Amethyst client keeps serving a link its creator believes is dead. + assertEquals(ConcordInviteBundleEvent.KIND, grave.kind) + assertEquals(minted.linkSignerPubKey, grave.pubKey, "a tombstone at a different author retires nothing") + assertEquals("", grave.content, "the grave carries no keys — nothing to encrypt") + assertEquals(listOf(listOf("d", ""), listOf("vsk", "9")), grave.tags.map { it.toList() }) + assertTrue(grave.verify(), "must be signed by the link signer the creator kept") + } + + @Test + fun aBuiltRevocationRetiresItsOwnLink() = + runTest { + val community = ConcordCommunityFactory.create(owner, "Nostrichs", createdAt = 1L, relays = listOf("wss://relay.example")) + val minted = ConcordInviteBundle.mintLink("https://vector.chat", inviteFor(community), createdAt = 1L, relays = listOf("wss://relay.example")) + + // End to end: what the creator publishes is what every redeemer then resolves. + val grave = ConcordInviteBundle.buildRevocation(minted.linkSignerPrivKey, createdAt = 2L) + assertEquals(InviteBundleStatus.Revoked, ConcordInviteBundle.classify(listOf(minted.bundleEvent, grave), minted.token)) + + // And a re-mint that lands AFTER the grave un-revokes the link, which is exactly why the + // refresh path must skip a coordinate it did not resolve Live first. + val remint = ConcordInviteBundle.build(minted.linkSignerPrivKey, minted.token, inviteFor(community), createdAt = 3L) + assertTrue(ConcordInviteBundle.classify(listOf(minted.bundleEvent, grave, remint), minted.token) is InviteBundleStatus.Live) + } + @Test fun realRelayopBundleIsUnreadable() { // The actual kind-33301 event behind the reported relayop.xyz/invite link (vsk=8), plus the From d27930fe761725896e8a7e8459f67da4f82b3429 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Mon, 10 Aug 2026 00:57:23 -0400 Subject: [PATCH 110/132] feat(concord): manage and revoke your invite links from Android MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revoking existed only in `amy` after the last commit, so the app could hand out a link it could never take back. This adds the Android half. `Invite links…` in a community's overflow menu opens a screen listing every link this account minted for it, read from the creator's own kind-13303 Invite List, each row offering Copy and Revoke. It shows only *our* links, because a link's `signer_sk` is what authors its coordinate and only the minting account ever held it — another admin's links are invisible here and un-revokable from here. That is the protocol, not a gap in the screen. Two deliberate choices: The entry point is NOT gated on CREATE_INVITE, unlike minting. Revoking acts on a key we hold rather than on the community, and gating it on the bit would mean a demoted admin could no longer retire the links they had already handed out — exactly when that matters most. An unreadable list is its own state, never an empty one. Telling a creator who came to kill a leaked link that they have no links would be a lie in the one direction that costs them something. `revokeConcordInvite` publishes the wire tombstone first and records the kind-13303 tombstone second, for the same reason the CLI does: the entry holds the only copy of the `signer_sk` the publish needs, and a merge drops a tombstoned token's entry terminally. A failed list write is reported as success because the link is already dead on the wire. Verified on a tablet against a local relay, cross-client with amy: the screen lists the two links the device minted (and not the one alice minted), the confirm dialog revokes exactly one coordinate — flipping it to vsk=9 with empty content while its siblings stay vsk=6 — the row disappears on reload, and a link revoked from the UI is then refused by `amy concord join` with `revoked` while the surviving link still joins. Co-Authored-By: Claude Opus 5 (1M context) --- .../amethyst/model/AccountConcordActions.kt | 69 +++++ .../amethyst/ui/navigation/AppNavigation.kt | 9 + .../amethyst/ui/navigation/routes/Routes.kt | 4 + .../concord/ConcordChannelListScreen.kt | 11 + .../concord/ConcordInviteLinksScreen.kt | 273 ++++++++++++++++++ amethyst/src/main/res/values/strings.xml | 11 + 6 files changed, 377 insertions(+) create mode 100644 amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordInviteLinksScreen.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt index daad3aa9c7..35816e52ce 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt @@ -44,6 +44,7 @@ import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteList import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteListDocument import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteListEntry import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteListEvent +import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteListTombstone import com.vitorpamplona.quartz.concord.cord05Invites.InviteBundleStatus import com.vitorpamplona.quartz.concord.cord05Invites.InviteRelayDictionary import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys @@ -339,6 +340,74 @@ class AccountConcordActions( return minted.url } + /** + * Every link this account minted for [communityId] that is still live, newest first — the + * backing list for the invite-links screen. + * + * Null means the list could not be read (no relay answered, or the signer refused the decrypt), + * which the UI must show as an error rather than as "you have no links": telling a creator their + * leaked link doesn't exist is worse than telling them we couldn't check. + * + * Retired tokens are filtered out here rather than rendered as dead rows — [ConcordInviteList] + * already drops a tombstoned entry on merge, so a tombstoned entry only appears in the window + * between our revoke and the next merge. + */ + suspend fun listConcordInviteLinks(communityId: String): List? { + val list = readConcordInviteList() ?: return null + val tombstoned = list.tombstones.mapTo(HashSet()) { it.token } + return list.entries + .filter { it.communityId == communityId && it.token !in tombstoned } + .sortedByDescending { it.createdAt } + } + + /** + * Retires the link [token] (CORD-05 §2): publishes a `vsk=9` tombstone at its coordinate, then + * records the retirement in the kind-13303 list. Returns false if the link could not be retired. + * + * No community permission is checked, deliberately. The coordinate is authored by the link + * signer, whose secret only the creator holds, so revoking is an act on your own key rather than + * on the community — and gating it on CREATE_INVITE would mean a demoted admin could no longer + * retire the links they had already handed out, which is precisely when they most need to. + * + * The wire tombstone goes first and the list second. That is the inverse of minting and it is + * deliberate: the entry holds the only copy of the `signer_sk` this needs, and a merge drops a + * tombstoned token's entry terminally, so recording first and then failing to publish would + * leave the link live with its signer gone and no way left to retire it. A failed list write is + * recoverable — the link is already dead on the wire, and the refresh path re-mints only a + * coordinate that still resolves Live. + */ + suspend fun revokeConcordInvite( + communityId: String, + token: String, + ): Boolean { + if (!account.isWriteable()) return false + val entry = + account.concordChannelList.liveCommunities.value + .firstOrNull { it.id == communityId } ?: return false + val link = + readConcordInviteList()?.entries?.firstOrNull { it.token == token && it.communityId == communityId } + ?: run { + Log.w("Concord") { "Cannot revoke $token: it is not in this account's invite list, so its link signer is unknown" } + return false + } + + val relays = entry.relays.mapNotNullTo(mutableSetOf()) { RelayUrlNormalizer.normalizeOrNull(it) }.ifEmpty { account.outboxRelays.flow.value } + if (relays.isEmpty()) return false + val published = + runCatching { + account.client.publish(ConcordActions.revokeBundleAt(link.signerSk.hexToByteArray(), TimeUtils.now()), relays) + true + }.onFailure { Log.w("Concord", "invite revocation failed for $communityId", it) }.getOrDefault(false) + if (!published) return false + + if (!publishConcordInviteList(ConcordInviteListDocument(tombstones = listOf(ConcordInviteListTombstone(token = token, communityId = communityId))))) { + // The link is already dead on the wire, so this is bookkeeping we can retry rather than a + // failed revocation. Reported as success for exactly that reason. + Log.w("Concord") { "Revoked $token on the wire but could not tombstone it in the invite list; a later revoke will record it" } + } + return true + } + /** Drop a joined Concord community from the private kind-13302 list by its id. */ suspend fun leaveConcordCommunity(communityId: String) = account.sendMyPublicAndPrivateOutbox(account.concordChannelList.unfollow(communityId)) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/AppNavigation.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/AppNavigation.kt index ee1ace2d8d..157a12d191 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/AppNavigation.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/AppNavigation.kt @@ -138,6 +138,7 @@ import com.vitorpamplona.amethyst.ui.screen.loggedIn.chats.publicChannels.concor import com.vitorpamplona.amethyst.ui.screen.loggedIn.chats.publicChannels.concord.ConcordCreateScreen import com.vitorpamplona.amethyst.ui.screen.loggedIn.chats.publicChannels.concord.ConcordEditScreen import com.vitorpamplona.amethyst.ui.screen.loggedIn.chats.publicChannels.concord.ConcordHomeScreen +import com.vitorpamplona.amethyst.ui.screen.loggedIn.chats.publicChannels.concord.ConcordInviteLinksScreen import com.vitorpamplona.amethyst.ui.screen.loggedIn.chats.publicChannels.concord.ConcordInviteScreen import com.vitorpamplona.amethyst.ui.screen.loggedIn.chats.publicChannels.concord.ConcordMembersScreen import com.vitorpamplona.amethyst.ui.screen.loggedIn.chats.publicChannels.ephemChat.EphemeralChatScreen @@ -747,6 +748,14 @@ fun BuildNavigation( ) } + composableFromEndArgs { + ConcordInviteLinksScreen( + communityId = it.communityId, + accountViewModel = accountViewModel, + nav = nav, + ) + } + composableFromEndArgs { ConcordEditScreen( communityId = it.communityId, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/routes/Routes.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/routes/Routes.kt index b1afa1c5b9..c267a23c73 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/routes/Routes.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/routes/Routes.kt @@ -830,6 +830,10 @@ sealed class Route { val communityId: String, ) : Route() + @Serializable data class ConcordInviteLinks( + val communityId: String, + ) : Route() + @Serializable object ConcordCreate : Route() // Deep-link target for a Concord invite link (naddr#fragment). Opens the join flow. diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordChannelListScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordChannelListScreen.kt index d33dfee5e2..57dffc1084 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordChannelListScreen.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordChannelListScreen.kt @@ -294,6 +294,17 @@ fun ConcordChannelListScreen( SymbolIcon(symbol = MaterialSymbols.MoreVert, contentDescription = stringRes(com.vitorpamplona.amethyst.R.string.more_options)) } DropdownMenu(expanded = menuOpen, onDismissRequest = { menuOpen = false }) { + // Deliberately not gated on CREATE_INVITE, unlike minting: the links listed + // there are this account's own, authored by link-signer keys only we hold. + // Gating on the bit would mean a demoted admin could no longer retire the + // links they had already handed out — exactly when that matters most. + DropdownMenuItem( + text = { Text(stringRes(com.vitorpamplona.amethyst.R.string.concord_invite_links_action)) }, + onClick = { + menuOpen = false + nav.nav(Route.ConcordInviteLinks(communityId)) + }, + ) DropdownMenuItem( text = { Text( diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordInviteLinksScreen.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordInviteLinksScreen.kt new file mode 100644 index 0000000000..ef10b8c2bb --- /dev/null +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/publicChannels/concord/ConcordInviteLinksScreen.kt @@ -0,0 +1,273 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.ui.screen.loggedIn.chats.publicChannels.concord + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.PaddingValues +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.lazy.LazyColumn +import androidx.compose.foundation.lazy.items +import androidx.compose.material3.AlertDialog +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.DropdownMenu +import androidx.compose.material3.DropdownMenuItem +import androidx.compose.material3.ExperimentalMaterial3Api +import androidx.compose.material3.HorizontalDivider +import androidx.compose.material3.IconButton +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Scaffold +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton +import androidx.compose.material3.TopAppBar +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableIntStateOf +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalClipboard +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp +import com.vitorpamplona.amethyst.R +import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols +import com.vitorpamplona.amethyst.ui.components.util.setText +import com.vitorpamplona.amethyst.ui.navigation.navs.INav +import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel +import com.vitorpamplona.amethyst.ui.stringRes +import com.vitorpamplona.quartz.concord.cord05Invites.ConcordInviteListEntry +import kotlinx.coroutines.launch +import java.text.DateFormat +import java.util.Date +import com.vitorpamplona.amethyst.commons.icons.symbols.Icon as SymbolIcon + +/** What the screen is currently showing. The unreadable case is deliberately not "empty" — see below. */ +private sealed interface LinksState { + data object Loading : LinksState + + data class Loaded( + val links: List, + ) : LinksState + + /** + * The kind-13303 list could not be read. Distinct from an empty list on purpose: rendering + * "no links yet" here would tell a creator that the link they came to kill does not exist. + */ + data object Unreadable : LinksState +} + +/** + * Every invite link this account minted for one community, with the ability to retire one + * (CORD-05 §2). + * + * The list is the creator's own kind-13303 Invite List, which is where a link's `signer_sk` lives — + * so this shows only links *this account* minted, from any of its devices. Another admin's links are + * invisible here and un-revokable from here, because the secret that authors their coordinate was + * never ours. That is a property of the protocol, not a gap in the screen. + * + * Fetched on entry rather than collected from a flow: nothing subscribes to kind 13303 (it is + * bookkeeping the user never sees), so there is no cache to observe. + */ +@OptIn(ExperimentalMaterial3Api::class) +@Composable +fun ConcordInviteLinksScreen( + communityId: String, + accountViewModel: AccountViewModel, + nav: INav, +) { + val account = accountViewModel.account + val scope = rememberCoroutineScope() + val clipboard = LocalClipboard.current + + var state by remember(communityId) { mutableStateOf(LinksState.Loading) } + var reloads by remember(communityId) { mutableIntStateOf(0) } + var confirming by remember { mutableStateOf(null) } + var revoking by remember { mutableStateOf(false) } + + LaunchedEffect(communityId, reloads) { + state = LinksState.Loading + state = account.concord.listConcordInviteLinks(communityId)?.let { LinksState.Loaded(it) } ?: LinksState.Unreadable + } + + val communityName = + remember(account, communityId) { + account.concordChannelList.liveCommunities.value + .firstOrNull { it.id == communityId } + ?.name + .orEmpty() + } + + Scaffold( + topBar = { + TopAppBar( + title = { + Column { + Text(stringRes(R.string.concord_invite_links_title), fontWeight = FontWeight.Bold) + if (communityName.isNotBlank()) { + Text(communityName, style = MaterialTheme.typography.bodySmall, maxLines = 1, overflow = TextOverflow.Ellipsis) + } + } + }, + navigationIcon = { + IconButton(onClick = { nav.popBack() }) { + SymbolIcon(symbol = MaterialSymbols.AutoMirrored.ArrowBack, contentDescription = stringRes(R.string.back)) + } + }, + ) + }, + ) { padding -> + when (val current = state) { + is LinksState.Loading -> + Box(Modifier.fillMaxSize().padding(padding), contentAlignment = Alignment.Center) { + CircularProgressIndicator() + } + + is LinksState.Unreadable -> CenteredMessage(padding, stringRes(R.string.concord_invite_links_unreadable)) + + is LinksState.Loaded -> + if (current.links.isEmpty()) { + CenteredMessage(padding, stringRes(R.string.concord_invite_links_empty)) + } else { + LazyColumn(Modifier.fillMaxSize().padding(padding)) { + items(current.links, key = { it.token }) { link -> + InviteLinkRow( + link = link, + enabled = !revoking, + onCopy = { scope.launch { clipboard.setText(link.url) } }, + onRevoke = { confirming = link }, + ) + HorizontalDivider() + } + } + } + } + } + + confirming?.let { link -> + AlertDialog( + onDismissRequest = { if (!revoking) confirming = null }, + title = { Text(stringRes(R.string.concord_invite_revoke_title)) }, + text = { Text(stringRes(R.string.concord_invite_revoke_explainer)) }, + confirmButton = { + TextButton( + enabled = !revoking, + onClick = { + revoking = true + scope.launch { + try { + val ok = account.concord.revokeConcordInvite(communityId, link.token) + accountViewModel.toastManager.toast( + R.string.concord_invite_links_title, + if (ok) R.string.concord_invite_revoked_ok else R.string.concord_invite_revoked_failed, + ) + // Re-read either way: on success the link is gone from the list, and on + // failure the list is the only thing that can say whether it changed. + reloads++ + } finally { + revoking = false + confirming = null + } + } + }, + ) { + Text(stringRes(R.string.concord_invite_revoke_confirm), color = MaterialTheme.colorScheme.error) + } + }, + dismissButton = { + TextButton(enabled = !revoking, onClick = { confirming = null }) { + Text(stringRes(R.string.cancel)) + } + }, + ) + } +} + +@Composable +private fun CenteredMessage( + padding: PaddingValues, + message: String, +) { + Box(Modifier.fillMaxSize().padding(padding).padding(24.dp), contentAlignment = Alignment.Center) { + Text( + message, + // This Box sits on the bare window background, so LocalContentColor is still the M3 + // default black — see the sibling invite screen, where that made the text invisible. + color = MaterialTheme.colorScheme.onBackground, + style = MaterialTheme.typography.bodyLarge, + ) + } +} + +@Composable +private fun InviteLinkRow( + link: ConcordInviteListEntry, + enabled: Boolean, + onCopy: () -> Unit, + onRevoke: () -> Unit, +) { + var menuOpen by remember { mutableStateOf(false) } + + Row( + modifier = Modifier.fillMaxWidth().padding(horizontal = 16.dp, vertical = 12.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.SpaceBetween, + ) { + Column(Modifier.weight(1f).padding(end = 8.dp)) { + // The token prefix is what tells two links to the same community apart; their URLs share + // a long prefix, so they are useless as labels until well past where the row wraps. + Text(link.token.take(8), fontWeight = FontWeight.Bold, style = MaterialTheme.typography.bodyLarge) + Text( + stringRes(R.string.concord_invite_links_created, DateFormat.getDateInstance(DateFormat.MEDIUM).format(Date(link.createdAt * 1000))), + style = MaterialTheme.typography.bodySmall, + ) + Text(link.url, style = MaterialTheme.typography.bodySmall, maxLines = 1, overflow = TextOverflow.Ellipsis) + } + + IconButton(enabled = enabled, onClick = { menuOpen = true }) { + SymbolIcon(symbol = MaterialSymbols.MoreVert, contentDescription = stringRes(R.string.more_options)) + } + DropdownMenu(expanded = menuOpen, onDismissRequest = { menuOpen = false }) { + DropdownMenuItem( + text = { Text(stringRes(R.string.copy_to_clipboard)) }, + onClick = { + menuOpen = false + onCopy() + }, + ) + DropdownMenuItem( + text = { Text(stringRes(R.string.concord_invite_revoke_action), color = MaterialTheme.colorScheme.error) }, + onClick = { + menuOpen = false + onRevoke() + }, + ) + } + } +} diff --git a/amethyst/src/main/res/values/strings.xml b/amethyst/src/main/res/values/strings.xml index 20e32ea0b7..9489821e4e 100644 --- a/amethyst/src/main/res/values/strings.xml +++ b/amethyst/src/main/res/values/strings.xml @@ -322,6 +322,17 @@ This invite link can\'t be opened. It may be outdated or already replaced by a newer one, or created with a newer version of the app. Ask for a fresh invite link. This invite link has been revoked and can no longer be used. Ask for a new one. This community has removed you. The link still works, but its member list does not admit you. + Invite links + Invite links… + Created %1$s + You haven\'t created any invite links for this community yet. Links other admins created are managed on their own devices. + Your invite links couldn\'t be loaded, so none can be revoked right now. Check your connection and try again. + Revoke link + Revoke this link? + Anyone still holding this link will no longer be able to join. People who already joined with it stay in the community. This can\'t be undone. + Revoke + Invite link revoked. + The link couldn\'t be revoked. Check your connection and try again. This invite link has expired and can no longer be used. Ask for a fresh link. Community name is only revealed after you join Joining connects to this invite\'s relays, publishes a join announcement signed by your account, and adds the community to your list. Nothing is sent until you tap Join. From 6e5f8c0fa076ca9d3f9439b79b4fc0db5a61bc79 Mon Sep 17 00:00:00 2001 From: nrobi144 Date: Thu, 6 Aug 2026 09:28:05 +0300 Subject: [PATCH 111/132] docs: upstream reconciliation plan for shared metadata loading (v2) Redo the commons extraction of the per-user + per-note finders on top of current upstream/main, which re-architected the subsystem (AccountScopedQuery attribution + SubPurpose/ExplainedFilter tagging). Key finding: attribution needs only a pubkey, so the narrow UserFinderAccount seam survives and the finder query states can drop AccountScopedQuery entirely. Co-Authored-By: Claude Opus 4.8 --- ...ta-loading-upstream-reconciliation-plan.md | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 desktopApp/plans/2026-08-06-shared-metadata-loading-upstream-reconciliation-plan.md diff --git a/desktopApp/plans/2026-08-06-shared-metadata-loading-upstream-reconciliation-plan.md b/desktopApp/plans/2026-08-06-shared-metadata-loading-upstream-reconciliation-plan.md new file mode 100644 index 0000000000..e30caa24c7 --- /dev/null +++ b/desktopApp/plans/2026-08-06-shared-metadata-loading-upstream-reconciliation-plan.md @@ -0,0 +1,110 @@ +# Shared Metadata Loading — Upstream Reconciliation Plan +*Redo the commons extraction of the per-user metadata finder and per-note event finder on top of current `upstream/main`, abandoning the stale rebase of `worktree-plan-shared-metadata-loading` (`682c6000f5`).* + +## 0. Executive summary of what changed under us + +Upstream re-architected the same subsystem but **also completed the model-sharing prerequisite we depended on**: + +- `amethyst.model.User`, `UserContext`, `Note` are now **typealiases** to `commons.model.*` (`amethyst/model/User.kt`). `SincePerRelayMap`, `MutableTime`, `EOSERelayList` are typealiases to `commons.relays.*` (`amethyst/service/relays/EOSE.kt`). Finder bodies are already type-compatible with commonMain. +- `LocalCache` is an `object` implementing `commons.model.cache.ICacheProvider` (exposes `getUserIfExists`, `getNoteIfExists`, `getOrCreateUser`, `relayHints`, …). The finders call exactly these. +- **Two new cross-cutting mechanisms**: + 1. `AccountScopedQuery { val account: amethyst.model.Account }` — implemented by `UserFinderQueryState`/`EventFinderQueryState` so base managers can attribute subscriptions to an account. + 2. `ExplainedFilter`/`SubPurpose` (already in commons) — every emitted filter is tagged with its purpose. + +**Most important finding:** the base **commons** managers never read `.account`. Attribution lives in *amethyst-side* managers (`PerUserEoseManager`, `PerUniqueIdEoseManager`, …) and each reads exactly `(key as? AccountScopedQuery)?.account?.userProfile()?.pubkeyHex` — **only a pubkey hex**. AND the four finder sub-assemblers extend the **commons** base managers, not the amethyst attribution managers — they attribute *inline* by computing `soleAccountPubKey` and passing `accountPubKeys` into their `ExplainedFilter`s. So the finders do not depend on the amethyst attribution managers; they need only a pubkey. + +## A. Map of upstream's current subsystem + +### User side — `amethyst/.../reqCommand/user/` +- `UserFinderFilterAssembler.kt` — `UserFinderQueryState(user, override val account: Account) : AccountScopedQuery`; groups `UserOutboxFinderSubAssembler`, `UserWatcherSubAssembler`, `UserReportsSubAssembler`, `UserCardsSubAssembler`. +- `UserFinderFilterAssemblerSubscription.kt` — composable entry (`(user, accountViewModel)`) + `UserFinderByParentFilterAssemblerSubscription`; uses `AccountViewModel.account`, `dataSources().userFinder`, commons `LifecycleAwareKeyDataSourceSubscription`. +- `UserObservers.kt` — `observeUser*`. +- `loaders/UserOutboxFinderSubAssembler.kt` — extends commons `BaseEoseManager`; reads `it.account` → `pickRelaysToLoadUsers(...)`; emits `ExplainedFilter(purpose = RELAY_LISTS, accountPubKeys = listOfNotNull(soleAccountPubKey))`. +- `watchers/UserWatcherSubAssembler.kt` — commons `BaseEoseManager`; reads `account.indexerRelayList.flow.value`; calls `filterUserMetadataForKey(...)`. +- `watchers/FilterUserMetadataForKey.kt` — emits `ExplainedFilter(PROFILE_METADATA, …)`; reads `LocalCache.relayHints.hintsForKey(...)` **statically** (the one non-injected cache ref). +- `watchers/UserReportsSubAssembler.kt` — commons `SingleSubEoseManager`; reads `account.declaredFollowsPerOutboxRelay.value`, `account.userProfile().pubkeyHex`. +- `watchers/UserCardsSubAssembler.kt` — commons `SingleSubEoseManager`; reads `account.homeRelays.flow.value`, `account.trustProviderList.liveUserRankProvider`, `…liveUserFollowerCount`, `account.userProfile().pubkeyHex`; emits `filterContactCardsToTargetKeysFromTrustedAccountsInTheRelay(...)` (already in commons `assemblers/ContactCardFilters.kt`). + +### Event side — `amethyst/.../reqCommand/event/` +- `EventFinderFilterAssembler.kt` — `EventFinderQueryState(note, override val account: Account) : AccountScopedQuery`; groups `NoteEventLoaderSubAssembler`, `EventWatcherSubAssembler`, `AddressableAuthorRelayLoaderSubAssembler(cache, ::allKeys, userFinder)`. +- `EventFinderFilterAssemblerSubscription.kt` — reads `accountViewModel.account`, `dataSources().eventFinder`. +- `EventObservers.kt` — `observeNote*` (one UI-only spot reads `accountViewModel.account.userProfile().pubkeyHex` for a moderator check). +- `loaders/NoteEventLoaderSubAssembler.kt`; `loaders/FilterMissingEvents.kt` (reads `key.account.followPlusAllMineWithSearch.flow.value`, `key.account.searchRelayList.flow.value`; `SubPurpose.REFERENCED_EVENTS`); `loaders/FilterMissingAddressables.kt` (`REFERENCED_EVENTS`). +- `loaders/AddressableAuthorRelayLoaderSubAssembler.kt` — constructs `UserFinderQueryState(author, key.account)`. +- `watchers/EventWatcherSubAssembler.kt` — commons `SingleSubEoseManager`; reads `it.account.userProfile().pubkeyHex` (attribution only); calls `filterRepliesAndReactionsToNotes/Addresses(...)`. +- `watchers/FilterRepliesAndReactionsToNotes.kt`, `FilterRepliesAndReactionsToAddresses.kt` — emit `ExplainedFilter(ENGAGEMENT, …)`. + +### Base managers and attribution +- commons `BaseEoseManager`, `PerKeyEoseManager`, `SingleSubEoseManager` — **account-agnostic**. +- amethyst `PerUserEoseManager`, `PerUniqueIdEoseManager`, `PerUserAndFollowListEoseManager`, `SingleSubNoEoseCacheEoseManager` — do `f.attributedTo(pk)` where `pk = (key as? AccountScopedQuery).account.userProfile().pubkeyHex`. The finder sub-assemblers do NOT use these; they attribute inline. +- `ExplainedFilter.attributedTo(accountPubKey: HexKey)`; `SubPurpose`/`SubPurposeGroup` — commonMain. + +## B. Account-seam decision — CHOSEN + +**Keep `UserFinderAccount` as the narrow commons seam, carrying the attribution pubkey via `userFinderPubkeyHex`. Do NOT move `AccountScopedQuery` to commons. Drop `AccountScopedQuery` from the two finder query states.** + +Justification (grounded in code): +1. Attribution reduces to `.userProfile().pubkeyHex` — a `HexKey`, already on `UserFinderAccount.userFinderPubkeyHex`. +2. The loaders' account reads are all snapshot relay-hint / follow-graph getters — exactly the `UserFinderAccount` surface (+2 additions). +3. `AccountScopedQuery` is declared over `amethyst.model.Account` and implemented by ~66 amethyst query states — can't move without dragging `Account`. +4. The finder query states never flow through the amethyst attribution managers, so they don't need `AccountScopedQuery` at all → dropping it removes the collision. + +Seam shape (commons `UserFinderAccount`), = our old branch + 2 additions: +```kotlin +interface UserFinderAccount { + val userFinderPubkeyHex: HexKey // attribution pubkey → ExplainedFilter.accountPubKeys + fun indexRelays(): Set + fun outboxHomeRelays(): Set + fun searchRelays(): Set + fun followPlusAllMineWithSearchRelays(): Set + fun commonRelays(): Set + fun cardHomeRelays(): Set + fun trustProvider(): ServiceProviderTag? + fun followerCountProvider(): ServiceProviderTag? // NEW (UserCards reads liveUserFollowerCount) + fun declaredFollowsByOutboxRelay(): Map> +} +``` +`EventFinderQueryState` reuses `UserFinderAccount` (needs only follow+search relays + pubkey). Attribution: `soleAccountPubKey = keys.map { it.account.userFinderPubkeyHex }.singleOrNull()` → `accountPubKeys = listOfNotNull(soleAccountPubKey)`. + +## C. SubPurpose mapping (preserve upstream tags across the move) + +| Moved helper | SubPurpose | +|---|---| +| `FilterUserMetadataForKey` (kind 0 bundle) | `PROFILE_METADATA` (runsInBackground) | +| `UserOutboxFinderSubAssembler` | `RELAY_LISTS` | +| `UserCardsSubAssembler` → `ContactCardFilters` (already commons) | unchanged | +| `UserReportsSubAssembler` → `filterReportsToKeysFromTrusted` | `MODERATION` | +| `EventWatcherSubAssembler` → replies/reactions | `ENGAGEMENT` | +| `NoteEventLoader` → `FilterMissingEvents/Addressables` | `REFERENCED_EVENTS` | + +`ExplainedFilter`/`SubPurpose`/`attributedTo`/`ContactCardFilters` are already commonMain + iOS-pure. Only change: `accountPubKeys` sourced from `userFinderPubkeyHex`. + +## D. Portable-unchanged vs must-rework + +**Unchanged (body identical after package move + seam swap):** the filter builders (`FilterUserMetadataForKey`, `FilterReportsToKey`, `FilterMissingEvents/Addressables`, `FilterRepliesAndReactionsToNotes/Addresses`), `observeUser*`/`observeNote*`, `UserFinderAccount` + CompositionLocals, `pickRelaysToLoadUsers` inner overload. + +**Must rework:** +1. **Move `EOSEAccountFast`** (`amethyst/service/relays/EOSE.kt`) to commons `relays/` + amethyst typealias — prereq for the loaders. (Purity risk: verify no `System.currentTimeMillis`/`Thread.sleep`.) +2. **`LocalCache.relayHints` static ref** in `FilterUserMetadataForKey` → injected `cache.relayHints` (add `val relayHints: HintIndexer` to `ICacheProvider`; LocalCache already has it). +3. **Account-seam swap** across the four user sub-assemblers + event loaders/watchers (getter-for-flow mapping listed in §B). +4. **`pickRelaysToLoadUsers`** — feed commons inner overload from `UserFinderAccount` getters (amethyst keeps a thin `Account`→relay-set wrapper). +5. **Drop `AccountScopedQuery`** from the two finder query states (verified no amethyst attribution manager consumes them). +6. `AddressableAuthorRelayLoaderSubAssembler` — `UserFinderQueryState(author, key.account)` now takes `UserFinderAccount`. Clean. + +**Desktop wiring (Phase 3/3b old plan) — unaffected:** `observeUser*`, `EventFinderFilterAssemblerSubscription(note)`, Locals, DM/search/notifications adoption, fast index-relay warm-up. `DesktopIAccount` adds `followerCountProvider() = null` (already degrades trust/declaredFollows). + +## E. Phase / commit sequence (fresh branch `feat/shared-metadata-loading-v2` off `upstream/main`) + +Cherry-pick *content*, not commits. Gates after each commons/amethyst commit: `:commons:verifyKmpPurity`, `:amethyst:compilePlayDebugKotlin`, `:commons:compileKotlinJvm` + `:desktopApp:compileKotlinJvm`. + +- **Phase 0 — prereqs:** (0a) move `EOSEAccountFast` → commons + typealias; (0b) add `relayHints` to `ICacheProvider`. +- **Phase 1 — seam:** (1a) add commons `UserFinderAccount` (+`followerCountProvider()`); (1b) `Account implements UserFinderAccount`. +- **Phase 2 — user finder → commons:** (2a) filter builders + `pickRelaysToLoadUsers` inner; (2b) the 4 sub-assemblers; (2c) `UserFinderFilterAssembler`+`UserFinderQueryState` (drop `AccountScopedQuery`) + amethyst typealias shim + commons `UserFinderSubscription`/Locals; keep composable subscription in amethyst delegating. +- **Phase 3 — event finder → commons:** (3a) filter builders; (3b) loaders/watchers + addressable bridge; (3c) assembler+state (drop `AccountScopedQuery`) + shim + `LocalEventFinder` + `observeNote*` to commons. +- **Phase 4 — Desktop wiring:** (4a) `DesktopIAccount implements UserFinderAccount`; (4b) provide Locals in `Main.kt`, wire DM/search/notifications + warm-up onto `observeUser*`/`EventFinderFilterAssemblerSubscription`. +- **Phase 5 — cleanup & tests:** delete dead originals; run `ExplainedFilterTest` + finder tests + full `:commons:check`. + +### Risk callouts +- `EOSEAccountFast` purity is the likeliest `verifyKmpPurity` tripwire. +- Preserve inline `soleAccountPubKey` attribution so "Active Relay Subscriptions" still files single-account REQs correctly (and shows "not attributed" when accounts pool relays — don't regress to per-account splitting). +- Do NOT re-introduce `AccountScopedQuery` on the two finder states; if a future upstream manager consumes them, add a thin amethyst adapter instead of widening the commons seam. From 89b2a273a06fc59bfd90713c528ae1e6f129d6d1 Mon Sep 17 00:00:00 2001 From: nrobi144 Date: Wed, 29 Jul 2026 12:26:18 +0300 Subject: [PATCH 112/132] refactor: move EOSEAccountFast to commons, KMP-purified (Phase 2 prep) EOSEAccountFast is used by the user/event finder assemblers that will move to commonMain. Relocate it to commons.relays with KmpLock instead of synchronized(...) so it passes the iOS verifyKmpPurity gate. The old amethyst.service.relays location keeps a typealias, so its 7 existing importers are untouched. (SincePerRelayMap/MutableTime/EOSERelayList were already commons typealiases.) Co-Authored-By: Claude Opus 4.8 --- .../amethyst/service/relays/EOSE.kt | 58 +----------- .../commons/relays/EOSEAccountFast.kt | 94 +++++++++++++++++++ 2 files changed, 95 insertions(+), 57 deletions(-) create mode 100644 commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relays/EOSEAccountFast.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relays/EOSE.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relays/EOSE.kt index 403fcaf39f..f6032b8800 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relays/EOSE.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relays/EOSE.kt @@ -28,6 +28,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl typealias EOSERelayList = com.vitorpamplona.amethyst.commons.relays.EOSERelayList typealias SincePerRelayMap = com.vitorpamplona.amethyst.commons.relays.SincePerRelayMap typealias MutableTime = com.vitorpamplona.amethyst.commons.relays.MutableTime +typealias EOSEAccountFast = com.vitorpamplona.amethyst.commons.relays.EOSEAccountFast open class EOSEByKey( cacheSize: Int = 200, @@ -113,60 +114,3 @@ open class EOSEAccountKey( time: Long, ) = addOrUpdate(user, listCode, relayUrl, time) } - -class EOSEAccountFast( - cacheSize: Int = 20, -) { - private val users: LruCache = LruCache(cacheSize) - private val lock = Any() - - fun addOrUpdate( - user: T, - relayUrl: NormalizedRelayUrl, - time: Long, - ) { - synchronized(lock) { - val relayList = users[user] - if (relayList == null) { - val newList = EOSERelayList() - users.put(user, newList) - - newList.addOrUpdate(relayUrl, time) - } else { - relayList.addOrUpdate(relayUrl, time) - } - } - } - - fun removeEveryoneBut(list: Set) { - synchronized(lock) { - users.snapshot().forEach { - if (it.key !in list) { - users.remove(it.key) - } - } - } - } - - fun removeDataFor(user: T) { - synchronized(lock) { - users.remove(user) - } - } - - fun since(key: T): SincePerRelayMap? = - synchronized(lock) { - users[key]?.relayList?.toMutableMap() - } - - fun sinceRelaySet(key: T): Set? = - synchronized(lock) { - users[key]?.relayList?.keys?.toSet() - } - - fun newEose( - user: T, - relayUrl: NormalizedRelayUrl, - time: Long, - ) = addOrUpdate(user, relayUrl, time) -} diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relays/EOSEAccountFast.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relays/EOSEAccountFast.kt new file mode 100644 index 0000000000..84d5e90773 --- /dev/null +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relays/EOSEAccountFast.kt @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.relays + +import androidx.collection.LruCache +import com.vitorpamplona.amethyst.commons.util.KmpLock +import com.vitorpamplona.amethyst.commons.util.withLock +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl + +/** + * Per-key EOSE tracker keyed by an arbitrary [T] (a `User`, a pubkey, …), used + * by the relay-subscription assemblers to remember which relays already EOSE'd + * for a given key so the next filter assembly can add a `since` and avoid a full + * re-download. + * + * KMP-pure: uses [KmpLock] instead of `synchronized(...)` so it compiles for the + * iOS targets `commons` builds for. Moved out of `amethyst.service.relays` so the + * shared user/event finder assemblers can live in `commonMain`. The old location + * keeps a `typealias` for source compatibility. + */ +class EOSEAccountFast( + cacheSize: Int = 20, +) { + private val users: LruCache = LruCache(cacheSize) + private val lock = KmpLock() + + fun addOrUpdate( + user: T, + relayUrl: NormalizedRelayUrl, + time: Long, + ) { + lock.withLock { + val relayList = users[user] + if (relayList == null) { + val newList = EOSERelayList() + users.put(user, newList) + + newList.addOrUpdate(relayUrl, time) + } else { + relayList.addOrUpdate(relayUrl, time) + } + } + } + + fun removeEveryoneBut(list: Set) { + lock.withLock { + users.snapshot().forEach { + if (it.key !in list) { + users.remove(it.key) + } + } + } + } + + fun removeDataFor(user: T) { + lock.withLock { + users.remove(user) + } + } + + fun since(key: T): SincePerRelayMap? = + lock.withLock { + users[key]?.relayList?.toMutableMap() + } + + fun sinceRelaySet(key: T): Set? = + lock.withLock { + users[key]?.relayList?.keys?.toSet() + } + + fun newEose( + user: T, + relayUrl: NormalizedRelayUrl, + time: Long, + ) = addOrUpdate(user, relayUrl, time) +} From 5a245c9f58e35e0bed6a1b5e327d387d6ccbfcab Mon Sep 17 00:00:00 2001 From: nrobi144 Date: Wed, 29 Jul 2026 14:12:24 +0300 Subject: [PATCH 113/132] refactor: add relayHints seam to ICacheProvider (Phase 2 prep) The shared user/event finder filter functions read LocalCache.relayHints statically to discover which relays likely hold a user's metadata or a missing event. To let those functions move to commonMain, expose the HintIndexer (a quartz type) on ICacheProvider. Implement it on Android LocalCache (override the existing field), add one to DesktopLocalCache, and satisfy the four commonTest stub caches. Co-Authored-By: Claude Opus 4.8 --- .../java/com/vitorpamplona/amethyst/model/LocalCache.kt | 2 +- .../amethyst/commons/model/cache/ICacheProvider.kt | 8 ++++++++ .../amethyst/commons/model/ThreadAssemblerTest.kt | 3 +++ .../commons/model/concord/ConcordChannelListLeaveTest.kt | 3 +++ .../commons/model/concord/ConcordListLateArrivalTest.kt | 3 +++ .../amethyst/commons/ui/note/ReplyContextTest.kt | 3 +++ .../amethyst/desktop/cache/DesktopLocalCache.kt | 4 ++++ 7 files changed, 25 insertions(+), 1 deletion(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/LocalCache.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/LocalCache.kt index 9df341c771..cd2963d6d7 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/LocalCache.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/LocalCache.kt @@ -479,7 +479,7 @@ object LocalCache : ILocalCache, ICacheProvider, Dao { @Volatile var lnurlEndpointResolver: LnurlEndpointResolver? = null - val relayHints = HintIndexer() + override val relayHints = HintIndexer() /** * Cashu mint URL directory, populated passively as diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/cache/ICacheProvider.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/cache/ICacheProvider.kt index 6bdb38532e..3f2e59bd64 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/cache/ICacheProvider.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/cache/ICacheProvider.kt @@ -27,6 +27,7 @@ import com.vitorpamplona.amethyst.commons.model.User import com.vitorpamplona.quartz.nip01Core.core.Address import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.hints.HintIndexer /** * Cache provider interface for accessing cached Notes, Users, and Channels. @@ -41,6 +42,13 @@ import com.vitorpamplona.quartz.nip01Core.core.HexKey * - Platform-agnostic model layer */ interface ICacheProvider { + /** + * NIP-hints index (event/address/pubkey → relay) accumulated from consumed + * events. Used by the shared user/event finder assemblers to discover which + * relays are likely to hold a given user's metadata or a missing event. + */ + val relayHints: HintIndexer + /** * Gets a channel by Note reference. * Used for resolving relay hints for channel messages. diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/ThreadAssemblerTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/ThreadAssemblerTest.kt index e2a25246a9..3bbc6bbc5d 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/ThreadAssemblerTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/ThreadAssemblerTest.kt @@ -25,6 +25,7 @@ import com.vitorpamplona.amethyst.commons.model.cache.ICacheProvider import com.vitorpamplona.quartz.nip01Core.core.Address import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.hints.HintIndexer import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent import com.vitorpamplona.quartz.nip22Comments.CommentEvent import com.vitorpamplona.quartz.nip25Reactions.ReactionEvent @@ -173,6 +174,8 @@ class ThreadAssemblerTest { ) : ICacheProvider { override fun getAnyChannel(note: Note): Channel? = null + override val relayHints = HintIndexer() + override fun getUserIfExists(pubkey: HexKey): User? = null override fun countUsers(predicate: (String, User) -> Boolean): Int = 0 diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordChannelListLeaveTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordChannelListLeaveTest.kt index 600a56a38b..298295d60c 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordChannelListLeaveTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordChannelListLeaveTest.kt @@ -33,6 +33,7 @@ import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair +import com.vitorpamplona.quartz.nip01Core.hints.HintIndexer import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -82,6 +83,8 @@ class ConcordChannelListLeaveTest { private class StubCache : ICacheProvider { override fun getAnyChannel(note: Note): Channel? = null + override val relayHints = HintIndexer() + override fun getUserIfExists(pubkey: HexKey): User? = null override fun countUsers(predicate: (String, User) -> Boolean): Int = 0 diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordListLateArrivalTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordListLateArrivalTest.kt index e4325e9280..6c682a88fc 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordListLateArrivalTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/model/concord/ConcordListLateArrivalTest.kt @@ -33,6 +33,7 @@ import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair +import com.vitorpamplona.quartz.nip01Core.hints.HintIndexer import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers @@ -80,6 +81,8 @@ class ConcordListLateArrivalTest { override fun getAnyChannel(note: Note): Channel? = null + override val relayHints = HintIndexer() + override fun getUserIfExists(pubkey: HexKey): User? = null override fun countUsers(predicate: (String, User) -> Boolean): Int = 0 diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/ui/note/ReplyContextTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/ui/note/ReplyContextTest.kt index 0a23769767..59d608d092 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/ui/note/ReplyContextTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/ui/note/ReplyContextTest.kt @@ -29,6 +29,7 @@ import com.vitorpamplona.amethyst.commons.model.cache.ICacheProvider import com.vitorpamplona.quartz.nip01Core.core.Address import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.hints.HintIndexer import com.vitorpamplona.quartz.nip10Notes.TextNoteEvent import kotlin.test.Test import kotlin.test.assertEquals @@ -113,6 +114,8 @@ class ReplyContextTest { ) : ICacheProvider { override fun getAnyChannel(note: Note): Channel? = null + override val relayHints = HintIndexer() + override fun getUserIfExists(pubkey: HexKey): User? = users[pubkey] override fun countUsers(predicate: (String, User) -> Boolean): Int = 0 diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/cache/DesktopLocalCache.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/cache/DesktopLocalCache.kt index b83d490a28..6c5979ac08 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/cache/DesktopLocalCache.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/cache/DesktopLocalCache.kt @@ -35,6 +35,7 @@ import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.crypto.checkSignature import com.vitorpamplona.quartz.nip01Core.crypto.verify +import com.vitorpamplona.quartz.nip01Core.hints.HintIndexer import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.tags.aTag.taggedAddresses @@ -90,6 +91,9 @@ class DesktopLocalCache : ICacheProvider { val addressableNotes = LargeSoftCache() private val deletedEvents = ConcurrentHashMap.newKeySet() + /** NIP-hints index accumulated from consumed events (event/address/pubkey → relay). */ + override val relayHints = HintIndexer() + val eventStream = DesktopCacheEventStream() /** Local relay store for persisting events to SQLite. Set from Main.kt on account login. */ From 1511a8018b4c60745eacc255860237768ddaea5a Mon Sep 17 00:00:00 2001 From: nrobi144 Date: Thu, 6 Aug 2026 10:00:47 +0300 Subject: [PATCH 114/132] feat: add UserFinderAccount seam + implement on Account (Phase 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The narrow read-only relay-hint seam the shared per-user/per-note finders need, so they can move to commons without the fat amethyst.model.Account. userFinderPubkeyHex doubles as the attribution pubkey for ExplainedFilter.accountPubKeys (upstream's subscription-attribution model needs only a pubkey, not the whole Account). Adds followerCountProvider() vs the prior design — upstream's UserCardsSubAssembler now reads trustProviderList.liveUserFollowerCount alongside liveUserRankProvider. Co-Authored-By: Claude Opus 4.8 --- .../vitorpamplona/amethyst/model/Account.kt | 32 ++++++- .../relayClient/user/UserFinderAccount.kt | 90 +++++++++++++++++++ 2 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/UserFinderAccount.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt index 57d6f64e5e..2260fb2b95 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt @@ -30,6 +30,9 @@ import com.vitorpamplona.amethyst.commons.connectedApps.nip46.Nip46ClientStore import com.vitorpamplona.amethyst.commons.connectedApps.signers.InMemoryNostrSignerPermissionStore import com.vitorpamplona.amethyst.commons.connectedApps.signers.NostrSignerPermissionLedger import com.vitorpamplona.amethyst.commons.connectedApps.signers.NostrSignerPermissionStore +import com.vitorpamplona.amethyst.commons.defaults.Constants +import com.vitorpamplona.amethyst.commons.defaults.DefaultIndexerRelayList +import com.vitorpamplona.amethyst.commons.defaults.DefaultSearchRelayList import com.vitorpamplona.amethyst.commons.marmot.MarmotManager import com.vitorpamplona.amethyst.commons.model.IAccount import com.vitorpamplona.amethyst.commons.model.buzz.BuzzRelayDialect @@ -59,6 +62,7 @@ import com.vitorpamplona.amethyst.commons.model.nip85TrustedAssertions.ContactCa import com.vitorpamplona.amethyst.commons.model.nip85TrustedAssertions.ContactCardsState import com.vitorpamplona.amethyst.commons.model.nip85TrustedAssertions.TrustProviderListDecryptionCache import com.vitorpamplona.amethyst.commons.model.privateChats.hasEncryptedContent +import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderAccount import com.vitorpamplona.amethyst.commons.relayauth.RelayAuthCustomToggles import com.vitorpamplona.amethyst.commons.relayauth.RelayAuthPermissionStore import com.vitorpamplona.amethyst.commons.richtext.RichTextParser @@ -286,6 +290,7 @@ import com.vitorpamplona.quartz.nip72ModCommunities.rules.CommunityRulesEvent import com.vitorpamplona.quartz.nip72ModCommunities.rules.tags.KindRuleTag import com.vitorpamplona.quartz.nip72ModCommunities.rules.tags.PubkeyRuleTag import com.vitorpamplona.quartz.nip72ModCommunities.rules.tags.WotTag +import com.vitorpamplona.quartz.nip85TrustedAssertions.list.tags.ServiceProviderTag import com.vitorpamplona.quartz.nip88Polls.poll.PollEvent import com.vitorpamplona.quartz.nip88Polls.response.PollResponseEvent import com.vitorpamplona.quartz.nip89AppHandlers.clientTag.NostrSignerWithClientTag @@ -354,7 +359,8 @@ class Account( relayAuthPermissionStore: RelayAuthPermissionStore = InMemoryRelayAuthPermissionStore(), signerPermissionStore: NostrSignerPermissionStore = InMemoryNostrSignerPermissionStore(), nip46ClientStore: Nip46ClientStore = InMemoryNip46ClientStore(), -) : IAccount { +) : IAccount, + UserFinderAccount { private var userProfileCache: User? = null override fun userProfile(): User = userProfileCache ?: cache.getOrCreateUser(signer.pubKey).also { userProfileCache = it } @@ -366,6 +372,30 @@ class Account( override val hiddenUsersHashCodes: Set get() = hiddenUsers.flow.value.hiddenUsersHashCodes override val spammersHashCodes: Set get() = hiddenUsers.flow.value.spammersHashCodes + // UserFinderAccount — narrow, read-only relay-hint view used by the shared + // per-user metadata + per-note event finders (moved to commons). Snapshot + // getters read `.value` fresh on every filter rebuild. userFinderPubkeyHex + // doubles as the attribution pubkey for ExplainedFilter.accountPubKeys. + override val userFinderPubkeyHex: HexKey get() = userProfile().pubkeyHex + + override fun indexRelays(): Set = indexerRelayList.flow.value.ifEmpty { DefaultIndexerRelayList } + + override fun outboxHomeRelays(): Set = nip65RelayList.allFlowNoDefaults.value + privateStorageRelayList.flow.value + localRelayList.flow.value + + override fun searchRelays(): Set = (trustedRelayList.flow.value + searchRelayList.flow.value.ifEmpty { DefaultSearchRelayList }).toSet() + + override fun followPlusAllMineWithSearchRelays(): Set = followPlusAllMineWithSearch.flow.value + + override fun commonRelays(): Set = followSharedOutboxesOrProxy.flow.value.ifEmpty { Constants.eventFinderRelays } + + override fun cardHomeRelays(): Set = homeRelays.flow.value + + override fun trustProvider(): ServiceProviderTag? = trustProviderList.liveUserRankProvider.value + + override fun followerCountProvider(): ServiceProviderTag? = trustProviderList.liveUserFollowerCount.value + + override fun declaredFollowsByOutboxRelay(): Map> = declaredFollowsPerOutboxRelay.value + val userMetadata = UserMetadataState(signer, cache, scope, settings) // Per-account NIP-42 ALLOW/DENY overrides, warm-cached in memory so a relay AUTH challenge is diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/UserFinderAccount.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/UserFinderAccount.kt new file mode 100644 index 0000000000..9f625e2839 --- /dev/null +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/UserFinderAccount.kt @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.relayClient.user + +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip85TrustedAssertions.list.tags.ServiceProviderTag + +/** + * Narrow, read-only view of a logged-in account that the user-finder + * subscription layer needs to route metadata / relay-list / report / + * contact-card REQs for *other* users. + * + * This is deliberately NOT part of [IAccount][com.vitorpamplona.amethyst.commons.model.IAccount]: + * `IAccount` is a behavioral capability interface for the *acting* user + * (sending DMs, gift wraps, MLS groups). The relay hints needed to *discover + * other users' metadata* are a separate concern, so they live on their own + * narrow interface (ISP). + * + * All accessors are **snapshot getters** read fresh on every filter rebuild — + * matching the prior direct `account.xxx.flow.value` reads. Relay-list changes + * therefore take effect on the next subscription invalidation without any + * captured-snapshot staleness. + * + * Platforms implement this on their concrete account (Android `Account`, + * Desktop `DesktopIAccount`). Fields with no backing on a platform degrade + * safely: Desktop has no NIP-85 trust-provider subsystem wired, so + * [trustProvider] returns null and [declaredFollowsByOutboxRelay] returns an + * empty map — contact-card and report discovery become best-effort there. + */ +interface UserFinderAccount { + /** This account's own pubkey (hex). */ + val userFinderPubkeyHex: HexKey + + /** Index/discovery relays, with the platform default fallback already applied. */ + fun indexRelays(): Set + + /** Home/write relays used for outbox discovery (nip65 + private storage + local). */ + fun outboxHomeRelays(): Set + + /** Search relays (trusted + search), with the default fallback applied. */ + fun searchRelays(): Set + + /** + * Follow + all-mine + search relays, used by the per-note event-finder to + * place "missing event" / "missing addressable" REQs (reactions, zaps, + * reposts, replies) when a note references content no relay has yet placed. + * Snapshot getter, same contract as the others. + */ + fun followPlusAllMineWithSearchRelays(): Set + + /** Shared-outbox / proxy relays used as the broad common fallback. */ + fun commonRelays(): Set + + /** Home relays used specifically for NIP-51 contact-card (kind 30382) discovery. */ + fun cardHomeRelays(): Set + + /** NIP-85 trusted-assertions rank provider, or null when unsupported (e.g. Desktop). */ + fun trustProvider(): ServiceProviderTag? + + /** + * NIP-85 follower-count rank provider, or null when unsupported (e.g. Desktop). + * Read by the contact-card sub-assembler alongside [trustProvider]. + */ + fun followerCountProvider(): ServiceProviderTag? + + /** + * Declared follows keyed by the relay they were declared on, used to trust + * report authors. Empty when the platform has no follow-graph-per-relay data. + */ + fun declaredFollowsByOutboxRelay(): Map> +} From 983fc1aab272a7eb83aee52fd4d6a3e6b5fb93b0 Mon Sep 17 00:00:00 2001 From: nrobi144 Date: Thu, 6 Aug 2026 10:41:12 +0300 Subject: [PATCH 115/132] refactor: move per-user metadata finder to commons on the new upstream model (Phase 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move UserFinderFilterAssembler(+QueryState), the four sub-assemblers (UserOutboxFinder, UserWatcher, UserReports, UserCards), FilterUserMetadataForKey, and FilterReportsToKey from amethyst/reqCommand/user/ into commons/relayClient/user/, plus the inner Account-free pickRelaysToLoadUsers overload into commons PickRelaysToLoadUsers.kt. Reconciled with upstream's re-architecture: - UserFinderQueryState carries the narrow UserFinderAccount (Phase 1) instead of the full Account; DROPS AccountScopedQuery. The finder sub-assemblers extend the commons base managers and attribute inline via soleAccountPubKey — now sourced from UserFinderAccount.userFinderPubkeyHex — so upstream's per-account attribution + ExplainedFilter/SubPurpose tags are preserved unchanged. - cache: LocalCache -> ICacheProvider; FilterUserMetadataForKey + pickRelaysToLoadUsers take an injected relayHints: HintIndexer instead of the static LocalCache.relayHints. - UserOutboxFinderSubAssembler inlines the relay-tier union over the UserFinderAccount getters (behaviour-preserving vs the old Account-based outer overload). Android unchanged: UserFinderShims.kt typealiases keep call sites (incl. reqCommand/event EventFinder*) compiling; UserFinderFilterAssemblerSubscription + UserObservers stay in amethyst (Account is-a UserFinderAccount). New commons UserFinderSubscription (LocalUserFinder/LocalUserFinderAccount + overloads) and UserMetadataObservers (observeUser*) added for Desktop. amethyst FilterFindFollowMetadataForKey's outer overload now delegates to the commons inner one. Green: commons JVM + iOS purity (verifyKmpPurity), :amethyst compilePlayDebugKotlin, :desktopApp compile, spotless. Co-Authored-By: Claude Opus 4.8 --- .../follows/FilterFindFollowMetadataForKey.kt | 121 +----------- .../reqCommand/user/UserFinderShims.kt | 30 +++ .../relayClient/user/PickRelaysToLoadUsers.kt | 147 +++++++++++++++ .../user/UserFinderFilterAssembler.kt | 22 +-- .../user/UserFinderSubscription.kt | 85 +++++++++ .../relayClient/user/UserMetadataObservers.kt | 177 ++++++++++++++++++ .../loaders/UserOutboxFinderSubAssembler.kt | 42 +++-- .../user/watchers/FilterReportsToKey.kt | 2 +- .../user/watchers/FilterUserMetadataForKey.kt | 11 +- .../user/watchers/UserCardsSubAssembler.kt | 24 +-- .../user/watchers/UserReportsSubAssembler.kt | 23 +-- .../user/watchers/UserWatcherSubAssembler.kt | 21 +-- 12 files changed, 521 insertions(+), 184 deletions(-) create mode 100644 amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/UserFinderShims.kt create mode 100644 commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/PickRelaysToLoadUsers.kt rename {amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand => commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient}/user/UserFinderFilterAssembler.kt (74%) create mode 100644 commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/UserFinderSubscription.kt create mode 100644 commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/UserMetadataObservers.kt rename {amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand => commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient}/user/loaders/UserOutboxFinderSubAssembler.kt (81%) rename {amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand => commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient}/user/watchers/FilterReportsToKey.kt (96%) rename {amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand => commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient}/user/watchers/FilterUserMetadataForKey.kt (94%) rename {amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand => commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient}/user/watchers/UserCardsSubAssembler.kt (88%) rename {amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand => commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient}/user/watchers/UserReportsSubAssembler.kt (89%) rename {amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand => commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient}/user/watchers/UserWatcherSubAssembler.kt (87%) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/follows/FilterFindFollowMetadataForKey.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/follows/FilterFindFollowMetadataForKey.kt index 4d3ed060cd..f466cb4f17 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/follows/FilterFindFollowMetadataForKey.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/follows/FilterFindFollowMetadataForKey.kt @@ -23,13 +23,13 @@ package com.vitorpamplona.amethyst.service.relayClient.reqCommand.account.follow import com.vitorpamplona.amethyst.commons.defaults.Constants import com.vitorpamplona.amethyst.commons.defaults.DefaultIndexerRelayList import com.vitorpamplona.amethyst.commons.defaults.DefaultSearchRelayList +import com.vitorpamplona.amethyst.commons.relayClient.user.pickRelaysToLoadUsers import com.vitorpamplona.amethyst.model.Account import com.vitorpamplona.amethyst.model.LocalCache import com.vitorpamplona.amethyst.model.User import com.vitorpamplona.amethyst.service.relays.EOSEAccountFast import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl -import com.vitorpamplona.quartz.utils.mapOfSet fun pickRelaysToLoadUsers( users: Set, @@ -68,6 +68,7 @@ fun pickRelaysToLoadUsers( return pickRelaysToLoadUsers( users, + LocalCache.relayHints, indexRelays - cannotConnectRelays, homeRelays - cannotConnectRelays, searchRelays - cannotConnectRelays, @@ -77,121 +78,3 @@ fun pickRelaysToLoadUsers( hasTried, ) } - -fun pickRelaysToLoadUsers( - users: Set, - indexRelays: Set, - homeRelays: Set, - searchRelays: Set, - connected: Set, - commonRelays: Set, - cannotConnectRelays: Set, - hasTried: EOSEAccountFast, -): Map> = - mapOfSet { - users.forEachIndexed { _, key -> - val tried = (hasTried.since(key)?.keys ?: emptySet()) + cannotConnectRelays - - val outbox = key.authorRelayList()?.writeRelaysNorm() - - if (!outbox.isNullOrEmpty()) { - // If there is a home, get from it. - - // if it tried all outbox relays, stop. - // the UserWatch will take over from here. - val leftToTry = (outbox - tried) - leftToTry.forEach { - add(it, key.pubkeyHex) - } - } else { - // if not, tries hints first. - val hints = key.allUsedRelays() + LocalCache.relayHints.hintsForKey(key.pubkeyHex) - - val leftToTryOnHints = hints - tried - - leftToTryOnHints.forEach { - add(it, key.pubkeyHex) - } - - // if there are only a few hints, broadens the search - if (leftToTryOnHints.size < 3) { - // This creates a pre-deterministic order of the array such that - // if this function is called twice, it returns the same arrays - // which gets ignored by the relay client if we send it twice - val indexRelaysLeftToTry = - (indexRelays - tried).sortedBy { relay -> - key.pubkeyHex.hashCode() xor relay.url.hashCode() - } - // This creates a pre-deterministic order of the array such that - // if this function is called twice, it returns the same arrays - // which gets ignored by the relay client if we send it twice - val homeRelaysLeftToTry = - (homeRelays - tried).sortedBy { relay -> - key.pubkeyHex.hashCode() xor relay.url.hashCode() - } - - // picks one at random to avoid overloading these relays - if (users.size > 300) { - if (indexRelaysLeftToTry.size >= 2) { - add(indexRelaysLeftToTry[0], key.pubkeyHex) - add(indexRelaysLeftToTry[1], key.pubkeyHex) - } else if (indexRelaysLeftToTry.size == 1) { - add(indexRelaysLeftToTry.first(), key.pubkeyHex) - } - - homeRelaysLeftToTry.forEach { - add(it, key.pubkeyHex) - } - } else { - indexRelaysLeftToTry.forEach { - add(it, key.pubkeyHex) - } - - homeRelaysLeftToTry.forEach { - add(it, key.pubkeyHex) - } - } - - if (indexRelaysLeftToTry.size < 2) { - val searchRelaysLeftToTry = searchRelays - tried - - searchRelaysLeftToTry.forEach { - add(it, key.pubkeyHex) - } - - val connectedRelaysLeftToTry = - (connected - tried) - .sortedBy { relay -> - key.pubkeyHex.hashCode() xor relay.url.hashCode() - }.take(100) - - // picks one at random to avoid overloading these relays - if (users.size > 300) { - connectedRelaysLeftToTry.take(20).forEach { - add(it, key.pubkeyHex) - } - } else { - connectedRelaysLeftToTry.forEach { - add(it, key.pubkeyHex) - } - } - - if (searchRelaysLeftToTry.size < 2) { - // This creates a pre-deterministic order of the array such that - // if this function is called twice, it returns the same arrays - // which gets ignored by the relay client if we send it twice - val allRelaysLeftToTry = - (commonRelays - tried) - .sortedBy { relay -> - key.pubkeyHex.hashCode() xor relay.url.hashCode() - }.take(100) - - allRelaysLeftToTry.forEach { - add(it, key.pubkeyHex) - } - } - } - } - } - } - } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/UserFinderShims.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/UserFinderShims.kt new file mode 100644 index 0000000000..c9d01b3b68 --- /dev/null +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/UserFinderShims.kt @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.service.relayClient.reqCommand.user + +/** + * Back-compat aliases: the per-user metadata finder moved to commons + * (`com.vitorpamplona.amethyst.commons.relayClient.user`). Existing Android call + * sites that reference these by their old names resolve here. + */ +typealias UserFinderFilterAssembler = com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderFilterAssembler + +typealias UserFinderQueryState = com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderQueryState diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/PickRelaysToLoadUsers.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/PickRelaysToLoadUsers.kt new file mode 100644 index 0000000000..289a2c96bf --- /dev/null +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/PickRelaysToLoadUsers.kt @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.relayClient.user + +import com.vitorpamplona.amethyst.commons.model.User +import com.vitorpamplona.amethyst.commons.relays.EOSEAccountFast +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.hints.HintIndexer +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.utils.mapOfSet + +fun pickRelaysToLoadUsers( + users: Set, + relayHints: HintIndexer, + indexRelays: Set, + homeRelays: Set, + searchRelays: Set, + connected: Set, + commonRelays: Set, + cannotConnectRelays: Set, + hasTried: EOSEAccountFast, +): Map> = + mapOfSet { + users.forEachIndexed { _, key -> + val tried = (hasTried.since(key)?.keys ?: emptySet()) + cannotConnectRelays + + val outbox = key.authorRelayList()?.writeRelaysNorm() + + if (!outbox.isNullOrEmpty()) { + // If there is a home, get from it. + + // if it tried all outbox relays, stop. + // the UserWatch will take over from here. + val leftToTry = (outbox - tried) + leftToTry.forEach { + add(it, key.pubkeyHex) + } + } else { + // if not, tries hints first. + val hints = key.allUsedRelays() + relayHints.hintsForKey(key.pubkeyHex) + + val leftToTryOnHints = hints - tried + + leftToTryOnHints.forEach { + add(it, key.pubkeyHex) + } + + // if there are only a few hints, broadens the search + if (leftToTryOnHints.size < 3) { + // This creates a pre-deterministic order of the array such that + // if this function is called twice, it returns the same arrays + // which gets ignored by the relay client if we send it twice + val indexRelaysLeftToTry = + (indexRelays - tried).sortedBy { relay -> + key.pubkeyHex.hashCode() xor relay.url.hashCode() + } + // This creates a pre-deterministic order of the array such that + // if this function is called twice, it returns the same arrays + // which gets ignored by the relay client if we send it twice + val homeRelaysLeftToTry = + (homeRelays - tried).sortedBy { relay -> + key.pubkeyHex.hashCode() xor relay.url.hashCode() + } + + // picks one at random to avoid overloading these relays + if (users.size > 300) { + if (indexRelaysLeftToTry.size >= 2) { + add(indexRelaysLeftToTry[0], key.pubkeyHex) + add(indexRelaysLeftToTry[1], key.pubkeyHex) + } else if (indexRelaysLeftToTry.size == 1) { + add(indexRelaysLeftToTry.first(), key.pubkeyHex) + } + + homeRelaysLeftToTry.forEach { + add(it, key.pubkeyHex) + } + } else { + indexRelaysLeftToTry.forEach { + add(it, key.pubkeyHex) + } + + homeRelaysLeftToTry.forEach { + add(it, key.pubkeyHex) + } + } + + if (indexRelaysLeftToTry.size < 2) { + val searchRelaysLeftToTry = searchRelays - tried + + searchRelaysLeftToTry.forEach { + add(it, key.pubkeyHex) + } + + val connectedRelaysLeftToTry = + (connected - tried) + .sortedBy { relay -> + key.pubkeyHex.hashCode() xor relay.url.hashCode() + }.take(100) + + // picks one at random to avoid overloading these relays + if (users.size > 300) { + connectedRelaysLeftToTry.take(20).forEach { + add(it, key.pubkeyHex) + } + } else { + connectedRelaysLeftToTry.forEach { + add(it, key.pubkeyHex) + } + } + + if (searchRelaysLeftToTry.size < 2) { + // This creates a pre-deterministic order of the array such that + // if this function is called twice, it returns the same arrays + // which gets ignored by the relay client if we send it twice + val allRelaysLeftToTry = + (commonRelays - tried) + .sortedBy { relay -> + key.pubkeyHex.hashCode() xor relay.url.hashCode() + }.take(100) + + allRelaysLeftToTry.forEach { + add(it, key.pubkeyHex) + } + } + } + } + } + } + } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/UserFinderFilterAssembler.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/UserFinderFilterAssembler.kt similarity index 74% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/UserFinderFilterAssembler.kt rename to commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/UserFinderFilterAssembler.kt index b0383beda5..9e92c5213a 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/UserFinderFilterAssembler.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/UserFinderFilterAssembler.kt @@ -18,18 +18,16 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.service.relayClient.reqCommand.user +package com.vitorpamplona.amethyst.commons.relayClient.user import androidx.compose.runtime.Stable +import com.vitorpamplona.amethyst.commons.model.User +import com.vitorpamplona.amethyst.commons.model.cache.ICacheProvider import com.vitorpamplona.amethyst.commons.relayClient.composeSubscriptionManagers.ComposeSubscriptionManager -import com.vitorpamplona.amethyst.model.Account -import com.vitorpamplona.amethyst.model.LocalCache -import com.vitorpamplona.amethyst.model.User -import com.vitorpamplona.amethyst.service.relayClient.AccountScopedQuery -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.loaders.UserOutboxFinderSubAssembler -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.watchers.UserCardsSubAssembler -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.watchers.UserReportsSubAssembler -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.watchers.UserWatcherSubAssembler +import com.vitorpamplona.amethyst.commons.relayClient.user.loaders.UserOutboxFinderSubAssembler +import com.vitorpamplona.amethyst.commons.relayClient.user.watchers.UserCardsSubAssembler +import com.vitorpamplona.amethyst.commons.relayClient.user.watchers.UserReportsSubAssembler +import com.vitorpamplona.amethyst.commons.relayClient.user.watchers.UserWatcherSubAssembler import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.RelayOfflineTracker @@ -37,13 +35,13 @@ import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.RelayOfflineT @Stable class UserFinderQueryState( val user: User, - override val account: Account, -) : AccountScopedQuery + val account: UserFinderAccount, +) @Stable class UserFinderFilterAssembler( client: INostrClient, - cache: LocalCache, + cache: ICacheProvider, failureTracker: RelayOfflineTracker, ) : ComposeSubscriptionManager() { val group = diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/UserFinderSubscription.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/UserFinderSubscription.kt new file mode 100644 index 0000000000..51dfc57ea3 --- /dev/null +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/UserFinderSubscription.kt @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.relayClient.user + +import androidx.compose.runtime.Composable +import androidx.compose.runtime.remember +import androidx.compose.runtime.staticCompositionLocalOf +import com.vitorpamplona.amethyst.commons.model.User +import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.LifecycleAwareKeyDataSourceSubscription + +/** + * The shared per-user metadata data source for the current front end. A front + * end provides this once near its composition root (Android via AppModules, + * Desktop via its subscriptions coordinator). Reading it without a provider is + * a programming error — the `observeUser*` composables must never be reachable + * from a composition that has no relay client (e.g. the Android `:napplet` + * sandbox process). + */ +val LocalUserFinder = + staticCompositionLocalOf { + error("LocalUserFinder not provided") + } + +/** + * The current logged-in account, in the narrow [UserFinderAccount] view the + * finder needs to route REQs. Provided alongside [LocalUserFinder]. + */ +val LocalUserFinderAccount = + staticCompositionLocalOf { + error("LocalUserFinderAccount not provided") + } + +/** + * Subscribes to relay updates for [user]'s metadata (and relay list / reports / + * contact cards) for as long as this composable is in composition, coalesced + * with every other on-screen user into batched REQs by [dataSource]. + * + * Because a `LazyColumn` composes only the visible window (+ a small prefetch + * buffer), this naturally means "load metadata only for users currently on + * screen" — the [LifecycleAwareKeyDataSourceSubscription] unsubscribes ~30s + * after the row leaves composition or the app is backgrounded. + */ +@Composable +fun UserFinderFilterAssemblerSubscription( + user: User, + account: UserFinderAccount, + dataSource: UserFinderFilterAssembler, +) { + // Different screens get their own query-state instance even when tracking + // the same user; the assembler dedups to one REQ per pubkey. + val state = remember(user, account) { UserFinderQueryState(user, account) } + + LifecycleAwareKeyDataSourceSubscription(state, dataSource) +} + +/** + * Convenience overload that reads the front end's [LocalUserFinder] and + * [LocalUserFinderAccount] from the composition. + */ +@Composable +fun UserFinderFilterAssemblerSubscription(user: User) { + UserFinderFilterAssemblerSubscription( + user = user, + account = LocalUserFinderAccount.current, + dataSource = LocalUserFinder.current, + ) +} diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/UserMetadataObservers.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/UserMetadataObservers.kt new file mode 100644 index 0000000000..c1d20a41d9 --- /dev/null +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/UserMetadataObservers.kt @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.relayClient.user + +import androidx.compose.runtime.Composable +import androidx.compose.runtime.State +import androidx.compose.runtime.remember +import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.vitorpamplona.amethyst.commons.model.User +import com.vitorpamplona.amethyst.commons.model.nip01Core.UserInfo +import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.map + +/** + * Shared, platform-agnostic observers for a single user's metadata (kind 0). + * + * Each observer both (a) opens a composition-scoped relay subscription for the + * user via [UserFinderFilterAssemblerSubscription] — so metadata is fetched only + * while the user is on screen — and (b) collects the resulting cache flow so the + * UI recomposes when the metadata arrives. The `(user)` overloads read the + * front end's [LocalUserFinder] / [LocalUserFinderAccount]; the explicit-param + * overloads are for callers that already hold both (and for tests). + * + * These are metadata-only. Richer per-user observers that depend on account + * subsystems not yet in commons (contact-card petnames, follow counts, + * bookmarks, statuses) remain in the Android layer for now and layer on top of + * the same subscription. + */ +@Composable +fun observeUserInfo( + user: User, + userFinder: UserFinderFilterAssembler, + account: UserFinderAccount, +): State { + UserFinderFilterAssemblerSubscription(user, account, userFinder) + return user.metadata().flow.collectAsStateWithLifecycle() +} + +@Composable +fun observeUserInfo(user: User): State = observeUserInfo(user, LocalUserFinder.current, LocalUserFinderAccount.current) + +@Composable +fun observeUserPicture( + user: User, + userFinder: UserFinderFilterAssembler, + account: UserFinderAccount, +): State { + UserFinderFilterAssemblerSubscription(user, account, userFinder) + + val flow = + remember(user) { + user + .metadata() + .flow + .map { it?.info?.picture } + .distinctUntilChanged() + } + + return flow.collectAsStateWithLifecycle( + user + .metadataOrNull() + ?.flow + ?.value + ?.info + ?.picture, + ) +} + +@Composable +fun observeUserPicture(user: User): State = observeUserPicture(user, LocalUserFinder.current, LocalUserFinderAccount.current) + +@Composable +fun observeUserBanner( + user: User, + userFinder: UserFinderFilterAssembler, + account: UserFinderAccount, +): State { + UserFinderFilterAssemblerSubscription(user, account, userFinder) + + val flow = + remember(user) { + user + .metadata() + .flow + .map { it?.info?.banner } + .distinctUntilChanged() + } + + return flow.collectAsStateWithLifecycle( + user + .metadataOrNull() + ?.flow + ?.value + ?.info + ?.banner, + ) +} + +@Composable +fun observeUserBanner(user: User): State = observeUserBanner(user, LocalUserFinder.current, LocalUserFinderAccount.current) + +@Composable +fun observeUserAboutMe( + user: User, + userFinder: UserFinderFilterAssembler, + account: UserFinderAccount, +): State { + UserFinderFilterAssemblerSubscription(user, account, userFinder) + + val flow = + remember(user) { + user + .metadata() + .flow + .map { it?.info?.about ?: "" } + .distinctUntilChanged() + } + + return flow.collectAsStateWithLifecycle( + user + .metadataOrNull() + ?.flow + ?.value + ?.info + ?.about ?: "", + ) +} + +@Composable +fun observeUserAboutMe(user: User): State = observeUserAboutMe(user, LocalUserFinder.current, LocalUserFinderAccount.current) + +/** + * The user's best available display name from their own metadata (kind 0), + * falling back to a truncated pubkey. Metadata-only: it does NOT apply the + * viewing account's private contact-card petname (that stays in the Android + * layer, which wraps this). + */ +@Composable +fun observeUserName( + user: User, + userFinder: UserFinderFilterAssembler, + account: UserFinderAccount, +): State { + UserFinderFilterAssemblerSubscription(user, account, userFinder) + + val flow = + remember(user) { + user + .metadata() + .flow + .map { it?.info?.bestName() ?: user.toBestDisplayName() } + .distinctUntilChanged() + } + + return flow.collectAsStateWithLifecycle(user.toBestDisplayName()) +} + +@Composable +fun observeUserName(user: User): State = observeUserName(user, LocalUserFinder.current, LocalUserFinderAccount.current) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/loaders/UserOutboxFinderSubAssembler.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/loaders/UserOutboxFinderSubAssembler.kt similarity index 81% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/loaders/UserOutboxFinderSubAssembler.kt rename to commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/loaders/UserOutboxFinderSubAssembler.kt index 7743fb7c7c..0243f98f69 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/loaders/UserOutboxFinderSubAssembler.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/loaders/UserOutboxFinderSubAssembler.kt @@ -18,18 +18,18 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.loaders +package com.vitorpamplona.amethyst.commons.relayClient.user.loaders import com.vitorpamplona.amethyst.commons.defaults.DefaultIndexerRelayList import com.vitorpamplona.amethyst.commons.defaults.DefaultSearchRelayList +import com.vitorpamplona.amethyst.commons.model.User +import com.vitorpamplona.amethyst.commons.model.cache.ICacheProvider import com.vitorpamplona.amethyst.commons.relayClient.eoseManagers.BaseEoseManager import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.ExplainedFilter import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.SubPurpose -import com.vitorpamplona.amethyst.model.LocalCache -import com.vitorpamplona.amethyst.model.User -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.account.follows.pickRelaysToLoadUsers -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.UserFinderQueryState -import com.vitorpamplona.amethyst.service.relays.EOSEAccountFast +import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderQueryState +import com.vitorpamplona.amethyst.commons.relayClient.user.pickRelaysToLoadUsers +import com.vitorpamplona.amethyst.commons.relays.EOSEAccountFast import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient @@ -44,7 +44,7 @@ import com.vitorpamplona.quartz.utils.TimeUtils class UserOutboxFinderSubAssembler( client: INostrClient, - val cache: LocalCache, + val cache: ICacheProvider, val failureTracker: RelayOfflineTracker, allKeys: () -> Set, ) : BaseEoseManager(client, allKeys) { @@ -108,19 +108,37 @@ class UserOutboxFinderSubAssembler( // and the users being resolved are whoever is on screen rather than anyone's follow list — so // with several accounts active there is no single honest owner for a given filter, and // splitting the sweep per account would re-issue the same lookups once per account. - // Deduped by pubkey, not by `Account`: that class uses identity equality, so two objects for - // the same logged-in user would look like two accounts and suppress attribution entirely. + // Deduped by pubkey, not by account identity: two objects for the same logged-in user would + // look like two accounts and suppress attribution entirely. val soleAccountPubKey = accounts - .mapTo(mutableSetOf()) { it.userProfile().pubkeyHex } + .mapTo(mutableSetOf()) { it.userFinderPubkeyHex } .singleOrNull() + // Union of every asking account's relay tiers. The UserFinderAccount getters already apply the + // platform default fallbacks (index/search), matching the prior outer pickRelaysToLoadUsers. + val cannotConnect = failureTracker.cannotConnectRelays + val indexRelays = mutableSetOf() + val homeRelays = mutableSetOf() + val searchRelays = mutableSetOf() + val commonRelays = mutableSetOf() + accounts.forEach { account -> + indexRelays.addAll(account.indexRelays()) + homeRelays.addAll(account.outboxHomeRelays()) + searchRelays.addAll(account.searchRelays()) + commonRelays.addAll(account.commonRelays()) + } + val perRelayKeysBoth = pickRelaysToLoadUsers( noOutboxList, - accounts, + cache.relayHints, + indexRelays - cannotConnect, + homeRelays - cannotConnect, + searchRelays - cannotConnect, connectedRelays, - failureTracker.cannotConnectRelays, + commonRelays - cannotConnect, + cannotConnect, hasTried, ) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/watchers/FilterReportsToKey.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/watchers/FilterReportsToKey.kt similarity index 96% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/watchers/FilterReportsToKey.kt rename to commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/watchers/FilterReportsToKey.kt index a369d824e7..d8b069cfa3 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/watchers/FilterReportsToKey.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/watchers/FilterReportsToKey.kt @@ -18,7 +18,7 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.watchers +package com.vitorpamplona.amethyst.commons.relayClient.user.watchers import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.ExplainedFilter import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.SubPurpose diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/watchers/FilterUserMetadataForKey.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/watchers/FilterUserMetadataForKey.kt similarity index 94% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/watchers/FilterUserMetadataForKey.kt rename to commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/watchers/FilterUserMetadataForKey.kt index e52167f763..23227231ad 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/watchers/FilterUserMetadataForKey.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/watchers/FilterUserMetadataForKey.kt @@ -18,16 +18,16 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.watchers +package com.vitorpamplona.amethyst.commons.relayClient.user.watchers +import com.vitorpamplona.amethyst.commons.model.User import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.ExplainedFilter import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.SubPurpose -import com.vitorpamplona.amethyst.model.LocalCache -import com.vitorpamplona.amethyst.model.User -import com.vitorpamplona.amethyst.service.relays.EOSEAccountFast +import com.vitorpamplona.amethyst.commons.relays.EOSEAccountFast import com.vitorpamplona.quartz.experimental.nipA3.PaymentTargetsEvent import com.vitorpamplona.quartz.marmot.mip00KeyPackages.KeyPackageRelayListEvent import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.hints.HintIndexer import com.vitorpamplona.quartz.nip01Core.metadata.MetadataEvent import com.vitorpamplona.quartz.nip01Core.relay.client.pool.RelayBasedFilter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl @@ -60,6 +60,7 @@ val UserMetadataForKeyKinds = fun filterUserMetadataForKey( authors: Set, + relayHints: HintIndexer, indexRelays: Set, cannotConnectRelays: Set, since: EOSEAccountFast, @@ -72,7 +73,7 @@ fun filterUserMetadataForKey( val relays = when { outbox == null -> - key.allUsedRelays() + LocalCache.relayHints.hintsForKey(key.pubkeyHex) + indexRelays + key.allUsedRelays() + relayHints.hintsForKey(key.pubkeyHex) + indexRelays // Outbox is published but exhausted (every relay either EOSE'd // or is known-unreachable) and metadata is still missing — // widen to indexers so a misconfigured outbox doesn't strand diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/watchers/UserCardsSubAssembler.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/watchers/UserCardsSubAssembler.kt similarity index 88% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/watchers/UserCardsSubAssembler.kt rename to commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/watchers/UserCardsSubAssembler.kt index 0404db46cb..9c4ffbc31e 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/watchers/UserCardsSubAssembler.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/watchers/UserCardsSubAssembler.kt @@ -18,16 +18,16 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.watchers +package com.vitorpamplona.amethyst.commons.relayClient.user.watchers +import com.vitorpamplona.amethyst.commons.model.User +import com.vitorpamplona.amethyst.commons.model.cache.ICacheProvider import com.vitorpamplona.amethyst.commons.model.toHexSet import com.vitorpamplona.amethyst.commons.relayClient.assemblers.filterContactCardsToTargetKeysFromTrustedAccountsInTheRelay import com.vitorpamplona.amethyst.commons.relayClient.eoseManagers.SingleSubEoseManager -import com.vitorpamplona.amethyst.model.LocalCache -import com.vitorpamplona.amethyst.model.User -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.UserFinderQueryState -import com.vitorpamplona.amethyst.service.relays.MutableTime -import com.vitorpamplona.amethyst.service.relays.SincePerRelayMap +import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderQueryState +import com.vitorpamplona.amethyst.commons.relays.MutableTime +import com.vitorpamplona.amethyst.commons.relays.SincePerRelayMap import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.pool.RelayBasedFilter @@ -38,7 +38,7 @@ import com.vitorpamplona.quartz.utils.mapOfSet class UserCardsSubAssembler( client: INostrClient, - val cache: LocalCache, + val cache: ICacheProvider, allKeys: () -> Set, ) : SingleSubEoseManager(client, allKeys) { override fun newEose( @@ -74,22 +74,22 @@ class UserCardsSubAssembler( // accounts, so with several active none of them owns a given filter. val soleAccountPubKey = accounts - .mapTo(mutableSetOf()) { it.userProfile().pubkeyHex } + .mapTo(mutableSetOf()) { it.userFinderPubkeyHex } .singleOrNull() val trustedAccounts: Map> = mapOfSet { accounts.forEach { account -> - account.homeRelays.flow.value.forEach { - add(it, account.userProfile().pubkeyHex) + account.cardHomeRelays().forEach { + add(it, account.userFinderPubkeyHex) } } - accounts.map { it.trustProviderList.liveUserRankProvider.value }.forEach { provider -> + accounts.map { it.trustProvider() }.forEach { provider -> if (provider != null) { add(provider.relayUrl, provider.pubkey) } } - accounts.map { it.trustProviderList.liveUserFollowerCount.value }.forEach { provider -> + accounts.map { it.followerCountProvider() }.forEach { provider -> if (provider != null) { add(provider.relayUrl, provider.pubkey) } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/watchers/UserReportsSubAssembler.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/watchers/UserReportsSubAssembler.kt similarity index 89% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/watchers/UserReportsSubAssembler.kt rename to commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/watchers/UserReportsSubAssembler.kt index 4371a4f822..5731d42b64 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/watchers/UserReportsSubAssembler.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/watchers/UserReportsSubAssembler.kt @@ -18,16 +18,16 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.watchers +package com.vitorpamplona.amethyst.commons.relayClient.user.watchers +import com.vitorpamplona.amethyst.commons.model.User +import com.vitorpamplona.amethyst.commons.model.cache.ICacheProvider import com.vitorpamplona.amethyst.commons.model.toHexSet import com.vitorpamplona.amethyst.commons.relayClient.eoseManagers.SingleSubEoseManager -import com.vitorpamplona.amethyst.model.Account -import com.vitorpamplona.amethyst.model.LocalCache -import com.vitorpamplona.amethyst.model.User -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.UserFinderQueryState -import com.vitorpamplona.amethyst.service.relays.MutableTime -import com.vitorpamplona.amethyst.service.relays.SincePerRelayMap +import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderAccount +import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderQueryState +import com.vitorpamplona.amethyst.commons.relays.MutableTime +import com.vitorpamplona.amethyst.commons.relays.SincePerRelayMap import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.pool.RelayBasedFilter import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter @@ -35,7 +35,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl class UserReportsSubAssembler( client: INostrClient, - val cache: LocalCache, + val cache: ICacheProvider, allKeys: () -> Set, ) : SingleSubEoseManager(client, allKeys) { override fun newEose( @@ -72,12 +72,13 @@ class UserReportsSubAssembler( } private fun filtersFor( - account: Account, + account: UserFinderAccount, lastUsersOnFilter: Set, ): List { - val accountPubKey = account.userProfile().pubkeyHex + val accountPubKey = account.userFinderPubkeyHex - return account.declaredFollowsPerOutboxRelay.value + return account + .declaredFollowsByOutboxRelay() .flatMap { (relay, trustedUsersInThisRelay) -> // this relay + accounts are where we could find reports. // we might have already loaded them, so let's separate new targets that were checked before from the others diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/watchers/UserWatcherSubAssembler.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/watchers/UserWatcherSubAssembler.kt similarity index 87% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/watchers/UserWatcherSubAssembler.kt rename to commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/watchers/UserWatcherSubAssembler.kt index f5381a9be1..673b71a977 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/user/watchers/UserWatcherSubAssembler.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/watchers/UserWatcherSubAssembler.kt @@ -18,14 +18,13 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.watchers +package com.vitorpamplona.amethyst.commons.relayClient.user.watchers -import com.vitorpamplona.amethyst.commons.defaults.DefaultIndexerRelayList +import com.vitorpamplona.amethyst.commons.model.User +import com.vitorpamplona.amethyst.commons.model.cache.ICacheProvider import com.vitorpamplona.amethyst.commons.relayClient.eoseManagers.BaseEoseManager -import com.vitorpamplona.amethyst.model.LocalCache -import com.vitorpamplona.amethyst.model.User -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.UserFinderQueryState -import com.vitorpamplona.amethyst.service.relays.EOSEAccountFast +import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderQueryState +import com.vitorpamplona.amethyst.commons.relays.EOSEAccountFast import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.RelayOfflineTracker @@ -37,7 +36,7 @@ import com.vitorpamplona.quartz.utils.TimeUtils class UserWatcherSubAssembler( client: INostrClient, - val cache: LocalCache, + val cache: ICacheProvider, val failureTracker: RelayOfflineTracker, allKeys: () -> Set, ) : BaseEoseManager(client, allKeys) { @@ -100,20 +99,18 @@ class UserWatcherSubAssembler( // account and the users are whoever is on screen, so with several askers none of them owns it. val soleAccountPubKey = keys - .mapTo(mutableSetOf()) { it.account.userProfile().pubkeyHex } + .mapTo(mutableSetOf()) { it.account.userFinderPubkeyHex } .singleOrNull() val indexRelays = mutableSetOf() keys.mapTo(mutableSetOf()) { it.account }.forEach { - indexRelays.addAll( - it.indexerRelayList.flow.value - .ifEmpty { DefaultIndexerRelayList }, - ) + indexRelays.addAll(it.indexRelays()) } val newFilters = filterUserMetadataForKey( users, + cache.relayHints, indexRelays, failureTracker.cannotConnectRelays, latestEOSEs, From 4d31cd7d25f049393ba9c4fcffe071b7ec2ef00d Mon Sep 17 00:00:00 2001 From: nrobi144 Date: Thu, 6 Aug 2026 11:02:09 +0300 Subject: [PATCH 116/132] refactor: move per-note event finder to commons on the new upstream model (Phase 3) Move EventFinderFilterAssembler(+QueryState), the loaders (NoteEventLoader, FilterMissingEvents, FilterMissingAddressables, AddressableAuthorRelayLoader) and watchers (EventWatcher, FilterRepliesAndReactionsToNotes/Addresses) from amethyst/reqCommand/event/ into commons/relayClient/event/, mirroring the Phase 2 user-finder move. - EventFinderQueryState carries the narrow UserFinderAccount; DROPS AccountScopedQuery. Attribution stays inline via userFinderPubkeyHex; ExplainedFilter/SubPurpose tags (REFERENCED_EVENTS / ENGAGEMENT) preserved. - cache: LocalCache -> ICacheProvider threaded into NoteEventLoaderSubAssembler + the FilterMissing* functions (which called the LocalCache singleton statically); getOrCreateUser is now nullable at these sites. Added ICacheProvider.checkGetOrCreateUser default; LocalCache.checkGetOrCreateUser now overrides it. - SingleSubNoEoseCacheEoseManager moved to commons (account-agnostic accountPubKeyOf); its two amethyst callers keep attribution via a new amethyst subclass AccountScopedSingleSubNoEoseCacheEoseManager (ChannelLoader) or the plain commons base (NWCPaymentWatcher, which never attributed). Android unchanged via EventFinderShims.kt (typealiases + AccountViewModel overload); EventObservers.kt stays in amethyst. New commons EventFinderFilterAssemblerSubscription (LocalEventFinder + (note) overload reusing LocalUserFinderAccount) for Desktop. Updated the direct loader-function callers (search/hashtag/thread sub-assemblers + the moved test) to import from commons and pass LocalCache. Green: commons JVM + iOS purity, :amethyst compilePlayDebugKotlin, :desktopApp compile, spotless. Co-Authored-By: Claude Opus 4.8 --- .../amethyst/model/LocalCache.kt | 2 +- ...ntScopedSingleSubNoEoseCacheEoseManager.kt | 46 ++++++++++ .../ChannelLoaderSubAssembler.kt | 4 +- ...lerSubscription.kt => EventFinderShims.kt} | 40 ++++----- .../nwc/NWCPaymentWatcherSubAssembler.kt | 2 +- .../subassemblies/FilterByAddress.kt | 6 +- .../subassemblies/FilterByEvent.kt | 8 +- .../hashtag/datasource/FilterHashtagLabels.kt | 6 +- .../FilterMissingEventsForThread.kt | 17 ++-- ...ssableAuthorRelayLoaderSubAssemblerTest.kt | 5 +- .../commons/model/cache/ICacheProvider.kt | 10 +++ .../SingleSubNoEoseCacheEoseManager.kt | 13 ++- .../event/EventFinderFilterAssembler.kt | 25 +++--- .../EventFinderFilterAssemblerSubscription.kt | 84 +++++++++++++++++++ ...ddressableAuthorRelayLoaderSubAssembler.kt | 16 ++-- .../loaders/FilterMissingAddressables.kt | 34 ++++---- .../event/loaders/FilterMissingEvents.kt | 38 +++++---- .../loaders/NoteEventLoaderSubAssembler.kt | 17 ++-- .../watchers/EventWatcherSubAssembler.kt | 16 ++-- .../FilterRepliesAndReactionsToAddresses.kt | 6 +- .../FilterRepliesAndReactionsToNotes.kt | 6 +- 21 files changed, 280 insertions(+), 121 deletions(-) create mode 100644 amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/AccountScopedSingleSubNoEoseCacheEoseManager.kt rename amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/{EventFinderFilterAssemblerSubscription.kt => EventFinderShims.kt} (61%) rename {amethyst/src/main/java/com/vitorpamplona/amethyst/service => commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons}/relayClient/eoseManagers/SingleSubNoEoseCacheEoseManager.kt (85%) rename {amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand => commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient}/event/EventFinderFilterAssembler.kt (70%) create mode 100644 commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/EventFinderFilterAssemblerSubscription.kt rename {amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand => commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient}/event/loaders/AddressableAuthorRelayLoaderSubAssembler.kt (89%) rename {amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand => commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient}/event/loaders/FilterMissingAddressables.kt (79%) rename {amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand => commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient}/event/loaders/FilterMissingEvents.kt (79%) rename {amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand => commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient}/event/loaders/NoteEventLoaderSubAssembler.kt (67%) rename {amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand => commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient}/event/watchers/EventWatcherSubAssembler.kt (89%) rename {amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand => commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient}/event/watchers/FilterRepliesAndReactionsToAddresses.kt (96%) rename {amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand => commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient}/event/watchers/FilterRepliesAndReactionsToNotes.kt (96%) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/LocalCache.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/LocalCache.kt index cd2963d6d7..4029469535 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/LocalCache.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/LocalCache.kt @@ -679,7 +679,7 @@ object LocalCache : ILocalCache, ICacheProvider, Dao { fun observeLatestNote(filter: Filter) = observeNotes(filter).map { it.firstOrNull() } - fun checkGetOrCreateUser(key: String): User? = runCatching { getOrCreateUser(key) }.getOrNull() + override fun checkGetOrCreateUser(key: String): User? = runCatching { getOrCreateUser(key) }.getOrNull() fun load(keys: List): List = keys.mapNotNull(::checkGetOrCreateUser) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/AccountScopedSingleSubNoEoseCacheEoseManager.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/AccountScopedSingleSubNoEoseCacheEoseManager.kt new file mode 100644 index 0000000000..c76800af32 --- /dev/null +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/AccountScopedSingleSubNoEoseCacheEoseManager.kt @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.service.relayClient.eoseManagers + +import com.vitorpamplona.amethyst.commons.relayClient.eoseManagers.SingleSubNoEoseCacheEoseManager +import com.vitorpamplona.amethyst.service.relayClient.AccountScopedQuery +import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient + +/** + * Amethyst variant of [SingleSubNoEoseCacheEoseManager] that restores single-account + * attribution for [AccountScopedQuery] keys. + * + * The commons base is account-agnostic (attribution defaults to null) so it can live in + * commonMain. Query states that carry an [Account] (home feed, channels, notifications, …) + * subclass this so their single-account REQs still show up attributed in "Active Relay + * Subscriptions". + * + * Keyed on [AccountScopedQuery] rather than a concrete query-state type: the home feed uses + * HomeQueryState, notifications use AccountQueryState, and checking one concrete class filed the + * other under "not attributed" despite both being built from a single account's data. + */ +abstract class AccountScopedSingleSubNoEoseCacheEoseManager( + client: INostrClient, + allKeys: () -> Set, + invalidateAfterEose: Boolean = false, +) : SingleSubNoEoseCacheEoseManager(client, allKeys, invalidateAfterEose) { + override fun accountPubKeyOf(key: Any?): String? = (key as? AccountScopedQuery)?.account?.userProfile()?.pubkeyHex +} diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/channel/nip28PublicChats/ChannelLoaderSubAssembler.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/channel/nip28PublicChats/ChannelLoaderSubAssembler.kt index c14d14bd19..f06d9a28d7 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/channel/nip28PublicChats/ChannelLoaderSubAssembler.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/channel/nip28PublicChats/ChannelLoaderSubAssembler.kt @@ -20,7 +20,7 @@ */ package com.vitorpamplona.amethyst.service.relayClient.reqCommand.channel.nip28PublicChats -import com.vitorpamplona.amethyst.service.relayClient.eoseManagers.SingleSubNoEoseCacheEoseManager +import com.vitorpamplona.amethyst.service.relayClient.eoseManagers.AccountScopedSingleSubNoEoseCacheEoseManager import com.vitorpamplona.amethyst.service.relayClient.reqCommand.channel.ChannelFinderQueryState import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.pool.RelayBasedFilter @@ -37,7 +37,7 @@ import com.vitorpamplona.quartz.nip01Core.relay.client.pool.RelayBasedFilter class ChannelLoaderSubAssembler( client: INostrClient, allKeys: () -> Set, -) : SingleSubNoEoseCacheEoseManager(client, allKeys, invalidateAfterEose = true) { +) : AccountScopedSingleSubNoEoseCacheEoseManager(client, allKeys, invalidateAfterEose = true) { override fun updateFilter(keys: List): List = filterMissingChannelsById(keys) override fun distinct(key: ChannelFinderQueryState) = key.channel diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/EventFinderFilterAssemblerSubscription.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/EventFinderShims.kt similarity index 61% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/EventFinderFilterAssemblerSubscription.kt rename to amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/EventFinderShims.kt index 005f4f203e..b3a9458bab 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/EventFinderFilterAssemblerSubscription.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/EventFinderShims.kt @@ -21,30 +21,30 @@ package com.vitorpamplona.amethyst.service.relayClient.reqCommand.event import androidx.compose.runtime.Composable -import androidx.compose.runtime.remember -import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.LifecycleAwareKeyDataSourceSubscription -import com.vitorpamplona.amethyst.model.Account +import com.vitorpamplona.amethyst.commons.relayClient.event.EventFinderFilterAssemblerSubscription import com.vitorpamplona.amethyst.model.Note import com.vitorpamplona.amethyst.ui.screen.loggedIn.AccountViewModel +/** + * Back-compat aliases: the per-note event finder moved to commons + * (`com.vitorpamplona.amethyst.commons.relayClient.event`). Existing Android call + * sites that reference these by their old names resolve here. + */ +typealias EventFinderFilterAssembler = com.vitorpamplona.amethyst.commons.relayClient.event.EventFinderFilterAssembler + +typealias EventFinderQueryState = com.vitorpamplona.amethyst.commons.relayClient.event.EventFinderQueryState + +/** + * Android convenience overload: pulls the account + shared event-finder data source + * out of [accountViewModel] and delegates to the commons subscription. `Account` + * is-a `UserFinderAccount`, so no adaptation is needed. + */ @Composable fun EventFinderFilterAssemblerSubscription( note: Note, accountViewModel: AccountViewModel, -) = EventFinderFilterAssemblerSubscription(note, accountViewModel.account, accountViewModel.dataSources().eventFinder) - -@Composable -fun EventFinderFilterAssemblerSubscription( - note: Note, - account: Account, - dataSource: EventFinderFilterAssembler, -) { - // different screens get different states - // even if they are tracking the same tag. - val state = - remember(note, account) { - EventFinderQueryState(note, account) - } - - LifecycleAwareKeyDataSourceSubscription(state, dataSource) -} +) = EventFinderFilterAssemblerSubscription( + note, + accountViewModel.account, + accountViewModel.dataSources().eventFinder, +) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/nwc/NWCPaymentWatcherSubAssembler.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/nwc/NWCPaymentWatcherSubAssembler.kt index b179e5a042..51e4ae0e67 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/nwc/NWCPaymentWatcherSubAssembler.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/nwc/NWCPaymentWatcherSubAssembler.kt @@ -20,7 +20,7 @@ */ package com.vitorpamplona.amethyst.service.relayClient.reqCommand.nwc -import com.vitorpamplona.amethyst.service.relayClient.eoseManagers.SingleSubNoEoseCacheEoseManager +import com.vitorpamplona.amethyst.commons.relayClient.eoseManagers.SingleSubNoEoseCacheEoseManager import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.pool.RelayBasedFilter diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/searchCommand/subassemblies/FilterByAddress.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/searchCommand/subassemblies/FilterByAddress.kt index a527046727..df31d45479 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/searchCommand/subassemblies/FilterByAddress.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/searchCommand/subassemblies/FilterByAddress.kt @@ -20,9 +20,9 @@ */ package com.vitorpamplona.amethyst.service.relayClient.searchCommand.subassemblies +import com.vitorpamplona.amethyst.commons.relayClient.event.loaders.filterMissingAddressables +import com.vitorpamplona.amethyst.commons.relayClient.event.loaders.potentialRelaysToFindAddress import com.vitorpamplona.amethyst.model.LocalCache -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.loaders.filterMissingAddressables -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.loaders.potentialRelaysToFindAddress import com.vitorpamplona.quartz.nip01Core.relay.client.pool.RelayBasedFilter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip19Bech32.entities.NAddress @@ -37,7 +37,7 @@ fun filterByAddress( val list = mapOfSet { if (note.event == null) { - potentialRelaysToFindAddress(note).ifEmpty { default }.forEach { relayUrl -> + potentialRelaysToFindAddress(LocalCache, note).ifEmpty { default }.forEach { relayUrl -> add(relayUrl, note.address) } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/searchCommand/subassemblies/FilterByEvent.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/searchCommand/subassemblies/FilterByEvent.kt index 0a56940449..ae225e1b27 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/searchCommand/subassemblies/FilterByEvent.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/searchCommand/subassemblies/FilterByEvent.kt @@ -20,10 +20,10 @@ */ package com.vitorpamplona.amethyst.service.relayClient.searchCommand.subassemblies +import com.vitorpamplona.amethyst.commons.relayClient.event.loaders.filterMissingEvents +import com.vitorpamplona.amethyst.commons.relayClient.event.loaders.potentialRelaysToFindEvent import com.vitorpamplona.amethyst.model.AddressableNote import com.vitorpamplona.amethyst.model.LocalCache -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.loaders.filterMissingEvents -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.loaders.potentialRelaysToFindEvent import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.relay.client.pool.RelayBasedFilter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl @@ -38,7 +38,7 @@ fun filterByEvent( val list = mapOfSet { if (note !is AddressableNote && note.event == null) { - potentialRelaysToFindEvent(note).ifEmpty { default }.forEach { relayUrl -> + potentialRelaysToFindEvent(LocalCache, note).ifEmpty { default }.forEach { relayUrl -> add(relayUrl, note.idHex) } } @@ -46,7 +46,7 @@ fun filterByEvent( // loads threading that is event-based note.replyTo?.forEach { parentNote -> if (parentNote !is AddressableNote && note.event == null) { - potentialRelaysToFindEvent(note).ifEmpty { default }.forEach { relayUrl -> + potentialRelaysToFindEvent(LocalCache, note).ifEmpty { default }.forEach { relayUrl -> add(relayUrl, note.idHex) } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/hashtag/datasource/FilterHashtagLabels.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/hashtag/datasource/FilterHashtagLabels.kt index 670eff235a..7106576dfb 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/hashtag/datasource/FilterHashtagLabels.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/hashtag/datasource/FilterHashtagLabels.kt @@ -20,12 +20,12 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.hashtag.datasource +import com.vitorpamplona.amethyst.commons.relayClient.event.loaders.filterMissingEvents +import com.vitorpamplona.amethyst.commons.relayClient.event.loaders.potentialRelaysToFindEvent import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.ExplainedFilter import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.SubPurpose import com.vitorpamplona.amethyst.model.Account import com.vitorpamplona.amethyst.model.LocalCache -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.loaders.filterMissingEvents -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.loaders.potentialRelaysToFindEvent import com.vitorpamplona.amethyst.service.relays.SincePerRelayMap import com.vitorpamplona.quartz.nip01Core.relay.client.pool.RelayBasedFilter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl @@ -93,7 +93,7 @@ fun filterHashtagLabels( val target = LocalCache.getNoteIfExists(targetId) if (target?.event == null) { val targetNote = LocalCache.getOrCreateNote(targetId) - potentialRelaysToFindEvent(targetNote).ifEmpty { relays }.forEach { relayUrl -> + potentialRelaysToFindEvent(LocalCache, targetNote).ifEmpty { relays }.forEach { relayUrl -> add(relayUrl, targetId) } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/threadview/datasources/subassembies/FilterMissingEventsForThread.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/threadview/datasources/subassembies/FilterMissingEventsForThread.kt index c2a7d7ba40..a9de4c769d 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/threadview/datasources/subassembies/FilterMissingEventsForThread.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/threadview/datasources/subassembies/FilterMissingEventsForThread.kt @@ -21,11 +21,12 @@ package com.vitorpamplona.amethyst.ui.screen.loggedIn.threadview.datasources.subassembies import com.vitorpamplona.amethyst.commons.model.ThreadAssembler +import com.vitorpamplona.amethyst.commons.relayClient.event.loaders.filterMissingAddressables +import com.vitorpamplona.amethyst.commons.relayClient.event.loaders.filterMissingEvents +import com.vitorpamplona.amethyst.commons.relayClient.event.loaders.potentialRelaysToFindAddress +import com.vitorpamplona.amethyst.commons.relayClient.event.loaders.potentialRelaysToFindEvent import com.vitorpamplona.amethyst.model.AddressableNote -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.loaders.filterMissingAddressables -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.loaders.filterMissingEvents -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.loaders.potentialRelaysToFindAddress -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.loaders.potentialRelaysToFindEvent +import com.vitorpamplona.amethyst.model.LocalCache import com.vitorpamplona.quartz.nip01Core.relay.client.pool.RelayBasedFilter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.utils.mapOfSet @@ -37,14 +38,14 @@ fun filterMissingEventsForThread( val missingEvents = mapOfSet { if (threadInfo.root.event == null && threadInfo.root !is AddressableNote) { - potentialRelaysToFindEvent(threadInfo.root).ifEmpty { defaultRelays }.forEach { relayUrl -> + potentialRelaysToFindEvent(LocalCache, threadInfo.root).ifEmpty { defaultRelays }.forEach { relayUrl -> add(relayUrl, threadInfo.root.idHex) } } threadInfo.allNotes.forEach { if (it !is AddressableNote && it.event == null) { - potentialRelaysToFindEvent(it).ifEmpty { defaultRelays }.forEach { relayUrl -> + potentialRelaysToFindEvent(LocalCache, it).ifEmpty { defaultRelays }.forEach { relayUrl -> add(relayUrl, it.idHex) } } @@ -59,14 +60,14 @@ fun filterMissingEventsForThread( // note's aTag idHex into the hex-keyed event-hint index, which throws // on the non-hex string and kills the whole filter build — leaving a // thread opened on an uncached naddr permanently unfetched. - potentialRelaysToFindAddress(rootNote).ifEmpty { defaultRelays }.forEach { relayUrl -> + potentialRelaysToFindAddress(LocalCache, rootNote).ifEmpty { defaultRelays }.forEach { relayUrl -> add(relayUrl, rootNote.address) } } threadInfo.allNotes.forEach { if (it is AddressableNote && it.event == null) { - potentialRelaysToFindAddress(it).ifEmpty { defaultRelays }.forEach { relayUrl -> + potentialRelaysToFindAddress(LocalCache, it).ifEmpty { defaultRelays }.forEach { relayUrl -> add(relayUrl, it.address) } } diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssemblerTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssemblerTest.kt index 842471e6a4..82fd16f813 100644 --- a/amethyst/src/test/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssemblerTest.kt +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssemblerTest.kt @@ -20,11 +20,12 @@ */ package com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.loaders +import com.vitorpamplona.amethyst.commons.relayClient.event.loaders.AddressableAuthorRelayLoaderSubAssembler +import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderFilterAssembler +import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderQueryState import com.vitorpamplona.amethyst.model.Account import com.vitorpamplona.amethyst.model.LocalCache import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.EventFinderQueryState -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.UserFinderFilterAssembler -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.UserFinderQueryState import com.vitorpamplona.quartz.nip01Core.core.Address import io.mockk.every import io.mockk.mockk diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/cache/ICacheProvider.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/cache/ICacheProvider.kt index 3f2e59bd64..7d9091967f 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/cache/ICacheProvider.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/cache/ICacheProvider.kt @@ -142,5 +142,15 @@ interface ICacheProvider { */ fun getOrCreateUser(pubkey: HexKey): User? + /** + * Gets or creates a User by public key hex, swallowing any failure. + * Used by the event-finder relay-hint scan, which touches many potentially + * malformed pubkeys and must never throw mid-scan. + * + * @param key The user's public key in hex format + * @return The User (existing or newly created), or null on failure + */ + fun checkGetOrCreateUser(key: HexKey): User? = runCatching { getOrCreateUser(key) }.getOrNull() + fun justConsumeMyOwnEvent(event: Event): Boolean } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/SingleSubNoEoseCacheEoseManager.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/eoseManagers/SingleSubNoEoseCacheEoseManager.kt similarity index 85% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/SingleSubNoEoseCacheEoseManager.kt rename to commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/eoseManagers/SingleSubNoEoseCacheEoseManager.kt index a80357e283..622d9f8fbb 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/SingleSubNoEoseCacheEoseManager.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/eoseManagers/SingleSubNoEoseCacheEoseManager.kt @@ -18,11 +18,9 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.service.relayClient.eoseManagers +package com.vitorpamplona.amethyst.commons.relayClient.eoseManagers -import com.vitorpamplona.amethyst.commons.relayClient.eoseManagers.BaseEoseManager import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.attributedTo -import com.vitorpamplona.amethyst.service.relayClient.AccountScopedQuery import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.pool.RelayBasedFilter @@ -85,9 +83,10 @@ abstract class SingleSubNoEoseCacheEoseManager( /** * The account behind [key], when the key is account-scoped. Null for keys about other users. * - * Keyed on [AccountScopedQuery] rather than a concrete query-state type: the home feed uses - * HomeQueryState, notifications use AccountQueryState, and checking one concrete class filed the - * other under "not attributed" despite both being built from a single account's data. + * Account-agnostic in commons: front ends that want single-account attribution override this + * (see the amethyst `AccountScopedSingleSubNoEoseCacheEoseManager`, which reads + * `(key as? AccountScopedQuery)?.account?.userProfile()?.pubkeyHex`). The default returns null, + * so pooled / cross-account subscriptions are filed as "not attributed". */ - private fun accountPubKeyOf(key: Any?): String? = (key as? AccountScopedQuery)?.account?.userProfile()?.pubkeyHex + open fun accountPubKeyOf(key: Any?): String? = null } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/EventFinderFilterAssembler.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/EventFinderFilterAssembler.kt similarity index 70% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/EventFinderFilterAssembler.kt rename to commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/EventFinderFilterAssembler.kt index f1f320ae50..a5e1ed022c 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/EventFinderFilterAssembler.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/EventFinderFilterAssembler.kt @@ -18,36 +18,35 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.service.relayClient.reqCommand.event +package com.vitorpamplona.amethyst.commons.relayClient.event import androidx.compose.runtime.Stable +import com.vitorpamplona.amethyst.commons.model.Note +import com.vitorpamplona.amethyst.commons.model.cache.ICacheProvider import com.vitorpamplona.amethyst.commons.relayClient.composeSubscriptionManagers.ComposeSubscriptionManager -import com.vitorpamplona.amethyst.model.Account -import com.vitorpamplona.amethyst.model.LocalCache -import com.vitorpamplona.amethyst.model.Note -import com.vitorpamplona.amethyst.service.relayClient.AccountScopedQuery -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.loaders.AddressableAuthorRelayLoaderSubAssembler -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.loaders.NoteEventLoaderSubAssembler -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.watchers.EventWatcherSubAssembler -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.UserFinderFilterAssembler +import com.vitorpamplona.amethyst.commons.relayClient.event.loaders.AddressableAuthorRelayLoaderSubAssembler +import com.vitorpamplona.amethyst.commons.relayClient.event.loaders.NoteEventLoaderSubAssembler +import com.vitorpamplona.amethyst.commons.relayClient.event.watchers.EventWatcherSubAssembler +import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderAccount +import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderFilterAssembler import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient // This allows multiple screen to be listening to tags, even the same tag @Stable class EventFinderQueryState( val note: Note, - override val account: Account, -) : AccountScopedQuery + val account: UserFinderAccount, +) @Stable class EventFinderFilterAssembler( client: INostrClient, - cache: LocalCache, + cache: ICacheProvider, userFinder: UserFinderFilterAssembler, ) : ComposeSubscriptionManager() { val group = listOf( - NoteEventLoaderSubAssembler(client, ::allKeys), + NoteEventLoaderSubAssembler(client, cache, ::allKeys), EventWatcherSubAssembler(client, ::allKeys), AddressableAuthorRelayLoaderSubAssembler(cache, ::allKeys, userFinder), ) diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/EventFinderFilterAssemblerSubscription.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/EventFinderFilterAssemblerSubscription.kt new file mode 100644 index 0000000000..bf9f637ec1 --- /dev/null +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/EventFinderFilterAssemblerSubscription.kt @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.relayClient.event + +import androidx.compose.runtime.Composable +import androidx.compose.runtime.remember +import androidx.compose.runtime.staticCompositionLocalOf +import com.vitorpamplona.amethyst.commons.model.Note +import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.LifecycleAwareKeyDataSourceSubscription +import com.vitorpamplona.amethyst.commons.relayClient.user.LocalUserFinderAccount +import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderAccount + +/** + * The shared per-note event data source (reactions / zaps / reposts / replies / + * OTS / references) for the current front end. Provided once near the composition + * root (Android via AppModules, Desktop via its subscriptions coordinator). + * Reading it without a provider is a programming error — the per-note observers + * must never be reachable from a composition that has no relay client (e.g. the + * Android `:napplet` sandbox process). + * + * The *account* half is reused from the user-finder: [LocalUserFinderAccount] + * already carries the narrow relay-hint seam the event loaders need. + */ +val LocalEventFinder = + staticCompositionLocalOf { + error("LocalEventFinder not provided") + } + +/** + * Subscribes to relay updates for [note]'s interactions (reactions, zaps, + * reposts, replies, …) for as long as this composable is in composition, + * coalesced with every other on-screen note into batched REQs by [dataSource]. + * + * Like the user-finder, because a `LazyColumn` composes only the visible window + * (+ a small prefetch buffer) this means "load interactions only for notes + * currently on screen" — [LifecycleAwareKeyDataSourceSubscription] unsubscribes + * ~30s after the row leaves composition or the app is backgrounded. + */ +@Composable +fun EventFinderFilterAssemblerSubscription( + note: Note, + account: UserFinderAccount, + dataSource: EventFinderFilterAssembler, +) { + // Different screens get their own query-state instance even when tracking + // the same note; the assembler dedups to one REQ per note. + val state = + remember(note, account) { + EventFinderQueryState(note, account) + } + + LifecycleAwareKeyDataSourceSubscription(state, dataSource) +} + +/** + * Convenience overload that reads the front end's [LocalEventFinder] and + * [LocalUserFinderAccount] from the composition. + */ +@Composable +fun EventFinderFilterAssemblerSubscription(note: Note) { + EventFinderFilterAssemblerSubscription( + note = note, + account = LocalUserFinderAccount.current, + dataSource = LocalEventFinder.current, + ) +} diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssembler.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/AddressableAuthorRelayLoaderSubAssembler.kt similarity index 89% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssembler.kt rename to commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/AddressableAuthorRelayLoaderSubAssembler.kt index 9dba6ed2bd..68e5a40544 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/AddressableAuthorRelayLoaderSubAssembler.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/AddressableAuthorRelayLoaderSubAssembler.kt @@ -18,15 +18,15 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.loaders +package com.vitorpamplona.amethyst.commons.relayClient.event.loaders +import com.vitorpamplona.amethyst.commons.model.AddressableNote +import com.vitorpamplona.amethyst.commons.model.cache.ICacheProvider import com.vitorpamplona.amethyst.commons.relayClient.eoseManagers.IEoseManager +import com.vitorpamplona.amethyst.commons.relayClient.event.EventFinderQueryState +import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderFilterAssembler +import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderQueryState import com.vitorpamplona.amethyst.commons.service.BundledUpdate -import com.vitorpamplona.amethyst.model.AddressableNote -import com.vitorpamplona.amethyst.model.LocalCache -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.EventFinderQueryState -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.UserFinderFilterAssembler -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.user.UserFinderQueryState import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.IO @@ -41,7 +41,7 @@ import kotlinx.coroutines.IO * relay list arrives, [EventFinderFilterAssembler] is invalidated and can query the correct relay. */ class AddressableAuthorRelayLoaderSubAssembler( - val cache: LocalCache, + val cache: ICacheProvider, val allKeys: () -> Set, val userFinder: UserFinderFilterAssembler, ) : IEoseManager { @@ -69,7 +69,7 @@ class AddressableAuthorRelayLoaderSubAssembler( val note = key.note if (note is AddressableNote && note.event == null) { val author = cache.getOrCreateUser(note.address.pubKeyHex) - if (author.authorRelayList() == null) { + if (author != null && author.authorRelayList() == null) { needed.add(UserFinderQueryState(author, key.account)) } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/FilterMissingAddressables.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/FilterMissingAddressables.kt similarity index 79% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/FilterMissingAddressables.kt rename to commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/FilterMissingAddressables.kt index 41eac22796..f52e1a3242 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/FilterMissingAddressables.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/FilterMissingAddressables.kt @@ -18,33 +18,36 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.loaders +package com.vitorpamplona.amethyst.commons.relayClient.event.loaders +import com.vitorpamplona.amethyst.commons.model.AddressableNote +import com.vitorpamplona.amethyst.commons.model.cache.ICacheProvider +import com.vitorpamplona.amethyst.commons.relayClient.event.EventFinderQueryState import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.ExplainedFilter import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.SubPurpose -import com.vitorpamplona.amethyst.model.AddressableNote -import com.vitorpamplona.amethyst.model.LocalCache -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.EventFinderQueryState import com.vitorpamplona.quartz.nip01Core.core.Address import com.vitorpamplona.quartz.nip01Core.relay.client.pool.RelayBasedFilter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.utils.mapOfSet -fun potentialRelaysToFindAddress(note: AddressableNote): Set { +fun potentialRelaysToFindAddress( + cache: ICacheProvider, + note: AddressableNote, +): Set { val set = mutableSetOf() - LocalCache.getOrCreateUser(note.address.pubKeyHex).outboxRelays()?.let { + cache.getOrCreateUser(note.address.pubKeyHex)?.outboxRelays()?.let { set.addAll(it) } - set.addAll(LocalCache.relayHints.hintsForAddress(note.idHex)) + set.addAll(cache.relayHints.hintsForAddress(note.idHex)) - LocalCache.getAnyChannel(note)?.relays()?.let { set.addAll(it) } + cache.getAnyChannel(note)?.relays()?.let { set.addAll(it) } note.replyTo?.forEach { parentNote -> set.addAll(parentNote.relays) - LocalCache.getAnyChannel(parentNote)?.relays()?.let { set.addAll(it) } + cache.getAnyChannel(parentNote)?.relays()?.let { set.addAll(it) } parentNote.author?.inboxRelays()?.let { set.addAll(it) } } @@ -52,7 +55,7 @@ fun potentialRelaysToFindAddress(note: AddressableNote): Set note.replies.forEach { childNote -> set.addAll(childNote.relays) - LocalCache.getAnyChannel(childNote)?.relays()?.let { set.addAll(it) } + cache.getAnyChannel(childNote)?.relays()?.let { set.addAll(it) } childNote.author?.outboxRelays()?.let { set.addAll(it) } } @@ -72,13 +75,16 @@ fun potentialRelaysToFindAddress(note: AddressableNote): Set return set } -fun filterMissingAddressables(keys: List): List { +fun filterMissingAddressables( + cache: ICacheProvider, + keys: List, +): List { val addressesPerRelay = mapOfSet { keys.forEach { key -> - val default = key.account.followPlusAllMineWithSearch.flow.value + val default = key.account.followPlusAllMineWithSearchRelays() if (key.note is AddressableNote && key.note.event == null) { - potentialRelaysToFindAddress(key.note).ifEmpty { default }.forEach { relayUrl -> + potentialRelaysToFindAddress(cache, key.note).ifEmpty { default }.forEach { relayUrl -> add(relayUrl, key.note.address) } } @@ -86,7 +92,7 @@ fun filterMissingAddressables(keys: List): List if (note is AddressableNote && note.event == null) { - potentialRelaysToFindAddress(note).ifEmpty { default }.forEach { relayUrl -> + potentialRelaysToFindAddress(cache, note).ifEmpty { default }.forEach { relayUrl -> add(relayUrl, note.address) } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/FilterMissingEvents.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/FilterMissingEvents.kt similarity index 79% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/FilterMissingEvents.kt rename to commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/FilterMissingEvents.kt index f9f545fc00..65cac46200 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/FilterMissingEvents.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/FilterMissingEvents.kt @@ -18,33 +18,36 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.loaders +package com.vitorpamplona.amethyst.commons.relayClient.event.loaders +import com.vitorpamplona.amethyst.commons.model.AddressableNote import com.vitorpamplona.amethyst.commons.model.Channel +import com.vitorpamplona.amethyst.commons.model.Note +import com.vitorpamplona.amethyst.commons.model.cache.ICacheProvider +import com.vitorpamplona.amethyst.commons.relayClient.event.EventFinderQueryState import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.ExplainedFilter import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.SubPurpose -import com.vitorpamplona.amethyst.model.AddressableNote -import com.vitorpamplona.amethyst.model.LocalCache -import com.vitorpamplona.amethyst.model.Note -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.EventFinderQueryState import com.vitorpamplona.quartz.nip01Core.hints.PubKeyHintProvider import com.vitorpamplona.quartz.nip01Core.relay.client.pool.RelayBasedFilter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.utils.mapOfSet -fun potentialRelaysToFindEvent(note: Note): Set { +fun potentialRelaysToFindEvent( + cache: ICacheProvider, + note: Note, +): Set { val set = mutableSetOf() - set.addAll(LocalCache.relayHints.hintsForEvent(note.idHex)) + set.addAll(cache.relayHints.hintsForEvent(note.idHex)) note.author?.outboxRelays()?.let { set.addAll(it) } - LocalCache.getAnyChannel(note)?.relays()?.let { set.addAll(it) } + cache.getAnyChannel(note)?.relays()?.let { set.addAll(it) } note.replyTo?.forEach { parentNote -> set.addAll(parentNote.relays) - LocalCache.getAnyChannel(parentNote)?.relays()?.let { set.addAll(it) } + cache.getAnyChannel(parentNote)?.relays()?.let { set.addAll(it) } parentNote.author?.inboxRelays()?.let { set.addAll(it) } } @@ -52,7 +55,7 @@ fun potentialRelaysToFindEvent(note: Note): Set { note.replies.forEach { childNote -> set.addAll(childNote.relays) - LocalCache.getAnyChannel(childNote)?.relays()?.let { set.addAll(it) } + cache.getAnyChannel(childNote)?.relays()?.let { set.addAll(it) } childNote.author?.outboxRelays()?.let { set.addAll(it) } } @@ -81,7 +84,7 @@ fun potentialRelaysToFindEvent(note: Note): Set { val noteEvent = parent.event if (noteEvent is PubKeyHintProvider) { noteEvent.linkedPubKeys().forEach { potentialAuthor -> - LocalCache.checkGetOrCreateUser(potentialAuthor)?.let { potentialAuthor -> + cache.checkGetOrCreateUser(potentialAuthor)?.let { potentialAuthor -> potentialAuthor.outboxRelays()?.let { set.addAll(it) } potentialAuthor.inboxRelays()?.let { set.addAll(it) } } @@ -98,18 +101,21 @@ fun potentialRelaysToFindEvent(note: Note): Set { return set } -fun filterMissingEvents(keys: List): List { +fun filterMissingEvents( + cache: ICacheProvider, + keys: List, +): List { val eventsPerRelay = mapOfSet { keys.forEach { key -> - val default = key.account.followPlusAllMineWithSearch.flow.value + val default = key.account.followPlusAllMineWithSearchRelays() if (key.note !is AddressableNote && key.note.event == null) { - potentialRelaysToFindEvent(key.note).ifEmpty { default }.forEach { relayUrl -> + potentialRelaysToFindEvent(cache, key.note).ifEmpty { default }.forEach { relayUrl -> add(relayUrl, key.note.idHex) } - key.account.searchRelayList.flow.value.forEach { relayUrl -> + key.account.searchRelays().forEach { relayUrl -> add(relayUrl, key.note.idHex) } } @@ -117,7 +123,7 @@ fun filterMissingEvents(keys: List): List if (note !is AddressableNote && note.event == null) { - potentialRelaysToFindEvent(note).ifEmpty { default }.forEach { relayUrl -> + potentialRelaysToFindEvent(cache, note).ifEmpty { default }.forEach { relayUrl -> add(relayUrl, note.idHex) } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/NoteEventLoaderSubAssembler.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/NoteEventLoaderSubAssembler.kt similarity index 67% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/NoteEventLoaderSubAssembler.kt rename to commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/NoteEventLoaderSubAssembler.kt index 9ff15e6407..172ba4d101 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/loaders/NoteEventLoaderSubAssembler.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/NoteEventLoaderSubAssembler.kt @@ -18,21 +18,28 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.loaders +package com.vitorpamplona.amethyst.commons.relayClient.event.loaders -import com.vitorpamplona.amethyst.service.relayClient.eoseManagers.SingleSubNoEoseCacheEoseManager -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.EventFinderQueryState +import com.vitorpamplona.amethyst.commons.model.cache.ICacheProvider +import com.vitorpamplona.amethyst.commons.relayClient.eoseManagers.SingleSubNoEoseCacheEoseManager +import com.vitorpamplona.amethyst.commons.relayClient.event.EventFinderQueryState import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient class NoteEventLoaderSubAssembler( client: INostrClient, + val cache: ICacheProvider, allKeys: () -> Set, ) : SingleSubNoEoseCacheEoseManager(client, allKeys, invalidateAfterEose = true) { override fun updateFilter(keys: List) = listOfNotNull( - filterMissingEvents(keys), - filterMissingAddressables(keys), + filterMissingEvents(cache, keys), + filterMissingAddressables(cache, keys), ).flatten() override fun distinct(key: EventFinderQueryState) = key.note + + // Attribute to the account that owns this subscription, when a single account is watching. + // Deduped by pubkey hex, not by account identity, so two objects for the same logged-in user + // don't look like two accounts and suppress attribution. + override fun accountPubKeyOf(key: Any?): String? = (key as? EventFinderQueryState)?.account?.userFinderPubkeyHex } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/watchers/EventWatcherSubAssembler.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/watchers/EventWatcherSubAssembler.kt similarity index 89% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/watchers/EventWatcherSubAssembler.kt rename to commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/watchers/EventWatcherSubAssembler.kt index e58b596d2d..25a9bb3b5b 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/watchers/EventWatcherSubAssembler.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/watchers/EventWatcherSubAssembler.kt @@ -18,15 +18,15 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.watchers +package com.vitorpamplona.amethyst.commons.relayClient.event.watchers +import com.vitorpamplona.amethyst.commons.model.AddressableNote +import com.vitorpamplona.amethyst.commons.model.Note import com.vitorpamplona.amethyst.commons.relayClient.eoseManagers.SingleSubEoseManager -import com.vitorpamplona.amethyst.model.AddressableNote -import com.vitorpamplona.amethyst.model.Note -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.EventFinderQueryState -import com.vitorpamplona.amethyst.service.relays.EOSEAccountFast -import com.vitorpamplona.amethyst.service.relays.MutableTime -import com.vitorpamplona.amethyst.service.relays.SincePerRelayMap +import com.vitorpamplona.amethyst.commons.relayClient.event.EventFinderQueryState +import com.vitorpamplona.amethyst.commons.relays.EOSEAccountFast +import com.vitorpamplona.amethyst.commons.relays.MutableTime +import com.vitorpamplona.amethyst.commons.relays.SincePerRelayMap import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.pool.RelayBasedFilter import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter @@ -67,7 +67,7 @@ class EventWatcherSubAssembler( // the same logged-in user would look like two accounts and suppress attribution entirely. val soleAccountPubKey = keys - .mapTo(mutableSetOf()) { it.account.userProfile().pubkeyHex } + .mapTo(mutableSetOf()) { it.account.userFinderPubkeyHex } .singleOrNull() return groupByRelayPresence(lastNotesOnFilter, latestEOSEs) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/watchers/FilterRepliesAndReactionsToAddresses.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/watchers/FilterRepliesAndReactionsToAddresses.kt similarity index 96% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/watchers/FilterRepliesAndReactionsToAddresses.kt rename to commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/watchers/FilterRepliesAndReactionsToAddresses.kt index cce973f915..16e0f9f435 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/watchers/FilterRepliesAndReactionsToAddresses.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/watchers/FilterRepliesAndReactionsToAddresses.kt @@ -18,12 +18,12 @@ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.watchers +package com.vitorpamplona.amethyst.commons.relayClient.event.watchers +import com.vitorpamplona.amethyst.commons.model.AddressableNote import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.ExplainedFilter import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.SubPurpose -import com.vitorpamplona.amethyst.model.AddressableNote -import com.vitorpamplona.amethyst.service.relays.SincePerRelayMap +import com.vitorpamplona.amethyst.commons.relays.SincePerRelayMap import com.vitorpamplona.quartz.experimental.attestations.attestation.AttestationEvent import com.vitorpamplona.quartz.experimental.zapPolls.ZapPollEvent import com.vitorpamplona.quartz.nip01Core.core.HexKey diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/watchers/FilterRepliesAndReactionsToNotes.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/watchers/FilterRepliesAndReactionsToNotes.kt similarity index 96% rename from amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/watchers/FilterRepliesAndReactionsToNotes.kt rename to commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/watchers/FilterRepliesAndReactionsToNotes.kt index 64510793f8..f96a5b46fb 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/event/watchers/FilterRepliesAndReactionsToNotes.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/watchers/FilterRepliesAndReactionsToNotes.kt @@ -20,12 +20,12 @@ */ @file:Suppress("DEPRECATION") -package com.vitorpamplona.amethyst.service.relayClient.reqCommand.event.watchers +package com.vitorpamplona.amethyst.commons.relayClient.event.watchers +import com.vitorpamplona.amethyst.commons.model.Note import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.ExplainedFilter import com.vitorpamplona.amethyst.commons.relayClient.subscriptions.SubPurpose -import com.vitorpamplona.amethyst.model.Note -import com.vitorpamplona.amethyst.service.relays.SincePerRelayMap +import com.vitorpamplona.amethyst.commons.relays.SincePerRelayMap import com.vitorpamplona.quartz.experimental.attestations.attestation.AttestationEvent import com.vitorpamplona.quartz.experimental.edits.TextNoteModificationEvent import com.vitorpamplona.quartz.experimental.zapPolls.ZapPollEvent From a867f9b2ca4ae58c018099721fba4b9cec00e9c1 Mon Sep 17 00:00:00 2001 From: nrobi144 Date: Mon, 10 Aug 2026 10:59:37 +0300 Subject: [PATCH 117/132] feat: wire Desktop to the shared per-visible metadata + reaction finders (Phase 4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-apply the Desktop adoption on top of the commons finders (now on upstream's model): - DesktopIAccount implements UserFinderAccount (connected-relays/nip65 relay hints; trustProvider/followerCountProvider = null, declaredFollowsByOutboxRelay = emptyMap — Desktop has no NIP-85 subsystem). followerCountProvider() is the new getter this model needs. - DesktopRelaySubscriptionsCoordinator builds userFinder + eventFinder (RelayOfflineTracker). - Main.kt provides LocalUserFinder/LocalUserFinderAccount/LocalEventFinder at both logged-in composition roots. - Per-visible adoption across feed (FeedNoteCardBody), NoteCard (profile/thread/bookmarks/ search/quoted), DM headers + conversation list, UserSearchCard (observeUserInfo), NotificationsScreen, thread replies — plus the fast index-relay warm-up on the feed. Metadata + reactions now load strictly per on-screen user/note on Desktop, coalesced into batched REQs by the shared finders. Green: :desktopApp compile, :amethyst compilePlayDebugKotlin, :commons verifyKmpPurity, spotless. Co-Authored-By: Claude Opus 4.8 --- .../commons/ui/components/UserSearchCard.kt | 16 ++++- .../vitorpamplona/amethyst/desktop/Main.kt | 9 +++ .../amethyst/desktop/model/DesktopIAccount.kt | 35 ++++++++- .../DesktopRelaySubscriptionsCoordinator.kt | 29 ++++++++ .../amethyst/desktop/ui/FeedScreen.kt | 72 ++++++++++++------- .../desktop/ui/NotificationsScreen.kt | 32 +++++---- .../amethyst/desktop/ui/SearchScreen.kt | 19 ++--- .../amethyst/desktop/ui/ThreadScreen.kt | 11 ++- .../desktop/ui/chats/ChatroomHeader.kt | 21 ++++-- .../desktop/ui/chats/ConversationListPane.kt | 6 +- .../amethyst/desktop/ui/note/NoteCard.kt | 22 ++++++ 11 files changed, 207 insertions(+), 65 deletions(-) diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/ui/components/UserSearchCard.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/ui/components/UserSearchCard.kt index bd7e82f54f..4badbacef1 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/ui/components/UserSearchCard.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/ui/components/UserSearchCard.kt @@ -32,6 +32,7 @@ import androidx.compose.material3.CardDefaults import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.text.font.FontFamily @@ -39,6 +40,7 @@ import androidx.compose.ui.unit.dp import com.vitorpamplona.amethyst.commons.icons.symbols.Icon import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols import com.vitorpamplona.amethyst.commons.model.User +import com.vitorpamplona.amethyst.commons.relayClient.user.observeUserInfo import com.vitorpamplona.amethyst.commons.resources.Res import com.vitorpamplona.amethyst.commons.resources.accessibility_navigate import com.vitorpamplona.amethyst.commons.resources.accessibility_user_avatar @@ -51,6 +53,12 @@ import org.jetbrains.compose.resources.stringResource * @param badge Optional overlay drawn on top of the avatar (bottom-right * by convention). Used by Desktop for the WoT trust-score chip; Android * call sites leave it null. Forwarded to [UserAvatar]. + * + * Loads the user's kind-0 metadata only while the card is composed via + * [observeUserInfo], so a search-results list only fetches metadata for the + * users currently on screen (coalesced into the shared finder's batched REQs). + * Requires [LocalUserFinder]/[LocalUserFinderAccount] in scope — provided at + * the Desktop logged-in roots; this card is Desktop-only. */ @Composable fun UserSearchCard( @@ -59,6 +67,8 @@ fun UserSearchCard( modifier: Modifier = Modifier, badge: @Composable (BoxScope.() -> Unit)? = null, ) { + val metadata by observeUserInfo(user) + Card( modifier = modifier @@ -76,7 +86,7 @@ fun UserSearchCard( ) { UserAvatar( userHex = user.pubkeyHex, - pictureUrl = user.profilePicture(), + pictureUrl = metadata?.info?.picture ?: user.profilePicture(), size = 40.dp, contentDescription = stringResource(Res.string.accessibility_user_avatar), badge = badge, @@ -84,11 +94,11 @@ fun UserSearchCard( Column(modifier = Modifier.weight(1f)) { Text( - user.toBestDisplayName(), + metadata?.info?.bestName() ?: user.toBestDisplayName(), style = MaterialTheme.typography.titleSmall, color = MaterialTheme.colorScheme.onSurface, ) - val nip05 = user.metadataOrNull()?.nip05() + val nip05 = metadata?.info?.nip05 ?: user.metadataOrNull()?.nip05() if (nip05 != null) { Text( nip05, diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/Main.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/Main.kt index 3c5faabd88..704c5f1a2c 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/Main.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/Main.kt @@ -81,8 +81,11 @@ import com.vitorpamplona.amethyst.commons.moderation.PreferencesHashtagSpamSetti import com.vitorpamplona.amethyst.commons.moderation.notifications.PreferencesNotificationReadState import com.vitorpamplona.amethyst.commons.moderation.notifications.PreferencesNotificationSettings import com.vitorpamplona.amethyst.commons.relayClient.auth.AuthApprovalBanner +import com.vitorpamplona.amethyst.commons.relayClient.event.LocalEventFinder import com.vitorpamplona.amethyst.commons.relayClient.nip17Dm.DmInboxRelayResolver import com.vitorpamplona.amethyst.commons.relayClient.nip17Dm.unwrapAndUnsealOrNull +import com.vitorpamplona.amethyst.commons.relayClient.user.LocalUserFinder +import com.vitorpamplona.amethyst.commons.relayClient.user.LocalUserFinderAccount import com.vitorpamplona.amethyst.commons.scheduledposts.ScheduledPostStatus import com.vitorpamplona.amethyst.commons.wot.LocalWoTReady import com.vitorpamplona.amethyst.commons.wot.LocalWoTService @@ -1470,6 +1473,9 @@ private fun AppInner( LocalNamecoinService provides namecoinService, LocalSpamExemptKeys provides spamExemptKeys, com.vitorpamplona.amethyst.desktop.model.LocalDesktopIAccount provides iAccount, + LocalUserFinder provides subscriptionsCoordinator.userFinder, + LocalUserFinderAccount provides iAccount, + LocalEventFinder provides subscriptionsCoordinator.eventFinder, ) { val pendingAuthApprovals by authCoordinator.pendingApprovals.collectAsState() Column(modifier = Modifier.fillMaxSize()) { @@ -2114,6 +2120,9 @@ fun MainContent( LocalRelayCategories provides relayCategories, LocalBlossomServers provides iAccount.blossomServerList.flow, com.vitorpamplona.amethyst.desktop.model.LocalDesktopIAccount provides iAccount, + LocalUserFinder provides subscriptionsCoordinator.userFinder, + LocalUserFinderAccount provides iAccount, + LocalEventFinder provides subscriptionsCoordinator.eventFinder, com.vitorpamplona.amethyst.desktop.ui.LocalSnackbarHost provides snackbarHostState, com.vitorpamplona.amethyst.desktop.ui.relay.LocalAccountRelays provides accountRelays, com.vitorpamplona.amethyst.desktop.ui.deck.LocalDesktopCache provides localCache, diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/model/DesktopIAccount.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/model/DesktopIAccount.kt index 3d9b28bbb2..c045e74df5 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/model/DesktopIAccount.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/model/DesktopIAccount.kt @@ -35,6 +35,7 @@ import com.vitorpamplona.amethyst.commons.model.nipB7Blossom.BlossomServerListSt import com.vitorpamplona.amethyst.commons.model.privateChats.ChatroomList import com.vitorpamplona.amethyst.commons.moderation.PreferencesSensitiveContentSettings import com.vitorpamplona.amethyst.commons.relayClient.nip17Dm.DmInboxRelayResolver +import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderAccount import com.vitorpamplona.amethyst.desktop.account.AccountState import com.vitorpamplona.amethyst.desktop.cache.DesktopLocalCache import com.vitorpamplona.amethyst.desktop.network.RelayConnectionManager @@ -64,6 +65,7 @@ import com.vitorpamplona.quartz.nip57Zaps.IPrivateZapsDecryptionCache import com.vitorpamplona.quartz.nip57Zaps.PrivateZapCache import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent import com.vitorpamplona.quartz.nip65RelayList.AdvertisedRelayListEvent +import com.vitorpamplona.quartz.nip85TrustedAssertions.list.tags.ServiceProviderTag import com.vitorpamplona.quartz.nip89AppHandlers.clientTag.NostrSignerWithClientTag import com.vitorpamplona.quartz.utils.DualCase import kotlinx.coroutines.CoroutineScope @@ -92,11 +94,42 @@ class DesktopIAccount( private val scope: CoroutineScope, private val accountRelays: DesktopAccountRelays? = null, val dmInboxResolver: DmInboxRelayResolver? = null, -) : IAccount { +) : IAccount, + UserFinderAccount { override val signer: NostrSigner = NostrSignerWithClientTag(accountState.signer, CLIENT_TAG_NAME) override val pubKey: String = accountState.pubKeyHex + // UserFinderAccount — Desktop's relay-hint view for the shared per-user + // metadata subscription layer. Desktop has no separate indexer/search relay + // lists nor a NIP-85 trust provider, so it routes discovery through its + // connected relays + NIP-65 outbox and degrades trust/reports to null/empty + // (contact-card ranking + report loading are best-effort here — see + // UserFinderAccount). + override val userFinderPubkeyHex: HexKey get() = pubKey + + override fun indexRelays(): Set = relayManager.connectedRelays.value + + override fun outboxHomeRelays(): Set = nip65RelayList.allFlowNoDefaults.value + relayManager.connectedRelays.value + + override fun searchRelays(): Set = relayManager.connectedRelays.value + + // Desktop has no merged follow/mine/search relay-list subsystem; route + // missing-event discovery through the connected relays (same degrade path + // as the other hints above). + override fun followPlusAllMineWithSearchRelays(): Set = relayManager.connectedRelays.value + + override fun commonRelays(): Set = relayManager.connectedRelays.value + + override fun cardHomeRelays(): Set = nip65RelayList.allFlowNoDefaults.value + + override fun trustProvider(): ServiceProviderTag? = null + + // Desktop has no NIP-85 rank/follower providers wired (same as trustProvider). + override fun followerCountProvider(): ServiceProviderTag? = null + + override fun declaredFollowsByOutboxRelay(): Map> = emptyMap() + // ----- State Classes (pin important notes via strong refs for GC retention) ----- val oldBookmarkState = OldBookmarkListState(signer, localCache, scope) diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/DesktopRelaySubscriptionsCoordinator.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/DesktopRelaySubscriptionsCoordinator.kt index 3a315d6507..78c1cffa71 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/DesktopRelaySubscriptionsCoordinator.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/subscriptions/DesktopRelaySubscriptionsCoordinator.kt @@ -22,8 +22,10 @@ package com.vitorpamplona.amethyst.desktop.subscriptions import com.vitorpamplona.amethyst.commons.model.Note import com.vitorpamplona.amethyst.commons.relayClient.assemblers.FeedMetadataCoordinator +import com.vitorpamplona.amethyst.commons.relayClient.event.EventFinderFilterAssembler import com.vitorpamplona.amethyst.commons.relayClient.preload.MetadataPreloader import com.vitorpamplona.amethyst.commons.relayClient.preload.MetadataRateLimiter +import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderFilterAssembler import com.vitorpamplona.amethyst.commons.service.BasicBundledInsert import com.vitorpamplona.amethyst.commons.wot.OutboxCacheGateway import com.vitorpamplona.amethyst.commons.wot.OutboxDispatcher @@ -32,6 +34,7 @@ import com.vitorpamplona.amethyst.desktop.model.DesktopDmRelayState import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.RelayOfflineTracker import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.fetchAll import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter @@ -83,6 +86,32 @@ class DesktopRelaySubscriptionsCoordinator( private val indexRelays: Set, private val localCache: DesktopLocalCache, ) { + /** + * Tracks relays that refuse to connect, so the shared user-finder skips + * them when picking discovery relays. Self-registers as a client listener. + */ + private val failureTracker = RelayOfflineTracker(client) + + /** + * The shared, composition-scoped per-user metadata subscription assembler + * (moved to commons). Desktop composables reach it via [LocalUserFinder] + * (provided in Main.kt) and subscribe per visible user through + * `observeUserPicture(user)` / `observeUserInfo(user)`, so metadata loads + * only for on-screen users. Coalesces every subscribed user into batched + * REQs — no per-avatar REQ storm. + */ + val userFinder = UserFinderFilterAssembler(client, localCache, failureTracker) + + /** + * The shared, composition-scoped per-note event subscription assembler + * (reactions / zaps / reposts / replies, moved to commons). Desktop note + * rows reach it via [LocalEventFinder] (provided in Main.kt) and subscribe + * per visible note through `EventFinderFilterAssemblerSubscription(note)`, so + * interactions load only for on-screen notes. Composes [userFinder] to + * resolve authors of not-yet-cached addressable notes. + */ + val eventFinder = EventFinderFilterAssembler(client, localCache, userFinder) + // Rate limiter: 20 requests per second to avoid flooding relays private val rateLimiter = MetadataRateLimiter(maxRequestsPerSecond = 20, scope = scope) diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/FeedScreen.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/FeedScreen.kt index 16696692b5..3c77deba9c 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/FeedScreen.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/FeedScreen.kt @@ -95,6 +95,10 @@ import com.vitorpamplona.amethyst.commons.model.nip02FollowList.FollowAction import com.vitorpamplona.amethyst.commons.model.nip05DnsIdentifiers.namecoin.NamecoinResolveState import com.vitorpamplona.amethyst.commons.model.nip25Reactions.ReactionAction import com.vitorpamplona.amethyst.commons.nip64Chess.RelaySyncStatus +import com.vitorpamplona.amethyst.commons.relayClient.event.EventFinderFilterAssemblerSubscription +import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderFilterAssemblerSubscription +import com.vitorpamplona.amethyst.commons.relayClient.user.observeUserName +import com.vitorpamplona.amethyst.commons.relayClient.user.observeUserPicture import com.vitorpamplona.amethyst.commons.richtext.UrlParser import com.vitorpamplona.amethyst.commons.search.AdvancedSearchBarState import com.vitorpamplona.amethyst.commons.search.QuerySerializer @@ -268,6 +272,20 @@ private fun FeedNoteCardBody( myPubKeyHex: String? = null, onFollow: ((String) -> Unit)? = null, ) { + // Load this note author's metadata (kind 0 + relay lists) only while this + // card is composed — i.e. on or near screen. The shared commons finder + // coalesces every visible author into batched REQs, giving per-row, + // visibility-scoped metadata loading that matches Android's model. + val cardAuthor = note.author + if (cardAuthor != null) { + UserFinderFilterAssemblerSubscription(cardAuthor) + } + + // Load this note's interactions (reactions / zaps / reposts / replies) only + // while the card is composed — the per-note counterpart to the author + // subscription above, coalesced into batched REQs by the shared event finder. + EventFinderFilterAssemblerSubscription(note) + if (event is PollEvent) { DesktopPollCard( note = note, @@ -751,17 +769,19 @@ fun FeedScreen( } } - // Viewport-aware metadata loading: only fetch for visible notes + buffer - // Uses snapshotFlow to avoid per-frame recomposition from scroll observation + // Fast first-paint warm-up: one batched kind-0 REQ straight to the index + // relays for the first visible authors. The per-row UserFinder subscriptions + // (in FeedNoteCardBody) are the source of truth — they do NIP-65 outbox + // routing and tear down off-screen — but their two-hop discovery is slower to + // first paint, so this immediate batch fills names/avatars instantly. Also + // prefetches reactions + referenced (repost/quote) notes for the initial set. LaunchedEffect(feedState, subscriptionsCoordinator) { if (subscriptionsCoordinator == null || feedState !is FeedState.Loaded) return@LaunchedEffect - val loadedFeed = feedState as FeedState.Loaded - // Initial load: batch metadata for first visible notes immediately val initialNotes = viewModel.feedState.visibleNotes().take(30) if (initialNotes.isNotEmpty()) { val authors = initialNotes.mapNotNull { it.author?.pubkeyHex }.distinct() - subscriptionsCoordinator.loadMetadataBatched(authors) + if (authors.isNotEmpty()) subscriptionsCoordinator.loadMetadataBatched(authors) subscriptionsCoordinator.loadMetadataForNotes(initialNotes) } } @@ -988,7 +1008,11 @@ fun FeedScreen( val loadedState by state.feed.collectAsState() val lazyListState = homeFeedLazyListState - // Viewport-aware scroll observation: fetch metadata for newly visible notes + // Fast-path warm-up on scroll: batch a kind-0 REQ to index + // relays for authors entering the viewport (+10 buffer), so + // names/avatars paint immediately. Complementary to the per-row + // FeedNoteCardBody subscriptions, which remain the source of + // truth (outbox routing + off-screen teardown). LaunchedEffect(lazyListState, loadedState) { if (subscriptionsCoordinator == null) return@LaunchedEffect val feedList = loadedState.list @@ -999,7 +1023,7 @@ fun FeedScreen( if (info.visibleItemsInfo.isEmpty()) return@snapshotFlow -1 to -1 info.visibleItemsInfo.first().index to info.visibleItemsInfo.last().index }.distinctUntilChanged() - .debounce(500) + .debounce(300) .collect { (first, last) -> if (first < 0) return@collect val from = (first - 10).coerceAtLeast(0) @@ -1974,15 +1998,9 @@ private fun ExpandedNoteContent( // Get reply notes from cache — recompute when replies change val replyNotes = remember(repliesState) { note.replies.sortedByDescending { it.createdAt() } } - // Load metadata for reply authors - LaunchedEffect(replyNotes, subscriptionsCoordinator) { - if (subscriptionsCoordinator != null && replyNotes.isNotEmpty()) { - val authors = replyNotes.mapNotNull { it.event?.pubKey }.distinct() - if (authors.isNotEmpty()) { - subscriptionsCoordinator.loadMetadataBatched(authors) - } - } - } + // Reply-author metadata (kind 0) is loaded per-row: each CommentItem below + // opens its own composition-scoped observeUser* subscription, so metadata + // loads for on-screen replies only and tears down when the thread closes. Column(modifier = Modifier.padding(top = 8.dp)) { // Comments card @@ -2022,24 +2040,30 @@ private fun ExpandedNoteContent( replyNotes.take(5).forEachIndexed { index, replyNote -> val replyEvent = replyNote.event val flowSet = remember(replyNote) { replyNote.flow() } - val metadataState by flowSet.metadata.stateFlow.collectAsState() val reactionsState by flowSet.reactions.stateFlow.collectAsState() val zapsState by flowSet.zaps.stateFlow.collectAsState() DisposableEffect(replyNote) { onDispose { replyNote.clearFlow() } } - val author = - remember(replyEvent?.pubKey, metadataState) { - replyEvent?.pubKey?.let { localCache.getUserIfExists(it) } - } + // Load this reply's own interactions (reactions/zaps) only + // while the comment row is composed. + EventFinderFilterAssemblerSubscription(replyNote) + + // Load + observe this reply author's metadata only while the + // comment row is composed; observeUser* both subscribes and + // drives recomposition when kind-0 arrives. + val replyAuthorPubKey = replyEvent?.pubKey + val author = remember(replyAuthorPubKey, localCache) { replyAuthorPubKey?.let { localCache.getOrCreateUser(it) } } + val authorName = author?.let { observeUserName(it).value } + val authorPicture = author?.let { observeUserPicture(it).value } val reactionCount = remember(reactionsState) { replyNote.countReactions() } val zapAmount = remember(zapsState) { replyNote.zapsAmount } CommentItem( - authorName = author?.toBestDisplayName() ?: replyEvent?.pubKey?.take(8) ?: "", + authorName = authorName ?: replyAuthorPubKey?.take(8) ?: "", authorHandle = author?.pubkeyNpub()?.take(16)?.let { "@$it..." } ?: "", - authorAvatarUrl = author?.profilePicture(), - authorPubKeyHex = replyEvent?.pubKey ?: "", + authorAvatarUrl = authorPicture, + authorPubKeyHex = replyAuthorPubKey ?: "", content = replyEvent?.content ?: "", timeAgo = (replyEvent?.createdAt ?: 0L).toTimeAgo(), reactionCount = reactionCount, diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/NotificationsScreen.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/NotificationsScreen.kt index be0adf4ad5..4bdb8f1083 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/NotificationsScreen.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/NotificationsScreen.kt @@ -67,6 +67,8 @@ import com.vitorpamplona.amethyst.commons.moderation.notifications.NotificationK import com.vitorpamplona.amethyst.commons.moderation.notifications.PreferencesNotificationReadState import com.vitorpamplona.amethyst.commons.moderation.notifications.PreferencesNotificationSettings import com.vitorpamplona.amethyst.commons.moderation.notifications.nowEpochSeconds +import com.vitorpamplona.amethyst.commons.relayClient.user.observeUserName +import com.vitorpamplona.amethyst.commons.relayClient.user.observeUserPicture import com.vitorpamplona.amethyst.commons.state.EventCollectionState import com.vitorpamplona.amethyst.commons.ui.components.EmptyState import com.vitorpamplona.amethyst.commons.ui.components.LoadingState @@ -654,10 +656,11 @@ private fun AggregateCard( horizontalArrangement = Arrangement.spacedBy((-4).dp), ) { reactorPubKeys.take(5).forEach { pk -> - val user = remember(pk, metadataVersion) { localCache.getUserIfExists(pk) } + val user = remember(pk, localCache) { localCache.getOrCreateUser(pk) } + val picture by observeUserPicture(user) UserAvatar( userHex = pk, - pictureUrl = user?.profilePicture(), + pictureUrl = picture, size = 20.dp, modifier = Modifier.clickable { onNavigateToProfile(pk) }, ) @@ -694,15 +697,17 @@ private fun AggregateCard( .clickable { onNavigateToProfile(pk) } .padding(vertical = 3.dp), ) { - val user = remember(pk, metadataVersion) { localCache.getUserIfExists(pk) } + val user = remember(pk, localCache) { localCache.getOrCreateUser(pk) } + val picture by observeUserPicture(user) + val name by observeUserName(user) UserAvatar( userHex = pk, - pictureUrl = user?.profilePicture(), + pictureUrl = picture, size = 22.dp, ) Spacer(Modifier.size(6.dp)) Text( - user?.toBestDisplayName() ?: pk.take(12), + name, style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.onSurface, ) @@ -821,14 +826,17 @@ fun NotificationCard( // actual zap sender lives in the nested zap request. effectiveAuthorPubKey // returns the right one per kind. val pk = notification.effectiveAuthorPubKey - val user = remember(pk, metadataVersion, localCache) { localCache?.getUserIfExists(pk) } + // Load + observe the actor's metadata only while this card is composed. + // localCache is only null in previews/defaults; guard the observers so we + // never touch LocalUserFinder off the provider tree. + val user = remember(pk, localCache) { localCache?.getOrCreateUser(pk) } + val observedName = user?.let { observeUserName(it).value } + val observedPicture = user?.let { observeUserPicture(it).value } val displayName = - remember(user, metadataVersion, pk) { - user?.toBestDisplayName() - ?: pk.hexToByteArrayOrNull()?.toNpub()?.take(12) - ?: pk.take(12) - } - val pictureUrl = remember(user, metadataVersion) { user?.profilePicture() } + observedName + ?: pk.hexToByteArrayOrNull()?.toNpub()?.take(12) + ?: pk.take(12) + val pictureUrl = observedPicture val unread by remember(notification.timestamp, lastReadAt) { derivedStateOf { notification.timestamp > lastReadAt } diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/SearchScreen.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/SearchScreen.kt index ba976472ea..cbddc13117 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/SearchScreen.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/SearchScreen.kt @@ -349,19 +349,12 @@ fun SearchScreen( } } - // Load author metadata for incoming note results. NIP-50 search relays - // typically don't return kind-0 metadata alongside notes, and the - // subscription explicitly drops MetadataEvents anyway — so display name - // and avatar arrive only after we explicitly fetch them from index - // relays via the coordinator. - LaunchedEffect(noteResults, subscriptionsCoordinator) { - val coordinator = subscriptionsCoordinator ?: return@LaunchedEffect - if (noteResults.isEmpty()) return@LaunchedEffect - val authors = noteResults.map { it.pubKey }.distinct() - if (authors.isNotEmpty()) { - coordinator.loadMetadataBatched(authors) - } - } + // Note-result author metadata (kind 0) is no longer batch-fetched here. + // Each result renders through NoteCard, which opens its own composition-scoped + // UserFinderFilterAssembler subscription — so the author's metadata is fetched + // from index/outbox relays per on-screen result and torn down when scrolled off. + // (NIP-50 search relays don't return kind-0 and the subscription drops + // MetadataEvents; the per-row finder covers that gap.) // Fetch interactions (incl. kind-1018 poll responses) for poll results so their // tallies populate — NIP-50 search returns the polls but not their responses. diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/ThreadScreen.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/ThreadScreen.kt index ecbd160bc1..dad4160970 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/ThreadScreen.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/ThreadScreen.kt @@ -39,7 +39,6 @@ import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Text import androidx.compose.runtime.Composable import androidx.compose.runtime.DisposableEffect -import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf @@ -194,12 +193,10 @@ fun ThreadScreen( onDispose { subId?.let { coordinator.releaseInteractions(it) } } } - // Load metadata for thread authors via coordinator - LaunchedEffect(threadNotes, subscriptionsCoordinator) { - if (subscriptionsCoordinator != null && threadNotes.isNotEmpty()) { - subscriptionsCoordinator.loadMetadataForNotes(threadNotes) - } - } + // Thread-author metadata + note interactions now load per row: each thread + // note renders through NoteCard, which opens composition-scoped UserFinder + + // EventFinder subscriptions. (requestInteractions above remains the thread's + // explicit interaction-refresh path.) // Fetch quoted notes referenced in thread content val quotedNoteIds = diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/chats/ChatroomHeader.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/chats/ChatroomHeader.kt index 96a771e6d3..3a3b724d41 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/chats/ChatroomHeader.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/chats/ChatroomHeader.kt @@ -29,12 +29,15 @@ import androidx.compose.foundation.layout.padding import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Text import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp import com.vitorpamplona.amethyst.commons.model.User +import com.vitorpamplona.amethyst.commons.relayClient.user.observeUserName +import com.vitorpamplona.amethyst.commons.relayClient.user.observeUserPicture import com.vitorpamplona.amethyst.commons.resources.Res import com.vitorpamplona.amethyst.commons.resources.accessibility_user_avatar import com.vitorpamplona.amethyst.commons.ui.components.UserAvatar @@ -59,6 +62,10 @@ fun ChatroomHeader( modifier: Modifier = ChatStdPadding, onClick: () -> Unit, ) { + // Load + observe the partner's metadata only while this header is composed. + val picture by observeUserPicture(user) + val name by observeUserName(user) + Column( Modifier .fillMaxWidth() @@ -68,14 +75,14 @@ fun ChatroomHeader( Row(verticalAlignment = Alignment.CenterVertically) { UserAvatar( userHex = user.pubkeyHex, - pictureUrl = user.profilePicture(), + pictureUrl = picture, size = ChatSize34dp, contentDescription = stringResource(Res.string.accessibility_user_avatar), ) Column(modifier = Modifier.padding(start = 10.dp)) { Text( - text = user.toBestDisplayName(), + text = name, style = MaterialTheme.typography.titleSmall, fontWeight = FontWeight.Bold, maxLines = 1, @@ -107,7 +114,12 @@ fun GroupChatroomHeader( modifier: Modifier = ChatStdPadding, onClick: () -> Unit, ) { - val participants = users.joinToString(", ") { it.toBestDisplayName() } + // Load + observe each participant's metadata only while this header is + // composed, so names and the group-icon avatar update as kind-0 arrives. + // forEach is inline, so the @Composable observeUserName call is legal here. + val participantNames = mutableListOf() + users.forEach { participantNames.add(observeUserName(it).value) } + val participants = participantNames.joinToString(", ") Column( modifier = Modifier @@ -121,9 +133,10 @@ fun GroupChatroomHeader( Row(verticalAlignment = Alignment.CenterVertically) { // Show first user's avatar as the group icon users.firstOrNull()?.let { firstUser -> + val firstUserPicture by observeUserPicture(firstUser) UserAvatar( userHex = firstUser.pubkeyHex, - pictureUrl = firstUser.profilePicture(), + pictureUrl = firstUserPicture, size = ChatSize34dp, contentDescription = stringResource(Res.string.accessibility_user_avatar), ) diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/chats/ConversationListPane.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/chats/ConversationListPane.kt index 337b738e8b..3ada46ff4d 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/chats/ConversationListPane.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/chats/ConversationListPane.kt @@ -66,6 +66,7 @@ import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp import com.vitorpamplona.amethyst.commons.icons.symbols.Icon import com.vitorpamplona.amethyst.commons.icons.symbols.MaterialSymbols +import com.vitorpamplona.amethyst.commons.relayClient.user.observeUserPicture import com.vitorpamplona.amethyst.commons.ui.components.UserAvatar import com.vitorpamplona.amethyst.desktop.ui.components.ToggleableTimeAgoText import com.vitorpamplona.quartz.nip17Dm.base.ChatroomKey @@ -320,9 +321,12 @@ private fun ConversationCard( val firstUser = item.users.firstOrNull() Box { if (firstUser != null) { + // Load + observe the conversation's primary user only while this + // row is composed (i.e. on screen in the list). + val firstUserPicture by observeUserPicture(firstUser) UserAvatar( userHex = firstUser.pubkeyHex, - pictureUrl = firstUser.profilePicture(), + pictureUrl = firstUserPicture, size = 40.dp, ) } else if (item.isGroup) { diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/note/NoteCard.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/note/NoteCard.kt index adf0b732ca..27d6ae795a 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/note/NoteCard.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/ui/note/NoteCard.kt @@ -56,6 +56,8 @@ import androidx.compose.ui.unit.dp import coil3.compose.AsyncImage import com.vitorpamplona.amethyst.commons.model.EmptyTagList import com.vitorpamplona.amethyst.commons.model.ImmutableListOfLists +import com.vitorpamplona.amethyst.commons.relayClient.event.EventFinderFilterAssemblerSubscription +import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderFilterAssemblerSubscription import com.vitorpamplona.amethyst.commons.richtext.CachedRichTextParser import com.vitorpamplona.amethyst.commons.richtext.RichTextParser import com.vitorpamplona.amethyst.commons.richtext.UrlParser @@ -63,6 +65,7 @@ import com.vitorpamplona.amethyst.commons.ui.note.ReplyContext import com.vitorpamplona.amethyst.commons.ui.note.ReplyToLabel import com.vitorpamplona.amethyst.desktop.cache.DesktopLocalCache import com.vitorpamplona.amethyst.desktop.ui.components.ToggleableTimeAgoText +import com.vitorpamplona.amethyst.desktop.ui.deck.LocalDesktopCache import com.vitorpamplona.amethyst.desktop.ui.media.AnimatedGifImage import com.vitorpamplona.amethyst.desktop.ui.media.AudioPlayer import com.vitorpamplona.amethyst.desktop.ui.media.DesktopVideoPlayer @@ -112,6 +115,25 @@ fun NoteCard( replyContext: ReplyContext? = null, onNavigateToThread: ((String) -> Unit)? = null, ) { + // Load the author's metadata (kind 0) only while this card is composed — + // i.e. on/near screen. This one call covers every screen that renders + // through NoteCard (profile, thread, bookmarks, search); the shared commons + // finder coalesces all visible authors into batched REQs. + val noteCardCache = LocalDesktopCache.current + val noteCardAuthor = remember(note.pubKeyHex, noteCardCache) { noteCardCache?.getOrCreateUser(note.pubKeyHex) } + if (noteCardAuthor != null) { + UserFinderFilterAssemblerSubscription(noteCardAuthor) + } + + // Load this note's interactions (reactions / zaps / reposts / replies) only + // while the card is composed — covers every NoteCard surface (profile, thread, + // bookmarks, search, quoted embeds). Guarded on cache presence so previews + // (no LocalDesktopCache → no LocalEventFinder) don't hit the provider default. + val noteCardNote = remember(note.id, noteCardCache) { noteCardCache?.getNoteIfExists(note.id) } + if (noteCardNote != null) { + EventFinderFilterAssemblerSubscription(noteCardNote) + } + val urls = remember(note.content) { UrlParser().parseValidUrls(note.content) } val imageUrls = remember(urls) { From fbe163e8f28db1ab40e4ec22ed6ecccdfb7595f7 Mon Sep 17 00:00:00 2001 From: nrobi144 Date: Mon, 10 Aug 2026 11:00:52 +0300 Subject: [PATCH 118/132] test+docs: commons finder test + document per-visible loading (Phase 5) - EventFinderFilterAssemblyTest (commons jvmTest): filterMissingEvents batches one ids-filter per relay with sorted ids; ICacheProvider.checkGetOrCreateUser default tolerates a throwing/null getOrCreateUser via a fake cache. - relay-client skill + commons/ARCHITECTURE.md: document observeUser*/ EventFinderFilterAssemblerSubscription/observeNote* as the canonical per-visible loading entry points and the LocalUserFinder/LocalUserFinderAccount/LocalEventFinder seams. Co-Authored-By: Claude Opus 4.8 --- .claude/skills/relay-client/SKILL.md | 26 ++++ commons/ARCHITECTURE.md | 2 +- .../event/EventFinderFilterAssemblyTest.kt | 119 ++++++++++++++++++ 3 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/EventFinderFilterAssemblyTest.kt diff --git a/.claude/skills/relay-client/SKILL.md b/.claude/skills/relay-client/SKILL.md index c1019cde8f..1b72fb6977 100644 --- a/.claude/skills/relay-client/SKILL.md +++ b/.claude/skills/relay-client/SKILL.md @@ -94,6 +94,32 @@ class MetadataFilterAssembler( Assemblers stay pure — no state, no I/O. They're the composition seam: `FeedMetadataCoordinator` takes a list of visible notes and assembles a single metadata filter covering every referenced pubkey. +## Per-visible loading — the canonical entry points (`observeUser*` / `observeNote*`) + +Prefer these over hand-rolled "load metadata for this list" calls. They are the shared, +KMP way to load data **only for what's on screen** — a composable subscribes while it is in +composition and unsubscribes ~30s after it leaves (or the app backgrounds). Both live in +`commons/relayClient/`: + +- **Per user** (`relayClient/user/`): `observeUserInfo/Picture/Banner/AboutMe/Name(user)` + each open a composition-scoped `UserFinderFilterAssemblerSubscription(user)` **and** return + reactive `State`. Metadata (kind 0 + relay lists) loads for on-screen users only, coalesced + into one batched REQ per relay for the whole visible set. +- **Per note** (`relayClient/event/`): `EventFinderFilterAssemblerSubscription(note)` loads a + note's interactions (reactions / zaps / reposts / replies) while it is composed. Android's + `observeNote*` display observers layer on top of the same subscription. + +Both read front-end-provided CompositionLocals — `LocalUserFinder` / `LocalUserFinderAccount` +(reused by the event finder) / `LocalEventFinder` — provided once near the composition root +(Android `AppModules`, Desktop `Main.kt` via its subscriptions coordinator). The account seam +is the narrow `UserFinderAccount` (snapshot relay-hint getters), NOT the fat `IAccount`. +`error()` defaults mean these must never be reached from a composition without a relay client +(e.g. the Android `:napplet` sandbox). + +The load-once, viewport-batch path (`FeedMetadataCoordinator.loadMetadataForNotes` / +`loadMetadataBatched`) is superseded for foreground loading; `MetadataPreloader` remains only +as an optional off-screen background warmer. + ## Preloaders `MetadataPreloader` is the "I need metadata for 200 pubkeys, but don't melt my CPU or the relay" path. It uses `MetadataRateLimiter` (token bucket) to throttle bulk fetches and group them into relay-friendly chunks. diff --git a/commons/ARCHITECTURE.md b/commons/ARCHITECTURE.md index b86c122c6f..25c594b532 100644 --- a/commons/ARCHITECTURE.md +++ b/commons/ARCHITECTURE.md @@ -101,7 +101,7 @@ they are shared across the GUI apps. Treat as GUI-shared, not strictly headless. ### Relay client | Package | UI? | Purpose | |----------------|-----|---------| -| `relayClient` | no | Compose-scoped subscription managers, filter assemblers, EOSE managers, preloaders. (Despite a `composeSubscriptionManagers` subpackage name, this is subscription-lifecycle logic, not UI.) | +| `relayClient` | no | Compose-scoped subscription managers, filter assemblers, EOSE managers, preloaders. (Despite a `composeSubscriptionManagers` subpackage name, this is subscription-lifecycle logic, not UI.) The canonical **per-visible loading** entry points live here: `relayClient/user/` (`observeUser*` — kind-0 metadata) and `relayClient/event/` (`EventFinderFilterAssemblerSubscription`/`observeNote*` — reactions/zaps/reposts). See the `relay-client` skill. | | `relays` | no | Low-level EOSE/relay-timing bookkeeping (`EOSECache`, `EOSERelayList`). | ### Platform abstractions (`expect`/`actual`) diff --git a/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/EventFinderFilterAssemblyTest.kt b/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/EventFinderFilterAssemblyTest.kt new file mode 100644 index 0000000000..034565a5d5 --- /dev/null +++ b/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/EventFinderFilterAssemblyTest.kt @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.relayClient.event + +import com.vitorpamplona.amethyst.commons.model.Channel +import com.vitorpamplona.amethyst.commons.model.Note +import com.vitorpamplona.amethyst.commons.model.User +import com.vitorpamplona.amethyst.commons.model.cache.ICacheEventStream +import com.vitorpamplona.amethyst.commons.model.cache.ICacheProvider +import com.vitorpamplona.amethyst.commons.relayClient.event.loaders.filterMissingEvents +import com.vitorpamplona.quartz.nip01Core.core.Address +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.hints.HintIndexer +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * Unit tests for the extracted (Phase 2b) per-note event-finder filter assembly + * and the [ICacheProvider] seam it relies on. + */ +class EventFinderFilterAssemblyTest { + private val relay1 = NormalizedRelayUrl("wss://relay1.test/") + private val relay2 = NormalizedRelayUrl("wss://relay2.test/") + + /** + * One batched `ids` filter per relay — NOT one filter per event id — and the + * ids are sorted deterministically so REQs dedup across rebuilds. + */ + @Test + fun `filterMissingEvents batches one filter per relay with sorted ids`() { + val filters = + filterMissingEvents( + mapOf( + relay1 to setOf("bbbb", "aaaa", "cccc"), + relay2 to setOf("dddd"), + ), + ) + + assertEquals("one batched filter per relay", 2, filters.size) + + val r1 = filters.first { it.relay == relay1 } + assertEquals(listOf("aaaa", "bbbb", "cccc"), r1.filter.ids) + + val r2 = filters.first { it.relay == relay2 } + assertEquals(listOf("dddd"), r2.filter.ids) + } + + @Test + fun `filterMissingEvents skips relays with no ids and empty input`() { + assertTrue(filterMissingEvents(emptyMap()).isEmpty()) + assertTrue(filterMissingEvents(mapOf(relay1 to emptySet())).isEmpty()) + } + + /** + * The new default [ICacheProvider.checkGetOrCreateUser] must swallow a + * malformed-key throw and return null (the event-finder follows pubkey hints + * parsed out of arbitrary events, some of which are junk). + */ + @Test + fun `checkGetOrCreateUser tolerates a throwing getOrCreateUser`() { + val throwing = + object : StubCache() { + override fun getOrCreateUser(pubkey: HexKey): User? = throw IllegalArgumentException("bad key") + } + assertNull(throwing.checkGetOrCreateUser("not-a-key")) + } + + @Test + fun `checkGetOrCreateUser passes through a null result`() { + assertNull(StubCache().checkGetOrCreateUser("00")) + } + + /** Minimal [ICacheProvider] that returns nothing; override per test. */ + private open class StubCache : ICacheProvider { + override val relayHints = HintIndexer() + + override fun getAnyChannel(note: Note): Channel? = null + + override fun getUserIfExists(pubkey: HexKey): User? = null + + override fun countUsers(predicate: (String, User) -> Boolean): Int = 0 + + override fun getNoteIfExists(hexKey: HexKey): Note? = null + + override fun checkGetOrCreateNote(hexKey: HexKey): Note? = null + + override fun getOrCreateAddressableNote(key: Address): com.vitorpamplona.amethyst.commons.model.AddressableNote = error("unused") + + override fun getEventStream(): ICacheEventStream = error("unused") + + override fun hasBeenDeleted(event: Any): Boolean = false + + override fun getOrCreateUser(pubkey: HexKey): User? = null + + override fun justConsumeMyOwnEvent(event: Event): Boolean = false + } +} From eaba40f2a6f320768906fa8536454549d82fbda9 Mon Sep 17 00:00:00 2001 From: nrobi144 Date: Mon, 10 Aug 2026 14:51:19 +0300 Subject: [PATCH 119/132] =?UTF-8?q?fix:=20address=20PR=20review=20?= =?UTF-8?q?=E2=80=94=20narrow=20search-relay=20seam,=20Local=20providers?= =?UTF-8?q?=20on=20Android,=20bug=20+=20test=20fixes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings from @davotoula: 1. Behaviour drift (real): the per-note event finder reused Account.searchRelays() (trusted + own search list), widening every missing-event REQ to trusted relays. Add a narrow UserFinderAccount.searchOnlyRelays() (= the pre-extraction account.searchRelayList read) and use it in FilterMissingEvents. Implemented on Account and DesktopIAccount. 2. Runtime trap: LocalUserFinder/LocalUserFinderAccount/LocalEventFinder error() when unprovided and were only provided on Desktop. Provide them on Android too, at the logged-in root (AppNavigation) from accountViewModel.dataSources() + .account, so any shared composable using the no-arg observeUser*/EventFinderFilterAssemblerSubscription overloads is safe on Android (the :napplet process never renders these). 3. Removed the redundant `.ifEmpty { DefaultSearchRelayList }` in Account.searchRelays() (SearchRelayListState.flow already applies that fallback) + its now-unused import. 4. Fixed a pre-existing shadowing bug on lines this PR touches: FilterByEvent's `note.replyTo?.forEach { parentNote -> }` used `note` in the body, so parent notes were never fetched — now uses `parentNote`. 5. Added a test at the layer where #1 lived: filterMissingEvents(cache, keys) fans a missing event to searchOnlyRelays + the follow/mine/search default, NOT the trusted relays that searchRelays would add. 6. Replaced an inline fully-qualified name in the test with an import (CLAUDE.md style). Green: commons jvmTest + verifyKmpPurity, :amethyst compilePlayDebugKotlin, :desktopApp compile, spotless. Co-Authored-By: Claude Opus 4.8 --- .../vitorpamplona/amethyst/model/Account.kt | 7 +- .../subassemblies/FilterByEvent.kt | 6 +- .../amethyst/ui/navigation/AppNavigation.kt | 12 ++++ .../event/loaders/FilterMissingEvents.kt | 2 +- .../relayClient/user/UserFinderAccount.kt | 11 ++- .../event/EventFinderFilterAssemblyTest.kt | 68 ++++++++++++++++++- .../amethyst/desktop/model/DesktopIAccount.kt | 3 + 7 files changed, 101 insertions(+), 8 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt index 2260fb2b95..4cfab50126 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/Account.kt @@ -32,7 +32,6 @@ import com.vitorpamplona.amethyst.commons.connectedApps.signers.NostrSignerPermi import com.vitorpamplona.amethyst.commons.connectedApps.signers.NostrSignerPermissionStore import com.vitorpamplona.amethyst.commons.defaults.Constants import com.vitorpamplona.amethyst.commons.defaults.DefaultIndexerRelayList -import com.vitorpamplona.amethyst.commons.defaults.DefaultSearchRelayList import com.vitorpamplona.amethyst.commons.marmot.MarmotManager import com.vitorpamplona.amethyst.commons.model.IAccount import com.vitorpamplona.amethyst.commons.model.buzz.BuzzRelayDialect @@ -382,7 +381,11 @@ class Account( override fun outboxHomeRelays(): Set = nip65RelayList.allFlowNoDefaults.value + privateStorageRelayList.flow.value + localRelayList.flow.value - override fun searchRelays(): Set = (trustedRelayList.flow.value + searchRelayList.flow.value.ifEmpty { DefaultSearchRelayList }).toSet() + // searchRelayList.flow already applies the DefaultSearchRelayList fallback internally + // (SearchRelayListState.normalizeSearchRelayListWithBackup), so no ifEmpty needed here. + override fun searchRelays(): Set = (trustedRelayList.flow.value + searchRelayList.flow.value).toSet() + + override fun searchOnlyRelays(): Set = searchRelayList.flow.value override fun followPlusAllMineWithSearchRelays(): Set = followPlusAllMineWithSearch.flow.value diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/searchCommand/subassemblies/FilterByEvent.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/searchCommand/subassemblies/FilterByEvent.kt index ae225e1b27..1bab42b2ef 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/searchCommand/subassemblies/FilterByEvent.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/searchCommand/subassemblies/FilterByEvent.kt @@ -45,9 +45,9 @@ fun filterByEvent( // loads threading that is event-based note.replyTo?.forEach { parentNote -> - if (parentNote !is AddressableNote && note.event == null) { - potentialRelaysToFindEvent(LocalCache, note).ifEmpty { default }.forEach { relayUrl -> - add(relayUrl, note.idHex) + if (parentNote !is AddressableNote && parentNote.event == null) { + potentialRelaysToFindEvent(LocalCache, parentNote).ifEmpty { default }.forEach { relayUrl -> + add(relayUrl, parentNote.idHex) } } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/AppNavigation.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/AppNavigation.kt index ee1ace2d8d..2f1287512d 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/AppNavigation.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/navigation/AppNavigation.kt @@ -50,6 +50,9 @@ import androidx.navigation.compose.composable import com.vitorpamplona.amethyst.Amethyst import com.vitorpamplona.amethyst.R import com.vitorpamplona.amethyst.commons.nipACWebRtcCalls.CallState +import com.vitorpamplona.amethyst.commons.relayClient.event.LocalEventFinder +import com.vitorpamplona.amethyst.commons.relayClient.user.LocalUserFinder +import com.vitorpamplona.amethyst.commons.relayClient.user.LocalUserFinderAccount import com.vitorpamplona.amethyst.service.crashreports.DisplayCrashMessages import com.vitorpamplona.amethyst.service.relayClient.notifyCommand.compose.DisplayNotifyMessages import com.vitorpamplona.amethyst.service.resourceusage.DisplayResourceUsageAlert @@ -341,6 +344,15 @@ fun AppNavigation( CompositionLocalProvider( LocalScreenLayout provides screenLayout, LocalTabReselectCoordinator provides tabReselectCoordinator, + // Provide the shared finder CompositionLocals so any commons composable that + // uses the no-arg observeUser*/EventFinderFilterAssemblerSubscription(note) + // overloads works when rendered on Android (they error() if unprovided). Android's + // own UI uses the AccountViewModel overloads and doesn't strictly need these, but + // providing them removes the runtime trap for shared composables reaching the + // logged-in tree. (The :napplet process never renders these composables.) + LocalUserFinder provides accountViewModel.dataSources().userFinder, + LocalUserFinderAccount provides accountViewModel.account, + LocalEventFinder provides accountViewModel.dataSources().eventFinder, ) { AccountSwitcherAndLeftDrawerLayout(accountViewModel, accountSessionManager, nav) { Box(Modifier.fillMaxSize()) { diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/FilterMissingEvents.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/FilterMissingEvents.kt index 65cac46200..6675cf7538 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/FilterMissingEvents.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/FilterMissingEvents.kt @@ -115,7 +115,7 @@ fun filterMissingEvents( add(relayUrl, key.note.idHex) } - key.account.searchRelays().forEach { relayUrl -> + key.account.searchOnlyRelays().forEach { relayUrl -> add(relayUrl, key.note.idHex) } } diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/UserFinderAccount.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/UserFinderAccount.kt index 9f625e2839..d1ecaae910 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/UserFinderAccount.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/user/UserFinderAccount.kt @@ -56,9 +56,18 @@ interface UserFinderAccount { /** Home/write relays used for outbox discovery (nip65 + private storage + local). */ fun outboxHomeRelays(): Set - /** Search relays (trusted + search), with the default fallback applied. */ + /** Search relays (trusted + own search list), for the user-finder's search-tier fallback. */ fun searchRelays(): Set + /** + * Just this account's own NIP-51 search relay list — WITHOUT the trusted-relay + * union that [searchRelays] adds. This is the narrow set the per-note event + * finder fans "missing event" REQs to, matching the pre-extraction + * `account.searchRelayList` read (reusing [searchRelays] there would have + * unintentionally widened the fan-out to trusted relays). + */ + fun searchOnlyRelays(): Set + /** * Follow + all-mine + search relays, used by the per-note event-finder to * place "missing event" / "missing addressable" REQs (reactions, zaps, diff --git a/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/EventFinderFilterAssemblyTest.kt b/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/EventFinderFilterAssemblyTest.kt index 034565a5d5..cfb3c42909 100644 --- a/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/EventFinderFilterAssemblyTest.kt +++ b/commons/src/jvmTest/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/EventFinderFilterAssemblyTest.kt @@ -20,18 +20,22 @@ */ package com.vitorpamplona.amethyst.commons.relayClient.event +import com.vitorpamplona.amethyst.commons.model.AddressableNote import com.vitorpamplona.amethyst.commons.model.Channel import com.vitorpamplona.amethyst.commons.model.Note import com.vitorpamplona.amethyst.commons.model.User import com.vitorpamplona.amethyst.commons.model.cache.ICacheEventStream import com.vitorpamplona.amethyst.commons.model.cache.ICacheProvider import com.vitorpamplona.amethyst.commons.relayClient.event.loaders.filterMissingEvents +import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderAccount import com.vitorpamplona.quartz.nip01Core.core.Address import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.hints.HintIndexer import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.nip85TrustedAssertions.list.tags.ServiceProviderTag import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse import org.junit.Assert.assertNull import org.junit.Assert.assertTrue import org.junit.Test @@ -73,6 +77,43 @@ class EventFinderFilterAssemblyTest { assertTrue(filterMissingEvents(mapOf(relay1 to emptySet())).isEmpty()) } + /** + * The per-note event finder fans a missing event out to the account's own + * search relays ([UserFinderAccount.searchOnlyRelays]) plus the + * follow/mine/search default — NOT the trusted-relay union that + * [UserFinderAccount.searchRelays] adds. Regression guard for reusing the + * wrong getter here (which would silently widen every missing-event REQ to + * trusted relays). + */ + @Test + fun `filterMissingEvents(keys) uses searchOnlyRelays, not the trusted searchRelays union`() { + val searchOnly = NormalizedRelayUrl("wss://search.test/") + val trustedExtra = NormalizedRelayUrl("wss://trusted.test/") // only in searchRelays() + val default = NormalizedRelayUrl("wss://default.test/") // followPlusAllMineWithSearchRelays() + + val account = + object : StubAccount() { + override fun searchOnlyRelays() = setOf(searchOnly) + + override fun searchRelays() = setOf(searchOnly, trustedExtra) + + override fun followPlusAllMineWithSearchRelays() = setOf(default) + } + + // event == null and not addressable → a "missing event"; no author/replies, + // and the stub cache has empty relay hints, so potentialRelaysToFindEvent is + // empty and the code falls back to the default relays + searchOnlyRelays. + val note = Note("a".repeat(64)) + + val filters = filterMissingEvents(StubCache(), listOf(EventFinderQueryState(note, account))) + val relays = filters.map { it.relay }.toSet() + + assertTrue("search-only relay carries the missing-event REQ", searchOnly in relays) + assertTrue("follow/mine/search default carries it", default in relays) + assertFalse("trusted relay (only in searchRelays) must NOT be fanned out to", trustedExtra in relays) + filters.forEach { assertEquals(listOf(note.idHex), it.filter.ids) } + } + /** * The new default [ICacheProvider.checkGetOrCreateUser] must swallow a * malformed-key throw and return null (the event-finder follows pubkey hints @@ -106,7 +147,7 @@ class EventFinderFilterAssemblyTest { override fun checkGetOrCreateNote(hexKey: HexKey): Note? = null - override fun getOrCreateAddressableNote(key: Address): com.vitorpamplona.amethyst.commons.model.AddressableNote = error("unused") + override fun getOrCreateAddressableNote(key: Address): AddressableNote = error("unused") override fun getEventStream(): ICacheEventStream = error("unused") @@ -116,4 +157,29 @@ class EventFinderFilterAssemblyTest { override fun justConsumeMyOwnEvent(event: Event): Boolean = false } + + /** Minimal [UserFinderAccount] returning nothing; override the relevant getters per test. */ + private open class StubAccount : UserFinderAccount { + override val userFinderPubkeyHex: HexKey = "00".repeat(32) + + override fun indexRelays(): Set = emptySet() + + override fun outboxHomeRelays(): Set = emptySet() + + override fun searchRelays(): Set = emptySet() + + override fun searchOnlyRelays(): Set = emptySet() + + override fun followPlusAllMineWithSearchRelays(): Set = emptySet() + + override fun commonRelays(): Set = emptySet() + + override fun cardHomeRelays(): Set = emptySet() + + override fun trustProvider(): ServiceProviderTag? = null + + override fun followerCountProvider(): ServiceProviderTag? = null + + override fun declaredFollowsByOutboxRelay(): Map> = emptyMap() + } } diff --git a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/model/DesktopIAccount.kt b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/model/DesktopIAccount.kt index c045e74df5..e68f95c233 100644 --- a/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/model/DesktopIAccount.kt +++ b/desktopApp/src/jvmMain/kotlin/com/vitorpamplona/amethyst/desktop/model/DesktopIAccount.kt @@ -114,6 +114,9 @@ class DesktopIAccount( override fun searchRelays(): Set = relayManager.connectedRelays.value + // Desktop has no separate NIP-51 search relay list; degrade to connected relays. + override fun searchOnlyRelays(): Set = relayManager.connectedRelays.value + // Desktop has no merged follow/mine/search relay-list subsystem; route // missing-event discovery through the connected relays (same degrade path // as the other hints above). From 70c53a8fcddd267c958bbb752965a3c08087a9df Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Mon, 10 Aug 2026 10:25:25 -0400 Subject: [PATCH 120/132] fix(concord): make the invite-list writes actually durable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A high-effort audit of the branch found that the durability guarantees the previous commits claimed were not the guarantees the code provided. Three of these are in the code written to close the last review, and they defeat exactly what those commits set out to fix. `INostrClient.publish` returns Unit — it queues an event and never reports acceptance; `publishAndConfirm` is the confirming variant. So every `runCatching { publish(...); true }` was true whenever local signing worked. That made minting's "record the link before handing out the URL" gate decorative, and made revoke worse than decorative: it reported success for a tombstone no relay stored, then recorded the kind-13303 tombstone, whose merge drops the entry — destroying the only `signer_sk` that could ever retire the link while the link stayed live. Both paths, and the Refounding re-mint, now confirm. `fetchAll` returns an empty list on cannot-connect / CLOSED / idle-timeout, so "a relay served us and had nothing" and "nobody answered" were the same observation. Reading the second as "no list yet" reintroduced, one layer below, the wipe the null-vs-empty work existed to prevent. `fetchAllWithHooks` gains a `doneOut` of per-relay terminal reasons plus `anyRelayServed()`, and both clients now only treat an empty read as an empty list when a relay actually reached EOSE. The rest: - `drainConcordRekeys` discarded the entry `adoptConcordRoot` now returns, so only the account that *launched* a rotation re-minted its links. An admin who was merely re-keyed left every link they had handed out on the dead root, and anyone stranded behind one could never recover — which is the branch's headline goal, holding only for the rotator. - The join-time ban gate fetched the Control Plane from `bundle.relays` alone (stale metadata refuses a community we can plainly reach) with a single un-paged REQ (truncated at the relay's filter cap, so a missing older ban edition fails the gate OPEN, re-admitting the account it exists to refuse). Now unions in the relays that just served the bundle, and pages. - `decodeOrNull` failed the whole document for one structurally incompatible entry. Since null now means "refuse to write", that converted the old silent data loss into a permanent write lock on a coordinate that never ages out. Unreadable entries are carried verbatim instead, so they neither block the account nor get dropped on re-encode. - The list read took the newest event of any kind and then cast, so one stray event at the coordinate read as "unreadable" forever. Filters by kind first. - The Refounding refresh did one full round trip per link, serially, inside a user-visible rotation. One pooled REQ over every link signer, then concurrent confirmed re-mints, classified per coordinate so one link's tombstone cannot decide another's status. Verified on a tablet: mint, list, revoke and the cross-client refusal still work end to end — and with the community relay killed, revoke now reports "The link couldn't be revoked" and leaves the entry intact, where before it would have claimed success and destroyed the key. Co-Authored-By: Claude Opus 5 (1M context) --- .../amethyst/model/AccountConcordActions.kt | 138 +++++++++++++----- .../com/vitorpamplona/amethyst/cli/Context.kt | 3 + .../amethyst/cli/commands/ConcordCommands.kt | 15 +- .../commons/actions/ConcordActions.kt | 10 ++ .../cord05Invites/ConcordInviteList.kt | 96 +++++++++--- .../NostrClientFetchAllWithHooksExt.kt | 21 +++ .../cord05Invites/ConcordInviteListTest.kt | 37 +++++ 7 files changed, 255 insertions(+), 65 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt index 35816e52ce..1bf9339ae4 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/AccountConcordActions.kt @@ -54,8 +54,11 @@ import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray import com.vitorpamplona.quartz.nip01Core.core.toHexKey +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.anyRelayServed import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.fetchAll import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.fetchAllPagesFromPool +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.fetchAllWithHooks +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.publishAndConfirm import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer @@ -65,6 +68,9 @@ import com.vitorpamplona.quartz.nipC7Chats.ChatEvent import com.vitorpamplona.quartz.utils.Log import com.vitorpamplona.quartz.utils.RandomInstance import com.vitorpamplona.quartz.utils.TimeUtils +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope import java.util.concurrent.ConcurrentHashMap /** Name of the default Concord community Admin role minted by "Make admin". */ @@ -191,12 +197,29 @@ class AccountConcordActions( val relays = account.outboxRelays.flow.value if (relays.isEmpty()) return null val filter = Filter(kinds = listOf(ConcordInviteListEvent.KIND), authors = listOf(account.signer.pubKey)) + // Terminal reasons, not just events: `fetchAll` returns an empty list both when a relay + // served us and had nothing AND when nothing answered at all (cannot-connect, CLOSED, idle + // timeout). Treating the second as "no list yet" is precisely how a read-merge-write wipes + // the signer_sk of every link it failed to read, so the two must be told apart. + val reasons = mutableMapOf() + val events = + account.client.fetchAllWithHooks( + filters = relays.associateWith { listOf(filter) }, + doneOut = reasons, + ) { _, _ -> true } + val newest = - account.client - .fetchAll(filters = relays.associateWith { listOf(filter) }) + events + .mapNotNull { it.second as? ConcordInviteListEvent } + // Filter by kind BEFORE picking the newest: taking the newest of anything and then + // casting means one stray event at this coordinate reads as "unreadable" forever. .maxByOrNull { it.createdAt } - ?: return ConcordInviteListDocument.EMPTY // nothing published yet — safe to start one - return (newest as? ConcordInviteListEvent)?.decrypt(account.signer) + ?: return if (reasons.anyRelayServed()) { + ConcordInviteListDocument.EMPTY // a relay answered and had nothing — safe to start one + } else { + null // nobody answered; we know nothing about what is published + } + return newest.decrypt(account.signer) } /** @@ -216,9 +239,11 @@ class AccountConcordActions( Log.w("Concord") { "Refusing to write the invite list: could not read the current one (would drop other links' signer_sk)" } return false } + // publishAndConfirm, never publish: `INostrClient.publish` returns Unit — it queues the event + // and never reports acceptance — so a `runCatching { publish(); true }` is true whenever + // local signing worked, and every caller's "did the record land?" gate becomes decorative. return runCatching { - account.client.publish(ConcordInviteListEvent.create(account.signer, ConcordInviteList.merge(base, patch), TimeUtils.now()), publishTo) - true + account.client.publishAndConfirm(ConcordInviteListEvent.create(account.signer, ConcordInviteList.merge(base, patch), TimeUtils.now()), publishTo) }.onFailure { Log.w("Concord", "invite list publish failed", it) }.getOrDefault(false) } @@ -243,30 +268,43 @@ class AccountConcordActions( val list = readConcordInviteList() ?: return 0 val tombstoned = list.tombstones.mapTo(HashSet()) { it.token } val now = TimeUtils.now() - var count = 0 - for (link in list.entries) { - if (link.communityId != entry.id) continue - // An elapsed or retired link can no longer be joined; re-posting it would only resurrect - // a dead URL at a live epoch. - if (link.isExpired(now) || link.token in tombstoned) continue - runCatching { - val token = link.token.hexToByteArray() - val wraps = account.client.fetchAll(filters = relays.associateWith { listOf(ConcordActions.bundleFilter(link.signerPubKeyHex())) }) - // Honour a revocation published at this coordinate, and carry the live bundle's own - // fields forward — only the epoch's key material changes. - val current = ConcordActions.classifyInvite(wraps, token) as? InviteBundleStatus.Live ?: return@runCatching - val moved = - current.invite.copy( - communityRoot = entry.root, - rootEpoch = entry.rootEpoch, - controlPk = entry.controlPk, - relays = entry.relays, - ) - account.client.publish(ConcordActions.remintBundleAt(link.signerSk.hexToByteArray(), token, moved, now), relays) - count++ - }.onFailure { Log.w("Concord", "invite refresh failed for ${entry.id}", it) } + + // An elapsed or retired link can no longer be joined; re-posting it would only resurrect a + // dead URL at a live epoch. + val links = list.entries.filter { it.communityId == entry.id && !it.isExpired(now) && it.token !in tombstoned } + if (links.isEmpty()) return 0 + + // One REQ for every link's bundle rather than a round trip each. This runs inside the + // user-visible Refounding, and a serial fetch per link makes a removal take time linear in + // how many links the creator ever minted, each able to wait out its own idle timeout. + val byAuthor = links.associateBy { it.signerPubKeyHex().lowercase() } + val wraps = account.client.fetchAll(filters = relays.associateWith { listOf(ConcordActions.bundlesFilter(byAuthor.keys.toList())) }) + val wrapsByAuthor = wraps.groupBy { it.pubKey.lowercase() } + + return coroutineScope { + byAuthor + .map { (author, link) -> + async { + runCatching { + val token = link.token.hexToByteArray() + // Classify per coordinate, never over the pooled set: one link's newer + // revocation tombstone must not decide another link's status. + val current = ConcordActions.classifyInvite(wrapsByAuthor[author].orEmpty(), token) as? InviteBundleStatus.Live ?: return@runCatching false + val moved = + current.invite.copy( + communityRoot = entry.root, + rootEpoch = entry.rootEpoch, + controlPk = entry.controlPk, + relays = entry.relays, + ) + // Confirmed: a link counted as moved but never stored is a link its + // holders can no longer redeem, reported as a success. + account.client.publishAndConfirm(ConcordActions.remintBundleAt(link.signerSk.hexToByteArray(), token, moved, now), relays) + }.onFailure { Log.w("Concord", "invite refresh failed for ${entry.id}", it) }.getOrDefault(false) + } + }.awaitAll() + .count { it } } - return count } /** @@ -393,10 +431,12 @@ class AccountConcordActions( val relays = entry.relays.mapNotNullTo(mutableSetOf()) { RelayUrlNormalizer.normalizeOrNull(it) }.ifEmpty { account.outboxRelays.flow.value } if (relays.isEmpty()) return false + // Confirmed, not fire-and-forget. A `publish` that returns Unit would report success for a + // tombstone no relay stored — and the list write below would then drop this entry on merge, + // destroying the only `signer_sk` that could ever retire the link while the link stays live. val published = runCatching { - account.client.publish(ConcordActions.revokeBundleAt(link.signerSk.hexToByteArray(), TimeUtils.now()), relays) - true + account.client.publishAndConfirm(ConcordActions.revokeBundleAt(link.signerSk.hexToByteArray(), TimeUtils.now()), relays) }.onFailure { Log.w("Concord", "invite revocation failed for $communityId", it) }.getOrDefault(false) if (!published) return false @@ -477,7 +517,15 @@ class AccountConcordActions( // other door into the same room. // // Fails CLOSED on an unreadable plane: the banlist is only knowable once the bundle yields - // the root, and no verdict means no join. + // the root, and no verdict means no join. Two things make that safe to insist on rather than + // a way to brick valid invites: + // + // - the plane is fetched over the SAME relays that just served the bundle, not the relay + // list inside the bundle alone, which can be stale (a moved relay, a link minted before a + // relay change) and would otherwise refuse a community we can plainly reach; + // - it is PAGED, because a single REQ is truncated at the relay's per-filter cap. A missing + // older ban edition fails the gate open — it re-admits the very account it exists to + // refuse — so the one direction we must not economise on is completeness. val joinKeys = ConcordActions.controlPlaneKeys( communityRoot = bundle.communityRoot.hexToByteArray(), @@ -485,12 +533,14 @@ class AccountConcordActions( rootEpoch = bundle.rootEpoch, controlPk = bundle.controlPk, ) - val joinRelays = bundle.relays.mapNotNullTo(mutableSetOf()) { RelayUrlNormalizer.normalizeOrNull(it) }.ifEmpty { relays } - val joinEditions = - ConcordActions.controlEditions( - account.client.fetchAll(filters = joinRelays.associateWith { listOf(ConcordActions.planeFilter(joinKeys.address)) }), - joinKeys, - ) + // Union, not `ifEmpty`: the relays that served the bundle are known-good for this community, + // and the bundle's own list is the one that goes stale. + val joinRelays = bundle.relays.mapNotNullTo(mutableSetOf()) { RelayUrlNormalizer.normalizeOrNull(it) } + relays + val planeWraps = mutableListOf() + account.client.fetchAllPagesFromPool( + filters = joinRelays.associateWith { listOf(ConcordActions.planeFilter(joinKeys.address)) }, + ) { event, _ -> planeWraps.add(event) } + val joinEditions = ConcordActions.controlEditions(planeWraps, joinKeys) if (joinEditions.isEmpty()) return ConcordInviteResult.NotReachable if (AuthorityResolver.resolve(joinEditions, bundle.owner).isBanned(account.signer.pubKey)) { return ConcordInviteResult.Banned @@ -1210,7 +1260,17 @@ class AccountConcordActions( // who has themselves been banned could still rotate the whole community. val authorized = authority.isOwner(received.rotator) || authority.hasPermission(received.rotator, ConcordPermissions.BAN) if (!authorized) continue - adoptConcordRoot(entry, received.newRoot, received.newEpoch, received.newControlPk, received.newControlRoot) + val adopted = adoptConcordRoot(entry, received.newRoot, received.newEpoch, received.newControlPk, received.newControlRoot) + + // Move our own links onto the epoch we just adopted. Rotating is not the only way to end + // up on a new epoch — being re-keyed is the common one — and a link creator who is merely + // re-keyed would otherwise leave every link they handed out pointing at the dead root, + // which is exactly the orphaning this branch exists to stop. Stranded recovery reads the + // bundle's epoch, so a link nobody re-mints is a member nobody can recover. + adopted?.let { next -> + val moved = refreshConcordInviteLinks(next) + if (moved > 0) Log.i("Concord") { "Rekey ${next.id}: refreshed $moved invite link(s) to epoch ${received.newEpoch}" } + } } } diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt index 02b482bebf..185cbc7f6d 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/Context.kt @@ -582,12 +582,15 @@ class Context( diagnoseSlow: Boolean = false, deadOut: MutableMap? = null, pendingOnAuthRequired: Boolean = false, + /** Per-relay terminal reason, so a caller can tell an empty answer from no answer. */ + doneOut: MutableMap? = null, ): List> = client.fetchAllWithHooks( filters = filters, idleTimeoutMs = idleTimeoutMs, pendingOnAuthRequired = pendingOnAuthRequired, deadOut = deadOut, + doneOut = doneOut, onTimeout = if (diagnoseSlow) { { stalled, doneReasons, collected -> logSlowDrain(idleTimeoutMs, stalled, doneReasons, collected) } diff --git a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt index 77047c4e5b..51ad02fd7e 100644 --- a/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt +++ b/cli/src/main/kotlin/com/vitorpamplona/amethyst/cli/commands/ConcordCommands.kt @@ -43,6 +43,7 @@ import com.vitorpamplona.quartz.concord.cord05Invites.InviteBundleStatus import com.vitorpamplona.quartz.concord.crypto.ControlPlaneKeys import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray import com.vitorpamplona.quartz.nip01Core.core.toHexKey +import com.vitorpamplona.quartz.nip01Core.relay.client.accessories.anyRelayServed import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer @@ -724,13 +725,19 @@ object ConcordCommands { val relays = ctx.outboxRelays() if (relays.isEmpty()) return null val filter = Filter(kinds = listOf(ConcordInviteListEvent.KIND), authors = listOf(ctx.signer.pubKey)) + // Terminal reasons, not just events: a drain returns nothing both when a relay served us and + // had nothing AND when nobody answered. Reading the second as "no list yet" is how the + // read-merge-write below wipes the signer_sk of every link it failed to read. + val reasons = mutableMapOf() val newest = ctx - .drain(relays.associateWith { listOf(filter) }) - .map { it.second } + .drain(relays.associateWith { listOf(filter) }, doneOut = reasons) + // Filter by kind BEFORE picking the newest — a stray event at this coordinate would + // otherwise make the list read as unreadable and refuse every later write. + .mapNotNull { it.second as? ConcordInviteListEvent } .maxByOrNull { it.createdAt } - ?: return ConcordInviteListDocument.EMPTY // nothing published yet — safe to start one - return (newest as? ConcordInviteListEvent)?.decrypt(ctx.signer) + ?: return if (reasons.anyRelayServed()) ConcordInviteListDocument.EMPTY else null + return newest.decrypt(ctx.signer) } /** diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt index dfb51935d7..e11a4278fa 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/actions/ConcordActions.kt @@ -201,6 +201,16 @@ object ConcordActions { /** The public invite bundle for a link signer. */ fun bundleFilter(linkSignerPubKeyHex: HexKey): Filter = Filter(kinds = listOf(ConcordInviteBundleEvent.KIND), authors = listOf(linkSignerPubKeyHex)) + /** + * The bundles of several links at once — one REQ over every link signer instead of a round trip + * per link, which is what a Refounding needs when it re-mints a creator's whole set. + * + * Partition the result by `pubKey` before classifying: [ConcordInviteBundle.classify] resolves a + * single coordinate, so handing it a pooled set would let one link's revocation tombstone decide + * another link's status purely by being newer. + */ + fun bundlesFilter(linkSignerPubKeyHexes: List): Filter = Filter(kinds = listOf(ConcordInviteBundleEvent.KIND), authors = linkSignerPubKeyHexes) + /** Pending direct invites addressed to the given member (indexed by k=3313). */ fun directInvitesFilter(memberPubKeyHex: HexKey): Filter = Filter(kinds = listOf(ConcordStreamEnvelope.KIND_WRAP), tags = mapOf("p" to listOf(memberPubKeyHex), "k" to listOf(ConcordDirectInvite.KIND.toString()))) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteList.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteList.kt index 5b930eb261..f136aed3fd 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteList.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteList.kt @@ -30,9 +30,11 @@ import kotlinx.serialization.KSerializer import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import kotlinx.serialization.descriptors.elementNames +import kotlinx.serialization.json.JsonArray import kotlinx.serialization.json.JsonElement import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonTransformingSerializer +import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonObject private val NoExtras: JsonObject = JsonObject(emptyMap()) @@ -77,11 +79,19 @@ class ConcordInviteListTombstone( val residue: JsonObject = NoExtras, ) -/** The decoded kind-13303 document: live [entries], [tombstones], and document-level [residue]. */ +/** + * The decoded kind-13303 document: live [entries], [tombstones], and document-level [residue]. + * + * [opaqueEntries] holds entries that did not type-check — a wrong-typed field from another client or + * a newer schema. They are carried verbatim rather than dropped (re-encoding without them would + * delete somebody's `signer_sk`) and rather than failing the whole read (which would refuse every + * future mint and revoke for this account until someone else repaired the list). + */ class ConcordInviteListDocument( val entries: List = emptyList(), val tombstones: List = emptyList(), val residue: JsonObject = NoExtras, + val opaqueEntries: List = emptyList(), ) { companion object { val EMPTY = ConcordInviteListDocument() @@ -160,41 +170,80 @@ object ConcordInviteList { private object WireDocumentSerializer : ExtrasPreserving(WireDocument.serializer()) /** - * Decodes the plaintext document, or **null** when it cannot be parsed. + * Decodes the plaintext document, or **null** when the document itself cannot be read. * * Null rather than an empty document on purpose: this list is replaceable, so a caller that * treats "I could not read it" as "it is empty" and republishes destroys every `signer_sk` it * did not manage to read — secrets that cannot be regenerated, orphaning every outstanding * invite at a dead epoch. Callers MUST distinguish the two (see [ConcordInviteList.merge]'s - * callers). Each field still defaults, so one odd entry does not abort the whole array. + * callers). + * + * Null is reserved for a *document-level* failure — not JSON, or `entries`/`tombstones` present + * but not arrays. A single entry that does not type-check is kept verbatim in + * [ConcordInviteListDocument.opaqueEntries] instead: failing the whole read for one odd row + * would refuse every future mint and revoke for the account, permanently, since a replaceable + * coordinate never ages out — turning the old silent data loss into a permanent write lock. */ fun decodeOrNull(json: String): ConcordInviteListDocument? = try { - val doc = ConcordJson.instance.decodeFromString(WireDocumentSerializer, json) - ConcordInviteListDocument( - entries = - doc.entries.map { + val root = ConcordJson.instance.parseToJsonElement(json).jsonObject + val opaque = mutableListOf() + + val entries = + (root["entries"]?.jsonArray ?: JsonArray(emptyList())).mapNotNull { element -> + val obj = element.jsonObject + try { + val it = ConcordJson.instance.decodeFromJsonElement(WireEntrySerializer, obj) ConcordInviteListEntry(it.token, it.signerSk, it.communityId, it.url, it.label, it.createdAt, it.expiresAt, it.extras) - }, - tombstones = doc.tombstones.map { ConcordInviteListTombstone(it.token, it.communityId, it.extras) }, - residue = doc.extras, + } catch (_: Exception) { + opaque.add(obj) + null + } + } + + val tombstones = + (root["tombstones"]?.jsonArray ?: JsonArray(emptyList())).mapNotNull { element -> + try { + val it = ConcordJson.instance.decodeFromJsonElement(WireTombstoneSerializer, element.jsonObject) + ConcordInviteListTombstone(it.token, it.communityId, it.extras) + } catch (_: Exception) { + // A tombstone we cannot read must not silently un-retire its link, but we + // have no token to key it by, so it can only ride along as document residue. + null + } + } + + ConcordInviteListDocument( + entries = entries, + tombstones = tombstones, + residue = JsonObject(root - "entries" - "tombstones"), + opaqueEntries = opaque, ) } catch (_: Exception) { null } - fun encode(doc: ConcordInviteListDocument): String = - ConcordJson.instance.encodeToString( - WireDocumentSerializer, - WireDocument( - entries = - doc.entries.map { - WireEntry(it.token, it.signerSk, it.communityId, it.url, it.label, it.createdAt, it.expiresAt, it.residue) - }, - tombstones = doc.tombstones.map { WireTombstone(it.token, it.communityId, it.residue) }, - extras = doc.residue, - ), - ) + fun encode(doc: ConcordInviteListDocument): String { + val wire = + ConcordJson.instance + .encodeToJsonElement( + WireDocumentSerializer, + WireDocument( + entries = + doc.entries.map { + WireEntry(it.token, it.signerSk, it.communityId, it.url, it.label, it.createdAt, it.expiresAt, it.residue) + }, + tombstones = doc.tombstones.map { WireTombstone(it.token, it.communityId, it.residue) }, + extras = doc.residue, + ), + ).jsonObject + + // Entries we could not type ride back out untouched. Dropping them here is the data loss + // this whole class exists to prevent — they are somebody's link signer too. + if (doc.opaqueEntries.isEmpty()) return ConcordJson.instance.encodeToString(JsonObject.serializer(), wire) + val entries = JsonArray((wire["entries"]?.jsonArray ?: JsonArray(emptyList())) + doc.opaqueEntries) + return ConcordJson.instance.encodeToString(JsonObject.serializer(), JsonObject(wire + ("entries" to entries))) + } /** * Merges [patch] onto [base], keyed by `token` — the spec's own merge key. A token present in @@ -218,6 +267,9 @@ object ConcordInviteList { entries = entries.values.toList(), tombstones = tombstones.values.toList(), residue = JsonObject(base.residue + patch.residue), + // Untyped entries survive the merge for the same reason they survive a decode: we cannot + // read them, so we are in no position to decide they are disposable. + opaqueEntries = (base.opaqueEntries + patch.opaqueEntries).distinct(), ) } } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllWithHooksExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllWithHooksExt.kt index 62d13ce5e9..f28864dca0 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllWithHooksExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientFetchAllWithHooksExt.kt @@ -79,6 +79,14 @@ suspend fun INostrClient.fetchAllWithHooks( subscriptionId: String = newSubId(), pendingOnAuthRequired: Boolean = false, deadOut: MutableMap? = null, + /** + * Receives the terminal reason per relay ("eose", "closed:…", "cannot:…"), so a caller can + * tell "a relay served us and had nothing" from "nobody served us". An empty result alone + * cannot: both look like zero events, and treating the second as the first is how a + * read-merge-write on a replaceable event destroys the entries it failed to read. See + * [anyRelayServed]. + */ + doneOut: MutableMap? = null, onTimeout: ((stalled: Set, doneReasons: Map, collected: List>) -> Unit)? = null, /** * Hard wall-clock ceiling. The idle window alone is unbounded when a relay @@ -237,9 +245,22 @@ suspend fun INostrClient.fetchAllWithHooks( classifyDrainFailure(reason)?.let { out[relay] = it } } } + doneOut?.putAll(doneReasons) return collected } +/** The terminal reason recorded when a relay finished serving a subscription normally. */ +const val DONE_REASON_EOSE = "eose" + +/** + * True when at least one relay completed the fetch normally, i.e. answered and reached EOSE. + * + * Read against the map filled by `fetchAllWithHooks`'s `doneOut`. An empty event list means + * "nothing matched" only when this is true; otherwise it means "nobody told us", and a caller + * that overwrites a replaceable event on that basis deletes whatever it could not read. + */ +fun Map.anyRelayServed(): Boolean = values.any { it == DONE_REASON_EOSE } + /** * [fetchAllPagesFromPool] with a suspending per-event hook: paginates every relay * to completion (each on its own `until` cursor, up to [maxConcurrentRelays] at diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListTest.kt index 7a3904bdbc..b3a6dbde5e 100644 --- a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListTest.kt +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/concord/cord05Invites/ConcordInviteListTest.kt @@ -123,6 +123,43 @@ class ConcordInviteListTest { assertEquals(null, ConcordInviteList.decodeOrNull("{\"entries\":\"wrong type\"}")) } + @Test + fun oneUnreadableEntryDoesNotFailTheWholeDocumentOrGetDropped() { + // One structurally incompatible entry — a newer schema turning a scalar into an object, the + // realistic version, since the lenient parser already coerces plain scalar mismatches — used + // to null the whole document. Because null now means "refuse to write", that turned a single + // odd row into a permanent lock on mint and revoke for the account: a replaceable coordinate + // never ages out, so nothing would ever clear it. + val mixed = + """ + { "entries": [ + { "token": "aa", "signer_sk": "bb", "community_id": "cc", "url": "u1" }, + { "token": {"v": "dd"}, "signer_sk": "dd", "community_id": "cc", "url": "u2", "mark": "keepme" } + ], + "tombstones": [] } + """.trimIndent() + + val doc = ConcordInviteList.decodeOrNull(mixed) + assertEquals(listOf("aa"), doc!!.entries.map { it.token }, "the readable entry still decodes") + assertEquals(1, doc.opaqueEntries.size, "the unreadable entry is kept, not discarded") + + // And it survives a re-encode: dropping it would delete somebody's signer_sk, which is the + // exact data loss this class exists to prevent. + assertTrue(ConcordInviteList.encode(doc).contains("keepme"), "unreadable entry lost on re-encode") + } + + @Test + fun aMergeCarriesUnreadableEntriesThrough() { + val base = ConcordInviteList.decodeOrNull("""{"entries":[{"token":{"v":7},"mark":"opaque"}],"tombstones":[]}""")!! + val patch = ConcordInviteListDocument(entries = listOf(ConcordInviteListEntry("t", "sk", "c", "u"))) + + val merged = ConcordInviteList.merge(base, patch) + + assertEquals(listOf("t"), merged.entries.map { it.token }) + // We cannot read it, so we are in no position to decide it is disposable. + assertTrue(ConcordInviteList.encode(merged).contains("opaque"), "merge dropped an unreadable entry") + } + @Test fun anUnreadableListIsDistinguishableFromAnEmptyOne() { // The whole point of the null: a caller must be able to tell "I could not read it" from From d3921cd7f547af9448ae3ddcd5570908e5aa53d3 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Mon, 10 Aug 2026 15:40:15 -0400 Subject: [PATCH 121/132] fix(ci): unbreak iOS compile and the arm64 desktop smoke test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two red checks on this branch, from two unrelated causes. 1. `test-quartz-ios` — AddressableAuthorRelayLoaderSubAssembler moved into commonMain still calling `synchronized(lock)`. That resolves from kotlin-stdlib-jvm with no import, so it compiles on Android/JVM and only fails at `:commons:compileKotlinIosSimulatorArm64`. Swapped to `KmpLock.withLock {}` (reentrant on every platform, and `withLock` is inline so `commit()`'s early `return` still works). The `verifyKmpPurity` gate missed it because it only forbade the `kotlin.jvm.Synchronized` *annotation*, not the bare call. Added `synchronized(` to the forbidden list in both :commons and :quartz so the next one fails in seconds instead of at the iOS compile step. 2. `release-deb-launch (ubuntu-24.04-arm)` — pre-existing infra break, not from this branch (same failure on other PRs since ~Aug 5). libskiko-linux-arm64.so needs libEGL.so.1 and the runner has no libegl1, so the app died at startup. create-release.yml already fixes this via scripts/add-deb-libegl-dep.sh; the smoke test never adopted it. Added that step, and switched the install from `dpkg -i` (which does not resolve dependencies) to `apt-get install ./x.deb` so the declared libegl1 is actually pulled in — this now exercises the same artifact release ships. Verified: :commons:compileKotlinIosSimulatorArm64, both verifyKmpPurity gates (and confirmed the new pattern fails when the bug is reintroduced), :commons:jvmTest, :amethyst:testPlayDebugUnitTest for the assembler test, and the .deb libegl mechanism end-to-end in an arm64 ubuntu:24.04 container. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/smoke-test-desktop.yml | 20 +++++++++++++++++-- commons/build.gradle.kts | 4 ++++ ...ddressableAuthorRelayLoaderSubAssembler.kt | 11 ++++++---- quartz/build.gradle.kts | 4 ++++ 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/.github/workflows/smoke-test-desktop.yml b/.github/workflows/smoke-test-desktop.yml index 6d8159ab0e..57b992ae73 100644 --- a/.github/workflows/smoke-test-desktop.yml +++ b/.github/workflows/smoke-test-desktop.yml @@ -92,13 +92,29 @@ jobs: chmod +x scripts/relax-deb-libicu.sh scripts/relax-deb-libicu.sh desktopApp/build/compose/binaries/main-release/deb/*.deb + # Mirrors the same step in create-release.yml so this job exercises the + # exact .deb release ships. libskiko-linux-arm64.so has libEGL.so.1 in + # DT_NEEDED and jpackage does not scan lib/app/ for Depends, so without + # this the arm64 app dies at startup with + # UnsatisfiedLinkError: libEGL.so.1: cannot open shared object file + # See scripts/add-deb-libegl-dep.sh for the full rationale. + - name: Add libegl1 dep to arm64 .deb + run: | + set -euo pipefail + chmod +x scripts/add-deb-libegl-dep.sh + scripts/add-deb-libegl-dep.sh desktopApp/build/compose/binaries/main-release/deb/*.deb + - name: Install .deb run: | + # Installed via apt (not `dpkg -i`) so the .deb's declared Depends are + # actually resolved — that is what pulls in libegl1 on the arm64 + # runner, which does not ship it preinstalled. + # # jpackage's post-install script runs xdg-desktop-menu which fails # on CI runners ("No writable system menu directory"). The files are # extracted successfully; only the menu registration fails. Allow the - # dpkg error, then verify the binary was actually installed. - sudo dpkg -i desktopApp/build/compose/binaries/main-release/deb/*.deb || true + # install error, then verify the binary was actually installed. + sudo apt-get install -y ./desktopApp/build/compose/binaries/main-release/deb/*.deb || true echo "Installed files:" dpkg -L amethyst | head -30 # Fail if the binary wasn't actually extracted diff --git a/commons/build.gradle.kts b/commons/build.gradle.kts index 2fccbea2ce..952b39c596 100644 --- a/commons/build.gradle.kts +++ b/commons/build.gradle.kts @@ -297,6 +297,10 @@ val verifyKmpPurity by tasks.registering { "Thread.sleep" to "use kotlinx.coroutines.delay or platform-specific actual", "java.util.UUID" to "use kotlin.uuid.Uuid", "kotlin.jvm.Synchronized" to "use KmpLock.withLock {}", + // The bare call, not just the annotation: `synchronized(lock) {}` resolves + // from kotlin-stdlib-jvm with no import, so it compiles on Android/JVM and + // only fails at the iOS compile step. Catch it here instead. + "synchronized(" to "`synchronized` is JVM-only — use KmpLock.withLock {}", "kotlin.jvm.Volatile" to "use kotlin.concurrent.Volatile", ) val offenders = diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/AddressableAuthorRelayLoaderSubAssembler.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/AddressableAuthorRelayLoaderSubAssembler.kt index 68e5a40544..46dbae7ac7 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/AddressableAuthorRelayLoaderSubAssembler.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/event/loaders/AddressableAuthorRelayLoaderSubAssembler.kt @@ -27,6 +27,8 @@ import com.vitorpamplona.amethyst.commons.relayClient.event.EventFinderQueryStat import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderFilterAssembler import com.vitorpamplona.amethyst.commons.relayClient.user.UserFinderQueryState import com.vitorpamplona.amethyst.commons.service.BundledUpdate +import com.vitorpamplona.amethyst.commons.util.KmpLock +import com.vitorpamplona.amethyst.commons.util.withLock import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.IO @@ -46,8 +48,9 @@ class AddressableAuthorRelayLoaderSubAssembler( val userFinder: UserFinderFilterAssembler, ) : IEoseManager { // Private monitor: @Synchronized locks on `this`, which leaves the instance's monitor - // reachable to anything holding a reference to this assembler. - private val lock = Any() + // reachable to anything holding a reference to this assembler. KmpLock (not `synchronized`) + // because this file lives in commonMain and must compile for the iOS targets too. + private val lock = KmpLock() // Only ever touched while holding [lock]. See commit() and destroy(). private var activeSubscriptions: Set = emptySet() @@ -92,7 +95,7 @@ class AddressableAuthorRelayLoaderSubAssembler( * never call back into this class. Revisit if that changes. */ private fun commit(needed: Set) { - synchronized(lock) { + lock.withLock { if (destroyed) return userFinder.subscribe((needed - activeSubscriptions).toList()) @@ -103,7 +106,7 @@ class AddressableAuthorRelayLoaderSubAssembler( } override fun destroy() { - synchronized(lock) { + lock.withLock { destroyed = true bundler.cancel() userFinder.unsubscribe(activeSubscriptions.toList()) diff --git a/quartz/build.gradle.kts b/quartz/build.gradle.kts index 8a93c5758d..4771e49055 100644 --- a/quartz/build.gradle.kts +++ b/quartz/build.gradle.kts @@ -424,6 +424,10 @@ val verifyKmpPurity by tasks.registering { "Thread.sleep" to "use kotlinx.coroutines.delay or platform-specific actual", "java.util.UUID" to "use kotlin.uuid.Uuid", "kotlin.jvm.Synchronized" to "use a KMP lock primitive", + // The bare call, not just the annotation: `synchronized(lock) {}` resolves + // from kotlin-stdlib-jvm with no import, so it compiles on Android/JVM and + // only fails at the iOS compile step. Catch it here instead. + "synchronized(" to "`synchronized` is JVM-only — use a KMP lock primitive", "kotlin.jvm.Volatile" to "use kotlin.concurrent.Volatile", ) val offenders = From 704198099695e03cbe9240e0ca92e402ce85178e Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Mon, 10 Aug 2026 21:08:05 +0000 Subject: [PATCH 122/132] negentropy: let a caller decline an id before the download REQ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit negentropySync names every id the relay has that the caller lacks, then fetches all of them. There is no point between those two steps where a caller can say "not that one" — onEvent is the first hook, and by then the body has already crossed the wire. That is a real cost for a mirror. A store keeping only the newest version of a replaceable event refuses every relay's older copy, and negentropy re-offers those copies on every sync because they are genuinely absent from the local id set. Measured on a downstream mirror against relay.damus.io, nos.lol and relay.primal.net: three passes over kinds 0/3/10002 produced 5, 18 and 29 REPLACED rejections, the same events re-downloaded each time. Adds an optional `wantId: ((HexKey) -> Boolean)?` to negentropySync, negentropySyncOrFetch and negentropySyncFanOut, consulted for each id before the REQ, plus a `skipped` count on the three result types. Default null keeps every existing call byte-identical. Three decisions worth stating: - The gate runs on a whole reconcile round's ids, BEFORE they are chunked into fetch batches. Gating after the chunking keeps the batch count and shrinks every batch instead — at the density this exists for, a fetchBatch of 500 becomes a hundred REQs of five ids each, turning a bandwidth saving into a latency regression. - `skipped` is reported apart from both `downloaded` and `needCount`. needCount stays the honest protocol diff whether or not the caller fetched it, and without a separate number an operator cannot tell a predicate that does nothing from one that eats everything — both are silent. - keep() returns an empty list, never null. A nullable return invites `gate?.keep(ids) ?: ids`, which reads as "no gate, keep everything" and means "everything was declined, so send everything". That elvis turned the fully-declining case into a full download during development; the non-null contract removes the trap rather than documenting it. The gate does not cover a window handed to onUnreconcilableWindow: that is drained over REQ, which names no ids before streaming bodies. Documented on both the base function and the combinator. --- .../accessories/NegentropyFanOutResult.kt | 7 + .../NostrClientNegentropyFanOutExt.kt | 10 ++ .../NostrClientNegentropySyncExt.kt | 114 ++++++++++++++- .../relay/client/accessories/NeedGateTest.kt | 137 ++++++++++++++++++ .../relay/NostrClientNegentropySyncTest.kt | 117 +++++++++++++++ 5 files changed, 382 insertions(+), 3 deletions(-) create mode 100644 quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NeedGateTest.kt diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropyFanOutResult.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropyFanOutResult.kt index 96392ace8a..a7cbb1678f 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropyFanOutResult.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NegentropyFanOutResult.kt @@ -36,4 +36,11 @@ class NegentropyFanOutResult( val downloaded: Int, val windows: Int, val connections: Int, + /** + * Ids `wantId` declined, so no `REQ` was ever issued for them. `0` when no + * predicate was passed. Same accounting as + * [NegentropySyncResult.skipped]: apart from [downloaded], and never folded + * into [needCount], which stays the honest protocol diff. + */ + val skipped: Int = 0, ) diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropyFanOutExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropyFanOutExt.kt index e941af4417..35583fe02c 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropyFanOutExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropyFanOutExt.kt @@ -84,6 +84,13 @@ suspend fun negentropySyncFanOut( fetchBatch: Int = 250, idleTimeoutMs: Long = 120_000L, reconcileConcurrency: Int = 2, + /** + * Same contract as [negentropySync]'s: consulted for every id the reconcile + * names, before the `REQ` that would fetch it. Declined ids are counted in + * [NegentropyFanOutResult.skipped]. Called from several reconciler + * coroutines at once, so it must be cheap and thread-safe. + */ + wantId: ((HexKey) -> Boolean)? = null, onProgress: ((needSoFar: Int, downloaded: Int) -> Unit)? = null, onEvent: suspend (Event) -> Unit, ): NegentropyFanOutResult { @@ -91,6 +98,7 @@ suspend fun negentropySyncFanOut( val need = AtomicInt(0) val have = AtomicInt(0) + val skipped = AtomicInt(0) val windows = AtomicInt(0) val used = AtomicInt(0) var downloaded = 0 @@ -155,6 +163,7 @@ suspend fun negentropySyncFanOut( need.addAndFetch(it) }, onHave = { have.addAndFetch(it) }, + gate = NeedGate(wantId) { skipped.addAndFetch(it) }, sendNeedBatch = { batch -> idBatches.send(batch) }, sendHaveBatch = if (localEntries.isEmpty() && localIndex == null) null else { _ -> }, ) @@ -192,6 +201,7 @@ suspend fun negentropySyncFanOut( downloaded = downloaded, windows = windows.load(), connections = used.load(), + skipped = skipped.load(), ) } diff --git a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropySyncExt.kt b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropySyncExt.kt index 51941e1856..570f56c147 100644 --- a/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropySyncExt.kt +++ b/quartz/src/commonMain/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NostrClientNegentropySyncExt.kt @@ -77,6 +77,18 @@ class NegentropySyncResult( val downloaded: Int, val windows: Int, val peerCap: Long? = null, + /** + * Ids the reconcile named that `wantId` declined, so no `REQ` was ever + * issued for them. Always `0` when no predicate was passed. + * + * Reported apart from [downloaded] and from [needCount] on purpose: + * [needCount] stays the honest protocol diff (what the relay has that the + * local set lacks) whether or not the caller chose to fetch it, and a + * skipped id was never a download. Folding either way would leave a caller + * unable to tell "my predicate is doing nothing" from "my predicate is + * eating everything" — and both are silent. + */ + val skipped: Int = 0, ) /** @@ -151,6 +163,26 @@ class NegentropySyncResult( * `limitation.max_subscriptions`; e.g. strfry defaults to 20) and exceeding the * cap can wedge the connection, not just fail the extra REQ — size the two knobs * to fit the target relay. + * @param wantId optional gate consulted for every id the reconcile names, + * BEFORE the `REQ` that would download it. Return `false` and the id is dropped + * from the fetch queue and counted in [NegentropySyncResult.skipped]; the + * reconcile itself is untouched, so [NegentropySyncResult.needCount] still + * reports the true diff. Called from the reconciler coroutines (possibly + * several at once when `reconcileConcurrency > 1`), so it must be cheap and + * thread-safe — a membership test, not a query. + * + * The case this exists for: a caller whose store will refuse an id no matter + * how often it arrives. A mirror that keeps only the newest version of a + * replaceable event is offered every relay's older copy on every sync, and + * without a hook here the only place to decline is after the body is already + * on the wire. `onEvent` is too late to save the bytes. + * + * **It does not cover a window handed to [onUnreconcilableWindow].** That + * window is drained by the caller over `REQ`, and a `REQ` names no ids before + * it streams bodies, so declined events in such a window arrive anyway and are + * not counted in [NegentropySyncResult.skipped]. Rare — it takes a single + * second denser than the relay's cap — but a caller treating the gate as an + * absolute bound on what it can receive would be wrong. * @param onProgress optional `(needSoFar, downloaded)` ticks as work proceeds. * @param onEvent called once per distinct event, serially, from the single * delivery consumer coroutine (not the relay reader thread) — so it never overlaps @@ -170,10 +202,12 @@ suspend fun INostrClient.negentropySync( localIndex: NegentropyLocalIndex? = null, targetWindow: Int = 0, onUnreconcilableWindow: (suspend (Filter) -> Unit)? = null, + wantId: ((HexKey) -> Boolean)? = null, onProgress: ((needSoFar: Int, downloaded: Int) -> Unit)? = null, onEvent: suspend (Event) -> Unit, ): NegentropySyncResult { val need = AtomicInt(0) + val skipped = AtomicInt(0) val windows = AtomicInt(0) var downloaded = 0 var peerCap: Long? = null @@ -213,6 +247,8 @@ suspend fun INostrClient.negentropySync( // single consumer loop below so the user callback is never // invoked from two coroutines at once. onNeed = { need.addAndFetch(it) }, + onSkipped = { skipped.addAndFetch(it) }, + wantId = wantId, deliver = { events.send(it) }, ) } finally { @@ -241,6 +277,7 @@ suspend fun INostrClient.negentropySync( downloaded = downloaded, windows = windows.load(), peerCap = peerCap, + skipped = skipped.load(), ) } @@ -257,6 +294,7 @@ suspend fun INostrClient.negentropySync( localIndex: NegentropyLocalIndex? = null, targetWindow: Int = 0, onUnreconcilableWindow: (suspend (Filter) -> Unit)? = null, + wantId: ((HexKey) -> Boolean)? = null, onProgress: ((needSoFar: Int, downloaded: Int) -> Unit)? = null, onEvent: suspend (Event) -> Unit, ): NegentropySyncResult = @@ -273,6 +311,7 @@ suspend fun INostrClient.negentropySync( localIndex = localIndex, targetWindow = targetWindow, onUnreconcilableWindow = onUnreconcilableWindow, + wantId = wantId, onProgress = onProgress, onEvent = onEvent, ) @@ -342,6 +381,14 @@ suspend fun INostrClient.negentropySyncOrFetch( localEntries: List = emptyList(), localIndex: NegentropyLocalIndex? = null, targetWindow: Int = 0, + /** + * Passed straight to [negentropySync]. Note it does NOT apply to the paged + * fallback: a `REQ` names no ids before it streams bodies, so there is + * nothing to gate there. A caller relying on this to bound its downloads + * should read [NegentropyOrFetchResult.pagedFallback] and expect the + * suppressed ids to arrive after all when a window pages. + */ + wantId: ((HexKey) -> Boolean)? = null, onProgress: ((needSoFar: Int, downloaded: Int) -> Unit)? = null, onEvent: suspend (Event) -> Unit, ): NegentropyOrFetchResult { @@ -397,6 +444,7 @@ suspend fun INostrClient.negentropySyncOrFetch( if (accept(event)) onProgress?.invoke(delivered, delivered) } }, + wantId = wantId, onProgress = onProgress, ) { accept(it) } NegentropyOrFetchResult( @@ -440,6 +488,7 @@ suspend fun INostrClient.negentropySyncOrFetch( localEntries: List = emptyList(), localIndex: NegentropyLocalIndex? = null, targetWindow: Int = 0, + wantId: ((HexKey) -> Boolean)? = null, onProgress: ((needSoFar: Int, downloaded: Int) -> Unit)? = null, onEvent: suspend (Event) -> Unit, ): NegentropyOrFetchResult = @@ -455,10 +504,53 @@ suspend fun INostrClient.negentropySyncOrFetch( localEntries = localEntries, localIndex = localIndex, targetWindow = targetWindow, + wantId = wantId, onProgress = onProgress, onEvent = onEvent, ) +/** + * Decides which of the ids a reconcile named are actually worth a `REQ`. + * + * Small and separate because it is the only place in the download path where + * an id can still be declined for free, and because the three things it does + * are each easy to get wrong in a lambda nobody can call from a test: apply + * the predicate, count what was dropped, and refuse to emit an empty batch. + * + * [keep] runs on the reconciler coroutines, so with `reconcileConcurrency > 1` + * it is called concurrently — hence the counting goes through [onSkipped], + * which the caller makes atomic, rather than a field here. + * + * **It is applied to a whole reconcile round's ids, before they are chunked + * into fetch batches.** Gating after the chunking instead would keep the batch + * COUNT and shrink every one of them: at the density this exists for (a mirror + * declining most of what it is offered) a `fetchBatch` of 500 would become a + * hundred `REQ`s of five ids apiece, each with its own EOSE round trip — + * turning a bandwidth saving into a latency regression. + */ +internal class NeedGate( + private val wantId: ((HexKey) -> Boolean)?, + private val onSkipped: (Int) -> Unit, +) { + /** + * The subset of [batch] to download. Empty when everything was declined — + * deliberately NOT null: the caller's chunk loop already does nothing with + * an empty list, and a nullable return here invites + * `gate?.keep(ids) ?: ids`, which reads as "no gate, keep everything" and + * silently means "everything was declined, so send everything". That exact + * elvis collapsed the fully-declining case into a full download once. + * + * With no predicate this returns [batch] itself — the unfiltered sync is + * the common case and must not pay a copy for a feature it is not using. + */ + fun keep(batch: List): List { + val wanted = if (wantId == null) batch else batch.filter(wantId) + val dropped = batch.size - wanted.size + if (dropped > 0) onSkipped(dropped) + return wanted + } +} + /** * The whole-sync pipeline: a single pool of [maxConcurrentReqs] download workers * fed by up to [reconcileConcurrency] concurrent window reconciliations through a @@ -494,6 +586,8 @@ private suspend fun INostrClient.syncPipeline( targetWindow: Int, onWindow: () -> Unit, onNeed: (Int) -> Unit, + onSkipped: (Int) -> Unit, + wantId: ((HexKey) -> Boolean)?, onPeerCap: ((Long) -> Unit)?, onUnreconcilableWindow: (suspend (Filter) -> Unit)?, deliver: suspend (Event) -> Unit, @@ -526,6 +620,9 @@ private suspend fun INostrClient.syncPipeline( onHave = {}, onPeerCap = onPeerCap, onUnreconcilableWindow = onUnreconcilableWindow, + // The gate is applied inside the reconcile, before these batches are + // cut — see NeedGate — so by here every id is one we want. + gate = NeedGate(wantId, onSkipped), sendNeedBatch = { batch -> idBatches.send(batch) }, sendHaveBatch = null, ) @@ -583,6 +680,9 @@ internal suspend fun reconcileWindows( // that hit the window, so a slow drain holds that reconciler — with // reconcileConcurrency = 1 the rest of the sweep waits for it. onUnreconcilableWindow: (suspend (Filter) -> Unit)? = null, + // Applied to each round's need ids before they are chunked. Null for the + // callers that hand the ids straight to their own consumer. + gate: NeedGate? = null, sendNeedBatch: suspend (List) -> Unit, sendHaveBatch: (suspend (List) -> Unit)?, ) = coroutineScope { @@ -703,6 +803,7 @@ internal suspend fun reconcileWindows( fetchBatch = batchSize, onNeed = onNeed, onHave = onHave, + gate = gate, sendNeedBatch = sendNeedBatch, sendHaveBatch = sendHaveBatch, ) @@ -1032,6 +1133,7 @@ private suspend fun INostrClient.reconcileStreaming( fetchBatch: Int, onNeed: (Int) -> Unit, onHave: (Int) -> Unit, + gate: NeedGate? = null, sendNeedBatch: suspend (List) -> Unit, sendHaveBatch: (suspend (List) -> Unit)?, ): ReconcileOutcome { @@ -1170,13 +1272,19 @@ private suspend fun INostrClient.reconcileStreaming( val result = session.processMessage(frame.payload) val needIds = result.needIds if (needIds.isNotEmpty()) { + // Counted before the gate: this is the protocol diff, and + // it is true whether or not the caller wants to fetch it. onNeed(needIds.size) + // Gated before the chunking, so a selective predicate + // yields FEWER full batches rather than the same number + // of nearly-empty ones — see NeedGate. + val wanted = if (gate == null) needIds else gate.keep(needIds) var i = 0 - while (i < needIds.size) { - val end = min(i + fetchBatch, needIds.size) + while (i < wanted.size) { + val end = min(i + fetchBatch, wanted.size) // Copy each batch so the frame's full id list can be freed // as soon as it is chunked; suspends under back-pressure. - sendNeedBatch(ArrayList(needIds.subList(i, end))) + sendNeedBatch(ArrayList(wanted.subList(i, end))) i = end } } diff --git a/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NeedGateTest.kt b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NeedGateTest.kt new file mode 100644 index 0000000000..f404e96887 --- /dev/null +++ b/quartz/src/commonTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/client/accessories/NeedGateTest.kt @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.quartz.nip01Core.relay.client.accessories + +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertSame +import kotlin.test.assertTrue + +/** + * The one point on the download path where an id can still be declined for + * free. Everything asserted here is a property the inline version of this code + * could break silently: a sync with no predicate must not start allocating, a + * fully-declined batch must not become an empty REQ, and the skip count must + * stay separate from the reconcile's own diff so an operator can tell a + * predicate that does nothing from one that eats everything. + */ +class NeedGateTest { + private fun id(n: Int): HexKey = n.toString().padStart(64, '0') + + private fun batch(range: IntRange) = range.map(::id) + + private class Skips { + var total = 0 + var calls = 0 + + val sink: (Int) -> Unit = { + total += it + calls++ + } + } + + @Test + fun `with no predicate the batch passes through untouched and uncopied`() { + // The unfiltered sync is the common case and must not pay a copy per + // batch for a feature it is not using. + val skips = Skips() + val input = batch(1..500) + val kept = NeedGate(null, skips.sink).keep(input) + + assertSame(input, kept, "an unfiltered batch must be the same list instance, not a copy") + assertEquals(0, skips.total) + assertEquals(0, skips.calls, "nothing was dropped so the counter should not even be touched") + } + + @Test + fun `a predicate keeps its subset in order`() { + val skips = Skips() + val wanted = setOf(id(2), id(4), id(6)) + val kept = NeedGate({ it in wanted }, skips.sink).keep(batch(1..6)) + + assertEquals(listOf(id(2), id(4), id(6)), kept) + assertEquals(3, skips.total) + } + + @Test + fun `a fully declined batch yields an empty list and never null`() { + // Non-null on purpose. A nullable return invites + // `gate?.keep(ids) ?: ids`, which reads as "no gate, keep everything" + // and silently means "everything declined, so send everything" — a + // fully-declining gate turning into a full download. + val skips = Skips() + val kept = NeedGate({ false }, skips.sink).keep(batch(1..10)) + + assertTrue(kept.isEmpty(), "nothing survived, so there is nothing to send") + assertEquals(10, skips.total) + } + + @Test + fun `an empty input yields empty and counts nothing`() { + val skips = Skips() + assertTrue(NeedGate({ true }, skips.sink).keep(emptyList()).isEmpty()) + assertEquals(0, skips.total) + } + + @Test + fun `a predicate that accepts everything drops nothing`() { + val skips = Skips() + val kept = NeedGate({ true }, skips.sink).keep(batch(1..20)) + + assertEquals(20, kept.size) + assertEquals(0, skips.total) + assertEquals(0, skips.calls) + } + + @Test + fun `skips accumulate across batches`() { + // One gate spans a whole sync, so the count has to survive more than + // the batch it was produced in. + val skips = Skips() + val gate = NeedGate({ it.endsWith("1") }, skips.sink) + gate.keep(batch(1..10)) + gate.keep(batch(11..20)) + + assertEquals(18, skips.total, "only ids 1 and 11 of the twenty end in 1") + } + + @Test + fun `the predicate sees every id in the batch exactly once`() { + val seen = mutableListOf() + NeedGate({ + seen.add(it) + true + }, Skips().sink).keep(batch(1..5)) + + assertEquals(batch(1..5), seen) + } + + @Test + fun `a sync result reports no skips unless a predicate declined something`() { + // Back-compat: every existing caller constructs this without the new + // field and must keep reading zero. + val result = NegentropySyncResult(needCount = 7, haveCount = 0, downloaded = 7, windows = 1) + + assertEquals(0, result.skipped) + assertTrue(result.needCount == 7, "the reconcile diff is unaffected by the gate") + } +} diff --git a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientNegentropySyncTest.kt b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientNegentropySyncTest.kt index 6f4b121146..4524099b2c 100644 --- a/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientNegentropySyncTest.kt +++ b/quartz/src/jvmAndroidTest/kotlin/com/vitorpamplona/quartz/nip01Core/relay/NostrClientNegentropySyncTest.kt @@ -360,6 +360,123 @@ class NostrClientNegentropySyncTest : RelayClientTest() { } } + /** + * `wantId` declines ids BEFORE the download REQ, so the events never cross + * the wire — the point of putting the hook here rather than at `onEvent`. + * + * The assertion that matters is the one on the recorded `REQ` filters: a + * gate that merely dropped events after delivery would satisfy every other + * check in this test while saving nothing. + */ + @Test + fun wantIdSkipsIdsBeforeTheyAreEverRequested() = + runBlocking { + // Every id the relay was actually asked for, straight off the wire. + val requested = java.util.Collections.synchronizedList(mutableListOf()) + val recording = + object : PassThroughPolicy() { + override fun accept(cmd: ReqCmd): PolicyResult { + cmd.filters.forEach { f -> f.ids?.let { requested.addAll(it) } } + return PolicyResult.Accepted(cmd) + } + } + + val hub = InProcessRelays(defaultPolicy = { recording }) + val scope = CoroutineScope(Dispatchers.Default + SupervisorJob()) + val client = NostrClient(hub, scope) + try { + val url = RelayUrlNormalizer.normalize("ws://127.0.0.1:7791/") + val events = (1..20).map { SyntheticEvents.fakeEvent(idSeed = it, kind = 1) } + hub.getOrCreate(url).preload(events) + + // Decline the first ten by id. + val unwanted = events.take(10).map { it.id }.toSet() + + val got = mutableListOf() + val result = + withTimeout(30_000) { + client.negentropySync( + relay = url, + filter = Filter(kinds = listOf(1)), + idleTimeoutMs = 10_000L, + wantId = { it !in unwanted }, + ) { got.add(it) } + } + + assertEquals(10, got.size, "only the wanted half is delivered") + assertTrue(got.none { it.id in unwanted }, "no declined event was delivered") + assertEquals(10, result.downloaded) + assertEquals(10, result.skipped, "the declined ids are reported apart from the download count") + assertEquals( + 20, + result.needCount, + "needCount stays the honest protocol diff — the relay really did have all 20 that we lacked", + ) + + // The whole point: the declined ids were never asked for. + assertTrue( + requested.none { it in unwanted }, + "a declined id must never reach a REQ; requested = ${requested.filter { it in unwanted }}", + ) + // Every wanted id WAS asked for — so the gate declined the right + // half rather than simply starving the download. Asserted as a + // subset rather than a count because negentropySync also opens a + // keep-alive subscription carrying a sentinel id. + assertTrue( + requested.containsAll(events.drop(10).map { it.id }), + "every wanted id should still have been requested", + ) + } finally { + client.disconnect() + scope.cancel() + hub.close() + } + } + + /** With no `wantId` nothing changes: every id is fetched and `skipped` is 0. */ + @Test + fun withoutWantIdEveryIdIsStillRequested() = + runBlocking { + defaultRelay.preload(SyntheticEvents.batch(12, kind = 1)) + + val got = mutableListOf() + val result = + withTimeout(20_000) { + client.negentropySync( + relay = defaultRelayUrl, + filter = Filter(kinds = listOf(1)), + ) { got.add(it) } + } + + assertEquals(12, got.size) + assertEquals(0, result.skipped, "no predicate means nothing is ever skipped") + } + + /** + * A gate that declines everything must finish cleanly rather than hang: the + * empty batches are dropped instead of being queued as REQs for no ids. + */ + @Test + fun wantIdDecliningEverythingDownloadsNothingAndStillCompletes() = + runBlocking { + defaultRelay.preload(SyntheticEvents.batch(15, kind = 1)) + + val got = mutableListOf() + val result = + withTimeout(20_000) { + client.negentropySync( + relay = defaultRelayUrl, + filter = Filter(kinds = listOf(1)), + wantId = { false }, + ) { got.add(it) } + } + + assertTrue(got.isEmpty(), "nothing was wanted, so nothing is delivered") + assertEquals(0, result.downloaded) + assertEquals(15, result.skipped) + assertEquals(15, result.needCount, "the reconcile still saw the full diff") + } + /** * On a relay that reconciles fine, [negentropySyncOrFetch] uses negentropy and * does not page. From 9ede348915062c5a7434b6a0af8ce0bf9eb48b7a Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Mon, 10 Aug 2026 17:57:12 -0400 Subject: [PATCH 123/132] fix(relays): seed search/indexer relay flows with the defaults, not empty MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both SearchRelayListState.flow and IndexerRelayListState.flow are supposed to never be empty: normalizeXxxWithBackup() substitutes the curated default set both when the account has no kind:10007 / kind:10086 and when the one it has decodes to zero relays (`?.ifEmpty { null } ?: Default…`). Every REQ assembler that reads `.flow` relies on that. The `stateIn` seed was the one value contradicting it. `flowOn(Dispatchers.IO)` means the first real emission can never be synchronous with `stateIn`, so `.value` was `emptySet()` until the collector ran — and for a NIP-46 signer whose list carries private entries that wait is a remote decrypt round trip, so the window is unbounded rather than sub-millisecond. A reader landing in that window queries nothing at all. AmethystAppFunctions (the system-assistant entry point, invoked cold) is the realistic case — it does `if (relays.isEmpty()) return SearchProfilesResult.empty()` directly under a comment asserting the flow "already resolves to a concrete relay set … or the curated default set", which is exactly the invariant that did not hold yet. Seeding with the defaults makes "never empty" true for the whole lifetime of the flow. Only `.flow` changes; `.flowNoDefaults` still seeds empty, so the editing and diffing paths (SearchRelayListViewModel, IndexerRelayListViewModel, AddInboxRelayForSearchCard, TrustedRelayListsState, Account.saveRelayList's diff) still see what the user actually configured. Consequence, and the intended trade: a reader in that window now queries the default relays instead of silently querying none. Co-Authored-By: Claude Opus 5 (1M context) --- .../indexerRelays/IndexerRelayListState.kt | 13 ++++++++++++- .../searchRelays/SearchRelayListState.kt | 15 ++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/nip51Lists/indexerRelays/IndexerRelayListState.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/nip51Lists/indexerRelays/IndexerRelayListState.kt index 40f30c1dbb..fcc0647212 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/nip51Lists/indexerRelays/IndexerRelayListState.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/nip51Lists/indexerRelays/IndexerRelayListState.kt @@ -62,6 +62,17 @@ class IndexerRelayListState( suspend fun normalizeIndexerRelayListWithBackupNoDefaults(note: Note): Set = indexListEvent(note)?.let { decryptionCache.relays(it) } ?: emptySet() + /** + * The account's indexer relays, **never empty** — [normalizeIndexerRelayListWithBackup] + * substitutes [DefaultIndexerRelayList] both when there is no kind:10086 and when the + * one we have decodes to zero relays. Callers assembling metadata / relay-list REQs read + * this and can rely on getting a usable set; use [flowNoDefaults] instead to show or diff + * what the user actually configured. + * + * Seeded with [DefaultIndexerRelayList] rather than `emptySet()`, for the same reason as + * the search list: `flowOn(IO)` makes the first real emission asynchronous, so an + * `emptySet()` seed left a window where `.value` contradicted the contract above. + */ val flow = getIndexerRelayListFlow() .map { normalizeIndexerRelayListWithBackup(it.note) } @@ -70,7 +81,7 @@ class IndexerRelayListState( .stateIn( scope, SharingStarted.Eagerly, - emptySet(), + DefaultIndexerRelayList, ) val flowNoDefaults = diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/nip51Lists/searchRelays/SearchRelayListState.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/nip51Lists/searchRelays/SearchRelayListState.kt index 3a227ef306..4d690210e3 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/nip51Lists/searchRelays/SearchRelayListState.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/nip51Lists/searchRelays/SearchRelayListState.kt @@ -62,6 +62,19 @@ class SearchRelayListState( suspend fun normalizeSearchRelayListWithBackupNoDefaults(note: Note): Set = searchListEvent(note)?.let { decryptionCache.relays(it) } ?: emptySet() + /** + * The account's search relays, **never empty** — [normalizeSearchRelayListWithBackup] + * substitutes [DefaultSearchRelayList] both when there is no kind:10007 and when the + * one we have decodes to zero relays. Callers assembling NIP-50 REQs read this and can + * rely on getting a usable set; use [flowNoDefaults] instead to show or diff what the + * user actually configured. + * + * Seeded with [DefaultSearchRelayList] rather than `emptySet()`: `flowOn(IO)` means the + * first real emission can never be synchronous with `stateIn`, so an `emptySet()` seed + * left a window where `.value` contradicted the "never empty" contract above and search + * silently queried nothing. That window is unbounded for a NIP-46 signer whose list has + * private entries, since the first emission waits on a remote decrypt. + */ val flow = getSearchRelayListFlow() .map { normalizeSearchRelayListWithBackup(it.note) } @@ -70,7 +83,7 @@ class SearchRelayListState( .stateIn( scope, SharingStarted.Eagerly, - emptySet(), + DefaultSearchRelayList, ) val flowNoDefaults = From 422ae2d2d6f7bfd2da8a5d559b471a39afdb26e6 Mon Sep 17 00:00:00 2001 From: vitorpamplona <532031+vitorpamplona@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:24:27 +0000 Subject: [PATCH 124/132] chore: sync Crowdin translations and seed translator npub placeholders --- amethyst/src/main/res/values-cs/strings.xml | 2 ++ amethyst/src/main/res/values-de-rDE/strings.xml | 16 ++++++++++++++++ amethyst/src/main/res/values-pt-rBR/strings.xml | 16 ++++++++++++++++ amethyst/src/main/res/values-sv-rSE/strings.xml | 2 ++ docs/changelog/translators.json | 2 ++ 5 files changed, 38 insertions(+) diff --git a/amethyst/src/main/res/values-cs/strings.xml b/amethyst/src/main/res/values-cs/strings.xml index 2fce7e78c2..358ff5201b 100644 --- a/amethyst/src/main/res/values-cs/strings.xml +++ b/amethyst/src/main/res/values-cs/strings.xml @@ -3977,6 +3977,8 @@ Sdílet s vývojáři Pokud se zdá, že Amethyst spotřebovává baterii nebo data, můžete tento report odeslat vývojářům v šifrované DM. Obsahuje pouze čísla na této obrazovce a technická počítadla za nimi — žádné příspěvky, kontakty ani podrobnosti o prohlížení. Nic se neodešle, dokud v obrazovce zprávy neklepnete na Odeslat. Odeslat report přes DM + Kopírovat + Sdílet Zjištěno vysoké využití prostředků Amethyst nedávno spotřeboval více, než se očekávalo: %1$s. Chcete vývojářům odeslat report o využití v šifrované DM? Před odesláním čehokoli uvidíte celý report. %1$s mobilních dat na pozadí za jeden den diff --git a/amethyst/src/main/res/values-de-rDE/strings.xml b/amethyst/src/main/res/values-de-rDE/strings.xml index b8f2041354..d97a88f44c 100644 --- a/amethyst/src/main/res/values-de-rDE/strings.xml +++ b/amethyst/src/main/res/values-de-rDE/strings.xml @@ -1949,6 +1949,7 @@ Stöbern Medien + Hashtags Themen Unterhaltung Suche @@ -1979,8 +1980,10 @@ Standortbasierte Räume für die Gebiete, denen du folgst, abgefragt bei den Relays, die sie führen. Chat und Zap-Ziele, die an Live-Streams hängen, die du geöffnet hast oder denen du folgst. DM-Posteingang + Wallet Nutzap-Posteingang Mint-Verzeichnis + Wallet Connect Community-Chats Community-Feeds + Backlog Workflow-Läufe Agentenarbeit Neuer Lauf @@ -4646,6 +4659,7 @@ Benötigt deine Genehmigung Wartet auf Genehmigung (keine Beschreibung) + Workflow: %1$s von wartet auf Du bist der Genehmigende, aber diese Anmeldung kann keine Entscheidung signieren. @@ -4681,11 +4695,13 @@ Noch keine Workflows. Öffne das Menü oben und wähle „Neue Definition…“, um einen zu erstellen, und löse ihn dann aus. Was soll er tun? Lauf auslösen + Workflow Noch keine Workflows definiert Workflow auswählen Neue Definition… Neue Workflow-Definition Benennt ihn für den Kanal und veröffentlicht sein YAML-Rezept (kind-30620). Ein echtes Buzz-Relay führt das YAML aus. Selbst gehostet führt der Runner seinen konfigurierten Befehl aus — hier benennt und katalogisiert die Definition den Lauf nur. + Name build-und-test YAML-Rezept Die Definition konnte nicht veröffentlicht werden — prüfe, ob du in diesem Workspace posten kannst. diff --git a/amethyst/src/main/res/values-pt-rBR/strings.xml b/amethyst/src/main/res/values-pt-rBR/strings.xml index 1ceae75a2e..a7309a0c0e 100644 --- a/amethyst/src/main/res/values-pt-rBR/strings.xml +++ b/amethyst/src/main/res/values-pt-rBR/strings.xml @@ -1941,8 +1941,13 @@ + + %1$s \u00b7 %2$d relé + %1$s \u00b7 %2$d relés + Navegação Mídia + Hashtags Tópicos Conversa Pesquisa @@ -1976,6 +1981,7 @@ Carteira Caixa de entrada de nutzaps Diretório de mints + Wallet Connect Chats de comunidades Feeds de comunidades + Backlog Execuções de fluxo de trabalho Trabalho do agente Nova execução diff --git a/amethyst/src/main/res/values-sv-rSE/strings.xml b/amethyst/src/main/res/values-sv-rSE/strings.xml index 59ec28bf1d..bbef7915c4 100644 --- a/amethyst/src/main/res/values-sv-rSE/strings.xml +++ b/amethyst/src/main/res/values-sv-rSE/strings.xml @@ -3839,6 +3839,8 @@ Dela med utvecklarna Om Amethyst verkar dra batteri eller data kan du skicka den här rapporten till utvecklarna i ett krypterat DM. Den innehåller bara siffrorna på den här skärmen och de tekniska räknarna bakom dem — inga inlägg, kontakter eller surfdetaljer. Ingenting skickas förrän du trycker på Skicka i meddelandeskärmen. Skicka rapport via DM + Kopiera + Dela Hög resursanvändning upptäckt Amethyst förbrukade mer än väntat nyligen: %1$s. Vill du skicka en användningsrapport till utvecklarna i ett krypterat DM? Du får se hela rapporten innan något skickas. %1$s mobildata i bakgrunden på en dag diff --git a/docs/changelog/translators.json b/docs/changelog/translators.json index 0697190820..6364d560b3 100644 --- a/docs/changelog/translators.json +++ b/docs/changelog/translators.json @@ -121,6 +121,8 @@ "user": "davotoula", "languages": [ "Czech", + "German", + "Portuguese, Brazilian", "Swedish" ] }, From 59b36592f344dc9fc4067cca5a3c0a26d61fe49b Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Mon, 10 Aug 2026 18:29:02 -0400 Subject: [PATCH 125/132] fix(relays): seed relay flows from precached tags, not just the defaults MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the previous commit, which seeded SearchRelayListState.flow and IndexerRelayListState.flow with the curated default sets so `.value` is never empty before the first async emission. Seeding with the defaults fixed the "queries nothing" hole but was blunt: an account with its own relays would briefly advertise the defaults instead. Seed from the precached resolution instead: searchListEvent(note)?.let { decryptionCache.cachedRelays(it) } ?.ifEmpty { null } ?: DefaultSearchRelayList `cachedRelays` is the non-suspending sibling of `relays` — public tags plus any already-decrypted private tags, and it never asks the signer, so it cannot block or trigger a NIP-46 round trip from a property initializer. At login the note's event is usually still null, so this resolves through settings.backupXxxRelayList (restored from LocalPreferences) and an account with public relays gets its own relays immediately. Only accounts whose relays are exclusively private still see the defaults for the window, and they get a working set rather than nothing. Same shape as the suspend normalizer, so all three arms still hold: no list → defaults, list with zero relays → defaults, list with relays → those relays. PrecachedRelayListSeedTest pins the property the refinement depends on — that public relays are readable with no signer involvement — plus the empty-list arm that hands over to the defaults, and a foreign-author read. Co-Authored-By: Claude Opus 5 (1M context) --- .../indexerRelays/IndexerRelayListState.kt | 17 +++- .../searchRelays/SearchRelayListState.kt | 18 +++- .../nip51Lists/PrecachedRelayListSeedTest.kt | 91 +++++++++++++++++++ 3 files changed, 120 insertions(+), 6 deletions(-) create mode 100644 amethyst/src/test/java/com/vitorpamplona/amethyst/model/nip51Lists/PrecachedRelayListSeedTest.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/nip51Lists/indexerRelays/IndexerRelayListState.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/nip51Lists/indexerRelays/IndexerRelayListState.kt index fcc0647212..1491bee591 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/nip51Lists/indexerRelays/IndexerRelayListState.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/nip51Lists/indexerRelays/IndexerRelayListState.kt @@ -62,6 +62,17 @@ class IndexerRelayListState( suspend fun normalizeIndexerRelayListWithBackupNoDefaults(note: Note): Set = indexListEvent(note)?.let { decryptionCache.relays(it) } ?: emptySet() + /** + * Same resolution as [normalizeIndexerRelayListWithBackup] but non-suspending, for use as the + * [flow] seed. Reads the event's public tags plus any *already decrypted* private tags; it + * never asks the signer, so it cannot block or hit a NIP-46 round trip. + * + * At login `indexerListNote.event` is usually still null and this resolves through + * `settings.backupIndexRelayList`, restored from LocalPreferences — so an account with public + * indexer relays gets its own relays immediately instead of the defaults. + */ + fun normalizeIndexerRelayListPrecached(note: Note): Set = indexListEvent(note)?.let { decryptionCache.cachedRelays(it) }?.ifEmpty { null } ?: DefaultIndexerRelayList + /** * The account's indexer relays, **never empty** — [normalizeIndexerRelayListWithBackup] * substitutes [DefaultIndexerRelayList] both when there is no kind:10086 and when the @@ -69,8 +80,8 @@ class IndexerRelayListState( * this and can rely on getting a usable set; use [flowNoDefaults] instead to show or diff * what the user actually configured. * - * Seeded with [DefaultIndexerRelayList] rather than `emptySet()`, for the same reason as - * the search list: `flowOn(IO)` makes the first real emission asynchronous, so an + * Seeded via [normalizeIndexerRelayListPrecached] rather than `emptySet()`, for the same + * reason as the search list: `flowOn(IO)` makes the first real emission asynchronous, so an * `emptySet()` seed left a window where `.value` contradicted the contract above. */ val flow = @@ -81,7 +92,7 @@ class IndexerRelayListState( .stateIn( scope, SharingStarted.Eagerly, - DefaultIndexerRelayList, + normalizeIndexerRelayListPrecached(indexerListNote), ) val flowNoDefaults = diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/nip51Lists/searchRelays/SearchRelayListState.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/nip51Lists/searchRelays/SearchRelayListState.kt index 4d690210e3..eb3714dc5c 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/nip51Lists/searchRelays/SearchRelayListState.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/nip51Lists/searchRelays/SearchRelayListState.kt @@ -62,6 +62,18 @@ class SearchRelayListState( suspend fun normalizeSearchRelayListWithBackupNoDefaults(note: Note): Set = searchListEvent(note)?.let { decryptionCache.relays(it) } ?: emptySet() + /** + * Same resolution as [normalizeSearchRelayListWithBackup] but non-suspending, for use as the + * [flow] seed. Reads the event's public tags plus any *already decrypted* private tags; it + * never asks the signer, so it cannot block or hit a NIP-46 round trip. + * + * At login `searchListNote.event` is usually still null and this resolves through + * `settings.backupSearchRelayList`, restored from LocalPreferences — so an account with public + * search relays gets its own relays immediately instead of the defaults. Accounts whose relays + * are exclusively private fall back to [DefaultSearchRelayList] until the first decrypt lands. + */ + fun normalizeSearchRelayListPrecached(note: Note): Set = searchListEvent(note)?.let { decryptionCache.cachedRelays(it) }?.ifEmpty { null } ?: DefaultSearchRelayList + /** * The account's search relays, **never empty** — [normalizeSearchRelayListWithBackup] * substitutes [DefaultSearchRelayList] both when there is no kind:10007 and when the @@ -69,8 +81,8 @@ class SearchRelayListState( * rely on getting a usable set; use [flowNoDefaults] instead to show or diff what the * user actually configured. * - * Seeded with [DefaultSearchRelayList] rather than `emptySet()`: `flowOn(IO)` means the - * first real emission can never be synchronous with `stateIn`, so an `emptySet()` seed + * Seeded via [normalizeSearchRelayListPrecached] rather than `emptySet()`: `flowOn(IO)` means + * the first real emission can never be synchronous with `stateIn`, so an `emptySet()` seed * left a window where `.value` contradicted the "never empty" contract above and search * silently queried nothing. That window is unbounded for a NIP-46 signer whose list has * private entries, since the first emission waits on a remote decrypt. @@ -83,7 +95,7 @@ class SearchRelayListState( .stateIn( scope, SharingStarted.Eagerly, - DefaultSearchRelayList, + normalizeSearchRelayListPrecached(searchListNote), ) val flowNoDefaults = diff --git a/amethyst/src/test/java/com/vitorpamplona/amethyst/model/nip51Lists/PrecachedRelayListSeedTest.kt b/amethyst/src/test/java/com/vitorpamplona/amethyst/model/nip51Lists/PrecachedRelayListSeedTest.kt new file mode 100644 index 0000000000..89c0df7647 --- /dev/null +++ b/amethyst/src/test/java/com/vitorpamplona/amethyst/model/nip51Lists/PrecachedRelayListSeedTest.kt @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.model.nip51Lists + +import com.vitorpamplona.amethyst.model.nip51Lists.searchRelays.SearchRelayListDecryptionCache +import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair +import com.vitorpamplona.quartz.nip01Core.relay.normalizer.RelayUrlNormalizer +import com.vitorpamplona.quartz.nip01Core.signers.NostrSignerInternal +import com.vitorpamplona.quartz.nip50Search.SearchRelayListEvent +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +/** + * Guards the non-suspending seed that [SearchRelayListState.flow] is initialized with. + * + * The seed exists so `.value` is never empty before the first async emission lands (see the + * KDoc on `flow`). For it to be an improvement over just using the curated defaults, reading an + * account's *public* relays must work with no signer involvement at all — otherwise a NIP-46 + * account would still block. These tests pin that property. + */ +class PrecachedRelayListSeedTest { + private val relayA = RelayUrlNormalizer.normalize("wss://relay.example.com") + private val relayB = RelayUrlNormalizer.normalize("wss://other.example.com") + + /** + * The load-bearing claim: public relay tags are readable synchronously. `cachedRelays` is + * non-suspending, so if this returned empty the seed would silently degrade to the defaults + * for every account. + */ + @Test + fun cachedRelays_readsPublicRelaysWithoutDecrypting() = + runTest { + val signer = NostrSignerInternal(KeyPair()) + val event = SearchRelayListEvent.create(relays = listOf(relayA, relayB), signer = signer) + + val relays = SearchRelayListDecryptionCache(signer).cachedRelays(event) + + assertEquals(setOf(relayA, relayB), relays) + } + + /** + * A kind:10007 with no relays must read as empty here, so the seed's `?.ifEmpty { null }` + * arm hands over to the curated defaults rather than seeding an empty set. + */ + @Test + fun cachedRelays_emptyListReadsEmptySoTheSeedCanFallBack() = + runTest { + val signer = NostrSignerInternal(KeyPair()) + val event = SearchRelayListEvent.create(relays = emptyList(), signer = signer) + + val relays = SearchRelayListDecryptionCache(signer).cachedRelays(event) + + assertTrue(relays.isEmpty()) + } + + /** + * Reading another account's list must not blow up or leak a decrypt attempt — the private + * cache refuses to build for a foreign pubkey, so only the public tags come back. + */ + @Test + fun cachedRelays_foreignAuthorStillYieldsPublicRelays() = + runTest { + val author = NostrSignerInternal(KeyPair()) + val reader = NostrSignerInternal(KeyPair()) + val event = SearchRelayListEvent.create(relays = listOf(relayA), signer = author) + + val relays = SearchRelayListDecryptionCache(reader).cachedRelays(event) + + assertEquals(setOf(relayA), relays) + } +} From 37f9c5ad85246e7b1bca0b77d39d7972a29270b8 Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Tue, 11 Aug 2026 00:40:26 -0400 Subject: [PATCH 126/132] fix(embed): restore the keyboard on tab return, and none for readonly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Device testing on a tablet (SM-T220, Android 14) walked every text-field focus path in the embedded tab. Two of them were wrong. **The tab-return restore never fired.** `noteKeyboardOnLeave` sampled `WindowInsets.imeAnimationTarget > 0 && isMirroringPageField()` inside `onDispose`, on the assumption that the dispose runs before anything hides the IME. It does not: by the time it runs, the nav transition has already snapped the animation target to 0 *and* taken focus off the view, so both halves read false and every tab was recorded as "left without a keyboard". Instrumented on device, the leave was `keyboardUp=false mirroring=false imeBottomPx=0` for all three nav-rail routes, so `pendingRestore` was false on every return and a tab left mid-typing always came back with the keyboard down. Ask the mirror what it *intends* instead of sampling the window at teardown: `RemoteImeView.keyboardWanted` is set when we raise the keyboard and cleared when the field blurs or the user puts the keyboard away, so it still reads true while the view is being torn down. Telling "user dismissed it" apart from "the tab went away" is what that clearing needs, and there is no key hook for it — Android 13+ routes the IME's back dismissal through OnBackInvokedCallback, so `onKeyPreIme` is never called (tried first; it silently never fired and the tab over-restored). The two cases are distinguishable by what else is true when the insets collapse, measured on device: dismiss: imeBottomPx=0 hasFocus=true mirrors=true tab switch: imeBottomPx=0 hasFocus=false mirrors=false so a collapse while we still mirror the field is the dismissal, and a switch never looks like one — the focus loss lands in the same frame as the insets. **A readonly field raised a keyboard that cannot type.** `isEditable` in the shim never looked at `readOnly`, so the host took the field and showed a keyboard whose keystrokes the page discards. Native, checked side by side in the full-screen WebView on the same page, focuses a readonly field without a keyboard. The field stays "editable" for selection (native offers handles and Copy there); only the raise is suppressed, via one guard in `raiseKeyboard` so the fresh-focus, tap-doorbell and tab-restore paths are all covered. Verified on device, 27/27 checks: fresh focus raises for text/textarea/ contenteditable/email/number/password/search/tel and not for disabled or readonly; BACK-dismiss then re-tap restores; re-tapping a field whose keyboard is up keeps it; leaving mid-typing restores on return (~1s, 5/5 runs) while a dismissed tab stays down; typing after either restore lands in the right field at the right caret; page-background tap blurs; address-bar keyboard never arms an embed restore; and the full-screen round trip leaves the embed IME working. `tools/ime-test/keyboard.html` is the page those checks drive: every field type plus a live focus readout and an event log that marks taps on an already-focused field, which is the case with no DOM event of its own. Co-Authored-By: Claude Opus 5 (1M context) --- .../loggedIn/embed/EmbeddedImeBridge.kt | 7 ++ .../screen/loggedIn/embed/EmbeddedTabLayer.kt | 22 ++-- .../ui/screen/loggedIn/embed/RemoteImeView.kt | 39 ++++++ .../composeResources/files/napplet/shim.js | 5 +- tools/ime-test/keyboard.html | 119 ++++++++++++++++++ 5 files changed, 183 insertions(+), 9 deletions(-) create mode 100644 tools/ime-test/keyboard.html diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedImeBridge.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedImeBridge.kt index c49a87b2ab..bb2673f7bb 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedImeBridge.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedImeBridge.kt @@ -96,6 +96,7 @@ private fun parseFocus(o: JSONObject) = inputType = o.optString("inputType", "text"), enterKeyHint = o.optString("enterKeyHint", ""), multiline = o.optBoolean("multiline", false), + readOnly = o.optBoolean("readOnly", false), text = o.optString("text", ""), selStart = o.optInt("selStart", 0), selEnd = o.optInt("selEnd", 0), @@ -109,6 +110,12 @@ sealed interface ImeEvent { val inputType: String, val enterKeyHint: String, val multiline: Boolean, + /** + * The field is `readonly`: focusable and selectable, but not typeable. Native Chrome focuses such a + * field without raising the keyboard, so the host must not either — otherwise the user gets a keyboard + * whose keystrokes the page discards. + */ + val readOnly: Boolean = false, val text: String, val selStart: Int, val selEnd: Int, diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabLayer.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabLayer.kt index 50bc57161f..d8441ae2ff 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabLayer.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabLayer.kt @@ -348,10 +348,14 @@ fun EmbeddedTabLayer(barFavoriteIds: List) { // the selection). All the show/hide state lives in one [SelectionUiState] so the rules — toolbar hides // while dragging or scrolling, handles hide while scrolling — are expressed in one place. val sel = remember { SelectionUiState() } - // Read at dispose time to record whether the keyboard was up as the user left this tab (see - // [EmbeddedTabHost.noteKeyboardOnLeave]). The dispose runs before anything hides the IME — our own - // onPageBlur below is what takes it down — so this still reads the pre-switch state. - val keyboardUp = rememberUpdatedState(imeBottomPx > 0) + // The keyboard collapsing while this view still mirrors the page field means the user put it away on a + // field they are still on (BACK, or the IME's own hide button) — record it so returning to this tab + // doesn't pop the keyboard back over the page. A tab switch also collapses the insets but does NOT + // look like this: measured on device, the switch takes focus off the view in the same frame, so + // isMirroringPageField() is already false there and the mark this tab was owed survives. + LaunchedEffect(activeId, imeBottomPx) { + if (imeBottomPx == 0 && imeView.isMirroringPageField()) imeView.noteKeyboardDismissed() + } DisposableEffect(imeBridge) { val boundId = activeId // A warm tab keeps its page focus while parked, so returning to it fires no focus event: ask the @@ -423,10 +427,14 @@ fun EmbeddedTabLayer(barFavoriteIds: List) { // way: blurring it here would fire the page's own blur handlers — validation, autocomplete // dismissal, submit-on-blur — for a switch the user never made inside the page. // - // Only a keyboard THIS mirror holds counts: the tab's own chrome has host-side fields (the - // browser's address bar), and typing in one of those must not arm a restore for a page field. + // Ask the mirror what it *intends* rather than sampling the window: by the time this dispose + // runs, the nav transition has already snapped `WindowInsets.imeAnimationTarget` to 0 and + // taken focus off the view, so both would report "no keyboard" for every tab the user left + // mid-typing — which is exactly the case this restore exists for. [wantsKeyboardForPageField] + // also answers the other half: only a keyboard THIS mirror holds counts, so typing in the + // browser's own address bar never arms a restore for a page field. if (boundId != null) { - EmbeddedTabHost.noteKeyboardOnLeave(boundId, keyboardUp.value && imeView.isMirroringPageField()) + EmbeddedTabHost.noteKeyboardOnLeave(boundId, imeView.wantsKeyboardForPageField()) } imeView.onPageBlur() imeView.bind(null) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/RemoteImeView.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/RemoteImeView.kt index bdb55e1aa0..27be249c47 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/RemoteImeView.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/RemoteImeView.kt @@ -72,6 +72,17 @@ class RemoteImeView( // ship the PREVIOUS tab's text to the page on the first keystroke. private var mirroring = false + // Whether the keyboard is *meant* to be up for the field we mirror — our own intent, not the window's + // current state. Deliberately not derived from the IME insets or [hasFocus]: by the time a tab switch + // tears this view down, `WindowInsets.imeAnimationTarget` has already snapped to 0 and the view has + // already lost focus, so anything sampled then reports "no keyboard" for a tab the user left mid-typing. + // Set when we raise the keyboard, cleared when the user dismisses it or the page field blurs. + private var keyboardWanted = false + + // The mirrored field is `readonly`. Kept here rather than checked at each call site so every raise path — + // a fresh focus, the tap doorbell, and a tab restore — is covered by the one guard in [raiseKeyboard]. + private var fieldReadOnly = false + private val imm get() = context.getSystemService(Context.INPUT_METHOD_SERVICE) as InputMethodManager private val flush = Runnable { flushState() } @@ -170,6 +181,7 @@ class RemoteImeView( ) { configureFor(focus) mirroring = true + fieldReadOnly = focus.readOnly // Focus the EditText BEFORE seeding text/selection. An EditText jumps its caret to the end when it // gains focus; if we seed first, that end-position then overrides the seed and gets shipped to the // page — so a tap mid-text lands the caret at the end of the field. Seeding AFTER focus makes the @@ -205,6 +217,13 @@ class RemoteImeView( /** True while the keyboard this view holds belongs to a page field — see [mirroring]. */ fun isMirroringPageField() = mirroring && hasFocus() + /** + * Whether this tab should come back with its keyboard up: we mirror a page field and the keyboard was + * meant to be showing when we were asked. Safe to call while the view is being torn down, which is the + * whole point — see [keyboardWanted]. + */ + fun wantsKeyboardForPageField() = mirroring && keyboardWanted + /** * Put the keyboard back on the field this view already mirrors — the user tapped it after dismissing the * keyboard, which leaves the page's focus (and this mirror) untouched, so there is nothing to re-seed. @@ -213,11 +232,29 @@ class RemoteImeView( */ @Suppress("DEPRECATION") // InputMethodManager.SHOW_IMPLICIT is deprecated; no equivalent flag on the newer API. fun raiseKeyboard() { + // A readonly field takes focus and can be selected/copied, but nothing can be typed into it — native + // Chrome shows no keyboard for one, so neither do we. + if (fieldReadOnly) return + keyboardWanted = true post { if (hasFocus()) imm.showSoftInput(this, InputMethodManager.SHOW_IMPLICIT) } } + /** + * The user put the keyboard away (BACK, or the IME's own hide affordance) while still on this field: a + * deliberate "I'm done typing", so returning to this tab must NOT pop the keyboard back up. + * + * Called by the layer when the IME insets collapse while this view still mirrors the page field. That + * condition is what separates a dismissal from a tab switch — on a switch the view has already lost focus + * by the time the insets collapse, so [isMirroringPageField] is false and this never fires. (Note there is + * no usable key hook for this: Android 13+ routes the IME's back-dismiss through OnBackInvokedCallback, so + * `onKeyPreIme` is never called.) + */ + fun noteKeyboardDismissed() { + keyboardWanted = false + } + // When the current selection first became a range, and how many of its collapse-abandonments we've // re-asserted. Chrome abandons a selection *immediately* (~60ms); a deliberate user tap-to-collapse comes // later — so we only re-assert within a short window of the range forming, bounded for safety. @@ -256,6 +293,8 @@ class RemoteImeView( /** The page field blurred: drop the keyboard. */ fun onPageBlur() { mirroring = false + keyboardWanted = false + fieldReadOnly = false removeCallbacks(reportRangeLost) if (hadRange) { hadRange = false diff --git a/commons/src/commonMain/composeResources/files/napplet/shim.js b/commons/src/commonMain/composeResources/files/napplet/shim.js index 6afbca5e04..4ac67e1cd1 100644 --- a/commons/src/commonMain/composeResources/files/napplet/shim.js +++ b/commons/src/commonMain/composeResources/files/napplet/shim.js @@ -376,7 +376,8 @@ var inputType = isCE(n) ? 'text' : (t === 'TEXTAREA' ? 'textarea' : (n.type || 'text').toLowerCase()); var sel = selOf(n); return { type:'ime.focus', inputType: inputType, enterKeyHint: (n.enterKeyHint || ''), - multiline: multiline, text: valOf(n), selStart: sel[0], selEnd: sel[1], geom: fieldGeom(n) }; + multiline: multiline, readOnly: !!n.readOnly, text: valOf(n), selStart: sel[0], + selEnd: sel[1], geom: fieldGeom(n) }; } // Like `ime.focus`, but for a field that is ALREADY focused: the answer to the host's `ime.resync`, which // it asks for when it needs to (re-)take a field whose focus never moved in the page. @@ -390,7 +391,7 @@ return { type:'ime.refocus', inputType: isCE(n) ? 'text' : (t === 'TEXTAREA' ? 'textarea' : (n.type || 'text').toLowerCase()), enterKeyHint: (n.enterKeyHint || ''), multiline: isCE(n) || t === 'TEXTAREA', - text: valOf(n), selStart: sel[0], selEnd: sel[1] }; + readOnly: !!n.readOnly, text: valOf(n), selStart: sel[0], selEnd: sel[1] }; } // Last selection we either applied (applyState) or already reported, so the asynchronous // selectionchange our own setSel triggers doesn't echo back to the host as a fresh edit. diff --git a/tools/ime-test/keyboard.html b/tools/ime-test/keyboard.html new file mode 100644 index 0000000000..a976e0626d --- /dev/null +++ b/tools/ime-test/keyboard.html @@ -0,0 +1,119 @@ + + +Keyboard focus matrix + + +
FOCUS: (none)
+ +
+
+
+
+
hello world
+
+
+
+
+
+
+
+
+ + + + +
+ +
+ + From b7ef983bd83768b15968bebff05c237acbec494e Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Tue, 11 Aug 2026 00:51:42 -0400 Subject: [PATCH 127/132] fix(embed): no caret handle or Cut/Paste on a readonly field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Suppressing the keyboard for a readonly field was only half of it. The selection UI still treated it as fully editable: a tap-and-hold raised the insertion caret handle, and its toolbar offered Cut and Paste — on a field the page will not let you modify. Cut appeared to work in the mirror while the page kept its text, so the two silently drifted apart. Native, on the same page in the full-screen WebView, gives a readonly field selection handles and a Copy / Select-all bar, and nothing else: no caret handle (there is no caret to place) and no editing actions. Carry the flag into SelectionUiState so the overlay can reason about it: `fieldReadOnly` gates the insertion handle (and with it the Paste/Select-all popup that hangs off it), and the field toolbar drops Cut and Paste. Selection, its handles, Copy and Select-all are untouched — that half is what native offers and it works today. `cutSelection`/`pasteClipboard` refuse on a readonly field too. The toolbar no longer offers them, so this is a backstop, placed next to the ops so a future call site can't reintroduce the drift. Device-verified: readonly long-press selects with handles and shows exactly "Copy | Select all"; a plain tap gives no keyboard and no caret droplet; an editable field still shows all four actions and keeps its caret handle. Co-Authored-By: Claude Opus 5 (1M context) --- .../screen/loggedIn/embed/EmbeddedTabLayer.kt | 26 ++++++++++++++----- .../ui/screen/loggedIn/embed/RemoteImeView.kt | 7 +++-- .../screen/loggedIn/embed/SelectionUiState.kt | 16 +++++++++++- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabLayer.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabLayer.kt index d8441ae2ff..f0b00c2319 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabLayer.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/embed/EmbeddedTabLayer.kt @@ -377,6 +377,7 @@ fun EmbeddedTabLayer(barFavoriteIds: List) { // Cancel any in-flight scroll-hide from the page phase: otherwise the field's own // selection-reveal scrolls keep it armed and the new field handles/toolbar never appear. sel.scrolling = false + sel.onFieldReadOnly(event.readOnly) sel.onFieldGeometry(event.geometry, event.text.isNotEmpty()) } ImeEvent.WantKeyboard -> { @@ -398,6 +399,7 @@ fun EmbeddedTabLayer(barFavoriteIds: List) { // Re-arm "the field has text" so a tap can show the insertion handle again (a tab // switch resets it). Geometry stays null here, so this can't pop a handle up on its // own — native shows nothing until the user touches the field. + sel.onFieldReadOnly(event.focus.readOnly) sel.onFieldGeometry(null, event.focus.text.isNotEmpty()) } ImeEvent.Blur -> { @@ -529,19 +531,29 @@ fun EmbeddedTabLayer(barFavoriteIds: List) { // In-field (/