From 6f9c5bbdf05389f1e68e4cc7f205a150f54d1706 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 30 May 2026 22:05:57 +0000 Subject: [PATCH] fix: complete DM windows on quiescence, add load-entire-history button MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fixed 15s window-load timeout fired mid-flood on accounts with a large DM history: a relay streaming thousands of stored gift wraps never EOSE'd within 15s, so the window was declared "loaded" while events were still pouring in and before they were decrypted into rooms. The rooms list still looked empty, so auto-fill widened again — re-issuing an ever-wider REQ that re-downloaded the whole history, over and over, every 15s. WindowLoadTracker now completes a window on activity quiescence instead of a wall clock: it stays loading until every expected relay EOSEs, or the event stream goes quiet for a few seconds. Every event (stored backfill included) bumps the idle timer via onActivity, so a relay mid-flood is never mistaken for a finished window; an absolute cap bounds pathological dribble. Both DM loaders feed event activity in (the NIP-04 loader now uses a custom listener so it sees stored events, not just live ones). Also add a "Load entire history" button to the rooms-list footer: it jumps the window straight to the max lookback (TimeWindowPagination.loadAll) so a single REQ pulls everything — the pre-windowing behavior — and marks the window exhausted so the auto-fill loop stops. --- .../eoseManagers/WindowLoadTracker.kt | 65 ++++++++++++++----- .../AccountGiftWrapsEoseManager.kt | 17 +++++ .../DMsFromUserFilterSubAssembler.kt | 41 +++++++++++- .../chats/rooms/feed/ChatroomListFeedView.kt | 33 ++++++++-- amethyst/src/main/res/values/strings.xml | 1 + .../pagination/TimeWindowPagination.kt | 5 ++ .../pagination/TimeWindowPaginationTest.kt | 13 ++++ 7 files changed, 152 insertions(+), 23 deletions(-) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/WindowLoadTracker.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/WindowLoadTracker.kt index 5e759592fd..7a8de53219 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/WindowLoadTracker.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/WindowLoadTracker.kt @@ -27,47 +27,73 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.isActive import kotlinx.coroutines.launch import kotlin.time.Duration +import kotlin.time.Duration.Companion.minutes import kotlin.time.Duration.Companion.seconds /** - * Tracks when one relay-subscription "window" has finished loading, so callers can wait for the - * WHOLE set of relays to answer instead of declaring victory on the first EOSE. + * Tracks when one relay-subscription "window" has finished loading, so callers (the rooms-screen + * auto-fill loop) can wait for the WHOLE response instead of declaring victory on the first EOSE. * * A subscription fans a single REQ out to several relays. The first EOSE is a misleading "done" * signal: a fast but near-empty relay can EOSE in milliseconds while the relay that actually holds - * the data is still connecting (or stuck in an auth handshake). An auto-fill loop driven by the - * first EOSE would therefore widen the time window again before the slow relay ever answered, - * walking the window back uselessly. + * the data is still connecting, stuck in an auth handshake, or busy streaming thousands of stored + * events. An auto-fill loop driven by the first EOSE — or by a fixed wall-clock timeout — would + * widen the window again mid-stream, before the events were even decrypted into rooms, re-issuing + * an ever-wider REQ that re-downloads the whole history over and over. * - * [loading] stays true until EVERY [setExpectedRelays] relay has answered (an EOSE, or a live event - * that implies the stored set already drained), or until [timeout] elapses as a backstop for relays - * that never answer (down, or looping on `auth-required`). + * So completion is **activity-based**: [loading] stays true until either every expected relay has + * EOSE'd, or the event stream has gone quiet for [idleTimeout] (a flood of events keeps resetting + * that timer via [onActivity], so a window that is still streaming is never declared done). An + * [absoluteCap] bounds the wait for pathological relays that dribble forever. */ class WindowLoadTracker( - private val timeout: Duration = 15.seconds, + private val idleTimeout: Duration = 3.seconds, + private val absoluteCap: Duration = 5.minutes, ) { private val _loading = MutableStateFlow(true) val loading: StateFlow = _loading.asStateFlow() private var expected: Set = emptySet() private val responded = mutableSetOf() - private var timeoutJob: Job? = null + private var watchdog: Job? = null - /** Begins a fresh window load: clears the responded set, raises [loading], and arms the timeout. */ + // Wall-clock of the last EOSE or event for the current window; the watchdog completes the + // window once this stops advancing for [idleTimeout]. Volatile so the hot per-event path + // ([onActivity]) stays lock-free. + @Volatile + private var lastActivityMs = 0L + + /** Begins a fresh window load: clears the responded set, raises [loading], and arms the watchdog. */ @Synchronized fun startLoading(scope: CoroutineScope) { responded.clear() + lastActivityMs = System.currentTimeMillis() _loading.value = true - timeoutJob?.cancel() - timeoutJob = + watchdog?.cancel() + watchdog = scope.launch { - delay(timeout) - finish() + val deadline = System.currentTimeMillis() + absoluteCap.inWholeMilliseconds + while (isActive && _loading.value) { + delay(IDLE_CHECK_MS) + val now = System.currentTimeMillis() + if (now - lastActivityMs >= idleTimeout.inWholeMilliseconds || now >= deadline) { + finish() + } + } } } + /** + * Records that the current window is still actively receiving events (stored OR live). Keeps the + * idle watchdog from completing while a relay is mid-flood. Lock-free: just bumps a timestamp. + */ + fun onActivity() { + lastActivityMs = System.currentTimeMillis() + } + /** Records which relays the current REQ was sent to. Completes immediately if there are none. */ @Synchronized fun setExpectedRelays(relays: Set) { @@ -78,6 +104,7 @@ class WindowLoadTracker( /** Marks [relay] as having answered (EOSE or live event). Completes once all expected have. */ @Synchronized fun onRelayResponded(relay: NormalizedRelayUrl) { + lastActivityMs = System.currentTimeMillis() responded.add(relay) if (expected.isNotEmpty() && responded.containsAll(expected)) finish() } @@ -85,7 +112,11 @@ class WindowLoadTracker( @Synchronized private fun finish() { _loading.value = false - timeoutJob?.cancel() - timeoutJob = null + watchdog?.cancel() + watchdog = null + } + + companion object { + private const val IDLE_CHECK_MS = 500L } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip59GiftWraps/AccountGiftWrapsEoseManager.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip59GiftWraps/AccountGiftWrapsEoseManager.kt index 335df0f9e6..23c7a274fb 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip59GiftWraps/AccountGiftWrapsEoseManager.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip59GiftWraps/AccountGiftWrapsEoseManager.kt @@ -128,6 +128,20 @@ class AccountGiftWrapsEoseManager( invalidateFilters() } + /** + * Jumps the gift-wrap window straight to the maximum lookback so a single REQ pulls the entire + * history (the pre-windowing behavior), and marks it [exhausted] so auto-fill stops. + */ + fun loadEverything(user: User) { + val window = windowFor(user) + if (window.isExhausted()) return + window.loadAll() + _exhausted.value = true + Log.d(TAG) { "loadEverything: pubkey=${user.pubkeyHex.take(8)}… loading full history since ${window.since}" } + scope?.let { windowLoad.startLoading(it) } + invalidateFilters() + } + override fun newEose( key: AccountQueryState, relay: NormalizedRelayUrl, @@ -195,6 +209,9 @@ class AccountGiftWrapsEoseManager( relay: NormalizedRelayUrl, forFilters: List?, ) { + // Every event (stored backfill included) keeps the window-load watchdog alive, + // so a relay mid-flood is never mistaken for a finished window. + windowLoad.onActivity() if (pubkey !in bootEoseLogged) { bootEventCount[pubkey] = (bootEventCount[pubkey] ?: 0) + 1 } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/datasource/DMsFromUserFilterSubAssembler.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/datasource/DMsFromUserFilterSubAssembler.kt index 79585ba6f0..4767490399 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/datasource/DMsFromUserFilterSubAssembler.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/datasource/DMsFromUserFilterSubAssembler.kt @@ -26,12 +26,15 @@ import com.vitorpamplona.amethyst.service.relayClient.eoseManagers.PerUserEoseMa import com.vitorpamplona.amethyst.service.relayClient.eoseManagers.WindowLoadTracker import com.vitorpamplona.amethyst.service.relayClient.reqCommand.account.nip59GiftWraps.AccountGiftWrapsEoseManager import com.vitorpamplona.amethyst.service.relays.SincePerRelayMap +import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.pool.RelayBasedFilter +import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener import com.vitorpamplona.quartz.nip01Core.relay.client.subscriptions.Subscription import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.utils.TimeUtils import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.FlowPreview @@ -103,6 +106,19 @@ class DMsFromUserFilterSubAssembler( invalidateFilters() } + /** + * Jumps the NIP-04 window straight to the maximum lookback so a single REQ pulls the entire + * history (the pre-windowing behavior), and marks it [exhausted] so auto-fill stops. + */ + fun loadEverything(user: User) { + val window = windowFor(user) + if (window.isExhausted()) return + window.loadAll() + _exhausted.value = true + scope?.let { windowLoad.startLoading(it) } + invalidateFilters() + } + override fun newEose( key: ChatroomListState, relay: NormalizedRelayUrl, @@ -137,7 +153,30 @@ class DMsFromUserFilterSubAssembler( }, ) - return super.newSub(key) + // Custom listener (vs super.newSub) so every event — stored backfill included — keeps the + // window-load watchdog alive; otherwise a NIP-04 flood would look "done" mid-stream. + return requestNewSubscription( + object : SubscriptionListener { + override fun onEose( + relay: NormalizedRelayUrl, + forFilters: List?, + ) { + newEose(key, relay, TimeUtils.now(), forFilters) + } + + override fun onEvent( + event: Event, + isLive: Boolean, + relay: NormalizedRelayUrl, + forFilters: List?, + ) { + windowLoad.onActivity() + if (isLive) { + newEose(key, relay, TimeUtils.now(), forFilters) + } + } + }, + ) } override fun endSub( diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/feed/ChatroomListFeedView.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/feed/ChatroomListFeedView.kt index f0620173e5..c814378dda 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/feed/ChatroomListFeedView.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/feed/ChatroomListFeedView.kt @@ -21,7 +21,7 @@ package com.vitorpamplona.amethyst.ui.screen.loggedIn.chats.rooms.feed import androidx.compose.animation.core.tween -import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.padding @@ -31,13 +31,18 @@ import androidx.compose.foundation.lazy.LazyListState import androidx.compose.foundation.lazy.items import androidx.compose.material3.CircularProgressIndicator import androidx.compose.material3.HorizontalDivider +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue import androidx.compose.runtime.remember import androidx.compose.runtime.snapshotFlow +import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.res.stringResource import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.vitorpamplona.amethyst.R import com.vitorpamplona.amethyst.commons.model.marmotGroups.MarmotGroupChatroom import com.vitorpamplona.amethyst.commons.ui.feeds.FeedContentState import com.vitorpamplona.amethyst.commons.ui.feeds.FeedState @@ -149,6 +154,9 @@ private fun FeedLoaded( val loadingGiftWraps by giftWraps.loadingMore.collectAsStateWithLifecycle() val loadingNip04 by nip04Dms.loadingMore.collectAsStateWithLifecycle() val loadingMore = loadingGiftWraps || loadingNip04 + val exhaustedGiftWraps by giftWraps.exhausted.collectAsStateWithLifecycle() + val exhaustedNip04 by nip04Dms.exhausted.collectAsStateWithLifecycle() + val historyExhausted = exhaustedGiftWraps && exhaustedNip04 LazyColumn( contentPadding = rememberFeedContentPadding(FeedPadding), @@ -171,13 +179,28 @@ private fun FeedLoaded( ) } - if (loadingMore) { + // Footer: shows the auto-fill / full-load spinner, and — while there is still older history + // to reach — a button to skip the windowed paging and pull the entire history at once. + if (loadingMore || !historyExhausted) { item(key = "loadingMoreFooter") { - Row( + Column( Modifier.fillMaxWidth().padding(vertical = Size10dp), - horizontalArrangement = Arrangement.Center, + horizontalAlignment = Alignment.CenterHorizontally, ) { - CircularProgressIndicator(Modifier.size(Size25dp)) + if (loadingMore) { + CircularProgressIndicator(Modifier.size(Size25dp)) + } + if (!historyExhausted) { + TextButton( + onClick = { + val user = accountViewModel.userProfile() + giftWraps.loadEverything(user) + nip04Dms.loadEverything(user) + }, + ) { + Text(stringResource(R.string.chats_load_entire_history)) + } + } } } } diff --git a/amethyst/src/main/res/values/strings.xml b/amethyst/src/main/res/values/strings.xml index 01d3b1c9ba..95fc7847f7 100644 --- a/amethyst/src/main/res/values/strings.xml +++ b/amethyst/src/main/res/values/strings.xml @@ -274,6 +274,7 @@ Generate a new key Loading feed Loading account + Load entire history "Error loading replies: " Try again No notifications yet. diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/pagination/TimeWindowPagination.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/pagination/TimeWindowPagination.kt index 1daae1e36b..6bbabd8cf5 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/pagination/TimeWindowPagination.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/pagination/TimeWindowPagination.kt @@ -63,6 +63,11 @@ class TimeWindowPagination( if (growthFactor > 1L) currentStep *= growthFactor } + /** Jumps straight to [maxLookback] so a single request pulls the entire history. */ + fun loadAll() { + since = floor() + } + /** True once the window has reached [maxLookback] — there is no older history to request. */ fun isExhausted(): Boolean = since <= floor() diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/relayClient/pagination/TimeWindowPaginationTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/relayClient/pagination/TimeWindowPaginationTest.kt index 18069b6ce8..cb79f93940 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/relayClient/pagination/TimeWindowPaginationTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/relayClient/pagination/TimeWindowPaginationTest.kt @@ -98,6 +98,19 @@ class TimeWindowPaginationTest { assertTrue("since must not go past the floor", pagination.since >= floor - 2 && pagination.since <= floor + 2) } + @Test + fun loadAllJumpsStraightToExhaustion() { + val maxLookback = 100L + val pagination = TimeWindowPagination(initialWindow = 10L, step = 10L, growthFactor = 2L, maxLookback = maxLookback) + assertTrue("not exhausted before loadAll", !pagination.isExhausted()) + + pagination.loadAll() + + assertTrue("loadAll exhausts the window in one step", pagination.isExhausted()) + val floor = TimeUtils.now() - maxLookback + assertTrue("since lands at the floor", kotlin.math.abs(pagination.since - floor) <= 2) + } + @Test fun resetClearsStepGrowth() { val pagination = TimeWindowPagination(initialWindow = 10L, step = 100L, growthFactor = 2L, maxLookback = Long.MAX_VALUE / 2)