diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/WindowLoadTracker.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/WindowLoadTracker.kt index 5e759592fd..7a8de53219 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/WindowLoadTracker.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/eoseManagers/WindowLoadTracker.kt @@ -27,47 +27,73 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.isActive import kotlinx.coroutines.launch import kotlin.time.Duration +import kotlin.time.Duration.Companion.minutes import kotlin.time.Duration.Companion.seconds /** - * Tracks when one relay-subscription "window" has finished loading, so callers can wait for the - * WHOLE set of relays to answer instead of declaring victory on the first EOSE. + * Tracks when one relay-subscription "window" has finished loading, so callers (the rooms-screen + * auto-fill loop) can wait for the WHOLE response instead of declaring victory on the first EOSE. * * A subscription fans a single REQ out to several relays. The first EOSE is a misleading "done" * signal: a fast but near-empty relay can EOSE in milliseconds while the relay that actually holds - * the data is still connecting (or stuck in an auth handshake). An auto-fill loop driven by the - * first EOSE would therefore widen the time window again before the slow relay ever answered, - * walking the window back uselessly. + * the data is still connecting, stuck in an auth handshake, or busy streaming thousands of stored + * events. An auto-fill loop driven by the first EOSE — or by a fixed wall-clock timeout — would + * widen the window again mid-stream, before the events were even decrypted into rooms, re-issuing + * an ever-wider REQ that re-downloads the whole history over and over. * - * [loading] stays true until EVERY [setExpectedRelays] relay has answered (an EOSE, or a live event - * that implies the stored set already drained), or until [timeout] elapses as a backstop for relays - * that never answer (down, or looping on `auth-required`). + * So completion is **activity-based**: [loading] stays true until either every expected relay has + * EOSE'd, or the event stream has gone quiet for [idleTimeout] (a flood of events keeps resetting + * that timer via [onActivity], so a window that is still streaming is never declared done). An + * [absoluteCap] bounds the wait for pathological relays that dribble forever. */ class WindowLoadTracker( - private val timeout: Duration = 15.seconds, + private val idleTimeout: Duration = 3.seconds, + private val absoluteCap: Duration = 5.minutes, ) { private val _loading = MutableStateFlow(true) val loading: StateFlow = _loading.asStateFlow() private var expected: Set = emptySet() private val responded = mutableSetOf() - private var timeoutJob: Job? = null + private var watchdog: Job? = null - /** Begins a fresh window load: clears the responded set, raises [loading], and arms the timeout. */ + // Wall-clock of the last EOSE or event for the current window; the watchdog completes the + // window once this stops advancing for [idleTimeout]. Volatile so the hot per-event path + // ([onActivity]) stays lock-free. + @Volatile + private var lastActivityMs = 0L + + /** Begins a fresh window load: clears the responded set, raises [loading], and arms the watchdog. */ @Synchronized fun startLoading(scope: CoroutineScope) { responded.clear() + lastActivityMs = System.currentTimeMillis() _loading.value = true - timeoutJob?.cancel() - timeoutJob = + watchdog?.cancel() + watchdog = scope.launch { - delay(timeout) - finish() + val deadline = System.currentTimeMillis() + absoluteCap.inWholeMilliseconds + while (isActive && _loading.value) { + delay(IDLE_CHECK_MS) + val now = System.currentTimeMillis() + if (now - lastActivityMs >= idleTimeout.inWholeMilliseconds || now >= deadline) { + finish() + } + } } } + /** + * Records that the current window is still actively receiving events (stored OR live). Keeps the + * idle watchdog from completing while a relay is mid-flood. Lock-free: just bumps a timestamp. + */ + fun onActivity() { + lastActivityMs = System.currentTimeMillis() + } + /** Records which relays the current REQ was sent to. Completes immediately if there are none. */ @Synchronized fun setExpectedRelays(relays: Set) { @@ -78,6 +104,7 @@ class WindowLoadTracker( /** Marks [relay] as having answered (EOSE or live event). Completes once all expected have. */ @Synchronized fun onRelayResponded(relay: NormalizedRelayUrl) { + lastActivityMs = System.currentTimeMillis() responded.add(relay) if (expected.isNotEmpty() && responded.containsAll(expected)) finish() } @@ -85,7 +112,11 @@ class WindowLoadTracker( @Synchronized private fun finish() { _loading.value = false - timeoutJob?.cancel() - timeoutJob = null + watchdog?.cancel() + watchdog = null + } + + companion object { + private const val IDLE_CHECK_MS = 500L } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip59GiftWraps/AccountGiftWrapsEoseManager.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip59GiftWraps/AccountGiftWrapsEoseManager.kt index 335df0f9e6..23c7a274fb 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip59GiftWraps/AccountGiftWrapsEoseManager.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip59GiftWraps/AccountGiftWrapsEoseManager.kt @@ -128,6 +128,20 @@ class AccountGiftWrapsEoseManager( invalidateFilters() } + /** + * Jumps the gift-wrap window straight to the maximum lookback so a single REQ pulls the entire + * history (the pre-windowing behavior), and marks it [exhausted] so auto-fill stops. + */ + fun loadEverything(user: User) { + val window = windowFor(user) + if (window.isExhausted()) return + window.loadAll() + _exhausted.value = true + Log.d(TAG) { "loadEverything: pubkey=${user.pubkeyHex.take(8)}… loading full history since ${window.since}" } + scope?.let { windowLoad.startLoading(it) } + invalidateFilters() + } + override fun newEose( key: AccountQueryState, relay: NormalizedRelayUrl, @@ -195,6 +209,9 @@ class AccountGiftWrapsEoseManager( relay: NormalizedRelayUrl, forFilters: List?, ) { + // Every event (stored backfill included) keeps the window-load watchdog alive, + // so a relay mid-flood is never mistaken for a finished window. + windowLoad.onActivity() if (pubkey !in bootEoseLogged) { bootEventCount[pubkey] = (bootEventCount[pubkey] ?: 0) + 1 } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/datasource/DMsFromUserFilterSubAssembler.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/datasource/DMsFromUserFilterSubAssembler.kt index 79585ba6f0..4767490399 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/datasource/DMsFromUserFilterSubAssembler.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/datasource/DMsFromUserFilterSubAssembler.kt @@ -26,12 +26,15 @@ import com.vitorpamplona.amethyst.service.relayClient.eoseManagers.PerUserEoseMa import com.vitorpamplona.amethyst.service.relayClient.eoseManagers.WindowLoadTracker import com.vitorpamplona.amethyst.service.relayClient.reqCommand.account.nip59GiftWraps.AccountGiftWrapsEoseManager import com.vitorpamplona.amethyst.service.relays.SincePerRelayMap +import com.vitorpamplona.quartz.nip01Core.core.Event import com.vitorpamplona.quartz.nip01Core.core.HexKey import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.pool.RelayBasedFilter +import com.vitorpamplona.quartz.nip01Core.relay.client.reqs.SubscriptionListener import com.vitorpamplona.quartz.nip01Core.relay.client.subscriptions.Subscription import com.vitorpamplona.quartz.nip01Core.relay.filters.Filter import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl +import com.vitorpamplona.quartz.utils.TimeUtils import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.FlowPreview @@ -103,6 +106,19 @@ class DMsFromUserFilterSubAssembler( invalidateFilters() } + /** + * Jumps the NIP-04 window straight to the maximum lookback so a single REQ pulls the entire + * history (the pre-windowing behavior), and marks it [exhausted] so auto-fill stops. + */ + fun loadEverything(user: User) { + val window = windowFor(user) + if (window.isExhausted()) return + window.loadAll() + _exhausted.value = true + scope?.let { windowLoad.startLoading(it) } + invalidateFilters() + } + override fun newEose( key: ChatroomListState, relay: NormalizedRelayUrl, @@ -137,7 +153,30 @@ class DMsFromUserFilterSubAssembler( }, ) - return super.newSub(key) + // Custom listener (vs super.newSub) so every event — stored backfill included — keeps the + // window-load watchdog alive; otherwise a NIP-04 flood would look "done" mid-stream. + return requestNewSubscription( + object : SubscriptionListener { + override fun onEose( + relay: NormalizedRelayUrl, + forFilters: List?, + ) { + newEose(key, relay, TimeUtils.now(), forFilters) + } + + override fun onEvent( + event: Event, + isLive: Boolean, + relay: NormalizedRelayUrl, + forFilters: List?, + ) { + windowLoad.onActivity() + if (isLive) { + newEose(key, relay, TimeUtils.now(), forFilters) + } + } + }, + ) } override fun endSub( diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/feed/ChatroomListFeedView.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/feed/ChatroomListFeedView.kt index f0620173e5..c814378dda 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/feed/ChatroomListFeedView.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/feed/ChatroomListFeedView.kt @@ -21,7 +21,7 @@ package com.vitorpamplona.amethyst.ui.screen.loggedIn.chats.rooms.feed import androidx.compose.animation.core.tween -import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.padding @@ -31,13 +31,18 @@ import androidx.compose.foundation.lazy.LazyListState import androidx.compose.foundation.lazy.items import androidx.compose.material3.CircularProgressIndicator import androidx.compose.material3.HorizontalDivider +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue import androidx.compose.runtime.remember import androidx.compose.runtime.snapshotFlow +import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.res.stringResource import androidx.lifecycle.compose.collectAsStateWithLifecycle +import com.vitorpamplona.amethyst.R import com.vitorpamplona.amethyst.commons.model.marmotGroups.MarmotGroupChatroom import com.vitorpamplona.amethyst.commons.ui.feeds.FeedContentState import com.vitorpamplona.amethyst.commons.ui.feeds.FeedState @@ -149,6 +154,9 @@ private fun FeedLoaded( val loadingGiftWraps by giftWraps.loadingMore.collectAsStateWithLifecycle() val loadingNip04 by nip04Dms.loadingMore.collectAsStateWithLifecycle() val loadingMore = loadingGiftWraps || loadingNip04 + val exhaustedGiftWraps by giftWraps.exhausted.collectAsStateWithLifecycle() + val exhaustedNip04 by nip04Dms.exhausted.collectAsStateWithLifecycle() + val historyExhausted = exhaustedGiftWraps && exhaustedNip04 LazyColumn( contentPadding = rememberFeedContentPadding(FeedPadding), @@ -171,13 +179,28 @@ private fun FeedLoaded( ) } - if (loadingMore) { + // Footer: shows the auto-fill / full-load spinner, and — while there is still older history + // to reach — a button to skip the windowed paging and pull the entire history at once. + if (loadingMore || !historyExhausted) { item(key = "loadingMoreFooter") { - Row( + Column( Modifier.fillMaxWidth().padding(vertical = Size10dp), - horizontalArrangement = Arrangement.Center, + horizontalAlignment = Alignment.CenterHorizontally, ) { - CircularProgressIndicator(Modifier.size(Size25dp)) + if (loadingMore) { + CircularProgressIndicator(Modifier.size(Size25dp)) + } + if (!historyExhausted) { + TextButton( + onClick = { + val user = accountViewModel.userProfile() + giftWraps.loadEverything(user) + nip04Dms.loadEverything(user) + }, + ) { + Text(stringResource(R.string.chats_load_entire_history)) + } + } } } } diff --git a/amethyst/src/main/res/values/strings.xml b/amethyst/src/main/res/values/strings.xml index 01d3b1c9ba..95fc7847f7 100644 --- a/amethyst/src/main/res/values/strings.xml +++ b/amethyst/src/main/res/values/strings.xml @@ -274,6 +274,7 @@ Generate a new key Loading feed Loading account + Load entire history "Error loading replies: " Try again No notifications yet. diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/pagination/TimeWindowPagination.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/pagination/TimeWindowPagination.kt index 1daae1e36b..6bbabd8cf5 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/pagination/TimeWindowPagination.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/relayClient/pagination/TimeWindowPagination.kt @@ -63,6 +63,11 @@ class TimeWindowPagination( if (growthFactor > 1L) currentStep *= growthFactor } + /** Jumps straight to [maxLookback] so a single request pulls the entire history. */ + fun loadAll() { + since = floor() + } + /** True once the window has reached [maxLookback] — there is no older history to request. */ fun isExhausted(): Boolean = since <= floor() diff --git a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/relayClient/pagination/TimeWindowPaginationTest.kt b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/relayClient/pagination/TimeWindowPaginationTest.kt index 18069b6ce8..cb79f93940 100644 --- a/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/relayClient/pagination/TimeWindowPaginationTest.kt +++ b/commons/src/commonTest/kotlin/com/vitorpamplona/amethyst/commons/relayClient/pagination/TimeWindowPaginationTest.kt @@ -98,6 +98,19 @@ class TimeWindowPaginationTest { assertTrue("since must not go past the floor", pagination.since >= floor - 2 && pagination.since <= floor + 2) } + @Test + fun loadAllJumpsStraightToExhaustion() { + val maxLookback = 100L + val pagination = TimeWindowPagination(initialWindow = 10L, step = 10L, growthFactor = 2L, maxLookback = maxLookback) + assertTrue("not exhausted before loadAll", !pagination.isExhausted()) + + pagination.loadAll() + + assertTrue("loadAll exhausts the window in one step", pagination.isExhausted()) + val floor = TimeUtils.now() - maxLookback + assertTrue("since lands at the floor", kotlin.math.abs(pagination.since - floor) <= 2) + } + @Test fun resetClearsStepGrowth() { val pagination = TimeWindowPagination(initialWindow = 10L, step = 100L, growthFactor = 2L, maxLookback = Long.MAX_VALUE / 2)