From aa6b9bc53eb06622a87fd25d3192f85ccce16e9b Mon Sep 17 00:00:00 2001 From: Vitor Pamplona Date: Mon, 8 Jun 2026 17:59:04 -0400 Subject: [PATCH] refactor(dm): centralize the DM history-window boundary + log prune rewinds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The "where the live tail ends and paged history begins" boundary (one week) was copy-pasted across the gift-wrap + NIP-04 live tails, the backward history pager floor, and the prune's recent/old split — easy to drift out of lockstep (overlap = double-load, gap = missed messages). - Add DmHistoryTuning: one place for liveTailSeconds + recentKeepCount, read by AccountGiftWrapsEoseManager, both NIP-04 SubAssemblers, BackwardRelayPager, and Chatroom.pruneMessagesToTheLatestOnly. Drops the duplicated LIVE_TAIL_SECONDS / DEFAULT_LIVE_TAIL_SECONDS constants. - Log the prune-time window realignment under DMPagination, per scope ([giftwrap] / [rooms.nip04] / [convo.nip04]): relay count + newest pruned timestamp, so the rewind is observable in logcat. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../amethyst/model/LocalCache.kt | 15 ++++-- .../AccountGiftWrapsEoseManager.kt | 8 ++- .../datasource/ChatroomNip04SubAssembler.kt | 6 +-- .../ChatroomListNip04SubAssembler.kt | 6 +-- .../commons/model/privateChats/Chatroom.kt | 9 ++-- .../model/privateChats/DmHistoryTuning.kt | 49 +++++++++++++++++++ .../relayClient/paging/BackwardRelayPager.kt | 8 ++- 7 files changed, 77 insertions(+), 24 deletions(-) create mode 100644 commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/privateChats/DmHistoryTuning.kt diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/LocalCache.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/LocalCache.kt index 920631cf99..a2bf434c66 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/model/LocalCache.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/model/LocalCache.kt @@ -2693,9 +2693,18 @@ object LocalCache : ILocalCache, ICacheProvider { // Realign the windows so a relay that already paged past (or `done` below) the dropped band // re-requests it on the next demand-advance instead of skipping the hole. - if (giftWrapPruned.isNotEmpty()) room.giftWrapHistory.rewindTo(giftWrapPruned) - if (accountNip04Pruned.isNotEmpty()) room.nip04History.rewindTo(accountNip04Pruned) - if (roomNip04Pruned.isNotEmpty()) chatroom.nip04History.rewindTo(roomNip04Pruned) + if (giftWrapPruned.isNotEmpty()) { + room.giftWrapHistory.rewindTo(giftWrapPruned) + Log.d("DMPagination") { "[giftwrap] window rewound after prune: ${giftWrapPruned.size} relay(s), newest pruned wrap @${giftWrapPruned.values.max()}" } + } + if (accountNip04Pruned.isNotEmpty()) { + room.nip04History.rewindTo(accountNip04Pruned) + Log.d("DMPagination") { "[rooms.nip04] window rewound after prune: ${accountNip04Pruned.size} relay(s), newest pruned @${accountNip04Pruned.values.max()}" } + } + if (roomNip04Pruned.isNotEmpty()) { + chatroom.nip04History.rewindTo(roomNip04Pruned) + Log.d("DMPagination") { "[convo.nip04] window rewound after prune of ${key.users.joinToString()}: ${roomNip04Pruned.size} relay(s), newest pruned @${roomNip04Pruned.values.max()}" } + } if (toBeRemoved.size > 1) { println( diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip59GiftWraps/AccountGiftWrapsEoseManager.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip59GiftWraps/AccountGiftWrapsEoseManager.kt index 5a9633e210..1ddf8e8028 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip59GiftWraps/AccountGiftWrapsEoseManager.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/service/relayClient/reqCommand/account/nip59GiftWraps/AccountGiftWrapsEoseManager.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.amethyst.service.relayClient.reqCommand.account.nip59GiftWraps +import com.vitorpamplona.amethyst.commons.model.privateChats.DmHistoryTuning import com.vitorpamplona.amethyst.commons.relayClient.nip17Dm.filterGiftWrapsToPubkey import com.vitorpamplona.amethyst.commons.relayClient.paging.WindowLoadTracker import com.vitorpamplona.amethyst.commons.relayClient.paging.trackingListener @@ -68,9 +69,9 @@ class AccountGiftWrapsEoseManager( } val relays = key.account.dmRelays.flow.value windowLoad.setExpectedRelays(relays.toSet()) - val sinceTime = TimeUtils.now() - LIVE_TAIL_SECONDS + val sinceTime = DmHistoryTuning.recentBoundary() DmRelayLog.log("giftwrap.live", key.account) - Log.d(TAG) { "[giftwrap.live] REQ since=$sinceTime (7d, no until) on ${relays.size} relay(s): ${relays.map { it.url }}" } + Log.d(TAG) { "[giftwrap.live] REQ since=$sinceTime (no until) on ${relays.size} relay(s): ${relays.map { it.url }}" } return relays.flatMap { relay -> filterGiftWrapsToPubkey(relay = relay, pubkey = user(key).pubkeyHex, since = sinceTime) } @@ -106,8 +107,5 @@ class AccountGiftWrapsEoseManager( companion object { private const val TAG = "DMPagination" - - // The live tail's fixed lower bound: one week of recent history, always open at the top. - val LIVE_TAIL_SECONDS = 7L * TimeUtils.ONE_DAY } } diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/privateDM/datasource/ChatroomNip04SubAssembler.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/privateDM/datasource/ChatroomNip04SubAssembler.kt index f8b233f7cd..f7c8bfb686 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/privateDM/datasource/ChatroomNip04SubAssembler.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/privateDM/datasource/ChatroomNip04SubAssembler.kt @@ -20,11 +20,11 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.chats.privateDM.datasource +import com.vitorpamplona.amethyst.commons.model.privateChats.DmHistoryTuning import com.vitorpamplona.amethyst.commons.relayClient.paging.WindowLoadTracker import com.vitorpamplona.amethyst.commons.relayClient.paging.trackingListener import com.vitorpamplona.amethyst.service.relayClient.eoseManagers.DmRelayLog import com.vitorpamplona.amethyst.service.relayClient.eoseManagers.PerUserAndFollowListEoseManager -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.account.nip59GiftWraps.AccountGiftWrapsEoseManager import com.vitorpamplona.amethyst.service.relays.SincePerRelayMap import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.pool.RelayBasedFilter @@ -50,11 +50,11 @@ class ChatroomNip04SubAssembler( since: SincePerRelayMap?, ): List? = if (key.account.isWriteable()) { - val sinceTime = TimeUtils.now() - AccountGiftWrapsEoseManager.LIVE_TAIL_SECONDS + val sinceTime = DmHistoryTuning.recentBoundary() val filters = filterNip04DMs(key.room.users, key.account, sinceTime) windowLoad.setExpectedRelays(filters?.mapTo(mutableSetOf()) { it.relay } ?: emptySet()) DmRelayLog.log("convo.nip04.live", key.account) - Log.d("DMPagination") { "[convo.nip04.live] REQ since=$sinceTime (7d, no until) on ${filters?.size ?: 0} relay-filter(s): ${filters?.map { it.relay.url }?.distinct()}" } + Log.d("DMPagination") { "[convo.nip04.live] REQ since=$sinceTime (no until) on ${filters?.size ?: 0} relay-filter(s): ${filters?.map { it.relay.url }?.distinct()}" } filters } else { windowLoad.setExpectedRelays(emptySet()) diff --git a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/datasource/ChatroomListNip04SubAssembler.kt b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/datasource/ChatroomListNip04SubAssembler.kt index 09e2910591..cd712abcaa 100644 --- a/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/datasource/ChatroomListNip04SubAssembler.kt +++ b/amethyst/src/main/java/com/vitorpamplona/amethyst/ui/screen/loggedIn/chats/rooms/datasource/ChatroomListNip04SubAssembler.kt @@ -20,12 +20,12 @@ */ package com.vitorpamplona.amethyst.ui.screen.loggedIn.chats.rooms.datasource +import com.vitorpamplona.amethyst.commons.model.privateChats.DmHistoryTuning import com.vitorpamplona.amethyst.commons.relayClient.paging.WindowLoadTracker import com.vitorpamplona.amethyst.commons.relayClient.paging.trackingListener import com.vitorpamplona.amethyst.model.User import com.vitorpamplona.amethyst.service.relayClient.eoseManagers.DmRelayLog import com.vitorpamplona.amethyst.service.relayClient.eoseManagers.PerUserEoseManager -import com.vitorpamplona.amethyst.service.relayClient.reqCommand.account.nip59GiftWraps.AccountGiftWrapsEoseManager import com.vitorpamplona.amethyst.service.relays.SincePerRelayMap import com.vitorpamplona.quartz.nip01Core.relay.client.INostrClient import com.vitorpamplona.quartz.nip01Core.relay.client.pool.RelayBasedFilter @@ -59,9 +59,9 @@ class ChatroomListNip04SubAssembler( val homeRelays = key.account.homeRelays.flow.value val dmRelays = key.account.dmRelays.flow.value windowLoad.setExpectedRelays((homeRelays + dmRelays).toSet()) - val sinceTime = TimeUtils.now() - AccountGiftWrapsEoseManager.LIVE_TAIL_SECONDS + val sinceTime = DmHistoryTuning.recentBoundary() DmRelayLog.log("rooms.nip04.live", key.account) - Log.d("DMPagination") { "[rooms.nip04.live] REQ since=$sinceTime (7d, no until) fromMe(outbox)=${homeRelays.map { it.url }} toMe(inbox)=${dmRelays.map { it.url }}" } + Log.d("DMPagination") { "[rooms.nip04.live] REQ since=$sinceTime (no until) fromMe(outbox)=${homeRelays.map { it.url }} toMe(inbox)=${dmRelays.map { it.url }}" } homeRelays.map { filterNip04DMsFromMe(key.account.userProfile(), it, sinceTime) } + dmRelays.map { filterNip04DMsToMe(key.account.userProfile(), it, sinceTime) } } else { diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/privateChats/Chatroom.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/privateChats/Chatroom.kt index 13d604208d..5c9f6cc7b3 100644 --- a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/privateChats/Chatroom.kt +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/privateChats/Chatroom.kt @@ -34,7 +34,6 @@ import com.vitorpamplona.quartz.nip01Core.relay.client.paging.RelayLoadingCursor import com.vitorpamplona.quartz.nip04Dm.messages.PrivateDmEvent import com.vitorpamplona.quartz.nip14Subject.subject import com.vitorpamplona.quartz.nip59Giftwrap.WrappedEvent -import com.vitorpamplona.quartz.utils.TimeUtils import kotlinx.coroutines.channels.BufferOverflow import kotlinx.coroutines.flow.MutableSharedFlow import kotlinx.coroutines.flow.MutableStateFlow @@ -140,11 +139,11 @@ class Chatroom : NotesGatherer { val sorted = messages.sortedWith(DefaultFeedOrder) val toKeep = - if ((sorted.firstOrNull()?.createdAt() ?: 0L) > TimeUtils.oneWeekAgo()) { - // Recent messages, keep last 100 - sorted.take(100).toSet() + if ((sorted.firstOrNull()?.createdAt() ?: 0L) > DmHistoryTuning.recentBoundary()) { + // Recent conversation, keep its newest N + sorted.take(DmHistoryTuning.recentKeepCount).toSet() } else { - // Old messages, keep the last one. + // Old conversation, keep the last one. sorted.take(1).toSet() } + sorted.filter { it.flowSet?.isInUse() ?: false } + sorted.filter { it.event !is PrivateDmEvent && it.event !is WrappedEvent } // Both DM protocols are pruned by the recency rule above: NIP-04 (PrivateDmEvent) and NIP-17 diff --git a/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/privateChats/DmHistoryTuning.kt b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/privateChats/DmHistoryTuning.kt new file mode 100644 index 0000000000..b2c1dae11c --- /dev/null +++ b/commons/src/commonMain/kotlin/com/vitorpamplona/amethyst/commons/model/privateChats/DmHistoryTuning.kt @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2025 Vitor Pamplona + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN + * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +package com.vitorpamplona.amethyst.commons.model.privateChats + +import com.vitorpamplona.quartz.utils.TimeUtils +import kotlin.concurrent.Volatile + +/** + * One shared boundary for the DM history window, so the three things that must agree on "where the + * live tail ends and paged history begins" actually read the same number: + * - the live-tail subscriptions' `since` floor (e.g. `AccountGiftWrapsEoseManager`), + * - the backward history pager's pinned floor (`BackwardRelayPager`), + * - the memory prune's recent/old split + retention cap (`Chatroom.pruneMessagesToTheLatestOnly`). + * + * Keeping them in one place avoids the live tail and history overlapping (double-loading) or leaving a + * gap when the boundary is tuned. The knobs are plain `@Volatile` vars (overridable once at startup — + * e.g. shrinking the window in a test to exercise pruning + the per-relay download-window realignment + * without needing week-old threads); they are not meant to change mid-session. + */ +object DmHistoryTuning { + /** Seconds below "now" where the live tail ends and paged history begins. Production: one week. */ + @Volatile + var liveTailSeconds: Long = 7L * TimeUtils.ONE_DAY + + /** How many newest messages a still-recent conversation keeps on a prune. Production: 100. */ + @Volatile + var recentKeepCount: Int = 100 + + /** The epoch-seconds boundary `now − [liveTailSeconds]` (recomputed each call against the clock). */ + fun recentBoundary(): Long = TimeUtils.now() - liveTailSeconds +} diff --git a/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/relayClient/paging/BackwardRelayPager.kt b/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/relayClient/paging/BackwardRelayPager.kt index b8b45ad107..62e675e244 100644 --- a/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/relayClient/paging/BackwardRelayPager.kt +++ b/commons/src/jvmAndroid/kotlin/com/vitorpamplona/amethyst/commons/relayClient/paging/BackwardRelayPager.kt @@ -20,6 +20,7 @@ */ package com.vitorpamplona.amethyst.commons.relayClient.paging +import com.vitorpamplona.amethyst.commons.model.privateChats.DmHistoryTuning import com.vitorpamplona.quartz.nip01Core.relay.client.paging.RelayLoadingCursors import com.vitorpamplona.quartz.nip01Core.relay.client.paging.RelayPagingProgress import com.vitorpamplona.quartz.nip01Core.relay.normalizer.NormalizedRelayUrl @@ -74,8 +75,8 @@ class BackwardRelayPager( // volume. A relay returning fewer is its own cap, NOT exhaustion — only an empty page ends a relay. val pageLimit: Int = DEFAULT_PAGE_LIMIT, // How far below "now" the history floor sits — paging starts here and walks backward. Defaults to - // the one-week live-tail boundary: everything newer is the always-on tail's job. - private val liveTailSeconds: Long = DEFAULT_LIVE_TAIL_SECONDS, + // the shared live-tail boundary ([DmHistoryTuning]): everything newer is the always-on tail's job. + private val liveTailSeconds: Long = DmHistoryTuning.liveTailSeconds, ) { private val loadTracker = PerRelayLoadTracker(name, onSilenced = ::onSilenced) @@ -300,8 +301,5 @@ class BackwardRelayPager( private const val TAG = "DMPagination" const val DEFAULT_PAGE_LIMIT = 10000 - - // One week — matches the DM live-tail floor (everything newer is the always-on tail's job). - const val DEFAULT_LIVE_TAIL_SECONDS = 7L * TimeUtils.ONE_DAY } }