From 9d684149124d1013d5b6740d8b592ab799772a1d Mon Sep 17 00:00:00 2001 From: DanConwayDev Date: Tue, 18 Aug 2026 13:49:09 +0000 Subject: [PATCH] fix(sync): keep repository inboxes live Repository owner and declared-maintainer read inboxes are authoritative sources for roots that may never reach a repository relay. Serving them only through a periodic history rotation leaves newly published roots and root-only status events unnecessarily stale. Map each accepted public Full repository into its owners and maintainers read or unmarked NIP-65 inbox targets, and merge every locally known repository root into the same target. The ordinary GRASP-02 target pipeline therefore keeps canonical repository and root filters in EssentialCore coverage, while compatibility variants remain ahead of recursive descendant fan-out. Write-only outboxes and non-root participant mailboxes retain the bounded history probe. This assumes NIP-65 read and unmarked relays are suitable inbox sources. Private mode continues to suppress repository coordinates, normal admission and persistent deletion tombstones remain authoritative, and this commit deliberately does not add permanent participant or write-only mailbox subscriptions or introduce cross-relay precedence over root-author inboxes. Validated with the full 819-test library suite, every Sync+ integration scenario, the complete sync integration binary (102 passed, one ignored), cargo fmt --check, and Clippy across all targets and features with warnings denied. Two unrelated timing tests failed once under the full workspace load, then passed individually and in the complete normal-parallel sync binary rerun. --- CHANGELOG.md | 14 +-- docs/explanation/architecture.md | 13 +-- .../grasp-03-proactive-sync-plus.md | 90 ++++++++++--------- src/sync/discovery.rs | 33 +++++++ src/sync/mod.rs | 63 ++++++++++--- tests/sync/proactive_sync_plus.rs | 27 +++++- 6 files changed, 174 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21a11de..e5d8b26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -137,12 +137,14 @@ Performance and Security fixes - along with other improvements; immediate upgrad ### Fixed -- Discover historical repository roots and descendants from the NIP-65 - mailboxes of accepted repository owners and maintainers. Public Sync+ - instances query exact repository coordinates and every known root for that - repository through the existing paced, byte-bounded history workers, while - private instances continue to withhold repository coordinates and the - ordinary write policy and persistent deletion tombstones remain authoritative. +- Discover historical and live repository roots and descendants from the + NIP-65 mailboxes of accepted repository owners and maintainers. Public Sync+ + instances make exact repository coordinates and every known root essential + live coverage on read/unmarked inboxes, above recursive descendant fan-out, + while their wider mailbox set continues through paced, byte-bounded history + workers. Private instances continue to withhold repository coordinates, and + the ordinary write policy and persistent deletion tombstones remain + authoritative. - Retire peer-closed outbound subscriptions from rust-nostr's desired registry without interrupting NIP-42: the first `auth-required` response keeps the diff --git a/docs/explanation/architecture.md b/docs/explanation/architecture.md index c85be87..01f7d49 100644 --- a/docs/explanation/architecture.md +++ b/docs/explanation/architecture.md @@ -685,11 +685,14 @@ participant mailboxes use independent, history-only `fetch_events` workers: at most one per relay and one new start per maintenance pass. They reuse the connection's ordinary pacing, subscription ledger, pagination and 30-second per-page terminal timeout without installing permanent participant -subscriptions or coupling progress between relays. On public instances, those -workers also query each accepted repository owner's and declared maintainer's -mailboxes for their exact Full repository coordinates and all locally known -roots in those repositories. This discovers previously unknown roots and -descendants that exist only on a maintainer mailbox without broadening an +subscriptions or coupling progress between relays. On public instances, each +accepted repository owner's and declared maintainer's NIP-65 read/unmarked +inboxes also become ordinary full repository sources for their exact +repositories and all locally known roots. Their canonical repository/root +filters are essential live coverage—equal to root-author canonical inbox +coverage and above recursive descendant fan-out—while their wider mailbox set +continues through the history workers. This discovers previously unknown roots +and descendants that exist only on a maintainer mailbox without broadening an ordinary participant mailbox. Private instances omit repository-coordinate mailbox expansion. diff --git a/docs/explanation/grasp-03-proactive-sync-plus.md b/docs/explanation/grasp-03-proactive-sync-plus.md index 2052b24..59b3904 100644 --- a/docs/explanation/grasp-03-proactive-sync-plus.md +++ b/docs/explanation/grasp-03-proactive-sync-plus.md @@ -18,17 +18,22 @@ existing index set for each author's latest profile kind `0` and NIP-65 kind `10002`; retain those replaceable events locally; keep accepted root authors' `read` and unmarked inboxes in ordinary GRASP-02 coverage; and probe accepted participants' `read`, `write` and unmarked conversation mailboxes with -repository and thread-reference filters. On public instances, the same -history-only probe associates accepted repository owners and declared -maintainers with those repositories and queries their mailboxes by repository -coordinate. This can discover a root which is not yet present locally. +repository and thread-reference filters. On public instances, accepted +repository owners' and declared maintainers' `read` and unmarked inboxes also +enter ordinary GRASP-02 repository coverage for their exact repositories; +their wider read/write mailbox set retains the history probe. Repository +coordinates on those inboxes can therefore discover a root which is not yet +present locally and continue receiving new roots live. Mailbox work reuses GRASP-02's connection safety, filter byte packing, pagination, subscription ledger, request pacing, event pipeline and write policy. The participant expansion is history-only: a non-root participant relay never becomes a permanent live source merely because an accepted author advertised it. Root-author inboxes retain the pre-existing live/rotating Sync+ -behavior. +behavior. Owner/maintainer repository inbox filters are essential core +coverage, equal to canonical root-author inbox filters and above recursive +descendant fan-out; compatibility tag variants may rotate before canonical +coverage is sacrificed. Authors without an accepted stored relay list are queried through the configured user-index set plus the bootstrap relay. Once a list is retained, discovery @@ -81,11 +86,12 @@ database before any network refresh, so serving established coverage does not depend on an external index remaining available. Remote discovery then refreshes that retained state on the normal cadence. -Root-author read/unmarked inboxes still use ordinary GRASP-02 coverage. Wider -participant mailboxes use history-only workers. A maintenance pass starts at -most one due relay, while each relay may have at most one worker in flight. -Relays do not share a mailbox lane or terminal state: a slow or unavailable -relay cannot prevent another relay from progressing on a later pass. +Root-author and repository owner/maintainer read/unmarked inboxes use ordinary +GRASP-02 coverage. Wider participant mailboxes and owner/maintainer write-only +outboxes use history-only workers. A maintenance pass starts at most one due +history relay, while each relay may have at most one worker in flight. Relays +do not share a mailbox lane or terminal state: a slow or unavailable relay +cannot prevent another relay from progressing on a later pass. Each worker selects one stable-sorted, byte-bounded filter and delegates its REQ lifecycle to the existing `RelayConnection::fetch_events` path. That path @@ -97,31 +103,34 @@ through the normal write policy and persistence pipeline. No mailbox-specific pending-batch kind, EOSE hook, close API, watchdog or cross-relay coordinator is added. -The probe covers accepted repository coordinates, root IDs, bounded recursive -descendant IDs, and descendant address coordinates using `a`/`A`/`q` and -`e`/`E`/`q`. A repository-scoped owner or maintainer mailbox also receives -event-reference filters for every currently known root in that repository. -This permits a status or other descendant stored only on that mailbox to be -found even when the root author's current relay list does not name it. A -participant-only mailbox remains constrained to its derived roots. A numeric -in-memory cursor gives each byte-bounded filter group a turn. A successful -complete rotation refreshes after 24 hours; a failed group advances the cursor -after a five-minute delay and is retried on a later rotation. This is -deliberately best effort rather than a durable exactly-once schedule. A relay -already needed for ordinary repository sync shares its connection; an -exclusively control-plane connection retires when its identity and mailbox -work is idle. +Coverage uses accepted repository coordinates, root IDs, bounded recursive +descendant IDs, and descendant address coordinates with `a`/`A`/`q` and +`e`/`E`/`q`. A repository-scoped owner or maintainer inbox receives the exact +repository and every currently known root in it as core live and historic +work. This permits a status or other descendant stored only on that inbox to +be found even when the root author's current relay list does not name it. The +wider mailbox history probe covers the same references and a participant-only +mailbox remains constrained to its derived roots. A numeric in-memory cursor +gives each byte-bounded history filter group a turn. A successful complete +rotation refreshes after 24 hours; a failed group advances the cursor after a +five-minute delay and is retried on a later rotation. A relay already needed +for ordinary repository sync shares its connection; an exclusively +history/control-plane connection retires when its identity and mailbox work is +idle. -Owner/maintainer expansion can increase a relay's history rotation in -proportion to the byte-packed repository coordinates and known roots assigned -to it. Shared relays and duplicate repository scopes are unioned before filter -construction. The expansion does not increase instantaneous worker -concurrency on any one relay: the manager still starts at most one due relay -per maintenance pass, each relay remains single-flight, and each worker fetches -one filter group through the existing background request pacer before -yielding. More distinct maintainer relays can nevertheless produce more active -per-relay workers across successive passes; the fixed-cardinality relay, -cursor, and active-worker metrics expose that growth for a production soak. +Owner/maintainer expansion adds a persistent managed connection and essential +live repository/root filters for each distinct read inbox, plus history +rotation proportional to the byte-packed repository coordinates and known +roots assigned to every mailbox. Shared relays and duplicate repository scopes +are unioned before filter construction. Canonical live filters consume normal +GRASP-02 subscription-ledger capacity; compatibility variants and recursive +descendants retain the existing priority cutoff and history fallback. History +still starts at most one due relay per maintenance pass, remains single-flight +per relay, and fetches one filter group through the background request pacer +before yielding. More distinct maintainer relays can therefore increase both +persistent connection/subscription load and active history workers; the +fixed-cardinality relay, cursor, worker, connection, and subscription metrics +must be watched during a production soak. On restart, accepted roots and retained kind `10002` events rebuild participant and mailbox ownership from LMDB. The in-memory group cursor is intentionally @@ -136,12 +145,13 @@ This makes a stuck worker or unexpected inventory expansion visible without putting peer URLs or public keys into metric labels. Mailbox replacement/removal changes desired ownership immediately. Root-author -inbox additions use ordinary coverage, while removal-only changes let existing -shared live subscriptions drain naturally. History-probe additions become due -promptly; removals prevent future groups while an already in-flight -`fetch_events` worker may finish naturally. Root deletion follows the existing -GRASP-02 root-index lifecycle and is reconstructed from retained accepted -events on restart; this change does not add a second deletion graph. +and owner/maintainer repository inbox additions use ordinary coverage, while +removal-only changes let existing shared live subscriptions drain naturally. +History-probe additions become due promptly; removals prevent future groups +while an already in-flight `fetch_events` worker may finish naturally. Root +deletion follows the existing GRASP-02 root-index lifecycle and is +reconstructed from retained accepted events on restart; this change does not +add a second deletion graph. ## Deliberately excluded diff --git a/src/sync/discovery.rs b/src/sync/discovery.rs index f1bf169..2897eb2 100644 --- a/src/sync/discovery.rs +++ b/src/sync/discovery.rs @@ -230,6 +230,22 @@ pub fn merge_inbox_roots( } } +/// Add accepted owner/maintainer inbox repositories to ordinary Full sync. +/// These targets receive canonical live coverage before auxiliary descendant +/// fan-out, while write-only mailboxes remain absent from this overlay. +pub fn merge_inbox_repositories( + targets: &mut HashMap, + inbox_repositories: &HashMap>, +) { + for (relay, repositories) in inbox_repositories { + targets + .entry(relay.clone()) + .or_default() + .repos + .extend(repositories.iter().cloned()); + } +} + pub fn build_inbox_root_overlay( author_roots: &HashMap>, author_inboxes: &HashMap>, @@ -543,6 +559,23 @@ mod tests { assert!(target.state_only_repos.is_empty()); } + #[test] + fn repository_inbox_overlay_adds_full_repository_work() { + let repository = "30617:owner:repo".to_string(); + let mut targets = HashMap::new(); + merge_inbox_repositories( + &mut targets, + &HashMap::from([( + "wss://inbox.example/".to_string(), + HashSet::from([repository.clone()]), + )]), + ); + let target = &targets["wss://inbox.example/"]; + assert_eq!(target.repos, HashSet::from([repository])); + assert!(target.root_events.is_empty()); + assert!(target.state_only_repos.is_empty()); + } + #[test] fn replacement_and_root_removal_subtract_from_desired_overlay() { let author = Keys::generate().public_key(); diff --git a/src/sync/mod.rs b/src/sync/mod.rs index 45f56b4..b5f0f3e 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -1932,7 +1932,12 @@ struct Nip65DiscoveryState { /// no accepted relay list. They use the operator's bounded fallback set /// until an accepted kind 10002 arrives. fallback_authors: HashSet, + /// Essential live repository and root coverage on accepted owners' and + /// maintainers' read/unmarked NIP-65 inboxes. Root-author inbox roots are + /// unioned into the same live target map, while participant write/outbox + /// coverage remains on the history-only mailbox path below. inbox_roots: HashMap>, + inbox_repositories: HashMap>, mailbox_roots: HashMap>, mailbox_repositories: HashMap>, mailbox_probe_next_at: HashMap, @@ -5647,6 +5652,7 @@ impl SyncManager { async fn derive_targets(&self) -> HashMap { let repo_index = self.repo_sync_index.read().await; let mut targets = algorithms::derive_relay_targets(&repo_index); + discovery::merge_inbox_repositories(&mut targets, &self.nip65_discovery.inbox_repositories); discovery::merge_inbox_roots(&mut targets, &self.nip65_discovery.inbox_roots); targets } @@ -5847,12 +5853,23 @@ impl SyncManager { }); self.nip65_discovery.next_inventory_at = Some(now + nip65_inventory_interval()); let fallback_relays = self.configured_nip65_fallback_relays(); - let inbox_overlay = discovery::build_inbox_root_overlay( + let mut inbox_overlay = discovery::build_inbox_root_overlay( &self.nip65_discovery.root_author_roots, &self.nip65_discovery.author_inboxes, &self.nip65_discovery.fallback_authors, &fallback_relays, ); + let inbox_repository_overlay = discovery::build_mailbox_repository_overlay( + &self.nip65_discovery.author_repositories, + &self.nip65_discovery.author_inboxes, + &self.nip65_discovery.fallback_authors, + &fallback_relays, + ); + merge_repository_roots_into_mailbox_overlay( + &mut inbox_overlay, + &inbox_repository_overlay, + &self.nip65_discovery.root_repositories, + ); let mut mailbox_overlay = discovery::build_inbox_root_overlay( &self.nip65_discovery.author_roots, &self.nip65_discovery.author_mailboxes, @@ -5870,7 +5887,8 @@ impl SyncManager { &mailbox_repository_overlay, &self.nip65_discovery.root_repositories, ); - self.install_nip65_inbox_overlay(inbox_overlay).await; + self.install_nip65_inbox_overlay(inbox_overlay, inbox_repository_overlay) + .await; self.install_nip65_mailbox_overlay(mailbox_overlay, mailbox_repository_overlay); tracing::info!( root_count = root_ids.len(), @@ -5969,19 +5987,30 @@ impl SyncManager { async fn install_nip65_inbox_overlay( &mut self, - new_overlay: HashMap>, + new_root_overlay: HashMap>, + new_repository_overlay: HashMap>, ) { - let old_overlay = std::mem::replace(&mut self.nip65_discovery.inbox_roots, new_overlay); - if old_overlay == self.nip65_discovery.inbox_roots { + let old_root_overlay = + std::mem::replace(&mut self.nip65_discovery.inbox_roots, new_root_overlay); + let old_repository_overlay = std::mem::replace( + &mut self.nip65_discovery.inbox_repositories, + new_repository_overlay, + ); + if old_root_overlay == self.nip65_discovery.inbox_roots + && old_repository_overlay == self.nip65_discovery.inbox_repositories + { return; } - let mut dirty_relays: HashSet = old_overlay.keys().cloned().collect(); + let mut dirty_relays: HashSet = old_root_overlay.keys().cloned().collect(); dirty_relays.extend(self.nip65_discovery.inbox_roots.keys().cloned()); + dirty_relays.extend(old_repository_overlay.keys().cloned()); + dirty_relays.extend(self.nip65_discovery.inbox_repositories.keys().cloned()); for relay in dirty_relays { - // Preserve the established root-author inbox behavior: additions - // enter ordinary live/rotating descendant coverage, while - // removal-only changes drain through that coverage's normal + // Repository owner/maintainer inboxes enter the same essential + // live target layer as root-author inboxes. Recursive descendant + // expansion remains auxiliary and may rotate under pressure. + // Removal-only changes drain through the ordinary coverage // lifecycle instead of churning shared subscriptions. self.recompute_new_sync_filters_for_relay(&relay).await; } @@ -6326,12 +6355,23 @@ impl SyncManager { return; } let fallback_relays = self.configured_nip65_fallback_relays(); - let inbox_overlay = discovery::build_inbox_root_overlay( + let mut inbox_overlay = discovery::build_inbox_root_overlay( &self.nip65_discovery.root_author_roots, &self.nip65_discovery.author_inboxes, &self.nip65_discovery.fallback_authors, &fallback_relays, ); + let inbox_repository_overlay = discovery::build_mailbox_repository_overlay( + &self.nip65_discovery.author_repositories, + &self.nip65_discovery.author_inboxes, + &self.nip65_discovery.fallback_authors, + &fallback_relays, + ); + merge_repository_roots_into_mailbox_overlay( + &mut inbox_overlay, + &inbox_repository_overlay, + &self.nip65_discovery.root_repositories, + ); let mut mailbox_overlay = discovery::build_inbox_root_overlay( &self.nip65_discovery.author_roots, &self.nip65_discovery.author_mailboxes, @@ -6349,7 +6389,8 @@ impl SyncManager { &mailbox_repository_overlay, &self.nip65_discovery.root_repositories, ); - self.install_nip65_inbox_overlay(inbox_overlay).await; + self.install_nip65_inbox_overlay(inbox_overlay, inbox_repository_overlay) + .await; self.install_nip65_mailbox_overlay(mailbox_overlay, mailbox_repository_overlay); tracing::info!( source = %result.source_relay, diff --git a/tests/sync/proactive_sync_plus.rs b/tests/sync/proactive_sync_plus.rs index 0cc72eb..391af2a 100644 --- a/tests/sync/proactive_sync_plus.rs +++ b/tests/sync/proactive_sync_plus.rs @@ -311,7 +311,7 @@ async fn participant_write_mailbox_fetches_child_of_direct_reaction() { } #[tokio::test] -async fn owner_mailbox_discovers_unknown_root_and_root_only_status() { +async fn owner_inbox_live_sync_discovers_unknown_root_and_root_only_status() { let index = MockRelay::start().await; let mailbox = MockRelay::start().await; let owner = Keys::generate(); @@ -319,7 +319,7 @@ async fn owner_mailbox_discovers_unknown_root_and_root_only_status() { let identifier = "owner-repository-mailbox"; let relay_list = EventBuilder::new(Kind::RelayList, "") - .tag(Tag::custom("r", vec![mailbox.url(), "write"])) + .tag(Tag::custom("r", vec![mailbox.url(), "read"])) .finalize(&owner) .expect("build owner relay list"); send_to_relay_url(index.url(), &relay_list) @@ -383,10 +383,29 @@ async fn owner_mailbox_discovers_unknown_root_and_root_only_status() { "repository mailbox ownership should remain observable without public-key labels" ); assert!( - !sync_log + sync_log .lines() .any(|line| { line.contains("Starting fresh_start") && line.contains(mailbox.url()) }), - "an owner mailbox must remain history-only rather than becoming a repository source" + "an owner inbox should enter ordinary repository live sync" + ); + + let live_issue = build_layer2_issue_event( + &root_author, + &repo_coord(&owner, identifier), + "root published after owner-inbox live coverage was installed", + ) + .expect("build live mailbox root"); + send_to_relay_url(mailbox.url(), &live_issue) + .await + .expect("publish root after live coverage"); + assert!( + wait_for_event_on_relay( + syncing.url(), + Filter::new().id(live_issue.id), + Duration::from_secs(5), + ) + .await, + "owner inbox repository coverage should remain live after initial history" ); syncing.stop().await;