From c0a023c7f2a321c9e10d806eda0df55f6f49515d Mon Sep 17 00:00:00 2001 From: DanConwayDev Date: Wed, 19 Aug 2026 15:47:44 +0000 Subject: [PATCH] fix(sync): keep maintainer repository inboxes historical The earlier commit in this PR made every accepted repository owner's and declared maintainer's read/unmarked inboxes ordinary persistent live repository sources. Combined with unbounded per-author relay lists this drove the soak's connection growth: merely owning a repository let an author's advertised inventory imply long-lived live subscriptions. The 2026-08-19 investigation showed one bot-owned repository contributing 983 relay URLs through exactly this path. Owner/maintainer repository scope is now historical-only. The live inbox_repositories tier and its merge into derive_targets are removed entirely, and the repository-scoped overlay moves onto the existing paced, byte-bounded mailbox history workers using each author's bounded, sanitized read/unmarked inbox selection. Repository coordinates and known roots are still probed there, so a root or status stored only on a maintainer mailbox is still discovered - on the recurring history cycle instead of a persistent subscription. Correctness assumptions: inbox_roots is the pre-existing root-author live tier and is deliberately untouched, so accepted root authors keep live coverage on their own bounded inboxes. Participant mailboxes were already history-only. public_repository_mailbox_scope still empties repository scope in private mode before any overlay is built, and the per-relay independent mailbox worker behavior is unchanged. Excluded scope: no changes to instance-own-relay or repository-advertised relay coverage, no event-triggered drains, negentropy routing, or scheduling changes; those remain follow-up work. Validation: cargo fmt, cargo clippy --all-targets (clean), cargo test --lib (884 passed), cargo test --test sync proactive_sync (4 passed; the owner scenario now proves the inbox is probed through the history path, that a later root still arrives via historical refresh, and that the inbox never enters ordinary live sync). --- CHANGELOG.md | 18 +++-- docs/explanation/architecture.md | 17 ++--- .../grasp-03-proactive-sync-plus.md | 67 ++++++++--------- src/sync/discovery.rs | 33 --------- src/sync/mod.rs | 73 ++++++------------- tests/sync/proactive_sync_plus.rs | 37 ++++++---- 6 files changed, 95 insertions(+), 150 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f1ca54..4031b87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -156,14 +156,16 @@ Performance and Security fixes - along with other improvements; immediate upgrad ### Fixed -- Discover historical and live repository roots and descendants from the - NIP-65 mailboxes of accepted repository owners and maintainers. Public Sync+ - instances make exact repository coordinates and every known root essential - live coverage on read/unmarked inboxes, above recursive descendant fan-out, - while their wider mailbox set continues through paced, byte-bounded history - workers. Private instances continue to withhold repository coordinates, and - the ordinary write policy and persistent deletion tombstones remain - authoritative. +- Discover historical repository roots and descendants from the NIP-65 + inboxes of accepted repository owners and maintainers. Public Sync+ + instances probe exact repository coordinates and every known root on those + authors' bounded, sanitized read/unmarked inboxes through paced, + byte-bounded history workers. This coverage is deliberately historical-only: + owning or maintaining a repository never adds an author's inbox relays to + ordinary persistent live repository targets, while accepted root authors + retain the pre-existing live inbox coverage. Private instances continue to + withhold repository coordinates, and the ordinary write policy and + persistent deletion tombstones remain authoritative. - Retire peer-closed outbound subscriptions from rust-nostr's desired registry without interrupting NIP-42: the first `auth-required` response keeps the diff --git a/docs/explanation/architecture.md b/docs/explanation/architecture.md index 6bf6a43..5eb6f98 100644 --- a/docs/explanation/architecture.md +++ b/docs/explanation/architecture.md @@ -693,15 +693,14 @@ filter cursor without waiting for a global relay rotation. They reuse the connection's ordinary pacing, subscription ledger, pagination and 30-second per-page terminal timeout without installing permanent participant subscriptions or coupling progress between relays. On public instances, each -accepted repository owner's and declared maintainer's NIP-65 read/unmarked -inboxes also become ordinary full repository sources for their exact -repositories and all locally known roots. Their canonical repository/root -filters are essential live coverage—equal to root-author canonical inbox -coverage and above recursive descendant fan-out—while their wider mailbox set -continues through the history workers. This discovers previously unknown roots -and descendants that exist only on a maintainer mailbox without broadening an -ordinary participant mailbox. Private instances omit repository-coordinate -mailbox expansion. +accepted repository owner's and declared maintainer's bounded, sanitized +read/unmarked NIP-65 inboxes receive repository-scoped historical mailbox +probes covering their exact repositories and all locally known roots. This +discovers previously unknown roots and descendants that exist only on a +maintainer mailbox without broadening an ordinary participant mailbox, and it +is deliberately historical-only: owning or maintaining a repository never adds +that author's inbox relays to ordinary persistent live repository targets. +Private instances omit repository-coordinate mailbox expansion. ### Rejected Events Index diff --git a/docs/explanation/grasp-03-proactive-sync-plus.md b/docs/explanation/grasp-03-proactive-sync-plus.md index 1ed1595..0497ac1 100644 --- a/docs/explanation/grasp-03-proactive-sync-plus.md +++ b/docs/explanation/grasp-03-proactive-sync-plus.md @@ -19,11 +19,11 @@ existing index set for each author's latest profile kind `0` and NIP-65 kind `read` and unmarked inboxes in ordinary GRASP-02 coverage; and probe accepted participants' `read`, `write` and unmarked conversation mailboxes with repository and thread-reference filters. On public instances, accepted -repository owners' and declared maintainers' `read` and unmarked inboxes also -enter ordinary GRASP-02 repository coverage for their exact repositories; -their wider read/write mailbox set retains the history probe. Repository -coordinates on those inboxes can therefore discover a root which is not yet -present locally and continue receiving new roots live. +repository owners' and declared maintainers' bounded `read` and unmarked +inboxes additionally receive repository-scoped historical mailbox probes. +Repository coordinates on those inboxes can therefore discover a root which is +not yet present locally on the recurring history cycle; the inboxes do not +enter ordinary GRASP-02 live repository coverage. Kind `10002` lists are untrusted peer input, so every relay URL they advertise passes single-URL target hygiene before it can occupy any selection slot, @@ -45,13 +45,10 @@ not subject to peer-input hygiene. Mailbox work reuses GRASP-02's connection safety, filter byte packing, pagination, subscription ledger, request pacing, event pipeline and write -policy. The participant expansion is history-only: a non-root participant +policy. The participant and owner/maintainer expansions are history-only: a relay never becomes a permanent live source merely because an accepted author -advertised it. Root-author inboxes retain the pre-existing live/rotating Sync+ -behavior. Owner/maintainer repository inbox filters are essential core -coverage, equal to canonical root-author inbox filters and above recursive -descendant fan-out; compatibility tag variants may rotate before canonical -coverage is sacrificed. +advertised it or because its author owns or maintains a repository. +Root-author inboxes retain the pre-existing live/rotating Sync+ behavior. Authors without an accepted stored relay list are queried through the configured user-index set plus the bootstrap relay. Once a list is retained, discovery @@ -104,9 +101,9 @@ database before any network refresh, so serving established coverage does not depend on an external index remaining available. Remote discovery then refreshes that retained state on the normal cadence. -Root-author and repository owner/maintainer read/unmarked inboxes use ordinary -GRASP-02 coverage. Wider participant mailboxes and owner/maintainer write-only -outboxes use history-only workers. A maintenance pass admits at most one new +Root-author read/unmarked inboxes use ordinary GRASP-02 coverage. Participant +mailboxes and owner/maintainer repository-scoped inboxes use history-only +workers. A maintenance pass admits at most one new due history relay, while each admitted relay may have at most one worker in flight and drains its own filter cursor without returning to a global round-robin between successful groups. Up to 32 relay workers may run at once @@ -128,11 +125,11 @@ or cross-relay coordinator is added. Coverage uses accepted repository coordinates, root IDs, bounded recursive descendant IDs, and descendant address coordinates with `a`/`A`/`q` and `e`/`E`/`q`. A repository-scoped owner or maintainer inbox receives the exact -repository and every currently known root in it as core live and historic -work. This permits a status or other descendant stored only on that inbox to -be found even when the root author's current relay list does not name it. The -wider mailbox history probe covers the same references and a participant-only -mailbox remains constrained to its derived roots. A numeric in-memory cursor +repository and every currently known root in it as historical probe work. This +permits a status or other descendant stored only on that inbox to be found on +the recurring history cycle even when the root author's current relay list +does not name it. A participant-only mailbox remains constrained to its +derived roots. A numeric in-memory cursor gives each byte-bounded history filter group a turn. A successful complete rotation refreshes after 24 hours; a failed group advances the cursor after a five-minute delay and is retried on a later rotation. A relay already needed @@ -140,20 +137,17 @@ for ordinary repository sync shares its connection; an exclusively history/control-plane connection retires when its identity and mailbox work is idle. -Owner/maintainer expansion adds a persistent managed connection and essential -live repository/root filters for each distinct read inbox, plus history -rotation proportional to the byte-packed repository coordinates and known -roots assigned to every mailbox. Shared relays and duplicate repository scopes -are unioned before filter construction. Canonical live filters consume normal -GRASP-02 subscription-ledger capacity; compatibility variants and recursive -descendants retain the existing priority cutoff and history fallback. History -still admits at most one new relay per maintenance pass and remains -single-flight per relay, but an admitted relay keeps its worker while its -successful filter groups drain. More distinct maintainer relays can therefore -increase both persistent connection/subscription load and active history -workers up to the process-wide safety ceiling; the -fixed-cardinality relay, cursor, worker, connection, and subscription metrics -must be watched during a production soak. +Owner/maintainer expansion adds history rotation proportional to the +byte-packed repository coordinates and known roots assigned to each of their +bounded read/unmarked inboxes; it adds no persistent live filters or +subscriptions. Shared relays and duplicate repository scopes are unioned +before filter construction. History still admits at most one new relay per +maintenance pass and remains single-flight per relay, but an admitted relay +keeps its worker while its successful filter groups drain. With hygiene and +the four-relay per-author bound applied before selection, distinct maintainer +relays increase only active history workers up to the process-wide safety +ceiling; the fixed-cardinality relay, cursor, worker, connection, and +subscription metrics must be watched during a production soak. On restart, accepted roots and retained kind `10002` events rebuild participant and mailbox ownership from LMDB. The in-memory group cursor is intentionally @@ -168,8 +162,8 @@ This makes a stuck worker or unexpected inventory expansion visible without putting peer URLs or public keys into metric labels. Mailbox replacement/removal changes desired ownership immediately. Root-author -and owner/maintainer repository inbox additions use ordinary coverage, while -removal-only changes let existing shared live subscriptions drain naturally. +inbox additions use ordinary coverage, while removal-only changes let existing +shared live subscriptions drain naturally. History-probe additions become due promptly; removals prevent future groups while an already in-flight `fetch_events` worker may finish naturally. Root deletion follows the existing GRASP-02 root-index lifecycle and is @@ -187,4 +181,5 @@ add a second deletion graph. - identity storage for authors outside accepted repositories and their locally accepted threads; - per-user fallback configuration knobs; and -- permanent non-root participant-mailbox live subscriptions. +- permanent live subscriptions on non-root participant mailboxes or on + owner/maintainer inboxes discovered only through repository ownership. diff --git a/src/sync/discovery.rs b/src/sync/discovery.rs index 3cea2ca..416cd88 100644 --- a/src/sync/discovery.rs +++ b/src/sync/discovery.rs @@ -279,22 +279,6 @@ pub fn merge_inbox_roots( } } -/// Add accepted owner/maintainer inbox repositories to ordinary Full sync. -/// These targets receive canonical live coverage before auxiliary descendant -/// fan-out, while write-only mailboxes remain absent from this overlay. -pub fn merge_inbox_repositories( - targets: &mut HashMap, - inbox_repositories: &HashMap>, -) { - for (relay, repositories) in inbox_repositories { - targets - .entry(relay.clone()) - .or_default() - .repos - .extend(repositories.iter().cloned()); - } -} - pub fn build_inbox_root_overlay( author_roots: &HashMap>, author_inboxes: &HashMap>, @@ -837,23 +821,6 @@ mod tests { assert!(target.state_only_repos.is_empty()); } - #[test] - fn repository_inbox_overlay_adds_full_repository_work() { - let repository = "30617:owner:repo".to_string(); - let mut targets = HashMap::new(); - merge_inbox_repositories( - &mut targets, - &HashMap::from([( - "wss://inbox.example".to_string(), - HashSet::from([repository.clone()]), - )]), - ); - let target = &targets["wss://inbox.example"]; - assert_eq!(target.repos, HashSet::from([repository])); - assert!(target.root_events.is_empty()); - assert!(target.state_only_repos.is_empty()); - } - #[test] fn replacement_and_root_removal_subtract_from_desired_overlay() { let author = Keys::generate().public_key(); diff --git a/src/sync/mod.rs b/src/sync/mod.rs index 8b643e7..1582ccb 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -1941,12 +1941,12 @@ struct Nip65DiscoveryState { /// no accepted relay list. They use the operator's bounded fallback set /// until an accepted kind 10002 arrives. fallback_authors: HashSet, - /// Essential live repository and root coverage on accepted owners' and - /// maintainers' read/unmarked NIP-65 inboxes. Root-author inbox roots are - /// unioned into the same live target map, while participant write/outbox - /// coverage remains on the history-only mailbox path below. + /// Essential live root coverage on accepted root authors' read/unmarked + /// NIP-65 inboxes. Owner/maintainer repository scope and participant + /// write/outbox coverage remain on the history-only mailbox path below: + /// owning or maintaining a repository never makes an author's inbox an + /// ordinary persistent live target. inbox_roots: HashMap>, - inbox_repositories: HashMap>, mailbox_roots: HashMap>, mailbox_repositories: HashMap>, mailbox_probe_next_at: HashMap, @@ -5671,7 +5671,6 @@ impl SyncManager { async fn derive_targets(&self) -> HashMap { let repo_index = self.repo_sync_index.read().await; let mut targets = algorithms::derive_relay_targets(&repo_index); - discovery::merge_inbox_repositories(&mut targets, &self.nip65_discovery.inbox_repositories); discovery::merge_inbox_roots(&mut targets, &self.nip65_discovery.inbox_roots); targets } @@ -5873,32 +5872,24 @@ impl SyncManager { }); self.nip65_discovery.next_inventory_at = Some(now + nip65_inventory_interval()); let fallback_relays = self.configured_nip65_fallback_relays(); - let mut inbox_overlay = discovery::build_inbox_root_overlay( + let inbox_overlay = discovery::build_inbox_root_overlay( &self.nip65_discovery.root_author_roots, &self.nip65_discovery.author_inboxes, &self.nip65_discovery.fallback_authors, &fallback_relays, ); - let inbox_repository_overlay = discovery::build_mailbox_repository_overlay( - &self.nip65_discovery.author_repositories, - &self.nip65_discovery.author_inboxes, - &self.nip65_discovery.fallback_authors, - &fallback_relays, - ); - merge_repository_roots_into_mailbox_overlay( - &mut inbox_overlay, - &inbox_repository_overlay, - &self.nip65_discovery.root_repositories, - ); let mut mailbox_overlay = discovery::build_inbox_root_overlay( &self.nip65_discovery.author_roots, &self.nip65_discovery.author_mailboxes, &self.nip65_discovery.fallback_authors, &fallback_relays, ); + // Owner/maintainer repository scope is historical-only, on their + // bounded read/unmarked inbox selection: owning a repository must + // not add an author's inboxes to persistent live targets. let mailbox_repository_overlay = discovery::build_mailbox_repository_overlay( &self.nip65_discovery.author_repositories, - &self.nip65_discovery.author_mailboxes, + &self.nip65_discovery.author_inboxes, &self.nip65_discovery.fallback_authors, &fallback_relays, ); @@ -5907,8 +5898,7 @@ impl SyncManager { &mailbox_repository_overlay, &self.nip65_discovery.root_repositories, ); - self.install_nip65_inbox_overlay(inbox_overlay, inbox_repository_overlay) - .await; + self.install_nip65_inbox_overlay(inbox_overlay).await; self.install_nip65_mailbox_overlay(mailbox_overlay, mailbox_repository_overlay); tracing::info!( root_count = root_ids.len(), @@ -6008,30 +5998,20 @@ impl SyncManager { async fn install_nip65_inbox_overlay( &mut self, new_root_overlay: HashMap>, - new_repository_overlay: HashMap>, ) { let old_root_overlay = std::mem::replace(&mut self.nip65_discovery.inbox_roots, new_root_overlay); - let old_repository_overlay = std::mem::replace( - &mut self.nip65_discovery.inbox_repositories, - new_repository_overlay, - ); - if old_root_overlay == self.nip65_discovery.inbox_roots - && old_repository_overlay == self.nip65_discovery.inbox_repositories - { + if old_root_overlay == self.nip65_discovery.inbox_roots { return; } let mut dirty_relays: HashSet = old_root_overlay.keys().cloned().collect(); dirty_relays.extend(self.nip65_discovery.inbox_roots.keys().cloned()); - dirty_relays.extend(old_repository_overlay.keys().cloned()); - dirty_relays.extend(self.nip65_discovery.inbox_repositories.keys().cloned()); for relay in dirty_relays { - // Repository owner/maintainer inboxes enter the same essential - // live target layer as root-author inboxes. Recursive descendant - // expansion remains auxiliary and may rotate under pressure. - // Removal-only changes drain through the ordinary coverage - // lifecycle instead of churning shared subscriptions. + // Root-author inbox roots are essential live coverage. Recursive + // descendant expansion remains auxiliary and may rotate under + // pressure. Removal-only changes drain through the ordinary + // coverage lifecycle instead of churning shared subscriptions. self.recompute_new_sync_filters_for_relay(&relay).await; } } @@ -6393,32 +6373,24 @@ impl SyncManager { return; } let fallback_relays = self.configured_nip65_fallback_relays(); - let mut inbox_overlay = discovery::build_inbox_root_overlay( + let inbox_overlay = discovery::build_inbox_root_overlay( &self.nip65_discovery.root_author_roots, &self.nip65_discovery.author_inboxes, &self.nip65_discovery.fallback_authors, &fallback_relays, ); - let inbox_repository_overlay = discovery::build_mailbox_repository_overlay( - &self.nip65_discovery.author_repositories, - &self.nip65_discovery.author_inboxes, - &self.nip65_discovery.fallback_authors, - &fallback_relays, - ); - merge_repository_roots_into_mailbox_overlay( - &mut inbox_overlay, - &inbox_repository_overlay, - &self.nip65_discovery.root_repositories, - ); let mut mailbox_overlay = discovery::build_inbox_root_overlay( &self.nip65_discovery.author_roots, &self.nip65_discovery.author_mailboxes, &self.nip65_discovery.fallback_authors, &fallback_relays, ); + // Owner/maintainer repository scope is historical-only, on their + // bounded read/unmarked inbox selection: owning a repository must + // not add an author's inboxes to persistent live targets. let mailbox_repository_overlay = discovery::build_mailbox_repository_overlay( &self.nip65_discovery.author_repositories, - &self.nip65_discovery.author_mailboxes, + &self.nip65_discovery.author_inboxes, &self.nip65_discovery.fallback_authors, &fallback_relays, ); @@ -6427,8 +6399,7 @@ impl SyncManager { &mailbox_repository_overlay, &self.nip65_discovery.root_repositories, ); - self.install_nip65_inbox_overlay(inbox_overlay, inbox_repository_overlay) - .await; + self.install_nip65_inbox_overlay(inbox_overlay).await; self.install_nip65_mailbox_overlay(mailbox_overlay, mailbox_repository_overlay); tracing::info!( source = %result.source_relay, diff --git a/tests/sync/proactive_sync_plus.rs b/tests/sync/proactive_sync_plus.rs index 2da1da3..3ff951c 100644 --- a/tests/sync/proactive_sync_plus.rs +++ b/tests/sync/proactive_sync_plus.rs @@ -315,7 +315,7 @@ async fn participant_write_mailbox_fetches_child_of_direct_reaction() { } #[tokio::test] -async fn owner_inbox_live_sync_discovers_unknown_root_and_root_only_status() { +async fn owner_inbox_history_discovers_unknown_root_without_live_sync() { let index = MockRelay::start().await; let mailbox = MockRelay::start().await; let owner = Keys::generate(); @@ -387,29 +387,40 @@ async fn owner_inbox_live_sync_discovers_unknown_root_and_root_only_status() { "repository mailbox ownership should remain observable without public-key labels" ); assert!( - sync_log - .lines() - .any(|line| { line.contains("Starting fresh_start") && line.contains(mailbox.url()) }), - "an owner inbox should enter ordinary repository live sync" + sync_log.lines().any(|line| { + line.contains("Started bounded participant mailbox fetch") + && line.contains(mailbox.url()) + }), + "the owner inbox should be probed through the repository-scoped history path" ); - let live_issue = build_layer2_issue_event( + // A root published later is still discovered, but only through the + // recurring historical probe cycle: the owner inbox never becomes an + // ordinary live repository source. + let later_issue = build_layer2_issue_event( &root_author, &repo_coord(&owner, identifier), - "root published after owner-inbox live coverage was installed", + "root published after initial owner-inbox history", ) - .expect("build live mailbox root"); - send_to_relay_url(mailbox.url(), &live_issue) + .expect("build later mailbox root"); + send_to_relay_url(mailbox.url(), &later_issue) .await - .expect("publish root after live coverage"); + .expect("publish root after initial history"); assert!( wait_for_event_on_relay( syncing.url(), - Filter::new().id(live_issue.id), - Duration::from_secs(5), + Filter::new().id(later_issue.id), + Duration::from_secs(30), ) .await, - "owner inbox repository coverage should remain live after initial history" + "owner inbox repository coverage should continue through historical refresh" + ); + let sync_log = std::fs::read_to_string(syncing.log_path()).expect("read syncing relay log"); + assert!( + !sync_log + .lines() + .any(|line| { line.contains("Starting fresh_start") && line.contains(mailbox.url()) }), + "an owner inbox must not enter ordinary repository live sync" ); syncing.stop().await;