feat(sync): cover roots with missing relay lists

Some accepted root authors do not publish a discoverable NIP-65 kind 10002, leaving their conversations outside Sync+ even after a successful user-index lookup. Use a bounded operator-configured fallback relay set only after such a successful empty lookup.

Feed fallback roots into the existing GRASP-02 target overlay, so historic, live, rotating, subscription-budget, and recovery behavior remain unified. A later accepted relay list removes the author from desired fallback coverage while shared live subscriptions drain naturally.

Keep eligibility restricted to accepted root authors and preserve single-flight identity discovery. This does not expand to response authors, add another scheduler, or infer fallback need from failed queries.

Validated with the end-to-end missing-list fallback and later-list replacement scenario, discovery/config unit tests, Nix module parsing, rustfmt, clippy with warnings denied, and diff checks.
This commit is contained in:
DanConwayDev
2026-08-12 17:44:00 +00:00
parent 6475e8e25f
commit e84be42446
10 changed files with 295 additions and 3 deletions
+5
View File
@@ -130,6 +130,11 @@
# Default: wss://purplepag.es,wss://index.hzrd149.com,wss://indexer.coracle.social
# NGIT_USER_INDEX_RELAYS=wss://purplepag.es,wss://index.hzrd149.com,wss://indexer.coracle.social
# Bounded inbox fallbacks for eligible authors whose NIP-65 list was not found
# CLI: --sync-plus-fallback-relays <comma-separated-websocket-urls>
# Default: wss://relay.ditto.pub,wss://relay.damus.io,wss://nos.lol,wss://relay.primal.net
# NGIT_SYNC_PLUS_FALLBACK_RELAYS=wss://relay.ditto.pub,wss://relay.damus.io,wss://nos.lol,wss://relay.primal.net
# Maximum backoff time in seconds for sync relay reconnection
# CLI: --sync-max-backoff-secs <seconds>
# Default: 3600 (1 hour)
+2
View File
@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- Add bounded, operator-configurable inbox fallback coverage when a successful
Sync+ user-index query finds no accepted NIP-65 relay list for a root author.
- Add a default-on `NGIT_SYNC_PLUS_ENABLED` opt-out and advertise GRASP-03 in
NIP-11 only when the Sync+ overlay is effective.
- Recover repository-event descendants which reference a direct thread member
@@ -27,6 +27,14 @@ follows its `write` and unmarked outboxes instead, allowing newer replacements
to converge without making arbitrary repository relays identity sources.
Discovery remains best effort and bounded to this operator-visible source graph.
If a successful user-index query returns no accepted kind `10002`, accepted
roots by that author temporarily use the operator-configured Sync+ fallback
relay set as inboxes. This is the same root-only overlay consumed by ordinary
GRASP-02 historic, live and rotating coverage; it creates no separate
subscription scheduler. An accepted relay list removes the author from desired
fallback coverage and installs its declared inboxes. Existing shared live
subscriptions still drain naturally, as with any other relay-list replacement.
The self-subscriber builds a compact root-candidate inventory during its
existing startup load and maintains it incrementally as roots arrive. StateOnly
candidates remain inert; promotion of their repository to Full makes them
@@ -73,5 +81,5 @@ accepted events on restart; this change does not add a second deletion graph.
- the old recursive `SyncScope` graph and response-author fan-out;
- maintainer mailbox expansion unrelated to an accepted root;
- identity storage for authors other than accepted roots;
- fallback lists and per-user configuration knobs; and
- per-user fallback configuration knobs; and
- a separate GRASP-03 subscription scheduler.
+19
View File
@@ -358,6 +358,25 @@ strings. Empty comma-separated entries and surrounding whitespace are ignored.
---
#### `NGIT_SYNC_PLUS_FALLBACK_RELAYS`
**Description:** Bounded inbox fallback relays for eligible Sync+ authors whose NIP-65 list was not found
**Type:** String list (comma-separated WebSocket URLs)
**Default:** `wss://relay.ditto.pub,wss://relay.damus.io,wss://nos.lol,wss://relay.primal.net`
**Required:** No
```bash
NGIT_SYNC_PLUS_FALLBACK_RELAYS=wss://relay.ditto.pub,wss://nos.lol
```
The corresponding NixOS option is `syncPlusFallbackRelays`. Fallback coverage
starts only after a successful user-index query returns no accepted kind
`10002` for an eligible author. A subsequently discovered relay list replaces
the fallback in desired coverage without eagerly closing shared subscriptions.
Set the list empty to disable this recovery while retaining ordinary Sync+.
---
#### `NGIT_SYNC_ALLOW_NON_GLOBAL_TARGETS`
**Description:** Allow event-directed sync targets that are not globally reachable
+16
View File
@@ -124,6 +124,20 @@ let
"Relays used to discover eligible accepted repository participants' NIP-65 relay lists";
};
syncPlusFallbackRelays = mkOption {
type = types.listOf types.str;
default = [
"wss://relay.ditto.pub"
"wss://relay.damus.io"
"wss://nos.lol"
"wss://relay.primal.net"
];
description = ''
Bounded inbox fallback relays used for eligible Sync+ authors only
after a user-index query succeeds without a NIP-65 relay list.
'';
};
databaseBackend = mkOption {
type = types.enum [ "lmdb" "memory" ];
default = "lmdb";
@@ -534,6 +548,8 @@ let
NGIT_RELAY_MAX_EVENT_SIZE_BYTES = toString cfg.relayMaxEventSizeBytes;
NGIT_RELAY_FILTER_LIMIT = toString cfg.relayFilterLimit;
NGIT_USER_INDEX_RELAYS = concatStringsSep "," cfg.userIndexRelays;
NGIT_SYNC_PLUS_FALLBACK_RELAYS =
concatStringsSep "," cfg.syncPlusFallbackRelays;
} // optionalAttrs (cfg.maxConnections != null) {
NGIT_MAX_CONNECTIONS = toString cfg.maxConnections;
} // optionalAttrs (cfg.relayName != null) {
+45
View File
@@ -18,6 +18,8 @@ const DEFAULT_DELETION_REQUEST_RETENTION_USED_UNSERVED_GATING_ADDITIONAL_SECS: u
90 * 24 * 60 * 60;
const DEFAULT_USER_INDEX_RELAYS: &str =
"wss://purplepag.es,wss://index.hzrd149.com,wss://indexer.coracle.social";
const DEFAULT_SYNC_PLUS_FALLBACK_RELAYS: &str =
"wss://relay.ditto.pub,wss://relay.damus.io,wss://nos.lol,wss://relay.primal.net";
/// Whitelist entry for repository/archive filtering
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
@@ -407,6 +409,14 @@ pub struct Config {
)]
pub user_index_relays: String,
/// Comma-separated inbox fallbacks for eligible authors whose NIP-65 relay list is absent.
#[arg(
long,
env = "NGIT_SYNC_PLUS_FALLBACK_RELAYS",
default_value = DEFAULT_SYNC_PLUS_FALLBACK_RELAYS
)]
pub sync_plus_fallback_relays: String,
/// Maximum backoff time in seconds for sync relay reconnection (default: 3600 = 1 hour)
#[arg(long, env = "NGIT_SYNC_MAX_BACKOFF_SECS", default_value_t = 3600)]
pub sync_max_backoff_secs: u64,
@@ -980,6 +990,16 @@ impl Config {
.collect()
}
/// Parse the comma-separated Sync+ fallback relay URLs.
pub fn parse_sync_plus_fallback_relays(&self) -> Vec<String> {
self.sync_plus_fallback_relays
.split(',')
.map(str::trim)
.filter(|relay| !relay.is_empty())
.map(str::to_owned)
.collect()
}
/// Get parsed archive configuration with computed read-only mode
///
/// Read-only mode defaults to true if archive mode is enabled, false otherwise.
@@ -1112,6 +1132,7 @@ impl Config {
sync_bootstrap_relay_url: None,
sync_plus_enabled: true,
user_index_relays: DEFAULT_USER_INDEX_RELAYS.to_string(),
sync_plus_fallback_relays: DEFAULT_SYNC_PLUS_FALLBACK_RELAYS.to_string(),
sync_max_backoff_secs: 3600,
sync_disconnect_check_interval_secs: 60,
sync_base_backoff_secs: 5,
@@ -1230,6 +1251,30 @@ mod tests {
);
}
#[test]
fn sync_plus_fallback_defaults_and_empty_override_are_explicit() {
let default = Config::try_parse_from(["ngit-grasp", "--domain", "example.com"])
.expect("Sync+ fallback defaults should parse");
assert_eq!(
default.parse_sync_plus_fallback_relays(),
vec![
"wss://relay.ditto.pub",
"wss://relay.damus.io",
"wss://nos.lol",
"wss://relay.primal.net",
]
);
let disabled = Config::try_parse_from([
"ngit-grasp",
"--domain",
"example.com",
"--sync-plus-fallback-relays=",
])
.expect("empty Sync+ fallback override should parse");
assert!(disabled.parse_sync_plus_fallback_relays().is_empty());
}
#[test]
fn sync_plus_is_enabled_by_default_and_can_be_disabled() {
let default = Config::try_parse_from(["ngit-grasp", "--domain", "example.com"])
+47 -1
View File
@@ -191,6 +191,8 @@ pub fn merge_inbox_roots(
pub fn build_inbox_root_overlay(
author_roots: &HashMap<PublicKey, HashSet<EventId>>,
author_inboxes: &HashMap<PublicKey, HashSet<String>>,
fallback_authors: &HashSet<PublicKey>,
fallback_relays: &HashSet<String>,
) -> HashMap<String, HashSet<EventId>> {
let mut overlay: HashMap<String, HashSet<EventId>> = HashMap::new();
for (author, inboxes) in author_inboxes {
@@ -204,6 +206,17 @@ pub fn build_inbox_root_overlay(
.extend(roots.iter().copied());
}
}
for author in fallback_authors {
let Some(roots) = author_roots.get(author) else {
continue;
};
for relay in fallback_relays {
overlay
.entry(relay.clone())
.or_default()
.extend(roots.iter().copied());
}
}
overlay
}
@@ -378,16 +391,26 @@ mod tests {
let first = build_inbox_root_overlay(
&roots,
&HashMap::from([(author, HashSet::from(["wss://a.example".to_string()]))]),
&HashSet::new(),
&HashSet::new(),
);
assert_eq!(first["wss://a.example"], HashSet::from([root]));
let replacement = build_inbox_root_overlay(
&roots,
&HashMap::from([(author, HashSet::from(["wss://b.example".to_string()]))]),
&HashSet::new(),
&HashSet::new(),
);
assert!(!replacement.contains_key("wss://a.example"));
assert_eq!(replacement["wss://b.example"], HashSet::from([root]));
assert!(build_inbox_root_overlay(&HashMap::new(), &HashMap::new()).is_empty());
assert!(build_inbox_root_overlay(
&HashMap::new(),
&HashMap::new(),
&HashSet::new(),
&HashSet::new()
)
.is_empty());
}
#[test]
@@ -437,10 +460,33 @@ mod tests {
let overlay = build_inbox_root_overlay(
&author_roots,
&HashMap::from([(author, HashSet::from(["wss://inbox.example".to_string()]))]),
&HashSet::new(),
&HashSet::new(),
);
assert_eq!(overlay["wss://inbox.example"], HashSet::from([root.id]));
}
#[test]
fn missing_list_authors_use_only_bounded_fallback_relays() {
let author = Keys::generate().public_key();
let unrelated = Keys::generate().public_key();
let root = EventId::from_byte_array([11; 32]);
let overlay = build_inbox_root_overlay(
&HashMap::from([(author, HashSet::from([root]))]),
&HashMap::new(),
&HashSet::from([author, unrelated]),
&HashSet::from([
"wss://fallback-one.example".to_string(),
"wss://fallback-two.example".to_string(),
]),
);
assert_eq!(overlay.len(), 2);
assert!(overlay
.values()
.all(|roots| roots == &HashSet::from([root])));
}
#[test]
fn identity_selection_keeps_latest_profile_and_relay_list_per_author() {
let author = Keys::generate();
+29 -1
View File
@@ -1333,6 +1333,10 @@ struct Nip65DiscoveryState {
author_sources: HashMap<PublicKey, HashSet<String>>,
relay_lists: HashMap<PublicKey, Event>,
author_inboxes: HashMap<PublicKey, HashSet<String>>,
/// Eligible authors for whom at least one successful index query returned
/// no accepted relay list. They use the operator's bounded fallback set
/// until an accepted kind 10002 arrives.
fallback_authors: HashSet<PublicKey>,
inbox_roots: HashMap<String, HashSet<EventId>>,
in_flight: HashSet<(String, PublicKey)>,
next_attempt_at: HashMap<(String, PublicKey), Instant>,
@@ -4815,6 +4819,14 @@ impl SyncManager {
targets
}
fn configured_nip65_fallback_relays(&self) -> HashSet<String> {
self.config
.parse_sync_plus_fallback_relays()
.into_iter()
.filter_map(|url| canonical_relay_key(&url).ok())
.collect()
}
async fn schedule_nip65_discovery(&mut self) {
// GRASP-03 is an optional overlay on the always-running GRASP-02
// manager. When disabled, do not inventory authors, retain identity
@@ -4925,6 +4937,10 @@ impl SyncManager {
self.nip65_discovery
.author_inboxes
.retain(|author, _| current_authors.contains(author));
self.nip65_discovery.fallback_authors.retain(|author| {
current_authors.contains(author)
&& !self.nip65_discovery.relay_lists.contains_key(author)
});
let current_sources = &self.nip65_discovery.author_sources;
self.nip65_discovery
.next_attempt_at
@@ -4937,6 +4953,8 @@ impl SyncManager {
let overlay = discovery::build_inbox_root_overlay(
&self.nip65_discovery.author_roots,
&self.nip65_discovery.author_inboxes,
&self.nip65_discovery.fallback_authors,
&self.configured_nip65_fallback_relays(),
);
self.install_nip65_overlay(overlay).await;
}
@@ -5089,6 +5107,8 @@ impl SyncManager {
.await
.unwrap_or_default();
let latest = discovery::latest_relay_lists(stored_relay_lists, &result.authors);
let authors_with_lists: HashSet<PublicKey> = latest.keys().copied().collect();
let previous_fallback_authors = self.nip65_discovery.fallback_authors.clone();
let now = Instant::now();
for author in &result.authors {
let retry_after =
@@ -5096,9 +5116,14 @@ impl SyncManager {
self.nip65_discovery
.next_attempt_at
.insert((result.source_relay.clone(), *author), now + retry_after);
if authors_with_lists.contains(author) {
self.nip65_discovery.fallback_authors.remove(author);
} else if query_succeeded && self.nip65_discovery.author_roots.contains_key(author) {
self.nip65_discovery.fallback_authors.insert(*author);
}
}
let mut changed = false;
let mut changed = previous_fallback_authors != self.nip65_discovery.fallback_authors;
for (author, candidate) in latest {
if !self.nip65_discovery.eligible_authors.contains(&author) {
continue;
@@ -5152,12 +5177,15 @@ impl SyncManager {
let overlay = discovery::build_inbox_root_overlay(
&self.nip65_discovery.author_roots,
&self.nip65_discovery.author_inboxes,
&self.nip65_discovery.fallback_authors,
&self.configured_nip65_fallback_relays(),
);
self.install_nip65_overlay(overlay).await;
tracing::info!(
source = %result.source_relay,
authors = result.authors.len(),
inbox_relays = self.nip65_discovery.inbox_roots.len(),
fallback_authors = self.nip65_discovery.fallback_authors.len(),
"Updated proactive inbox coverage from NIP-65"
);
+27
View File
@@ -73,6 +73,7 @@ pub struct TestRelay {
#[derive(Default, Clone)]
struct RelayOptions {
bootstrap_relay_url: Option<String>,
sync_plus_fallback_relays: Option<String>,
disable_negentropy: bool,
archive_all: bool,
archive_read_only: bool,
@@ -426,6 +427,28 @@ impl TestRelay {
.await
}
/// Start a persistent syncing relay with an explicit Sync+ fallback set.
pub async fn start_on_reservation_persistent_sync_with_fallback(
reservation: PortReservation,
bootstrap_relay_url: String,
fallback_relay_url: String,
git_data_path: PathBuf,
relay_data_path: PathBuf,
) -> Self {
Self::start_internal(
reservation,
RelayOptions {
bootstrap_relay_url: Some(bootstrap_relay_url),
sync_plus_fallback_relays: Some(fallback_relay_url),
lmdb_backend: true,
git_data_path: Some(git_data_path),
relay_data_path: Some(relay_data_path),
..RelayOptions::default()
},
)
.await
}
/// Start a relay with every configurable option, on a pre-reserved port.
///
/// Prefer the narrower constructors above — this exists so the option
@@ -552,6 +575,7 @@ impl TestRelay {
// to the production user-index defaults. A configured loopback
// bootstrap below becomes the scenario's sole index source.
.env("NGIT_USER_INDEX_RELAYS", "")
.env("NGIT_SYNC_PLUS_FALLBACK_RELAYS", "")
.env("NGIT_SYNC_STARTUP_DELAY_SECS", "0") // No startup delay for faster tests
.env("NGIT_SYNC_STARTUP_JITTER_MS", "0") // No jitter for tests
.env("NGIT_SYNC_DISCONNECT_CHECK_INTERVAL_SECS", "1") // Fast reconnect attempts for tests
@@ -585,6 +609,9 @@ impl TestRelay {
cmd.env("NGIT_SYNC_BOOTSTRAP_RELAY_URL", bootstrap_url)
.env("NGIT_USER_INDEX_RELAYS", bootstrap_url);
}
if let Some(ref fallback_relays) = options.sync_plus_fallback_relays {
cmd.env("NGIT_SYNC_PLUS_FALLBACK_RELAYS", fallback_relays);
}
// The test infrastructure runs entirely on loopback, which the
// production outbound target policy rejects for event-directed sync.
+96
View File
@@ -206,3 +206,99 @@ async fn root_author_inbox_reuses_existing_root_sync_pipeline() {
outbox.stop().await;
inbox.stop().await;
}
#[tokio::test]
async fn missing_relay_list_uses_bounded_fallback_coverage() {
let index = MockRelay::start().await;
let fallback = MockRelay::start().await;
let discovered_inbox = MockRelay::start().await;
let owner = Keys::generate();
let root_author = Keys::generate();
let identifier = "missing-list-fallback";
let syncing_git_dir = tempfile::tempdir().expect("create persistent git directory");
let syncing_relay_dir = tempfile::tempdir().expect("create persistent relay directory");
let syncing = TestRelay::start_on_reservation_persistent_sync_with_fallback(
reserve_port(),
index.url().to_string(),
fallback.url().to_string(),
syncing_git_dir.path().to_path_buf(),
syncing_relay_dir.path().to_path_buf(),
)
.await;
let syncing_domain = syncing.domain();
let (_announcement, _git_dir) =
setup_announcement_on_relay(&syncing, &owner, &[&syncing_domain], identifier).await;
let issue = build_layer2_issue_event(
&root_author,
&repo_coord(&owner, identifier),
"root with no published relay list",
)
.expect("build accepted root");
let client = TestClient::new(syncing.url(), root_author.clone())
.await
.expect("connect to target");
client.send_event(&issue).await.expect("publish root");
let reply = EventBuilder::new(Kind::TextNote, "fallback-only reply")
.tag(Tag::custom("e", vec![issue.id.to_hex()]))
.finalize(&root_author)
.expect("build reply");
send_to_relay_url(fallback.url(), &reply)
.await
.expect("seed fallback relay");
assert!(
wait_for_event_on_relay(
syncing.url(),
Filter::new().id(reply.id),
Duration::from_secs(20),
)
.await,
"a successful empty index lookup should activate bounded fallback coverage"
);
let sync_log = std::fs::read_to_string(syncing.log_path()).expect("read syncing relay log");
assert!(
sync_log.lines().any(|line| {
line.contains("Updated proactive inbox coverage from NIP-65")
&& line.contains("fallback_authors=1")
}),
"fallback activation should remain observable"
);
let relay_list = EventBuilder::new(Kind::RelayList, "")
.tag(Tag::custom("r", vec![discovered_inbox.url(), "read"]))
.finalize(&root_author)
.expect("build later relay list");
send_to_relay_url(index.url(), &relay_list)
.await
.expect("publish later relay list to index");
let declared_reply = EventBuilder::new(Kind::TextNote, "declared-inbox reply")
.tag(Tag::custom("e", vec![issue.id.to_hex()]))
.finalize(&root_author)
.expect("build declared-inbox reply");
send_to_relay_url(discovered_inbox.url(), &declared_reply)
.await
.expect("seed declared inbox");
assert!(
wait_for_event_on_relay(
syncing.url(),
Filter::new().id(declared_reply.id),
Duration::from_secs(20),
)
.await,
"a later accepted relay list should replace desired fallback coverage"
);
let sync_log = std::fs::read_to_string(syncing.log_path()).expect("read updated relay log");
assert!(
sync_log.lines().any(|line| {
line.contains("Updated proactive inbox coverage from NIP-65")
&& line.contains("fallback_authors=0")
}),
"accepted NIP-65 ownership should retire the author from desired fallback coverage"
);
syncing.stop().await;
discovered_inbox.stop().await;
fallback.stop().await;
index.stop().await;
}