From 0a0f7c3570027e8f2a5fa07ab4de0f84bdd1bcf4 Mon Sep 17 00:00:00 2001 From: DanConwayDev Date: Thu, 18 Jun 2026 11:28:47 +0000 Subject: [PATCH] feat(deletion): add startup whitelist parity cleanup --- docs/explanation/deletion-requests.md | 20 ++--- src/main.rs | 16 ++++ src/nostr/builder.rs | 105 +++++++++++++++++++++++++ src/nostr/holding.rs | 2 + src/nostr/policy/deletion.rs | 35 +++++++++ tests/nip09_blacklist_ops.rs | 109 +++++++++++++++++++++++++- 6 files changed, 275 insertions(+), 12 deletions(-) diff --git a/docs/explanation/deletion-requests.md b/docs/explanation/deletion-requests.md index 09e7c75..9df4e0f 100644 --- a/docs/explanation/deletion-requests.md +++ b/docs/explanation/deletion-requests.md @@ -128,17 +128,19 @@ NIP-11 currently omits 9 and 62 in this mode (`src/http/nip11.rs:115`). ### 3) Startup policy parity flow -On startup, ngit-grasp currently runs a **blacklist parity** pass: it scans -stored announcements against blacklist config and deletes matches through the -same cascade + holding + archive path, marking metadata source as `blacklist` -(`src/nostr/builder.rs:140`, `src/nostr/policy/deletion.rs:817`). +On startup, ngit-grasp runs parity cleanup for both policy lists: -This gives parity for already-stored repos that became blacklisted while the -relay was offline. +- **Blacklist parity:** scans stored announcements against blacklist config and + deletes matches through the same cascade + holding + archive path, marking + metadata source as `blacklist` (`src/nostr/builder.rs:140`, + `src/nostr/policy/deletion.rs:817`). +- **Whitelist parity:** scans stored announcements that list this relay’s + service domain and deletes entries that no longer match + `repository_whitelist`, marking metadata source as `whitelist` + (`src/nostr/builder.rs:232`, `src/nostr/policy/deletion.rs:850`). -Repository whitelist enforcement is currently admission-time (new/replacement -announcements) and does **not** yet run a startup cleanup sweep for previously -stored announcements that no longer match whitelist policy. +This gives startup-time parity for repositories that became out-of-policy while +the relay was offline. ### 4) NIP-62 vanish flow diff --git a/src/main.rs b/src/main.rs index 34984c3..9145818 100644 --- a/src/main.rs +++ b/src/main.rs @@ -245,6 +245,22 @@ async fn run_relay(config: Config) -> Result<()> { ); } + let whitelist_stats = relay_with_db + .write_policy + .run_startup_whitelist_parity_pass() + .await; + if whitelist_stats.scanned_announcements > 0 || whitelist_stats.mismatched_announcements > 0 + { + info!( + scanned = whitelist_stats.scanned_announcements, + mismatched = whitelist_stats.mismatched_announcements, + attempted = whitelist_stats.attempted_deletions, + succeeded = whitelist_stats.successful_deletions, + failed = whitelist_stats.failed_deletions, + "Startup whitelist parity pass completed" + ); + } + // Wire the GRASP-06 `/prs/` filesystem cleanup context into // purgatory so the standard expiry sweep can delete dangling // refs/nostr/ refs (and zero-ref bare repos) when a diff --git a/src/nostr/builder.rs b/src/nostr/builder.rs index 2ae9099..58d528b 100644 --- a/src/nostr/builder.rs +++ b/src/nostr/builder.rs @@ -58,6 +58,15 @@ pub struct BlacklistParityStats { pub failed_deletions: usize, } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct WhitelistParityStats { + pub scanned_announcements: usize, + pub mismatched_announcements: usize, + pub attempted_deletions: usize, + pub successful_deletions: usize, + pub failed_deletions: usize, +} + impl std::fmt::Debug for Nip34WritePolicy { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Nip34WritePolicy") @@ -221,6 +230,102 @@ impl Nip34WritePolicy { stats } + /// Startup-only whitelist parity pass. + /// + /// Scans already-stored kind-30617 announcements that list this relay's + /// service domain and removes repositories that no longer match + /// `repository_whitelist`, using the same cascade + holding/archive flow as + /// NIP-09 but with `DeletionSource::Whitelist`. + pub async fn run_startup_whitelist_parity_pass(&self) -> WhitelistParityStats { + let whitelist = self.ctx.config.repository_config(); + if !whitelist.enabled() { + return WhitelistParityStats::default(); + } + + let blacklist = self.ctx.config.blacklist_config(); + let announcements = match self + .ctx + .database + .query(Filter::new().kind(Kind::GitRepoAnnouncement)) + .await + { + Ok(events) => events, + Err(e) => { + tracing::error!(error = %e, "Whitelist startup parity scan failed to query announcements"); + return WhitelistParityStats::default(); + } + }; + + let mut stats = WhitelistParityStats { + scanned_announcements: announcements.len(), + ..WhitelistParityStats::default() + }; + + for announcement in announcements { + let parsed = match RepositoryAnnouncement::from_event(announcement.clone()) { + Ok(p) => p, + Err(e) => { + tracing::warn!( + event_id = %announcement.id.to_hex(), + error = %e, + "Whitelist startup parity: skipping unparseable announcement" + ); + continue; + } + }; + + if !parsed.lists_service(&self.ctx.domain) { + continue; + } + + let npub = announcement + .pubkey + .to_bech32() + .expect("public key to bech32 should be infallible"); + + // Blacklist precedence: if this repository is blacklisted, it belongs to + // the blacklist parity path and should be attributed there. + if blacklist.check(&npub, &parsed.identifier).is_some() { + continue; + } + + if whitelist.matches(&npub, &parsed.identifier) { + continue; + } + + stats.mismatched_announcements += 1; + stats.attempted_deletions += 1; + + match self + .deletion_policy + .apply_whitelist_deletion_for_announcement(&announcement) + .await + { + Ok(()) => { + stats.successful_deletions += 1; + tracing::info!( + event_id = %announcement.id.to_hex(), + owner = %announcement.pubkey.to_hex(), + identifier = %parsed.identifier, + "Whitelist startup parity: deleted stored repository" + ); + } + Err(e) => { + stats.failed_deletions += 1; + tracing::error!( + event_id = %announcement.id.to_hex(), + owner = %announcement.pubkey.to_hex(), + identifier = %parsed.identifier, + error = %e, + "Whitelist startup parity: deletion failed" + ); + } + } + } + + stats + } + /// Set the local relay for purgatory notifications. /// /// This must be called after the relay is created since the relay depends diff --git a/src/nostr/holding.rs b/src/nostr/holding.rs index dd5780e..d7f7a3b 100644 --- a/src/nostr/holding.rs +++ b/src/nostr/holding.rs @@ -35,6 +35,7 @@ pub const DEFAULT_CLEANUP_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60) pub enum DeletionSource { Nip09, Blacklist, + Whitelist, } impl DeletionSource { @@ -42,6 +43,7 @@ impl DeletionSource { match self { Self::Nip09 => "nip09", Self::Blacklist => "blacklist", + Self::Whitelist => "whitelist", } } } diff --git a/src/nostr/policy/deletion.rs b/src/nostr/policy/deletion.rs index b8bf54e..dc36c79 100644 --- a/src/nostr/policy/deletion.rs +++ b/src/nostr/policy/deletion.rs @@ -844,6 +844,41 @@ impl DeletionPolicy { Ok(()) } + /// Apply operator-driven whitelist deletion for a stored kind-30617 + /// announcement. + /// + /// Uses the same cascade + holding/archive flow as NIP-09 deletions, but + /// marks holding metadata with `DeletionSource::Whitelist`. + pub async fn apply_whitelist_deletion_for_announcement( + &self, + announcement: &Event, + ) -> anyhow::Result<()> { + if announcement.kind != Kind::GitRepoAnnouncement { + return Err(anyhow::anyhow!( + "whitelist deletion requires kind 30617 announcement, got {}", + announcement.kind.as_u16() + )); + } + + let Some(identifier) = identifier_from_event(announcement) else { + return Err(anyhow::anyhow!( + "announcement {} missing identifier tag", + announcement.id.to_hex() + )); + }; + + let coordinate = format!("30617:{}:{}", announcement.pubkey.to_hex(), identifier); + self.cascade_delete_announcement( + &announcement.pubkey, + &coordinate, + Timestamp::now(), + DeletionSource::Whitelist, + ) + .await; + + Ok(()) + } + /// Delete kind-30618 state events for `identifier` when the repository is /// now unanchored (no surviving kind-30617 announcement with that `d` tag). /// diff --git a/tests/nip09_blacklist_ops.rs b/tests/nip09_blacklist_ops.rs index 8a70130..fd95609 100644 --- a/tests/nip09_blacklist_ops.rs +++ b/tests/nip09_blacklist_ops.rs @@ -35,17 +35,21 @@ fn repo_identifier(event: &Event) -> String { .expect("announcement identifier tag") } -fn make_announcement(keys: &Keys, identifier: &str) -> Event { +fn make_announcement_with_domain(keys: &Keys, identifier: &str, domain: &str) -> Event { EventBuilder::new(Kind::GitRepoAnnouncement, "") .tags(vec![ Tag::identifier(identifier), - Tag::custom("clone", vec!["https://example.com/repo.git"]), - Tag::custom("relays", vec!["wss://example.com"]), + Tag::custom("clone", vec![format!("https://{domain}/repo.git")]), + Tag::custom("relays", vec![format!("wss://{domain}")]), ]) .finalize(keys) .expect("build announcement") } +fn make_announcement(keys: &Keys, identifier: &str) -> Event { + make_announcement_with_domain(keys, identifier, "example.com") +} + fn make_issue(keys: &Keys, announcement: &Event) -> Event { let coordinate = format!( "30617:{}:{}", @@ -233,6 +237,105 @@ async fn startup_blacklist_scan_leaves_non_matching_repositories_untouched() { .is_empty()); } +#[tokio::test] +async fn startup_whitelist_scan_deletes_non_matching_repositories_via_holding_archive_path() { + let relay_dir = tempfile::tempdir().expect("relay tempdir"); + let git_dir = tempfile::tempdir().expect("git tempdir"); + + let db: SharedDatabase = Arc::new(nostr_memory::MemoryDatabase::unbounded()); + let holding = HoldingStore::open_lmdb(relay_dir.path(), git_dir.path()) + .await + .expect("open holding lmdb"); + + let owner = Keys::generate(); + let announcement = make_announcement_with_domain(&owner, "not-whitelisted", "test.example.com"); + let issue = make_issue(&owner, &announcement); + + db.save_event(&announcement) + .await + .expect("save announcement"); + db.save_event(&issue).await.expect("save issue"); + + let owner_npub = owner.public_key().to_bech32().expect("npub"); + std::fs::create_dir_all( + git_dir + .path() + .join(owner_npub) + .join("not-whitelisted.git") + .join("refs"), + ) + .expect("create bare repo dir"); + + let mut config = base_config(); + config.repository_whitelist = "allowed-repo".to_string(); + + let policy = make_policy(config, db.clone(), holding.clone(), git_dir.path()); + let stats = policy.run_startup_whitelist_parity_pass().await; + + assert_eq!(stats.mismatched_announcements, 1); + assert_eq!(stats.successful_deletions, 1); + assert!( + db.event_by_id(&announcement.id) + .await + .expect("query announcement") + .is_none(), + "non-whitelisted announcement must be deleted from main DB" + ); + assert!( + db.event_by_id(&issue.id) + .await + .expect("query issue") + .is_none(), + "dependent events must be cascade-deleted from main DB" + ); + + assert!(holding.has_event(&announcement.id).await); + assert!(holding.has_event(&issue.id).await); + let announcement_meta = holding.metadata_for_event(&announcement.id).await; + assert!( + announcement_meta + .iter() + .any(|m| metadata_has_tag(m, "holding-source", Some("whitelist"))), + "holding metadata must mark whitelist deletion source" + ); +} + +#[tokio::test] +async fn startup_whitelist_scan_leaves_matching_repositories_untouched() { + let relay_dir = tempfile::tempdir().expect("relay tempdir"); + let git_dir = tempfile::tempdir().expect("git tempdir"); + + let db: SharedDatabase = Arc::new(nostr_memory::MemoryDatabase::unbounded()); + let holding = HoldingStore::open_lmdb(relay_dir.path(), git_dir.path()) + .await + .expect("open holding lmdb"); + + let owner = Keys::generate(); + let announcement = make_announcement_with_domain(&owner, "matching-repo", "test.example.com"); + db.save_event(&announcement) + .await + .expect("save announcement"); + + let mut config = base_config(); + config.repository_whitelist = "matching-repo".to_string(); + + let policy = make_policy(config, db.clone(), holding.clone(), git_dir.path()); + let stats = policy.run_startup_whitelist_parity_pass().await; + + assert_eq!(stats.mismatched_announcements, 0); + assert!( + db.event_by_id(&announcement.id) + .await + .expect("query announcement") + .is_some(), + "whitelisted repository must remain in main DB" + ); + assert!(holding + .metadata_for_event(&announcement.id) + .await + .is_empty()); +} + #[tokio::test] async fn manual_ejection_removes_holding_and_archive_idempotently() { let relay_dir = tempfile::tempdir().expect("relay tempdir");