From 299ad4fde96d231ce5b8273e41e0f7c7aca1d0c4 Mon Sep 17 00:00:00 2001 From: DanConwayDev Date: Thu, 18 Jun 2026 16:33:24 +0000 Subject: [PATCH] feat(blacklist): add startup auto-restore from holding --- .env.example | 7 + docs/explanation/deletion-requests.md | 38 ++-- docs/reference/configuration.md | 34 ++++ nix/module.nix | 12 ++ src/config.rs | 6 + src/main.rs | 23 +++ src/metrics/mod.rs | 38 ++++ src/nostr/builder.rs | 123 ++++++++++++- src/nostr/holding.rs | 110 +++++++++++- tests/nip09_blacklist_ops.rs | 242 +++++++++++++++++++++++++- 10 files changed, 612 insertions(+), 21 deletions(-) diff --git a/.env.example b/.env.example index 5692263..b3ee8bb 100644 --- a/.env.example +++ b/.env.example @@ -331,6 +331,13 @@ # NGIT_REPOSITORY_BLACKLIST=malware-repo,spam-repo # NGIT_REPOSITORY_BLACKLIST= +# Automatically restore repositories that were previously deleted by blacklist +# parity when they are no longer blacklisted and still within holding retention. +# Runs once on startup as part of startup parity orchestration. +# CLI: --blacklist-auto-restore +# Default: false +# NGIT_BLACKLIST_AUTO_RESTORE=false + # ============================================================================ # EVENT BLACKLIST # ============================================================================ diff --git a/docs/explanation/deletion-requests.md b/docs/explanation/deletion-requests.md index cf33551..8e3c268 100644 --- a/docs/explanation/deletion-requests.md +++ b/docs/explanation/deletion-requests.md @@ -114,8 +114,11 @@ data during the retention window. - Use case: Confirmed malware requiring immediate permanent deletion - Mechanism: `ngit-grasp holding-eject --owner --identifier ` - Logged for audit trail -- **Manual restoration:** Operator restores blacklisted repo after removal from blacklist - - Currently no dedicated restore CLI (restore happens through normal recovery paths) +- **Blacklist restoration policy:** optional startup auto-restore + - Controlled by `NGIT_BLACKLIST_AUTO_RESTORE` (default `false`) + - Restores only `holding-source=blacklist` scopes that are now unblacklisted + and still within holding retention + - Still-blacklisted scopes are skipped ## Deletion Flow @@ -270,19 +273,21 @@ deletions (announcement + state). ### Blacklist Recovery -When a repository is removed from the blacklist: +When a repository is removed from the blacklist, startup behavior depends on +`NGIT_BLACKLIST_AUTO_RESTORE`: -**Option 1: Manual Restoration (Current Behavior)** -- Operator removes from blacklist config -- Operator manually restores from holding DB if desired -- Provides explicit control over recovery decisions +**Disabled (default):** +- No startup restore sweep +- Repository remains in holding/archive unless recovered by another trigger -**Option 2: Automatic Restoration (Future Enhancement)** -- On startup, detect repos in holding area no longer blacklisted -- Automatically restore to main DB if within retention period -- Requires careful design to prevent unwanted restorations +**Enabled:** +- Startup scans holding metadata with `holding-source=blacklist` +- Deduplicates owner+identifier scopes within retention +- Restores scopes that no longer match current blacklist +- Skips scopes still blacklisted -**Current behavior:** manual restoration workflows only. +This keeps recovery opt-in and operator-controlled while removing manual restore +steps for common unblacklist workflows. ### Manual Ejection from Holding Area @@ -323,7 +328,7 @@ Blacklist-triggered deletions use the **same infrastructure** as NIP-09 deletion | **Author validation** | Required (pubkey match) | Not applicable (operator decision) | | **Disrespector mode** | Prevents deletion | Does NOT prevent deletion | | **Purpose** | User agency | Moderation/safety | -| **Recovery** | Automatic (re-publish) | Manual (operator decision) | +| **Recovery** | Automatic (re-publish) | Startup auto-restore (optional) | | **Metadata** | Links to Kind 5 event | Marks "blacklist-triggered" | ### Why Disrespector Doesn't Prevent Blacklist Deletion @@ -359,9 +364,12 @@ In the same startup parity pass, repository whitelist mismatches are also reconciled (for announcements that list this relay service but no longer match `repository_whitelist`). +When `NGIT_BLACKLIST_AUTO_RESTORE=true`, startup also runs a blacklist restore +pass between blacklist parity delete and whitelist parity delete. + **Future Enhancement (Dynamic Updates):** - Watch for configuration file changes -- Trigger deletion immediately on blacklist addition +- Trigger deletion/restore immediately on blacklist addition/removal - Requires careful design to avoid race conditions ## Cascade Deletion Strategy @@ -550,6 +558,7 @@ This allows clients to discover whether a relay respects deletion requests. **Prometheus Metrics (Currently Exposed):** - `ngit_blacklist_deletions_total{phase,result}` +- `ngit_blacklist_startup_restore_total{result,reason}` - `ngit_holding_cleanup_runs_total` - `ngit_holding_cleanup_deleted_total{type}` - `ngit_holding_cleanup_last_run_deleted{type}` @@ -602,7 +611,6 @@ work): ### Blacklist lifecycle improvements - **Dynamic blacklist updates:** apply blacklist additions/removals without restart. -- **Automatic blacklist recovery (optional policy):** configurable auto-restore for unblacklisted repos within retention. - **Dedicated restore CLI:** explicit operator restore commands with clear scope selection and outcomes. ### Operator controls and UX diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 45bdc38..bbb445a 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -996,6 +996,40 @@ Blacklist does **not** affect NIP-11 metadata: - Blacklist is transparent to clients (rejected with specific reason) - Operators can use blacklist without advertising curation +#### `NGIT_BLACKLIST_AUTO_RESTORE` + +**Description:** On startup, restore repositories previously deleted by blacklist parity when they are no longer blacklisted and still within holding retention +**Type:** Boolean +**Default:** `false` +**Required:** No +**CLI:** `--blacklist-auto-restore` + +**Behavior:** + +- When `false` (default): startup runs delete-only parity reconciliation for blacklist/whitelist; unblacklisted repositories in holding are restored only by existing recovery triggers (for example re-announcement). +- When `true`: startup also scans holding metadata with `holding-source=blacklist`, deduplicates owner+identifier scopes, and attempts restore for scopes that: + - are still within `NGIT_HOLDING_RETENTION_SECS`, and + - no longer match `NGIT_REPOSITORY_BLACKLIST`. +- Scopes that are still blacklisted are skipped (not restored). + +**Startup ordering:** + +1. Blacklist parity delete +2. Blacklist auto-restore (if enabled) +3. Whitelist parity delete + +This ordering keeps startup reconciliation deterministic when multiple policies interact. + +**Examples:** + +```bash +# Enable startup auto-restore for unblacklisted repos +NGIT_BLACKLIST_AUTO_RESTORE=true + +# Default behavior +NGIT_BLACKLIST_AUTO_RESTORE=false +``` + --- ### Event Blacklist diff --git a/nix/module.nix b/nix/module.nix index ad9a0b7..4b2d036 100644 --- a/nix/module.nix +++ b/nix/module.nix @@ -306,6 +306,16 @@ let ''; }; + blacklistAutoRestore = mkOption { + type = types.bool; + default = false; + description = '' + Automatically restore repositories deleted by blacklist parity on + startup when they are no longer blacklisted and still within the + holding retention window. + ''; + }; + eventBlacklist = mkOption { type = types.listOf types.str; default = [ ]; @@ -392,6 +402,8 @@ let concatStringsSep "," cfg.archiveGraspServices; NGIT_REPOSITORY_WHITELIST = concatStringsSep "," cfg.repositoryWhitelist; NGIT_REPOSITORY_BLACKLIST = concatStringsSep "," cfg.repositoryBlacklist; + NGIT_BLACKLIST_AUTO_RESTORE = + if cfg.blacklistAutoRestore then "true" else "false"; NGIT_EVENT_BLACKLIST = concatStringsSep "," cfg.eventBlacklist; NGIT_LOG_LEVEL = cfg.logLevel; NGIT_GRASP06_ENABLE = if cfg.grasp06Enable then "true" else "false"; diff --git a/src/config.rs b/src/config.rs index d7d10b4..9da32a1 100644 --- a/src/config.rs +++ b/src/config.rs @@ -492,6 +492,11 @@ pub struct Config { #[arg(long, env = "NGIT_REPOSITORY_BLACKLIST", default_value = "")] pub repository_blacklist: String, + /// Automatically restore blacklist-deleted repositories on startup when + /// they are no longer blacklisted and still within holding retention. + #[arg(long, env = "NGIT_BLACKLIST_AUTO_RESTORE", default_value_t = false)] + pub blacklist_auto_restore: bool, + /// Event blacklist: comma-separated list of npubs whose events are rejected /// All events from these authors are blocked from both relay storage and purgatory #[arg(long, env = "NGIT_EVENT_BLACKLIST", default_value = "")] @@ -830,6 +835,7 @@ impl Config { grasp06_enable: false, repository_whitelist: String::new(), repository_blacklist: String::new(), + blacklist_auto_restore: false, event_blacklist: String::new(), deletion_request_disrespector: false, max_connections: None, diff --git a/src/main.rs b/src/main.rs index 0e2e915..7ff3d52 100644 --- a/src/main.rs +++ b/src/main.rs @@ -230,6 +230,12 @@ async fn run_relay(config: Config) -> Result<()> { .write_policy .set_local_relay(relay_with_db.relay.clone()); + // Startup policy ordering (explicit): + // 1) blacklist parity delete (remove currently-blacklisted repos) + // 2) blacklist auto-restore (optional, restore previously blacklisted + // repos now unblacklisted and still within holding retention) + // 3) whitelist parity delete (final acceptance reconciliation for + // service-listed repos, preventing restore/re-delete churn on later starts) let blacklist_stats = relay_with_db .write_policy .run_startup_blacklist_parity_pass() @@ -245,6 +251,23 @@ async fn run_relay(config: Config) -> Result<()> { ); } + let blacklist_restore_stats = relay_with_db + .write_policy + .run_startup_blacklist_restore_pass() + .await; + if blacklist_restore_stats.scanned_scopes > 0 + || blacklist_restore_stats.attempted_restores > 0 + { + info!( + scanned_scopes = blacklist_restore_stats.scanned_scopes, + attempted = blacklist_restore_stats.attempted_restores, + succeeded = blacklist_restore_stats.successful_restores, + failed = blacklist_restore_stats.failed_restores, + skipped = blacklist_restore_stats.skipped_scopes, + "Startup blacklist restore pass completed" + ); + } + let whitelist_stats = relay_with_db .write_policy .run_startup_whitelist_parity_pass() diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index 046094d..7c13ded 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -46,6 +46,20 @@ lazy_static! { .expect("register blacklist deletion metric"); metric }; + static ref BLACKLIST_STARTUP_RESTORE_TOTAL: CounterVec = { + let metric = CounterVec::new( + Opts::new( + "ngit_blacklist_startup_restore_total", + "Blacklist startup restore operations by result and reason", + ), + &["result", "reason"], + ) + .expect("build blacklist startup restore metric"); + REGISTRY + .register(Box::new(metric.clone())) + .expect("register blacklist startup restore metric"); + metric + }; static ref HOLDING_CLEANUP_RUNS_TOTAL: Counter = { let metric = Counter::with_opts(Opts::new( "ngit_holding_cleanup_runs_total", @@ -144,6 +158,30 @@ pub fn record_blacklist_deletion_failure(phase: &str) { .inc(); } +pub fn record_blacklist_startup_restore_attempt() { + BLACKLIST_STARTUP_RESTORE_TOTAL + .with_label_values(&["attempted", "none"]) + .inc(); +} + +pub fn record_blacklist_startup_restore_success() { + BLACKLIST_STARTUP_RESTORE_TOTAL + .with_label_values(&["succeeded", "none"]) + .inc(); +} + +pub fn record_blacklist_startup_restore_failure() { + BLACKLIST_STARTUP_RESTORE_TOTAL + .with_label_values(&["failed", "none"]) + .inc(); +} + +pub fn record_blacklist_startup_restore_skipped(reason: &str) { + BLACKLIST_STARTUP_RESTORE_TOTAL + .with_label_values(&["skipped", reason]) + .inc(); +} + pub fn record_holding_cleanup_run( metadata_deleted: usize, archived_events_deleted: usize, diff --git a/src/nostr/builder.rs b/src/nostr/builder.rs index 58d528b..e115815 100644 --- a/src/nostr/builder.rs +++ b/src/nostr/builder.rs @@ -19,7 +19,7 @@ use tar::Archive as TarArchive; use crate::config::{Config, DatabaseBackend}; use crate::nostr::events::RepositoryAnnouncement; use crate::nostr::history::ReplaceableHistoryStore; -use crate::nostr::holding::RecoveryMetadataRecord; +use crate::nostr::holding::{DeletionSource, RecoveryMetadataRecord}; use crate::nostr::policy::{ accepted_purgatory, duplicate, reject_error, reject_invalid, reject_restricted, AnnouncementPolicy, AnnouncementResult, DeletionPolicy, PolicyContext, PrEventPolicy, @@ -67,6 +67,15 @@ pub struct WhitelistParityStats { pub failed_deletions: usize, } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct BlacklistRestoreStats { + pub scanned_scopes: usize, + pub attempted_restores: usize, + pub successful_restores: usize, + pub failed_restores: usize, + pub skipped_scopes: usize, +} + impl std::fmt::Debug for Nip34WritePolicy { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Nip34WritePolicy") @@ -326,6 +335,105 @@ impl Nip34WritePolicy { stats } + /// Startup-only blacklist restore pass. + /// + /// Scans holding metadata for scopes deleted by blacklist parity and restores + /// owner+identifier scopes that are no longer blacklisted and still inside + /// holding retention. + pub async fn run_startup_blacklist_restore_pass(&self) -> BlacklistRestoreStats { + if !self.ctx.config.blacklist_auto_restore { + return BlacklistRestoreStats::default(); + } + + let now = Timestamp::now(); + let retention = self.ctx.config.holding_retention(); + let scopes = self + .ctx + .holding + .eligible_recovery_scopes_by_source(DeletionSource::Blacklist, now, retention) + .await; + + let blacklist = self.ctx.config.blacklist_config(); + let mut stats = BlacklistRestoreStats { + scanned_scopes: scopes.len(), + ..BlacklistRestoreStats::default() + }; + + for scope in scopes { + let owner_pubkey = match PublicKey::from_hex(&scope.owner_pubkey_hex) { + Ok(pubkey) => pubkey, + Err(e) => { + stats.skipped_scopes += 1; + crate::metrics::record_blacklist_startup_restore_skipped("invalid_owner"); + tracing::warn!( + owner = %scope.owner_pubkey_hex, + identifier = %scope.identifier, + reason = "invalid_owner", + error = %e, + "Blacklist startup restore: skipping scope" + ); + continue; + } + }; + + let owner_npub = match owner_pubkey.to_bech32() { + Ok(npub) => npub, + Err(e) => { + stats.skipped_scopes += 1; + crate::metrics::record_blacklist_startup_restore_skipped("owner_bech32_failed"); + tracing::warn!( + owner = %scope.owner_pubkey_hex, + identifier = %scope.identifier, + reason = "owner_bech32_failed", + error = %e, + "Blacklist startup restore: skipping scope" + ); + continue; + } + }; + + if let Some(reason) = blacklist.check(&owner_npub, &scope.identifier) { + stats.skipped_scopes += 1; + crate::metrics::record_blacklist_startup_restore_skipped("still_blacklisted"); + tracing::info!( + owner = %scope.owner_pubkey_hex, + identifier = %scope.identifier, + reason = %reason, + "Blacklist startup restore: skipping scope still blacklisted" + ); + continue; + } + + stats.attempted_restores += 1; + crate::metrics::record_blacklist_startup_restore_attempt(); + + if self + .maybe_recover_deleted_repository_for_scope(&owner_pubkey, &scope.identifier) + .await + { + stats.successful_restores += 1; + crate::metrics::record_blacklist_startup_restore_success(); + tracing::info!( + owner = %scope.owner_pubkey_hex, + identifier = %scope.identifier, + reason = "unblacklisted_within_retention", + "Blacklist startup restore: restored repository scope" + ); + } else { + stats.failed_restores += 1; + crate::metrics::record_blacklist_startup_restore_failure(); + tracing::error!( + owner = %scope.owner_pubkey_hex, + identifier = %scope.identifier, + reason = "recovery_pipeline_failed", + "Blacklist startup restore: recovery pipeline failed" + ); + } + } + + stats + } + /// Set the local relay for purgatory notifications. /// /// This must be called after the relay is created since the relay depends @@ -1645,7 +1753,16 @@ impl Nip34WritePolicy { event: &Event, identifier: &str, ) -> bool { - let owner_hex = event.pubkey.to_hex(); + self.maybe_recover_deleted_repository_for_scope(&event.pubkey, identifier) + .await + } + + async fn maybe_recover_deleted_repository_for_scope( + &self, + owner_pubkey: &PublicKey, + identifier: &str, + ) -> bool { + let owner_hex = owner_pubkey.to_hex(); let records = self .ctx .holding @@ -1681,7 +1798,7 @@ impl Nip34WritePolicy { ); } - let owner_component = Self::owner_directory_component(&event.pubkey); + let owner_component = Self::owner_directory_component(owner_pubkey); let mut git_ready = true; let mut archive_restored = false; let mut selected_archive_abs: Option = None; diff --git a/src/nostr/holding.rs b/src/nostr/holding.rs index d7f7a3b..9f0dd8e 100644 --- a/src/nostr/holding.rs +++ b/src/nostr/holding.rs @@ -21,6 +21,7 @@ pub const HOLDING_DIR: &str = "holding"; /// Internal metadata event kind stored alongside archived events. pub const HOLDING_METADATA_KIND: u16 = 9905; +pub const HOLDING_SOURCE_TAG: &str = "holding-source"; pub const HOLDING_ARCHIVE_PATH_TAG: &str = "holding-archive-path"; pub const HOLDING_ARCHIVE_CREATED_AT_TAG: &str = "holding-archive-created-at"; pub const HOLDING_OWNER_PUBKEY_TAG: &str = "holding-owner-pubkey"; @@ -85,6 +86,12 @@ pub struct RecoveryMetadataRecord { pub archive_relative_path: Option, } +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct RecoveryScope { + pub owner_pubkey_hex: String, + pub identifier: String, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub struct CleanupStats { pub metadata_examined: usize, @@ -155,7 +162,10 @@ impl HoldingStore { let mut tags = vec![ Tag::event(event.id), - Tag::custom("holding-source", vec![metadata.source.as_str().to_string()]), + Tag::custom( + HOLDING_SOURCE_TAG, + vec![metadata.source.as_str().to_string()], + ), Tag::custom( "holding-deleted-at", vec![metadata.deleted_at.as_secs().to_string()], @@ -233,6 +243,22 @@ impl HoldingStore { }) } + fn parse_holding_source(metadata_event: &Event) -> Option { + metadata_event.tags.iter().find_map(|tag| { + let v = tag.as_slice(); + if v.len() < 2 || v[0] != HOLDING_SOURCE_TAG { + return None; + } + + match v[1].as_str() { + "nip09" => Some(DeletionSource::Nip09), + "blacklist" => Some(DeletionSource::Blacklist), + "whitelist" => Some(DeletionSource::Whitelist), + _ => None, + } + }) + } + fn parse_archived_event_id(metadata_event: &Event) -> Option { metadata_event.tags.iter().find_map(|tag| { let v = tag.as_slice(); @@ -288,6 +314,40 @@ impl HoldingStore { }) } + fn parse_scope_from_coordinate(metadata_event: &Event) -> Option { + let coordinate = Self::parse_coordinate(metadata_event)?; + let mut parts = coordinate.splitn(3, ':'); + let kind = parts.next()?; + let owner = parts.next()?; + let identifier = parts.next()?; + + if kind != "30617" || owner.is_empty() || identifier.is_empty() { + return None; + } + + Some(RecoveryScope { + owner_pubkey_hex: owner.to_string(), + identifier: identifier.to_string(), + }) + } + + fn parse_recovery_scope(metadata_event: &Event) -> Option { + if let Some(scope) = Self::parse_scope_from_coordinate(metadata_event) { + return Some(scope); + } + + let owner_pubkey_hex = Self::parse_owner_pubkey(metadata_event)?; + let identifier = Self::parse_identifier(metadata_event)?; + if owner_pubkey_hex.is_empty() || identifier.is_empty() { + return None; + } + + Some(RecoveryScope { + owner_pubkey_hex, + identifier, + }) + } + fn metadata_matches_recovery_scope( metadata_event: &Event, owner_pubkey_hex: &str, @@ -358,6 +418,54 @@ impl HoldingStore { .collect() } + /// Find distinct owner+identifier scopes eligible for recovery from a + /// specific deletion source. + /// + /// Eligibility requires all of: + /// - metadata has the requested `holding-source`, + /// - metadata has a parseable `holding-deleted-at`, + /// - metadata is within `retention` from `now`, + /// - metadata yields a parseable owner+identifier recovery scope. + pub async fn eligible_recovery_scopes_by_source( + &self, + source: DeletionSource, + now: Timestamp, + retention: Duration, + ) -> Vec { + let cutoff = now.as_secs().saturating_sub(retention.as_secs()); + let filter = Filter::new().kind(Kind::from(HOLDING_METADATA_KIND)); + + let metadata_events = match self.db.query(filter).await { + Ok(events) => events, + Err(e) => { + tracing::error!(error = %e, "Holding recovery failed to query metadata scopes"); + return Vec::new(); + } + }; + + let mut scopes = BTreeSet::new(); + for metadata_event in metadata_events { + if Self::parse_holding_source(&metadata_event) != Some(source) { + continue; + } + + let Some(deleted_at) = Self::parse_holding_deleted_at(&metadata_event) else { + continue; + }; + if deleted_at.as_secs() <= cutoff { + continue; + } + + let Some(scope) = Self::parse_recovery_scope(&metadata_event) else { + continue; + }; + + scopes.insert(scope); + } + + scopes.into_iter().collect() + } + /// Resolve an archive-relative path to an absolute path under `.archive`. pub fn archive_absolute_path(&self, relative_path: &str) -> Option { self.resolve_archive_path(relative_path) diff --git a/tests/nip09_blacklist_ops.rs b/tests/nip09_blacklist_ops.rs index fd95609..0024320 100644 --- a/tests/nip09_blacklist_ops.rs +++ b/tests/nip09_blacklist_ops.rs @@ -7,7 +7,7 @@ use std::time::Duration; use clap::Parser; use ngit_grasp::config::Config; use ngit_grasp::grasp06::receive::new_repo_init_locks; -use ngit_grasp::metrics::{self, Metrics}; +use ngit_grasp::metrics::{self, REGISTRY}; use ngit_grasp::nostr::builder::{Nip34WritePolicy, SharedDatabase}; use ngit_grasp::nostr::history::ReplaceableHistoryStore; use ngit_grasp::nostr::holding::{ @@ -16,6 +16,7 @@ use ngit_grasp::nostr::holding::{ use ngit_grasp::nostr::tombstones::Tombstones; use ngit_grasp::purgatory::Purgatory; use nostr_sdk::prelude::*; +use prometheus::{Encoder, TextEncoder}; fn metadata_has_tag(event: &Event, key: &str, value: Option<&str>) -> bool { event.tags.iter().any(|tag| { @@ -35,6 +36,35 @@ fn repo_identifier(event: &Event) -> String { .expect("announcement identifier tag") } +fn metric_value(rendered: &str, metric_name: &str, labels: &[(&str, &str)]) -> f64 { + rendered + .lines() + .find_map(|line| { + if !line.starts_with(metric_name) { + return None; + } + + let labels_match = labels + .iter() + .all(|(k, v)| line.contains(&format!("{k}=\"{v}\""))); + if !labels_match { + return None; + } + + line.split_whitespace().last()?.parse::().ok() + }) + .unwrap_or(0.0) +} + +fn render_metrics_snapshot() -> String { + let families = REGISTRY.gather(); + let mut bytes = Vec::new(); + TextEncoder::new() + .encode(&families, &mut bytes) + .expect("encode prometheus metrics"); + String::from_utf8(bytes).expect("utf8 metrics") +} + fn make_announcement_with_domain(keys: &Keys, identifier: &str, domain: &str) -> Event { EventBuilder::new(Kind::GitRepoAnnouncement, "") .tags(vec![ @@ -336,6 +366,213 @@ async fn startup_whitelist_scan_leaves_matching_repositories_untouched() { .is_empty()); } +#[tokio::test] +async fn startup_blacklist_restore_recovers_unblacklisted_scope_and_is_idempotent() { + let relay_dir = tempfile::tempdir().expect("relay tempdir"); + let git_dir = tempfile::tempdir().expect("git tempdir"); + let db: SharedDatabase = Arc::new(nostr_memory::MemoryDatabase::unbounded()); + let holding = HoldingStore::open_lmdb(relay_dir.path(), git_dir.path()) + .await + .expect("open holding lmdb"); + + let owner = Keys::generate(); + let owner_npub = owner.public_key().to_bech32().expect("owner npub"); + let announcement = make_announcement(&owner, "blacklist-restore-repo"); + let issue = make_issue(&owner, &announcement); + + db.save_event(&announcement) + .await + .expect("save announcement"); + db.save_event(&issue).await.expect("save issue"); + + let repo_path = git_dir + .path() + .join(owner_npub.clone()) + .join("blacklist-restore-repo.git"); + std::fs::create_dir_all(repo_path.join("refs")).expect("create bare repo dir"); + + let deletion_policy = make_policy( + Config { + repository_blacklist: owner_npub.clone(), + ..base_config() + }, + db.clone(), + holding.clone(), + git_dir.path(), + ); + let deletion_stats = deletion_policy.run_startup_blacklist_parity_pass().await; + assert_eq!(deletion_stats.successful_deletions, 1); + + if repo_path.exists() { + std::fs::remove_dir_all(&repo_path).expect("remove stale repo directory"); + } + + let announcement_meta = holding.metadata_for_event(&announcement.id).await; + let archive_rel = announcement_meta + .iter() + .flat_map(|m| m.tags.iter()) + .find_map(|tag| { + let v = tag.as_slice(); + (v.len() >= 2 && v[0] == HOLDING_ARCHIVE_PATH_TAG).then(|| v[1].clone()) + }) + .expect("blacklist deletion should record archive path"); + let archive_abs = holding + .archive_absolute_path(&archive_rel) + .expect("resolve archive path"); + assert!(archive_abs.exists(), "archive must exist before restore"); + + let restore_policy = make_policy( + Config { + blacklist_auto_restore: true, + ..base_config() + }, + db.clone(), + holding.clone(), + git_dir.path(), + ); + + let before = render_metrics_snapshot(); + let restore_stats = restore_policy.run_startup_blacklist_restore_pass().await; + assert_eq!(restore_stats.scanned_scopes, 1); + assert_eq!(restore_stats.attempted_restores, 1); + assert_eq!(restore_stats.successful_restores, 1); + assert_eq!(restore_stats.failed_restores, 0); + assert_eq!(restore_stats.skipped_scopes, 0); + + assert!( + db.event_by_id(&announcement.id) + .await + .expect("query announcement") + .is_some(), + "announcement should be restored from holding" + ); + assert!( + db.event_by_id(&issue.id) + .await + .expect("query issue") + .is_some(), + "dependent events should be restored from holding" + ); + assert!( + holding + .metadata_for_event(&announcement.id) + .await + .is_empty(), + "holding metadata should be cleaned after restore" + ); + assert!(!archive_abs.exists(), "consumed archive should be deleted"); + + let after = render_metrics_snapshot(); + let attempted_before = metric_value( + &before, + "ngit_blacklist_startup_restore_total", + &[("result", "attempted"), ("reason", "none")], + ); + let attempted_after = metric_value( + &after, + "ngit_blacklist_startup_restore_total", + &[("result", "attempted"), ("reason", "none")], + ); + assert_eq!(attempted_after, attempted_before + 1.0); + + let success_before = metric_value( + &before, + "ngit_blacklist_startup_restore_total", + &[("result", "succeeded"), ("reason", "none")], + ); + let success_after = metric_value( + &after, + "ngit_blacklist_startup_restore_total", + &[("result", "succeeded"), ("reason", "none")], + ); + assert_eq!(success_after, success_before + 1.0); + + let second_stats = restore_policy.run_startup_blacklist_restore_pass().await; + assert_eq!(second_stats.scanned_scopes, 0); + assert_eq!(second_stats.attempted_restores, 0); + assert_eq!(second_stats.successful_restores, 0); + assert_eq!(second_stats.failed_restores, 0); + assert_eq!(second_stats.skipped_scopes, 0); +} + +#[tokio::test] +async fn startup_blacklist_restore_skips_scopes_still_blacklisted() { + let relay_dir = tempfile::tempdir().expect("relay tempdir"); + let git_dir = tempfile::tempdir().expect("git tempdir"); + let db: SharedDatabase = Arc::new(nostr_memory::MemoryDatabase::unbounded()); + let holding = HoldingStore::open_lmdb(relay_dir.path(), git_dir.path()) + .await + .expect("open holding lmdb"); + + let owner = Keys::generate(); + let owner_npub = owner.public_key().to_bech32().expect("owner npub"); + let announcement = make_announcement(&owner, "blacklist-still-blocked-repo"); + db.save_event(&announcement) + .await + .expect("save announcement"); + + std::fs::create_dir_all( + git_dir + .path() + .join(owner_npub.clone()) + .join("blacklist-still-blocked-repo.git") + .join("refs"), + ) + .expect("create bare repo dir"); + + let deletion_policy = make_policy( + Config { + repository_blacklist: owner_npub.clone(), + ..base_config() + }, + db.clone(), + holding.clone(), + git_dir.path(), + ); + let deletion_stats = deletion_policy.run_startup_blacklist_parity_pass().await; + assert_eq!(deletion_stats.successful_deletions, 1); + + let restore_policy = make_policy( + Config { + repository_blacklist: owner_npub, + blacklist_auto_restore: true, + ..base_config() + }, + db.clone(), + holding, + git_dir.path(), + ); + + let before = render_metrics_snapshot(); + let restore_stats = restore_policy.run_startup_blacklist_restore_pass().await; + assert_eq!(restore_stats.scanned_scopes, 1); + assert_eq!(restore_stats.attempted_restores, 0); + assert_eq!(restore_stats.successful_restores, 0); + assert_eq!(restore_stats.failed_restores, 0); + assert_eq!(restore_stats.skipped_scopes, 1); + + assert!( + db.event_by_id(&announcement.id) + .await + .expect("query announcement") + .is_none(), + "still-blacklisted scope must not be restored" + ); + + let after = render_metrics_snapshot(); + let skipped_before = metric_value( + &before, + "ngit_blacklist_startup_restore_total", + &[("result", "skipped"), ("reason", "still_blacklisted")], + ); + let skipped_after = metric_value( + &after, + "ngit_blacklist_startup_restore_total", + &[("result", "skipped"), ("reason", "still_blacklisted")], + ); + assert_eq!(skipped_after, skipped_before + 1.0); +} + #[tokio::test] async fn manual_ejection_removes_holding_and_archive_idempotently() { let relay_dir = tempfile::tempdir().expect("relay tempdir"); @@ -459,8 +696,9 @@ async fn metrics_increment_on_blacklist_cleanup_recovery_and_manual_ejection_pat .await .expect("manual ejection"); - let rendered = Metrics::new(10, None).render(); + let rendered = render_metrics_snapshot(); assert!(rendered.contains("ngit_blacklist_deletions_total")); + assert!(rendered.contains("ngit_blacklist_startup_restore_total")); assert!(rendered.contains("ngit_holding_cleanup_runs_total")); assert!(rendered.contains("ngit_recovery_total")); assert!(rendered.contains("ngit_manual_ejections_total"));