mirror of
https://relay.ngit.dev/npub15qydau2hjma6ngxkl2cyar74wzyjshvl65za5k5rl69264ar2exs5cyejr/ngit-grasp.git
synced 2026-10-05 23:18:24 +00:00
feat(deletion): add git archive lifecycle for holding cleanup
This commit is contained in:
Generated
+32
@@ -591,6 +591,16 @@ dependencies = [
|
||||
"getrandom 0.3.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "filetime"
|
||||
version = "0.2.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.9"
|
||||
@@ -1403,6 +1413,7 @@ dependencies = [
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tar",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -2351,6 +2362,17 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tar"
|
||||
version = "0.4.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840"
|
||||
dependencies = [
|
||||
"filetime",
|
||||
"libc",
|
||||
"xattr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.27.0"
|
||||
@@ -3199,6 +3221,16 @@ version = "0.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
|
||||
|
||||
[[package]]
|
||||
name = "xattr"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rustix",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "yoke"
|
||||
version = "0.8.3"
|
||||
|
||||
@@ -29,6 +29,7 @@ nostr-memory = "0.45.0-alpha.1"
|
||||
futures-util = "0.3"
|
||||
base64 = "0.22"
|
||||
flate2 = "1.0"
|
||||
tar = "0.4"
|
||||
|
||||
# Metrics
|
||||
prometheus = "0.14"
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# Deletion Request Support (NIP-09)
|
||||
|
||||
**Status:** ✅ **PARTIALLY IMPLEMENTED (holding DB cleanup engine live)**
|
||||
**Status:** ✅ **PARTIALLY IMPLEMENTED (Phases 1–4 live; recovery deferred)**
|
||||
|
||||
This document now reflects both the implemented single-node deletion path
|
||||
(main DB + tombstones + holding DB cleanup engine) and the still-planned
|
||||
expiry/archive/recovery architecture.
|
||||
(main DB + tombstones + holding DB + git archive lifecycle) and the still-planned
|
||||
recovery architecture.
|
||||
|
||||
---
|
||||
|
||||
@@ -15,7 +15,7 @@ expiry/archive/recovery architecture.
|
||||
> What follows in this section is what is **actually built today** and is the
|
||||
> foundation that work builds on. The `deletion-request-disrespector` archival
|
||||
> mode and the holding-DB move semantics are now implemented (see below);
|
||||
> cleanup expiry is now implemented; git archival and recovery remain planned.
|
||||
> cleanup expiry and git archival lifecycle are now implemented; recovery remains planned.
|
||||
|
||||
Up to the rust-nostr 0.45 bump, ngit-grasp relied on the LMDB backend's
|
||||
*automatic* NIP-09 / NIP-62 processing (`NostrLmdb` defaults
|
||||
@@ -562,7 +562,7 @@ When implementation is complete, the following documentation will be updated:
|
||||
|
||||
### Completed in the current implementation
|
||||
|
||||
The following are implemented now (archive/recovery still pending):
|
||||
The following are implemented now (recovery still pending):
|
||||
|
||||
- ngit-grasp-owned NIP-09/NIP-62 handling (backend auto-processing disabled)
|
||||
- tombstone persistence and deletion re-submission gate
|
||||
@@ -573,6 +573,15 @@ The following are implemented now (archive/recovery still pending):
|
||||
- periodic expiry cleanup task,
|
||||
- graceful shutdown signal handling for the cleanup task,
|
||||
- idempotent handling of partial/missing holding data during cleanup
|
||||
- git archive lifecycle for repository-announcement deletions:
|
||||
- creates `.archive/<owner>/<identifier>-<timestamp>.tar.gz` snapshots,
|
||||
- stores archive linkage tags in holding metadata (`holding-archive-path`,
|
||||
archive creation timestamp, owner linkage),
|
||||
- integrates archive file deletion into holding-expiry cleanup,
|
||||
- tolerates missing/corrupt archive files during cleanup (log + continue)
|
||||
to avoid hard-failure loops,
|
||||
- applies **fail-safe preservation**: when archive creation fails while live
|
||||
git data exists, destructive announcement deletion is skipped
|
||||
- announcement cascade deletion with multi-maintainer retention semantics
|
||||
- extended dependent-kind coverage, including PR chain kinds
|
||||
(`1618`/`1619`/`1631`/`1632`)
|
||||
@@ -612,10 +621,12 @@ using served PR/PR-update fixtures (event + git-ref promotion).
|
||||
- Added focused cleanup tests in
|
||||
[`tests/nip09_holding_cleanup.rs`](../../tests/nip09_holding_cleanup.rs).
|
||||
|
||||
**Phase 4: Git archive + metadata lifecycle** 🔄
|
||||
- Add git archive creation (`.tar.gz`) on deletion and archive metadata linkage.
|
||||
- Integrate archive cleanup with holding-DB expiry pipeline.
|
||||
- Add failure-mode handling (archive create/extract errors, missing files).
|
||||
**Phase 4: Git archive + metadata lifecycle** ✅
|
||||
- Git archive creation (`.tar.gz`) on announcement deletion paths is implemented.
|
||||
- Archive metadata linkage is persisted in holding metadata events.
|
||||
- Archive cleanup is integrated with holding-DB expiry cleanup.
|
||||
- Failure-mode handling is implemented with fail-safe preservation on archive-create
|
||||
failure and best-effort cleanup on missing archive files.
|
||||
|
||||
**Phase 5: Recovery workflow** 🔄
|
||||
- Re-announcement detection and eligibility checks (within retention window,
|
||||
@@ -629,15 +640,13 @@ using served PR/PR-update fixtures (event + git-ref promotion).
|
||||
- Observability: metrics for holding DB size/count, cleanup, recoveries, ejections.
|
||||
- Concurrency/race analysis, max-depth/scale limits, lock strategy finalization.
|
||||
|
||||
### Deferred items after Phase 3
|
||||
### Deferred items after Phase 4
|
||||
|
||||
The next phases still intentionally deferred are:
|
||||
|
||||
1. **Git archive lifecycle (Phase 4):** create and clean up `.tar.gz` git
|
||||
archives linked to holding entries.
|
||||
2. **Recovery workflow (Phase 5):** restore events/git data from holding/archive
|
||||
1. **Recovery workflow (Phase 5):** restore events/git data from holding/archive
|
||||
on explicit/operator-approved flows.
|
||||
3. **Blacklist runtime deletion parity (Phase 6):** there is currently no active
|
||||
2. **Blacklist runtime deletion parity (Phase 6):** there is currently no active
|
||||
runtime blacklist-triggered deletion path in the codebase; when introduced,
|
||||
it must route through holding with `DeletionSource::Blacklist`.
|
||||
|
||||
|
||||
@@ -942,7 +942,11 @@ pub async fn create_relay(
|
||||
)
|
||||
})?;
|
||||
let tombstones = crate::nostr::tombstones::Tombstones::open_lmdb(db_path).await?;
|
||||
let holding = crate::nostr::holding::HoldingStore::open_lmdb(db_path).await?;
|
||||
let holding = crate::nostr::holding::HoldingStore::open_lmdb(
|
||||
db_path,
|
||||
Path::new(&config.effective_git_data_path()),
|
||||
)
|
||||
.await?;
|
||||
(Arc::new(db), tombstones, holding)
|
||||
}
|
||||
};
|
||||
|
||||
+101
-6
@@ -4,7 +4,7 @@
|
||||
//! - copy/move deleted events out of the main DB into a dedicated holding DB,
|
||||
//! - persist minimal per-event metadata for later retention/recovery phases.
|
||||
|
||||
use std::path::Path;
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -20,6 +20,9 @@ pub const HOLDING_DIR: &str = "holding";
|
||||
|
||||
/// Internal metadata event kind stored alongside archived events.
|
||||
pub const HOLDING_METADATA_KIND: u16 = 9905;
|
||||
pub const HOLDING_ARCHIVE_PATH_TAG: &str = "holding-archive-path";
|
||||
pub const HOLDING_ARCHIVE_CREATED_AT_TAG: &str = "holding-archive-created-at";
|
||||
pub const HOLDING_OWNER_PUBKEY_TAG: &str = "holding-owner-pubkey";
|
||||
|
||||
/// Default retention window for deleted events kept in the holding DB.
|
||||
pub const DEFAULT_RETENTION: Duration = Duration::from_secs(90 * 24 * 60 * 60);
|
||||
@@ -48,18 +51,28 @@ pub struct HoldingMetadata {
|
||||
pub source: DeletionSource,
|
||||
pub coordinate: Option<String>,
|
||||
pub identifier: Option<String>,
|
||||
pub owner_pubkey: Option<String>,
|
||||
pub git_archive: Option<GitArchiveMetadata>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct GitArchiveMetadata {
|
||||
pub relative_path: String,
|
||||
pub created_at: Timestamp,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct HoldingStore {
|
||||
db: Arc<dyn NostrDatabase>,
|
||||
metadata_signer: Keys,
|
||||
archive_root: Option<PathBuf>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ExpiredMetadataRecord {
|
||||
pub metadata_event_id: EventId,
|
||||
pub archived_event_id: Option<EventId>,
|
||||
pub archive_relative_path: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
@@ -68,6 +81,7 @@ pub struct CleanupStats {
|
||||
pub expired_records: usize,
|
||||
pub metadata_deleted: usize,
|
||||
pub archived_events_deleted: usize,
|
||||
pub archive_files_deleted: usize,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for HoldingStore {
|
||||
@@ -77,7 +91,7 @@ impl std::fmt::Debug for HoldingStore {
|
||||
}
|
||||
|
||||
impl HoldingStore {
|
||||
pub async fn open_lmdb(relay_data_path: &Path) -> anyhow::Result<Self> {
|
||||
pub async fn open_lmdb(relay_data_path: &Path, git_data_path: &Path) -> anyhow::Result<Self> {
|
||||
let path = relay_data_path.join(HOLDING_DIR);
|
||||
std::fs::create_dir_all(&path).map_err(|e| {
|
||||
anyhow::anyhow!(
|
||||
@@ -99,6 +113,7 @@ impl HoldingStore {
|
||||
Ok(Self {
|
||||
db: Arc::new(db),
|
||||
metadata_signer: Keys::generate(),
|
||||
archive_root: Some(git_data_path.join(".archive")),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -106,6 +121,7 @@ impl HoldingStore {
|
||||
Self {
|
||||
db: Arc::new(MemoryDatabase::unbounded()),
|
||||
metadata_signer: Keys::generate(),
|
||||
archive_root: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -134,6 +150,22 @@ impl HoldingStore {
|
||||
if let Some(identifier) = &metadata.identifier {
|
||||
tags.push(Tag::custom("d", vec![identifier.clone()]));
|
||||
}
|
||||
if let Some(owner_pubkey) = &metadata.owner_pubkey {
|
||||
tags.push(Tag::custom(
|
||||
HOLDING_OWNER_PUBKEY_TAG,
|
||||
vec![owner_pubkey.clone()],
|
||||
));
|
||||
}
|
||||
if let Some(git_archive) = &metadata.git_archive {
|
||||
tags.push(Tag::custom(
|
||||
HOLDING_ARCHIVE_PATH_TAG,
|
||||
vec![git_archive.relative_path.clone()],
|
||||
));
|
||||
tags.push(Tag::custom(
|
||||
HOLDING_ARCHIVE_CREATED_AT_TAG,
|
||||
vec![git_archive.created_at.as_secs().to_string()],
|
||||
));
|
||||
}
|
||||
|
||||
let metadata_event = EventBuilder::new(Kind::from(HOLDING_METADATA_KIND), "")
|
||||
.tags(tags)
|
||||
@@ -194,6 +226,38 @@ impl HoldingStore {
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_archive_relative_path(metadata_event: &Event) -> Option<String> {
|
||||
metadata_event.tags.iter().find_map(|tag| {
|
||||
let v = tag.as_slice();
|
||||
if v.len() >= 2 && v[0] == HOLDING_ARCHIVE_PATH_TAG {
|
||||
Some(v[1].clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn resolve_archive_path(&self, relative_path: &str) -> Option<PathBuf> {
|
||||
let root = self.archive_root.as_ref()?;
|
||||
let rel = Path::new(relative_path);
|
||||
if rel.is_absolute() {
|
||||
tracing::warn!(path = %relative_path, "Ignoring absolute holding archive path");
|
||||
return None;
|
||||
}
|
||||
|
||||
if rel.components().any(|c| {
|
||||
matches!(
|
||||
c,
|
||||
Component::ParentDir | Component::RootDir | Component::Prefix(_)
|
||||
)
|
||||
}) {
|
||||
tracing::warn!(path = %relative_path, "Ignoring unsafe holding archive path");
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(root.join(rel))
|
||||
}
|
||||
|
||||
/// Find expired metadata records in holding based on `holding-deleted-at`
|
||||
/// and a retention window.
|
||||
///
|
||||
@@ -223,6 +287,7 @@ impl HoldingStore {
|
||||
Some(ExpiredMetadataRecord {
|
||||
metadata_event_id: metadata_event.id,
|
||||
archived_event_id: Self::parse_archived_event_id(&metadata_event),
|
||||
archive_relative_path: Self::parse_archive_relative_path(&metadata_event),
|
||||
})
|
||||
} else {
|
||||
None
|
||||
@@ -238,9 +303,33 @@ impl HoldingStore {
|
||||
pub async fn delete_expired_record(
|
||||
&self,
|
||||
record: ExpiredMetadataRecord,
|
||||
) -> anyhow::Result<(bool, bool)> {
|
||||
) -> anyhow::Result<(bool, bool, bool)> {
|
||||
let mut metadata_deleted = false;
|
||||
let mut event_deleted = false;
|
||||
let mut archive_deleted = false;
|
||||
|
||||
if let Some(relative_path) = record.archive_relative_path.as_deref() {
|
||||
if let Some(absolute_path) = self.resolve_archive_path(relative_path) {
|
||||
match std::fs::remove_file(&absolute_path) {
|
||||
Ok(()) => {
|
||||
archive_deleted = true;
|
||||
}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
|
||||
tracing::warn!(
|
||||
path = %absolute_path.display(),
|
||||
"Holding cleanup archive file missing; continuing"
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
path = %absolute_path.display(),
|
||||
error = %e,
|
||||
"Holding cleanup failed to delete archive file; continuing"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(event_id) = record.archived_event_id {
|
||||
let exists = self.db.event_by_id(&event_id).await.map_err(|e| {
|
||||
@@ -280,7 +369,7 @@ impl HoldingStore {
|
||||
metadata_deleted = true;
|
||||
}
|
||||
|
||||
Ok((metadata_deleted, event_deleted))
|
||||
Ok((metadata_deleted, event_deleted, archive_deleted))
|
||||
}
|
||||
|
||||
/// Cleanup pass for expired holding records.
|
||||
@@ -310,13 +399,17 @@ impl HoldingStore {
|
||||
};
|
||||
|
||||
for record in expired {
|
||||
let (metadata_deleted, event_deleted) = self.delete_expired_record(record).await?;
|
||||
let (metadata_deleted, event_deleted, archive_deleted) =
|
||||
self.delete_expired_record(record).await?;
|
||||
if metadata_deleted {
|
||||
stats.metadata_deleted += 1;
|
||||
}
|
||||
if event_deleted {
|
||||
stats.archived_events_deleted += 1;
|
||||
}
|
||||
if archive_deleted {
|
||||
stats.archive_files_deleted += 1;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(stats)
|
||||
@@ -370,6 +463,8 @@ mod tests {
|
||||
source: DeletionSource::Nip09,
|
||||
coordinate: None,
|
||||
identifier: None,
|
||||
owner_pubkey: None,
|
||||
git_archive: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -30,14 +30,20 @@
|
||||
/// Author ownership is enforced before any main-DB deletion or tombstone
|
||||
/// recording: only the original author may delete their event.
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs::File;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use flate2::write::GzEncoder;
|
||||
use flate2::Compression;
|
||||
use nostr::nips::nip19::ToBech32;
|
||||
use nostr_relay_builder::prelude::{
|
||||
Alphabet, Event, EventId, Filter, Kind, PublicKey, SingleLetterTag, Timestamp,
|
||||
WritePolicyResult,
|
||||
};
|
||||
use tar::Builder as TarBuilder;
|
||||
|
||||
use super::{PolicyContext, RetentionReason};
|
||||
use crate::nostr::holding::{DeletionSource, HoldingMetadata};
|
||||
use crate::nostr::holding::{DeletionSource, GitArchiveMetadata, HoldingMetadata};
|
||||
use crate::nostr::policy::reject_invalid;
|
||||
|
||||
/// NIP-34 event kinds (other than the kind-30617 announcement and kind-30618
|
||||
@@ -183,6 +189,8 @@ impl DeletionPolicy {
|
||||
source: DeletionSource::Nip09,
|
||||
coordinate: None,
|
||||
identifier: None,
|
||||
owner_pubkey: None,
|
||||
git_archive: None,
|
||||
};
|
||||
self.archive_and_delete_filter(filter, &metadata, moved_ids, "NIP-09 e-tag deletion")
|
||||
.await;
|
||||
@@ -248,10 +256,11 @@ impl DeletionPolicy {
|
||||
/// 5. Hard-delete the orphaned event ids plus the announcement coordinate
|
||||
/// itself from the main DB.
|
||||
///
|
||||
/// Phase-2 holding orchestration is active in this cascade path via
|
||||
/// Holding orchestration is active in this cascade path via
|
||||
/// [`Self::archive_and_delete_filter`]: every deleted orphan/coordinate
|
||||
/// target is moved into holding before main-DB deletion. Archive-file
|
||||
/// lifecycle and recovery orchestration remain future phases.
|
||||
/// target is moved into holding before main-DB deletion, and announcement
|
||||
/// deletions include git archive metadata linkage for cleanup lifecycle.
|
||||
/// Recovery orchestration remains a future phase.
|
||||
async fn cascade_delete_announcement(
|
||||
&self,
|
||||
author: &PublicKey,
|
||||
@@ -383,6 +392,14 @@ impl DeletionPolicy {
|
||||
source: DeletionSource::Nip09,
|
||||
coordinate: Some(announcement_addr.to_string()),
|
||||
identifier: Some(identifier.to_string()),
|
||||
owner_pubkey: Some(author.to_hex()),
|
||||
git_archive: self
|
||||
.ensure_repo_archive(
|
||||
&owner_directory_component(author),
|
||||
identifier,
|
||||
deletion_created_at,
|
||||
)
|
||||
.await,
|
||||
};
|
||||
self.archive_and_delete_filter(
|
||||
filter,
|
||||
@@ -460,6 +477,8 @@ impl DeletionPolicy {
|
||||
source,
|
||||
coordinate: None,
|
||||
identifier: Some(identifier.to_string()),
|
||||
owner_pubkey: None,
|
||||
git_archive: None,
|
||||
};
|
||||
self.archive_and_delete_filter(
|
||||
state_filter,
|
||||
@@ -542,7 +561,37 @@ impl DeletionPolicy {
|
||||
source,
|
||||
coordinate: Some(coordinate.to_string()),
|
||||
identifier: Some(identifier.to_string()),
|
||||
owner_pubkey: Some(author.to_hex()),
|
||||
git_archive: if kind == Kind::GitRepoAnnouncement {
|
||||
self.ensure_repo_archive(
|
||||
&owner_directory_component(author),
|
||||
identifier,
|
||||
deletion_created_at,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
None
|
||||
},
|
||||
};
|
||||
|
||||
// Fail-safe preservation policy: when deleting an announcement, if git
|
||||
// archival was required but failed, skip the destructive event delete.
|
||||
if kind == Kind::GitRepoAnnouncement
|
||||
&& self.repo_exists_for_owner_and_identifier(
|
||||
&owner_directory_component(author),
|
||||
identifier,
|
||||
)
|
||||
&& metadata.git_archive.is_none()
|
||||
{
|
||||
tracing::warn!(
|
||||
author = %author.to_hex(),
|
||||
identifier = %identifier,
|
||||
op = "coordinate deletion",
|
||||
"Skipping announcement deletion because git archival failed (fail-safe preservation)"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
self.archive_and_delete_filter(filter, &metadata, moved_ids, "coordinate deletion")
|
||||
.await;
|
||||
}
|
||||
@@ -568,8 +617,45 @@ impl DeletionPolicy {
|
||||
|
||||
let mut deletable = Vec::with_capacity(matches.len());
|
||||
for event in matches {
|
||||
let mut event_metadata = metadata.clone();
|
||||
|
||||
if event.kind == Kind::GitRepoAnnouncement {
|
||||
let owner_hex = event.pubkey.to_hex();
|
||||
let owner_dir = owner_directory_component(&event.pubkey);
|
||||
if let Some(identifier) = identifier_from_event(&event) {
|
||||
event_metadata.identifier = Some(identifier.clone());
|
||||
event_metadata.owner_pubkey = Some(owner_hex.clone());
|
||||
|
||||
if event_metadata.git_archive.is_none() {
|
||||
event_metadata.git_archive = self
|
||||
.ensure_repo_archive(&owner_dir, &identifier, metadata.deleted_at)
|
||||
.await;
|
||||
}
|
||||
|
||||
// Fail-safe policy applies here too: if this announcement has
|
||||
// on-disk git data but we couldn't archive it, skip delete.
|
||||
if self.repo_exists_for_owner_and_identifier(&owner_dir, &identifier)
|
||||
&& event_metadata.git_archive.is_none()
|
||||
{
|
||||
tracing::warn!(
|
||||
event_id = %event.id.to_hex(),
|
||||
owner = %owner_hex,
|
||||
identifier = %identifier,
|
||||
op = %context,
|
||||
"Skipping announcement deletion because git archival failed (fail-safe preservation)"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if moved_ids.insert(event.id) {
|
||||
if let Err(e) = self.ctx.holding.archive_event(&event, metadata).await {
|
||||
if let Err(e) = self
|
||||
.ctx
|
||||
.holding
|
||||
.archive_event(&event, &event_metadata)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(
|
||||
error = %e,
|
||||
event_id = %event.id.to_hex(),
|
||||
@@ -592,6 +678,115 @@ impl DeletionPolicy {
|
||||
}
|
||||
}
|
||||
|
||||
fn archive_output_path(
|
||||
&self,
|
||||
owner_path_component: &str,
|
||||
identifier: &str,
|
||||
deleted_at: Timestamp,
|
||||
) -> PathBuf {
|
||||
self.ctx
|
||||
.git_data_path
|
||||
.join(".archive")
|
||||
.join(owner_path_component)
|
||||
.join(format!("{}-{}.tar.gz", identifier, deleted_at.as_secs()))
|
||||
}
|
||||
|
||||
fn repo_path_for_owner_and_identifier(
|
||||
&self,
|
||||
owner_path_component: &str,
|
||||
identifier: &str,
|
||||
) -> PathBuf {
|
||||
self.ctx
|
||||
.git_data_path
|
||||
.join(owner_path_component)
|
||||
.join(format!("{}.git", identifier))
|
||||
}
|
||||
|
||||
fn repo_exists_for_owner_and_identifier(
|
||||
&self,
|
||||
owner_path_component: &str,
|
||||
identifier: &str,
|
||||
) -> bool {
|
||||
self.repo_path_for_owner_and_identifier(owner_path_component, identifier)
|
||||
.is_dir()
|
||||
}
|
||||
|
||||
async fn ensure_repo_archive(
|
||||
&self,
|
||||
owner_path_component: &str,
|
||||
identifier: &str,
|
||||
deleted_at: Timestamp,
|
||||
) -> Option<GitArchiveMetadata> {
|
||||
let repo_path = self.repo_path_for_owner_and_identifier(owner_path_component, identifier);
|
||||
if !repo_path.is_dir() {
|
||||
tracing::debug!(
|
||||
owner = %owner_path_component,
|
||||
identifier = %identifier,
|
||||
path = %repo_path.display(),
|
||||
"No on-disk git repository found for announcement deletion; skipping archive creation"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
let archive_path = self.archive_output_path(owner_path_component, identifier, deleted_at);
|
||||
let relative_path = format!(
|
||||
"{}/{}-{}.tar.gz",
|
||||
owner_path_component,
|
||||
identifier,
|
||||
deleted_at.as_secs()
|
||||
);
|
||||
|
||||
if archive_path.exists() {
|
||||
return Some(GitArchiveMetadata {
|
||||
relative_path,
|
||||
created_at: Timestamp::now(),
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(parent) = archive_path.parent() {
|
||||
if let Err(e) = std::fs::create_dir_all(parent) {
|
||||
tracing::warn!(
|
||||
owner = %owner_path_component,
|
||||
identifier = %identifier,
|
||||
path = %parent.display(),
|
||||
error = %e,
|
||||
"Failed to create archive directory"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
}
|
||||
|
||||
let tmp_path = archive_path.with_extension("tar.gz.tmp");
|
||||
let result = (|| -> anyhow::Result<()> {
|
||||
let archive_file = File::create(&tmp_path)?;
|
||||
let encoder = GzEncoder::new(archive_file, Compression::default());
|
||||
let mut tar = TarBuilder::new(encoder);
|
||||
tar.append_dir_all(format!("{}.git", identifier), &repo_path)?;
|
||||
let encoder = tar.into_inner()?;
|
||||
encoder.finish()?;
|
||||
std::fs::rename(&tmp_path, &archive_path)?;
|
||||
Ok(())
|
||||
})();
|
||||
|
||||
if let Err(e) = result {
|
||||
let _ = std::fs::remove_file(&tmp_path);
|
||||
tracing::warn!(
|
||||
owner = %owner_path_component,
|
||||
identifier = %identifier,
|
||||
repo = %repo_path.display(),
|
||||
archive = %archive_path.display(),
|
||||
error = %e,
|
||||
"Failed to archive git repository for deletion"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(GitArchiveMetadata {
|
||||
relative_path,
|
||||
created_at: Timestamp::now(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Remove any purgatory entries targeted by this deletion event.
|
||||
///
|
||||
/// Handles both reference styles from NIP-09:
|
||||
@@ -848,6 +1043,21 @@ fn announcement_address_of(event: &Event) -> String {
|
||||
format!("30617:{}:{}", event.pubkey.to_hex(), identifier)
|
||||
}
|
||||
|
||||
fn identifier_from_event(event: &Event) -> Option<String> {
|
||||
event.tags.iter().find_map(|tag| {
|
||||
let v = tag.as_slice();
|
||||
if v.len() >= 2 && v[0] == "d" {
|
||||
Some(v[1].clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn owner_directory_component(pubkey: &PublicKey) -> String {
|
||||
pubkey.to_bech32().unwrap_or_else(|_| pubkey.to_hex())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
//! Integration tests for Phase 4 git archive lifecycle.
|
||||
|
||||
mod common;
|
||||
|
||||
use common::{announcement_coordinate, build_deletion, publish_served_repo, TestRelay};
|
||||
use grasp_audit::{AuditClient, AuditConfig};
|
||||
use ngit_grasp::nostr::holding::{
|
||||
HoldingStore, HOLDING_ARCHIVE_PATH_TAG, HOLDING_METADATA_KIND, HOLDING_OWNER_PUBKEY_TAG,
|
||||
};
|
||||
use nostr_sdk::prelude::*;
|
||||
use std::time::Duration;
|
||||
|
||||
fn metadata_tag_value(event: &Event, key: &str) -> Option<String> {
|
||||
event.tags.iter().find_map(|tag| {
|
||||
let v = tag.as_slice();
|
||||
if v.len() >= 2 && v[0] == key {
|
||||
Some(v[1].clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async fn open_holding(relay: &TestRelay) -> HoldingStore {
|
||||
HoldingStore::open_lmdb(relay.relay_data_path(), relay.git_data_path())
|
||||
.await
|
||||
.expect("open holding db")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn announcement_deletion_creates_git_archive_and_metadata_linkage() {
|
||||
let relay = TestRelay::start_with_lmdb().await;
|
||||
let client = AuditClient::new(relay.url(), AuditConfig::isolated())
|
||||
.await
|
||||
.expect("create audit client");
|
||||
|
||||
let (announcement, repo_id) = publish_served_repo(&client, "git-archive-create").await;
|
||||
let coordinate = announcement_coordinate(&announcement, &repo_id);
|
||||
|
||||
let deletion = build_deletion(&client, &[], std::slice::from_ref(&coordinate));
|
||||
client.send_event(deletion).await.expect("send deletion");
|
||||
tokio::time::sleep(Duration::from_millis(600)).await;
|
||||
|
||||
let store = open_holding(&relay).await;
|
||||
let metadata = store.metadata_for_event(&announcement.id).await;
|
||||
assert!(
|
||||
!metadata.is_empty(),
|
||||
"announcement must have holding metadata"
|
||||
);
|
||||
|
||||
let archive_rel_path = metadata
|
||||
.iter()
|
||||
.find_map(|m| metadata_tag_value(m, HOLDING_ARCHIVE_PATH_TAG))
|
||||
.expect("holding metadata must include archive path");
|
||||
let owner = metadata
|
||||
.iter()
|
||||
.find_map(|m| metadata_tag_value(m, HOLDING_OWNER_PUBKEY_TAG))
|
||||
.expect("holding metadata must include owner linkage");
|
||||
|
||||
let archive_path = relay
|
||||
.git_data_path()
|
||||
.join(".archive")
|
||||
.join(&archive_rel_path);
|
||||
assert!(archive_path.exists(), "archive file must exist on disk");
|
||||
assert_eq!(owner, client.public_key().to_hex());
|
||||
|
||||
relay.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_removes_expired_archive_file_and_metadata() {
|
||||
let relay = TestRelay::start_with_lmdb().await;
|
||||
let client = AuditClient::new(relay.url(), AuditConfig::isolated())
|
||||
.await
|
||||
.expect("create audit client");
|
||||
|
||||
let (announcement, repo_id) = publish_served_repo(&client, "git-archive-cleanup").await;
|
||||
let coordinate = announcement_coordinate(&announcement, &repo_id);
|
||||
|
||||
let deletion = build_deletion(&client, &[], std::slice::from_ref(&coordinate));
|
||||
client.send_event(deletion).await.expect("send deletion");
|
||||
tokio::time::sleep(Duration::from_millis(600)).await;
|
||||
|
||||
let store = open_holding(&relay).await;
|
||||
let metadata = store.metadata_for_event(&announcement.id).await;
|
||||
let archive_rel_path = metadata
|
||||
.iter()
|
||||
.find_map(|m| metadata_tag_value(m, HOLDING_ARCHIVE_PATH_TAG))
|
||||
.expect("holding metadata must include archive path");
|
||||
let archive_path = relay
|
||||
.git_data_path()
|
||||
.join(".archive")
|
||||
.join(&archive_rel_path);
|
||||
assert!(
|
||||
archive_path.exists(),
|
||||
"archive file should exist before cleanup"
|
||||
);
|
||||
|
||||
let first = store
|
||||
.cleanup_expired(
|
||||
Timestamp::from_secs(Timestamp::now().as_secs() + 10_000),
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.await
|
||||
.expect("cleanup should succeed");
|
||||
|
||||
assert!(
|
||||
first.archive_files_deleted >= 1,
|
||||
"cleanup should delete at least one archive file"
|
||||
);
|
||||
assert!(
|
||||
!archive_path.exists(),
|
||||
"archive file must be removed by cleanup"
|
||||
);
|
||||
assert!(store.metadata_for_event(&announcement.id).await.is_empty());
|
||||
|
||||
let second = store
|
||||
.cleanup_expired(
|
||||
Timestamp::from_secs(Timestamp::now().as_secs() + 10_000),
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.await
|
||||
.expect("second cleanup should succeed");
|
||||
assert_eq!(second.expired_records, 0);
|
||||
assert_eq!(second.metadata_deleted, 0);
|
||||
assert_eq!(second.archive_files_deleted, 0);
|
||||
|
||||
relay.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_tolerates_missing_archive_file_and_removes_stale_metadata() {
|
||||
let relay = TestRelay::start_with_lmdb().await;
|
||||
let client = AuditClient::new(relay.url(), AuditConfig::isolated())
|
||||
.await
|
||||
.expect("create audit client");
|
||||
|
||||
let (announcement, repo_id) = publish_served_repo(&client, "git-archive-missing").await;
|
||||
let coordinate = announcement_coordinate(&announcement, &repo_id);
|
||||
|
||||
let deletion = build_deletion(&client, &[], std::slice::from_ref(&coordinate));
|
||||
client.send_event(deletion).await.expect("send deletion");
|
||||
tokio::time::sleep(Duration::from_millis(600)).await;
|
||||
|
||||
let store = open_holding(&relay).await;
|
||||
let metadata = store.metadata_for_event(&announcement.id).await;
|
||||
let archive_rel_path = metadata
|
||||
.iter()
|
||||
.find_map(|m| metadata_tag_value(m, HOLDING_ARCHIVE_PATH_TAG))
|
||||
.expect("holding metadata must include archive path");
|
||||
let archive_path = relay
|
||||
.git_data_path()
|
||||
.join(".archive")
|
||||
.join(&archive_rel_path);
|
||||
std::fs::remove_file(&archive_path).expect("remove archive file to simulate missing file");
|
||||
|
||||
let stats = store
|
||||
.cleanup_expired(
|
||||
Timestamp::from_secs(Timestamp::now().as_secs() + 10_000),
|
||||
Duration::from_secs(1),
|
||||
)
|
||||
.await
|
||||
.expect("cleanup should succeed when archive file is missing");
|
||||
|
||||
assert!(
|
||||
stats.metadata_deleted >= 1,
|
||||
"stale metadata must be removed"
|
||||
);
|
||||
assert_eq!(stats.archive_files_deleted, 0);
|
||||
assert!(store.metadata_for_event(&announcement.id).await.is_empty());
|
||||
|
||||
relay.stop().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn disrespector_mode_does_not_archive_repository_data() {
|
||||
let relay = TestRelay::start_with_lmdb_deletion_disrespector().await;
|
||||
let client = AuditClient::new(relay.url(), AuditConfig::isolated())
|
||||
.await
|
||||
.expect("create audit client");
|
||||
|
||||
let (announcement, repo_id) = publish_served_repo(&client, "git-archive-disrespector").await;
|
||||
let coordinate = announcement_coordinate(&announcement, &repo_id);
|
||||
let deletion = build_deletion(
|
||||
&client,
|
||||
&[announcement.id],
|
||||
std::slice::from_ref(&coordinate),
|
||||
);
|
||||
client.send_event(deletion).await.expect("send deletion");
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
|
||||
assert!(
|
||||
client
|
||||
.is_event_on_relay(announcement.id)
|
||||
.await
|
||||
.expect("query announcement after disrespector deletion"),
|
||||
"announcement must remain served in disrespector mode"
|
||||
);
|
||||
|
||||
let store = open_holding(&relay).await;
|
||||
let metadata = store
|
||||
.metadata_for_event(&announcement.id)
|
||||
.await
|
||||
.into_iter()
|
||||
.filter(|e| e.kind == Kind::from(HOLDING_METADATA_KIND))
|
||||
.collect::<Vec<_>>();
|
||||
assert!(
|
||||
metadata.is_empty(),
|
||||
"disrespector mode must not move to holding"
|
||||
);
|
||||
|
||||
let archive_root = relay.git_data_path().join(".archive");
|
||||
assert!(
|
||||
!archive_root.exists(),
|
||||
"disrespector mode must not create archive files"
|
||||
);
|
||||
|
||||
relay.stop().await;
|
||||
}
|
||||
@@ -26,6 +26,8 @@ async fn cleanup_removes_expired_holding_events_and_metadata() {
|
||||
source: DeletionSource::Nip09,
|
||||
coordinate: None,
|
||||
identifier: None,
|
||||
owner_pubkey: None,
|
||||
git_archive: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -56,6 +58,8 @@ async fn cleanup_preserves_non_expired_entries() {
|
||||
source: DeletionSource::Nip09,
|
||||
coordinate: None,
|
||||
identifier: None,
|
||||
owner_pubkey: None,
|
||||
git_archive: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -76,7 +80,7 @@ async fn startup_catchup_cleanup_removes_expired_entries_after_restart() {
|
||||
let temp = tempfile::tempdir().expect("create temp dir");
|
||||
|
||||
let event_id = {
|
||||
let store = HoldingStore::open_lmdb(temp.path())
|
||||
let store = HoldingStore::open_lmdb(temp.path(), temp.path())
|
||||
.await
|
||||
.expect("open holding");
|
||||
let event = build_test_event("restart-expired", Timestamp::from_secs(10));
|
||||
@@ -89,6 +93,8 @@ async fn startup_catchup_cleanup_removes_expired_entries_after_restart() {
|
||||
source: DeletionSource::Nip09,
|
||||
coordinate: None,
|
||||
identifier: None,
|
||||
owner_pubkey: None,
|
||||
git_archive: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -97,7 +103,7 @@ async fn startup_catchup_cleanup_removes_expired_entries_after_restart() {
|
||||
};
|
||||
|
||||
// Simulate process restart: reopen store and run startup catch-up pass.
|
||||
let reopened = HoldingStore::open_lmdb(temp.path())
|
||||
let reopened = HoldingStore::open_lmdb(temp.path(), temp.path())
|
||||
.await
|
||||
.expect("reopen holding");
|
||||
let stats = reopened
|
||||
@@ -119,6 +125,8 @@ async fn cleanup_is_idempotent_with_partial_missing_event_data() {
|
||||
source: DeletionSource::Nip09,
|
||||
coordinate: None,
|
||||
identifier: None,
|
||||
owner_pubkey: None,
|
||||
git_archive: None,
|
||||
};
|
||||
|
||||
// Two metadata rows referencing the same archived event: first delete removes
|
||||
|
||||
@@ -14,7 +14,7 @@ use nostr_sdk::prelude::*;
|
||||
use std::time::Duration;
|
||||
|
||||
async fn open_holding(relay: &TestRelay) -> HoldingStore {
|
||||
HoldingStore::open_lmdb(relay.relay_data_path())
|
||||
HoldingStore::open_lmdb(relay.relay_data_path(), relay.git_data_path())
|
||||
.await
|
||||
.expect("open holding db")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user