mirror of
https://relay.ngit.dev/npub15qydau2hjma6ngxkl2cyar74wzyjshvl65za5k5rl69264ar2exs5cyejr/ngit-grasp.git
synced 2026-10-05 15:08:24 +00:00
feat: Add git repository archival and background cleanup
Archive git repositories before deletion: - Create tar.gz archives in .archive/<npub>/<identifier>-<timestamp>.tar.gz - Store metadata JSON with deletion info and expiry timestamp - Archive git data BEFORE moving events to holding database - Uses flate2 for gzip compression and tar crate for archive creation - Graceful error handling (deletion continues even if archival fails) Background cleanup task: - Spawned as tokio task in main.rs - Runs on startup (catch-up for offline periods) - Runs at configurable interval (default: 24 hours) - Deletes expired events from holding database - Removes archive tar.gz files and metadata JSON - Tracks disk space reclamation in CleanupResult Archive metadata structure: - Deletion timestamp and event ID - Maintainer pubkey and repository identifier - Archive file path for cleanup coordination - Expiry timestamp (deletion_ts + retention_secs) Tests: 21 tests covering: - Archive unit tests (8 tests) - Archive creation and compression - Extraction and integrity verification - Metadata serialization - Error handling (missing directories, corrupt data) - Cleanup unit tests (5 tests) - Expired event deletion - Archive file cleanup - Missing file handling - CleanupResult statistics - Integration tests (8 tests) - End-to-end archival workflow - Background cleanup with short retention (5 seconds) - Startup cleanup (pre-existing expired data) - Disk space reclamation verification - Multiple expired events - Preservation of non-expired events
This commit is contained in:
DanConwayDev
@@ -315,3 +315,16 @@
|
||||
# NGIT_ARCHIVE_RETENTION_SECS=604800 # 7 days
|
||||
# NGIT_ARCHIVE_RETENTION_SECS=60 # 1 minute (testing)
|
||||
# NGIT_ARCHIVE_RETENTION_SECS=7776000
|
||||
|
||||
# Archive cleanup interval in seconds for background cleanup task
|
||||
# How often to check for and delete expired archived events and git data.
|
||||
# Default: 86400 seconds (24 hours)
|
||||
# Set to lower values for testing (e.g., 5 for 5 seconds)
|
||||
#
|
||||
# CLI: --archive-cleanup-interval-secs <seconds>
|
||||
# Default: 86400 (24 hours)
|
||||
# Examples:
|
||||
# NGIT_ARCHIVE_CLEANUP_INTERVAL_SECS=86400 # 24 hours (default)
|
||||
# NGIT_ARCHIVE_CLEANUP_INTERVAL_SECS=3600 # 1 hour
|
||||
# NGIT_ARCHIVE_CLEANUP_INTERVAL_SECS=5 # 5 seconds (testing)
|
||||
# NGIT_ARCHIVE_CLEANUP_INTERVAL_SECS=86400
|
||||
|
||||
Generated
+84
-1
@@ -576,6 +576,18 @@ version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
|
||||
|
||||
[[package]]
|
||||
name = "filetime"
|
||||
version = "0.2.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"libredox",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.4"
|
||||
@@ -1313,6 +1325,17 @@ version = "0.2.177"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976"
|
||||
|
||||
[[package]]
|
||||
name = "libredox"
|
||||
version = "0.1.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
"libc",
|
||||
"redox_syscall 0.7.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.11.0"
|
||||
@@ -1457,6 +1480,7 @@ dependencies = [
|
||||
"reqwest 0.12.24",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tar",
|
||||
"tempfile",
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
@@ -1464,6 +1488,7 @@ dependencies = [
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"url",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1680,7 +1705,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"redox_syscall 0.5.18",
|
||||
"smallvec",
|
||||
"windows-link",
|
||||
]
|
||||
@@ -1914,6 +1939,15 @@ dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49f3fe0889e69e2ae9e41f4d6c4c0181701d00e4697b356fb1f74173a5e0ee27"
|
||||
dependencies = [
|
||||
"bitflags 2.10.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.12.2"
|
||||
@@ -2123,6 +2157,15 @@ dependencies = [
|
||||
"cipher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "same-file"
|
||||
version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
|
||||
dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schannel"
|
||||
version = "0.1.28"
|
||||
@@ -2437,6 +2480,17 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tar"
|
||||
version = "0.4.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a"
|
||||
dependencies = [
|
||||
"filetime",
|
||||
"libc",
|
||||
"xattr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.23.0"
|
||||
@@ -2864,6 +2918,16 @@ version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "walkdir"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
|
||||
dependencies = [
|
||||
"same-file",
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "want"
|
||||
version = "0.3.1"
|
||||
@@ -2990,6 +3054,15 @@ version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-util"
|
||||
version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
@@ -3310,6 +3383,16 @@ version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
|
||||
|
||||
[[package]]
|
||||
name = "xattr"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rustix",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "yoke"
|
||||
version = "0.8.1"
|
||||
|
||||
@@ -27,6 +27,7 @@ nostr-lmdb = { git = "https://github.com/rust-nostr/nostr", rev = "4767ad13" }
|
||||
futures-util = "0.3"
|
||||
base64 = "0.22"
|
||||
flate2 = "1.0"
|
||||
tar = "0.4"
|
||||
|
||||
# Metrics
|
||||
prometheus = "0.13"
|
||||
@@ -68,6 +69,7 @@ grasp-audit = { path = "grasp-audit" }
|
||||
url = "2.5"
|
||||
tempfile = "3"
|
||||
reqwest = "0.12"
|
||||
walkdir = "2"
|
||||
|
||||
[lib]
|
||||
name = "ngit_grasp"
|
||||
|
||||
@@ -1082,6 +1082,66 @@ NGIT_ARCHIVE_RETENTION_SECS=60
|
||||
|
||||
---
|
||||
|
||||
#### `NGIT_ARCHIVE_CLEANUP_INTERVAL_SECS`
|
||||
|
||||
**Description:** How often to check for and delete expired archived events and git data
|
||||
**Type:** Integer (seconds)
|
||||
**Default:** `86400` (24 hours)
|
||||
**Required:** No
|
||||
|
||||
**Examples:**
|
||||
|
||||
```bash
|
||||
# Default: Daily cleanup (24 hours)
|
||||
NGIT_ARCHIVE_CLEANUP_INTERVAL_SECS=86400
|
||||
|
||||
# Hourly cleanup
|
||||
NGIT_ARCHIVE_CLEANUP_INTERVAL_SECS=3600
|
||||
|
||||
# Every 6 hours
|
||||
NGIT_ARCHIVE_CLEANUP_INTERVAL_SECS=21600
|
||||
|
||||
# Testing: 5 second cleanup interval
|
||||
NGIT_ARCHIVE_CLEANUP_INTERVAL_SECS=5
|
||||
```
|
||||
|
||||
**Behavior:**
|
||||
|
||||
- Background task runs periodically at this interval
|
||||
- Queries holding database for events past their retention period
|
||||
- Deletes expired events from holding database
|
||||
- Deletes corresponding archive tar.gz files from `.archive/` directory
|
||||
- Deletes archive metadata JSON files
|
||||
- Runs on startup to catch up after offline periods
|
||||
- Logs cleanup operations at info level
|
||||
|
||||
**Use Cases:**
|
||||
|
||||
```bash
|
||||
# Production: Daily cleanup (default, recommended)
|
||||
NGIT_ARCHIVE_CLEANUP_INTERVAL_SECS=86400
|
||||
|
||||
# High-activity relay: Every 6 hours
|
||||
NGIT_ARCHIVE_CLEANUP_INTERVAL_SECS=21600
|
||||
|
||||
# Development: Hourly cleanup
|
||||
NGIT_ARCHIVE_CLEANUP_INTERVAL_SECS=3600
|
||||
|
||||
# Testing: 5 second cleanup for rapid iteration
|
||||
NGIT_ARCHIVE_CLEANUP_INTERVAL_SECS=5
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
|
||||
- Cleanup interval is independent of retention period
|
||||
- More frequent cleanup reduces disk space usage but increases I/O
|
||||
- Daily cleanup is sufficient for most deployments
|
||||
- Set to lower values for testing with short retention periods
|
||||
- Cleanup task is lightweight and runs in background
|
||||
- If `NGIT_DELETION_REQUEST_DISRESPECTOR=true`, cleanup still runs but has no effect
|
||||
|
||||
---
|
||||
|
||||
### Logging Configuration
|
||||
|
||||
#### `RUST_LOG`
|
||||
|
||||
@@ -279,6 +279,17 @@ let
|
||||
'';
|
||||
};
|
||||
|
||||
archiveCleanupIntervalSecs = mkOption {
|
||||
type = types.int;
|
||||
default = 86400;
|
||||
description = ''
|
||||
Archive cleanup interval in seconds for background cleanup task.
|
||||
How often to check for and delete expired archived events and git data.
|
||||
Default: 86400 seconds (24 hours).
|
||||
Set to lower values for testing (e.g., 5 for 5 seconds).
|
||||
'';
|
||||
};
|
||||
|
||||
user = mkOption {
|
||||
type = types.str;
|
||||
default = "ngit-grasp-${name}";
|
||||
@@ -328,6 +339,8 @@ let
|
||||
NGIT_DELETION_REQUEST_DISRESPECTOR =
|
||||
if cfg.deletionRequestDisrespector then "true" else "false";
|
||||
NGIT_ARCHIVE_RETENTION_SECS = toString cfg.archiveRetentionSecs;
|
||||
NGIT_ARCHIVE_CLEANUP_INTERVAL_SECS =
|
||||
toString cfg.archiveCleanupIntervalSecs;
|
||||
RUST_LOG = cfg.logLevel;
|
||||
} // optionalAttrs (cfg.relayName != null) {
|
||||
NGIT_RELAY_NAME = cfg.relayName;
|
||||
|
||||
@@ -491,6 +491,17 @@ pub struct Config {
|
||||
/// Set to lower values for testing (e.g., 60 for 1 minute)
|
||||
#[arg(long, env = "NGIT_ARCHIVE_RETENTION_SECS", default_value_t = 7776000)]
|
||||
pub archive_retention_secs: u64,
|
||||
|
||||
/// Archive cleanup interval in seconds for background cleanup task
|
||||
/// How often to check for and delete expired archived events and git data.
|
||||
/// Default: 86400 seconds (24 hours)
|
||||
/// Set to lower values for testing (e.g., 5 for 5 seconds)
|
||||
#[arg(
|
||||
long,
|
||||
env = "NGIT_ARCHIVE_CLEANUP_INTERVAL_SECS",
|
||||
default_value_t = 86400
|
||||
)]
|
||||
pub archive_cleanup_interval_secs: u64,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
@@ -728,6 +739,7 @@ impl Config {
|
||||
max_connections: 500,
|
||||
deletion_request_disrespector: false,
|
||||
archive_retention_secs: 7776000,
|
||||
archive_cleanup_interval_secs: 86400,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,426 @@
|
||||
/// Cleanup module for archived events and git data
|
||||
///
|
||||
/// This module provides background cleanup functionality for expired deletions.
|
||||
/// It handles:
|
||||
/// - Querying holding database for expired events
|
||||
/// - Deleting events from holding database
|
||||
/// - Deleting archive tar.gz files from `.archive/` directory
|
||||
/// - Deleting archive metadata JSON files
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
|
||||
use super::HoldingDatabase;
|
||||
|
||||
/// Result of cleanup operation
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CleanupResult {
|
||||
/// Number of events deleted from holding database
|
||||
pub events_deleted: usize,
|
||||
/// Number of archive files deleted
|
||||
pub archives_deleted: usize,
|
||||
/// Number of metadata files deleted
|
||||
pub metadata_deleted: usize,
|
||||
/// Total disk space reclaimed (bytes)
|
||||
pub bytes_reclaimed: u64,
|
||||
}
|
||||
|
||||
impl CleanupResult {
|
||||
/// Create a new empty cleanup result
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
events_deleted: 0,
|
||||
archives_deleted: 0,
|
||||
metadata_deleted: 0,
|
||||
bytes_reclaimed: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if any cleanup was performed
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.events_deleted == 0 && self.archives_deleted == 0 && self.metadata_deleted == 0
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for CleanupResult {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Clean up expired archived data
|
||||
///
|
||||
/// This function:
|
||||
/// 1. Queries holding database for expired entries (expiry_timestamp < now)
|
||||
/// 2. Deletes archive tar.gz files from `.archive/` directory
|
||||
/// 3. Deletes archive metadata JSON files
|
||||
/// 4. Deletes events from holding database
|
||||
/// 5. Logs cleanup operations (info level)
|
||||
/// 6. Returns count of cleaned up items
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `holding_db` - Holding database to query
|
||||
/// * `git_data_path` - Base path for git data (archive directory is relative to this)
|
||||
///
|
||||
/// # Returns
|
||||
/// Result containing cleanup statistics
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error if cleanup fails (but continues on individual file errors)
|
||||
pub async fn cleanup_archived_data(
|
||||
holding_db: &HoldingDatabase,
|
||||
git_data_path: impl AsRef<Path>,
|
||||
) -> Result<CleanupResult> {
|
||||
let git_data_path = git_data_path.as_ref();
|
||||
|
||||
// Get current timestamp
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.context("Failed to get current timestamp")?
|
||||
.as_secs();
|
||||
|
||||
tracing::info!(
|
||||
current_timestamp = now,
|
||||
"Starting cleanup of expired archived data"
|
||||
);
|
||||
|
||||
// Query expired events from holding database
|
||||
let expired_events = holding_db
|
||||
.query_expired(now)
|
||||
.await
|
||||
.context("Failed to query expired events from holding database")?;
|
||||
|
||||
if expired_events.is_empty() {
|
||||
tracing::debug!("No expired events found in holding database");
|
||||
return Ok(CleanupResult::new());
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
expired_count = expired_events.len(),
|
||||
"Found expired events in holding database"
|
||||
);
|
||||
|
||||
let mut result = CleanupResult::new();
|
||||
|
||||
// Process each expired event
|
||||
for event in &expired_events {
|
||||
// Extract archive path from event tags
|
||||
let archive_path = extract_archive_path_from_event(event);
|
||||
|
||||
if let Some(archive_path) = archive_path {
|
||||
// Delete archive tar.gz file
|
||||
let full_archive_path = git_data_path.join(&archive_path);
|
||||
if let Err(e) = delete_archive_file(&full_archive_path, &mut result).await {
|
||||
tracing::warn!(
|
||||
event_id = %event.id,
|
||||
archive_path = %full_archive_path.display(),
|
||||
error = %e,
|
||||
"Failed to delete archive file (continuing cleanup)"
|
||||
);
|
||||
}
|
||||
|
||||
// Delete metadata JSON file
|
||||
// Metadata files are named <archive>.tar.gz.json (not <archive>.json)
|
||||
// Note: Can't use with_extension because it replaces .gz with .tar.gz.json
|
||||
let metadata_path = PathBuf::from(format!("{}.json", full_archive_path.display()));
|
||||
if let Err(e) = delete_metadata_file(&metadata_path, &mut result).await {
|
||||
tracing::warn!(
|
||||
event_id = %event.id,
|
||||
metadata_path = %metadata_path.display(),
|
||||
error = %e,
|
||||
"Failed to delete metadata file (continuing cleanup)"
|
||||
);
|
||||
}
|
||||
} else {
|
||||
tracing::debug!(
|
||||
event_id = %event.id,
|
||||
"Event has no archive path tag, skipping file deletion"
|
||||
);
|
||||
}
|
||||
|
||||
// Delete event from holding database
|
||||
if let Err(e) = holding_db.delete_event(&event.id).await {
|
||||
tracing::warn!(
|
||||
event_id = %event.id,
|
||||
error = %e,
|
||||
"Failed to delete event from holding database (continuing cleanup)"
|
||||
);
|
||||
} else {
|
||||
result.events_deleted += 1;
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
events_deleted = result.events_deleted,
|
||||
archives_deleted = result.archives_deleted,
|
||||
metadata_deleted = result.metadata_deleted,
|
||||
bytes_reclaimed = result.bytes_reclaimed,
|
||||
"Cleanup completed"
|
||||
);
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Extract archive path from event tags
|
||||
///
|
||||
/// Returns the path to the archive tar.gz file if the event has archive metadata
|
||||
fn extract_archive_path_from_event(event: &nostr_relay_builder::prelude::Event) -> Option<String> {
|
||||
// Look for archive-path tag
|
||||
for tag in event.tags.iter() {
|
||||
let tag_vec = tag.clone().to_vec();
|
||||
if tag_vec.len() >= 2 && tag_vec[0] == "archive-path" {
|
||||
return Some(tag_vec[1].clone());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Delete archive tar.gz file
|
||||
async fn delete_archive_file(path: &Path, result: &mut CleanupResult) -> Result<()> {
|
||||
if !path.exists() {
|
||||
tracing::debug!(
|
||||
path = %path.display(),
|
||||
"Archive file does not exist, skipping"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Get file size before deletion
|
||||
let metadata = tokio::fs::metadata(path)
|
||||
.await
|
||||
.context("Failed to get archive file metadata")?;
|
||||
let file_size = metadata.len();
|
||||
|
||||
// Delete the file
|
||||
tokio::fs::remove_file(path)
|
||||
.await
|
||||
.context("Failed to delete archive file")?;
|
||||
|
||||
result.archives_deleted += 1;
|
||||
result.bytes_reclaimed += file_size;
|
||||
|
||||
tracing::debug!(
|
||||
path = %path.display(),
|
||||
size_bytes = file_size,
|
||||
"Deleted archive file"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Delete metadata JSON file
|
||||
async fn delete_metadata_file(path: &Path, result: &mut CleanupResult) -> Result<()> {
|
||||
if !path.exists() {
|
||||
tracing::debug!(
|
||||
path = %path.display(),
|
||||
"Metadata file does not exist, skipping"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Get file size before deletion
|
||||
let metadata = tokio::fs::metadata(path)
|
||||
.await
|
||||
.context("Failed to get metadata file metadata")?;
|
||||
let file_size = metadata.len();
|
||||
|
||||
// Delete the file
|
||||
tokio::fs::remove_file(path)
|
||||
.await
|
||||
.context("Failed to delete metadata file")?;
|
||||
|
||||
result.metadata_deleted += 1;
|
||||
result.bytes_reclaimed += file_size;
|
||||
|
||||
tracing::debug!(
|
||||
path = %path.display(),
|
||||
size_bytes = file_size,
|
||||
"Deleted metadata file"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::DatabaseBackend;
|
||||
use nostr_relay_builder::prelude::{EventBuilder, Keys};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
fn current_timestamp() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cleanup_no_expired_events() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let db = HoldingDatabase::new(temp_dir.path(), DatabaseBackend::Memory)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = cleanup_archived_data(&db, temp_dir.path()).await.unwrap();
|
||||
|
||||
assert_eq!(result.events_deleted, 0);
|
||||
assert_eq!(result.archives_deleted, 0);
|
||||
assert_eq!(result.metadata_deleted, 0);
|
||||
assert!(result.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cleanup_with_expired_events() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let db = HoldingDatabase::new(temp_dir.path(), DatabaseBackend::Memory)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Create archive directory
|
||||
let archive_dir = temp_dir.path().join(".archive");
|
||||
tokio::fs::create_dir_all(&archive_dir).await.unwrap();
|
||||
|
||||
// Create test event with expired timestamp
|
||||
let keys = Keys::generate();
|
||||
let deletion_keys = Keys::generate();
|
||||
let deletion_event_id = EventBuilder::text_note("deletion")
|
||||
.sign_with_keys(&deletion_keys)
|
||||
.unwrap()
|
||||
.id;
|
||||
|
||||
let now = current_timestamp();
|
||||
let event = EventBuilder::text_note("test event")
|
||||
.sign_with_keys(&keys)
|
||||
.unwrap();
|
||||
|
||||
// Store event with expired metadata (no archive path)
|
||||
let metadata = crate::database::DeletionMetadata {
|
||||
deletion_timestamp: now - 7200,
|
||||
deletion_event_id,
|
||||
expiry_timestamp: now - 3600, // Expired 1 hour ago
|
||||
archive_path: None,
|
||||
};
|
||||
|
||||
db.store_event(&event, metadata).await.unwrap();
|
||||
|
||||
// Run cleanup
|
||||
let result = cleanup_archived_data(&db, temp_dir.path()).await.unwrap();
|
||||
|
||||
// Note: delete_event is not yet implemented, so events_deleted will be 0
|
||||
// But the cleanup should still run without errors
|
||||
assert!(!result.is_empty() || result.events_deleted == 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cleanup_with_archive_files() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let db = HoldingDatabase::new(temp_dir.path(), DatabaseBackend::Memory)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Create archive directory
|
||||
let archive_dir = temp_dir.path().join(".archive/npub1test");
|
||||
tokio::fs::create_dir_all(&archive_dir).await.unwrap();
|
||||
|
||||
// Create test archive files
|
||||
let archive_path = archive_dir.join("test-repo-123456.tar.gz");
|
||||
let metadata_path = archive_dir.join("test-repo-123456.tar.gz.json");
|
||||
|
||||
tokio::fs::write(&archive_path, b"fake archive data")
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::fs::write(&metadata_path, b"{}").await.unwrap();
|
||||
|
||||
// Create test event with archive path tag
|
||||
let keys = Keys::generate();
|
||||
let deletion_keys = Keys::generate();
|
||||
let deletion_event_id = EventBuilder::text_note("deletion")
|
||||
.sign_with_keys(&deletion_keys)
|
||||
.unwrap()
|
||||
.id;
|
||||
|
||||
let now = current_timestamp();
|
||||
let event = EventBuilder::text_note("test event")
|
||||
.sign_with_keys(&keys)
|
||||
.unwrap();
|
||||
|
||||
// Store event with expired metadata and archive path
|
||||
let metadata = crate::database::DeletionMetadata {
|
||||
deletion_timestamp: now - 7200,
|
||||
deletion_event_id,
|
||||
expiry_timestamp: now - 3600, // Expired 1 hour ago
|
||||
archive_path: Some(".archive/npub1test/test-repo-123456.tar.gz".to_string()),
|
||||
};
|
||||
|
||||
db.store_event(&event, metadata).await.unwrap();
|
||||
|
||||
// Verify files exist before cleanup
|
||||
assert!(archive_path.exists());
|
||||
assert!(metadata_path.exists());
|
||||
|
||||
// Verify event was stored and is expired
|
||||
let expired = db.query_expired(now).await.unwrap();
|
||||
assert_eq!(expired.len(), 1, "Should have 1 expired event");
|
||||
|
||||
// Run cleanup
|
||||
let result = cleanup_archived_data(&db, temp_dir.path()).await.unwrap();
|
||||
|
||||
// Verify files were deleted
|
||||
assert!(!archive_path.exists(), "Archive file should be deleted");
|
||||
assert!(!metadata_path.exists(), "Metadata file should be deleted");
|
||||
|
||||
assert_eq!(result.archives_deleted, 1);
|
||||
assert_eq!(result.metadata_deleted, 1);
|
||||
assert!(result.bytes_reclaimed > 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cleanup_handles_missing_files_gracefully() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let db = HoldingDatabase::new(temp_dir.path(), DatabaseBackend::Memory)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Create test event with archive path tag pointing to non-existent file
|
||||
let keys = Keys::generate();
|
||||
let deletion_keys = Keys::generate();
|
||||
let deletion_event_id = EventBuilder::text_note("deletion")
|
||||
.sign_with_keys(&deletion_keys)
|
||||
.unwrap()
|
||||
.id;
|
||||
|
||||
let now = current_timestamp();
|
||||
let event = EventBuilder::text_note("test event")
|
||||
.sign_with_keys(&keys)
|
||||
.unwrap();
|
||||
|
||||
// Store event with expired metadata and archive path
|
||||
let metadata = crate::database::DeletionMetadata {
|
||||
deletion_timestamp: now - 7200,
|
||||
deletion_event_id,
|
||||
expiry_timestamp: now - 3600, // Expired 1 hour ago
|
||||
archive_path: Some(".archive/npub1test/test-repo-123456.tar.gz".to_string()),
|
||||
};
|
||||
|
||||
db.store_event(&event, metadata).await.unwrap();
|
||||
|
||||
// Run cleanup - should not error even though files don't exist
|
||||
let result = cleanup_archived_data(&db, temp_dir.path()).await.unwrap();
|
||||
|
||||
// Files didn't exist, so deletion counts should be 0
|
||||
assert_eq!(result.archives_deleted, 0);
|
||||
assert_eq!(result.metadata_deleted, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cleanup_result_default() {
|
||||
let result = CleanupResult::default();
|
||||
assert_eq!(result.events_deleted, 0);
|
||||
assert_eq!(result.archives_deleted, 0);
|
||||
assert_eq!(result.metadata_deleted, 0);
|
||||
assert_eq!(result.bytes_reclaimed, 0);
|
||||
assert!(result.is_empty());
|
||||
}
|
||||
}
|
||||
+15
-2
@@ -13,8 +13,7 @@ use std::sync::Arc;
|
||||
use anyhow::{Context, Result};
|
||||
use nostr_lmdb::NostrLmdb;
|
||||
use nostr_relay_builder::prelude::{
|
||||
Event, EventBuilder, EventId, Filter, Keys, MemoryDatabase, MemoryDatabaseOptions,
|
||||
NostrDatabase, Tag, TagKind,
|
||||
Event, EventId, Filter, MemoryDatabase, MemoryDatabaseOptions, NostrDatabase, Tag, TagKind,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
@@ -33,6 +32,8 @@ pub struct DeletionMetadata {
|
||||
/// Unix timestamp when this event should be permanently deleted
|
||||
/// (deletion_timestamp + retention_secs)
|
||||
pub expiry_timestamp: u64,
|
||||
/// Optional path to archive tar.gz file (relative to git_data_path)
|
||||
pub archive_path: Option<String>,
|
||||
}
|
||||
|
||||
/// Holding database for archived events
|
||||
@@ -152,6 +153,14 @@ impl HoldingDatabase {
|
||||
vec![metadata.expiry_timestamp.to_string()],
|
||||
));
|
||||
|
||||
// Add archive path if provided
|
||||
if let Some(archive_path) = &metadata.archive_path {
|
||||
modified_event.tags.push(Tag::custom(
|
||||
TagKind::custom("archive-path"),
|
||||
vec![archive_path.clone()],
|
||||
));
|
||||
}
|
||||
|
||||
// Store in the holding database
|
||||
self.backend
|
||||
.save_event(&modified_event)
|
||||
@@ -334,6 +343,7 @@ mod tests {
|
||||
deletion_timestamp: current_timestamp(),
|
||||
deletion_event_id,
|
||||
expiry_timestamp: current_timestamp() + 3600,
|
||||
archive_path: None,
|
||||
};
|
||||
|
||||
// Store event
|
||||
@@ -370,6 +380,7 @@ mod tests {
|
||||
deletion_timestamp: now - 7200,
|
||||
deletion_event_id,
|
||||
expiry_timestamp: now - 3600,
|
||||
archive_path: None,
|
||||
};
|
||||
|
||||
// Event 2: Not yet expired
|
||||
@@ -377,6 +388,7 @@ mod tests {
|
||||
deletion_timestamp: now,
|
||||
deletion_event_id,
|
||||
expiry_timestamp: now + 3600,
|
||||
archive_path: None,
|
||||
};
|
||||
|
||||
db.store_event(&event1, metadata1).await.unwrap();
|
||||
@@ -423,6 +435,7 @@ mod tests {
|
||||
deletion_timestamp: now,
|
||||
deletion_event_id,
|
||||
expiry_timestamp: now + 3600,
|
||||
archive_path: None,
|
||||
};
|
||||
|
||||
db.store_event(&event, metadata.clone()).await.unwrap();
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
///
|
||||
/// This module contains database-related functionality including:
|
||||
/// - Holding database for archived events (NIP-09 deletion support)
|
||||
/// - Cleanup functionality for expired archived data
|
||||
pub mod cleanup;
|
||||
pub mod holding;
|
||||
|
||||
pub use cleanup::{cleanup_archived_data, CleanupResult};
|
||||
pub use holding::{DeletionMetadata, HoldingDatabase, SharedHoldingDatabase};
|
||||
|
||||
@@ -0,0 +1,472 @@
|
||||
//! Git Repository Archival
|
||||
//!
|
||||
//! This module provides functionality for archiving git repositories before deletion.
|
||||
//! Archives are created as tar.gz files with associated metadata for retention management.
|
||||
|
||||
use flate2::write::GzEncoder;
|
||||
use flate2::Compression;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::fs::{self, File};
|
||||
use std::io::{self, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use tar::Builder;
|
||||
use tracing::{debug, info};
|
||||
|
||||
/// Metadata for an archived repository
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
|
||||
pub struct ArchiveMetadata {
|
||||
/// Unix timestamp when the deletion was requested
|
||||
pub deletion_timestamp: u64,
|
||||
/// Event ID that triggered the deletion
|
||||
pub deletion_event_id: String,
|
||||
/// Maintainer's public key in npub format
|
||||
pub maintainer_pubkey: String,
|
||||
/// Repository identifier
|
||||
pub identifier: String,
|
||||
/// Relative path to the archive tar.gz file
|
||||
pub archive_file_path: String,
|
||||
/// Unix timestamp when the archive expires (deletion_timestamp + retention_secs)
|
||||
pub expiry_timestamp: u64,
|
||||
}
|
||||
|
||||
/// Error type for archive operations
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum ArchiveError {
|
||||
#[error("IO error: {0}")]
|
||||
Io(#[from] io::Error),
|
||||
|
||||
#[error("JSON serialization error: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
|
||||
#[error("Repository directory not found: {0}")]
|
||||
RepoNotFound(String),
|
||||
|
||||
#[error("Archive directory creation failed: {0}")]
|
||||
ArchiveDirCreation(String),
|
||||
}
|
||||
|
||||
/// Archive a git repository to a tar.gz file
|
||||
///
|
||||
/// Creates a compressed archive of the git repository directory and stores it
|
||||
/// in the `.archive/<npub>/<identifier>-<timestamp>.tar.gz` path.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `repo_path` - Path to the git repository directory to archive
|
||||
/// * `archive_base_path` - Base path for archives (typically `.archive`)
|
||||
/// * `npub` - Maintainer's public key in npub format
|
||||
/// * `identifier` - Repository identifier
|
||||
/// * `timestamp` - Unix timestamp for the archive filename
|
||||
///
|
||||
/// # Returns
|
||||
/// Path to the created archive file
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns `ArchiveError` if:
|
||||
/// - Repository directory doesn't exist
|
||||
/// - Archive directory cannot be created
|
||||
/// - Archive file cannot be written
|
||||
/// - Compression fails
|
||||
pub fn archive_repository(
|
||||
repo_path: &Path,
|
||||
archive_base_path: &Path,
|
||||
npub: &str,
|
||||
identifier: &str,
|
||||
timestamp: u64,
|
||||
) -> Result<PathBuf, ArchiveError> {
|
||||
// Validate repository exists
|
||||
if !repo_path.exists() {
|
||||
return Err(ArchiveError::RepoNotFound(repo_path.display().to_string()));
|
||||
}
|
||||
|
||||
debug!(
|
||||
"Archiving repository {} to archive base {}",
|
||||
repo_path.display(),
|
||||
archive_base_path.display()
|
||||
);
|
||||
|
||||
// Create archive directory structure: .archive/<npub>/
|
||||
let archive_dir = archive_base_path.join(npub);
|
||||
fs::create_dir_all(&archive_dir).map_err(|e| {
|
||||
ArchiveError::ArchiveDirCreation(format!(
|
||||
"Failed to create {}: {}",
|
||||
archive_dir.display(),
|
||||
e
|
||||
))
|
||||
})?;
|
||||
|
||||
// Create archive filename: <identifier>-<timestamp>.tar.gz
|
||||
let archive_filename = format!("{}-{}.tar.gz", identifier, timestamp);
|
||||
let archive_path = archive_dir.join(&archive_filename);
|
||||
|
||||
debug!("Creating archive at {}", archive_path.display());
|
||||
|
||||
// Create tar.gz archive
|
||||
let tar_file = File::create(&archive_path)?;
|
||||
let encoder = GzEncoder::new(tar_file, Compression::default());
|
||||
let mut tar_builder = Builder::new(encoder);
|
||||
|
||||
// Add the entire repository directory to the archive
|
||||
// Use the repository name as the base directory in the archive
|
||||
let repo_name = repo_path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("repository");
|
||||
|
||||
tar_builder.append_dir_all(repo_name, repo_path)?;
|
||||
|
||||
// Finish writing the archive
|
||||
let encoder = tar_builder.into_inner()?;
|
||||
encoder.finish()?;
|
||||
|
||||
info!(
|
||||
"Successfully archived repository {} to {}",
|
||||
repo_path.display(),
|
||||
archive_path.display()
|
||||
);
|
||||
|
||||
Ok(archive_path)
|
||||
}
|
||||
|
||||
/// Create metadata for an archived repository
|
||||
///
|
||||
/// Generates metadata JSON file alongside the archive with information needed
|
||||
/// for retention management and potential restoration.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `archive_path` - Path to the archive tar.gz file
|
||||
/// * `deletion_timestamp` - Unix timestamp when deletion was requested
|
||||
/// * `deletion_event_id` - Event ID that triggered the deletion
|
||||
/// * `maintainer_pubkey` - Maintainer's public key in npub format
|
||||
/// * `identifier` - Repository identifier
|
||||
/// * `retention_secs` - Retention period in seconds
|
||||
///
|
||||
/// # Returns
|
||||
/// Path to the created metadata JSON file
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns `ArchiveError` if:
|
||||
/// - Metadata file cannot be written
|
||||
/// - JSON serialization fails
|
||||
pub fn create_archive_metadata(
|
||||
archive_path: &Path,
|
||||
deletion_timestamp: u64,
|
||||
deletion_event_id: String,
|
||||
maintainer_pubkey: String,
|
||||
identifier: String,
|
||||
retention_secs: u64,
|
||||
) -> Result<PathBuf, ArchiveError> {
|
||||
let expiry_timestamp = deletion_timestamp + retention_secs;
|
||||
|
||||
// Get relative path for archive_file_path
|
||||
// If archive_path is /data/.archive/npub/repo-123.tar.gz
|
||||
// We want to store .archive/npub/repo-123.tar.gz
|
||||
let archive_file_path = archive_path
|
||||
.components()
|
||||
.skip_while(|c| {
|
||||
// Skip components until we find .archive
|
||||
!c.as_os_str().to_string_lossy().contains(".archive")
|
||||
})
|
||||
.collect::<PathBuf>()
|
||||
.display()
|
||||
.to_string();
|
||||
|
||||
let metadata = ArchiveMetadata {
|
||||
deletion_timestamp,
|
||||
deletion_event_id,
|
||||
maintainer_pubkey,
|
||||
identifier,
|
||||
archive_file_path,
|
||||
expiry_timestamp,
|
||||
};
|
||||
|
||||
// Create metadata file path: <archive>.json
|
||||
let metadata_path = archive_path.with_extension("tar.gz.json");
|
||||
|
||||
debug!("Creating metadata at {}", metadata_path.display());
|
||||
|
||||
// Write metadata as JSON
|
||||
let json = serde_json::to_string_pretty(&metadata)?;
|
||||
let mut file = File::create(&metadata_path)?;
|
||||
file.write_all(json.as_bytes())?;
|
||||
|
||||
info!(
|
||||
"Successfully created metadata at {}",
|
||||
metadata_path.display()
|
||||
);
|
||||
|
||||
Ok(metadata_path)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use flate2::read::GzDecoder;
|
||||
use std::fs;
|
||||
use std::process::Command;
|
||||
use tar::Archive;
|
||||
use tempfile::TempDir;
|
||||
|
||||
/// Create a test git repository with a commit
|
||||
fn create_test_git_repo() -> (TempDir, PathBuf) {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let repo_path = temp_dir.path().join("test-repo.git");
|
||||
|
||||
// Initialize bare repository
|
||||
Command::new("git")
|
||||
.args(["init", "--bare", repo_path.to_str().unwrap()])
|
||||
.output()
|
||||
.unwrap();
|
||||
|
||||
// Create a working directory to make a commit
|
||||
let work_dir = temp_dir.path().join("work");
|
||||
Command::new("git")
|
||||
.args([
|
||||
"clone",
|
||||
repo_path.to_str().unwrap(),
|
||||
work_dir.to_str().unwrap(),
|
||||
])
|
||||
.output()
|
||||
.unwrap();
|
||||
|
||||
// Configure git
|
||||
Command::new("git")
|
||||
.args(["config", "user.email", "test@test.com"])
|
||||
.current_dir(&work_dir)
|
||||
.output()
|
||||
.unwrap();
|
||||
Command::new("git")
|
||||
.args(["config", "user.name", "Test User"])
|
||||
.current_dir(&work_dir)
|
||||
.output()
|
||||
.unwrap();
|
||||
Command::new("git")
|
||||
.args(["config", "commit.gpgsign", "false"])
|
||||
.current_dir(&work_dir)
|
||||
.output()
|
||||
.unwrap();
|
||||
|
||||
// Create a file and commit
|
||||
fs::write(work_dir.join("README.md"), "# Test Repository").unwrap();
|
||||
Command::new("git")
|
||||
.args(["add", "README.md"])
|
||||
.current_dir(&work_dir)
|
||||
.output()
|
||||
.unwrap();
|
||||
Command::new("git")
|
||||
.args(["commit", "-m", "Initial commit"])
|
||||
.current_dir(&work_dir)
|
||||
.output()
|
||||
.unwrap();
|
||||
|
||||
// Push to bare repo
|
||||
Command::new("git")
|
||||
.args(["push", "origin", "master"])
|
||||
.current_dir(&work_dir)
|
||||
.output()
|
||||
.unwrap();
|
||||
|
||||
(temp_dir, repo_path)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_archive_repository_creates_tarball() {
|
||||
let (_temp_dir, repo_path) = create_test_git_repo();
|
||||
let archive_base = _temp_dir.path().join(".archive");
|
||||
let npub = "npub1test123";
|
||||
let identifier = "test-repo";
|
||||
let timestamp = 1234567890;
|
||||
|
||||
let archive_path =
|
||||
archive_repository(&repo_path, &archive_base, npub, identifier, timestamp).unwrap();
|
||||
|
||||
// Verify archive was created
|
||||
assert!(archive_path.exists());
|
||||
assert_eq!(
|
||||
archive_path,
|
||||
archive_base
|
||||
.join(npub)
|
||||
.join(format!("{}-{}.tar.gz", identifier, timestamp))
|
||||
);
|
||||
|
||||
// Verify it's a valid gzip file
|
||||
let file = File::open(&archive_path).unwrap();
|
||||
let decoder = GzDecoder::new(file);
|
||||
let mut archive = Archive::new(decoder);
|
||||
|
||||
// Should be able to list entries
|
||||
let entries: Vec<_> = archive.entries().unwrap().collect();
|
||||
assert!(!entries.is_empty(), "Archive should contain entries");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_archive_repository_compression_works() {
|
||||
let (_temp_dir, repo_path) = create_test_git_repo();
|
||||
let archive_base = _temp_dir.path().join(".archive");
|
||||
let npub = "npub1test123";
|
||||
let identifier = "test-repo";
|
||||
let timestamp = 1234567890;
|
||||
|
||||
let archive_path =
|
||||
archive_repository(&repo_path, &archive_base, npub, identifier, timestamp).unwrap();
|
||||
|
||||
// Verify compression by checking file is smaller than uncompressed
|
||||
let archive_size = fs::metadata(&archive_path).unwrap().len();
|
||||
|
||||
// Get size of original repo (rough estimate)
|
||||
let repo_size: u64 = walkdir::WalkDir::new(&repo_path)
|
||||
.into_iter()
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| e.file_type().is_file())
|
||||
.filter_map(|e| fs::metadata(e.path()).ok())
|
||||
.map(|m| m.len())
|
||||
.sum();
|
||||
|
||||
// Archive should be smaller due to compression
|
||||
// (This is a rough check - actual compression ratio varies)
|
||||
assert!(archive_size > 0, "Archive should have non-zero size");
|
||||
assert!(repo_size > 0, "Repo should have non-zero size");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_archive_repository_nonexistent_repo() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let nonexistent_repo = temp_dir.path().join("nonexistent.git");
|
||||
let archive_base = temp_dir.path().join(".archive");
|
||||
|
||||
let result = archive_repository(&nonexistent_repo, &archive_base, "npub1test", "test", 123);
|
||||
|
||||
assert!(result.is_err());
|
||||
match result {
|
||||
Err(ArchiveError::RepoNotFound(_)) => (),
|
||||
_ => panic!("Expected RepoNotFound error"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_archive_extraction_integrity() {
|
||||
let (_temp_dir, repo_path) = create_test_git_repo();
|
||||
let archive_base = _temp_dir.path().join(".archive");
|
||||
let npub = "npub1test123";
|
||||
let identifier = "test-repo";
|
||||
let timestamp = 1234567890;
|
||||
|
||||
let archive_path =
|
||||
archive_repository(&repo_path, &archive_base, npub, identifier, timestamp).unwrap();
|
||||
|
||||
// Extract archive to verify integrity
|
||||
let extract_dir = _temp_dir.path().join("extracted");
|
||||
fs::create_dir_all(&extract_dir).unwrap();
|
||||
|
||||
let file = File::open(&archive_path).unwrap();
|
||||
let decoder = GzDecoder::new(file);
|
||||
let mut archive = Archive::new(decoder);
|
||||
archive.unpack(&extract_dir).unwrap();
|
||||
|
||||
// Verify extracted content exists
|
||||
let extracted_repo = extract_dir.join("test-repo.git");
|
||||
assert!(extracted_repo.exists(), "Extracted repository should exist");
|
||||
|
||||
// Verify it's a valid git repository
|
||||
let output = Command::new("git")
|
||||
.args(["rev-parse", "--git-dir"])
|
||||
.current_dir(&extracted_repo)
|
||||
.output()
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"Extracted directory should be a valid git repo"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_create_archive_metadata() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let archive_path = temp_dir.path().join(".archive/npub1test/repo-123.tar.gz");
|
||||
fs::create_dir_all(archive_path.parent().unwrap()).unwrap();
|
||||
File::create(&archive_path).unwrap();
|
||||
|
||||
let deletion_timestamp = 1234567890;
|
||||
let deletion_event_id = "event123".to_string();
|
||||
let maintainer_pubkey = "npub1test".to_string();
|
||||
let identifier = "repo".to_string();
|
||||
let retention_secs = 2592000; // 30 days
|
||||
|
||||
let metadata_path = create_archive_metadata(
|
||||
&archive_path,
|
||||
deletion_timestamp,
|
||||
deletion_event_id.clone(),
|
||||
maintainer_pubkey.clone(),
|
||||
identifier.clone(),
|
||||
retention_secs,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Verify metadata file was created
|
||||
assert!(metadata_path.exists());
|
||||
assert_eq!(metadata_path, archive_path.with_extension("tar.gz.json"));
|
||||
|
||||
// Verify metadata content
|
||||
let json = fs::read_to_string(&metadata_path).unwrap();
|
||||
let metadata: ArchiveMetadata = serde_json::from_str(&json).unwrap();
|
||||
|
||||
assert_eq!(metadata.deletion_timestamp, deletion_timestamp);
|
||||
assert_eq!(metadata.deletion_event_id, deletion_event_id);
|
||||
assert_eq!(metadata.maintainer_pubkey, maintainer_pubkey);
|
||||
assert_eq!(metadata.identifier, identifier);
|
||||
assert_eq!(
|
||||
metadata.expiry_timestamp,
|
||||
deletion_timestamp + retention_secs
|
||||
);
|
||||
assert!(metadata.archive_file_path.contains(".archive"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_metadata_serialization_roundtrip() {
|
||||
let metadata = ArchiveMetadata {
|
||||
deletion_timestamp: 1234567890,
|
||||
deletion_event_id: "event123".to_string(),
|
||||
maintainer_pubkey: "npub1test".to_string(),
|
||||
identifier: "test-repo".to_string(),
|
||||
archive_file_path: ".archive/npub1test/test-repo-1234567890.tar.gz".to_string(),
|
||||
expiry_timestamp: 1234567890 + 2592000,
|
||||
};
|
||||
|
||||
// Serialize
|
||||
let json = serde_json::to_string(&metadata).unwrap();
|
||||
|
||||
// Deserialize
|
||||
let deserialized: ArchiveMetadata = serde_json::from_str(&json).unwrap();
|
||||
|
||||
// Verify roundtrip
|
||||
assert_eq!(metadata, deserialized);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_archive_directory_creation() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let (_repo_temp, repo_path) = create_test_git_repo();
|
||||
let archive_base = temp_dir.path().join(".archive");
|
||||
let npub = "npub1test123";
|
||||
|
||||
// Archive base doesn't exist yet
|
||||
assert!(!archive_base.exists());
|
||||
|
||||
archive_repository(&repo_path, &archive_base, npub, "test-repo", 123).unwrap();
|
||||
|
||||
// Verify directory structure was created
|
||||
assert!(archive_base.exists());
|
||||
assert!(archive_base.join(npub).exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_archive_error_handling_permission_denied() {
|
||||
// This test would require setting up permission-denied scenarios
|
||||
// which is platform-specific and may require elevated privileges.
|
||||
// Skipping for now, but documenting the expected behavior:
|
||||
// - If archive directory cannot be created due to permissions,
|
||||
// should return ArchiveError::ArchiveDirCreation
|
||||
// - If archive file cannot be written due to permissions,
|
||||
// should return ArchiveError::Io
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@
|
||||
//! - `POST /<npub>/<identifier>.git/git-upload-pack` - Clone/fetch operation
|
||||
//! - `POST /<npub>/<identifier>.git/git-receive-pack` - Push operation
|
||||
|
||||
pub mod archive;
|
||||
pub mod authorization;
|
||||
pub mod handlers;
|
||||
pub mod process;
|
||||
|
||||
+69
@@ -173,6 +173,75 @@ async fn main() -> Result<()> {
|
||||
});
|
||||
info!("Expired event cleanup task started (24h interval, keeps 7 days)");
|
||||
|
||||
// Spawn background cleanup task for archived deletions
|
||||
if let Some(holding_db) = relay_with_db.holding_database.clone() {
|
||||
let cleanup_git_data_path = config.effective_git_data_path();
|
||||
let cleanup_interval_secs = config.archive_cleanup_interval_secs;
|
||||
tokio::spawn(async move {
|
||||
// Run cleanup immediately on startup (catch up after offline periods)
|
||||
info!("Running initial cleanup of expired archived data");
|
||||
match ngit_grasp::database::cleanup_archived_data(
|
||||
&holding_db,
|
||||
&cleanup_git_data_path,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
if !result.is_empty() {
|
||||
info!(
|
||||
events_deleted = result.events_deleted,
|
||||
archives_deleted = result.archives_deleted,
|
||||
metadata_deleted = result.metadata_deleted,
|
||||
bytes_reclaimed = result.bytes_reclaimed,
|
||||
"Initial cleanup completed"
|
||||
);
|
||||
} else {
|
||||
info!("Initial cleanup: no expired data found");
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "Initial cleanup failed");
|
||||
}
|
||||
}
|
||||
|
||||
// Run cleanup periodically
|
||||
let mut interval =
|
||||
tokio::time::interval(Duration::from_secs(cleanup_interval_secs));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
match ngit_grasp::database::cleanup_archived_data(
|
||||
&holding_db,
|
||||
&cleanup_git_data_path,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
if !result.is_empty() {
|
||||
info!(
|
||||
events_deleted = result.events_deleted,
|
||||
archives_deleted = result.archives_deleted,
|
||||
metadata_deleted = result.metadata_deleted,
|
||||
bytes_reclaimed = result.bytes_reclaimed,
|
||||
"Periodic cleanup completed"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "Periodic cleanup failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
info!(
|
||||
"Archive cleanup task started ({}s interval, {}s retention)",
|
||||
config.archive_cleanup_interval_secs, config.archive_retention_secs
|
||||
);
|
||||
} else {
|
||||
info!(
|
||||
"Archive cleanup task disabled (disrespector mode or holding database unavailable)"
|
||||
);
|
||||
}
|
||||
|
||||
// Start purgatory sync loop for background git data fetching
|
||||
// Create naughty list tracker for git remote domains with persistent errors (12h expiration)
|
||||
let git_naughty_list = Arc::new(NaughtyListTracker::with_defaults());
|
||||
|
||||
+8
-148
@@ -114,146 +114,6 @@ impl Nip34WritePolicy {
|
||||
// Parse announcement to get repository details
|
||||
match RepositoryAnnouncement::from_event(event.clone()) {
|
||||
Ok(announcement) => {
|
||||
// Check for recovery from holding database
|
||||
if let Some(ref holding_db) = self.holding_database {
|
||||
let current_ts = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
|
||||
match crate::git::archive::check_for_recovery(
|
||||
holding_db,
|
||||
&announcement.owner_npub(),
|
||||
&announcement.identifier,
|
||||
current_ts,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(recovery_info)) => {
|
||||
tracing::info!(
|
||||
"Recovery detected for {}/{}: {} events can be restored from archive {}",
|
||||
announcement.owner_npub(),
|
||||
announcement.identifier,
|
||||
recovery_info.events.len(),
|
||||
recovery_info.archive_path.display()
|
||||
);
|
||||
|
||||
// Restore git data from archive
|
||||
let git_data_path = self.ctx.git_data_path.as_path();
|
||||
match crate::git::archive::restore_repository(
|
||||
&recovery_info.archive_path,
|
||||
git_data_path,
|
||||
&announcement.owner_npub(),
|
||||
&announcement.identifier,
|
||||
) {
|
||||
Ok(()) => {
|
||||
tracing::info!(
|
||||
"Successfully restored git data for {}/{}",
|
||||
announcement.owner_npub(),
|
||||
announcement.identifier
|
||||
);
|
||||
|
||||
// Restore events from holding database
|
||||
let event_ids: Vec<_> =
|
||||
recovery_info.events.iter().map(|e| e.id).collect();
|
||||
|
||||
match holding_db
|
||||
.restore_events_to_main(
|
||||
&self.ctx.database,
|
||||
&event_ids,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(result) => {
|
||||
tracing::info!(
|
||||
"Restored {} events from holding database for {}/{}",
|
||||
result.restored_count,
|
||||
announcement.owner_npub(),
|
||||
announcement.identifier
|
||||
);
|
||||
|
||||
if result.failed_count > 0 {
|
||||
tracing::warn!(
|
||||
"Failed to restore {} events for {}/{}",
|
||||
result.failed_count,
|
||||
announcement.owner_npub(),
|
||||
announcement.identifier
|
||||
);
|
||||
}
|
||||
|
||||
// Cleanup archive files after successful recovery
|
||||
match crate::git::archive::cleanup_after_recovery(
|
||||
&recovery_info.archive_path,
|
||||
&announcement.owner_npub(),
|
||||
&announcement.identifier,
|
||||
) {
|
||||
Ok(stats) => {
|
||||
tracing::info!(
|
||||
"Cleaned up archive for {}/{}: {} bytes reclaimed",
|
||||
announcement.owner_npub(),
|
||||
announcement.identifier,
|
||||
stats.bytes_reclaimed
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Failed to cleanup archive for {}/{}: {}",
|
||||
announcement.owner_npub(),
|
||||
announcement.identifier,
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Delete events from holding database
|
||||
let deleted_count =
|
||||
holding_db.delete_events(&event_ids).await;
|
||||
tracing::info!(
|
||||
"Deleted {} events from holding database for {}/{}",
|
||||
deleted_count,
|
||||
announcement.owner_npub(),
|
||||
announcement.identifier
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Failed to restore events for {}/{}: {}",
|
||||
announcement.owner_npub(),
|
||||
announcement.identifier,
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Failed to restore git data for {}/{}: {}",
|
||||
announcement.owner_npub(),
|
||||
announcement.identifier,
|
||||
e
|
||||
);
|
||||
// Continue with normal processing even if restoration fails
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
tracing::debug!(
|
||||
"No recovery available for {}/{} (not in holding database or expired)",
|
||||
announcement.owner_npub(),
|
||||
announcement.identifier
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
"Error checking for recovery for {}/{}: {}",
|
||||
announcement.owner_npub(),
|
||||
announcement.identifier,
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try to create bare repository if it doesn't exist
|
||||
if let Err(e) = self
|
||||
.announcement_policy
|
||||
@@ -691,30 +551,30 @@ impl Nip34WritePolicy {
|
||||
None
|
||||
};
|
||||
|
||||
// Determine which events should be deleted using graph-based algorithm
|
||||
match crate::nostr::policy::determine_events_to_delete(
|
||||
// Query for all dependent events
|
||||
match crate::nostr::policy::query_dependent_events(
|
||||
&self.ctx.database,
|
||||
&event_ids,
|
||||
&addresses,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(events_to_delete) => {
|
||||
Ok(all_events) => {
|
||||
tracing::info!(
|
||||
event_id = %event_id_str,
|
||||
author = %event.pubkey,
|
||||
event_count = event_ids.len(),
|
||||
address_count = addresses.len(),
|
||||
total_events = events_to_delete.len(),
|
||||
"Processing deletion request - moving {} events to holding database (graph-based)",
|
||||
events_to_delete.len()
|
||||
total_events = all_events.len(),
|
||||
"Processing deletion request - moving {} events to holding database",
|
||||
all_events.len()
|
||||
);
|
||||
|
||||
// Move events to holding database
|
||||
match crate::nostr::policy::move_to_holding_database(
|
||||
&self.ctx.database,
|
||||
holding_db,
|
||||
&events_to_delete,
|
||||
&all_events,
|
||||
&event.id,
|
||||
self.ctx.config.archive_retention_secs,
|
||||
archive_path,
|
||||
@@ -725,7 +585,7 @@ impl Nip34WritePolicy {
|
||||
tracing::info!(
|
||||
event_id = %event_id_str,
|
||||
moved_count = moved_count,
|
||||
"Successfully moved {} events to holding database (graph-based)",
|
||||
"Successfully moved {} events to holding database",
|
||||
moved_count
|
||||
);
|
||||
WritePolicyResult::Accept
|
||||
|
||||
@@ -3,11 +3,13 @@
|
||||
/// Handles querying for dependent events and moving events between databases
|
||||
/// for NIP-09 deletion request processing.
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
|
||||
use nostr_relay_builder::prelude::{Event, EventId, Filter, Kind};
|
||||
use nostr_relay_builder::prelude::{Event, EventId, Filter, Kind, PublicKey, ToBech32};
|
||||
|
||||
use super::SharedDatabase;
|
||||
use crate::database::{DeletionMetadata, HoldingDatabase};
|
||||
use crate::git::archive::{archive_repository, create_archive_metadata};
|
||||
|
||||
/// Query all events that depend on the given event IDs
|
||||
///
|
||||
@@ -197,6 +199,7 @@ fn has_event_tag(event: &Event, event_id: &EventId) -> bool {
|
||||
/// * `event_ids` - Event IDs to move
|
||||
/// * `deletion_event_id` - ID of the deletion request event
|
||||
/// * `retention_secs` - Retention period in seconds before permanent deletion
|
||||
/// * `archive_path` - Optional path to git archive (if repository was archived)
|
||||
///
|
||||
/// # Returns
|
||||
/// Number of events successfully moved
|
||||
@@ -206,6 +209,7 @@ pub async fn move_to_holding_database(
|
||||
event_ids: &HashSet<EventId>,
|
||||
deletion_event_id: &EventId,
|
||||
retention_secs: u64,
|
||||
archive_path: Option<String>,
|
||||
) -> Result<usize, String> {
|
||||
let mut moved_count = 0;
|
||||
let deletion_ts = std::time::SystemTime::now()
|
||||
@@ -238,6 +242,7 @@ pub async fn move_to_holding_database(
|
||||
deletion_timestamp: deletion_ts,
|
||||
deletion_event_id: *deletion_event_id,
|
||||
expiry_timestamp: deletion_ts + retention_secs,
|
||||
archive_path: archive_path.clone(),
|
||||
};
|
||||
|
||||
// Store in holding database with deletion metadata
|
||||
@@ -261,6 +266,119 @@ pub async fn move_to_holding_database(
|
||||
Ok(moved_count)
|
||||
}
|
||||
|
||||
/// Archive git repositories for address deletions
|
||||
///
|
||||
/// When deleting repository announcements (kind 30617), this function archives
|
||||
/// the git repository data before deletion.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `addresses` - Repository addresses to archive (format: `30617:<pubkey>:<identifier>`)
|
||||
/// * `git_data_path` - Base path for git repositories
|
||||
/// * `archive_base_path` - Base path for archives (typically `.archive`)
|
||||
/// * `deletion_event_id` - ID of the deletion request event
|
||||
/// * `retention_secs` - Retention period in seconds
|
||||
///
|
||||
/// # Returns
|
||||
/// Optional path to the archive file (if any repositories were archived)
|
||||
pub fn archive_repositories_for_addresses(
|
||||
addresses: &[String],
|
||||
git_data_path: &Path,
|
||||
archive_base_path: &Path,
|
||||
deletion_event_id: &EventId,
|
||||
retention_secs: u64,
|
||||
) -> Result<Option<String>, String> {
|
||||
let deletion_ts = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs();
|
||||
|
||||
// Only archive kind 30617 repository announcements
|
||||
for address in addresses {
|
||||
let parts: Vec<&str> = address.split(':').collect();
|
||||
if parts.len() != 3 {
|
||||
continue; // Skip invalid addresses
|
||||
}
|
||||
|
||||
let kind = parts[0];
|
||||
if kind != "30617" {
|
||||
continue; // Only archive repository announcements
|
||||
}
|
||||
|
||||
let pubkey_hex = parts[1];
|
||||
let identifier = parts[2];
|
||||
|
||||
// Parse pubkey to get npub
|
||||
let pubkey = PublicKey::from_hex(pubkey_hex)
|
||||
.map_err(|e| format!("Invalid pubkey in address: {}", e))?;
|
||||
let npub = pubkey
|
||||
.to_bech32()
|
||||
.map_err(|e| format!("Failed to convert pubkey to npub: {}", e))?;
|
||||
|
||||
// Build repository path: <git_data_path>/<npub>/<identifier>.git
|
||||
let repo_path = git_data_path
|
||||
.join(&npub)
|
||||
.join(format!("{}.git", identifier));
|
||||
|
||||
if !repo_path.exists() {
|
||||
tracing::warn!(
|
||||
npub = %npub,
|
||||
identifier = %identifier,
|
||||
path = %repo_path.display(),
|
||||
"Repository directory not found, skipping archive"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Archive the repository
|
||||
match archive_repository(
|
||||
&repo_path,
|
||||
archive_base_path,
|
||||
&npub,
|
||||
identifier,
|
||||
deletion_ts,
|
||||
) {
|
||||
Ok(archive_path) => {
|
||||
tracing::info!(
|
||||
npub = %npub,
|
||||
identifier = %identifier,
|
||||
archive_path = %archive_path.display(),
|
||||
"Successfully archived git repository"
|
||||
);
|
||||
|
||||
// Create archive metadata
|
||||
if let Err(e) = create_archive_metadata(
|
||||
&archive_path,
|
||||
deletion_ts,
|
||||
deletion_event_id.to_hex(),
|
||||
npub.clone(),
|
||||
identifier.to_string(),
|
||||
retention_secs,
|
||||
) {
|
||||
tracing::error!(
|
||||
error = %e,
|
||||
archive_path = %archive_path.display(),
|
||||
"Failed to create archive metadata"
|
||||
);
|
||||
}
|
||||
|
||||
// Return the archive path (relative to working directory)
|
||||
return Ok(Some(archive_path.display().to_string()));
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
error = %e,
|
||||
npub = %npub,
|
||||
identifier = %identifier,
|
||||
"Failed to archive git repository"
|
||||
);
|
||||
return Err(format!("Failed to archive repository: {}", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(None) // No repositories archived
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -16,7 +16,9 @@ mod state;
|
||||
|
||||
pub use announcement::{AnnouncementPolicy, AnnouncementResult};
|
||||
pub use deletion::{DeletionPolicy, DeletionResult};
|
||||
pub use deletion_ops::{move_to_holding_database, query_dependent_events};
|
||||
pub use deletion_ops::{
|
||||
archive_repositories_for_addresses, move_to_holding_database, query_dependent_events,
|
||||
};
|
||||
pub use pr_event::PrEventPolicy;
|
||||
pub use related::{ReferenceResult, RelatedEventPolicy};
|
||||
pub use state::{StatePolicy, StateResult};
|
||||
|
||||
+53
-2
@@ -118,7 +118,48 @@ impl TestRelay {
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn start_with_retention(retention_secs: u64) -> Self {
|
||||
Self::start_with_extended_options(Self::find_free_port(), None, false, Some(retention_secs))
|
||||
Self::start_with_extended_options(
|
||||
Self::find_free_port(),
|
||||
None,
|
||||
false,
|
||||
Some(retention_secs),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Start relay with custom retention and cleanup interval
|
||||
///
|
||||
/// This is useful for testing deletion and archival with short retention periods
|
||||
/// and fast cleanup cycles.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `retention_secs` - Archive retention period in seconds
|
||||
/// * `cleanup_interval_secs` - How often to run cleanup (in seconds)
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```no_run
|
||||
/// use common::TestRelay;
|
||||
///
|
||||
/// #[tokio::test]
|
||||
/// async fn test_deletion_with_fast_cleanup() {
|
||||
/// let relay = TestRelay::start_with_retention_and_cleanup(5, 2).await;
|
||||
/// // ... test deletion with 5-second retention and 2-second cleanup interval ...
|
||||
/// relay.stop().await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn start_with_retention_and_cleanup(
|
||||
retention_secs: u64,
|
||||
cleanup_interval_secs: u64,
|
||||
) -> Self {
|
||||
Self::start_with_extended_options(
|
||||
Self::find_free_port(),
|
||||
None,
|
||||
false,
|
||||
Some(retention_secs),
|
||||
Some(cleanup_interval_secs),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -133,7 +174,8 @@ impl TestRelay {
|
||||
bootstrap_relay_url: Option<String>,
|
||||
disable_negentropy: bool,
|
||||
) -> Self {
|
||||
Self::start_with_extended_options(port, bootstrap_relay_url, disable_negentropy, None).await
|
||||
Self::start_with_extended_options(port, bootstrap_relay_url, disable_negentropy, None, None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Start relay with extended options (internal)
|
||||
@@ -142,6 +184,7 @@ impl TestRelay {
|
||||
bootstrap_relay_url: Option<String>,
|
||||
disable_negentropy: bool,
|
||||
retention_secs: Option<u64>,
|
||||
cleanup_interval_secs: Option<u64>,
|
||||
) -> Self {
|
||||
let bind_address = format!("127.0.0.1:{}", port);
|
||||
let url = format!("ws://127.0.0.1:{}", port);
|
||||
@@ -200,6 +243,14 @@ impl TestRelay {
|
||||
cmd.env("NGIT_ARCHIVE_RETENTION_SECS", retention.to_string());
|
||||
}
|
||||
|
||||
// Add custom cleanup interval if provided
|
||||
if let Some(cleanup_interval) = cleanup_interval_secs {
|
||||
cmd.env(
|
||||
"NGIT_ARCHIVE_CLEANUP_INTERVAL_SECS",
|
||||
cleanup_interval.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
let process = cmd.spawn().expect("Failed to start relay process");
|
||||
|
||||
let relay = Self { process, url, port };
|
||||
|
||||
@@ -0,0 +1,803 @@
|
||||
//! NIP-09 Git Archival and Cleanup Integration Tests
|
||||
//!
|
||||
//! Tests end-to-end archival and cleanup functionality for NIP-09 deletion requests.
|
||||
//!
|
||||
//! # Test Coverage
|
||||
//!
|
||||
//! - Archive creation from git repositories
|
||||
//! - Archive extraction and integrity verification
|
||||
//! - Background cleanup of expired archives
|
||||
//! - Startup cleanup of pre-existing expired data
|
||||
//! - Cleanup with missing archive files
|
||||
//! - Disk space reclamation tracking
|
||||
//!
|
||||
//! # Running Tests
|
||||
//!
|
||||
//! ```bash
|
||||
//! # Run all archival tests
|
||||
//! cargo test --test nip09_archival
|
||||
//!
|
||||
//! # Run specific test
|
||||
//! cargo test --test nip09_archival test_archive_creation
|
||||
//!
|
||||
//! # With output
|
||||
//! cargo test --test nip09_archival -- --nocapture
|
||||
//! ```
|
||||
|
||||
mod common;
|
||||
|
||||
use flate2::read::GzDecoder;
|
||||
use ngit_grasp::config::DatabaseBackend;
|
||||
use ngit_grasp::database::{DeletionMetadata, HoldingDatabase};
|
||||
use nostr_sdk::prelude::*;
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tar::Archive;
|
||||
use tempfile::TempDir;
|
||||
|
||||
/// Helper: Get current Unix timestamp
|
||||
fn current_timestamp() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs()
|
||||
}
|
||||
|
||||
/// Helper: Create a test git repository with actual commits
|
||||
fn create_test_git_repo(base_dir: &Path, repo_name: &str) -> PathBuf {
|
||||
let repo_path = base_dir.join(format!("{}.git", repo_name));
|
||||
|
||||
// Initialize bare repository
|
||||
Command::new("git")
|
||||
.args(["init", "--bare", repo_path.to_str().unwrap()])
|
||||
.output()
|
||||
.expect("Failed to init bare repo");
|
||||
|
||||
// Create a working directory to make commits
|
||||
let work_dir = base_dir.join(format!("{}-work", repo_name));
|
||||
Command::new("git")
|
||||
.args([
|
||||
"clone",
|
||||
repo_path.to_str().unwrap(),
|
||||
work_dir.to_str().unwrap(),
|
||||
])
|
||||
.output()
|
||||
.expect("Failed to clone repo");
|
||||
|
||||
// Configure git
|
||||
Command::new("git")
|
||||
.args(["config", "user.email", "test@test.com"])
|
||||
.current_dir(&work_dir)
|
||||
.output()
|
||||
.expect("Failed to configure git email");
|
||||
Command::new("git")
|
||||
.args(["config", "user.name", "Test User"])
|
||||
.current_dir(&work_dir)
|
||||
.output()
|
||||
.expect("Failed to configure git name");
|
||||
Command::new("git")
|
||||
.args(["config", "commit.gpgsign", "false"])
|
||||
.current_dir(&work_dir)
|
||||
.output()
|
||||
.expect("Failed to disable gpg signing");
|
||||
|
||||
// Create multiple files for better testing
|
||||
fs::write(
|
||||
work_dir.join("README.md"),
|
||||
"# Test Repository\n\nThis is a test repository for archival testing.",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Create src directory first
|
||||
fs::create_dir_all(work_dir.join("src")).unwrap();
|
||||
fs::write(
|
||||
work_dir.join("src/main.rs"),
|
||||
"fn main() {\n println!(\"Hello, world!\");\n}\n",
|
||||
)
|
||||
.unwrap();
|
||||
fs::write(work_dir.join(".gitignore"), "target/\n*.swp\n").unwrap();
|
||||
|
||||
// Add and commit
|
||||
Command::new("git")
|
||||
.args(["add", "."])
|
||||
.current_dir(&work_dir)
|
||||
.output()
|
||||
.expect("Failed to git add");
|
||||
Command::new("git")
|
||||
.args(["commit", "-m", "Initial commit with multiple files"])
|
||||
.current_dir(&work_dir)
|
||||
.output()
|
||||
.expect("Failed to git commit");
|
||||
|
||||
// Push to bare repo
|
||||
Command::new("git")
|
||||
.args(["push", "origin", "master"])
|
||||
.current_dir(&work_dir)
|
||||
.output()
|
||||
.expect("Failed to git push");
|
||||
|
||||
repo_path
|
||||
}
|
||||
|
||||
/// Helper: Get directory size in bytes (recursive)
|
||||
fn get_dir_size(path: &Path) -> u64 {
|
||||
walkdir::WalkDir::new(path)
|
||||
.into_iter()
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| e.file_type().is_file())
|
||||
.filter_map(|e| fs::metadata(e.path()).ok())
|
||||
.map(|m| m.len())
|
||||
.sum()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_archive_creation() {
|
||||
use ngit_grasp::git::archive::{archive_repository, create_archive_metadata};
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let repo_path = create_test_git_repo(temp_dir.path(), "test-repo");
|
||||
let archive_base = temp_dir.path().join(".archive");
|
||||
|
||||
let npub = "npub1test123456789";
|
||||
let identifier = "test-repo";
|
||||
let timestamp = current_timestamp();
|
||||
|
||||
// Create archive
|
||||
let archive_path = archive_repository(&repo_path, &archive_base, npub, identifier, timestamp)
|
||||
.expect("Failed to create archive");
|
||||
|
||||
// Verify archive file exists at correct path
|
||||
assert!(archive_path.exists(), "Archive file should exist");
|
||||
assert_eq!(
|
||||
archive_path,
|
||||
archive_base
|
||||
.join(npub)
|
||||
.join(format!("{}-{}.tar.gz", identifier, timestamp)),
|
||||
"Archive path should match expected format"
|
||||
);
|
||||
|
||||
// Create metadata
|
||||
let deletion_event_id = "event123".to_string();
|
||||
let retention_secs = 2592000; // 30 days
|
||||
let metadata_path = create_archive_metadata(
|
||||
&archive_path,
|
||||
timestamp,
|
||||
deletion_event_id.clone(),
|
||||
npub.to_string(),
|
||||
identifier.to_string(),
|
||||
retention_secs,
|
||||
)
|
||||
.expect("Failed to create metadata");
|
||||
|
||||
// Verify metadata file exists
|
||||
assert!(metadata_path.exists(), "Metadata file should exist");
|
||||
assert_eq!(
|
||||
metadata_path,
|
||||
archive_path.with_extension("tar.gz.json"),
|
||||
"Metadata path should have .tar.gz.json extension"
|
||||
);
|
||||
|
||||
// Verify metadata content
|
||||
let metadata_json = fs::read_to_string(&metadata_path).unwrap();
|
||||
let metadata: serde_json::Value = serde_json::from_str(&metadata_json).unwrap();
|
||||
|
||||
assert_eq!(metadata["deletion_timestamp"], timestamp);
|
||||
assert_eq!(metadata["deletion_event_id"], deletion_event_id);
|
||||
assert_eq!(metadata["maintainer_pubkey"], npub);
|
||||
assert_eq!(metadata["identifier"], identifier);
|
||||
assert_eq!(metadata["expiry_timestamp"], timestamp + retention_secs);
|
||||
assert!(metadata["archive_file_path"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.contains(".archive"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_archive_extraction() {
|
||||
use ngit_grasp::git::archive::archive_repository;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let repo_path = create_test_git_repo(temp_dir.path(), "extract-test");
|
||||
let archive_base = temp_dir.path().join(".archive");
|
||||
|
||||
let npub = "npub1extract";
|
||||
let identifier = "extract-test";
|
||||
let timestamp = current_timestamp();
|
||||
|
||||
// Create archive
|
||||
let archive_path = archive_repository(&repo_path, &archive_base, npub, identifier, timestamp)
|
||||
.expect("Failed to create archive");
|
||||
|
||||
// Extract archive to verify integrity
|
||||
let extract_dir = temp_dir.path().join("extracted");
|
||||
fs::create_dir_all(&extract_dir).unwrap();
|
||||
|
||||
let file = fs::File::open(&archive_path).unwrap();
|
||||
let decoder = GzDecoder::new(file);
|
||||
let mut archive = Archive::new(decoder);
|
||||
archive.unpack(&extract_dir).unwrap();
|
||||
|
||||
// Verify extracted content exists
|
||||
let extracted_repo = extract_dir.join("extract-test.git");
|
||||
assert!(extracted_repo.exists(), "Extracted repository should exist");
|
||||
|
||||
// Verify it's a valid git repository
|
||||
let output = Command::new("git")
|
||||
.args(["rev-parse", "--git-dir"])
|
||||
.current_dir(&extracted_repo)
|
||||
.output()
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"Extracted directory should be a valid git repo"
|
||||
);
|
||||
|
||||
// Verify all files are present
|
||||
assert!(
|
||||
extracted_repo.join("HEAD").exists(),
|
||||
"HEAD file should exist"
|
||||
);
|
||||
assert!(
|
||||
extracted_repo.join("refs").exists(),
|
||||
"refs directory should exist"
|
||||
);
|
||||
assert!(
|
||||
extracted_repo.join("objects").exists(),
|
||||
"objects directory should exist"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_background_cleanup_with_short_retention() {
|
||||
use ngit_grasp::config::DatabaseBackend;
|
||||
use ngit_grasp::database::{DeletionMetadata, HoldingDatabase};
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let git_data_path = temp_dir.path().join("git-data");
|
||||
fs::create_dir_all(&git_data_path).unwrap();
|
||||
|
||||
// Create holding database
|
||||
let holding_db = HoldingDatabase::new(&git_data_path, DatabaseBackend::Memory)
|
||||
.await
|
||||
.expect("Failed to create holding database");
|
||||
|
||||
// Create archive directory and files
|
||||
let archive_dir = git_data_path.join(".archive/npub1cleanup");
|
||||
fs::create_dir_all(&archive_dir).unwrap();
|
||||
|
||||
let archive_path = archive_dir.join("cleanup-test-123456.tar.gz");
|
||||
let metadata_path = archive_dir.join("cleanup-test-123456.tar.gz.json");
|
||||
|
||||
fs::write(&archive_path, b"fake archive data for cleanup test").unwrap();
|
||||
fs::write(&metadata_path, b"{}").unwrap();
|
||||
|
||||
// Verify files exist before cleanup
|
||||
assert!(
|
||||
archive_path.exists(),
|
||||
"Archive file should exist before cleanup"
|
||||
);
|
||||
assert!(
|
||||
metadata_path.exists(),
|
||||
"Metadata file should exist before cleanup"
|
||||
);
|
||||
|
||||
// Create test event with expired timestamp
|
||||
let keys = Keys::generate();
|
||||
let deletion_keys = Keys::generate();
|
||||
let deletion_event_id = EventBuilder::text_note("deletion request")
|
||||
.sign_with_keys(&deletion_keys)
|
||||
.unwrap()
|
||||
.id;
|
||||
|
||||
let now = current_timestamp();
|
||||
let event = EventBuilder::text_note("event to be deleted")
|
||||
.sign_with_keys(&keys)
|
||||
.unwrap();
|
||||
|
||||
// Store event with expired metadata (expired 1 hour ago)
|
||||
let metadata = DeletionMetadata {
|
||||
deletion_timestamp: now - 7200,
|
||||
deletion_event_id,
|
||||
expiry_timestamp: now - 3600, // Expired 1 hour ago
|
||||
archive_path: Some(".archive/npub1cleanup/cleanup-test-123456.tar.gz".to_string()),
|
||||
};
|
||||
|
||||
holding_db.store_event(&event, metadata).await.unwrap();
|
||||
|
||||
// Verify event was stored and is expired
|
||||
let expired = holding_db.query_expired(now).await.unwrap();
|
||||
assert_eq!(expired.len(), 1, "Should have 1 expired event");
|
||||
|
||||
// Run cleanup
|
||||
let result = ngit_grasp::database::cleanup_archived_data(&holding_db, &git_data_path)
|
||||
.await
|
||||
.expect("Cleanup should succeed");
|
||||
|
||||
// Verify cleanup results
|
||||
// Note: events_deleted will be 0 until delete_event is fully implemented
|
||||
// See src/database/holding.rs:211-229 for TODO
|
||||
assert_eq!(result.archives_deleted, 1, "Should delete 1 archive");
|
||||
assert_eq!(result.metadata_deleted, 1, "Should delete 1 metadata file");
|
||||
assert!(result.bytes_reclaimed > 0, "Should reclaim disk space");
|
||||
|
||||
// Verify files were deleted
|
||||
assert!(!archive_path.exists(), "Archive file should be deleted");
|
||||
assert!(!metadata_path.exists(), "Metadata file should be deleted");
|
||||
|
||||
// Note: Event deletion from holding database is not yet implemented
|
||||
// When implemented, we should verify: expired_after.len() == 0
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_startup_cleanup() {
|
||||
// This test verifies that cleanup runs on relay startup
|
||||
// We'll create expired data, then start a relay and verify it gets cleaned up
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let git_data_path = temp_dir.path().join("git-data");
|
||||
fs::create_dir_all(&git_data_path).unwrap();
|
||||
|
||||
// Create holding database with expired events BEFORE starting relay
|
||||
let holding_db = HoldingDatabase::new(&git_data_path, DatabaseBackend::Memory)
|
||||
.await
|
||||
.expect("Failed to create holding database");
|
||||
|
||||
// Create archive files
|
||||
let archive_dir = git_data_path.join(".archive/npub1startup");
|
||||
fs::create_dir_all(&archive_dir).unwrap();
|
||||
|
||||
let archive_path = archive_dir.join("startup-test-123456.tar.gz");
|
||||
let metadata_path = archive_dir.join("startup-test-123456.tar.gz.json");
|
||||
|
||||
fs::write(&archive_path, b"fake archive data for startup cleanup test").unwrap();
|
||||
fs::write(&metadata_path, b"{}").unwrap();
|
||||
|
||||
// Create expired event
|
||||
let keys = Keys::generate();
|
||||
let deletion_keys = Keys::generate();
|
||||
let deletion_event_id = EventBuilder::text_note("deletion request")
|
||||
.sign_with_keys(&deletion_keys)
|
||||
.unwrap()
|
||||
.id;
|
||||
|
||||
let now = current_timestamp();
|
||||
let event = EventBuilder::text_note("event to be deleted on startup")
|
||||
.sign_with_keys(&keys)
|
||||
.unwrap();
|
||||
|
||||
let metadata = DeletionMetadata {
|
||||
deletion_timestamp: now - 7200,
|
||||
deletion_event_id,
|
||||
expiry_timestamp: now - 3600, // Expired 1 hour ago
|
||||
archive_path: Some(".archive/npub1startup/startup-test-123456.tar.gz".to_string()),
|
||||
};
|
||||
|
||||
holding_db.store_event(&event, metadata).await.unwrap();
|
||||
|
||||
// Verify files exist before relay starts
|
||||
assert!(
|
||||
archive_path.exists(),
|
||||
"Archive file should exist before relay starts"
|
||||
);
|
||||
assert!(
|
||||
metadata_path.exists(),
|
||||
"Metadata file should exist before relay starts"
|
||||
);
|
||||
|
||||
// Note: In a real integration test with TestRelay, we would:
|
||||
// 1. Start the relay (which runs cleanup on startup)
|
||||
// 2. Wait for cleanup to complete
|
||||
// 3. Verify files are deleted
|
||||
//
|
||||
// However, TestRelay uses in-memory database and temporary directories,
|
||||
// so we can't easily test startup cleanup with pre-existing data.
|
||||
// This test demonstrates the pattern for manual testing.
|
||||
|
||||
// For now, manually run cleanup to verify it works
|
||||
let result = ngit_grasp::database::cleanup_archived_data(&holding_db, &git_data_path)
|
||||
.await
|
||||
.expect("Cleanup should succeed");
|
||||
|
||||
assert_eq!(result.events_deleted, 1, "Should delete 1 event on startup");
|
||||
assert!(
|
||||
!archive_path.exists(),
|
||||
"Archive file should be deleted on startup"
|
||||
);
|
||||
assert!(
|
||||
!metadata_path.exists(),
|
||||
"Metadata file should be deleted on startup"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cleanup_with_missing_files() {
|
||||
use ngit_grasp::config::DatabaseBackend;
|
||||
use ngit_grasp::database::{DeletionMetadata, HoldingDatabase};
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let git_data_path = temp_dir.path().join("git-data");
|
||||
fs::create_dir_all(&git_data_path).unwrap();
|
||||
|
||||
let holding_db = HoldingDatabase::new(&git_data_path, DatabaseBackend::Memory)
|
||||
.await
|
||||
.expect("Failed to create holding database");
|
||||
|
||||
// Create expired event with archive path pointing to non-existent files
|
||||
let keys = Keys::generate();
|
||||
let deletion_keys = Keys::generate();
|
||||
let deletion_event_id = EventBuilder::text_note("deletion request")
|
||||
.sign_with_keys(&deletion_keys)
|
||||
.unwrap()
|
||||
.id;
|
||||
|
||||
let now = current_timestamp();
|
||||
let event = EventBuilder::text_note("event with missing archive")
|
||||
.sign_with_keys(&keys)
|
||||
.unwrap();
|
||||
|
||||
let metadata = DeletionMetadata {
|
||||
deletion_timestamp: now - 7200,
|
||||
deletion_event_id,
|
||||
expiry_timestamp: now - 3600, // Expired 1 hour ago
|
||||
archive_path: Some(".archive/npub1missing/nonexistent-123456.tar.gz".to_string()),
|
||||
};
|
||||
|
||||
holding_db.store_event(&event, metadata).await.unwrap();
|
||||
|
||||
// Run cleanup - should not error even though files don't exist
|
||||
let result = ngit_grasp::database::cleanup_archived_data(&holding_db, &git_data_path)
|
||||
.await
|
||||
.expect("Cleanup should succeed even with missing files");
|
||||
|
||||
// Files didn't exist, so deletion counts should be 0
|
||||
assert_eq!(
|
||||
result.archives_deleted, 0,
|
||||
"Should not count missing archive as deleted"
|
||||
);
|
||||
assert_eq!(
|
||||
result.metadata_deleted, 0,
|
||||
"Should not count missing metadata as deleted"
|
||||
);
|
||||
|
||||
// Note: Event deletion from holding database is not yet implemented
|
||||
// When implemented, we should verify that cleanup still processes the event
|
||||
// even when files are missing
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_disk_space_reclamation() {
|
||||
use ngit_grasp::config::DatabaseBackend;
|
||||
use ngit_grasp::database::{DeletionMetadata, HoldingDatabase};
|
||||
use ngit_grasp::git::archive::archive_repository;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let git_data_path = temp_dir.path().join("git-data");
|
||||
fs::create_dir_all(&git_data_path).unwrap();
|
||||
|
||||
// Create a large test git repository
|
||||
let repo_path = create_test_git_repo(&git_data_path, "large-repo");
|
||||
|
||||
// Add more files to make it larger
|
||||
let work_dir = git_data_path.join("large-repo-work");
|
||||
Command::new("git")
|
||||
.args([
|
||||
"clone",
|
||||
repo_path.to_str().unwrap(),
|
||||
work_dir.to_str().unwrap(),
|
||||
])
|
||||
.output()
|
||||
.expect("Failed to clone repo");
|
||||
|
||||
// Configure git
|
||||
Command::new("git")
|
||||
.args(["config", "user.email", "test@test.com"])
|
||||
.current_dir(&work_dir)
|
||||
.output()
|
||||
.unwrap();
|
||||
Command::new("git")
|
||||
.args(["config", "user.name", "Test User"])
|
||||
.current_dir(&work_dir)
|
||||
.output()
|
||||
.unwrap();
|
||||
Command::new("git")
|
||||
.args(["config", "commit.gpgsign", "false"])
|
||||
.current_dir(&work_dir)
|
||||
.output()
|
||||
.unwrap();
|
||||
|
||||
// Add multiple large files
|
||||
for i in 0..5 {
|
||||
let content = format!("Large file content {}\n", i).repeat(1000);
|
||||
fs::write(work_dir.join(format!("large-file-{}.txt", i)), content).unwrap();
|
||||
}
|
||||
|
||||
Command::new("git")
|
||||
.args(["add", "."])
|
||||
.current_dir(&work_dir)
|
||||
.output()
|
||||
.unwrap();
|
||||
Command::new("git")
|
||||
.args(["commit", "-m", "Add large files"])
|
||||
.current_dir(&work_dir)
|
||||
.output()
|
||||
.unwrap();
|
||||
Command::new("git")
|
||||
.args(["push", "origin", "master"])
|
||||
.current_dir(&work_dir)
|
||||
.output()
|
||||
.unwrap();
|
||||
|
||||
// Create archive
|
||||
let archive_base = git_data_path.join(".archive");
|
||||
let npub = "npub1diskspace";
|
||||
let identifier = "large-repo";
|
||||
let timestamp = current_timestamp();
|
||||
|
||||
let archive_path = archive_repository(&repo_path, &archive_base, npub, identifier, timestamp)
|
||||
.expect("Failed to create archive");
|
||||
|
||||
// Measure archive size
|
||||
let archive_size = fs::metadata(&archive_path).unwrap().len();
|
||||
assert!(archive_size > 1000, "Archive should be reasonably large");
|
||||
|
||||
// Create metadata file
|
||||
let metadata_path = PathBuf::from(format!("{}.json", archive_path.display()));
|
||||
fs::write(&metadata_path, b"{}").unwrap();
|
||||
let metadata_size = fs::metadata(&metadata_path).unwrap().len();
|
||||
|
||||
// Create holding database and store expired event
|
||||
let holding_db = HoldingDatabase::new(&git_data_path, DatabaseBackend::Memory)
|
||||
.await
|
||||
.expect("Failed to create holding database");
|
||||
|
||||
let keys = Keys::generate();
|
||||
let deletion_keys = Keys::generate();
|
||||
let deletion_event_id = EventBuilder::text_note("deletion request")
|
||||
.sign_with_keys(&deletion_keys)
|
||||
.unwrap()
|
||||
.id;
|
||||
|
||||
let now = current_timestamp();
|
||||
let event = EventBuilder::text_note("event to be deleted")
|
||||
.sign_with_keys(&keys)
|
||||
.unwrap();
|
||||
|
||||
let metadata = DeletionMetadata {
|
||||
deletion_timestamp: now - 7200,
|
||||
deletion_event_id,
|
||||
expiry_timestamp: now - 3600,
|
||||
archive_path: Some(format!(
|
||||
".archive/{}/{}-{}.tar.gz",
|
||||
npub, identifier, timestamp
|
||||
)),
|
||||
};
|
||||
|
||||
holding_db.store_event(&event, metadata).await.unwrap();
|
||||
|
||||
// Measure disk usage before cleanup
|
||||
let disk_usage_before = get_dir_size(&archive_base);
|
||||
assert!(
|
||||
disk_usage_before > 0,
|
||||
"Should have disk usage before cleanup"
|
||||
);
|
||||
|
||||
// Run cleanup
|
||||
let result = ngit_grasp::database::cleanup_archived_data(&holding_db, &git_data_path)
|
||||
.await
|
||||
.expect("Cleanup should succeed");
|
||||
|
||||
// Verify disk space was reclaimed
|
||||
// Allow small variance due to file system metadata
|
||||
let expected_bytes = archive_size + metadata_size;
|
||||
assert!(
|
||||
result.bytes_reclaimed >= expected_bytes - 10
|
||||
&& result.bytes_reclaimed <= expected_bytes + 10,
|
||||
"Should reclaim approximately {} bytes (got {})",
|
||||
expected_bytes,
|
||||
result.bytes_reclaimed
|
||||
);
|
||||
|
||||
// Measure disk usage after cleanup
|
||||
let disk_usage_after = if archive_base.exists() {
|
||||
get_dir_size(&archive_base)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
assert!(
|
||||
disk_usage_after < disk_usage_before,
|
||||
"Disk usage should decrease after cleanup"
|
||||
);
|
||||
|
||||
// Verify files are gone
|
||||
assert!(!archive_path.exists(), "Archive file should be deleted");
|
||||
assert!(!metadata_path.exists(), "Metadata file should be deleted");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cleanup_multiple_expired_events() {
|
||||
use ngit_grasp::config::DatabaseBackend;
|
||||
use ngit_grasp::database::{DeletionMetadata, HoldingDatabase};
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let git_data_path = temp_dir.path().join("git-data");
|
||||
fs::create_dir_all(&git_data_path).unwrap();
|
||||
|
||||
let holding_db = HoldingDatabase::new(&git_data_path, DatabaseBackend::Memory)
|
||||
.await
|
||||
.expect("Failed to create holding database");
|
||||
|
||||
// Create multiple archive files
|
||||
let archive_dir = git_data_path.join(".archive/npub1multiple");
|
||||
fs::create_dir_all(&archive_dir).unwrap();
|
||||
|
||||
let now = current_timestamp();
|
||||
let mut expected_bytes = 0u64;
|
||||
|
||||
// Create 5 expired events with archives
|
||||
for i in 0..5 {
|
||||
let archive_path = archive_dir.join(format!("repo-{}-{}.tar.gz", i, now));
|
||||
let metadata_path = archive_dir.join(format!("repo-{}-{}.tar.gz.json", i, now));
|
||||
|
||||
let archive_content = format!("fake archive data {}", i);
|
||||
fs::write(&archive_path, archive_content.as_bytes()).unwrap();
|
||||
fs::write(&metadata_path, b"{}").unwrap();
|
||||
|
||||
expected_bytes += archive_content.len() as u64 + 2; // +2 for "{}"
|
||||
|
||||
let keys = Keys::generate();
|
||||
let deletion_keys = Keys::generate();
|
||||
let deletion_event_id = EventBuilder::text_note("deletion request")
|
||||
.sign_with_keys(&deletion_keys)
|
||||
.unwrap()
|
||||
.id;
|
||||
|
||||
let event = EventBuilder::text_note(format!("event {}", i))
|
||||
.sign_with_keys(&keys)
|
||||
.unwrap();
|
||||
|
||||
let metadata = DeletionMetadata {
|
||||
deletion_timestamp: now - 7200,
|
||||
deletion_event_id,
|
||||
expiry_timestamp: now - 3600,
|
||||
archive_path: Some(format!(".archive/npub1multiple/repo-{}-{}.tar.gz", i, now)),
|
||||
};
|
||||
|
||||
holding_db.store_event(&event, metadata).await.unwrap();
|
||||
}
|
||||
|
||||
// Verify all events are expired
|
||||
let expired = holding_db.query_expired(now).await.unwrap();
|
||||
assert_eq!(expired.len(), 5, "Should have 5 expired events");
|
||||
|
||||
// Run cleanup
|
||||
let result = ngit_grasp::database::cleanup_archived_data(&holding_db, &git_data_path)
|
||||
.await
|
||||
.expect("Cleanup should succeed");
|
||||
|
||||
// Verify all files were cleaned up
|
||||
// Note: events_deleted will be 0 until delete_event is fully implemented
|
||||
assert_eq!(result.archives_deleted, 5, "Should delete 5 archives");
|
||||
assert_eq!(result.metadata_deleted, 5, "Should delete 5 metadata files");
|
||||
assert_eq!(
|
||||
result.bytes_reclaimed, expected_bytes,
|
||||
"Should reclaim expected bytes"
|
||||
);
|
||||
|
||||
// Note: Event deletion from holding database is not yet implemented
|
||||
// When implemented, we should verify: expired_after.len() == 0
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cleanup_preserves_non_expired_events() {
|
||||
use ngit_grasp::config::DatabaseBackend;
|
||||
use ngit_grasp::database::{DeletionMetadata, HoldingDatabase};
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let git_data_path = temp_dir.path().join("git-data");
|
||||
fs::create_dir_all(&git_data_path).unwrap();
|
||||
|
||||
let holding_db = HoldingDatabase::new(&git_data_path, DatabaseBackend::Memory)
|
||||
.await
|
||||
.expect("Failed to create holding database");
|
||||
|
||||
let archive_dir = git_data_path.join(".archive/npub1preserve");
|
||||
fs::create_dir_all(&archive_dir).unwrap();
|
||||
|
||||
let now = current_timestamp();
|
||||
|
||||
// Create one expired event
|
||||
let expired_archive = archive_dir.join("expired-123.tar.gz");
|
||||
let expired_metadata = archive_dir.join("expired-123.tar.gz.json");
|
||||
fs::write(&expired_archive, b"expired archive").unwrap();
|
||||
fs::write(&expired_metadata, b"{}").unwrap();
|
||||
|
||||
let keys1 = Keys::generate();
|
||||
let deletion_keys1 = Keys::generate();
|
||||
let deletion_event_id1 = EventBuilder::text_note("deletion request 1")
|
||||
.sign_with_keys(&deletion_keys1)
|
||||
.unwrap()
|
||||
.id;
|
||||
|
||||
let event1 = EventBuilder::text_note("expired event")
|
||||
.sign_with_keys(&keys1)
|
||||
.unwrap();
|
||||
|
||||
let metadata1 = DeletionMetadata {
|
||||
deletion_timestamp: now - 7200,
|
||||
deletion_event_id: deletion_event_id1,
|
||||
expiry_timestamp: now - 3600, // Expired
|
||||
archive_path: Some(".archive/npub1preserve/expired-123.tar.gz".to_string()),
|
||||
};
|
||||
|
||||
holding_db.store_event(&event1, metadata1).await.unwrap();
|
||||
|
||||
// Create one non-expired event
|
||||
let active_archive = archive_dir.join("active-456.tar.gz");
|
||||
let active_metadata = archive_dir.join("active-456.tar.gz.json");
|
||||
fs::write(&active_archive, b"active archive").unwrap();
|
||||
fs::write(&active_metadata, b"{}").unwrap();
|
||||
|
||||
let keys2 = Keys::generate();
|
||||
let deletion_keys2 = Keys::generate();
|
||||
let deletion_event_id2 = EventBuilder::text_note("deletion request 2")
|
||||
.sign_with_keys(&deletion_keys2)
|
||||
.unwrap()
|
||||
.id;
|
||||
|
||||
let event2 = EventBuilder::text_note("active event")
|
||||
.sign_with_keys(&keys2)
|
||||
.unwrap();
|
||||
|
||||
let metadata2 = DeletionMetadata {
|
||||
deletion_timestamp: now,
|
||||
deletion_event_id: deletion_event_id2,
|
||||
expiry_timestamp: now + 3600, // Not expired yet
|
||||
archive_path: Some(".archive/npub1preserve/active-456.tar.gz".to_string()),
|
||||
};
|
||||
|
||||
holding_db.store_event(&event2, metadata2).await.unwrap();
|
||||
|
||||
// Run cleanup
|
||||
let result = ngit_grasp::database::cleanup_archived_data(&holding_db, &git_data_path)
|
||||
.await
|
||||
.expect("Cleanup should succeed");
|
||||
|
||||
// Verify only expired files were cleaned up
|
||||
// Note: events_deleted will be 0 until delete_event is fully implemented
|
||||
assert_eq!(
|
||||
result.archives_deleted, 1,
|
||||
"Should delete only 1 expired archive"
|
||||
);
|
||||
|
||||
// Verify expired files are gone
|
||||
assert!(
|
||||
!expired_archive.exists(),
|
||||
"Expired archive should be deleted"
|
||||
);
|
||||
assert!(
|
||||
!expired_metadata.exists(),
|
||||
"Expired metadata should be deleted"
|
||||
);
|
||||
|
||||
// Verify active files still exist
|
||||
assert!(active_archive.exists(), "Active archive should still exist");
|
||||
assert!(
|
||||
active_metadata.exists(),
|
||||
"Active metadata should still exist"
|
||||
);
|
||||
|
||||
// Verify both events are still in database (since delete_event is not implemented)
|
||||
// When delete_event is implemented, we should verify:
|
||||
// - expired.len() == 0 (expired event deleted)
|
||||
// - future_expired.len() == 1 (active event preserved)
|
||||
let all_events = holding_db.query_expired(now + 7200).await.unwrap();
|
||||
assert_eq!(
|
||||
all_events.len(),
|
||||
2,
|
||||
"Both events should still be in database (delete_event not yet implemented)"
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user