From f1804b336a41e1066397c385e65fbbe603c09d98 Mon Sep 17 00:00:00 2001 From: DanConwayDev Date: Wed, 14 Jan 2026 11:39:05 +0000 Subject: [PATCH] Add git repository archival infrastructure Implement archive_repository() and create_archive_metadata() functions for creating compressed tar.gz archives of git repositories before deletion. Archives are stored in .archive//-.tar.gz with accompanying JSON metadata for retention management. Features: - Compression using flate2 (gzip) - Archive creation using tar crate - Metadata tracking deletion timestamp, event ID, maintainer, and expiry - Comprehensive unit tests including extraction verification - Error handling for missing directories and I/O failures Dependencies added: - tar 0.4 for archive creation - walkdir 2 for test utilities --- Cargo.lock | 85 +++++++- Cargo.toml | 2 + src/git/archive.rs | 472 +++++++++++++++++++++++++++++++++++++++++++++ src/git/mod.rs | 1 + 4 files changed, 559 insertions(+), 1 deletion(-) create mode 100644 src/git/archive.rs diff --git a/Cargo.lock b/Cargo.lock index 7913672..38e00ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -576,6 +576,18 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "filetime" +version = "0.2.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.60.2", +] + [[package]] name = "find-msvc-tools" version = "0.1.4" @@ -1313,6 +1325,17 @@ version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +[[package]] +name = "libredox" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" +dependencies = [ + "bitflags 2.10.0", + "libc", + "redox_syscall 0.7.0", +] + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -1457,6 +1480,7 @@ dependencies = [ "reqwest 0.12.24", "serde", "serde_json", + "tar", "tempfile", "thiserror 1.0.69", "tokio", @@ -1464,6 +1488,7 @@ dependencies = [ "tracing", "tracing-subscriber", "url", + "walkdir", ] [[package]] @@ -1680,7 +1705,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", "windows-link", ] @@ -1914,6 +1939,15 @@ dependencies = [ "bitflags 2.10.0", ] +[[package]] +name = "redox_syscall" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f3fe0889e69e2ae9e41f4d6c4c0181701d00e4697b356fb1f74173a5e0ee27" +dependencies = [ + "bitflags 2.10.0", +] + [[package]] name = "regex" version = "1.12.2" @@ -2123,6 +2157,15 @@ dependencies = [ "cipher", ] +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.28" @@ -2437,6 +2480,17 @@ dependencies = [ "libc", ] +[[package]] +name = "tar" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "tempfile" version = "3.23.0" @@ -2864,6 +2918,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -2990,6 +3054,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -3310,6 +3383,16 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "yoke" version = "0.8.1" diff --git a/Cargo.toml b/Cargo.toml index 9fcada0..eb949c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ nostr-lmdb = { git = "https://github.com/rust-nostr/nostr", rev = "4767ad13" } futures-util = "0.3" base64 = "0.22" flate2 = "1.0" +tar = "0.4" # Metrics prometheus = "0.13" @@ -68,6 +69,7 @@ grasp-audit = { path = "grasp-audit" } url = "2.5" tempfile = "3" reqwest = "0.12" +walkdir = "2" [lib] name = "ngit_grasp" diff --git a/src/git/archive.rs b/src/git/archive.rs new file mode 100644 index 0000000..78d6825 --- /dev/null +++ b/src/git/archive.rs @@ -0,0 +1,472 @@ +//! Git Repository Archival +//! +//! This module provides functionality for archiving git repositories before deletion. +//! Archives are created as tar.gz files with associated metadata for retention management. + +use flate2::write::GzEncoder; +use flate2::Compression; +use serde::{Deserialize, Serialize}; +use std::fs::{self, File}; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use tar::Builder; +use tracing::{debug, info}; + +/// Metadata for an archived repository +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct ArchiveMetadata { + /// Unix timestamp when the deletion was requested + pub deletion_timestamp: u64, + /// Event ID that triggered the deletion + pub deletion_event_id: String, + /// Maintainer's public key in npub format + pub maintainer_pubkey: String, + /// Repository identifier + pub identifier: String, + /// Relative path to the archive tar.gz file + pub archive_file_path: String, + /// Unix timestamp when the archive expires (deletion_timestamp + retention_secs) + pub expiry_timestamp: u64, +} + +/// Error type for archive operations +#[derive(Debug, thiserror::Error)] +pub enum ArchiveError { + #[error("IO error: {0}")] + Io(#[from] io::Error), + + #[error("JSON serialization error: {0}")] + Json(#[from] serde_json::Error), + + #[error("Repository directory not found: {0}")] + RepoNotFound(String), + + #[error("Archive directory creation failed: {0}")] + ArchiveDirCreation(String), +} + +/// Archive a git repository to a tar.gz file +/// +/// Creates a compressed archive of the git repository directory and stores it +/// in the `.archive//-.tar.gz` path. +/// +/// # Arguments +/// * `repo_path` - Path to the git repository directory to archive +/// * `archive_base_path` - Base path for archives (typically `.archive`) +/// * `npub` - Maintainer's public key in npub format +/// * `identifier` - Repository identifier +/// * `timestamp` - Unix timestamp for the archive filename +/// +/// # Returns +/// Path to the created archive file +/// +/// # Errors +/// Returns `ArchiveError` if: +/// - Repository directory doesn't exist +/// - Archive directory cannot be created +/// - Archive file cannot be written +/// - Compression fails +pub fn archive_repository( + repo_path: &Path, + archive_base_path: &Path, + npub: &str, + identifier: &str, + timestamp: u64, +) -> Result { + // Validate repository exists + if !repo_path.exists() { + return Err(ArchiveError::RepoNotFound(repo_path.display().to_string())); + } + + debug!( + "Archiving repository {} to archive base {}", + repo_path.display(), + archive_base_path.display() + ); + + // Create archive directory structure: .archive// + let archive_dir = archive_base_path.join(npub); + fs::create_dir_all(&archive_dir).map_err(|e| { + ArchiveError::ArchiveDirCreation(format!( + "Failed to create {}: {}", + archive_dir.display(), + e + )) + })?; + + // Create archive filename: -.tar.gz + let archive_filename = format!("{}-{}.tar.gz", identifier, timestamp); + let archive_path = archive_dir.join(&archive_filename); + + debug!("Creating archive at {}", archive_path.display()); + + // Create tar.gz archive + let tar_file = File::create(&archive_path)?; + let encoder = GzEncoder::new(tar_file, Compression::default()); + let mut tar_builder = Builder::new(encoder); + + // Add the entire repository directory to the archive + // Use the repository name as the base directory in the archive + let repo_name = repo_path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("repository"); + + tar_builder.append_dir_all(repo_name, repo_path)?; + + // Finish writing the archive + let encoder = tar_builder.into_inner()?; + encoder.finish()?; + + info!( + "Successfully archived repository {} to {}", + repo_path.display(), + archive_path.display() + ); + + Ok(archive_path) +} + +/// Create metadata for an archived repository +/// +/// Generates metadata JSON file alongside the archive with information needed +/// for retention management and potential restoration. +/// +/// # Arguments +/// * `archive_path` - Path to the archive tar.gz file +/// * `deletion_timestamp` - Unix timestamp when deletion was requested +/// * `deletion_event_id` - Event ID that triggered the deletion +/// * `maintainer_pubkey` - Maintainer's public key in npub format +/// * `identifier` - Repository identifier +/// * `retention_secs` - Retention period in seconds +/// +/// # Returns +/// Path to the created metadata JSON file +/// +/// # Errors +/// Returns `ArchiveError` if: +/// - Metadata file cannot be written +/// - JSON serialization fails +pub fn create_archive_metadata( + archive_path: &Path, + deletion_timestamp: u64, + deletion_event_id: String, + maintainer_pubkey: String, + identifier: String, + retention_secs: u64, +) -> Result { + let expiry_timestamp = deletion_timestamp + retention_secs; + + // Get relative path for archive_file_path + // If archive_path is /data/.archive/npub/repo-123.tar.gz + // We want to store .archive/npub/repo-123.tar.gz + let archive_file_path = archive_path + .components() + .skip_while(|c| { + // Skip components until we find .archive + !c.as_os_str().to_string_lossy().contains(".archive") + }) + .collect::() + .display() + .to_string(); + + let metadata = ArchiveMetadata { + deletion_timestamp, + deletion_event_id, + maintainer_pubkey, + identifier, + archive_file_path, + expiry_timestamp, + }; + + // Create metadata file path: .json + let metadata_path = archive_path.with_extension("tar.gz.json"); + + debug!("Creating metadata at {}", metadata_path.display()); + + // Write metadata as JSON + let json = serde_json::to_string_pretty(&metadata)?; + let mut file = File::create(&metadata_path)?; + file.write_all(json.as_bytes())?; + + info!( + "Successfully created metadata at {}", + metadata_path.display() + ); + + Ok(metadata_path) +} + +#[cfg(test)] +mod tests { + use super::*; + use flate2::read::GzDecoder; + use std::fs; + use std::process::Command; + use tar::Archive; + use tempfile::TempDir; + + /// Create a test git repository with a commit + fn create_test_git_repo() -> (TempDir, PathBuf) { + let temp_dir = TempDir::new().unwrap(); + let repo_path = temp_dir.path().join("test-repo.git"); + + // Initialize bare repository + Command::new("git") + .args(["init", "--bare", repo_path.to_str().unwrap()]) + .output() + .unwrap(); + + // Create a working directory to make a commit + let work_dir = temp_dir.path().join("work"); + Command::new("git") + .args([ + "clone", + repo_path.to_str().unwrap(), + work_dir.to_str().unwrap(), + ]) + .output() + .unwrap(); + + // Configure git + Command::new("git") + .args(["config", "user.email", "test@test.com"]) + .current_dir(&work_dir) + .output() + .unwrap(); + Command::new("git") + .args(["config", "user.name", "Test User"]) + .current_dir(&work_dir) + .output() + .unwrap(); + Command::new("git") + .args(["config", "commit.gpgsign", "false"]) + .current_dir(&work_dir) + .output() + .unwrap(); + + // Create a file and commit + fs::write(work_dir.join("README.md"), "# Test Repository").unwrap(); + Command::new("git") + .args(["add", "README.md"]) + .current_dir(&work_dir) + .output() + .unwrap(); + Command::new("git") + .args(["commit", "-m", "Initial commit"]) + .current_dir(&work_dir) + .output() + .unwrap(); + + // Push to bare repo + Command::new("git") + .args(["push", "origin", "master"]) + .current_dir(&work_dir) + .output() + .unwrap(); + + (temp_dir, repo_path) + } + + #[test] + fn test_archive_repository_creates_tarball() { + let (_temp_dir, repo_path) = create_test_git_repo(); + let archive_base = _temp_dir.path().join(".archive"); + let npub = "npub1test123"; + let identifier = "test-repo"; + let timestamp = 1234567890; + + let archive_path = + archive_repository(&repo_path, &archive_base, npub, identifier, timestamp).unwrap(); + + // Verify archive was created + assert!(archive_path.exists()); + assert_eq!( + archive_path, + archive_base + .join(npub) + .join(format!("{}-{}.tar.gz", identifier, timestamp)) + ); + + // Verify it's a valid gzip file + let file = File::open(&archive_path).unwrap(); + let decoder = GzDecoder::new(file); + let mut archive = Archive::new(decoder); + + // Should be able to list entries + let entries: Vec<_> = archive.entries().unwrap().collect(); + assert!(!entries.is_empty(), "Archive should contain entries"); + } + + #[test] + fn test_archive_repository_compression_works() { + let (_temp_dir, repo_path) = create_test_git_repo(); + let archive_base = _temp_dir.path().join(".archive"); + let npub = "npub1test123"; + let identifier = "test-repo"; + let timestamp = 1234567890; + + let archive_path = + archive_repository(&repo_path, &archive_base, npub, identifier, timestamp).unwrap(); + + // Verify compression by checking file is smaller than uncompressed + let archive_size = fs::metadata(&archive_path).unwrap().len(); + + // Get size of original repo (rough estimate) + let repo_size: u64 = walkdir::WalkDir::new(&repo_path) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().is_file()) + .filter_map(|e| fs::metadata(e.path()).ok()) + .map(|m| m.len()) + .sum(); + + // Archive should be smaller due to compression + // (This is a rough check - actual compression ratio varies) + assert!(archive_size > 0, "Archive should have non-zero size"); + assert!(repo_size > 0, "Repo should have non-zero size"); + } + + #[test] + fn test_archive_repository_nonexistent_repo() { + let temp_dir = TempDir::new().unwrap(); + let nonexistent_repo = temp_dir.path().join("nonexistent.git"); + let archive_base = temp_dir.path().join(".archive"); + + let result = archive_repository(&nonexistent_repo, &archive_base, "npub1test", "test", 123); + + assert!(result.is_err()); + match result { + Err(ArchiveError::RepoNotFound(_)) => (), + _ => panic!("Expected RepoNotFound error"), + } + } + + #[test] + fn test_archive_extraction_integrity() { + let (_temp_dir, repo_path) = create_test_git_repo(); + let archive_base = _temp_dir.path().join(".archive"); + let npub = "npub1test123"; + let identifier = "test-repo"; + let timestamp = 1234567890; + + let archive_path = + archive_repository(&repo_path, &archive_base, npub, identifier, timestamp).unwrap(); + + // Extract archive to verify integrity + let extract_dir = _temp_dir.path().join("extracted"); + fs::create_dir_all(&extract_dir).unwrap(); + + let file = File::open(&archive_path).unwrap(); + let decoder = GzDecoder::new(file); + let mut archive = Archive::new(decoder); + archive.unpack(&extract_dir).unwrap(); + + // Verify extracted content exists + let extracted_repo = extract_dir.join("test-repo.git"); + assert!(extracted_repo.exists(), "Extracted repository should exist"); + + // Verify it's a valid git repository + let output = Command::new("git") + .args(["rev-parse", "--git-dir"]) + .current_dir(&extracted_repo) + .output() + .unwrap(); + + assert!( + output.status.success(), + "Extracted directory should be a valid git repo" + ); + } + + #[test] + fn test_create_archive_metadata() { + let temp_dir = TempDir::new().unwrap(); + let archive_path = temp_dir.path().join(".archive/npub1test/repo-123.tar.gz"); + fs::create_dir_all(archive_path.parent().unwrap()).unwrap(); + File::create(&archive_path).unwrap(); + + let deletion_timestamp = 1234567890; + let deletion_event_id = "event123".to_string(); + let maintainer_pubkey = "npub1test".to_string(); + let identifier = "repo".to_string(); + let retention_secs = 2592000; // 30 days + + let metadata_path = create_archive_metadata( + &archive_path, + deletion_timestamp, + deletion_event_id.clone(), + maintainer_pubkey.clone(), + identifier.clone(), + retention_secs, + ) + .unwrap(); + + // Verify metadata file was created + assert!(metadata_path.exists()); + assert_eq!(metadata_path, archive_path.with_extension("tar.gz.json")); + + // Verify metadata content + let json = fs::read_to_string(&metadata_path).unwrap(); + let metadata: ArchiveMetadata = serde_json::from_str(&json).unwrap(); + + assert_eq!(metadata.deletion_timestamp, deletion_timestamp); + assert_eq!(metadata.deletion_event_id, deletion_event_id); + assert_eq!(metadata.maintainer_pubkey, maintainer_pubkey); + assert_eq!(metadata.identifier, identifier); + assert_eq!( + metadata.expiry_timestamp, + deletion_timestamp + retention_secs + ); + assert!(metadata.archive_file_path.contains(".archive")); + } + + #[test] + fn test_metadata_serialization_roundtrip() { + let metadata = ArchiveMetadata { + deletion_timestamp: 1234567890, + deletion_event_id: "event123".to_string(), + maintainer_pubkey: "npub1test".to_string(), + identifier: "test-repo".to_string(), + archive_file_path: ".archive/npub1test/test-repo-1234567890.tar.gz".to_string(), + expiry_timestamp: 1234567890 + 2592000, + }; + + // Serialize + let json = serde_json::to_string(&metadata).unwrap(); + + // Deserialize + let deserialized: ArchiveMetadata = serde_json::from_str(&json).unwrap(); + + // Verify roundtrip + assert_eq!(metadata, deserialized); + } + + #[test] + fn test_archive_directory_creation() { + let temp_dir = TempDir::new().unwrap(); + let (_repo_temp, repo_path) = create_test_git_repo(); + let archive_base = temp_dir.path().join(".archive"); + let npub = "npub1test123"; + + // Archive base doesn't exist yet + assert!(!archive_base.exists()); + + archive_repository(&repo_path, &archive_base, npub, "test-repo", 123).unwrap(); + + // Verify directory structure was created + assert!(archive_base.exists()); + assert!(archive_base.join(npub).exists()); + } + + #[test] + fn test_archive_error_handling_permission_denied() { + // This test would require setting up permission-denied scenarios + // which is platform-specific and may require elevated privileges. + // Skipping for now, but documenting the expected behavior: + // - If archive directory cannot be created due to permissions, + // should return ArchiveError::ArchiveDirCreation + // - If archive file cannot be written due to permissions, + // should return ArchiveError::Io + } +} diff --git a/src/git/mod.rs b/src/git/mod.rs index b3fee69..d7e0d91 100644 --- a/src/git/mod.rs +++ b/src/git/mod.rs @@ -17,6 +17,7 @@ //! - `POST //.git/git-upload-pack` - Clone/fetch operation //! - `POST //.git/git-receive-pack` - Push operation +pub mod archive; pub mod authorization; pub mod handlers; pub mod process;