laantungir/ngit-grasp
1
0
Fork 0
mirror of https://relay.ngit.dev/npub15qydau2hjma6ngxkl2cyar74wzyjshvl65za5k5rl69264ar2exs5cyejr/ngit-grasp.git synced 2026-10-05 15:08:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add git repository archival infrastructure

Browse Source 
Implement archive_repository() and create_archive_metadata() functions
for creating compressed tar.gz archives of git repositories before
deletion. Archives are stored in .archive/<npub>/<identifier>-<timestamp>.tar.gz
with accompanying JSON metadata for retention management.

Features:
- Compression using flate2 (gzip)
- Archive creation using tar crate
- Metadata tracking deletion timestamp, event ID, maintainer, and expiry
- Comprehensive unit tests including extraction verification
- Error handling for missing directories and I/O failures

Dependencies added:
- tar 0.4 for archive creation
- walkdir 2 for test utilities
This commit is contained in:
DanConwayDev
2026-01-14 11:39:05 +00:00
parent dc86593fa9
commit f1804b336a
4 changed files with 559 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+84 -1
View File
@@ -576,6 +576,18 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "filetime"
version = "0.2.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc0505cd1b6fa6580283f6bdf70a73fcf4aba1184038c90902b92b3dd0df63ed"
dependencies = [
"cfg-if",
"libc",
"libredox",
"windows-sys 0.60.2",
]
[[package]]
name = "find-msvc-tools"
version = "0.1.4"
@@ -1313,6 +1325,17 @@ version = "0.2.177"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976"
[[package]]
name = "libredox"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616"
dependencies = [
"bitflags 2.10.0",
"libc",
"redox_syscall 0.7.0",
]
[[package]]
name = "linux-raw-sys"
version = "0.11.0"
@@ -1457,6 +1480,7 @@ dependencies = [
"reqwest 0.12.24",
"serde",
"serde_json",
"tar",
"tempfile",
"thiserror 1.0.69",
"tokio",
@@ -1464,6 +1488,7 @@ dependencies = [
"tracing",
"tracing-subscriber",
"url",
"walkdir",
]
[[package]]
@@ -1680,7 +1705,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"redox_syscall 0.5.18",
"smallvec",
"windows-link",
]
@@ -1914,6 +1939,15 @@ dependencies = [
"bitflags 2.10.0",
]
[[package]]
name = "redox_syscall"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49f3fe0889e69e2ae9e41f4d6c4c0181701d00e4697b356fb1f74173a5e0ee27"
dependencies = [
"bitflags 2.10.0",
]
[[package]]
name = "regex"
version = "1.12.2"
@@ -2123,6 +2157,15 @@ dependencies = [
"cipher",
]
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "schannel"
version = "0.1.28"
@@ -2437,6 +2480,17 @@ dependencies = [
"libc",
]
[[package]]
name = "tar"
version = "0.4.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a"
dependencies = [
"filetime",
"libc",
"xattr",
]
[[package]]
name = "tempfile"
version = "3.23.0"
@@ -2864,6 +2918,16 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "walkdir"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
dependencies = [
"same-file",
"winapi-util",
]
[[package]]
name = "want"
version = "0.3.1"
@@ -2990,6 +3054,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
@@ -3310,6 +3383,16 @@ version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
[[package]]
name = "xattr"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
dependencies = [
"libc",
"rustix",
]
[[package]]
name = "yoke"
version = "0.8.1"
Cargo.toml
+2
View File
@@ -27,6 +27,7 @@ nostr-lmdb = { git = "https://github.com/rust-nostr/nostr", rev = "4767ad13" }
futures-util = "0.3"
base64 = "0.22"
flate2 = "1.0"
tar = "0.4"
# Metrics
prometheus = "0.13"
@@ -68,6 +69,7 @@ grasp-audit = { path = "grasp-audit" }
url = "2.5"
tempfile = "3"
reqwest = "0.12"
walkdir = "2"
[lib]
name = "ngit_grasp"
src/git/archive.rs
+472
View File
@@ -0,0 +1,472 @@
//! Git Repository Archival
//!
//! This module provides functionality for archiving git repositories before deletion.
//! Archives are created as tar.gz files with associated metadata for retention management.
use flate2::write::GzEncoder;
use flate2::Compression;
use serde::{Deserialize, Serialize};
use std::fs::{self, File};
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use tar::Builder;
use tracing::{debug, info};
/// Metadata for an archived repository
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
pub struct ArchiveMetadata {
/// Unix timestamp when the deletion was requested
pub deletion_timestamp: u64,
/// Event ID that triggered the deletion
pub deletion_event_id: String,
/// Maintainer's public key in npub format
pub maintainer_pubkey: String,
/// Repository identifier
pub identifier: String,
/// Relative path to the archive tar.gz file
pub archive_file_path: String,
/// Unix timestamp when the archive expires (deletion_timestamp + retention_secs)
pub expiry_timestamp: u64,
}
/// Error type for archive operations
#[derive(Debug, thiserror::Error)]
pub enum ArchiveError {
#[error("IO error: {0}")]
Io(#[from] io::Error),
#[error("JSON serialization error: {0}")]
Json(#[from] serde_json::Error),
#[error("Repository directory not found: {0}")]
RepoNotFound(String),
#[error("Archive directory creation failed: {0}")]
ArchiveDirCreation(String),
}
/// Archive a git repository to a tar.gz file
///
/// Creates a compressed archive of the git repository directory and stores it
/// in the `.archive/<npub>/<identifier>-<timestamp>.tar.gz` path.
///
/// # Arguments
/// * `repo_path` - Path to the git repository directory to archive
/// * `archive_base_path` - Base path for archives (typically `.archive`)
/// * `npub` - Maintainer's public key in npub format
/// * `identifier` - Repository identifier
/// * `timestamp` - Unix timestamp for the archive filename
///
/// # Returns
/// Path to the created archive file
///
/// # Errors
/// Returns `ArchiveError` if:
/// - Repository directory doesn't exist
/// - Archive directory cannot be created
/// - Archive file cannot be written
/// - Compression fails
pub fn archive_repository(
repo_path: &Path,
archive_base_path: &Path,
npub: &str,
identifier: &str,
timestamp: u64,
) -> Result<PathBuf, ArchiveError> {
// Validate repository exists
if !repo_path.exists() {
return Err(ArchiveError::RepoNotFound(repo_path.display().to_string()));
}
debug!(
"Archiving repository {} to archive base {}",
repo_path.display(),
archive_base_path.display()
);
// Create archive directory structure: .archive/<npub>/
let archive_dir = archive_base_path.join(npub);
fs::create_dir_all(&archive_dir).map_err(|e| {
ArchiveError::ArchiveDirCreation(format!(
"Failed to create {}: {}",
archive_dir.display(),
e
))
})?;
// Create archive filename: <identifier>-<timestamp>.tar.gz
let archive_filename = format!("{}-{}.tar.gz", identifier, timestamp);
let archive_path = archive_dir.join(&archive_filename);
debug!("Creating archive at {}", archive_path.display());
// Create tar.gz archive
let tar_file = File::create(&archive_path)?;
let encoder = GzEncoder::new(tar_file, Compression::default());
let mut tar_builder = Builder::new(encoder);
// Add the entire repository directory to the archive
// Use the repository name as the base directory in the archive
let repo_name = repo_path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("repository");
tar_builder.append_dir_all(repo_name, repo_path)?;
// Finish writing the archive
let encoder = tar_builder.into_inner()?;
encoder.finish()?;
info!(
"Successfully archived repository {} to {}",
repo_path.display(),
archive_path.display()
);
Ok(archive_path)
}
/// Create metadata for an archived repository
///
/// Generates metadata JSON file alongside the archive with information needed
/// for retention management and potential restoration.
///
/// # Arguments
/// * `archive_path` - Path to the archive tar.gz file
/// * `deletion_timestamp` - Unix timestamp when deletion was requested
/// * `deletion_event_id` - Event ID that triggered the deletion
/// * `maintainer_pubkey` - Maintainer's public key in npub format
/// * `identifier` - Repository identifier
/// * `retention_secs` - Retention period in seconds
///
/// # Returns
/// Path to the created metadata JSON file
///
/// # Errors
/// Returns `ArchiveError` if:
/// - Metadata file cannot be written
/// - JSON serialization fails
pub fn create_archive_metadata(
archive_path: &Path,
deletion_timestamp: u64,
deletion_event_id: String,
maintainer_pubkey: String,
identifier: String,
retention_secs: u64,
) -> Result<PathBuf, ArchiveError> {
let expiry_timestamp = deletion_timestamp + retention_secs;
// Get relative path for archive_file_path
// If archive_path is /data/.archive/npub/repo-123.tar.gz
// We want to store .archive/npub/repo-123.tar.gz
let archive_file_path = archive_path
.components()
.skip_while(|c| {
// Skip components until we find .archive
!c.as_os_str().to_string_lossy().contains(".archive")
})
.collect::<PathBuf>()
.display()
.to_string();
let metadata = ArchiveMetadata {
deletion_timestamp,
deletion_event_id,
maintainer_pubkey,
identifier,
archive_file_path,
expiry_timestamp,
};
// Create metadata file path: <archive>.json
let metadata_path = archive_path.with_extension("tar.gz.json");
debug!("Creating metadata at {}", metadata_path.display());
// Write metadata as JSON
let json = serde_json::to_string_pretty(&metadata)?;
let mut file = File::create(&metadata_path)?;
file.write_all(json.as_bytes())?;
info!(
"Successfully created metadata at {}",
metadata_path.display()
);
Ok(metadata_path)
}
#[cfg(test)]
mod tests {
use super::*;
use flate2::read::GzDecoder;
use std::fs;
use std::process::Command;
use tar::Archive;
use tempfile::TempDir;
/// Create a test git repository with a commit
fn create_test_git_repo() -> (TempDir, PathBuf) {
let temp_dir = TempDir::new().unwrap();
let repo_path = temp_dir.path().join("test-repo.git");
// Initialize bare repository
Command::new("git")
.args(["init", "--bare", repo_path.to_str().unwrap()])
.output()
.unwrap();
// Create a working directory to make a commit
let work_dir = temp_dir.path().join("work");
Command::new("git")
.args([
"clone",
repo_path.to_str().unwrap(),
work_dir.to_str().unwrap(),
])
.output()
.unwrap();
// Configure git
Command::new("git")
.args(["config", "user.email", "test@test.com"])
.current_dir(&work_dir)
.output()
.unwrap();
Command::new("git")
.args(["config", "user.name", "Test User"])
.current_dir(&work_dir)
.output()
.unwrap();
Command::new("git")
.args(["config", "commit.gpgsign", "false"])
.current_dir(&work_dir)
.output()
.unwrap();
// Create a file and commit
fs::write(work_dir.join("README.md"), "# Test Repository").unwrap();
Command::new("git")
.args(["add", "README.md"])
.current_dir(&work_dir)
.output()
.unwrap();
Command::new("git")
.args(["commit", "-m", "Initial commit"])
.current_dir(&work_dir)
.output()
.unwrap();
// Push to bare repo
Command::new("git")
.args(["push", "origin", "master"])
.current_dir(&work_dir)
.output()
.unwrap();
(temp_dir, repo_path)
}
#[test]
fn test_archive_repository_creates_tarball() {
let (_temp_dir, repo_path) = create_test_git_repo();
let archive_base = _temp_dir.path().join(".archive");
let npub = "npub1test123";
let identifier = "test-repo";
let timestamp = 1234567890;
let archive_path =
archive_repository(&repo_path, &archive_base, npub, identifier, timestamp).unwrap();
// Verify archive was created
assert!(archive_path.exists());
assert_eq!(
archive_path,
archive_base
.join(npub)
.join(format!("{}-{}.tar.gz", identifier, timestamp))
);
// Verify it's a valid gzip file
let file = File::open(&archive_path).unwrap();
let decoder = GzDecoder::new(file);
let mut archive = Archive::new(decoder);
// Should be able to list entries
let entries: Vec<_> = archive.entries().unwrap().collect();
assert!(!entries.is_empty(), "Archive should contain entries");
}
#[test]
fn test_archive_repository_compression_works() {
let (_temp_dir, repo_path) = create_test_git_repo();
let archive_base = _temp_dir.path().join(".archive");
let npub = "npub1test123";
let identifier = "test-repo";
let timestamp = 1234567890;
let archive_path =
archive_repository(&repo_path, &archive_base, npub, identifier, timestamp).unwrap();
// Verify compression by checking file is smaller than uncompressed
let archive_size = fs::metadata(&archive_path).unwrap().len();
// Get size of original repo (rough estimate)
let repo_size: u64 = walkdir::WalkDir::new(&repo_path)
.into_iter()
.filter_map(|e| e.ok())
.filter(|e| e.file_type().is_file())
.filter_map(|e| fs::metadata(e.path()).ok())
.map(|m| m.len())
.sum();
// Archive should be smaller due to compression
// (This is a rough check - actual compression ratio varies)
assert!(archive_size > 0, "Archive should have non-zero size");
assert!(repo_size > 0, "Repo should have non-zero size");
}
#[test]
fn test_archive_repository_nonexistent_repo() {
let temp_dir = TempDir::new().unwrap();
let nonexistent_repo = temp_dir.path().join("nonexistent.git");
let archive_base = temp_dir.path().join(".archive");
let result = archive_repository(&nonexistent_repo, &archive_base, "npub1test", "test", 123);
assert!(result.is_err());
match result {
Err(ArchiveError::RepoNotFound(_)) => (),
_ => panic!("Expected RepoNotFound error"),
}
}
#[test]
fn test_archive_extraction_integrity() {
let (_temp_dir, repo_path) = create_test_git_repo();
let archive_base = _temp_dir.path().join(".archive");
let npub = "npub1test123";
let identifier = "test-repo";
let timestamp = 1234567890;
let archive_path =
archive_repository(&repo_path, &archive_base, npub, identifier, timestamp).unwrap();
// Extract archive to verify integrity
let extract_dir = _temp_dir.path().join("extracted");
fs::create_dir_all(&extract_dir).unwrap();
let file = File::open(&archive_path).unwrap();
let decoder = GzDecoder::new(file);
let mut archive = Archive::new(decoder);
archive.unpack(&extract_dir).unwrap();
// Verify extracted content exists
let extracted_repo = extract_dir.join("test-repo.git");
assert!(extracted_repo.exists(), "Extracted repository should exist");
// Verify it's a valid git repository
let output = Command::new("git")
.args(["rev-parse", "--git-dir"])
.current_dir(&extracted_repo)
.output()
.unwrap();
assert!(
output.status.success(),
"Extracted directory should be a valid git repo"
);
}
#[test]
fn test_create_archive_metadata() {
let temp_dir = TempDir::new().unwrap();
let archive_path = temp_dir.path().join(".archive/npub1test/repo-123.tar.gz");
fs::create_dir_all(archive_path.parent().unwrap()).unwrap();
File::create(&archive_path).unwrap();
let deletion_timestamp = 1234567890;
let deletion_event_id = "event123".to_string();
let maintainer_pubkey = "npub1test".to_string();
let identifier = "repo".to_string();
let retention_secs = 2592000; // 30 days
let metadata_path = create_archive_metadata(
&archive_path,
deletion_timestamp,
deletion_event_id.clone(),
maintainer_pubkey.clone(),
identifier.clone(),
retention_secs,
)
.unwrap();
// Verify metadata file was created
assert!(metadata_path.exists());
assert_eq!(metadata_path, archive_path.with_extension("tar.gz.json"));
// Verify metadata content
let json = fs::read_to_string(&metadata_path).unwrap();
let metadata: ArchiveMetadata = serde_json::from_str(&json).unwrap();
assert_eq!(metadata.deletion_timestamp, deletion_timestamp);
assert_eq!(metadata.deletion_event_id, deletion_event_id);
assert_eq!(metadata.maintainer_pubkey, maintainer_pubkey);
assert_eq!(metadata.identifier, identifier);
assert_eq!(
metadata.expiry_timestamp,
deletion_timestamp + retention_secs
);
assert!(metadata.archive_file_path.contains(".archive"));
}
#[test]
fn test_metadata_serialization_roundtrip() {
let metadata = ArchiveMetadata {
deletion_timestamp: 1234567890,
deletion_event_id: "event123".to_string(),
maintainer_pubkey: "npub1test".to_string(),
identifier: "test-repo".to_string(),
archive_file_path: ".archive/npub1test/test-repo-1234567890.tar.gz".to_string(),
expiry_timestamp: 1234567890 + 2592000,
};
// Serialize
let json = serde_json::to_string(&metadata).unwrap();
// Deserialize
let deserialized: ArchiveMetadata = serde_json::from_str(&json).unwrap();
// Verify roundtrip
assert_eq!(metadata, deserialized);
}
#[test]
fn test_archive_directory_creation() {
let temp_dir = TempDir::new().unwrap();
let (_repo_temp, repo_path) = create_test_git_repo();
let archive_base = temp_dir.path().join(".archive");
let npub = "npub1test123";
// Archive base doesn't exist yet
assert!(!archive_base.exists());
archive_repository(&repo_path, &archive_base, npub, "test-repo", 123).unwrap();
// Verify directory structure was created
assert!(archive_base.exists());
assert!(archive_base.join(npub).exists());
}
#[test]
fn test_archive_error_handling_permission_denied() {
// This test would require setting up permission-denied scenarios
// which is platform-specific and may require elevated privileges.
// Skipping for now, but documenting the expected behavior:
// - If archive directory cannot be created due to permissions,
// should return ArchiveError::ArchiveDirCreation
// - If archive file cannot be written due to permissions,
// should return ArchiveError::Io
}
}
src/git/mod.rs
+1
View File
@@ -17,6 +17,7 @@
//! - `POST /<npub>/<identifier>.git/git-upload-pack` - Clone/fetch operation
//! - `POST /<npub>/<identifier>.git/git-receive-pack` - Push operation
pub mod archive;
pub mod authorization;
pub mod handlers;
pub mod process;