Add repository recovery detection and git restoration

Implements Phase 4A of NIP-09 deletion request support:
- Add check_for_recovery() to detect recoverable repos in holding database
- Add restore_repository() to extract and restore git data from archives
- Integrate recovery detection into announcement processing
- Query holding database when new announcements arrive
- Restore archived git data within retention period
- Add comprehensive tests for all recovery scenarios
This commit is contained in:
DanConwayDev
2026-01-14 15:06:32 +00:00
parent 7dd25cd41f
commit 5b6cc95a19
2 changed files with 735 additions and 2 deletions
+666 -2
View File
@@ -3,14 +3,15 @@
//! This module provides functionality for archiving git repositories before deletion.
//! Archives are created as tar.gz files with associated metadata for retention management.
use flate2::read::GzDecoder;
use flate2::write::GzEncoder;
use flate2::Compression;
use serde::{Deserialize, Serialize};
use std::fs::{self, File};
use std::io::{self, Write};
use std::path::{Path, PathBuf};
use tar::Builder;
use tracing::{debug, info};
use tar::{Archive, Builder};
use tracing::{debug, info, warn};
/// Metadata for an archived repository
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
@@ -43,6 +44,18 @@ pub enum ArchiveError {
#[error("Archive directory creation failed: {0}")]
ArchiveDirCreation(String),
#[error("Archive file not found: {0}")]
ArchiveNotFound(String),
#[error("Archive is corrupt or invalid: {0}")]
CorruptArchive(String),
#[error("Invalid git repository in archive: {0}")]
InvalidGitData(String),
#[error("Database query error: {0}")]
DatabaseError(String),
}
/// Archive a git repository to a tar.gz file
@@ -312,6 +325,339 @@ pub fn cleanup_after_recovery(
})
}
/// Information about a recoverable repository
#[derive(Debug, Clone)]
pub struct RecoveryInfo {
/// Path to the archived tar.gz file
pub archive_path: PathBuf,
/// Events that should be restored from holding database
pub events: Vec<nostr_relay_builder::prelude::Event>,
/// Deletion metadata from the holding database
pub deletion_metadata: Vec<crate::database::holding::DeletionMetadata>,
}
/// Check if a repository can be recovered from the holding database
///
/// Queries the holding database for events matching the given npub and identifier
/// that are still within the retention period.
///
/// # Arguments
/// * `holding_database` - The holding database to query
/// * `npub` - Maintainer's public key in npub format
/// * `identifier` - Repository identifier
/// * `current_timestamp` - Current unix timestamp for checking expiry
///
/// # Returns
/// `Some(RecoveryInfo)` if recovery is possible, `None` if no recoverable events found
///
/// # Errors
/// Returns `ArchiveError` if the database query fails
pub async fn check_for_recovery(
holding_database: &crate::database::holding::HoldingDatabase,
npub: &str,
identifier: &str,
current_timestamp: u64,
) -> Result<Option<RecoveryInfo>, ArchiveError> {
use nostr_relay_builder::prelude::{Filter, PublicKey};
debug!(
"Checking for recovery: npub={}, identifier={}",
npub, identifier
);
// Parse npub to get the public key
let pubkey = PublicKey::parse(npub).map_err(|e| {
ArchiveError::DatabaseError(format!("Failed to parse npub {}: {}", npub, e))
})?;
// Query for kind 30617 events (repository announcements) from this author
// with the matching d-tag (identifier)
let filter = Filter::new()
.kind(nostr_relay_builder::prelude::Kind::from(30617))
.author(pubkey)
.custom_tag(
nostr_relay_builder::prelude::SingleLetterTag::lowercase(
nostr_relay_builder::prelude::Alphabet::D,
),
identifier.to_string(),
);
let events = holding_database.query_events(filter).await.map_err(|e| {
ArchiveError::DatabaseError(format!("Failed to query holding database: {}", e))
})?;
if events.is_empty() {
debug!(
"No events found in holding database for {}/{}",
npub, identifier
);
return Ok(None);
}
// Extract deletion metadata and filter by expiry
let mut valid_events = Vec::new();
let mut deletion_metadata_list = Vec::new();
let mut archive_path: Option<PathBuf> = None;
for event in events {
// Extract metadata from tags
let mut deletion_ts = None;
let mut deletion_event_id = None;
let mut expiry_ts = None;
let mut event_archive_path = None;
for tag in event.tags.iter() {
let tag_vec = tag.clone().to_vec();
if tag_vec.len() >= 2 {
match tag_vec[0].as_str() {
"deletion-ts" => {
deletion_ts = tag_vec[1].parse::<u64>().ok();
}
"deletion-event" => {
deletion_event_id =
nostr_relay_builder::prelude::EventId::parse(&tag_vec[1]).ok();
}
"expiry-ts" => {
expiry_ts = tag_vec[1].parse::<u64>().ok();
}
"archive-path" => {
event_archive_path = Some(tag_vec[1].clone());
}
_ => {}
}
}
}
// Check if event is still within retention period
if let Some(expiry) = expiry_ts {
if expiry > current_timestamp {
// Event is still valid for recovery
if let (Some(del_ts), Some(del_event_id)) = (deletion_ts, deletion_event_id) {
valid_events.push(event);
deletion_metadata_list.push(crate::database::holding::DeletionMetadata {
deletion_timestamp: del_ts,
deletion_event_id: del_event_id,
expiry_timestamp: expiry,
archive_path: event_archive_path.clone(),
});
// Store the archive path (should be the same for all events from this repo)
if archive_path.is_none() {
if let Some(path_str) = event_archive_path {
archive_path = Some(PathBuf::from(path_str));
}
}
}
} else {
debug!(
"Event {} has expired (expiry: {}, current: {})",
event.id, expiry, current_timestamp
);
}
}
}
if valid_events.is_empty() {
debug!(
"No valid recoverable events found for {}/{} (all expired)",
npub, identifier
);
return Ok(None);
}
// We need an archive path to recover git data
let archive_path = archive_path.ok_or_else(|| {
ArchiveError::DatabaseError(
"Events found in holding database but no archive path stored".to_string(),
)
})?;
info!(
"Found {} recoverable events for {}/{} with archive at {}",
valid_events.len(),
npub,
identifier,
archive_path.display()
);
Ok(Some(RecoveryInfo {
archive_path,
events: valid_events,
deletion_metadata: deletion_metadata_list,
}))
}
/// Restore a git repository from an archive
///
/// Extracts the tar.gz archive to a temporary directory, verifies it's a valid
/// git repository, and moves it to the target location.
///
/// # Arguments
/// * `archive_path` - Path to the archive tar.gz file
/// * `git_data_path` - Base path for git repositories
/// * `npub` - Maintainer's public key in npub format
/// * `identifier` - Repository identifier
///
/// # Returns
/// `Ok(())` if restoration is successful
///
/// # Errors
/// Returns `ArchiveError` if:
/// - Archive file doesn't exist
/// - Archive is corrupt or cannot be extracted
/// - Extracted data is not a valid git repository
/// - Target directory cannot be created
/// - Files cannot be moved to target location
pub fn restore_repository(
archive_path: &Path,
git_data_path: &Path,
npub: &str,
identifier: &str,
) -> Result<(), ArchiveError> {
// Verify archive exists
if !archive_path.exists() {
return Err(ArchiveError::ArchiveNotFound(
archive_path.display().to_string(),
));
}
info!(
"Restoring repository {}/{} from archive {}",
npub,
identifier,
archive_path.display()
);
// Create a temporary directory for extraction
let temp_dir = std::env::temp_dir().join(format!(
"ngit-restore-{}-{}-{}",
npub,
identifier,
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_nanos()
));
fs::create_dir_all(&temp_dir)?;
debug!("Extracting archive to {}", temp_dir.display());
// Extract the archive
let tar_file = File::open(archive_path)?;
let decoder = GzDecoder::new(tar_file);
let mut archive = Archive::new(decoder);
archive
.unpack(&temp_dir)
.map_err(|e| ArchiveError::CorruptArchive(format!("Failed to extract archive: {}", e)))?;
// Find the git repository in the extracted directory
// The archive contains a directory with the repo name (e.g., "repo.git")
let extracted_entries: Vec<_> = fs::read_dir(&temp_dir)
.map_err(|e| {
ArchiveError::InvalidGitData(format!("Failed to read extracted directory: {}", e))
})?
.filter_map(|entry| entry.ok())
.collect();
if extracted_entries.is_empty() {
return Err(ArchiveError::InvalidGitData("Archive is empty".to_string()));
}
// The first entry should be the repository directory
let extracted_repo = extracted_entries[0].path();
if !extracted_repo.is_dir() {
return Err(ArchiveError::InvalidGitData(
"Archive does not contain a directory".to_string(),
));
}
debug!("Found extracted repository at {}", extracted_repo.display());
// Verify it's a valid git repository
// Check for either .git directory (non-bare) or git objects (bare repo)
let is_bare = extracted_repo.join("objects").exists() && extracted_repo.join("refs").exists();
let is_non_bare = extracted_repo.join(".git").exists();
if !is_bare && !is_non_bare {
return Err(ArchiveError::InvalidGitData(
"Archive does not contain a valid git repository (missing .git directory or bare repo structure)"
.to_string(),
));
}
debug!(
"Verified git repository (bare: {}, non-bare: {})",
is_bare, is_non_bare
);
// Create target directory structure: <git_data_path>/<npub>/
let target_dir = git_data_path.join(npub);
fs::create_dir_all(&target_dir).map_err(|e| {
ArchiveError::ArchiveDirCreation(format!(
"Failed to create target directory {}: {}",
target_dir.display(),
e
))
})?;
// Target path: <git_data_path>/<npub>/<identifier>.git
let target_repo = target_dir.join(format!("{}.git", identifier));
// If target already exists, warn and remove it
if target_repo.exists() {
warn!(
"Target repository {} already exists, removing it for restoration",
target_repo.display()
);
fs::remove_dir_all(&target_repo).map_err(|e| {
ArchiveError::Io(io::Error::new(
io::ErrorKind::Other,
format!("Failed to remove existing repository: {}", e),
))
})?;
}
// Move the extracted repository to the target location
debug!(
"Moving repository from {} to {}",
extracted_repo.display(),
target_repo.display()
);
fs::rename(&extracted_repo, &target_repo).map_err(|e| {
ArchiveError::Io(io::Error::new(
io::ErrorKind::Other,
format!(
"Failed to move repository from {} to {}: {}",
extracted_repo.display(),
target_repo.display(),
e
),
))
})?;
info!(
"Successfully restored repository {}/{} to {}",
npub,
identifier,
target_repo.display()
);
// Clean up temporary directory
if let Err(e) = fs::remove_dir_all(&temp_dir) {
warn!(
"Failed to remove temporary directory {}: {}",
temp_dir.display(),
e
);
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
@@ -735,4 +1081,322 @@ mod tests {
assert_eq!(stats.bytes_reclaimed, 1024 + 256);
}
// Recovery detection and restoration tests
#[tokio::test]
async fn test_check_for_recovery_no_events() {
use crate::config::DatabaseBackend;
use crate::database::holding::HoldingDatabase;
use nostr_relay_builder::prelude::{Keys, ToBech32};
use std::time::{SystemTime, UNIX_EPOCH};
let temp_dir = TempDir::new().unwrap();
let holding_db = HoldingDatabase::new(temp_dir.path(), DatabaseBackend::Memory)
.await
.unwrap();
let current_ts = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs();
// Generate a valid npub for testing
let keys = Keys::generate();
let npub = keys.public_key().to_bech32().unwrap();
// Query for non-existent repository
let result = check_for_recovery(&holding_db, &npub, "nonexistent", current_ts)
.await
.unwrap();
assert!(result.is_none(), "Should return None for non-existent repo");
}
#[tokio::test]
async fn test_check_for_recovery_valid_events() {
use crate::config::DatabaseBackend;
use crate::database::holding::{DeletionMetadata, HoldingDatabase};
use nostr_relay_builder::prelude::{EventBuilder, Keys, Kind, Tag, TagKind, ToBech32};
use std::time::{SystemTime, UNIX_EPOCH};
let temp_dir = TempDir::new().unwrap();
let holding_db = HoldingDatabase::new(temp_dir.path(), DatabaseBackend::Memory)
.await
.unwrap();
let keys = Keys::generate();
let npub = keys.public_key().to_bech32().unwrap();
let identifier = "test-repo";
let current_ts = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs();
// Create a repository announcement event
let event = EventBuilder::new(Kind::from(30617), "Test repository")
.tag(Tag::custom(
TagKind::custom("d"),
vec![identifier.to_string()],
))
.tag(Tag::custom(
TagKind::custom("clone"),
vec!["https://example.com/repo.git".to_string()],
))
.sign_with_keys(&keys)
.unwrap();
// Create deletion metadata with 1 hour retention
let deletion_keys = Keys::generate();
let deletion_event_id = EventBuilder::text_note("deletion")
.sign_with_keys(&deletion_keys)
.unwrap()
.id;
let metadata = DeletionMetadata {
deletion_timestamp: current_ts,
deletion_event_id,
expiry_timestamp: current_ts + 3600,
archive_path: Some(".archive/npub1test/test-repo-123.tar.gz".to_string()),
};
// Store in holding database
holding_db.store_event(&event, metadata).await.unwrap();
// Check for recovery
let result = check_for_recovery(&holding_db, &npub, identifier, current_ts)
.await
.unwrap();
assert!(result.is_some(), "Should find recoverable events");
let recovery_info = result.unwrap();
assert_eq!(recovery_info.events.len(), 1);
assert_eq!(recovery_info.deletion_metadata.len(), 1);
assert!(recovery_info
.archive_path
.to_string_lossy()
.contains("test-repo"));
}
#[tokio::test]
async fn test_check_for_recovery_expired_events() {
use crate::config::DatabaseBackend;
use crate::database::holding::{DeletionMetadata, HoldingDatabase};
use nostr_relay_builder::prelude::{EventBuilder, Keys, Kind, Tag, TagKind, ToBech32};
use std::time::{SystemTime, UNIX_EPOCH};
let temp_dir = TempDir::new().unwrap();
let holding_db = HoldingDatabase::new(temp_dir.path(), DatabaseBackend::Memory)
.await
.unwrap();
let keys = Keys::generate();
let npub = keys.public_key().to_bech32().unwrap();
let identifier = "expired-repo";
let current_ts = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs();
// Create a repository announcement event
let event = EventBuilder::new(Kind::from(30617), "Expired repository")
.tag(Tag::custom(
TagKind::custom("d"),
vec![identifier.to_string()],
))
.tag(Tag::custom(
TagKind::custom("clone"),
vec!["https://example.com/repo.git".to_string()],
))
.sign_with_keys(&keys)
.unwrap();
// Create deletion metadata that expired 1 hour ago
let deletion_keys = Keys::generate();
let deletion_event_id = EventBuilder::text_note("deletion")
.sign_with_keys(&deletion_keys)
.unwrap()
.id;
let metadata = DeletionMetadata {
deletion_timestamp: current_ts - 7200,
deletion_event_id,
expiry_timestamp: current_ts - 3600, // Expired
archive_path: Some(".archive/npub1test/expired-repo-123.tar.gz".to_string()),
};
// Store in holding database
holding_db.store_event(&event, metadata).await.unwrap();
// Check for recovery - should return None because event is expired
let result = check_for_recovery(&holding_db, &npub, identifier, current_ts)
.await
.unwrap();
assert!(
result.is_none(),
"Should return None for expired events (all past retention)"
);
}
#[test]
fn test_restore_repository_missing_archive() {
let temp_dir = TempDir::new().unwrap();
let nonexistent_archive = temp_dir.path().join("nonexistent.tar.gz");
let git_data_path = temp_dir.path().join("git");
let result = restore_repository(
&nonexistent_archive,
&git_data_path,
"npub1test",
"test-repo",
);
assert!(result.is_err());
match result {
Err(ArchiveError::ArchiveNotFound(_)) => (),
_ => panic!("Expected ArchiveNotFound error"),
}
}
#[test]
fn test_restore_repository_corrupt_archive() {
let temp_dir = TempDir::new().unwrap();
// Create a corrupt archive (just empty file)
let corrupt_archive = temp_dir.path().join("corrupt.tar.gz");
File::create(&corrupt_archive).unwrap();
let git_data_path = temp_dir.path().join("git");
let result = restore_repository(&corrupt_archive, &git_data_path, "npub1test", "test-repo");
assert!(result.is_err());
match result {
Err(ArchiveError::CorruptArchive(_)) => (),
_ => panic!("Expected CorruptArchive error"),
}
}
#[test]
fn test_restore_repository_invalid_git_data() {
let temp_dir = TempDir::new().unwrap();
// Create an archive with non-git content
let archive_path = temp_dir.path().join("invalid.tar.gz");
let tar_file = File::create(&archive_path).unwrap();
let encoder = GzEncoder::new(tar_file, Compression::default());
let mut tar_builder = Builder::new(encoder);
// Add a directory without git structure
let fake_repo = temp_dir.path().join("fake-repo.git");
fs::create_dir_all(&fake_repo).unwrap();
fs::write(fake_repo.join("README.md"), "not a git repo").unwrap();
tar_builder
.append_dir_all("fake-repo.git", &fake_repo)
.unwrap();
let encoder = tar_builder.into_inner().unwrap();
encoder.finish().unwrap();
let git_data_path = temp_dir.path().join("git");
let result = restore_repository(&archive_path, &git_data_path, "npub1test", "test-repo");
assert!(result.is_err());
match result {
Err(ArchiveError::InvalidGitData(_)) => (),
_ => panic!("Expected InvalidGitData error"),
}
}
#[test]
fn test_restore_repository_success() {
let temp_dir = TempDir::new().unwrap();
// Create a real git repository and archive it
let (_repo_temp, repo_path) = create_test_git_repo();
let archive_base = temp_dir.path().join("archives");
let npub = "npub1test123";
let identifier = "test-repo";
let timestamp = 1234567890;
let archive_path =
archive_repository(&repo_path, &archive_base, npub, identifier, timestamp).unwrap();
// Now try to restore it
let git_data_path = temp_dir.path().join("git");
let result = restore_repository(&archive_path, &git_data_path, npub, identifier);
assert!(result.is_ok(), "Restoration should succeed");
// Verify the repository was restored to the correct location
let restored_repo = git_data_path.join(npub).join(format!("{}.git", identifier));
assert!(
restored_repo.exists(),
"Restored repository should exist at expected path"
);
// Verify it's a valid git repository
let output = Command::new("git")
.args(["rev-parse", "--git-dir"])
.current_dir(&restored_repo)
.output()
.unwrap();
assert!(
output.status.success(),
"Restored repository should be a valid git repo"
);
}
#[test]
fn test_restore_repository_overwrites_existing() {
let temp_dir = TempDir::new().unwrap();
// Create a git repository and archive it
let (_repo_temp, repo_path) = create_test_git_repo();
let archive_base = temp_dir.path().join("archives");
let npub = "npub1test123";
let identifier = "test-repo";
let timestamp = 1234567890;
let archive_path =
archive_repository(&repo_path, &archive_base, npub, identifier, timestamp).unwrap();
let git_data_path = temp_dir.path().join("git");
// Create an existing repository at the target location
let target_repo = git_data_path.join(npub).join(format!("{}.git", identifier));
fs::create_dir_all(&target_repo).unwrap();
fs::write(target_repo.join("marker.txt"), "existing").unwrap();
// Restore should overwrite
let result = restore_repository(&archive_path, &git_data_path, npub, identifier);
assert!(
result.is_ok(),
"Restoration should succeed even with existing repo"
);
// Verify the marker file is gone (overwritten)
assert!(
!target_repo.join("marker.txt").exists(),
"Old marker file should be removed"
);
// Verify it's a valid git repository
let output = Command::new("git")
.args(["rev-parse", "--git-dir"])
.current_dir(&target_repo)
.output()
.unwrap();
assert!(
output.status.success(),
"Restored repository should be a valid git repo"
);
}
}
+69
View File
@@ -114,6 +114,75 @@ impl Nip34WritePolicy {
// Parse announcement to get repository details
match RepositoryAnnouncement::from_event(event.clone()) {
Ok(announcement) => {
// Check for recovery from holding database
if let Some(ref holding_db) = self.holding_database {
let current_ts = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
match crate::git::archive::check_for_recovery(
holding_db,
&announcement.owner_npub(),
&announcement.identifier,
current_ts,
)
.await
{
Ok(Some(recovery_info)) => {
tracing::info!(
"Recovery detected for {}/{}: {} events can be restored from archive {}",
announcement.owner_npub(),
announcement.identifier,
recovery_info.events.len(),
recovery_info.archive_path.display()
);
// Restore git data from archive
let git_data_path = self.ctx.git_data_path.as_path();
match crate::git::archive::restore_repository(
&recovery_info.archive_path,
git_data_path,
&announcement.owner_npub(),
&announcement.identifier,
) {
Ok(()) => {
tracing::info!(
"Successfully restored git data for {}/{}",
announcement.owner_npub(),
announcement.identifier
);
// Note: Event restoration will be handled in Phase 4B
}
Err(e) => {
tracing::warn!(
"Failed to restore git data for {}/{}: {}",
announcement.owner_npub(),
announcement.identifier,
e
);
// Continue with normal processing even if restoration fails
}
}
}
Ok(None) => {
tracing::debug!(
"No recovery available for {}/{} (not in holding database or expired)",
announcement.owner_npub(),
announcement.identifier
);
}
Err(e) => {
tracing::warn!(
"Error checking for recovery for {}/{}: {}",
announcement.owner_npub(),
announcement.identifier,
e
);
}
}
}
// Try to create bare repository if it doesn't exist
if let Err(e) = self
.announcement_policy