mirror of
https://relay.ngit.dev/npub15qydau2hjma6ngxkl2cyar74wzyjshvl65za5k5rl69264ar2exs5cyejr/ngit-grasp.git
synced 2026-10-05 15:08:24 +00:00
DanConwayDev
25c535f96a
A primary and its delayed hedge could remain silent indefinitely, while cancellation killed only the direct Git child. In either case helpers could retain repository resources after admission guards returned and prevent alternative sources from making progress. Keep a synchronous process-group guard armed from spawn through completion. Cancellation kills the group before Rust releases repository and domain guards. Five minutes without output initiates explicit SIGTERM cleanup; the unreaped leader anchors the group identity through a ten-second grace period before SIGKILL and wait. An unproductive pair then tries remaining URLs sequentially. Normal completion disarms immediately when Git has joined its helpers, avoiding any signal after the leader is reaped and its PGID could be reused. Continuously active large transfers remain unbounded; activity is not source scoring. Validated by focused tests proving active output survives, inactivity forcibly reaps a TERM-resistant descendant, cancellation reaps descendants before follow-up work, a failed pair reaches a third source, and git diff --check passes.
1570 lines
53 KiB
Rust
1570 lines
53 KiB
Rust
//! Core sync functions for identifier-based purgatory synchronization.
|
|
//!
|
|
//! This module provides the two main functions that both the main sync loop
|
|
//! and `DomainThrottle` queue processing use:
|
|
//!
|
|
//! - [`sync_identifier_next_url`]: Pure URL selection logic - finds next URL to try
|
|
//! - [`sync_identifier_from_url`]: Pure fetch logic - fetches from a specific URL
|
|
//!
|
|
//! The separation enables:
|
|
//! - Main sync loop to try non-throttled URLs immediately
|
|
//! - DomainThrottle to process queued identifiers when capacity frees
|
|
//! - Clean testability with mocked SyncContext
|
|
|
|
use std::collections::{HashMap, HashSet};
|
|
use std::sync::Arc;
|
|
use tracing::debug;
|
|
|
|
use super::context::{GitFetchRole, SyncContext};
|
|
use super::throttle::ThrottleManager;
|
|
use crate::sync::naughty_list::NaughtyListTracker;
|
|
|
|
/// A primary gets a substantial head start before one alternative server is
|
|
/// allowed to duplicate the same object demand. This is a hedge trigger, not
|
|
/// a timeout: neither legitimate long-running fetch is killed.
|
|
const REPOSITORY_FETCH_HEDGE_DELAY: std::time::Duration = std::time::Duration::from_secs(30);
|
|
|
|
#[derive(Default)]
|
|
struct RepositoryFetchOutcome {
|
|
fetched: usize,
|
|
attempted_urls: Vec<String>,
|
|
}
|
|
|
|
/// Extract domain from a URL.
|
|
///
|
|
/// Supports HTTP(S) URLs. SSH URLs (git@...) are not supported.
|
|
///
|
|
/// # Examples
|
|
///
|
|
/// ```ignore
|
|
/// assert_eq!(extract_domain("https://github.com/foo/bar.git"), Some("github.com".to_string()));
|
|
/// assert_eq!(extract_domain("http://example.com:8080/repo.git"), Some("example.com".to_string()));
|
|
/// assert_eq!(extract_domain("git@github.com:foo/bar.git"), None); // SSH URLs not supported
|
|
/// ```
|
|
pub(crate) fn extract_domain(url: &str) -> Option<String> {
|
|
// Simple URL parsing for HTTP(S) URLs
|
|
// Format: scheme://[user@]host[:port]/path
|
|
let url = url
|
|
.strip_prefix("https://")
|
|
.or_else(|| url.strip_prefix("http://"))?;
|
|
|
|
// Remove user info if present (e.g., "user@host" -> "host")
|
|
let url = url.split('@').next_back()?;
|
|
|
|
// Extract host (before first '/' or ':')
|
|
let host = url.split('/').next()?;
|
|
let host = host.split(':').next()?;
|
|
|
|
if host.is_empty() {
|
|
None
|
|
} else {
|
|
Some(host.to_string())
|
|
}
|
|
}
|
|
|
|
/// Find the next URL to try for an identifier.
|
|
///
|
|
/// This is pure URL selection logic with no side effects. It:
|
|
/// 1. Checks if there are pending events for the identifier
|
|
/// 2. Checks if there are OIDs still needed
|
|
/// 3. Gets repository data and extracts clone URLs
|
|
/// 4. Filters out our own domain and already-tried URLs
|
|
/// 5. Filters out naughty domains (with persistent SSL/DNS errors)
|
|
/// 6. Returns the first non-throttled URL (when `domain` is None)
|
|
/// or a URL from the specified domain (when `domain` is Some)
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `ctx` - The sync context providing repository data and OID information
|
|
/// * `identifier` - The repository identifier (d-tag value)
|
|
/// * `domain` - If Some, only return URLs from this specific domain.
|
|
/// If None, return any non-throttled URL.
|
|
/// * `tried_urls` - URLs that have already been tried (will be skipped)
|
|
/// * `throttle_manager` - Used to check if domains are throttled (when domain is None)
|
|
/// * `git_naughty_list` - Used to filter out domains with persistent errors
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// * `Some(url)` - The next URL to try
|
|
/// * `None` - No suitable URL found (all tried, all throttled, or no URLs available)
|
|
pub async fn sync_identifier_next_url<C: SyncContext + ?Sized>(
|
|
ctx: &C,
|
|
identifier: &str,
|
|
domain: Option<&str>,
|
|
tried_urls: &HashSet<String>,
|
|
throttle_manager: &ThrottleManager,
|
|
git_naughty_list: &NaughtyListTracker,
|
|
) -> Option<String> {
|
|
// 1. Check if we still have pending events
|
|
if !ctx.has_pending_events(identifier) {
|
|
debug!(
|
|
identifier = %identifier,
|
|
"No pending events - skipping URL selection"
|
|
);
|
|
return None;
|
|
}
|
|
|
|
// 2. Collect needed OIDs
|
|
let needed_oids = ctx.collect_needed_oids(identifier);
|
|
if needed_oids.is_empty() {
|
|
debug!(
|
|
identifier = %identifier,
|
|
"No OIDs needed - sync is complete"
|
|
);
|
|
return None;
|
|
}
|
|
|
|
// 3. Get repository data
|
|
let repo_data = match ctx.fetch_repository_data_with_purgatory(identifier).await {
|
|
Ok(data) => data,
|
|
Err(e) => {
|
|
debug!(
|
|
identifier = %identifier,
|
|
error = %e,
|
|
"Failed to fetch repository data"
|
|
);
|
|
return None;
|
|
}
|
|
};
|
|
|
|
// 4. Collect clone URLs from announcements AND PR events in purgatory
|
|
let our_domain = ctx.our_domain();
|
|
|
|
// Get clone URLs from repository announcements
|
|
let announcement_urls: HashSet<String> = repo_data
|
|
.announcements
|
|
.iter()
|
|
.flat_map(|a| a.clone_urls.iter().cloned())
|
|
.collect();
|
|
|
|
// Get clone URLs from PR events in purgatory
|
|
let pr_urls = ctx.collect_pr_clone_urls(identifier);
|
|
|
|
// Merge and filter out our domain (exact host:port match, not substring)
|
|
let all_urls: HashSet<String> = announcement_urls
|
|
.union(&pr_urls)
|
|
.filter(|url| {
|
|
our_domain.is_none_or(|d| !crate::outbound::url_matches_service_domain(url, d))
|
|
})
|
|
.cloned()
|
|
.collect();
|
|
|
|
if all_urls.is_empty() {
|
|
debug!(
|
|
identifier = %identifier,
|
|
"No clone URLs available (after filtering our domain)"
|
|
);
|
|
return None;
|
|
}
|
|
|
|
// 5. Group by domain
|
|
let urls_by_domain: HashMap<String, Vec<String>> =
|
|
all_urls.iter().fold(HashMap::new(), |mut acc, url| {
|
|
if let Some(d) = extract_domain(url) {
|
|
acc.entry(d).or_default().push(url.clone());
|
|
}
|
|
acc
|
|
});
|
|
|
|
// 6. Find an available URL
|
|
match domain {
|
|
Some(specific_domain) => {
|
|
// Only look at URLs from this specific domain
|
|
urls_by_domain
|
|
.get(specific_domain)
|
|
.and_then(|urls| urls.iter().find(|url| !tried_urls.contains(*url)).cloned())
|
|
}
|
|
None => {
|
|
// Try any non-throttled, non-naughty domain
|
|
for (d, domain_urls) in &urls_by_domain {
|
|
if throttle_manager.is_throttled(d) {
|
|
debug!(
|
|
identifier = %identifier,
|
|
domain = %d,
|
|
"Domain is throttled - skipping"
|
|
);
|
|
continue;
|
|
}
|
|
|
|
// NEW: Skip naughty domains
|
|
if git_naughty_list.is_naughty(d) {
|
|
debug!(
|
|
identifier = %identifier,
|
|
domain = %d,
|
|
"Domain is on git naughty list - skipping"
|
|
);
|
|
continue;
|
|
}
|
|
|
|
if let Some(url) = domain_urls.iter().find(|url| !tried_urls.contains(*url)) {
|
|
return Some(url.clone());
|
|
}
|
|
}
|
|
None
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Information about throttled domains with untried URLs.
|
|
///
|
|
/// Used by the main sync loop to know which `DomainThrottle` queues
|
|
/// to add the identifier to when it can't complete immediately.
|
|
#[derive(Debug, Clone)]
|
|
pub struct ThrottledDomainInfo {
|
|
/// The throttled domain name
|
|
pub domain: String,
|
|
/// URLs from this domain that have already been tried
|
|
pub tried_urls_for_domain: HashSet<String>,
|
|
}
|
|
|
|
/// Get information about throttled domains that have untried URLs.
|
|
///
|
|
/// Called by main sync loop to know which `DomainThrottle` queues to add
|
|
/// the identifier to when non-throttled URLs are exhausted.
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `ctx` - The sync context providing repository data
|
|
/// * `identifier` - The repository identifier
|
|
/// * `tried_urls` - All URLs that have been tried (across all domains)
|
|
/// * `throttle_manager` - Used to check which domains are throttled
|
|
/// * `git_naughty_list` - Used to filter out domains with persistent errors
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// A list of throttled domains that still have untried URLs, along with
|
|
/// the tried URLs for each domain (for proper queue state).
|
|
pub async fn get_throttled_domains_with_untried_urls<C: SyncContext + ?Sized>(
|
|
ctx: &C,
|
|
identifier: &str,
|
|
tried_urls: &HashSet<String>,
|
|
throttle_manager: &ThrottleManager,
|
|
git_naughty_list: &NaughtyListTracker,
|
|
) -> Vec<ThrottledDomainInfo> {
|
|
let repo_data = match ctx.fetch_repository_data_with_purgatory(identifier).await {
|
|
Ok(data) => data,
|
|
Err(_) => return vec![],
|
|
};
|
|
|
|
let our_domain = ctx.our_domain();
|
|
|
|
// Get clone URLs from repository announcements
|
|
let announcement_urls: HashSet<String> = repo_data
|
|
.announcements
|
|
.iter()
|
|
.flat_map(|a| a.clone_urls.iter().cloned())
|
|
.collect();
|
|
|
|
// Get clone URLs from PR events in purgatory
|
|
let pr_urls = ctx.collect_pr_clone_urls(identifier);
|
|
|
|
// Merge and filter out our domain (exact host:port match, not substring)
|
|
let all_urls: HashSet<String> = announcement_urls
|
|
.union(&pr_urls)
|
|
.filter(|url| {
|
|
our_domain.is_none_or(|d| !crate::outbound::url_matches_service_domain(url, d))
|
|
})
|
|
.cloned()
|
|
.collect();
|
|
|
|
let urls_by_domain: HashMap<String, Vec<String>> =
|
|
all_urls.iter().fold(HashMap::new(), |mut acc, url| {
|
|
if let Some(d) = extract_domain(url) {
|
|
acc.entry(d).or_default().push(url.clone());
|
|
}
|
|
acc
|
|
});
|
|
|
|
urls_by_domain
|
|
.into_iter()
|
|
.filter_map(|(domain, domain_urls)| {
|
|
if !throttle_manager.is_throttled(&domain) {
|
|
return None; // Not throttled, skip
|
|
}
|
|
|
|
// Skip naughty domains
|
|
if git_naughty_list.is_naughty(&domain) {
|
|
return None; // On naughty list, skip
|
|
}
|
|
|
|
let untried: Vec<_> = domain_urls
|
|
.iter()
|
|
.filter(|url| !tried_urls.contains(*url))
|
|
.collect();
|
|
|
|
if untried.is_empty() {
|
|
return None; // All URLs tried for this domain
|
|
}
|
|
|
|
// Collect tried URLs that belong to this domain
|
|
let tried_urls_for_domain: HashSet<String> = tried_urls
|
|
.iter()
|
|
.filter(|url| extract_domain(url).as_deref() == Some(domain.as_str()))
|
|
.cloned()
|
|
.collect();
|
|
|
|
Some(ThrottledDomainInfo {
|
|
domain,
|
|
tried_urls_for_domain,
|
|
})
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
/// Fetch git data from a specific URL for an identifier.
|
|
///
|
|
/// This function:
|
|
/// 1. Records the request with the throttle manager (for rate limiting)
|
|
/// 2. Performs the actual git fetch via the context
|
|
/// 3. Processes any events that can now be satisfied
|
|
/// 4. Records request completion
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `ctx` - The sync context providing fetch and processing capabilities
|
|
/// * `identifier` - The repository identifier
|
|
/// * `url` - The remote URL to fetch from
|
|
/// * `throttle_manager` - Used to track request start/completion for rate limiting
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// The number of OIDs successfully fetched (0 on failure)
|
|
pub async fn sync_identifier_from_url<C: SyncContext + ?Sized>(
|
|
ctx: &C,
|
|
identifier: &str,
|
|
url: &str,
|
|
throttle_manager: &Arc<ThrottleManager>,
|
|
) -> usize {
|
|
sync_identifier_from_urls(ctx, identifier, vec![url.to_string()], throttle_manager)
|
|
.await
|
|
.fetched
|
|
}
|
|
|
|
async fn fetch_from_url<C: SyncContext + ?Sized>(
|
|
ctx: &C,
|
|
repo: &std::path::Path,
|
|
url: &str,
|
|
needed_oids: &[String],
|
|
throttle_manager: &Arc<ThrottleManager>,
|
|
role: GitFetchRole,
|
|
) -> Vec<String> {
|
|
let domain = match extract_domain(url) {
|
|
Some(d) => d,
|
|
None => return Vec::new(),
|
|
};
|
|
|
|
let _fetch_permit = throttle_manager.acquire_fetch_pass(&domain).await;
|
|
match ctx.fetch_oids_with_role(repo, url, needed_oids, role).await {
|
|
Ok(fetched) => fetched,
|
|
Err(error) => {
|
|
debug!(url = %url, %error, "Fetch failed");
|
|
Vec::new()
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn process_unique_fetched<C: SyncContext + ?Sized>(
|
|
ctx: &C,
|
|
repo: &std::path::Path,
|
|
fetched: Vec<String>,
|
|
processed: &mut HashSet<String>,
|
|
) -> usize {
|
|
let new_oids: HashSet<String> = fetched
|
|
.into_iter()
|
|
.filter(|oid| !processed.contains(oid))
|
|
.collect();
|
|
if new_oids.is_empty() {
|
|
return 0;
|
|
}
|
|
if let Err(error) = ctx.process_newly_available_git_data(repo, &new_oids).await {
|
|
debug!(%error, "Failed to process newly available git data");
|
|
return 0;
|
|
}
|
|
processed.extend(new_oids.iter().cloned());
|
|
new_oids.len()
|
|
}
|
|
|
|
async fn sync_identifier_from_urls<C: SyncContext + ?Sized>(
|
|
ctx: &C,
|
|
identifier: &str,
|
|
mut urls: Vec<String>,
|
|
throttle_manager: &Arc<ThrottleManager>,
|
|
) -> RepositoryFetchOutcome {
|
|
let mut outcome = RepositoryFetchOutcome::default();
|
|
|
|
// Get repository data for target repo path
|
|
let repo_data = match ctx.fetch_repository_data_with_purgatory(identifier).await {
|
|
Ok(data) => data,
|
|
Err(e) => {
|
|
debug!(
|
|
identifier = %identifier,
|
|
error = %e,
|
|
"Failed to fetch repo data"
|
|
);
|
|
return outcome;
|
|
}
|
|
};
|
|
|
|
let target_repo = match ctx.find_target_repo(&repo_data) {
|
|
Some(path) => path,
|
|
None => {
|
|
debug!(identifier = %identifier, "No target repo found");
|
|
return outcome;
|
|
}
|
|
};
|
|
|
|
// The path, rather than the identifier or remote domain, is the local
|
|
// mutation boundary. A waiting caller re-checks demand after the active
|
|
// wave, coalescing duplicate notifications without losing newly arrived
|
|
// OIDs.
|
|
let _repository = throttle_manager.coordinate_repository(&target_repo).await;
|
|
|
|
// Collect needed OIDs
|
|
let needed_oids: Vec<String> = ctx.collect_needed_oids(identifier).into_iter().collect();
|
|
if needed_oids.is_empty() {
|
|
debug!(
|
|
identifier = %identifier,
|
|
"No OIDs needed - nothing to fetch"
|
|
);
|
|
return outcome;
|
|
}
|
|
|
|
urls.retain(|url| extract_domain(url).is_some());
|
|
let Some(primary_url) = urls.first().cloned() else {
|
|
return outcome;
|
|
};
|
|
let primary_domain = extract_domain(&primary_url).unwrap();
|
|
let hedge_url = urls
|
|
.iter()
|
|
.skip(1)
|
|
.find(|url| extract_domain(url).as_deref() != Some(primary_domain.as_str()))
|
|
.cloned();
|
|
|
|
outcome.attempted_urls.push(primary_url.clone());
|
|
let primary = fetch_from_url(
|
|
ctx,
|
|
&target_repo,
|
|
&primary_url,
|
|
&needed_oids,
|
|
throttle_manager,
|
|
GitFetchRole::Primary,
|
|
);
|
|
tokio::pin!(primary);
|
|
let mut processed = HashSet::new();
|
|
|
|
let delay = tokio::time::sleep(REPOSITORY_FETCH_HEDGE_DELAY);
|
|
tokio::pin!(delay);
|
|
tokio::select! {
|
|
biased;
|
|
fetched = &mut primary => {
|
|
outcome.fetched += process_unique_fetched(ctx, &target_repo, fetched, &mut processed).await;
|
|
return outcome;
|
|
}
|
|
_ = &mut delay => {}
|
|
}
|
|
|
|
let Some(hedge_url) = hedge_url else {
|
|
let fetched = primary.await;
|
|
outcome.fetched += process_unique_fetched(ctx, &target_repo, fetched, &mut processed).await;
|
|
return outcome;
|
|
};
|
|
outcome.attempted_urls.push(hedge_url.clone());
|
|
let hedge = fetch_from_url(
|
|
ctx,
|
|
&target_repo,
|
|
&hedge_url,
|
|
&needed_oids,
|
|
throttle_manager,
|
|
GitFetchRole::Hedge,
|
|
);
|
|
tokio::pin!(hedge);
|
|
|
|
tokio::select! {
|
|
biased;
|
|
fetched = &mut primary => {
|
|
outcome.fetched += process_unique_fetched(ctx, &target_repo, fetched, &mut processed).await;
|
|
let fetched = hedge.await;
|
|
outcome.fetched += process_unique_fetched(ctx, &target_repo, fetched, &mut processed).await;
|
|
}
|
|
fetched = &mut hedge => {
|
|
outcome.fetched += process_unique_fetched(ctx, &target_repo, fetched, &mut processed).await;
|
|
let fetched = primary.await;
|
|
outcome.fetched += process_unique_fetched(ctx, &target_repo, fetched, &mut processed).await;
|
|
}
|
|
}
|
|
|
|
// A stalled subprocess is returned as an ordinary failed attempt after
|
|
// its process group has been reaped. If neither member of the bounded
|
|
// two-source wave supplied an object, keep walking the remaining sources
|
|
// sequentially rather than leaving this repository pinned to the pair.
|
|
// No third fetch overlaps the primary and hedge.
|
|
if outcome.fetched == 0 {
|
|
for fallback_url in urls.into_iter().skip(1) {
|
|
if fallback_url == hedge_url {
|
|
continue;
|
|
}
|
|
outcome.attempted_urls.push(fallback_url.clone());
|
|
let fetched = fetch_from_url(
|
|
ctx,
|
|
&target_repo,
|
|
&fallback_url,
|
|
&needed_oids,
|
|
throttle_manager,
|
|
GitFetchRole::Primary,
|
|
)
|
|
.await;
|
|
outcome.fetched +=
|
|
process_unique_fetched(ctx, &target_repo, fetched, &mut processed).await;
|
|
if outcome.fetched > 0 {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
outcome
|
|
}
|
|
|
|
/// Sync git data for an identifier.
|
|
///
|
|
/// This is the main orchestration function called by the sync loop. It:
|
|
/// 1. Tries all non-throttled, non-naughty URLs in sequence
|
|
/// 2. After each fetch, checks if sync is complete (no pending events or no needed OIDs)
|
|
/// 3. When no non-throttled URLs remain, enqueues with throttled domains for later processing
|
|
/// 4. Returns without waiting for throttled domains to complete
|
|
///
|
|
/// # Arguments
|
|
///
|
|
/// * `ctx` - The sync context providing repository data and OID information
|
|
/// * `identifier` - The repository identifier (d-tag value)
|
|
/// * `throttle_manager` - Used for rate limiting and domain queue management
|
|
/// * `git_naughty_list` - Used to filter out domains with persistent errors
|
|
///
|
|
/// # Returns
|
|
///
|
|
/// * `true` - Sync completed (no pending events or all OIDs fetched)
|
|
/// * `false` - Events remain in purgatory (will be retried after backoff, or processed
|
|
/// by throttled domain queues)
|
|
pub async fn sync_identifier<C: SyncContext + ?Sized>(
|
|
ctx: &C,
|
|
identifier: &str,
|
|
throttle_manager: &Arc<ThrottleManager>,
|
|
git_naughty_list: &NaughtyListTracker,
|
|
) -> bool {
|
|
let mut tried_urls: HashSet<String> = HashSet::new();
|
|
|
|
debug!(
|
|
identifier = %identifier,
|
|
"Starting sync for identifier"
|
|
);
|
|
|
|
// Try all non-throttled, non-naughty URLs
|
|
loop {
|
|
match sync_identifier_next_url(
|
|
ctx,
|
|
identifier,
|
|
None,
|
|
&tried_urls,
|
|
throttle_manager,
|
|
git_naughty_list,
|
|
)
|
|
.await
|
|
{
|
|
Some(url) => {
|
|
debug!(
|
|
identifier = %identifier,
|
|
url = %url,
|
|
"Found non-throttled URL to try"
|
|
);
|
|
|
|
// Build one wave from currently admissible sources. The
|
|
// coordinator starts the preferred primary immediately and
|
|
// at most one distinct-domain hedge after its head start.
|
|
let mut wave_urls = vec![url.clone()];
|
|
let mut discovered = tried_urls.clone();
|
|
discovered.insert(url);
|
|
while let Some(candidate) = sync_identifier_next_url(
|
|
ctx,
|
|
identifier,
|
|
None,
|
|
&discovered,
|
|
throttle_manager,
|
|
git_naughty_list,
|
|
)
|
|
.await
|
|
{
|
|
discovered.insert(candidate.clone());
|
|
wave_urls.push(candidate);
|
|
}
|
|
let outcome =
|
|
sync_identifier_from_urls(ctx, identifier, wave_urls, throttle_manager).await;
|
|
tried_urls.extend(outcome.attempted_urls);
|
|
|
|
// Check if sync is now complete
|
|
if !ctx.has_pending_events(identifier) {
|
|
debug!(
|
|
identifier = %identifier,
|
|
"Sync complete - no pending events"
|
|
);
|
|
return true;
|
|
}
|
|
|
|
let needed_oids = ctx.collect_needed_oids(identifier);
|
|
if needed_oids.is_empty() {
|
|
debug!(
|
|
identifier = %identifier,
|
|
"Sync complete - all OIDs available"
|
|
);
|
|
return true;
|
|
}
|
|
|
|
// Continue trying more URLs
|
|
}
|
|
None => {
|
|
// No more non-throttled URLs available
|
|
debug!(
|
|
identifier = %identifier,
|
|
tried_count = tried_urls.len(),
|
|
"No more non-throttled URLs available"
|
|
);
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Check if we're done (no pending events or no needed OIDs)
|
|
if !ctx.has_pending_events(identifier) {
|
|
debug!(
|
|
identifier = %identifier,
|
|
"Sync complete after exhausting URLs - no pending events"
|
|
);
|
|
return true;
|
|
}
|
|
|
|
let needed_oids = ctx.collect_needed_oids(identifier);
|
|
if needed_oids.is_empty() {
|
|
debug!(
|
|
identifier = %identifier,
|
|
"Sync complete after exhausting URLs - all OIDs available"
|
|
);
|
|
return true;
|
|
}
|
|
|
|
// Enqueue with any throttled domains that have untried URLs
|
|
let throttled_domains = get_throttled_domains_with_untried_urls(
|
|
ctx,
|
|
identifier,
|
|
&tried_urls,
|
|
throttle_manager,
|
|
git_naughty_list,
|
|
)
|
|
.await;
|
|
|
|
for info in throttled_domains {
|
|
debug!(
|
|
identifier = %identifier,
|
|
domain = %info.domain,
|
|
"Enqueueing identifier with throttled domain"
|
|
);
|
|
throttle_manager.enqueue_identifier(
|
|
&info.domain,
|
|
identifier.to_string(),
|
|
info.tried_urls_for_domain,
|
|
);
|
|
}
|
|
|
|
// Return false - events remain, will retry after backoff
|
|
// (throttled domains will process independently)
|
|
debug!(
|
|
identifier = %identifier,
|
|
"Sync incomplete - returning false for backoff"
|
|
);
|
|
false
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::purgatory::sync::MockSyncContext;
|
|
|
|
#[tokio::test]
|
|
async fn next_url_skips_throttled_domains() {
|
|
// Set up mock with URLs from two domains
|
|
let mock = MockSyncContext::new()
|
|
.with_urls(&[
|
|
"https://github.com/foo/bar.git",
|
|
"https://gitlab.com/foo/bar.git",
|
|
])
|
|
.with_needed_oids(&["abc123"])
|
|
.with_pending_events(true);
|
|
|
|
// Create throttle manager and throttle github.com
|
|
let throttle_manager = ThrottleManager::new(1, 100);
|
|
let naughty_list = NaughtyListTracker::with_defaults();
|
|
|
|
// Saturate github.com by starting a request
|
|
throttle_manager.start_request("github.com");
|
|
|
|
// Should return gitlab.com URL since github.com is throttled
|
|
let tried_urls = HashSet::new();
|
|
let result = sync_identifier_next_url(
|
|
&mock,
|
|
"test-repo",
|
|
None,
|
|
&tried_urls,
|
|
&throttle_manager,
|
|
&naughty_list,
|
|
)
|
|
.await;
|
|
|
|
assert!(result.is_some());
|
|
let url = result.unwrap();
|
|
assert!(
|
|
url.contains("gitlab.com"),
|
|
"Expected gitlab.com URL, got: {}",
|
|
url
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn next_url_skips_tried_urls() {
|
|
// Set up mock with two URLs from same domain
|
|
let mock = MockSyncContext::new()
|
|
.with_urls(&[
|
|
"https://github.com/foo/bar.git",
|
|
"https://github.com/foo/bar2.git",
|
|
])
|
|
.with_needed_oids(&["abc123"])
|
|
.with_pending_events(true);
|
|
|
|
let throttle_manager = ThrottleManager::new(5, 100);
|
|
let naughty_list = NaughtyListTracker::with_defaults();
|
|
|
|
// Mark first URL as tried
|
|
let mut tried_urls = HashSet::new();
|
|
tried_urls.insert("https://github.com/foo/bar.git".to_string());
|
|
|
|
// Should return the second URL
|
|
let result = sync_identifier_next_url(
|
|
&mock,
|
|
"test-repo",
|
|
None,
|
|
&tried_urls,
|
|
&throttle_manager,
|
|
&naughty_list,
|
|
)
|
|
.await;
|
|
|
|
assert!(result.is_some());
|
|
let url = result.unwrap();
|
|
assert_eq!(url, "https://github.com/foo/bar2.git");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn next_url_returns_none_when_no_pending_events() {
|
|
let mock = MockSyncContext::new()
|
|
.with_urls(&["https://github.com/foo/bar.git"])
|
|
.with_needed_oids(&["abc123"])
|
|
.with_pending_events(false); // No pending events
|
|
|
|
let throttle_manager = ThrottleManager::new(5, 100);
|
|
let naughty_list = NaughtyListTracker::with_defaults();
|
|
let tried_urls = HashSet::new();
|
|
|
|
let result = sync_identifier_next_url(
|
|
&mock,
|
|
"test-repo",
|
|
None,
|
|
&tried_urls,
|
|
&throttle_manager,
|
|
&naughty_list,
|
|
)
|
|
.await;
|
|
|
|
assert!(result.is_none());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn next_url_returns_none_when_no_oids_needed() {
|
|
let mock = MockSyncContext::new()
|
|
.with_urls(&["https://github.com/foo/bar.git"])
|
|
.with_needed_oids(&[]) // No OIDs needed
|
|
.with_pending_events(true);
|
|
|
|
let throttle_manager = ThrottleManager::new(5, 100);
|
|
let naughty_list = NaughtyListTracker::with_defaults();
|
|
let tried_urls = HashSet::new();
|
|
|
|
let result = sync_identifier_next_url(
|
|
&mock,
|
|
"test-repo",
|
|
None,
|
|
&tried_urls,
|
|
&throttle_manager,
|
|
&naughty_list,
|
|
)
|
|
.await;
|
|
|
|
assert!(result.is_none());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn next_url_filters_our_domain() {
|
|
let mock = MockSyncContext::new()
|
|
.with_urls(&[
|
|
"https://our-relay.com/foo/bar.git",
|
|
"https://github.com/foo/bar.git",
|
|
])
|
|
.with_needed_oids(&["abc123"])
|
|
.with_pending_events(true)
|
|
.with_our_domain("our-relay.com");
|
|
|
|
let throttle_manager = ThrottleManager::new(5, 100);
|
|
let naughty_list = NaughtyListTracker::with_defaults();
|
|
let tried_urls = HashSet::new();
|
|
|
|
let result = sync_identifier_next_url(
|
|
&mock,
|
|
"test-repo",
|
|
None,
|
|
&tried_urls,
|
|
&throttle_manager,
|
|
&naughty_list,
|
|
)
|
|
.await;
|
|
|
|
assert!(result.is_some());
|
|
let url = result.unwrap();
|
|
assert!(
|
|
url.contains("github.com"),
|
|
"Expected github.com URL (our domain filtered), got: {}",
|
|
url
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn next_url_with_specific_domain() {
|
|
let mock = MockSyncContext::new()
|
|
.with_urls(&[
|
|
"https://github.com/foo/bar.git",
|
|
"https://gitlab.com/foo/bar.git",
|
|
])
|
|
.with_needed_oids(&["abc123"])
|
|
.with_pending_events(true);
|
|
|
|
let throttle_manager = ThrottleManager::new(5, 100);
|
|
let naughty_list = NaughtyListTracker::with_defaults();
|
|
let tried_urls = HashSet::new();
|
|
|
|
// Request specific domain
|
|
let result = sync_identifier_next_url(
|
|
&mock,
|
|
"test-repo",
|
|
Some("gitlab.com"),
|
|
&tried_urls,
|
|
&throttle_manager,
|
|
&naughty_list,
|
|
)
|
|
.await;
|
|
|
|
assert!(result.is_some());
|
|
let url = result.unwrap();
|
|
assert!(
|
|
url.contains("gitlab.com"),
|
|
"Expected gitlab.com URL, got: {}",
|
|
url
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn from_url_fetches_and_processes_on_success() {
|
|
// Set up mock that can provide the needed OID
|
|
let mock = MockSyncContext::new()
|
|
.with_urls(&["https://github.com/foo/bar.git"])
|
|
.with_needed_oids(&["abc123"])
|
|
.with_pending_events(true)
|
|
.url_provides("https://github.com/foo/bar.git", &["abc123"]);
|
|
|
|
let throttle_manager = Arc::new(ThrottleManager::new(5, 100));
|
|
|
|
// Fetch from the URL
|
|
let fetched = sync_identifier_from_url(
|
|
&mock,
|
|
"test-repo",
|
|
"https://github.com/foo/bar.git",
|
|
&throttle_manager,
|
|
)
|
|
.await;
|
|
|
|
// Should have fetched 1 OID
|
|
assert_eq!(fetched, 1);
|
|
|
|
// Should have logged the fetch attempt
|
|
let fetch_log = mock.fetch_log();
|
|
assert_eq!(fetch_log.len(), 1);
|
|
assert_eq!(fetch_log[0], "https://github.com/foo/bar.git");
|
|
|
|
// OID should no longer be needed
|
|
assert!(mock.current_needed_oids().is_empty());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn from_url_returns_zero_on_failure() {
|
|
let mock = MockSyncContext::new()
|
|
.with_urls(&["https://bad-server.com/repo.git"])
|
|
.with_needed_oids(&["abc123"])
|
|
.with_pending_events(true)
|
|
.url_should_fail("https://bad-server.com/repo.git");
|
|
|
|
let throttle_manager = Arc::new(ThrottleManager::new(5, 100));
|
|
|
|
let fetched = sync_identifier_from_url(
|
|
&mock,
|
|
"test-repo",
|
|
"https://bad-server.com/repo.git",
|
|
&throttle_manager,
|
|
)
|
|
.await;
|
|
|
|
// Should return 0 on failure
|
|
assert_eq!(fetched, 0);
|
|
|
|
// OID should still be needed
|
|
assert!(mock.current_needed_oids().contains("abc123"));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn from_url_tracks_throttle_requests() {
|
|
let mock = MockSyncContext::new()
|
|
.with_urls(&["https://github.com/foo/bar.git"])
|
|
.with_needed_oids(&["abc123"])
|
|
.with_pending_events(true)
|
|
.url_provides("https://github.com/foo/bar.git", &["abc123"]);
|
|
|
|
let throttle_manager = Arc::new(ThrottleManager::new(1, 100));
|
|
|
|
// First request should work
|
|
let fetched = sync_identifier_from_url(
|
|
&mock,
|
|
"test-repo",
|
|
"https://github.com/foo/bar.git",
|
|
&throttle_manager,
|
|
)
|
|
.await;
|
|
assert_eq!(fetched, 1);
|
|
|
|
// After completion, domain should not be throttled
|
|
assert!(!throttle_manager.is_throttled("github.com"));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn duplicate_repo_demand_joins_one_fetch_wave() {
|
|
let url = "https://primary.example/repo.git";
|
|
let mock = Arc::new(
|
|
MockSyncContext::new()
|
|
.with_needed_oids(&["abc123"])
|
|
.url_provides(url, &["abc123"])
|
|
.url_waits_for_release(url),
|
|
);
|
|
let manager = Arc::new(ThrottleManager::new(5, 100));
|
|
|
|
let first_mock = mock.clone();
|
|
let first_manager = manager.clone();
|
|
let first = tokio::spawn(async move {
|
|
sync_identifier_from_url(&*first_mock, "repo", url, &first_manager).await
|
|
});
|
|
mock.wait_for_fetches(1).await;
|
|
|
|
let second_mock = mock.clone();
|
|
let second_manager = manager.clone();
|
|
let second = tokio::spawn(async move {
|
|
sync_identifier_from_url(&*second_mock, "repo", url, &second_manager).await
|
|
});
|
|
tokio::task::yield_now().await;
|
|
assert_eq!(mock.fetch_log().len(), 1);
|
|
|
|
mock.release_fetch(url);
|
|
assert_eq!(first.await.unwrap(), 1);
|
|
assert_eq!(second.await.unwrap(), 0);
|
|
assert_eq!(mock.fetch_log().len(), 1);
|
|
assert_eq!(mock.process_call_count(), 1);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn delayed_distinct_domain_hedge_can_supply_the_oid() {
|
|
let primary = "https://a.example/repo.git";
|
|
let hedge = "https://b.example/repo.git";
|
|
let mock = Arc::new(
|
|
MockSyncContext::new()
|
|
.with_needed_oids(&["abc123"])
|
|
.url_provides(primary, &["abc123"])
|
|
.url_provides(hedge, &["abc123"])
|
|
.url_waits_for_release(primary),
|
|
);
|
|
let manager = Arc::new(ThrottleManager::new(5, 100));
|
|
let task_mock = mock.clone();
|
|
let task_manager = manager.clone();
|
|
let task = tokio::spawn(async move {
|
|
sync_identifier_from_urls(
|
|
&*task_mock,
|
|
"repo",
|
|
vec![primary.to_string(), hedge.to_string()],
|
|
&task_manager,
|
|
)
|
|
.await
|
|
});
|
|
mock.wait_for_fetches(1).await;
|
|
tokio::time::advance(REPOSITORY_FETCH_HEDGE_DELAY).await;
|
|
mock.wait_for_fetches(2).await;
|
|
assert_eq!(mock.process_call_count(), 1, "hedge should supply the OID");
|
|
mock.release_fetch(primary);
|
|
assert_eq!(task.await.unwrap().fetched, 1);
|
|
assert_eq!(mock.process_call_count(), 1, "same OID is promoted once");
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn empty_primary_does_not_hide_a_productive_hedge() {
|
|
let primary = "https://a.example/repo.git";
|
|
let productive_hedge = "https://b.example/repo.git";
|
|
let mock = Arc::new(
|
|
MockSyncContext::new()
|
|
.with_needed_oids(&["abc123"])
|
|
.url_provides(productive_hedge, &["abc123"])
|
|
.url_waits_for_release(primary),
|
|
);
|
|
let manager = Arc::new(ThrottleManager::new(5, 100));
|
|
let task_mock = mock.clone();
|
|
let task_manager = manager.clone();
|
|
let task = tokio::spawn(async move {
|
|
sync_identifier_from_urls(
|
|
&*task_mock,
|
|
"repo",
|
|
vec![primary.to_string(), productive_hedge.to_string()],
|
|
&task_manager,
|
|
)
|
|
.await
|
|
});
|
|
mock.wait_for_fetches(1).await;
|
|
tokio::time::advance(REPOSITORY_FETCH_HEDGE_DELAY).await;
|
|
mock.wait_for_fetches(2).await;
|
|
mock.release_fetch(primary);
|
|
assert_eq!(task.await.unwrap().fetched, 1);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn failed_pair_falls_back_to_a_third_source() {
|
|
let primary = "https://a.example/repo.git";
|
|
let hedge = "https://b.example/repo.git";
|
|
let fallback = "https://c.example/repo.git";
|
|
let mock = Arc::new(
|
|
MockSyncContext::new()
|
|
.with_needed_oids(&["abc123"])
|
|
.url_should_fail(primary)
|
|
.url_should_fail(hedge)
|
|
.url_waits_for_release(primary)
|
|
.url_provides(fallback, &["abc123"]),
|
|
);
|
|
let manager = Arc::new(ThrottleManager::new(5, 100));
|
|
let task_mock = mock.clone();
|
|
let task_manager = manager.clone();
|
|
let task = tokio::spawn(async move {
|
|
sync_identifier_from_urls(
|
|
&*task_mock,
|
|
"repo",
|
|
vec![primary.to_string(), hedge.to_string(), fallback.to_string()],
|
|
&task_manager,
|
|
)
|
|
.await
|
|
});
|
|
mock.wait_for_fetches(1).await;
|
|
tokio::time::advance(REPOSITORY_FETCH_HEDGE_DELAY).await;
|
|
mock.wait_for_fetches(2).await;
|
|
mock.release_fetch(primary);
|
|
|
|
let outcome = task.await.unwrap();
|
|
assert_eq!(outcome.fetched, 1);
|
|
assert_eq!(
|
|
outcome.attempted_urls,
|
|
vec![primary.to_string(), hedge.to_string(), fallback.to_string()]
|
|
);
|
|
assert_eq!(mock.process_call_count(), 1);
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn shared_object_observation_is_not_treated_as_source_attribution() {
|
|
let primary = "https://a.example/repo.git";
|
|
let observer = "https://b.example/repo.git";
|
|
let mock = Arc::new(
|
|
MockSyncContext::new()
|
|
.with_needed_oids(&["abc123"])
|
|
.url_provides(primary, &["abc123"])
|
|
.url_waits_after_fetch(primary)
|
|
.url_reports_shared_objects(observer),
|
|
);
|
|
let manager = Arc::new(ThrottleManager::new(5, 100));
|
|
let task_mock = mock.clone();
|
|
let task_manager = manager.clone();
|
|
let task = tokio::spawn(async move {
|
|
sync_identifier_from_urls(
|
|
&*task_mock,
|
|
"repo",
|
|
vec![primary.to_string(), observer.to_string()],
|
|
&task_manager,
|
|
)
|
|
.await
|
|
});
|
|
|
|
mock.wait_for_fetches(1).await;
|
|
tokio::time::advance(REPOSITORY_FETCH_HEDGE_DELAY).await;
|
|
mock.wait_for_fetches(2).await;
|
|
mock.release_fetch(primary);
|
|
assert_eq!(task.await.unwrap().fetched, 1);
|
|
assert_eq!(mock.process_call_count(), 1);
|
|
// The observer returned the primary's shared object. There is
|
|
// deliberately no winner score or reorder API to receive that
|
|
// ambiguous result.
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn two_slow_fetches_never_run_a_third_concurrently() {
|
|
let first = "https://a.example/repo.git";
|
|
let second = "https://b.example/repo.git";
|
|
let third = "https://c.example/repo.git";
|
|
let mock = Arc::new(
|
|
MockSyncContext::new()
|
|
.with_needed_oids(&["abc123"])
|
|
.url_waits_for_release(first)
|
|
.url_waits_for_release(second),
|
|
);
|
|
let manager = Arc::new(ThrottleManager::new(5, 100));
|
|
let task_mock = mock.clone();
|
|
let task_manager = manager.clone();
|
|
let task = tokio::spawn(async move {
|
|
sync_identifier_from_urls(
|
|
&*task_mock,
|
|
"repo",
|
|
vec![first.to_string(), second.to_string(), third.to_string()],
|
|
&task_manager,
|
|
)
|
|
.await
|
|
});
|
|
mock.wait_for_fetches(1).await;
|
|
tokio::time::advance(REPOSITORY_FETCH_HEDGE_DELAY).await;
|
|
mock.wait_for_fetches(2).await;
|
|
tokio::time::advance(REPOSITORY_FETCH_HEDGE_DELAY * 10).await;
|
|
assert_eq!(mock.fetch_log().len(), 2);
|
|
mock.release_fetch(first);
|
|
mock.release_fetch(second);
|
|
task.await.unwrap();
|
|
assert!(mock.fetch_log().iter().any(|url| url == third));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn different_repository_paths_fetch_independently() {
|
|
let first_url = "https://a.example/one.git";
|
|
let second_url = "https://b.example/two.git";
|
|
let first = Arc::new(
|
|
MockSyncContext::new()
|
|
.with_target_repo("/tmp/repo-one")
|
|
.with_needed_oids(&["one"])
|
|
.url_waits_for_release(first_url),
|
|
);
|
|
let second = Arc::new(
|
|
MockSyncContext::new()
|
|
.with_target_repo("/tmp/repo-two")
|
|
.with_needed_oids(&["two"])
|
|
.url_waits_for_release(second_url),
|
|
);
|
|
let manager = Arc::new(ThrottleManager::new(5, 100));
|
|
|
|
let first_task = {
|
|
let ctx = first.clone();
|
|
let manager = manager.clone();
|
|
tokio::spawn(async move {
|
|
sync_identifier_from_url(&*ctx, "one", first_url, &manager).await
|
|
})
|
|
};
|
|
let second_task = {
|
|
let ctx = second.clone();
|
|
let manager = manager.clone();
|
|
tokio::spawn(async move {
|
|
sync_identifier_from_url(&*ctx, "two", second_url, &manager).await
|
|
})
|
|
};
|
|
first.wait_for_fetches(1).await;
|
|
second.wait_for_fetches(1).await;
|
|
first.release_fetch(first_url);
|
|
second.release_fetch(second_url);
|
|
first_task.await.unwrap();
|
|
second_task.await.unwrap();
|
|
}
|
|
|
|
#[tokio::test(start_paused = true)]
|
|
async fn same_domain_alternative_is_not_a_hedge() {
|
|
let primary = "https://same.example/one.git";
|
|
let alternative = "https://same.example/two.git";
|
|
let mock = Arc::new(
|
|
MockSyncContext::new()
|
|
.with_needed_oids(&["abc123"])
|
|
.url_waits_for_release(primary),
|
|
);
|
|
let manager = Arc::new(ThrottleManager::new(5, 100));
|
|
let task_mock = mock.clone();
|
|
let task_manager = manager.clone();
|
|
let task = tokio::spawn(async move {
|
|
sync_identifier_from_urls(
|
|
&*task_mock,
|
|
"repo",
|
|
vec![primary.to_string(), alternative.to_string()],
|
|
&task_manager,
|
|
)
|
|
.await
|
|
});
|
|
mock.wait_for_fetches(1).await;
|
|
tokio::time::advance(REPOSITORY_FETCH_HEDGE_DELAY * 2).await;
|
|
assert_eq!(mock.fetch_log(), vec![primary.to_string()]);
|
|
mock.release_fetch(primary);
|
|
task.await.unwrap();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn get_throttled_domains_returns_only_throttled_with_untried() {
|
|
let mock = MockSyncContext::new()
|
|
.with_urls(&[
|
|
"https://github.com/foo/bar.git",
|
|
"https://gitlab.com/foo/bar.git",
|
|
"https://bitbucket.org/foo/bar.git",
|
|
])
|
|
.with_needed_oids(&["abc123"])
|
|
.with_pending_events(true);
|
|
|
|
let throttle_manager = ThrottleManager::new(1, 100);
|
|
let naughty_list = NaughtyListTracker::with_defaults();
|
|
|
|
// Throttle github.com and gitlab.com
|
|
throttle_manager.start_request("github.com");
|
|
throttle_manager.start_request("gitlab.com");
|
|
|
|
// Mark github.com URL as already tried
|
|
let mut tried_urls = HashSet::new();
|
|
tried_urls.insert("https://github.com/foo/bar.git".to_string());
|
|
|
|
let throttled = get_throttled_domains_with_untried_urls(
|
|
&mock,
|
|
"test-repo",
|
|
&tried_urls,
|
|
&throttle_manager,
|
|
&naughty_list,
|
|
)
|
|
.await;
|
|
|
|
// Should only include gitlab.com (throttled with untried URLs)
|
|
// github.com is throttled but URL was tried
|
|
// bitbucket.org is not throttled
|
|
assert_eq!(throttled.len(), 1);
|
|
assert_eq!(throttled[0].domain, "gitlab.com");
|
|
assert!(throttled[0].tried_urls_for_domain.is_empty());
|
|
}
|
|
|
|
// =========================================================================
|
|
// Phase 6: sync_identifier tests
|
|
// =========================================================================
|
|
|
|
#[tokio::test]
|
|
async fn sync_identifier_tries_multiple_urls_until_complete() {
|
|
// Set up mock with 3 URLs, each providing partial OIDs
|
|
// URL1 provides abc123, URL2 provides def456, URL3 provides ghi789
|
|
let mock = MockSyncContext::new()
|
|
.with_urls(&[
|
|
"https://server1.com/repo.git",
|
|
"https://server2.com/repo.git",
|
|
"https://server3.com/repo.git",
|
|
])
|
|
.with_needed_oids(&["abc123", "def456", "ghi789"])
|
|
.with_pending_events(true)
|
|
.url_provides("https://server1.com/repo.git", &["abc123"])
|
|
.url_provides("https://server2.com/repo.git", &["def456"])
|
|
.url_provides("https://server3.com/repo.git", &["ghi789"]);
|
|
|
|
let throttle_manager = Arc::new(ThrottleManager::new(5, 100));
|
|
let naughty_list = NaughtyListTracker::with_defaults();
|
|
|
|
// Run sync_identifier
|
|
let complete = sync_identifier(&mock, "test-repo", &throttle_manager, &naughty_list).await;
|
|
|
|
// Should return true (sync complete)
|
|
assert!(complete, "Expected sync to complete after trying all URLs");
|
|
|
|
// Should have tried all 3 URLs
|
|
let fetch_log = mock.fetch_log();
|
|
assert_eq!(
|
|
fetch_log.len(),
|
|
3,
|
|
"Expected 3 fetch attempts, got: {:?}",
|
|
fetch_log
|
|
);
|
|
|
|
// All OIDs should now be fetched
|
|
assert!(
|
|
mock.current_needed_oids().is_empty(),
|
|
"Expected all OIDs to be fetched"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn sync_identifier_enqueues_throttled_domains_when_incomplete() {
|
|
// Set up mock with URLs from two domains
|
|
// Only github.com can provide the OID, but it will be throttled
|
|
let mock = MockSyncContext::new()
|
|
.with_urls(&[
|
|
"https://github.com/foo/bar.git",
|
|
"https://gitlab.com/foo/bar.git",
|
|
])
|
|
.with_needed_oids(&["abc123"])
|
|
.with_pending_events(true)
|
|
.url_provides("https://github.com/foo/bar.git", &["abc123"]);
|
|
// Note: gitlab.com doesn't provide any OIDs
|
|
|
|
let throttle_manager = Arc::new(ThrottleManager::new(1, 100));
|
|
let naughty_list = NaughtyListTracker::with_defaults();
|
|
|
|
// Throttle github.com by starting a request
|
|
throttle_manager.start_request("github.com");
|
|
|
|
// Run sync_identifier
|
|
let complete = sync_identifier(&mock, "test-repo", &throttle_manager, &naughty_list).await;
|
|
|
|
// Should return false (sync incomplete - github.com is throttled)
|
|
assert!(
|
|
!complete,
|
|
"Expected sync to be incomplete when required domain is throttled"
|
|
);
|
|
|
|
// Should have tried gitlab.com (not throttled) but it doesn't have the OID
|
|
let fetch_log = mock.fetch_log();
|
|
assert_eq!(
|
|
fetch_log.len(),
|
|
1,
|
|
"Expected 1 fetch attempt (gitlab.com), got: {:?}",
|
|
fetch_log
|
|
);
|
|
assert!(
|
|
fetch_log[0].contains("gitlab.com"),
|
|
"Expected gitlab.com to be tried first"
|
|
);
|
|
|
|
// OID should still be needed
|
|
assert!(
|
|
mock.current_needed_oids().contains("abc123"),
|
|
"Expected OID to still be needed"
|
|
);
|
|
|
|
// github.com should have the identifier enqueued
|
|
// We can verify this by checking if github.com is still throttled (it should be,
|
|
// since the identifier was enqueued but not processed yet)
|
|
assert!(
|
|
throttle_manager.is_throttled("github.com"),
|
|
"Expected github.com to still be throttled"
|
|
);
|
|
}
|
|
|
|
// =========================================================================
|
|
// PR Clone URL Tests
|
|
// =========================================================================
|
|
|
|
#[tokio::test]
|
|
async fn test_collect_pr_clone_urls_returns_configured_urls() {
|
|
// Test that MockSyncContext returns configured PR clone URLs
|
|
let mock = MockSyncContext::new().with_pr_clone_urls(&[
|
|
"https://pr-server.com/fork.git",
|
|
"https://another-server.com/fork.git",
|
|
]);
|
|
|
|
let pr_urls = mock.collect_pr_clone_urls("test-repo");
|
|
|
|
assert_eq!(pr_urls.len(), 2);
|
|
assert!(pr_urls.contains("https://pr-server.com/fork.git"));
|
|
assert!(pr_urls.contains("https://another-server.com/fork.git"));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_sync_identifier_next_url_includes_pr_clone_urls() {
|
|
// Set up mock with announcement URLs and PR clone URLs
|
|
let mock = MockSyncContext::new()
|
|
.with_urls(&["https://github.com/owner/repo.git"]) // From announcement
|
|
.with_pr_clone_urls(&["https://pr-author.com/fork.git"]) // From PR event
|
|
.with_needed_oids(&["abc123"])
|
|
.with_pending_events(true);
|
|
|
|
let throttle_manager = ThrottleManager::new(5, 100);
|
|
let naughty_list = NaughtyListTracker::with_defaults();
|
|
let tried_urls = HashSet::new();
|
|
|
|
// Get first URL
|
|
let first_url = sync_identifier_next_url(
|
|
&mock,
|
|
"test-repo",
|
|
None,
|
|
&tried_urls,
|
|
&throttle_manager,
|
|
&naughty_list,
|
|
)
|
|
.await
|
|
.expect("Should return a URL");
|
|
|
|
// Try the first URL
|
|
let mut tried = HashSet::new();
|
|
tried.insert(first_url.clone());
|
|
|
|
// Get second URL
|
|
let second_url = sync_identifier_next_url(
|
|
&mock,
|
|
"test-repo",
|
|
None,
|
|
&tried,
|
|
&throttle_manager,
|
|
&naughty_list,
|
|
)
|
|
.await
|
|
.expect("Should return a second URL");
|
|
|
|
// Both URLs should be available (one from announcement, one from PR)
|
|
let both_urls = [first_url, second_url];
|
|
assert!(
|
|
both_urls.iter().any(|u| u.contains("github.com")),
|
|
"Should include announcement URL"
|
|
);
|
|
assert!(
|
|
both_urls.iter().any(|u| u.contains("pr-author.com")),
|
|
"Should include PR clone URL"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_pr_clone_urls_filtered_by_our_domain() {
|
|
// Set up mock with PR clone URL pointing to our domain
|
|
let mock = MockSyncContext::new()
|
|
.with_urls(&["https://github.com/owner/repo.git"])
|
|
.with_pr_clone_urls(&[
|
|
"https://our-relay.com/fork.git", // Should be filtered
|
|
"https://external.com/fork.git", // Should be included
|
|
])
|
|
.with_our_domain("our-relay.com")
|
|
.with_needed_oids(&["abc123"])
|
|
.with_pending_events(true);
|
|
|
|
let throttle_manager = ThrottleManager::new(5, 100);
|
|
let naughty_list = NaughtyListTracker::with_defaults();
|
|
let mut tried_urls = HashSet::new();
|
|
|
|
// Collect all available URLs
|
|
let mut available_urls = Vec::new();
|
|
while let Some(url) = sync_identifier_next_url(
|
|
&mock,
|
|
"test-repo",
|
|
None,
|
|
&tried_urls,
|
|
&throttle_manager,
|
|
&naughty_list,
|
|
)
|
|
.await
|
|
{
|
|
available_urls.push(url.clone());
|
|
tried_urls.insert(url);
|
|
}
|
|
|
|
// Should have 2 URLs (github.com and external.com), not 3
|
|
assert_eq!(
|
|
available_urls.len(),
|
|
2,
|
|
"Expected 2 URLs after filtering our domain, got: {:?}",
|
|
available_urls
|
|
);
|
|
|
|
// our-relay.com should be filtered out
|
|
assert!(
|
|
!available_urls.iter().any(|u| u.contains("our-relay.com")),
|
|
"Our domain should be filtered out"
|
|
);
|
|
|
|
// github.com and external.com should be present
|
|
assert!(
|
|
available_urls.iter().any(|u| u.contains("github.com")),
|
|
"github.com should be present"
|
|
);
|
|
assert!(
|
|
available_urls.iter().any(|u| u.contains("external.com")),
|
|
"external.com should be present"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_get_throttled_domains_includes_pr_clone_urls() {
|
|
// Set up mock with throttled PR clone URL domain
|
|
let mock = MockSyncContext::new()
|
|
.with_urls(&["https://github.com/owner/repo.git"])
|
|
.with_pr_clone_urls(&["https://pr-server.com/fork.git"])
|
|
.with_needed_oids(&["abc123"])
|
|
.with_pending_events(true);
|
|
|
|
let throttle_manager = ThrottleManager::new(1, 100);
|
|
let naughty_list = NaughtyListTracker::with_defaults();
|
|
|
|
// Throttle both domains
|
|
throttle_manager.start_request("github.com");
|
|
throttle_manager.start_request("pr-server.com");
|
|
|
|
let tried_urls = HashSet::new();
|
|
|
|
let throttled = get_throttled_domains_with_untried_urls(
|
|
&mock,
|
|
"test-repo",
|
|
&tried_urls,
|
|
&throttle_manager,
|
|
&naughty_list,
|
|
)
|
|
.await;
|
|
|
|
// Should include both throttled domains
|
|
let domains: Vec<&str> = throttled.iter().map(|t| t.domain.as_str()).collect();
|
|
assert!(domains.contains(&"github.com"), "Should include github.com");
|
|
assert!(
|
|
domains.contains(&"pr-server.com"),
|
|
"Should include pr-server.com from PR clone URLs"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn test_sync_identifier_uses_pr_clone_urls_when_announcement_urls_fail() {
|
|
// Set up mock where only PR clone URL can provide the needed OID
|
|
let mock = MockSyncContext::new()
|
|
.with_urls(&["https://github.com/owner/repo.git"]) // Doesn't have the OID
|
|
.with_pr_clone_urls(&["https://pr-author.com/fork.git"]) // Has the OID
|
|
.with_needed_oids(&["pr-commit-123"])
|
|
.with_pending_events(true)
|
|
.url_provides("https://pr-author.com/fork.git", &["pr-commit-123"]);
|
|
// Note: github.com doesn't provide any OIDs
|
|
|
|
let throttle_manager = Arc::new(ThrottleManager::new(5, 100));
|
|
let naughty_list = NaughtyListTracker::with_defaults();
|
|
|
|
// Run sync_identifier
|
|
let complete = sync_identifier(&mock, "test-repo", &throttle_manager, &naughty_list).await;
|
|
|
|
// Should complete successfully using PR clone URL
|
|
assert!(complete, "Sync should complete using PR clone URL");
|
|
|
|
// Verify PR clone URL was tried
|
|
let fetch_log = mock.fetch_log();
|
|
assert!(
|
|
fetch_log.iter().any(|u| u.contains("pr-author.com")),
|
|
"PR clone URL should have been tried: {:?}",
|
|
fetch_log
|
|
);
|
|
|
|
// OID should be fetched
|
|
assert!(
|
|
mock.current_needed_oids().is_empty(),
|
|
"OID should be fetched from PR clone URL"
|
|
);
|
|
}
|
|
}
|