mirror of
https://relay.ngit.dev/npub15qydau2hjma6ngxkl2cyar74wzyjshvl65za5k5rl69264ar2exs5cyejr/ngit-grasp.git
synced 2026-10-05 23:18:24 +00:00
feat(metrics): expose retained sync state totals
The peer-state audit found that important long-lived collections were visible only through incidental logs. That made slow cardinality growth difficult to distinguish from expected event throughput during production soaks. Export one fixed-label aggregate gauge and refresh eight operational classes from the existing two-second health pass: pending batches and subscriptions, auth retries, connection attempts, purgatory dependency attempts, temporary dependency relays, deferred consolidations, and descendant rotations. No relay, event, or repository identifier enters a label. The metric is observational only and introduces no admission limit. Its class vocabulary is deliberately fixed in the sync manager so monitoring cannot recreate the peer-controlled cardinality problem removed by the preceding metrics fix. Validated with a registry test for aggregate values and series count, cargo fmt, and cargo clippy --locked --lib -- -D warnings.
This commit is contained in:
@@ -42,6 +42,8 @@ pub struct SyncMetrics {
|
||||
relays_connected_total: IntGauge,
|
||||
/// Relays marked as dead
|
||||
relays_dead_total: IntGauge,
|
||||
/// Aggregate cardinality of long-lived sync-manager state by fixed class.
|
||||
retained_state_current: IntGaugeVec,
|
||||
|
||||
// === Rejected Events Index Metrics (unified with event_type label) ===
|
||||
/// Current number of entries in hot cache (by event_type: announcement, state)
|
||||
@@ -144,6 +146,15 @@ impl SyncMetrics {
|
||||
))?;
|
||||
registry.register(Box::new(relays_dead_total.clone()))?;
|
||||
|
||||
let retained_state_current = IntGaugeVec::new(
|
||||
Opts::new(
|
||||
"ngit_sync_retained_state_current",
|
||||
"Current retained sync-manager entries by fixed state class",
|
||||
),
|
||||
&["class"],
|
||||
)?;
|
||||
registry.register(Box::new(retained_state_current.clone()))?;
|
||||
|
||||
// Rejected events metrics (unified with event_type label)
|
||||
let rejected_hot_cache_current = IntGaugeVec::new(
|
||||
Opts::new(
|
||||
@@ -228,6 +239,7 @@ impl SyncMetrics {
|
||||
relays_tracked_total,
|
||||
relays_connected_total,
|
||||
relays_dead_total,
|
||||
retained_state_current,
|
||||
rejected_hot_cache_current,
|
||||
rejected_hot_cache_hits_total,
|
||||
rejected_hot_cache_misses_total,
|
||||
@@ -425,6 +437,14 @@ impl SyncMetrics {
|
||||
self.relays_dead_total.get()
|
||||
}
|
||||
|
||||
/// Record one aggregate long-lived state cardinality. Callers must use the
|
||||
/// fixed class vocabulary documented by the sync manager.
|
||||
pub fn set_retained_state(&self, class: &str, count: usize) {
|
||||
self.retained_state_current
|
||||
.with_label_values(&[class])
|
||||
.set(count as i64);
|
||||
}
|
||||
|
||||
// === Rejected Events Recording Methods (unified with event_type parameter) ===
|
||||
|
||||
/// Update hot cache current size gauge for a specific event type.
|
||||
@@ -663,6 +683,30 @@ mod tests {
|
||||
assert!(metrics2.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retained_state_metric_uses_fixed_aggregate_classes() {
|
||||
let registry = create_test_registry();
|
||||
let metrics = SyncMetrics::register(®istry).unwrap();
|
||||
|
||||
metrics.set_retained_state("pending_batches", 12);
|
||||
metrics.set_retained_state("queued_connection_attempts", 3);
|
||||
|
||||
let family = registry
|
||||
.gather()
|
||||
.into_iter()
|
||||
.find(|family| family.name() == "ngit_sync_retained_state_current")
|
||||
.expect("retained-state metric must be exported");
|
||||
assert_eq!(family.get_metric().len(), 2);
|
||||
assert_eq!(
|
||||
family
|
||||
.get_metric()
|
||||
.iter()
|
||||
.map(|metric| metric.get_gauge().value() as i64)
|
||||
.sum::<i64>(),
|
||||
15
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rejected_events_metrics() {
|
||||
let registry = create_test_registry();
|
||||
|
||||
@@ -1803,6 +1803,30 @@ async fn run_health_and_metrics_checker(
|
||||
let entries = naughty_list.get_all();
|
||||
metrics.update_naughty_list(entries);
|
||||
}
|
||||
|
||||
let (pending_batches, pending_subscriptions) = {
|
||||
let pending = manager.pending_sync_index.read().await;
|
||||
(
|
||||
pending.values().map(Vec::len).sum(),
|
||||
pending
|
||||
.values()
|
||||
.flatten()
|
||||
.map(|batch| batch.outstanding_subs.len())
|
||||
.sum(),
|
||||
)
|
||||
};
|
||||
for (class, count) in [
|
||||
("pending_batches", pending_batches),
|
||||
("pending_subscriptions", pending_subscriptions),
|
||||
("auth_retries", manager.auth_required_attempts.len()),
|
||||
("queued_connection_attempts", manager.in_flight_connect_attempts.len()),
|
||||
("purgatory_dependencies", manager.purgatory_dependency_attempts.len()),
|
||||
("dependency_relays", manager.dependency_relay_deadlines.len()),
|
||||
("deferred_consolidations", manager.deferred_consolidations.relays.len()),
|
||||
("descendant_rotations", manager.descendant_sync_rotations.len()),
|
||||
] {
|
||||
metrics.set_retained_state(class, count);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = shutdown_rx.recv() => {
|
||||
|
||||
Reference in New Issue
Block a user