mirror of
https://relay.ngit.dev/npub15qydau2hjma6ngxkl2cyar74wzyjshvl65za5k5rl69264ar2exs5cyejr/ngit-grasp.git
synced 2026-10-05 15:08:24 +00:00
feat(metrics): export service resource usage
The stabilization acceptance criteria require sustained CPU and memory evidence, but ngit-grasp currently exposes neither through its Prometheus endpoint. Manual systemd sampling is service-specific but does not provide a durable application time series. Enable rust-prometheus's Linux process collector and explicitly register it with ngit-grasp's private registry. Also read the service's cgroup v2 CPU and memory counters so the headline metrics include every child Git process. The process-only metrics remain available to distinguish daemon cost from Git work. Document both levels and the PromQL rate used to derive CPU use. The cgroup metrics are omitted gracefully when the binary is not running in a readable Linux cgroup v2 hierarchy. They follow the existing global metrics switch: no separate configuration surface is added. Host-wide contention and unrelated services remain deliberately excluded. Validated with: - git diff --check - nix develop -c cargo test --lib metrics::tests::test_metrics_with_repository_counting The repository-wide cargo fmt check still reports pre-existing formatting in unrelated sync modules and tests; src/metrics/mod.rs was formatted directly.
This commit is contained in:
@@ -12,6 +12,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- Log routine Git `info/refs` requests for repositories not yet available
|
||||
locally at DEBUG rather than WARN; this is expected during propagation.
|
||||
|
||||
### Added
|
||||
|
||||
- Export total cgroup CPU and memory usage, including child Git processes,
|
||||
through the Prometheus metrics endpoint on Linux. Process-only CPU, memory,
|
||||
thread, and file-descriptor metrics provide diagnostic detail.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Preserve purgatory and rejected-event recovery across abrupt termination.
|
||||
|
||||
Generated
+52
-3
@@ -897,6 +897,12 @@ dependencies = [
|
||||
"heed-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hex"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
|
||||
|
||||
[[package]]
|
||||
name = "hex-conservative"
|
||||
version = "0.2.2"
|
||||
@@ -1199,6 +1205,12 @@ version = "0.2.186"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.4.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.12.1"
|
||||
@@ -1652,6 +1664,28 @@ dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "procfs"
|
||||
version = "0.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"hex",
|
||||
"procfs-core",
|
||||
"rustix 0.38.44",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "procfs-core"
|
||||
version = "0.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"hex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prometheus"
|
||||
version = "0.14.0"
|
||||
@@ -1661,8 +1695,10 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"fnv",
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"memchr",
|
||||
"parking_lot",
|
||||
"procfs",
|
||||
"protobuf",
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
@@ -1880,6 +1916,19 @@ dependencies = [
|
||||
"semver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.38.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys 0.4.15",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "1.1.4"
|
||||
@@ -1889,7 +1938,7 @@ dependencies = [
|
||||
"bitflags",
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"linux-raw-sys 0.12.1",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
@@ -2197,7 +2246,7 @@ dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.4.2",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"rustix 1.1.4",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
@@ -2992,7 +3041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rustix",
|
||||
"rustix 1.1.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
+1
-1
@@ -39,7 +39,7 @@ tar = "0.4"
|
||||
fs2 = "0.4"
|
||||
|
||||
# Metrics
|
||||
prometheus = "0.14"
|
||||
prometheus = { version = "0.14", features = ["process"] }
|
||||
dashmap = "6"
|
||||
lazy_static = "1.4"
|
||||
|
||||
|
||||
@@ -138,6 +138,22 @@ scrape_configs:
|
||||
- `ngit_uptime_seconds` - Server uptime
|
||||
- `ngit_build_info` - Version and commit info
|
||||
- `ngit_repositories_total` - Total hosted repositories
|
||||
- `ngit_cgroup_cpu_seconds_total` - Cumulative CPU time used by ngit-grasp and
|
||||
its child Git processes; use `rate(ngit_cgroup_cpu_seconds_total[5m])` for
|
||||
total service CPU cores consumed over time
|
||||
- `ngit_cgroup_memory_current_bytes` - Current memory used by ngit-grasp and
|
||||
its child Git processes
|
||||
- `process_cpu_seconds_total` - Cumulative CPU time used by the ngit-grasp process;
|
||||
use `rate(process_cpu_seconds_total[5m])` for CPU cores consumed over time
|
||||
- `process_resident_memory_bytes` - Resident memory used by the ngit-grasp process
|
||||
- `process_virtual_memory_bytes` - Virtual address space used by the process
|
||||
- `process_threads` - Number of operating-system threads in the process
|
||||
- `process_open_fds` / `process_max_fds` - Current and maximum file descriptors
|
||||
|
||||
Use the `ngit_cgroup_*` metrics as the headline service resource measurements.
|
||||
The `process_*` metrics exclude child Git processes and are useful for separating
|
||||
daemon resource usage from Git work. Cgroup metrics are omitted when ngit-grasp
|
||||
is not running in a readable Linux cgroup v2 hierarchy.
|
||||
|
||||
## Example Alerts
|
||||
|
||||
@@ -175,4 +191,4 @@ groups:
|
||||
## See Also
|
||||
|
||||
- [Monitoring Overview](../explanation/monitoring.md) - Architecture and design
|
||||
- [Configuration Reference](../reference/configuration.md) - All config options
|
||||
- [Configuration Reference](../reference/configuration.md) - All config options
|
||||
|
||||
+80
-1
@@ -20,6 +20,9 @@ pub use crate::sync::SyncMetrics;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
use std::path::PathBuf;
|
||||
|
||||
use lazy_static::lazy_static;
|
||||
use prometheus::{
|
||||
Counter, CounterVec, Encoder, Gauge, GaugeVec, Histogram, HistogramOpts, HistogramVec, Opts,
|
||||
@@ -29,9 +32,53 @@ use prometheus::{
|
||||
use bandwidth::BandwidthTracker;
|
||||
use connection::ConnectionTracker;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn cgroup_v2_directory() -> Option<PathBuf> {
|
||||
let membership = std::fs::read_to_string("/proc/self/cgroup").ok()?;
|
||||
let relative = membership.lines().find_map(|line| {
|
||||
let mut fields = line.splitn(3, ':');
|
||||
match (fields.next(), fields.next(), fields.next()) {
|
||||
(Some("0"), Some(""), Some(path)) => Some(path.trim_start_matches('/')),
|
||||
_ => None,
|
||||
}
|
||||
})?;
|
||||
Some(PathBuf::from("/sys/fs/cgroup").join(relative))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn cgroup_cpu_seconds(directory: &std::path::Path) -> Option<f64> {
|
||||
let stat = std::fs::read_to_string(directory.join("cpu.stat")).ok()?;
|
||||
let usage_usec = stat.lines().find_map(|line| {
|
||||
let mut fields = line.split_whitespace();
|
||||
match (fields.next(), fields.next()) {
|
||||
(Some("usage_usec"), Some(value)) => value.parse::<u64>().ok(),
|
||||
_ => None,
|
||||
}
|
||||
})?;
|
||||
Some(usage_usec as f64 / 1_000_000.0)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn cgroup_memory_bytes(directory: &std::path::Path) -> Option<u64> {
|
||||
std::fs::read_to_string(directory.join("memory.current"))
|
||||
.ok()?
|
||||
.trim()
|
||||
.parse()
|
||||
.ok()
|
||||
}
|
||||
|
||||
lazy_static! {
|
||||
/// Global Prometheus registry for ngit-grasp metrics
|
||||
pub static ref REGISTRY: Registry = Registry::new();
|
||||
pub static ref REGISTRY: Registry = {
|
||||
let registry = Registry::new();
|
||||
#[cfg(target_os = "linux")]
|
||||
registry
|
||||
.register(Box::new(
|
||||
prometheus::process_collector::ProcessCollector::for_self(),
|
||||
))
|
||||
.expect("register process metrics");
|
||||
registry
|
||||
};
|
||||
static ref BLACKLIST_DELETIONS_TOTAL: CounterVec = {
|
||||
let metric = CounterVec::new(
|
||||
Opts::new(
|
||||
@@ -674,6 +721,25 @@ impl Metrics {
|
||||
uptime
|
||||
));
|
||||
|
||||
// A systemd service's cgroup includes ngit-grasp and the Git processes
|
||||
// it launches, making these the headline measures of total service
|
||||
// cost. ProcessCollector metrics remain useful for separating the
|
||||
// daemon's own usage from its children. Omit these metrics gracefully
|
||||
// outside a readable cgroup v2 hierarchy.
|
||||
#[cfg(target_os = "linux")]
|
||||
if let Some(directory) = cgroup_v2_directory() {
|
||||
if let Some(cpu_seconds) = cgroup_cpu_seconds(&directory) {
|
||||
output.push_str(&format!(
|
||||
"# HELP ngit_cgroup_cpu_seconds_total Total CPU time used by ngit-grasp and child processes\n# TYPE ngit_cgroup_cpu_seconds_total counter\nngit_cgroup_cpu_seconds_total {cpu_seconds}\n"
|
||||
));
|
||||
}
|
||||
if let Some(memory_bytes) = cgroup_memory_bytes(&directory) {
|
||||
output.push_str(&format!(
|
||||
"# HELP ngit_cgroup_memory_current_bytes Current memory used by ngit-grasp and child processes\n# TYPE ngit_cgroup_memory_current_bytes gauge\nngit_cgroup_memory_current_bytes {memory_bytes}\n"
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
output
|
||||
}
|
||||
|
||||
@@ -1010,6 +1076,19 @@ mod tests {
|
||||
// render() recounts from disk)
|
||||
let output = metrics.render();
|
||||
assert!(output.contains("ngit_repositories_total 0"));
|
||||
assert!(output.contains("process_cpu_seconds_total"));
|
||||
assert!(output.contains("process_resident_memory_bytes"));
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
if let Some(directory) = cgroup_v2_directory() {
|
||||
if cgroup_cpu_seconds(&directory).is_some() {
|
||||
assert!(output.contains("ngit_cgroup_cpu_seconds_total"));
|
||||
}
|
||||
if cgroup_memory_bytes(&directory).is_some() {
|
||||
assert!(output.contains("ngit_cgroup_memory_current_bytes"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create some repositories
|
||||
let npub1 = git_data_path.join("npub1test");
|
||||
|
||||
Reference in New Issue
Block a user