diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e83d95..47bd5f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Log routine Git `info/refs` requests for repositories not yet available locally at DEBUG rather than WARN; this is expected during propagation. +### Added + +- Export total cgroup CPU and memory usage, including child Git processes, + through the Prometheus metrics endpoint on Linux. Process-only CPU, memory, + thread, and file-descriptor metrics provide diagnostic detail. + ### Fixed - Preserve purgatory and rejected-event recovery across abrupt termination. diff --git a/Cargo.lock b/Cargo.lock index 284b4b3..f118dc7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -897,6 +897,12 @@ dependencies = [ "heed-traits", ] +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hex-conservative" version = "0.2.2" @@ -1199,6 +1205,12 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -1652,6 +1664,28 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "procfs" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc5b72d8145275d844d4b5f6d4e1eef00c8cd889edb6035c21675d1bb1f45c9f" +dependencies = [ + "bitflags", + "hex", + "procfs-core", + "rustix 0.38.44", +] + +[[package]] +name = "procfs-core" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "239df02d8349b06fc07398a3a1697b06418223b1c7725085e801e7c0fc6a12ec" +dependencies = [ + "bitflags", + "hex", +] + [[package]] name = "prometheus" version = "0.14.0" @@ -1661,8 +1695,10 @@ dependencies = [ "cfg-if", "fnv", "lazy_static", + "libc", "memchr", "parking_lot", + "procfs", "protobuf", "thiserror 2.0.18", ] @@ -1880,6 +1916,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.52.0", +] + [[package]] name = "rustix" version = "1.1.4" @@ -1889,7 +1938,7 @@ dependencies = [ "bitflags", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] @@ -2197,7 +2246,7 @@ dependencies = [ "fastrand", "getrandom 0.4.2", "once_cell", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] @@ -2992,7 +3041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "rustix", + "rustix 1.1.4", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fbbdcc0..10ea6fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,7 +39,7 @@ tar = "0.4" fs2 = "0.4" # Metrics -prometheus = "0.14" +prometheus = { version = "0.14", features = ["process"] } dashmap = "6" lazy_static = "1.4" diff --git a/docs/how-to/prometheus-setup.md b/docs/how-to/prometheus-setup.md index 53ec480..cee1657 100644 --- a/docs/how-to/prometheus-setup.md +++ b/docs/how-to/prometheus-setup.md @@ -138,6 +138,22 @@ scrape_configs: - `ngit_uptime_seconds` - Server uptime - `ngit_build_info` - Version and commit info - `ngit_repositories_total` - Total hosted repositories +- `ngit_cgroup_cpu_seconds_total` - Cumulative CPU time used by ngit-grasp and + its child Git processes; use `rate(ngit_cgroup_cpu_seconds_total[5m])` for + total service CPU cores consumed over time +- `ngit_cgroup_memory_current_bytes` - Current memory used by ngit-grasp and + its child Git processes +- `process_cpu_seconds_total` - Cumulative CPU time used by the ngit-grasp process; + use `rate(process_cpu_seconds_total[5m])` for CPU cores consumed over time +- `process_resident_memory_bytes` - Resident memory used by the ngit-grasp process +- `process_virtual_memory_bytes` - Virtual address space used by the process +- `process_threads` - Number of operating-system threads in the process +- `process_open_fds` / `process_max_fds` - Current and maximum file descriptors + +Use the `ngit_cgroup_*` metrics as the headline service resource measurements. +The `process_*` metrics exclude child Git processes and are useful for separating +daemon resource usage from Git work. Cgroup metrics are omitted when ngit-grasp +is not running in a readable Linux cgroup v2 hierarchy. ## Example Alerts @@ -175,4 +191,4 @@ groups: ## See Also - [Monitoring Overview](../explanation/monitoring.md) - Architecture and design -- [Configuration Reference](../reference/configuration.md) - All config options \ No newline at end of file +- [Configuration Reference](../reference/configuration.md) - All config options diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index a264a93..908f3da 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -20,6 +20,9 @@ pub use crate::sync::SyncMetrics; use std::sync::Arc; use std::time::Instant; +#[cfg(target_os = "linux")] +use std::path::PathBuf; + use lazy_static::lazy_static; use prometheus::{ Counter, CounterVec, Encoder, Gauge, GaugeVec, Histogram, HistogramOpts, HistogramVec, Opts, @@ -29,9 +32,53 @@ use prometheus::{ use bandwidth::BandwidthTracker; use connection::ConnectionTracker; +#[cfg(target_os = "linux")] +fn cgroup_v2_directory() -> Option { + let membership = std::fs::read_to_string("/proc/self/cgroup").ok()?; + let relative = membership.lines().find_map(|line| { + let mut fields = line.splitn(3, ':'); + match (fields.next(), fields.next(), fields.next()) { + (Some("0"), Some(""), Some(path)) => Some(path.trim_start_matches('/')), + _ => None, + } + })?; + Some(PathBuf::from("/sys/fs/cgroup").join(relative)) +} + +#[cfg(target_os = "linux")] +fn cgroup_cpu_seconds(directory: &std::path::Path) -> Option { + let stat = std::fs::read_to_string(directory.join("cpu.stat")).ok()?; + let usage_usec = stat.lines().find_map(|line| { + let mut fields = line.split_whitespace(); + match (fields.next(), fields.next()) { + (Some("usage_usec"), Some(value)) => value.parse::().ok(), + _ => None, + } + })?; + Some(usage_usec as f64 / 1_000_000.0) +} + +#[cfg(target_os = "linux")] +fn cgroup_memory_bytes(directory: &std::path::Path) -> Option { + std::fs::read_to_string(directory.join("memory.current")) + .ok()? + .trim() + .parse() + .ok() +} + lazy_static! { /// Global Prometheus registry for ngit-grasp metrics - pub static ref REGISTRY: Registry = Registry::new(); + pub static ref REGISTRY: Registry = { + let registry = Registry::new(); + #[cfg(target_os = "linux")] + registry + .register(Box::new( + prometheus::process_collector::ProcessCollector::for_self(), + )) + .expect("register process metrics"); + registry + }; static ref BLACKLIST_DELETIONS_TOTAL: CounterVec = { let metric = CounterVec::new( Opts::new( @@ -674,6 +721,25 @@ impl Metrics { uptime )); + // A systemd service's cgroup includes ngit-grasp and the Git processes + // it launches, making these the headline measures of total service + // cost. ProcessCollector metrics remain useful for separating the + // daemon's own usage from its children. Omit these metrics gracefully + // outside a readable cgroup v2 hierarchy. + #[cfg(target_os = "linux")] + if let Some(directory) = cgroup_v2_directory() { + if let Some(cpu_seconds) = cgroup_cpu_seconds(&directory) { + output.push_str(&format!( + "# HELP ngit_cgroup_cpu_seconds_total Total CPU time used by ngit-grasp and child processes\n# TYPE ngit_cgroup_cpu_seconds_total counter\nngit_cgroup_cpu_seconds_total {cpu_seconds}\n" + )); + } + if let Some(memory_bytes) = cgroup_memory_bytes(&directory) { + output.push_str(&format!( + "# HELP ngit_cgroup_memory_current_bytes Current memory used by ngit-grasp and child processes\n# TYPE ngit_cgroup_memory_current_bytes gauge\nngit_cgroup_memory_current_bytes {memory_bytes}\n" + )); + } + } + output } @@ -1010,6 +1076,19 @@ mod tests { // render() recounts from disk) let output = metrics.render(); assert!(output.contains("ngit_repositories_total 0")); + assert!(output.contains("process_cpu_seconds_total")); + assert!(output.contains("process_resident_memory_bytes")); + #[cfg(target_os = "linux")] + { + if let Some(directory) = cgroup_v2_directory() { + if cgroup_cpu_seconds(&directory).is_some() { + assert!(output.contains("ngit_cgroup_cpu_seconds_total")); + } + if cgroup_memory_bytes(&directory).is_some() { + assert!(output.contains("ngit_cgroup_memory_current_bytes")); + } + } + } // Create some repositories let npub1 = git_data_path.join("npub1test");