diff --git a/grasp-audit/README.md b/grasp-audit/README.md index 2604e67..51ce3a9 100644 --- a/grasp-audit/README.md +++ b/grasp-audit/README.md @@ -90,8 +90,18 @@ grasp-audit audit --relay ws://localhost:7334 --spec nip01-smoke # Run with isolated fixtures (for testing/debugging) grasp-audit audit --relay ws://localhost:7334 --mode isolated --spec push-auth + +# Emit one stable JSON document for a scheduled full audit +grasp-audit audit --relay wss://relay.ngit.dev --spec all --json ``` +`audit --json` writes one schema-versioned document to stdout and suppresses +the human progress/report output. Completed runs include tool and run metadata, +pinned GRASP revisions, aggregate counts, and per-test outcomes. Invalid +arguments and setup/runtime errors use the same envelope with an `error` +outcome. Exit status is `0` for a passing audit, `1` for completed test +failures, and `2` for invocation or runtime errors. + ### Audit identity Probe write checks and full audits can use an explicit identity. Key sources diff --git a/grasp-audit/src/bin/grasp-audit.rs b/grasp-audit/src/bin/grasp-audit.rs index 6eb69f8..1457b73 100644 --- a/grasp-audit/src/bin/grasp-audit.rs +++ b/grasp-audit/src/bin/grasp-audit.rs @@ -1,13 +1,13 @@ //! GRASP Audit CLI Tool -use clap::{Args, CommandFactory, Parser, Subcommand}; +use clap::{Args, CommandFactory, Parser, Subcommand, ValueEnum}; use grasp_audit::*; use std::ffi::OsString; use std::path::PathBuf; -use std::time::Duration; +use std::time::{Duration, Instant}; #[derive(Parser)] -#[command(name = "grasp-audit")] +#[command(name = "grasp-audit", version)] #[command(about = "GRASP audit and compliance testing tool", long_about = None)] struct Cli { #[command(subcommand)] @@ -26,6 +26,30 @@ struct KeyArgs { nsec_file: Option, } +#[derive(Debug, Clone, Copy, ValueEnum)] +enum AuditSelection { + Nip01Smoke, + Nip11, + EventAcceptance, + Cors, + GitClone, + GitFilter, + PushAuth, + RepoCreation, + Purgatory, + Grasp06, + All, +} + +impl std::fmt::Display for AuditSelection { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let value = self + .to_possible_value() + .expect("audit selection variants have possible values"); + formatter.write_str(value.get_name()) + } +} + #[derive(Subcommand)] enum Commands { /// Run a probe/smoke test against a server @@ -67,20 +91,21 @@ enum Commands { relay: String, /// Fixture mode: shared (default) or isolated - /// - /// - shared: Fixtures are cached and reused across tests (efficient for sequential test runs) - /// - isolated: Each test creates fresh fixtures (for parallel tests like cargo test) #[arg(short, long, default_value = "shared")] mode: String, - /// Spec to test (nip01-smoke, nip11, event-acceptance, cors, git-clone, git-filter, push-auth, repo-creation, purgatory, grasp06, all) - #[arg(short, long, default_value = "all")] - spec: String, + /// Audit specification or suite to run. + #[arg(short, long, value_enum, default_value_t = AuditSelection::All)] + spec: AuditSelection, /// Git data directory (required for cors, git-clone, push-auth, repo-creation specs) #[arg(short, long)] git_data_dir: Option, + /// Output one stable machine-readable JSON document. + #[arg(long, default_value_t = false)] + json: bool, + #[command(flatten)] keys: KeyArgs, }, @@ -88,13 +113,11 @@ enum Commands { #[tokio::main] async fn main() -> Result<()> { - // Probe output is self-contained — library chatter (nostr_relay_pool etc.) - // adds no value and clutters both human and JSON output. Skip the tracing - // subscriber entirely for the probe subcommand; initialise it normally for - // audit subcommands where verbose output is expected. - let is_probe = std::env::args().nth(1).as_deref() == Some("probe"); + let args: Vec = std::env::args_os().collect(); + let is_probe = args.get(1).is_some_and(|arg| arg == "probe"); if !is_probe { tracing_subscriber::fmt() + .with_writer(std::io::stderr) .with_env_filter( tracing_subscriber::EnvFilter::from_default_env() .add_directive(tracing::Level::INFO.into()), @@ -102,7 +125,33 @@ async fn main() -> Result<()> { .init(); } - let cli = Cli::parse(); + let audit_json = audit_json_requested(&args); + let parse_started_at = unix_timestamp(); + let parse_started = Instant::now(); + let cli = match Cli::try_parse_from(&args) { + Ok(cli) => cli, + Err(error) + if matches!( + error.kind(), + clap::error::ErrorKind::DisplayHelp | clap::error::ErrorKind::DisplayVersion + ) => + { + error.print()?; + return Ok(()); + } + Err(error) if audit_json => { + let report = AuditReport::error( + requested_relay(&args), + "", + parse_started_at, + parse_started.elapsed(), + format!("invalid command line ({:?})", error.kind()), + ); + println!("{}", serde_json::to_string(&report)?); + std::process::exit(2); + } + Err(error) => error.exit(), + }; match cli.command { Commands::Probe { @@ -115,36 +164,26 @@ async fn main() -> Result<()> { harden_network, } => { let relay = match relay { - Some(r) => r, + Some(relay) => relay, None => { - // Print probe-specific help and exit cleanly - let mut cmd = Cli::command(); - let _ = cmd.find_subcommand_mut("probe").unwrap().print_help(); + let mut command = Cli::command(); + let _ = command.find_subcommand_mut("probe").unwrap().print_help(); println!(); return Ok(()); } }; - let keys = load_keys(&keys)?; - - // read_only is the default; --create-repo opts into the write path let read_only = !create_repo; - - // Overall probe timeout: min(20s, watch_interval) to prevent - // overlapping runs under --watch or cron scheduling. - let overall_secs = match watch { - Some(interval) => interval.min(20), - None => 20, - }; + let overall_secs = watch.map_or(20, |interval| interval.min(20)); let options = ProbeOptions { harden_network }; if let Some(interval) = watch { - let mut run = 1u64; + let mut run = 1_u64; loop { if !json { - println!("\n[Run {}]", run); + println!("\n[Run {run}]"); } - let report = grasp_audit::probe::run_probe_with_options( + let report = run_probe_with_options( &relay, keys.clone(), read_only, @@ -162,15 +201,9 @@ async fn main() -> Result<()> { tokio::time::sleep(Duration::from_secs(interval)).await; } } else { - let report = grasp_audit::probe::run_probe_with_options( - &relay, - keys, - read_only, - timeout, - overall_secs, - options, - ) - .await; + let report = + run_probe_with_options(&relay, keys, read_only, timeout, overall_secs, options) + .await; if json { report.print_json(); } else { @@ -181,213 +214,261 @@ async fn main() -> Result<()> { } } } - Commands::Audit { relay, mode, spec, git_data_dir, + json, keys, - } => { - let keys = load_keys(&keys)?; - let mut config = match mode.as_str() { - "shared" => AuditConfig::shared(), - "isolated" => AuditConfig::isolated(), - // Backwards compatibility aliases - "ci" => AuditConfig::isolated(), - "production" => AuditConfig::shared(), - _ => { - return Err(anyhow!( - "Invalid mode: {}. Use 'shared' or 'isolated'", - mode - )) - } - }; - - // Audit needs to create events to test the relay, so disable read-only mode - config.read_only = false; - - // Derive relay_domain from relay URL (e.g., "ws://localhost:8081" -> "localhost:8081") - let relay_domain = relay - .replace("ws://", "") - .replace("wss://", "") - .trim_end_matches('/') - .to_string(); - - println!("🔍 GRASP Audit Tool"); - println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"); - println!("Relay: {}", relay); - println!("Mode: {}", mode); - println!("Spec: {}", spec); - println!("Run ID: {}", config.run_id); - if let Some(ref dir) = git_data_dir { - println!("Git Dir: {}", dir.display()); - } - println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"); - println!(); - - println!("Connecting to relay..."); - let client = match keys { - Some(keys) => AuditClient::new_with_keys(&relay, config, keys).await, - None => AuditClient::new(&relay, config).await, - } - .map_err(|e| anyhow!("Failed to connect to relay: {}", e))?; - - if !client.is_connected().await { - return Err(anyhow!("Could not establish connection to relay")); - } - - println!("✓ Connected\n"); - - let results = match spec.as_str() { - "nip01-smoke" => { - println!("Running NIP-01 smoke tests...\n"); - specs::Nip01SmokeTests::run_all(&client).await - } - "nip11" => { - println!("Running NIP-11 document tests...\n"); - specs::Nip11DocumentTests::run_all(&client).await - } - "event-acceptance" => { - println!("Running event acceptance policy tests...\n"); - specs::EventAcceptancePolicyTests::run_all(&client).await - } - "cors" => { - println!("Running CORS tests...\n"); - specs::CorsTests::run_all(&client, &relay_domain).await - } - "git-clone" => { - println!("Running Git clone tests...\n"); - specs::GitCloneTests::run_all(&client, &relay_domain).await - } - "git-filter" => { - println!("Running Git filter capability tests...\n"); - specs::GitFilterTests::run_all(&client, &relay_domain).await - } - "push-auth" => { - println!("Running push authorization tests...\n"); - specs::PushAuthorizationTests::run_all(&client, &relay_domain).await - } - "repo-creation" => { - println!("Running repository creation tests...\n"); - specs::RepositoryCreationTests::run_all(&client, &relay_domain).await - } - "purgatory" => { - println!("Running purgatory tests...\n"); - specs::PurgatoryTests::run_all(&client).await - } - "grasp06" => { - println!("Running GRASP-06 tests...\n"); - // GRASP-06 has its own report renderer (sections differ - // from GRASP-01). Print it directly and short-circuit the - // shared print_report path below. - let grasp06_results = specs::Grasp06Tests::run_all(&client).await; - specs::Grasp06Tests::print_report(&grasp06_results); - if !grasp06_results.all_passed() { - println!("❌ Some tests failed"); - std::process::exit(1); - } else { - println!("✅ All tests passed!"); - } - return Ok(()); - } - "all" => { - println!("Running all tests...\n"); - let mut all_results = AuditResult::new("All GRASP-01 Tests"); - - // NIP-01 smoke tests (stateless - no shared fixture dependencies) - println!(" → NIP-01 smoke tests..."); - let nip01_results = specs::Nip01SmokeTests::run_all(&client).await; - all_results.merge(nip01_results); - - // NIP-11 document tests (stateless) - println!(" → NIP-11 document tests..."); - let nip11_results = specs::Nip11DocumentTests::run_all(&client).await; - all_results.merge(nip11_results); - - // CORS tests (stateless HTTP checks) - println!(" → CORS tests..."); - let cors_results = specs::CorsTests::run_all(&client, &relay_domain).await; - all_results.merge(cors_results); - - // Repository creation tests (uses ValidRepoSent only - no state events) - println!(" → Repository creation tests..."); - let repo_results = specs::RepositoryCreationTests::run_all(&client, &relay_domain).await; - all_results.merge(repo_results); - - // Git clone tests (uses ValidRepoSent only - no state events) - println!(" → Git clone tests..."); - let clone_results = specs::GitCloneTests::run_all(&client, &relay_domain).await; - all_results.merge(clone_results); - - // Git filter capability tests (uses ValidRepoSent only - no state events) - println!(" → Git filter capability tests..."); - let filter_results = specs::GitFilterTests::run_all(&client, &relay_domain).await; - all_results.merge(filter_results); - - // Event acceptance policy tests (uses ValidRepoServed - no extra state events) - println!(" → Event acceptance policy tests..."); - let event_results = specs::EventAcceptancePolicyTests::run_all(&client).await; - all_results.merge(event_results); - - // Purgatory tests MUST run before push-auth. - // Push-auth sends new replaceable state events (kind 30618) for the same - // repo_id as OwnerStateDataPushed (e.g. test_head_set_after_git_push_with_required_oids - // sends a develop1 state event that displaces the original). If purgatory ran - // after push-auth, is_event_on_relay(original_id) would return false because - // the original state event has been replaced on the relay. - println!(" → Purgatory tests..."); - let purgatory_results = specs::PurgatoryTests::run_all(&client).await; - all_results.merge(purgatory_results); - - // Push authorization tests (mutates shared state - must run last among git specs) - println!(" → Push authorization tests..."); - let push_results = specs::PushAuthorizationTests::run_all(&client, &relay_domain).await; - all_results.merge(push_results); - - // GRASP-06 tests live in their own spec family with their - // own report renderer. Print the GRASP-01 block first - // (via the default `print_report` below), then the - // GRASP-06 block separately. - println!(" → GRASP-06 tests..."); - let grasp06_results = specs::Grasp06Tests::run_all(&client).await; - - println!(); - all_results.print_report(); - specs::Grasp06Tests::print_report(&grasp06_results); - - let combined_ok = - all_results.all_passed() && grasp06_results.all_passed(); - if !combined_ok { - println!("❌ Some tests failed"); - std::process::exit(1); - } else { - println!("✅ All tests passed!"); - } - return Ok(()); - } - _ => { - return Err(anyhow!( - "Unknown spec: {}. Use 'nip01-smoke', 'nip11', 'event-acceptance', 'cors', 'git-clone', 'git-filter', 'push-auth', 'repo-creation', 'purgatory', 'grasp06', or 'all'", - spec - )) - } - }; - - results.print_report(); - - if !results.all_passed() { - println!("❌ Some tests failed"); - std::process::exit(1); - } else { - println!("✅ All tests passed!"); - } - } + } => run_audit(relay, mode, spec, git_data_dir, json, keys).await?, } Ok(()) } +fn audit_json_requested(args: &[OsString]) -> bool { + args.get(1).is_some_and(|arg| arg == "audit") && args.iter().skip(2).any(|arg| arg == "--json") +} + +fn requested_relay(args: &[OsString]) -> String { + for (index, arg) in args.iter().enumerate().skip(2) { + if arg == "--relay" || arg == "-r" { + return args + .get(index + 1) + .map(|value| value.to_string_lossy().into_owned()) + .unwrap_or_default(); + } + if let Some(value) = arg.to_str().and_then(|arg| arg.strip_prefix("--relay=")) { + return value.to_string(); + } + } + String::new() +} + +async fn run_audit( + relay: String, + mode: String, + spec: AuditSelection, + git_data_dir: Option, + json: bool, + key_args: KeyArgs, +) -> Result<()> { + let started_at = unix_timestamp(); + let started = Instant::now(); + let mut config = match mode.as_str() { + "shared" | "production" => AuditConfig::shared(), + "isolated" | "ci" => AuditConfig::isolated(), + _ => audit_runtime_error( + json, + &relay, + "", + started_at, + started, + anyhow!("Invalid mode: {mode}. Use 'shared' or 'isolated'"), + ), + }; + config.read_only = false; + let run_id = config.run_id.clone(); + + let keys = match load_keys(&key_args) { + Ok(keys) => keys, + Err(error) => audit_runtime_error(json, &relay, &run_id, started_at, started, error), + }; + let relay_domain = relay + .trim_start_matches("ws://") + .trim_start_matches("wss://") + .trim_end_matches('/') + .to_string(); + + if !json { + println!("🔍 GRASP Audit Tool"); + println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"); + println!("Relay: {relay}"); + println!("Mode: {mode}"); + println!("Spec: {spec}"); + println!("Run ID: {run_id}"); + if let Some(ref dir) = git_data_dir { + println!("Git Dir: {}", dir.display()); + } + println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"); + println!("Connecting to relay..."); + } + + let client_result = match keys { + Some(keys) => AuditClient::new_with_keys(&relay, config, keys).await, + None => AuditClient::new(&relay, config).await, + }; + let client = match client_result { + Ok(client) => client, + Err(error) => audit_runtime_error( + json, + &relay, + &run_id, + started_at, + started, + anyhow!("Failed to connect to relay: {error}"), + ), + }; + if !client.is_connected().await { + audit_runtime_error( + json, + &relay, + &run_id, + started_at, + started, + anyhow!("Could not establish connection to relay"), + ); + } + if !json { + println!("✓ Connected\n"); + } + + let mut results = Vec::new(); + match spec { + AuditSelection::Nip01Smoke => { + status(json, "Running NIP-01 smoke tests..."); + results.push(( + AuditSuite::Nip01Smoke, + specs::Nip01SmokeTests::run_all(&client).await, + )); + } + AuditSelection::Nip11 => { + status(json, "Running NIP-11 document tests..."); + results.push(( + AuditSuite::Nip11Document, + specs::Nip11DocumentTests::run_all(&client).await, + )); + } + AuditSelection::EventAcceptance => { + status(json, "Running event acceptance policy tests..."); + results.push(( + AuditSuite::EventAcceptance, + specs::EventAcceptancePolicyTests::run_all(&client).await, + )); + } + AuditSelection::Cors => { + status(json, "Running CORS tests..."); + results.push(( + AuditSuite::Cors, + specs::CorsTests::run_all(&client, &relay_domain).await, + )); + } + AuditSelection::GitClone => { + status(json, "Running Git clone tests..."); + results.push(( + AuditSuite::GitClone, + specs::GitCloneTests::run_all(&client, &relay_domain).await, + )); + } + AuditSelection::GitFilter => { + status(json, "Running Git filter capability tests..."); + results.push(( + AuditSuite::GitFilter, + specs::GitFilterTests::run_all(&client, &relay_domain).await, + )); + } + AuditSelection::PushAuth => { + status(json, "Running push authorization tests..."); + results.push(( + AuditSuite::PushAuthorization, + specs::PushAuthorizationTests::run_all(&client, &relay_domain).await, + )); + } + AuditSelection::RepoCreation => { + status(json, "Running repository creation tests..."); + results.push(( + AuditSuite::RepositoryCreation, + specs::RepositoryCreationTests::run_all(&client, &relay_domain).await, + )); + } + AuditSelection::Purgatory => { + status(json, "Running purgatory tests..."); + results.push(( + AuditSuite::Purgatory, + specs::PurgatoryTests::run_all(&client).await, + )); + } + AuditSelection::Grasp06 => { + status(json, "Running GRASP-06 tests..."); + results.push(( + AuditSuite::Grasp06All, + specs::Grasp06Tests::run_all(&client).await, + )); + } + AuditSelection::All => { + status(json, "Running all tests..."); + let mut grasp01 = AuditResult::new("All GRASP-01 Tests"); + progress(json, " → NIP-01 smoke tests..."); + grasp01.merge(specs::Nip01SmokeTests::run_all(&client).await); + progress(json, " → NIP-11 document tests..."); + grasp01.merge(specs::Nip11DocumentTests::run_all(&client).await); + progress(json, " → CORS tests..."); + grasp01.merge(specs::CorsTests::run_all(&client, &relay_domain).await); + progress(json, " → Repository creation tests..."); + grasp01.merge(specs::RepositoryCreationTests::run_all(&client, &relay_domain).await); + progress(json, " → Git clone tests..."); + grasp01.merge(specs::GitCloneTests::run_all(&client, &relay_domain).await); + progress(json, " → Git filter capability tests..."); + grasp01.merge(specs::GitFilterTests::run_all(&client, &relay_domain).await); + progress(json, " → Event acceptance policy tests..."); + grasp01.merge(specs::EventAcceptancePolicyTests::run_all(&client).await); + // Purgatory must run before push authorization because the latter + // replaces shared state events used by the purgatory assertions. + progress(json, " → Purgatory tests..."); + grasp01.merge(specs::PurgatoryTests::run_all(&client).await); + progress(json, " → Push authorization tests..."); + grasp01.merge(specs::PushAuthorizationTests::run_all(&client, &relay_domain).await); + results.push((AuditSuite::Grasp01All, grasp01)); + progress(json, " → GRASP-06 tests..."); + results.push(( + AuditSuite::Grasp06All, + specs::Grasp06Tests::run_all(&client).await, + )); + } + } + + let all_passed = results.iter().all(|(_, result)| result.all_passed()); + if json { + let specs = results + .iter() + .map(|(suite, result)| SpecReport::from_result(*suite, result)) + .collect(); + let report = AuditReport::completed(&relay, &run_id, started_at, started.elapsed(), specs); + println!("{}", serde_json::to_string(&report)?); + } else { + for (suite, result) in &results { + if *suite == AuditSuite::Grasp06All { + specs::Grasp06Tests::print_report(result); + } else { + result.print_report(); + } + } + if all_passed { + println!("✅ All tests passed!"); + } else { + println!("❌ Some tests failed"); + } + } + + if !all_passed { + std::process::exit(1); + } + Ok(()) +} + +fn status(json: bool, message: &str) { + if !json { + println!("{message}\n"); + } +} + +fn progress(json: bool, message: &str) { + if !json { + println!("{message}"); + } +} + fn load_keys(args: &KeyArgs) -> Result> { load_keys_with_env(args, std::env::var_os(GRASP_AUDIT_NSEC_ENV)) } @@ -425,6 +506,32 @@ fn load_keys_with_env(args: &KeyArgs, env_value: Option) -> Result ! { + if json { + let report = AuditReport::error( + relay, + run_id, + started_at, + started.elapsed(), + error.to_string(), + ); + println!( + "{}", + serde_json::to_string(&report).expect("audit error report is serializable") + ); + } else { + eprintln!("{error:#}"); + } + std::process::exit(2); +} + #[cfg(test)] mod tests { use super::*; @@ -540,4 +647,34 @@ mod tests { ); } } + + #[test] + fn audit_selection_is_validated_by_clap() { + let parsed = Cli::try_parse_from([ + "grasp-audit", + "audit", + "--relay", + "wss://example.com", + "--spec", + "not-an-audit", + ]); + + assert!(parsed.is_err()); + } + + #[test] + fn json_parse_error_helpers_detect_audit_and_relay() { + let args: Vec = [ + "grasp-audit", + "audit", + "--json", + "--relay=wss://relay.example", + ] + .into_iter() + .map(OsString::from) + .collect(); + + assert!(audit_json_requested(&args)); + assert_eq!(requested_relay(&args), "wss://relay.example"); + } } diff --git a/grasp-audit/src/lib.rs b/grasp-audit/src/lib.rs index e9f60be..0bde633 100644 --- a/grasp-audit/src/lib.rs +++ b/grasp-audit/src/lib.rs @@ -70,7 +70,8 @@ pub use probe::{ run_probe, run_probe_with_options, ProbeCheck, ProbeCheckName, ProbeOptions, ProbeReport, }; pub use report::{ - AuditOutcome, AuditSpec, AuditSuite, ResultCounts, SpecReport, TestOutcome, TestReport, + unix_timestamp, AuditOutcome, AuditReport, AuditRunOutcome, AuditSpec, AuditSuite, + ResultCounts, SpecReport, TestOutcome, TestReport, ToolInfo, AUDIT_REPORT_SCHEMA_VERSION, }; pub use result::{AuditResult, TestResult}; diff --git a/grasp-audit/src/report.rs b/grasp-audit/src/report.rs index f4bb6b2..bc87ccf 100644 --- a/grasp-audit/src/report.rs +++ b/grasp-audit/src/report.rs @@ -1,15 +1,17 @@ //! Stable, serializable results for compliance audit consumers. //! -//! These types describe completed spec and test results. They deliberately do -//! not include invocation details such as the relay URL, run timestamps, or a -//! top-level schema version; a CLI machine-output contract can wrap them later -//! without changing the audit domain model. +//! [`SpecReport`] and [`TestReport`] describe completed audit results, while +//! [`AuditReport`] adds the versioned run metadata used by the CLI's machine +//! output contract. use crate::result::{AuditResult, TestResult}; use crate::specs::grasp01::GRASP_COMMIT_ID; use crate::specs::grasp06::GRASP_06_COMMIT_ID; use serde::Serialize; -use std::time::Duration; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +/// Version of the full-audit JSON document emitted by the CLI. +pub const AUDIT_REPORT_SCHEMA_VERSION: u16 = 1; /// Maximum UTF-8 byte length retained for failures and skip reasons. pub const MAX_RESULT_MESSAGE_BYTES: usize = 4096; @@ -91,6 +93,22 @@ pub enum AuditOutcome { Fail, } +/// Outcome of a full audit invocation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AuditRunOutcome { + Pass, + Fail, + Error, +} + +/// Identity of the tool that produced a full audit report. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ToolInfo { + pub name: &'static str, + pub version: &'static str, +} + /// Outcome of a single audit test. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "snake_case")] @@ -135,6 +153,74 @@ pub struct SpecReport { pub tests: Vec, } +/// Stable machine-readable envelope for one full audit CLI invocation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct AuditReport { + pub schema_version: u16, + pub tool: ToolInfo, + pub relay_url: String, + pub run_id: String, + pub started_at: u64, + pub finished_at: u64, + pub duration_ms: u64, + pub outcome: AuditRunOutcome, + pub specs: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +impl AuditReport { + /// Build a report for an audit that reached test execution. + pub fn completed( + relay_url: impl Into, + run_id: impl Into, + started_at: u64, + duration: Duration, + specs: Vec, + ) -> Self { + let outcome = if specs.iter().all(|spec| spec.outcome == AuditOutcome::Pass) { + AuditRunOutcome::Pass + } else { + AuditRunOutcome::Fail + }; + + Self { + schema_version: AUDIT_REPORT_SCHEMA_VERSION, + tool: tool_info(), + relay_url: relay_url.into(), + run_id: run_id.into(), + started_at, + finished_at: unix_timestamp(), + duration_ms: duration_ms(duration), + outcome, + specs, + error: None, + } + } + + /// Build a report for a parse, setup, or runtime error. + pub fn error( + relay_url: impl Into, + run_id: impl Into, + started_at: u64, + duration: Duration, + error: impl AsRef, + ) -> Self { + Self { + schema_version: AUDIT_REPORT_SCHEMA_VERSION, + tool: tool_info(), + relay_url: relay_url.into(), + run_id: run_id.into(), + started_at, + finished_at: unix_timestamp(), + duration_ms: duration_ms(duration), + outcome: AuditRunOutcome::Error, + specs: Vec::new(), + error: Some(bounded_message(error.as_ref())), + } + } +} + impl SpecReport { /// Convert runtime results using the central metadata for `suite`. pub fn from_result(suite: AuditSuite, result: &AuditResult) -> Self { @@ -186,6 +272,21 @@ fn duration_ms(duration: Duration) -> u64 { duration.as_millis().min(u64::MAX as u128) as u64 } +/// Current Unix timestamp in seconds. +pub fn unix_timestamp() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +fn tool_info() -> ToolInfo { + ToolInfo { + name: "grasp-audit", + version: env!("CARGO_PKG_VERSION"), + } +} + fn bounded_message(value: &str) -> String { if value.len() <= MAX_RESULT_MESSAGE_BYTES { return value.to_owned(); @@ -258,4 +359,51 @@ mod tests { assert_eq!(TestReport::from(&result).duration_ms, 42); } + + #[test] + fn audit_report_wraps_stable_spec_reports() { + let mut passing = AuditResult::new("passing"); + passing.add(TestResult::new("passes", SpecRef::NostrRelayNip01Compliant, "passes").pass()); + let mut failing = AuditResult::new("failing"); + failing.add( + TestResult::new("fails", SpecRef::NostrRelayNip01Compliant, "fails").fail("broken"), + ); + + let report = AuditReport::completed( + "wss://relay.example", + "audit-1", + 1, + Duration::from_millis(42), + vec![ + SpecReport::from_result(AuditSuite::Grasp01All, &passing), + SpecReport::from_result(AuditSuite::Grasp06All, &failing), + ], + ); + let json = serde_json::to_value(&report).unwrap(); + + assert_eq!(json["schema_version"], AUDIT_REPORT_SCHEMA_VERSION); + assert_eq!(json["tool"]["name"], "grasp-audit"); + assert_eq!(json["relay_url"], "wss://relay.example"); + assert_eq!(json["run_id"], "audit-1"); + assert_eq!(json["duration_ms"], 42); + assert_eq!(json["outcome"], "fail"); + assert_eq!(json["specs"][0]["suite"], "grasp01-all"); + assert_eq!(json["specs"][1]["suite"], "grasp06-all"); + assert!(json.get("error").is_none()); + } + + #[test] + fn audit_error_reports_are_bounded_and_have_no_specs() { + let report = AuditReport::error( + "wss://relay.example", + "audit-1", + 1, + Duration::ZERO, + "é".repeat(MAX_RESULT_MESSAGE_BYTES), + ); + + assert_eq!(report.outcome, AuditRunOutcome::Error); + assert!(report.specs.is_empty()); + assert!(report.error.unwrap().len() <= MAX_RESULT_MESSAGE_BYTES); + } } diff --git a/grasp-audit/tests/cli_json.rs b/grasp-audit/tests/cli_json.rs new file mode 100644 index 0000000..efacf72 --- /dev/null +++ b/grasp-audit/tests/cli_json.rs @@ -0,0 +1,81 @@ +use std::process::Command; + +fn run_invalid_audit(args: &[&str]) -> serde_json::Value { + let output = Command::new(env!("CARGO_BIN_EXE_grasp-audit")) + .args(args) + .output() + .unwrap(); + + assert_eq!(output.status.code(), Some(2)); + assert!( + output.stderr.is_empty(), + "unexpected stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8(output.stdout).unwrap(); + assert_eq!(stdout.lines().count(), 1); + serde_json::from_str(&stdout).unwrap() +} + +#[test] +fn missing_required_argument_uses_audit_json_schema() { + let report = run_invalid_audit(&["audit", "--json"]); + + assert_eq!(report["schema_version"], 1); + assert_eq!(report["outcome"], "error"); + assert_eq!(report["relay_url"], ""); + assert_eq!(report["specs"], serde_json::json!([])); + assert!(report["error"] + .as_str() + .unwrap() + .contains("MissingRequiredArgument")); +} + +#[test] +fn unknown_argument_uses_audit_json_schema() { + let report = run_invalid_audit(&[ + "audit", + "--json", + "--relay", + "wss://relay.example", + "--unknown", + ]); + + assert_eq!(report["outcome"], "error"); + assert_eq!(report["relay_url"], "wss://relay.example"); + assert!(report["error"] + .as_str() + .unwrap() + .contains("UnknownArgument")); +} + +#[test] +fn invalid_audit_spec_uses_audit_json_schema() { + let report = run_invalid_audit(&[ + "audit", + "--json", + "--relay", + "wss://relay.example", + "--spec", + "not-an-audit", + ]); + + assert_eq!(report["outcome"], "error"); + assert!(report["error"].as_str().unwrap().contains("InvalidValue")); +} + +#[test] +fn invalid_mode_uses_audit_json_schema_without_connecting() { + let report = run_invalid_audit(&[ + "audit", + "--json", + "--relay", + "wss://relay.example", + "--mode", + "invalid", + ]); + + assert_eq!(report["outcome"], "error"); + assert_eq!(report["relay_url"], "wss://relay.example"); + assert!(report["error"].as_str().unwrap().contains("Invalid mode")); +}