diff --git a/CHANGELOG.md b/CHANGELOG.md index c6b312cd..df91d24d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,16 @@ ## [Unreleased] +### Added + +- **`rivet coverage --aggregate ...`** (#188 sub-issue 3). File-based + cross-repo V&V matrix aggregator: each repo's CI emits its + `rivet coverage --matrix --format json`, a top-level job merges them + into one matrix (text / markdown / html / json). Needs no GitHub API + access — inputs are plain files; duplicate `(repo, id)` rows are + coalesced so re-runs are idempotent, and the merged JSON re-feeds the + aggregator unchanged. + ## [0.9.0] — 2026-05-11 Theme: backlog drain. Ships the rivet-bundle command, the s-expr diff --git a/rivet-cli/src/main.rs b/rivet-cli/src/main.rs index 898f93d7..e01cf124 100644 --- a/rivet-cli/src/main.rs +++ b/rivet-cli/src/main.rs @@ -423,6 +423,15 @@ enum Command { /// markdown, or html output. Mutually exclusive with `--tests`. #[arg(long, conflicts_with = "tests")] matrix: bool, + + /// Aggregate one or more JSON matrices produced by + /// `rivet coverage --matrix --format json` (rivet#188 sub-issue 3, + /// the file-based cross-repo aggregator). Each CI job emits its + /// repo's JSON; a top-level job merges them with this flag. Implies + /// `--matrix`; the local project is not read. Repeat the flag or + /// pass several paths. + #[arg(long = "aggregate", value_name = "FILE", num_args = 1.., conflicts_with = "tests")] + aggregate: Vec, }, /// Generate a traceability matrix @@ -1732,8 +1741,11 @@ fn run(cli: Cli) -> Result { scan_paths, baseline, matrix, + aggregate, } => { - if *matrix { + if !aggregate.is_empty() { + cmd_coverage_matrix_aggregate(format, aggregate) + } else if *matrix { cmd_coverage_matrix(&cli, format, baseline.as_deref()) } else if *tests { cmd_coverage_tests(&cli, format, scan_paths) @@ -5827,6 +5839,104 @@ fn cmd_coverage_matrix(cli: &Cli, format: &str, baseline_name: Option<&str>) -> Ok(true) } +// ── V&V coverage matrix — cross-repo aggregator (rivet#188 sub-issue 3) ─ +// +// The file-based aggregator: each repo's CI runs +// `rivet coverage --matrix --format json` and uploads the result; a +// top-level job collects those JSON files and merges them with +// `rivet coverage --aggregate a.json b.json ... --format {text,markdown,html,json}`. +// No GitHub API access is needed — the inputs are plain files. + +fn json_string_list(value: Option<&serde_json::Value>) -> Vec { + value + .and_then(serde_json::Value::as_array) + .map(|seq| { + seq.iter() + .filter_map(|v| v.as_str()) + .map(str::to_owned) + .collect() + }) + .unwrap_or_default() +} + +/// Parse one JSON matrix file (the envelope emitted by `render_matrix_json`) +/// into `RepoStatusRow`s. +fn parse_matrix_json_file(path: &std::path::Path) -> Result> { + let text = std::fs::read_to_string(path) + .with_context(|| format!("reading matrix JSON {}", path.display()))?; + let value: serde_json::Value = serde_json::from_str(&text) + .with_context(|| format!("parsing matrix JSON {}", path.display()))?; + let repos = value + .get("repos") + .and_then(serde_json::Value::as_array) + .ok_or_else(|| { + anyhow::anyhow!( + "{}: not a `rivet coverage --matrix --format json` document \ + (missing `repos` array)", + path.display() + ) + })?; + let mut rows = Vec::with_capacity(repos.len()); + for repo in repos { + let repo_name = repo.get("repo").and_then(serde_json::Value::as_str); + let id = repo.get("id").and_then(serde_json::Value::as_str); + let (id, repo_name) = match (id, repo_name) { + (Some(i), Some(r)) => (i.to_owned(), r.to_owned()), + (Some(i), None) => (i.to_owned(), i.to_owned()), + (None, Some(r)) => (r.to_owned(), r.to_owned()), + (None, None) => { + anyhow::bail!( + "{}: a `repos` entry has neither `id` nor `repo`", + path.display() + ) + } + }; + let notes = repo + .get("notes") + .and_then(serde_json::Value::as_str) + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(str::to_owned); + rows.push(RepoStatusRow { + id, + repo: repo_name, + applied: json_string_list(repo.get("techniques_applied")), + gated: json_string_list(repo.get("techniques_gated_in_ci")), + notes, + }); + } + Ok(rows) +} + +/// Merge several JSON matrices into one and render it. +fn cmd_coverage_matrix_aggregate(format: &str, paths: &[PathBuf]) -> Result { + validate_format(format, &["text", "json", "markdown", "html"])?; + + let mut rows: Vec = Vec::new(); + let mut seen: std::collections::HashSet<(String, String)> = std::collections::HashSet::new(); + for path in paths { + for row in parse_matrix_json_file(path)? { + // First occurrence of a (repo, id) pair wins; later files that + // re-state the same row are ignored so re-running the aggregator + // over overlapping inputs is idempotent. + if seen.insert((row.repo.clone(), row.id.clone())) { + rows.push(row); + } + } + } + rows.sort_by(|a, b| a.repo.cmp(&b.repo).then_with(|| a.id.cmp(&b.id))); + let cols = matrix_columns(&rows); + + match format { + "json" => render_matrix_json(&rows, &cols), + "markdown" => render_matrix_markdown(&rows, &cols), + "html" => render_matrix_html(&rows, &cols), + _ => render_matrix_text(&rows, &cols), + } + + Ok(true) +} + /// Generate a traceability matrix. fn cmd_matrix( cli: &Cli, diff --git a/rivet-cli/tests/cli_commands.rs b/rivet-cli/tests/cli_commands.rs index a5120cc0..30cc9146 100644 --- a/rivet-cli/tests/cli_commands.rs +++ b/rivet-cli/tests/cli_commands.rs @@ -1948,6 +1948,180 @@ fn coverage_matrix_empty_project() { ); } +// ── rivet coverage --aggregate (rivet#188 sub-issue 3) ───────────────── + +/// Write two single-repo matrix JSON files (the shape +/// `rivet coverage --matrix --format json` emits) into a fresh tmpdir and +/// return `(tmpdir, path_a, path_b)`. +fn aggregate_inputs() -> (tempfile::TempDir, std::path::PathBuf, std::path::PathBuf) { + let tmp = tempfile::tempdir().expect("create temp dir"); + let a = tmp.path().join("rivet.json"); + let b = tmp.path().join("loom.json"); + std::fs::write( + &a, + r#"{"command":"coverage-matrix","columns":["kani","proptest"], + "repos":[{"id":"RS-RIVET","repo":"pulseengine/rivet", + "techniques_applied":["kani","proptest"], + "techniques_gated_in_ci":["proptest"],"notes":"ref", + "cells":[{"technique":"kani","status":"applied"}, + {"technique":"proptest","status":"gated"}]}]}"#, + ) + .expect("write a.json"); + std::fs::write( + &b, + r#"{"command":"coverage-matrix","columns":["miri"], + "repos":[{"id":"RS-LOOM","repo":"pulseengine/loom", + "techniques_applied":["miri","kani"],"techniques_gated_in_ci":[], + "notes":null,"cells":[{"technique":"miri","status":"applied"}]}]}"#, + ) + .expect("write b.json"); + (tmp, a, b) +} + +/// `--aggregate a.json b.json --format markdown` merges both repos into one +/// table whose columns are the union of every input's techniques. +#[test] +fn coverage_aggregate_markdown_merges_repos() { + let (tmp, a, b) = aggregate_inputs(); + let out = Command::new(rivet_bin()) + .args([ + "coverage", + "--aggregate", + a.to_str().unwrap(), + b.to_str().unwrap(), + "--format", + "markdown", + ]) + .output() + .expect("coverage --aggregate --format markdown"); + let stdout = String::from_utf8_lossy(&out.stdout); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + out.status.success(), + "must exit 0. stdout:\n{stdout}\nstderr:\n{stderr}" + ); + assert!( + stdout.contains("# V&V coverage matrix"), + "heading. {stdout}" + ); + for repo in ["pulseengine/rivet", "pulseengine/loom"] { + assert!( + stdout.contains(&format!("| {} |", repo)), + "row for {repo}. {stdout}" + ); + } + // Union of columns across both files, sorted. + let header = stdout + .lines() + .find(|l| l.starts_with("| repo |")) + .expect("header row"); + assert_eq!( + header, "| repo | kani | miri | proptest |", + "merged columns" + ); + // rivet has proptest CI-gated. + assert!(stdout.contains('●'), "gated glyph somewhere. {stdout}"); + drop(tmp); +} + +/// The aggregate JSON output uses the same envelope as the per-repo +/// command, so it can be fed straight back into `--aggregate`; duplicate +/// (repo, id) rows are coalesced so re-runs are idempotent. +#[test] +fn coverage_aggregate_json_roundtrips_and_dedups() { + let (tmp, a, b) = aggregate_inputs(); + // Pass `a` twice plus `b`; the duplicate must not produce a second row. + let out = Command::new(rivet_bin()) + .args([ + "coverage", + "--aggregate", + a.to_str().unwrap(), + a.to_str().unwrap(), + b.to_str().unwrap(), + "--format", + "json", + ]) + .output() + .expect("coverage --aggregate --format json"); + assert!( + out.status.success(), + "exit 0: {}", + String::from_utf8_lossy(&out.stderr) + ); + let stdout = String::from_utf8_lossy(&out.stdout); + let parsed: serde_json::Value = serde_json::from_str(&stdout).expect("aggregate JSON parses"); + assert_eq!( + parsed.get("command").and_then(|v| v.as_str()), + Some("coverage-matrix") + ); + let repos = parsed + .get("repos") + .and_then(|v| v.as_array()) + .expect("repos array"); + assert_eq!(repos.len(), 2, "duplicate row coalesced. {stdout}"); + assert_eq!( + parsed + .get("columns") + .and_then(|v| v.as_array()) + .map(Vec::len), + Some(3), + "merged column count. {stdout}" + ); + + // Re-feed the merged output back into the aggregator: same result. + let merged = tmp.path().join("merged.json"); + std::fs::write(&merged, stdout.as_bytes()).expect("write merged.json"); + let out2 = Command::new(rivet_bin()) + .args([ + "coverage", + "--aggregate", + merged.to_str().unwrap(), + "--format", + "json", + ]) + .output() + .expect("re-aggregate"); + assert!(out2.status.success(), "re-aggregate exit 0"); + let reparsed: serde_json::Value = + serde_json::from_str(&String::from_utf8_lossy(&out2.stdout)).expect("re-parse"); + assert_eq!( + reparsed + .get("repos") + .and_then(|v| v.as_array()) + .map(Vec::len), + Some(2), + "round-trip preserves rows" + ); +} + +/// A non-JSON or wrong-shaped input fails with a diagnostic naming the file. +#[test] +fn coverage_aggregate_bad_input_fails() { + let tmp = tempfile::tempdir().expect("temp dir"); + let bad = tmp.path().join("nope.json"); + std::fs::write(&bad, "this is not json").expect("write"); + let out = Command::new(rivet_bin()) + .args(["coverage", "--aggregate", bad.to_str().unwrap()]) + .output() + .expect("coverage --aggregate bad"); + assert!(!out.status.success(), "must fail on bad input"); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!(stderr.contains("nope.json"), "names the file. {stderr}"); + + let wrong = tmp.path().join("wrong.json"); + std::fs::write(&wrong, r#"{"hello":"world"}"#).expect("write"); + let out = Command::new(rivet_bin()) + .args(["coverage", "--aggregate", wrong.to_str().unwrap()]) + .output() + .expect("coverage --aggregate wrong"); + assert!(!out.status.success(), "must fail on wrong shape"); + let stderr = String::from_utf8_lossy(&out.stderr); + assert!( + stderr.contains("wrong.json") && stderr.contains("repos"), + "explains missing repos. {stderr}" + ); +} + /// `rivet stats --format json` exposes diagnostic counts so consumers /// don't need a second `rivet validate --format json` call just to /// get the severity breakdown.