diff --git a/kani-driver/src/args/mod.rs b/kani-driver/src/args/mod.rs index 13e1321d6494..2a0123929b4a 100644 --- a/kani-driver/src/args/mod.rs +++ b/kani-driver/src/args/mod.rs @@ -242,6 +242,11 @@ pub struct VerificationArgs { #[arg(long)] pub default_unwind: Option, + /// Output the verification results to a JSON file at the specified path. + /// This feature is unstable and it requires `-Z unstable-options` to be used + #[arg(long)] + pub export_json: Option, + /// When specified, the harness filter will only match the exact fully qualified name of a harness #[arg(long, requires("harnesses"))] pub exact: bool, @@ -734,6 +739,12 @@ impl ValidateArgs for VerificationArgs { UnstableFeature::UnstableOptions, )?; + self.common_args.check_unstable( + self.export_json.is_some(), + "export-json", + UnstableFeature::UnstableOptions, + )?; + Ok(()) }; @@ -777,6 +788,15 @@ impl ValidateArgs for VerificationArgs { "Conflicting options: --sarif isn't compatible with --output-format=old.", )); } + // `--output-format=old` bypasses CBMC's structured output entirely: `run_cbmc` mocks a + // result with no properties, and treats a timeout as success. An export produced from + // that would be indistinguishable from a real clean run. + if self.export_json.is_some() && self.output_format == OutputFormat::Old { + return Err(Error::raw( + ErrorKind::ArgumentConflict, + "Conflicting options: --export-json isn't compatible with --output-format=old.", + )); + } if self.concrete_playback.is_some() && self.jobs().will_multithread() { // Concrete playback currently embeds a lot of assumptions about the order in which harnesses get called. return Err(Error::raw( @@ -790,6 +810,19 @@ impl ValidateArgs for VerificationArgs { "Conflicting options: --sarif isn't compatible with --only-codegen.", )); } + // Neither code-generation-only mode runs verification, so there is nothing to export. + // `--only-codegen` would otherwise succeed without writing the file the user asked for, + // and `--no-codegen` would write a document describing a run that never happened. + if self.export_json.is_some() && (self.only_codegen || self.no_codegen) { + let incompatible = + if self.only_codegen { "--only-codegen" } else { "--no-codegen" }; + return Err(Error::raw( + ErrorKind::ArgumentConflict, + format!( + "Conflicting options: --export-json isn't compatible with {incompatible}." + ), + )); + } if self.jobs().will_multithread() && self.output_format != OutputFormat::Terse { // More verbose output formats make it hard to interpret output right now when run in parallel. // This can be removed when we change up how results are printed. @@ -1133,6 +1166,32 @@ mod tests { check_unstable_flag!("--no-slice-formula", no_slice_formula); } + #[test] + fn check_export_json_conflicts() { + expect_validation_error( + "kani file.rs -Z unstable-options --export-json out.json --output-format=old", + ErrorKind::ArgumentConflict, + ); + expect_validation_error( + "kani file.rs -Z unstable-options --export-json out.json --only-codegen", + ErrorKind::ArgumentConflict, + ); + expect_validation_error( + "kani file.rs -Z unstable-options --export-json out.json --no-codegen", + ErrorKind::ArgumentConflict, + ); + } + + #[test] + fn check_export_json_unstable() { + check_opt!( + "--export-json results.json", + Some(UnstableFeature::UnstableOptions), + export_json, + Some(PathBuf::from("results.json")) + ); + } + #[test] fn check_concrete_playback_unstable() { let check = |input: &str| { diff --git a/kani-driver/src/call_cbmc.rs b/kani-driver/src/call_cbmc.rs index 6891d49fd66a..dc4ae72b2db6 100644 --- a/kani-driver/src/call_cbmc.rs +++ b/kani-driver/src/call_cbmc.rs @@ -18,7 +18,7 @@ use tokio::process::Command as TokioCommand; use crate::args::common::Verbosity; use crate::args::{OutputFormat, VerificationArgs}; use crate::cbmc_output_parser::{ - CheckStatus, Property, VerificationOutput, extract_results, process_cbmc_output, + CheckStatus, ParserItem, Property, VerificationOutput, extract_results, process_cbmc_output, }; use crate::cbmc_property_renderer::{format_coverage, format_result, kani_cbmc_output_filter}; use crate::coverage::cov_results::{CoverageCheck, CoverageResults}; @@ -26,6 +26,141 @@ use crate::coverage::cov_results::{CoverageRegion, CoverageTerm}; use crate::session::KaniSession; use crate::util::render_command; +/// CBMC version and system information +#[derive(Debug, Clone)] +pub struct CbmcInfo { + pub version: String, + pub os_info: String, +} + +/// CBMC runtime and execution statistics +#[derive(Debug, Clone, Default)] +pub struct CbmcStats { + pub runtime_symex_s: Option, + pub size_program_expression: Option, + pub slicing_removed_assignments: Option, + pub vccs_generated: Option, + pub vccs_remaining: Option, + pub runtime_postprocess_equation_s: Option, + pub runtime_convert_ssa_s: Option, + pub runtime_post_process_s: Option, + pub runtime_solver_s: Option, + pub runtime_decision_procedure_s: Option, +} + +impl KaniSession { + /// Get CBMC version and system information + pub fn get_cbmc_info(&self) -> Result { + let output = std::process::Command::new("cbmc") + .arg("--version") + .output() + .map_err(|_| anyhow::Error::msg("Failed to run cbmc --version"))?; + + let version_output = String::from_utf8_lossy(&output.stdout); + let lines: Vec<&str> = version_output.lines().collect(); + + // Extract version from first line (e.g., "6.7.1 (cbmc-6.7.1)") + let version = lines + .first() + .and_then(|line| line.split_whitespace().next()) + .unwrap_or("unknown") + .to_string(); + + // For OS info, we'll use the system information since CBMC --version doesn't provide it + let os_info = format!( + "{} {} {}", + std::env::consts::ARCH, + std::env::consts::OS, + std::env::consts::FAMILY + ); + + Ok(CbmcInfo { version, os_info }) + } +} + +/// Collect the statistics CBMC reports for a single verification run. +/// +/// CBMC reports these as free-text status messages. `--json-ui`, which Kani always passes, wraps +/// each message in a JSON envelope carrying `messageType` and `messageText`, but it does not break +/// the numbers out into fields of their own, so the message text remains the only source available +/// (CBMC's `structured_datat` mechanism, which would render as real JSON fields, is not used by +/// these call sites). What the envelope does buy us is the ability to require a status message and +/// to anchor on CBMC's exact label, rather than searching arbitrary output for a loose pattern. +/// +/// Returns `None` when no message carried statistics, which is the case whenever CBMC did not get +/// far enough to report any. +fn merge_cbmc_stats(items: &[ParserItem]) -> Option { + let mut stats = CbmcStats::default(); + let mut found_any = false; + + for item in items { + if let ParserItem::Message { message_text, message_type } = item + && message_type == "STATUS-MESSAGE" + { + found_any |= record_cbmc_stat(message_text, &mut stats); + } + } + + found_any.then_some(stats) +} + +/// Record the statistic a single CBMC status message carries, if it carries one. Later messages win, +/// matching CBMC's own behaviour of reporting a running figure more than once. +/// Returns whether this message was recognized. +fn record_cbmc_stat(message: &str, stats: &mut CbmcStats) -> bool { + // "Generated 1 VCC(s), 1 remaining after simplification" + if let Some(counts) = message + .strip_prefix("Generated ") + .and_then(|rest| rest.strip_suffix(" remaining after simplification")) + && let Some((generated, remaining)) = counts.split_once(" VCC(s), ") + { + stats.vccs_generated = generated.parse().ok(); + stats.vccs_remaining = remaining.parse().ok(); + return stats.vccs_generated.is_some() || stats.vccs_remaining.is_some(); + } + + // "slicing removed 81 assignments", or "simple slicing removed 5 assignments" when only the + // simple slicer ran. CBMC emits one or the other; our schema has a single field for both. + if let Some(rest) = message.strip_suffix(" assignments") + && let Some(count) = rest + .strip_prefix("slicing removed ") + .or_else(|| rest.strip_prefix("simple slicing removed ")) + { + stats.slicing_removed_assignments = count.parse().ok(); + return stats.slicing_removed_assignments.is_some(); + } + + // Everything else is reported as "