From 22e119bbacd3aa11db04d84947b4fb2c5ccb0435 Mon Sep 17 00:00:00 2001 From: Zalathar Date: Mon, 12 Jun 2023 18:07:04 +1000 Subject: [PATCH] Add a custom `run-coverage` mode to compiletest --- src/tools/compiletest/src/common.rs | 3 + src/tools/compiletest/src/header.rs | 37 ++++- src/tools/compiletest/src/runtest.rs | 195 ++++++++++++++++++++++++++- 3 files changed, 232 insertions(+), 3 deletions(-) diff --git a/src/tools/compiletest/src/common.rs b/src/tools/compiletest/src/common.rs index 85a8fbcffbe2..81684c6f9f9e 100644 --- a/src/tools/compiletest/src/common.rs +++ b/src/tools/compiletest/src/common.rs @@ -66,6 +66,7 @@ string_enum! { JsDocTest => "js-doc-test", MirOpt => "mir-opt", Assembly => "assembly", + RunCoverage => "run-coverage", } } @@ -626,6 +627,7 @@ pub const UI_EXTENSIONS: &[&str] = &[ UI_STDERR_64, UI_STDERR_32, UI_STDERR_16, + UI_COVERAGE, ]; pub const UI_STDERR: &str = "stderr"; pub const UI_STDOUT: &str = "stdout"; @@ -635,6 +637,7 @@ pub const UI_RUN_STDOUT: &str = "run.stdout"; pub const UI_STDERR_64: &str = "64bit.stderr"; pub const UI_STDERR_32: &str = "32bit.stderr"; pub const UI_STDERR_16: &str = "16bit.stderr"; +pub const UI_COVERAGE: &str = "coverage"; /// Absolute path to the directory where all output for all tests in the given /// `relative_dir` group should reside. Example: diff --git a/src/tools/compiletest/src/header.rs b/src/tools/compiletest/src/header.rs index 00720e1c4db7..b699accf34c1 100644 --- a/src/tools/compiletest/src/header.rs +++ b/src/tools/compiletest/src/header.rs @@ -612,10 +612,25 @@ pub fn line_directive<'line>( } fn iter_header(testfile: &Path, rdr: R, it: &mut dyn FnMut(Option<&str>, &str, usize)) { + iter_header_extra(testfile, rdr, &[], it) +} + +fn iter_header_extra( + testfile: &Path, + rdr: impl Read, + extra_directives: &[&str], + it: &mut dyn FnMut(Option<&str>, &str, usize), +) { if testfile.is_dir() { return; } + // Process any extra directives supplied by the caller (e.g. because they + // are implied by the test mode), with a dummy line number of 0. + for directive in extra_directives { + it(None, directive, 0); + } + let comment = if testfile.extension().map(|e| e == "rs") == Some(true) { "//" } else { "#" }; let mut rdr = BufReader::new(rdr); @@ -894,7 +909,27 @@ pub fn make_test_description( let mut ignore_message = None; let mut should_fail = false; - iter_header(path, src, &mut |revision, ln, line_number| { + let extra_directives: &[&str] = match config.mode { + // The run-coverage tests are treated as having these extra directives, + // without needing to specify them manually in every test file. + // (Some of the comments below have been copied over from + // `tests/run-make/coverage-reports/Makefile`.) + Mode::RunCoverage => { + &[ + "needs-profiler-support", + // FIXME(mati865): MinGW GCC miscompiles compiler-rt profiling library but with Clang it works + // properly. Since we only have GCC on the CI ignore the test for now. + "ignore-windows-gnu", + // FIXME(pietroalbini): this test currently does not work on cross-compiled + // targets because remote-test is not capable of sending back the *.profraw + // files generated by the LLVM instrumentation. + "ignore-cross-compile", + ] + } + _ => &[], + }; + + iter_header_extra(path, src, extra_directives, &mut |revision, ln, line_number| { if revision.is_some() && revision != cfg { return; } diff --git a/src/tools/compiletest/src/runtest.rs b/src/tools/compiletest/src/runtest.rs index 91b9f668ce51..920db24a1a4f 100644 --- a/src/tools/compiletest/src/runtest.rs +++ b/src/tools/compiletest/src/runtest.rs @@ -6,8 +6,8 @@ use crate::common::{Assembly, Incremental, JsDocTest, MirOpt, RunMake, RustdocJs use crate::common::{Codegen, CodegenUnits, DebugInfo, Debugger, Rustdoc}; use crate::common::{CompareMode, FailMode, PassMode}; use crate::common::{Config, TestPaths}; -use crate::common::{Pretty, RunPassValgrind}; -use crate::common::{UI_RUN_STDERR, UI_RUN_STDOUT}; +use crate::common::{Pretty, RunCoverage, RunPassValgrind}; +use crate::common::{UI_COVERAGE, UI_RUN_STDERR, UI_RUN_STDOUT}; use crate::compute_diff::{write_diff, write_filtered_diff}; use crate::errors::{self, Error, ErrorKind}; use crate::header::TestProps; @@ -253,6 +253,7 @@ impl<'test> TestCx<'test> { MirOpt => self.run_mir_opt_test(), Assembly => self.run_assembly_test(), JsDocTest => self.run_js_doc_test(), + RunCoverage => self.run_coverage_test(), } } @@ -465,6 +466,184 @@ impl<'test> TestCx<'test> { } } + fn run_coverage_test(&self) { + let should_run = self.run_if_enabled(); + let proc_res = self.compile_test(should_run, Emit::None); + + if !proc_res.status.success() { + self.fatal_proc_rec("compilation failed!", &proc_res); + } + drop(proc_res); + + if let WillExecute::Disabled = should_run { + return; + } + + let profraw_path = self.output_base_dir().join("default.profraw"); + let profdata_path = self.output_base_dir().join("default.profdata"); + + // Delete any existing profraw/profdata files to rule out unintended + // interference between repeated test runs. + if profraw_path.exists() { + std::fs::remove_file(&profraw_path).unwrap(); + } + if profdata_path.exists() { + std::fs::remove_file(&profdata_path).unwrap(); + } + + let proc_res = self.exec_compiled_test_general( + &[("LLVM_PROFILE_FILE", &profraw_path.to_str().unwrap())], + false, + ); + if self.props.failure_status.is_some() { + self.check_correct_failure_status(&proc_res); + } else if !proc_res.status.success() { + self.fatal_proc_rec("test run failed!", &proc_res); + } + drop(proc_res); + + // Run `llvm-profdata merge` to index the raw coverage output. + let proc_res = self.run_llvm_tool("llvm-profdata", |cmd| { + cmd.args(["merge", "--sparse", "--output"]); + cmd.arg(&profdata_path); + cmd.arg(&profraw_path); + }); + if !proc_res.status.success() { + self.fatal_proc_rec("llvm-profdata merge failed!", &proc_res); + } + drop(proc_res); + + // Run `llvm-cov show` to produce a coverage report in text format. + let proc_res = self.run_llvm_tool("llvm-cov", |cmd| { + cmd.args(["show", "--format=text", "--show-line-counts-or-regions"]); + + cmd.arg("--Xdemangler"); + cmd.arg(self.config.rust_demangler_path.as_ref().unwrap()); + + cmd.arg("--instr-profile"); + cmd.arg(&profdata_path); + + cmd.arg("--object"); + cmd.arg(&self.make_exe_name()); + }); + if !proc_res.status.success() { + self.fatal_proc_rec("llvm-cov show failed!", &proc_res); + } + + let kind = UI_COVERAGE; + + let expected_coverage = self.load_expected_output(kind); + let normalized_actual_coverage = + self.normalize_coverage_output(&proc_res.stdout).unwrap_or_else(|err| { + self.fatal_proc_rec(&err, &proc_res); + }); + + let coverage_errors = self.compare_output( + kind, + &normalized_actual_coverage, + &expected_coverage, + self.props.compare_output_lines_by_subset, + ); + + if coverage_errors > 0 { + self.fatal_proc_rec( + &format!("{} errors occurred comparing coverage output.", coverage_errors), + &proc_res, + ); + } + } + + fn run_llvm_tool(&self, name: &str, configure_cmd_fn: impl FnOnce(&mut Command)) -> ProcRes { + let tool_path = self + .config + .llvm_bin_dir + .as_ref() + .expect("this test expects the LLVM bin dir to be available") + .join(name); + + let mut cmd = Command::new(tool_path); + configure_cmd_fn(&mut cmd); + + let output = cmd.output().unwrap_or_else(|_| panic!("failed to exec `{cmd:?}`")); + + let proc_res = ProcRes { + status: output.status, + stdout: String::from_utf8(output.stdout).unwrap(), + stderr: String::from_utf8(output.stderr).unwrap(), + cmdline: format!("{cmd:?}"), + }; + self.dump_output(&proc_res.stdout, &proc_res.stderr); + + proc_res + } + + fn normalize_coverage_output(&self, coverage: &str) -> Result { + let normalized = self.normalize_output(coverage, &[]); + + let mut lines = normalized.lines().collect::>(); + + Self::sort_coverage_subviews(&mut lines)?; + + let joined_lines = lines.iter().flat_map(|line| [line, "\n"]).collect::(); + Ok(joined_lines) + } + + fn sort_coverage_subviews(coverage_lines: &mut Vec<&str>) -> Result<(), String> { + let mut output_lines = Vec::new(); + + // We accumulate a list of zero or more "subviews", where each + // subview is a list of one or more lines. + let mut subviews: Vec> = Vec::new(); + + fn flush<'a>(subviews: &mut Vec>, output_lines: &mut Vec<&'a str>) { + if subviews.is_empty() { + return; + } + + // Take and clear the list of accumulated subviews. + let mut subviews = std::mem::take(subviews); + + // The last "subview" should be just a boundary line on its own, + // so exclude it when sorting the other subviews. + let except_last = subviews.len() - 1; + (&mut subviews[..except_last]).sort(); + + for view in subviews { + for line in view { + output_lines.push(line); + } + } + } + + for (line, line_num) in coverage_lines.iter().zip(1..) { + if line.starts_with(" ------------------") { + // This is a subview boundary line, so start a new subview. + subviews.push(vec![line]); + } else if line.starts_with(" |") { + // Add this line to the current subview. + subviews + .last_mut() + .ok_or(format!( + "unexpected subview line outside of a subview on line {line_num}" + ))? + .push(line); + } else { + // This line is not part of a subview, so sort and print any + // accumulated subviews, and then print the line as-is. + flush(&mut subviews, &mut output_lines); + output_lines.push(line); + } + } + + flush(&mut subviews, &mut output_lines); + assert!(subviews.is_empty()); + + assert_eq!(output_lines.len(), coverage_lines.len()); + *coverage_lines = output_lines; + + Ok(()) + } + fn run_pretty_test(&self) { if self.props.pp_exact.is_some() { logv(self.config, "testing for exact pretty-printing".to_owned()); @@ -1822,6 +2001,7 @@ impl<'test> TestCx<'test> { || self.is_vxworks_pure_static() || self.config.target.contains("bpf") || !self.config.target_cfg().dynamic_linking + || self.config.mode == RunCoverage { // We primarily compile all auxiliary libraries as dynamic libraries // to avoid code size bloat and large binaries as much as possible @@ -1832,6 +2012,10 @@ impl<'test> TestCx<'test> { // dynamic libraries so we just go back to building a normal library. Note, // however, that for MUSL if the library is built with `force_host` then // it's ok to be a dylib as the host should always support dylibs. + // + // Coverage tests want static linking by default so that coverage + // mappings in auxiliary libraries can be merged into the final + // executable. (false, Some("lib")) } else { (true, Some("dylib")) @@ -2009,6 +2193,10 @@ impl<'test> TestCx<'test> { } } DebugInfo => { /* debuginfo tests must be unoptimized */ } + RunCoverage => { + // Coverage reports are affected by optimization level, and + // the current snapshots assume no optimization by default. + } _ => { rustc.arg("-O"); } @@ -2075,6 +2263,9 @@ impl<'test> TestCx<'test> { rustc.arg(dir_opt); } + RunCoverage => { + rustc.arg("-Cinstrument-coverage"); + } RunPassValgrind | Pretty | DebugInfo | Codegen | Rustdoc | RustdocJson | RunMake | CodegenUnits | JsDocTest | Assembly => { // do not use JSON output