From 147e2da13af98af32689bb8b3a3bb2b92862b8d4 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Thu, 6 Oct 2016 23:30:38 -0700 Subject: [PATCH] rustbuild: Use `cargo metadata` to learn about DAG This updates the commit to use workspaces to use `cargo metadata` instead of hardcoded lists about what to test. This should help us be resilient to updates in the future on behalf of the crate DAG and minimize the amount of files that need to be touched. --- src/Cargo.lock | 34 +++++++---- src/bootstrap/check.rs | 131 ++++++++++++++++++++++------------------- 2 files changed, 91 insertions(+), 74 deletions(-) diff --git a/src/Cargo.lock b/src/Cargo.lock index cd7a9a8d5e09..ecd94e5b7e92 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -87,6 +87,14 @@ dependencies = [ "rustc_unicode 0.0.0", ] +[[package]] +name = "compiler_builtins" +version = "0.0.0" +dependencies = [ + "core 0.0.0", + "gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "compiletest" version = "0.0.0" @@ -250,6 +258,13 @@ dependencies = [ [[package]] name = "proc_macro" version = "0.0.0" +dependencies = [ + "syntax 0.0.0", +] + +[[package]] +name = "proc_macro_plugin" +version = "0.0.0" dependencies = [ "log 0.0.0", "rustc_plugin 0.0.0", @@ -264,14 +279,6 @@ dependencies = [ "core 0.0.0", ] -[[package]] -name = "rbml" -version = "0.0.0" -dependencies = [ - "log 0.0.0", - "serialize 0.0.0", -] - [[package]] name = "regex" version = "0.1.73" @@ -302,7 +309,6 @@ dependencies = [ "fmt_macros 0.0.0", "graphviz 0.0.0", "log 0.0.0", - "rbml 0.0.0", "rustc_back 0.0.0", "rustc_bitflags 0.0.0", "rustc_const_math 0.0.0", @@ -395,7 +401,7 @@ dependencies = [ "flate 0.0.0", "graphviz 0.0.0", "log 0.0.0", - "proc_macro 0.0.0", + "proc_macro_plugin 0.0.0", "rustc 0.0.0", "rustc_back 0.0.0", "rustc_borrowck 0.0.0", @@ -434,7 +440,6 @@ version = "0.0.0" dependencies = [ "graphviz 0.0.0", "log 0.0.0", - "rbml 0.0.0", "rustc 0.0.0", "rustc_data_structures 0.0.0", "serialize 0.0.0", @@ -469,16 +474,16 @@ version = "0.0.0" dependencies = [ "flate 0.0.0", "log 0.0.0", - "rbml 0.0.0", + "proc_macro 0.0.0", "rustc 0.0.0", "rustc_back 0.0.0", - "rustc_bitflags 0.0.0", "rustc_const_math 0.0.0", "rustc_data_structures 0.0.0", "rustc_errors 0.0.0", "rustc_llvm 0.0.0", "serialize 0.0.0", "syntax 0.0.0", + "syntax_ext 0.0.0", "syntax_pos 0.0.0", ] @@ -647,6 +652,7 @@ dependencies = [ "alloc_system 0.0.0", "build_helper 0.1.0", "collections 0.0.0", + "compiler_builtins 0.0.0", "core 0.0.0", "gcc 0.3.35 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.0.0", @@ -661,6 +667,7 @@ dependencies = [ name = "std_shim" version = "0.1.0" dependencies = [ + "core 0.0.0", "std 0.0.0", ] @@ -681,6 +688,7 @@ version = "0.0.0" dependencies = [ "fmt_macros 0.0.0", "log 0.0.0", + "proc_macro 0.0.0", "rustc_errors 0.0.0", "syntax 0.0.0", "syntax_pos 0.0.0", diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index 8d160a993524..b8d0eb3ff996 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -13,19 +13,44 @@ //! This file implements the various regression test suites that we execute on //! our CI. +use std::collections::{HashMap, HashSet}; use std::env; -use std::fs::{self, File}; -use std::io::prelude::*; +use std::fs; use std::path::{PathBuf, Path}; use std::process::Command; use build_helper::output; +use rustc_serialize::json; use {Build, Compiler, Mode}; use util::{self, dylib_path, dylib_path_var}; const ADB_TEST_DIR: &'static str = "/data/tmp"; +#[derive(RustcDecodable)] +struct Output { + packages: Vec, + resolve: Resolve, +} + +#[derive(RustcDecodable)] +struct Package { + id: String, + name: String, + source: Option, +} + +#[derive(RustcDecodable)] +struct Resolve { + nodes: Vec, +} + +#[derive(RustcDecodable)] +struct ResolveNode { + id: String, + dependencies: Vec, +} + /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler. /// /// This tool in `src/tools` will verify the validity of all our links in the @@ -263,90 +288,74 @@ fn markdown_test(build: &Build, compiler: &Compiler, markdown: &Path) { /// It essentially is the driver for running `cargo test`. /// /// Currently this runs all tests for a DAG by passing a bunch of `-p foo` -/// arguments, and those arguments are discovered from `Cargo.lock`. +/// arguments, and those arguments are discovered from `cargo metadata`. pub fn krate(build: &Build, compiler: &Compiler, target: &str, mode: Mode) { - let (name, path, features, excluded) = match mode { + let (name, path, features, root) = match mode { Mode::Libstd => { - let excluded = vec![ - "alloc_jemalloc", "arena", "bootstrap", "cargotest", "compiletest", - "error_index_generator", "flate", "fmt_macros", "getopts", "graphviz", - "linkchecker", "log", "proc_macro", "rbml", "rustbook", "rustc", "rustc-main", - "rustc_back", "rustc_bitflags", "rustc_borrowck", "rustc_const_eval", - "rustc_const_math", "rustc_data_structures", "rustc_driver", "rustc_errors", - "rustc_incremental", "rustc_lint", "rustc_llvm", "rustc_metadata", "rustc_mir", - "rustc_passes", "rustc_platform_intrinsics", "rustc_plugin", "rustc_privacy", - "rustc_resolve", "rustc_save_analysis", "rustc_trans", "rustc_typeck", "rustdoc", - "serialize", "syntax", "syntax_ext", "syntax_pos", "term", "test", "test_shim", - "tidy", "unwind", - ]; - ("libstd", "src/rustc/std_shim", build.std_features(), excluded) + ("libstd", "src/rustc/std_shim", build.std_features(), "std_shim") } Mode::Libtest => { - let excluded = vec![ - "alloc", "alloc_jemalloc", "alloc_system", "arena", "bootstrap", "build_helper", - "cargotest", "collections", "compiletest", "core", "error_index_generator", - "flate", "fmt_macros", "graphviz", "libc", "linkchecker", "log", "panic_abort", - "panic_unwind", "proc_macro", "rand", "rbml", "rustbook", "rustc", "rustc-main", - "rustc_back", "rustc_bitflags", "rustc_borrowck", "rustc_const_eval", - "rustc_const_math", "rustc_data_structures", "rustc_driver", "rustc_errors", - "rustc_incremental", "rustc_lint", "rustc_llvm", "rustc_metadata", "rustc_mir", - "rustc_passes", "rustc_platform_intrinsics", "rustc_plugin", "rustc_privacy", - "rustc_resolve", "rustc_save_analysis", "rustc_trans", "rustc_typeck", - "rustc_unicode", "rustdoc", "serialize", "std", "std_shim", "syntax", "syntax_ext", - "syntax_pos", "tidy", "unwind", - ]; - ("libtest", "src/rustc/test_shim", String::new(), excluded) + ("libtest", "src/rustc/test_shim", String::new(), "test_shim") } Mode::Librustc => { - let excluded = vec![ - "alloc", "alloc_jemalloc", "alloc_system", "bootstrap", "cargotest", "collections", - "compiletest", "core", "error_index_generator", "getopts", "libc", "linkchecker", - "panic_abort", "panic_unwind", "rand", "rustbook", "rustc_unicode", "std", - "std_shim", "term", "test", "test_shim", "tidy", "unwind", - ]; - ("librustc", "src/rustc", build.rustc_features(), excluded) + ("librustc", "src/rustc", build.rustc_features(), "rustc-main") } _ => panic!("can only test libraries"), }; println!("Testing {} stage{} ({} -> {})", name, compiler.stage, compiler.host, target); + // Run `cargo metadata` to figure out what crates we're testing. + // + // Down below we're going to call `cargo test`, but to test the right set + // of packages we're going to have to know what `-p` arguments to pass it + // to know what crates to test. Here we run `cargo metadata` to learn about + // the dependency graph and what `-p` arguments there are. + let mut cargo = Command::new(&build.cargo); + cargo.arg("metadata") + .arg("--manifest-path").arg(build.src.join(path).join("Cargo.toml")); + let output = output(&mut cargo); + let output: Output = json::decode(&output).unwrap(); + let id2pkg = output.packages.iter() + .map(|pkg| (&pkg.id, pkg)) + .collect::>(); + let id2deps = output.resolve.nodes.iter() + .map(|node| (&node.id, &node.dependencies)) + .collect::>(); + // Build up the base `cargo test` command. + // + // Pass in some standard flags then iterate over the graph we've discovered + // in `cargo metadata` with the maps above and figure out what `-p` + // arguments need to get passed. let mut cargo = build.cargo(compiler, mode, target, "test"); cargo.arg("--manifest-path") .arg(build.src.join(path).join("Cargo.toml")) .arg("--features").arg(features); - // Generate a list of `-p` arguments to pass to the `cargo test` invocation - // by crawling the corresponding Cargo.lock file. - let lockfile = build.src.join("src").join("Cargo.lock"); - let mut contents = String::new(); - t!(t!(File::open(&lockfile)).read_to_string(&mut contents)); - let mut lines = contents.lines(); - while let Some(line) = lines.next() { - let prefix = "name = \""; - if !line.starts_with(prefix) { + let mut visited = HashSet::new(); + let root_pkg = output.packages.iter().find(|p| p.name == root).unwrap(); + let mut next = vec![&root_pkg.id]; + while let Some(id) = next.pop() { + // Skip any packages with sources listed, as these come from crates.io + // and we shouldn't be testing them. + if id2pkg[id].source.is_some() { continue } - lines.next(); // skip `version = ...` - - // skip crates.io or otherwise non-path crates - if let Some(line) = lines.next() { - if line.starts_with("source") { - continue + // Right now jemalloc is our only target-specific crate in the sense + // that it's not present on all platforms. Custom skip it here for now, + // but if we add more this probably wants to get more generalized. + if !id.contains("jemalloc") { + cargo.arg("-p").arg(&id2pkg[id].name); + } + for dep in id2deps[id] { + if visited.insert(dep) { + next.push(dep); } } - - let crate_name = &line[prefix.len()..line.len() - 1]; - - if excluded.contains(&crate_name) { - continue - } - - cargo.arg("-p").arg(crate_name); } // The tests are going to run with the *target* libraries, so we need to