From 50876d1ca47ab34473b70c7364b1bfb835f5c25a Mon Sep 17 00:00:00 2001 From: Mark Mansi Date: Mon, 26 Feb 2018 23:45:04 -0600 Subject: [PATCH] Only check the whitelist for some crates --- src/tools/tidy/src/deps.rs | 495 +++++++++++-------------------------- 1 file changed, 151 insertions(+), 344 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 4c9c0d8cf668..0d351c9e7350 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -28,10 +28,10 @@ static LICENSES: &'static [&'static str] = &[ "Unlicense/MIT", ]; -// These are exceptions to Rust's permissive licensing policy, and -// should be considered bugs. Exceptions are only allowed in Rust -// tooling. It is _crucial_ that no exception crates be dependencies -// of the Rust runtime (std / test). +/// These are exceptions to Rust's permissive licensing policy, and +/// should be considered bugs. Exceptions are only allowed in Rust +/// tooling. It is _crucial_ that no exception crates be dependencies +/// of the Rust runtime (std / test). static EXCEPTIONS: &'static [&'static str] = &[ "mdbook", // MPL2, mdbook "openssl", // BSD+advertising clause, cargo, mdbook @@ -48,353 +48,122 @@ static EXCEPTIONS: &'static [&'static str] = &[ "clippy_lints", // MPL-2.0 rls ]; -// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible. -static WHITELIST: &'static [(&'static str, &'static str)] = &[ -// ("advapi32-sys", "0.2.0"), -// ("aho-corasick", "0.5.3"), -// ("aho-corasick", "0.6.4"), -// ("alloc", "0.0.0"), -// ("alloc_jemalloc", "0.0.0"), -// ("alloc_system", "0.0.0"), -// ("ansi_term", "0.10.2"), -// ("ar", "0.3.1"), -// ("arena", "0.0.0"), -// ("atty", "0.2.6"), -// ("backtrace", "0.3.5"), -// ("backtrace-sys", "0.1.16"), -// ("bin_lib", "0.1.0"), -// ("bitflags", "0.7.0"), -// ("bitflags", "0.9.1"), -// ("bitflags", "1.0.1"), -// ("bootstrap", "0.0.0"), -// ("borrow_error", "0.1.0"), -// ("bufstream", "0.1.3"), -// ("build-manifest", "0.1.0"), -// ("build_helper", "0.1.0"), -// ("byteorder", "1.2.1"), -// ("cargo", "0.26.0"), -// ("cargo_metadata", "0.2.3"), -// ("cargo_metadata", "0.4.0"), -// ("cargotest", "0.1.0"), -// ("cargotest2", "0.1.0"), -// ("cc", "1.0.4"), -// ("cfg-if", "0.1.2"), -// ("chrono", "0.4.0"), -// ("clap", "2.29.0"), -// ("clippy", "0.0.186"), -// ("clippy-mini-macro-test", "0.2.0"), -// ("clippy_lints", "0.0.186"), -// ("cmake", "0.1.29"), -// ("coco", "0.1.1"), -// ("commoncrypto", "0.2.0"), -// ("commoncrypto-sys", "0.2.0"), -// ("compiler_builtins", "0.0.0"), -// ("compiletest", "0.0.0"), -// ("compiletest_rs", "0.3.6"), -// ("completion", "0.1.0"), -// ("core", "0.0.0"), -// ("core-foundation", "0.4.6"), -// ("core-foundation-sys", "0.4.6"), -// ("crates-io", "0.15.0"), -// ("crossbeam", "0.2.12"), -// ("crossbeam", "0.3.2"), -// ("crypto-hash", "0.3.0"), -// ("curl", "0.4.11"), -// ("curl-sys", "0.4.1"), -// ("deglob", "0.1.0"), -// ("derive-new", "0.5.0"), -// ("diff", "0.1.11"), -// ("dlmalloc", "0.0.0"), -// ("docopt", "0.8.3"), -// ("dtoa", "0.4.2"), -// ("duct", "0.8.2"), -// ("either", "1.4.0"), -// ("endian-type", "0.1.2"), -// ("enum_primitive", "0.1.1"), -// ("env_logger", "0.3.5"), -// ("env_logger", "0.4.3"), -// ("env_logger", "0.5.3"), -// ("error-chain", "0.11.0"), -// ("error-chain", "0.8.1"), -// ("error_index_generator", "0.0.0"), -// ("failure", "0.1.1"), -// ("failure_derive", "0.1.1"), -// ("features", "0.1.0"), -// ("filetime", "0.1.15"), -// ("find_all_refs_no_cfg_test", "0.1.0"), -// ("find_impls", "0.1.0"), -// ("flate2", "1.0.1"), -// ("fmt_macros", "0.0.0"), -// ("fnv", "1.0.6"), -// ("foreign-types", "0.3.2"), -// ("foreign-types-shared", "0.1.1"), -// ("fs2", "0.4.3"), -// ("fuchsia-zircon", "0.3.3"), -// ("fuchsia-zircon-sys", "0.3.3"), -// ("futures", "0.1.17"), -// ("getopts", "0.2.15"), -// ("git2", "0.6.11"), -// ("git2-curl", "0.7.0"), -// ("glob", "0.2.11"), -// ("globset", "0.2.1"), -// ("graphviz", "0.0.0"), -// ("hamcrest", "0.1.1"), -// ("handlebars", "0.29.1"), -// ("hex", "0.2.0"), -// ("hex", "0.3.1"), -// ("home", "0.3.0"), -// ("idna", "0.1.4"), -// ("if_chain", "0.1.2"), -// ("ignore", "0.3.1"), -// ("infer_bin", "0.1.0"), -// ("infer_custom_bin", "0.1.0"), -// ("infer_lib", "0.1.0"), -// ("installer", "0.0.0"), -// ("is-match", "0.1.0"), -// ("itertools", "0.6.5"), -// ("itertools", "0.7.6"), -// ("itoa", "0.3.4"), -// ("jobserver", "0.1.9"), -// ("json", "0.11.12"), -// ("jsonrpc-core", "8.0.1"), -// ("kernel32-sys", "0.2.2"), -// ("languageserver-types", "0.30.0"), -// ("lazy_static", "0.2.11"), -// ("lazy_static", "1.0.0"), -// ("lazycell", "0.5.1"), -// ("libc", "0.0.0"), -// ("libc", "0.2.36"), -// ("libgit2-sys", "0.6.19"), -// ("libssh2-sys", "0.2.6"), -// ("libz-sys", "1.0.18"), -// ("linkchecker", "0.1.0"), -// ("log", "0.3.9"), -// ("log", "0.4.1"), -// ("log_settings", "0.1.1"), -// ("lzma-sys", "0.1.9"), -// ("matches", "0.1.6"), -// ("mdbook", "0.1.2"), -// ("memchr", "0.1.11"), -// ("memchr", "2.0.1"), -// ("miniz-sys", "0.1.10"), -// ("miow", "0.2.1"), -// ("miri", "0.1.0"), -// ("multiple_bins", "0.1.0"), -// ("net2", "0.2.31"), -// ("nibble_vec", "0.0.3"), -// ("nix", "0.8.1"), -// ("num", "0.1.41"), -// ("num-bigint", "0.1.41"), -// ("num-complex", "0.1.41"), -// ("num-integer", "0.1.35"), -// ("num-iter", "0.1.34"), -// ("num-rational", "0.1.40"), -// ("num-traits", "0.1.41"), -// ("num_cpus", "1.8.0"), -// ("open", "1.2.1"), -// ("openssl", "0.9.23"), -// ("openssl-probe", "0.1.2"), -// ("openssl-sys", "0.9.24"), -// ("os_pipe", "0.5.1"), -// ("owning_ref", "0.3.3"), -// ("panic_abort", "0.0.0"), -// ("panic_unwind", "0.0.0"), -// ("parking_lot", "0.5.3"), -// ("parking_lot_core", "0.2.9"), -// ("percent-encoding", "1.0.1"), -// ("pest", "0.3.3"), -// ("pkg-config", "0.3.9"), -// ("proc_macro", "0.0.0"), -// ("profiler_builtins", "0.0.0"), -// ("pulldown-cmark", "0.0.15"), -// ("pulldown-cmark", "0.1.0"), -// ("quick-error", "1.2.1"), -// ("quine-mc_cluskey", "0.2.4"), -// ("quote", "0.3.15"), -// ("racer", "2.0.12"), -// ("radix_trie", "0.1.2"), -// ("rand", "0.3.20"), -// ("rayon", "0.9.0"), -// ("rayon-core", "1.3.0"), -// ("redox_syscall", "0.1.37"), -// ("redox_termios", "0.1.1"), -// ("reformat", "0.1.0"), -// ("reformat_with_range", "0.1.0"), -// ("regex", "0.1.80"), -// ("regex", "0.2.5"), -// ("regex-syntax", "0.3.9"), -// ("regex-syntax", "0.4.2"), -// ("remote-test-client", "0.1.0"), -// ("remote-test-server", "0.1.0"), -// ("rls", "0.125.0"), -// ("rls-analysis", "0.11.0"), -// ("rls-blacklist", "0.1.0"), -// ("rls-data", "0.15.0"), -// ("rls-rustc", "0.2.1"), -// ("rls-span", "0.4.0"), -// ("rls-vfs", "0.4.4"), -// ("rustbook", "0.1.0"), -// ("rustc", "0.0.0"), -// ("rustc-ap-rustc_cratesio_shim", "29.0.0"), -// ("rustc-ap-rustc_data_structures", "29.0.0"), -// ("rustc-ap-rustc_errors", "29.0.0"), -// ("rustc-ap-serialize", "29.0.0"), -// ("rustc-ap-syntax", "29.0.0"), -// ("rustc-ap-syntax_pos", "29.0.0"), -// ("rustc-demangle", "0.1.5"), -// ("rustc-main", "0.0.0"), -// ("rustc-serialize", "0.3.24"), -// ("rustc_allocator", "0.0.0"), -// ("rustc_apfloat", "0.0.0"), -// ("rustc_asan", "0.0.0"), -// ("rustc_back", "0.0.0"), -// ("rustc_binaryen", "0.0.0"), -// ("rustc_borrowck", "0.0.0"), -// ("rustc_const_eval", "0.0.0"), -// ("rustc_const_math", "0.0.0"), -// ("rustc_cratesio_shim", "0.0.0"), -// ("rustc_data_structures", "0.0.0"), -// ("rustc_driver", "0.0.0"), -// ("rustc_errors", "0.0.0"), -// ("rustc_incremental", "0.0.0"), -// ("rustc_lint", "0.0.0"), -// ("rustc_llvm", "0.0.0"), -// ("rustc_lsan", "0.0.0"), -// ("rustc_metadata", "0.0.0"), -// ("rustc_mir", "0.0.0"), -// ("rustc_msan", "0.0.0"), -// ("rustc_passes", "0.0.0"), -// ("rustc_platform_intrinsics", "0.0.0"), -// ("rustc_plugin", "0.0.0"), -// ("rustc_privacy", "0.0.0"), -// ("rustc_resolve", "0.0.0"), -// ("rustc_save_analysis", "0.0.0"), -// ("rustc_trans", "0.0.0"), -// ("rustc_trans_utils", "0.0.0"), -// ("rustc_tsan", "0.0.0"), -// ("rustc_typeck", "0.0.0"), -// ("rustdoc", "0.0.0"), -// ("rustdoc-themes", "0.1.0"), -// ("rustdoc-tool", "0.0.0"), -// ("rustfmt-nightly", "0.3.8"), -// ("same-file", "0.1.3"), -// ("same-file", "1.0.2"), -// ("schannel", "0.1.10"), -// ("scoped-tls", "0.1.0"), -// ("scopeguard", "0.1.2"), -// ("scopeguard", "0.3.3"), -// ("semver", "0.6.0"), -// ("semver", "0.8.0"), -// ("semver", "0.9.0"), -// ("semver-parser", "0.7.0"), -// ("serde", "1.0.27"), -// ("serde_derive", "1.0.27"), -// ("serde_derive_internals", "0.19.0"), -// ("serde_ignored", "0.0.4"), -// ("serde_json", "1.0.9"), -// ("serialize", "0.0.0"), -// ("shared_child", "0.2.1"), -// ("shell-escape", "0.1.3"), -// ("shlex", "0.1.1"), -// ("smallvec", "0.6.0"), -// ("socket2", "0.3.0"), -// ("stable_deref_trait", "1.0.0"), -// ("std", "0.0.0"), -// ("std_unicode", "0.0.0"), -// ("strsim", "0.6.0"), -// ("syn", "0.11.11"), -// ("synom", "0.11.3"), -// ("synstructure", "0.6.1"), -// ("syntax", "0.0.0"), -// ("syntax_ext", "0.0.0"), -// ("syntax_pos", "0.0.0"), -// ("syntex_errors", "0.52.0"), -// ("syntex_pos", "0.52.0"), -// ("syntex_syntax", "0.52.0"), -// ("tar", "0.4.14"), -// ("tempdir", "0.3.5"), -// ("term", "0.0.0"), -// ("term", "0.4.6"), -// ("termcolor", "0.3.3"), -// ("termion", "1.5.1"), -// ("test", "0.0.0"), -// ("textwrap", "0.9.0"), -// ("thread-id", "2.0.0"), -// ("thread_local", "0.2.7"), -// ("thread_local", "0.3.5"), -// ("tidy", "0.1.0"), -// ("time", "0.1.39"), -// ("toml", "0.2.1"), -// ("toml", "0.4.5"), -// ("toml-query", "0.6.0"), -// ("unicode-bidi", "0.3.4"), -// ("unicode-normalization", "0.1.5"), -// ("unicode-segmentation", "1.2.0"), -// ("unicode-width", "0.1.4"), -// ("unicode-xid", "0.0.3"), -// ("unicode-xid", "0.0.4"), -// ("unreachable", "1.0.0"), -// ("unstable-book-gen", "0.1.0"), -// ("unwind", "0.0.0"), -// ("url", "1.6.0"), -// ("url_serde", "0.2.0"), -// ("userenv-sys", "0.2.0"), -// ("utf8-ranges", "0.1.3"), -// ("utf8-ranges", "1.0.0"), -// ("vcpkg", "0.2.2"), -// ("vec_map", "0.8.0"), -// ("void", "1.0.2"), -// ("walkdir", "1.0.7"), -// ("walkdir", "2.0.1"), -// ("winapi", "0.2.8"), -// ("winapi", "0.3.4"), -// ("winapi-build", "0.1.1"), -// ("winapi-i686-pc-windows-gnu", "0.4.0"), -// ("winapi-x86_64-pc-windows-gnu", "0.4.0"), -// ("wincolor", "0.1.4"), -// ("workspace_symbol", "0.1.0"), -// ("ws2_32-sys", "0.2.1"), -// ("xattr", "0.1.11"), -// ("xz2", "0.1.4"), -// ("yaml-rust", "0.3.5"), +/// Which crates to check against the whitelist? +static WHITELIST_CRATES: &'static [Crate] = + &[Crate("rustc", "0.0.0"), Crate("rustc_trans", "0.0.0")]; + +/// Whitelist of crates rustc is allowed to depend on. Avoid adding to the list if possible. +static WHITELIST: &'static [Crate] = &[ +// Crate("ar", "0.3.1"), +// Crate("arena", "0.0.0"), +// Crate("backtrace", "0.3.5"), +// Crate("backtrace-sys", "0.1.16"), +// Crate("bitflags", "1.0.1"), +// Crate("build_helper", "0.1.0"), +// Crate("byteorder", "1.2.1"), +// Crate("cc", "1.0.4"), +// Crate("cfg-if", "0.1.2"), +// Crate("cmake", "0.1.29"), +// Crate("filetime", "0.1.15"), +// Crate("flate2", "1.0.1"), +// Crate("fmt_macros", "0.0.0"), +// Crate("fuchsia-zircon", "0.3.3"), +// Crate("fuchsia-zircon-sys", "0.3.3"), +// Crate("graphviz", "0.0.0"), +// Crate("jobserver", "0.1.9"), +// Crate("kernel32-sys", "0.2.2"), +// Crate("lazy_static", "0.2.11"), +// Crate("libc", "0.2.36"), +// Crate("log", "0.4.1"), +// Crate("log_settings", "0.1.1"), +// Crate("miniz-sys", "0.1.10"), +// Crate("num_cpus", "1.8.0"), +// Crate("owning_ref", "0.3.3"), +// Crate("parking_lot", "0.5.3"), +// Crate("parking_lot_core", "0.2.9"), +// Crate("rand", "0.3.20"), +// Crate("redox_syscall", "0.1.37"), +// Crate("rustc", "0.0.0"), +// Crate("rustc-demangle", "0.1.5"), +// Crate("rustc_allocator", "0.0.0"), +// Crate("rustc_apfloat", "0.0.0"), +// Crate("rustc_back", "0.0.0"), +// Crate("rustc_binaryen", "0.0.0"), +// Crate("rustc_const_eval", "0.0.0"), +// Crate("rustc_const_math", "0.0.0"), +// Crate("rustc_cratesio_shim", "0.0.0"), +// Crate("rustc_data_structures", "0.0.0"), +// Crate("rustc_errors", "0.0.0"), +// Crate("rustc_incremental", "0.0.0"), +// Crate("rustc_llvm", "0.0.0"), +// Crate("rustc_mir", "0.0.0"), +// Crate("rustc_platform_intrinsics", "0.0.0"), +// Crate("rustc_trans", "0.0.0"), +// Crate("rustc_trans_utils", "0.0.0"), +// Crate("serialize", "0.0.0"), +// Crate("smallvec", "0.6.0"), +// Crate("stable_deref_trait", "1.0.0"), +// Crate("syntax", "0.0.0"), +// Crate("syntax_pos", "0.0.0"), +// Crate("tempdir", "0.3.5"), +// Crate("unicode-width", "0.1.4"), +// Crate("winapi", "0.2.8"), +// Crate("winapi", "0.3.4"), +// Crate("winapi-build", "0.1.1"), +// Crate("winapi-i686-pc-windows-gnu", "0.4.0"), +// Crate("winapi-x86_64-pc-windows-gnu", "0.4.0"), ]; // Some types for Serde to deserialize the output of `cargo metadata` to... #[derive(Deserialize)] struct Output { - packages: Vec, + resolve: Resolve, // Not used, but needed to not confuse serde :P - #[allow(dead_code)] resolve: Resolve, -} - -#[derive(Deserialize)] -struct Package { - name: String, - version: String, - - // Not used, but needed to not confuse serde :P - #[allow(dead_code)] id: String, - #[allow(dead_code)] source: Option, - #[allow(dead_code)] manifest_path: String, + #[allow(dead_code)] packages: Vec, } // Not used, but needed to not confuse serde :P #[allow(dead_code)] +#[derive(Deserialize)] +struct Package { + name: String, + version: String, + id: String, + source: Option, + manifest_path: String, +} + #[derive(Deserialize)] struct Resolve { nodes: Vec, } -// Not used, but needed to not confuse serde :P -#[allow(dead_code)] #[derive(Deserialize)] struct ResolveNode { id: String, dependencies: Vec, } +/// A unique identifier for a crate +#[derive(Copy, Clone, PartialOrd, Ord, PartialEq, Eq, Debug, Hash)] +struct Crate<'a>(&'a str, &'a str); // (name, version) + +impl<'a> Crate<'a> { + pub fn from_str(s: &'a str) -> Self { + let mut parts = s.split(" "); + let name = parts.next().unwrap(); + let version = parts.next().unwrap(); + + Crate(name, version) + } + + pub fn id_str(&self) -> String { + format!("{} {}", self.0, self.1) + } +} + /// Checks the dependency at the given path. Changes `bad` to `true` if a check failed. /// /// Specifically, this checks that the license is correct. @@ -423,25 +192,27 @@ pub fn check(path: &Path, bad: &mut bool) { assert!(saw_dir, "no vendored source"); } -/// Checks the dependency at the given path. Changes `bad` to `true` if a check failed. +/// Checks the dependency of WHITELIST_CRATES at the given path. Changes `bad` to `true` if a check +/// failed. /// -/// Specifically, this checks that the dependencies are on the whitelist. +/// Specifically, this checks that the dependencies are on the WHITELIST. pub fn check_whitelist(path: &Path, cargo: &Path, bad: &mut bool) { - // Check dependencies - let deps: HashSet<_> = get_deps(&path, &cargo) - .into_iter() - .map(|Package { name, version, .. }| (name, version)) - .collect(); - let whitelist: HashSet<(String, String)> = WHITELIST - .iter() - .map(|&(n, v)| (n.to_owned(), v.to_owned())) - .collect(); + // Get dependencies from cargo metadata + let resolve = get_deps(path, cargo); - // Dependencies not in the whitelist - let mut unapproved: Vec<_> = deps.difference(&whitelist).collect(); + // Get the whitelist into a convenient form + let whitelist: HashSet<_> = WHITELIST.iter().cloned().collect(); + + // Check dependencies + let mut unapproved = Vec::new(); + for &krate in WHITELIST_CRATES.iter() { + let mut bad = check_crate_whitelist(&whitelist, &resolve, krate); + unapproved.append(&mut bad); + } // For ease of reading - unapproved.sort(); + unapproved.sort_unstable(); + unapproved.dedup(); if unapproved.len() > 0 { println!("Dependencies not on the whitelist:"); @@ -492,7 +263,7 @@ fn extract_license(line: &str) -> String { } /// Get the dependencies of the crate at the given path using `cargo metadata`. -fn get_deps(path: &Path, cargo: &Path) -> Vec { +fn get_deps(path: &Path, cargo: &Path) -> Resolve { // Run `cargo metadata` to get the set of dependencies let output = Command::new(cargo) .arg("metadata") @@ -506,5 +277,41 @@ fn get_deps(path: &Path, cargo: &Path) -> Vec { let output = String::from_utf8_lossy(&output); let output: Output = serde_json::from_str(&output).unwrap(); - output.packages + output.resolve +} + +/// Checks the dependencies of the given crate from the given cargo metadata to see if they are on +/// the whitelist. Returns a list of illegal dependencies. +fn check_crate_whitelist<'a>( + whitelist: &'a HashSet, + resolve: &'a Resolve, + krate: Crate<'a>, +) -> Vec> { + // Will contain bad deps + let mut unapproved = Vec::new(); + + // If this dependency is not on the WHITELIST, add to bad set + if !whitelist.contains(&krate) { + unapproved.push(krate); + } + + // Do a DFS in the crate graph (it's a DAG, so we know we have no cycles!) + let to_check = resolve + .nodes + .iter() + .find(|n| n.id.starts_with(&krate.id_str())) + .expect("crate does not exist"); + + for dep in to_check.dependencies.iter() { + let krate = Crate::from_str(dep); + let mut bad = check_crate_whitelist(whitelist, resolve, krate); + + unapproved.append(&mut bad); + } + + // Remove duplicates + unapproved.sort_unstable(); + unapproved.dedup(); + + unapproved }