From c8f02aa2f438da9ee2e66d6223dfff3ac0424df2 Mon Sep 17 00:00:00 2001 From: bors Date: Thu, 9 Jun 2022 18:20:08 +0000 Subject: [PATCH] Auto merge of #97911 - dtolnay:numcpu, r=Mark-Simulacrum Revert "remove num_cpus dependency" in rustc and update cargo Fixes #97549. This PR reverts #94524 and does a Cargo update to pull in rust-lang/cargo#10737. Rust 1.61.0 has a regression in which it misidentifies the number of available CPUs in some environments, leading to enormously increased memory usage and failing builds. In between Rust 1.60 and 1.61 both rustc and cargo replaced some uses of `num_cpus` with `available_parallelism`, which eliminated support for cgroupv1, still apparently in common use. This PR switches both rustc and cargo back to using `num_cpus` in order to support environments where the available parallelism is controlled by cgroupv1. Both can use `available_parallism` again once it handles cgroupv1 (if ever). I have confirmed that the rustc part of this PR fixes the memory usage regression in my non-Cargo environment, and others have confirmed in #97549 that the Cargo regression was at fault for the memory usage regression in their environments. --- Cargo.lock | 4 ++++ compiler/rustc_session/Cargo.toml | 1 + compiler/rustc_session/src/options.rs | 2 +- src/bootstrap/Cargo.toml | 1 + src/bootstrap/config.rs | 2 +- src/bootstrap/flags.rs | 2 +- src/bootstrap/lib.rs | 4 +--- src/tools/build-manifest/Cargo.toml | 1 + src/tools/build-manifest/src/main.rs | 2 +- src/tools/cargo | 2 +- 10 files changed, 13 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f143b097b7f..a186e173ccf8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -218,6 +218,7 @@ dependencies = [ "getopts", "ignore", "libc", + "num_cpus", "once_cell", "opener", "pretty_assertions", @@ -247,6 +248,7 @@ dependencies = [ "anyhow", "flate2", "hex 0.4.2", + "num_cpus", "rayon", "serde", "serde_json", @@ -346,6 +348,7 @@ dependencies = [ "libgit2-sys", "log", "memchr", + "num_cpus", "opener", "openssl", "os_info", @@ -4338,6 +4341,7 @@ name = "rustc_session" version = "0.0.0" dependencies = [ "getopts", + "num_cpus", "rustc_ast", "rustc_data_structures", "rustc_errors", diff --git a/compiler/rustc_session/Cargo.toml b/compiler/rustc_session/Cargo.toml index 6b1eaa4d399d..37cfc4a0dc3c 100644 --- a/compiler/rustc_session/Cargo.toml +++ b/compiler/rustc_session/Cargo.toml @@ -15,5 +15,6 @@ rustc_serialize = { path = "../rustc_serialize" } rustc_data_structures = { path = "../rustc_data_structures" } rustc_span = { path = "../rustc_span" } rustc_fs_util = { path = "../rustc_fs_util" } +num_cpus = "1.0" rustc_ast = { path = "../rustc_ast" } rustc_lint_defs = { path = "../rustc_lint_defs" } diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 14e918660dd3..5c404b6cd228 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -578,7 +578,7 @@ mod parse { crate fn parse_threads(slot: &mut usize, v: Option<&str>) -> bool { match v.and_then(|s| s.parse().ok()) { Some(0) => { - *slot = std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get); + *slot = ::num_cpus::get(); true } Some(i) => { diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index dea8d998bded..cea7b7c949e2 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -37,6 +37,7 @@ test = false [dependencies] cmake = "0.1.38" filetime = "0.2" +num_cpus = "1.0" getopts = "0.2.19" cc = "1.0.69" libc = "0.2" diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index e39c9fa1c5a6..5457cb52cc8b 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -1345,7 +1345,7 @@ fn set(field: &mut T, val: Option) { fn threads_from_config(v: u32) -> u32 { match v { - 0 => std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32, + 0 => num_cpus::get() as u32, n => n, } } diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 4cd835ade642..bddf70424bec 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -210,7 +210,7 @@ To learn more about a subcommand, run `./x.py -h`", let j_msg = format!( "number of jobs to run in parallel; \ defaults to {} (this host's logical CPU count)", - std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) + num_cpus::get() ); opts.optopt("j", "jobs", &j_msg, "JOBS"); opts.optflag("h", "help", "print this help message"); diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 5d32b4f801a1..68d387b04257 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -993,9 +993,7 @@ impl Build { /// Returns the number of parallel jobs that have been configured for this /// build. fn jobs(&self) -> u32 { - self.config.jobs.unwrap_or_else(|| { - std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32 - }) + self.config.jobs.unwrap_or_else(|| num_cpus::get() as u32) } fn debuginfo_map_to(&self, which: GitRepo) -> Option { diff --git a/src/tools/build-manifest/Cargo.toml b/src/tools/build-manifest/Cargo.toml index c437bde5ae69..c022d3aa0acd 100644 --- a/src/tools/build-manifest/Cargo.toml +++ b/src/tools/build-manifest/Cargo.toml @@ -13,3 +13,4 @@ tar = "0.4.29" sha2 = "0.10.1" rayon = "1.5.1" hex = "0.4.2" +num_cpus = "1.13.0" diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index a1dfbef0601a..6338e467055c 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -210,7 +210,7 @@ fn main() { let num_threads = if let Some(num) = env::var_os("BUILD_MANIFEST_NUM_THREADS") { num.to_str().unwrap().parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS") } else { - std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) + num_cpus::get() }; rayon::ThreadPoolBuilder::new() .num_threads(num_threads) diff --git a/src/tools/cargo b/src/tools/cargo index 4751950ccc94..a748cf5a3e66 160000 --- a/src/tools/cargo +++ b/src/tools/cargo @@ -1 +1 @@ -Subproject commit 4751950ccc948c07047e62c20adf423d7e5f668c +Subproject commit a748cf5a3e666bc2dcdf54f37adef8ef22196452