Auto merge of #77762 - pietroalbini:dist-build-manifest, r=Mark-Simulacrum

Dist build manifest

This PR makes two changes that should remove a significant chunk of the time spent in our release process: cloning the `rust-lang/rust` monorepo, all its submodules, and building `bootstrap` to then invoke `build-manifest`:

* `build-manifest` doesn't rely on a clone of the monorepo being present anymore. The only remaining bit of information it fetched from it (the Rust version) is instead bundled in the binary.
* A new "component" is added, `build-manifest`. That component includes a prebuilt version of the tool, and it's *not* included in the Rustup manifest. This will allow `promote-release` to directly invoke the tool without interacting with our build system.
* The Linux x86_64 CI is changed to also build the component mentioned above. It's the only CI builder tasked to do so, and to cleanly support this a new `--include-default-paths` flag was added to `./x.py`.
* The `BUILD_MANIFEST_NUM_THREADS` environment variable is added to configure the number of threads at runtime.

This PR is best reviewed commit-by-commit.
r? `@Mark-Simulacrum`

src/tools/build-manifest/Cargo.toml

@@ -14,3 +14,4 @@ tar = "0.4.29"
 sha2 = "0.9.1"
 rayon = "1.3.1"
 hex = "0.4.2"
+num_cpus = "1.13.0"

src/tools/build-manifest/README.md

@@ -21,8 +21,8 @@ Then, you can generate the manifest and all the packages from `path/to/dist` to
 
 ```
 $ cargo +nightly run path/to/dist path/to/output 1970-01-01 http://example.com \
-    CHANNEL path/to/rust/repo
+    CHANNEL VERSION
 ```
 
 Remember to replace `CHANNEL` with the channel you produced dist artifacts of
-and `path/to/rust/repo` with the path to your checkout of the Rust repository.
+and `VERSION` with the current Rust version.

src/tools/build-manifest/src/main.rs

@@ -205,15 +205,20 @@ fn main() {
     //
     // Once the old release process is fully decommissioned, the environment variable, all the
     // related code in this tool and ./x.py dist hash-and-sign can be removed.
-    let legacy = env::var("BUILD_MANIFEST_LEGACY").is_ok();
+    let legacy = env::var_os("BUILD_MANIFEST_LEGACY").is_some();
 
-    // Avoid overloading the old server in legacy mode.
-    if legacy {
-        rayon::ThreadPoolBuilder::new()
-            .num_threads(1)
-            .build_global()
-            .expect("failed to initialize Rayon");
-    }
+    let num_threads = if legacy {
+        // Avoid overloading the old server in legacy mode.
+        1
+    } else if let Some(num) = env::var_os("BUILD_MANIFEST_NUM_THREADS") {
+        num.to_str().unwrap().parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS")
+    } else {
+        num_cpus::get()
+    };
+    rayon::ThreadPoolBuilder::new()
+        .num_threads(num_threads)
+        .build_global()
+        .expect("failed to initialize Rayon");
 
     let mut args = env::args().skip(1);
     let input = PathBuf::from(args.next().unwrap());
@@ -221,7 +226,6 @@ fn main() {
     let date = args.next().unwrap();
     let s3_address = args.next().unwrap();
     let channel = args.next().unwrap();
-    let monorepo_path = args.next().unwrap();
 
     // Do not ask for a passphrase while manually testing
     let mut passphrase = String::new();
@@ -231,7 +235,7 @@ fn main() {
     }
 
     Builder {
-        versions: Versions::new(&channel, &input, Path::new(&monorepo_path)).unwrap(),
+        versions: Versions::new(&channel, &input).unwrap(),
 
         input,
         output,

src/tools/build-manifest/src/versions.rs

@@ -1,4 +1,4 @@
-use anyhow::{Context, Error};
+use anyhow::Error;
 use flate2::read::GzDecoder;
 use std::collections::HashMap;
 use std::fs::File;
@@ -7,6 +7,7 @@ use std::path::{Path, PathBuf};
 use tar::Archive;
 
 const DEFAULT_TARGET: &str = "x86_64-unknown-linux-gnu";
+const RUSTC_VERSION: &str = include_str!("../../../version");
 
 #[derive(Debug, Hash, Eq, PartialEq, Clone)]
 pub(crate) enum PkgType {
@@ -87,26 +88,13 @@ pub(crate) struct VersionInfo {
 
 pub(crate) struct Versions {
     channel: String,
-    rustc_version: String,
     dist_path: PathBuf,
     versions: HashMap<PkgType, VersionInfo>,
 }
 
 impl Versions {
-    pub(crate) fn new(
-        channel: &str,
-        dist_path: &Path,
-        monorepo_root: &Path,
-    ) -> Result<Self, Error> {
-        Ok(Self {
-            channel: channel.into(),
-            rustc_version: std::fs::read_to_string(monorepo_root.join("src").join("version"))
-                .context("failed to read the rustc version from src/version")?
-                .trim()
-                .to_string(),
-            dist_path: dist_path.into(),
-            versions: HashMap::new(),
-        })
+    pub(crate) fn new(channel: &str, dist_path: &Path) -> Result<Self, Error> {
+        Ok(Self { channel: channel.into(), dist_path: dist_path.into(), versions: HashMap::new() })
     }
 
     pub(crate) fn channel(&self) -> &str {
@@ -184,10 +172,10 @@ impl Versions {
     ) -> Result<String, Error> {
         let component_name = package.tarball_component_name();
         let version = match self.channel.as_str() {
-            "stable" => self.rustc_version.clone(),
+            "stable" => RUSTC_VERSION.into(),
             "beta" => "beta".into(),
             "nightly" => "nightly".into(),
-            _ => format!("{}-dev", self.rustc_version),
+            _ => format!("{}-dev", RUSTC_VERSION),
         };
 
         if package.target_independent() {
@@ -198,6 +186,6 @@ impl Versions {
     }
 
     pub(crate) fn rustc_version(&self) -> &str {
-        &self.rustc_version
+        RUSTC_VERSION
     }
 }