Auto merge of #150538 - Kobzol:dist-cg-gcc-component, r=jieyouxu

Add a dist component for cg_gcc

try-job: dist-x86_64-linux

src/bootstrap/src/core/build_steps/compile.rs

@@ -19,7 +19,7 @@ use serde_derive::Deserialize;
 #[cfg(feature = "tracing")]
 use tracing::span;
 
-use crate::core::build_steps::gcc::{Gcc, GccOutput, GccTargetPair, add_cg_gcc_cargo_flags};
+use crate::core::build_steps::gcc::{Gcc, GccOutput, GccTargetPair};
 use crate::core::build_steps::tool::{RustcPrivateCompilers, SourceType, copy_lld_artifacts};
 use crate::core::build_steps::{dist, llvm};
 use crate::core::builder;
@@ -1569,21 +1569,29 @@ impl Step for RustcLink {
 }
 
 /// Set of `libgccjit` dylibs that can be used by `cg_gcc` to compile code for a set of targets.
+/// `libgccjit` requires a separate build for each `(host, target)` pair.
+/// So if you are on linux-x64 and build for linux-aarch64, you will need at least:
+/// - linux-x64 -> linux-x64 libgccjit (for building host code like proc macros)
+/// - linux-x64 -> linux-aarch64 libgccjit (for the aarch64 target code)
 #[derive(Clone)]
 pub struct GccDylibSet {
     dylibs: BTreeMap<GccTargetPair, GccOutput>,
-    host_pair: GccTargetPair,
 }
 
 impl GccDylibSet {
-    /// Returns the libgccjit.so dylib that corresponds to a host target on which `cg_gcc` will be
-    /// executed, and which will target the host. So e.g. if `cg_gcc` will be executed on
-    /// x86_64-unknown-linux-gnu, the host dylib will be for compilation pair
-    /// `(x86_64-unknown-linux-gnu, x86_64-unknown-linux-gnu)`.
-    fn host_dylib(&self) -> &GccOutput {
-        self.dylibs.get(&self.host_pair).unwrap_or_else(|| {
-            panic!("libgccjit.so was not built for host target {}", self.host_pair)
-        })
+    /// Build a set of libgccjit dylibs that will be executed on `host` and will generate code for
+    /// each specified target.
+    pub fn build(
+        builder: &Builder<'_>,
+        host: TargetSelection,
+        targets: Vec<TargetSelection>,
+    ) -> Self {
+        let dylibs = targets
+            .iter()
+            .map(|t| GccTargetPair::for_target_pair(host, *t))
+            .map(|target_pair| (target_pair, builder.ensure(Gcc { target_pair })))
+            .collect();
+        Self { dylibs }
     }
 
     /// Install the libgccjit dylibs to the corresponding target directories of the given compiler.
@@ -1626,39 +1634,34 @@ impl GccDylibSet {
 
 /// Output of the `compile::GccCodegenBackend` step.
 ///
-/// It contains paths to all built libgccjit libraries on which this backend depends here.
+/// It contains a build stamp with the path to the built cg_gcc dylib.
 #[derive(Clone)]
 pub struct GccCodegenBackendOutput {
     stamp: BuildStamp,
-    dylib_set: GccDylibSet,
+}
+
+impl GccCodegenBackendOutput {
+    pub fn stamp(&self) -> &BuildStamp {
+        &self.stamp
+    }
 }
 
 /// Builds the GCC codegen backend (`cg_gcc`).
-/// The `cg_gcc` backend uses `libgccjit`, which requires a separate build for each
-/// `host -> target` pair. So if you are on linux-x64 and build for linux-aarch64,
-/// you will need at least:
-/// - linux-x64 -> linux-x64 libgccjit (for building host code like proc macros)
-/// - linux-x64 -> linux-aarch64 libgccjit (for the aarch64 target code)
-///
-/// We model this by having a single cg_gcc for a given host target, which contains one
-/// libgccjit per (host, target) pair.
-/// Note that the host target is taken from `self.compilers.target_compiler.host`.
+/// Note that this **does not** build libgccjit, which is a dependency of cg_gcc.
+/// That has to be built separately, because a separate copy of libgccjit is required
+/// for each (host, target) compilation pair.
+/// cg_gcc goes to great lengths to ensure that it does not *directly* link to libgccjit,
+/// so we respect that here and allow building cg_gcc without building libgccjit itself.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct GccCodegenBackend {
     compilers: RustcPrivateCompilers,
-    targets: Vec<TargetSelection>,
+    target: TargetSelection,
 }
 
 impl GccCodegenBackend {
-    /// Build `cg_gcc` that will run on host `H` (`compilers.target_compiler.host`) and will be
-    /// able to produce code target pairs (`H`, `T`) for all `T` from `targets`.
-    pub fn for_targets(
-        compilers: RustcPrivateCompilers,
-        mut targets: Vec<TargetSelection>,
-    ) -> Self {
-        // Sort targets to improve step cache hits
-        targets.sort();
-        Self { compilers, targets }
+    /// Build `cg_gcc` that will run on the given host target.
+    pub fn for_target(compilers: RustcPrivateCompilers, target: TargetSelection) -> Self {
+        Self { compilers, target }
     }
 }
 
@@ -1672,10 +1675,8 @@ impl Step for GccCodegenBackend {
     }
 
     fn make_run(run: RunConfig<'_>) {
-        // By default, build cg_gcc that will only be able to compile native code for the given
-        // host target.
         let compilers = RustcPrivateCompilers::new(run.builder, run.builder.top_stage, run.target);
-        run.builder.ensure(GccCodegenBackend { compilers, targets: vec![run.target] });
+        run.builder.ensure(GccCodegenBackend::for_target(compilers, run.target));
     }
 
     fn run(self, builder: &Builder<'_>) -> Self::Output {
@@ -1689,19 +1690,7 @@ impl Step for GccCodegenBackend {
             &CodegenBackendKind::Gcc,
         );
 
-        let dylib_set = GccDylibSet {
-            dylibs: self
-                .targets
-                .iter()
-                .map(|&target| {
-                    let target_pair = GccTargetPair::for_target_pair(host, target);
-                    (target_pair, builder.ensure(Gcc { target_pair }))
-                })
-                .collect(),
-            host_pair: GccTargetPair::for_native_build(host),
-        };
-
-        if builder.config.keep_stage.contains(&build_compiler.stage) {
+        if builder.config.keep_stage.contains(&build_compiler.stage) && stamp.path().exists() {
             trace!("`keep-stage` requested");
             builder.info(
                 "WARNING: Using a potentially old codegen backend. \
@@ -1709,7 +1698,7 @@ impl Step for GccCodegenBackend {
             );
             // Codegen backends are linked separately from this step today, so we don't do
             // anything here.
-            return GccCodegenBackendOutput { stamp, dylib_set };
+            return GccCodegenBackendOutput { stamp };
         }
 
         let mut cargo = builder::Cargo::new(
@@ -1723,15 +1712,12 @@ impl Step for GccCodegenBackend {
         cargo.arg("--manifest-path").arg(builder.src.join("compiler/rustc_codegen_gcc/Cargo.toml"));
         rustc_cargo_env(builder, &mut cargo, host);
 
-        add_cg_gcc_cargo_flags(&mut cargo, dylib_set.host_dylib());
-
         let _guard =
             builder.msg(Kind::Build, "codegen backend gcc", Mode::Codegen, build_compiler, host);
         let files = run_cargo(builder, cargo, vec![], &stamp, vec![], false, false);
 
         GccCodegenBackendOutput {
             stamp: write_codegen_backend_stamp(stamp, files, builder.config.dry_run()),
-            dylib_set,
         }
     }
 
@@ -2457,12 +2443,18 @@ impl Step for Assemble {
                         // GCC dylibs built below by taking a look at the current stage and whether
                         // cg_gcc is used as the default codegen backend.
 
+                        // First, the easy part: build cg_gcc
                         let compilers = prepare_compilers();
+                        let cg_gcc = builder
+                            .ensure(GccCodegenBackend::for_target(compilers, target_compiler.host));
+                        copy_codegen_backends_to_sysroot(builder, cg_gcc.stamp, target_compiler);
+
+                        // Then, the hard part: prepare all required libgccjit dylibs.
 
                         // The left side of the target pairs below is implied. It has to match the
-                        // host target on which cg_gcc will run, which is the host target of
+                        // host target on which libgccjit will be used, which is the host target of
                         // `target_compiler`. We only pass the right side of the target pairs to
-                        // the `GccCodegenBackend` constructor.
+                        // the `GccDylibSet` constructor.
                         let mut targets = HashSet::new();
                         // Add all host targets, so that we are able to build host code in this
                         // bootstrap invocation using cg_gcc.
@@ -2477,14 +2469,16 @@ impl Step for Assemble {
                         // host code (e.g. proc macros) using cg_gcc.
                         targets.insert(compilers.target_compiler().host);
 
-                        let output = builder.ensure(GccCodegenBackend::for_targets(
-                            compilers,
+                        // Now build all the required libgccjit dylibs
+                        let dylib_set = GccDylibSet::build(
+                            builder,
+                            compilers.target_compiler().host,
                             targets.into_iter().collect(),
-                        ));
-                        copy_codegen_backends_to_sysroot(builder, output.stamp, target_compiler);
-                        // Also copy all requires libgccjit dylibs to the corresponding
-                        // library sysroots, so that they are available for the codegen backend.
-                        output.dylib_set.install_to(builder, target_compiler);
+                        );
+
+                        // And then copy all the dylibs to the corresponding
+                        // library sysroots, so that they are available for cg_gcc.
+                        dylib_set.install_to(builder, target_compiler);
                     }
                     CodegenBackendKind::Llvm | CodegenBackendKind::Custom(_) => continue,
                 }

src/bootstrap/src/core/build_steps/dist.rs

@@ -1652,23 +1652,7 @@ impl Step for CraneliftCodegenBackend {
             return None;
         }
 
-        // Get the relative path of where the codegen backend should be stored.
-        let backends_dst = builder.sysroot_codegen_backends(compilers.target_compiler());
-        let backends_rel = backends_dst
-            .strip_prefix(builder.sysroot(compilers.target_compiler()))
-            .unwrap()
-            .strip_prefix(builder.sysroot_libdir_relative(compilers.target_compiler()))
-            .unwrap();
-        // Don't use custom libdir here because ^lib/ will be resolved again with installer
-        let backends_dst = PathBuf::from("lib").join(backends_rel);
-
-        let codegen_backend_dylib = get_codegen_backend_file(&stamp);
-        tarball.add_renamed_file(
-            &codegen_backend_dylib,
-            &backends_dst,
-            &normalize_codegen_backend_name(builder, &codegen_backend_dylib),
-            FileType::NativeLibrary,
-        );
+        add_codegen_backend_to_tarball(builder, &tarball, compilers.target_compiler(), &stamp);
 
         Some(tarball.generate())
     }
@@ -1681,6 +1665,113 @@ impl Step for CraneliftCodegenBackend {
     }
 }
 
+/// Builds a dist component containing the GCC codegen backend.
+/// Note that for this backend to work, it must have a set of libgccjit dylibs available
+/// at runtime.
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
+pub struct GccCodegenBackend {
+    pub compilers: RustcPrivateCompilers,
+    pub target: TargetSelection,
+}
+
+impl Step for GccCodegenBackend {
+    type Output = Option<GeneratedTarball>;
+    const IS_HOST: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.alias("rustc_codegen_gcc")
+    }
+
+    fn is_default_step(builder: &Builder<'_>) -> bool {
+        // We only want to build the gcc backend in `x dist` if the backend was enabled
+        // in rust.codegen-backends.
+        // Sadly, we don't have access to the actual target for which we're disting clif here..
+        // So we just use the host target.
+        builder
+            .config
+            .enabled_codegen_backends(builder.host_target)
+            .contains(&CodegenBackendKind::Gcc)
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(GccCodegenBackend {
+            compilers: RustcPrivateCompilers::new(run.builder, run.builder.top_stage, run.target),
+            target: run.target,
+        });
+    }
+
+    fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+        // This prevents rustc_codegen_gcc from being built for "dist"
+        // or "install" on the stable/beta channels. It is not yet stable and
+        // should not be included.
+        if !builder.build.unstable_features() {
+            return None;
+        }
+
+        let target = self.target;
+        if target != "x86_64-unknown-linux-gnu" {
+            builder
+                .info(&format!("target `{target}` not supported by rustc_codegen_gcc. skipping"));
+            return None;
+        }
+
+        let mut tarball = Tarball::new(builder, "rustc-codegen-gcc", &target.triple);
+        tarball.set_overlay(OverlayKind::RustcCodegenGcc);
+        tarball.is_preview(true);
+        tarball.add_legal_and_readme_to("share/doc/rustc_codegen_gcc");
+
+        let compilers = self.compilers;
+        let backend = builder.ensure(compile::GccCodegenBackend::for_target(compilers, target));
+
+        if builder.config.dry_run() {
+            return None;
+        }
+
+        add_codegen_backend_to_tarball(
+            builder,
+            &tarball,
+            compilers.target_compiler(),
+            backend.stamp(),
+        );
+
+        Some(tarball.generate())
+    }
+
+    fn metadata(&self) -> Option<StepMetadata> {
+        Some(
+            StepMetadata::dist("rustc_codegen_gcc", self.target)
+                .built_by(self.compilers.build_compiler()),
+        )
+    }
+}
+
+/// Add a codegen backend built for `compiler`, with its artifacts stored in `stamp`, to the given
+/// `tarball` at the correct place.
+fn add_codegen_backend_to_tarball(
+    builder: &Builder<'_>,
+    tarball: &Tarball<'_>,
+    compiler: Compiler,
+    stamp: &BuildStamp,
+) {
+    // Get the relative path of where the codegen backend should be stored.
+    let backends_dst = builder.sysroot_codegen_backends(compiler);
+    let backends_rel = backends_dst
+        .strip_prefix(builder.sysroot(compiler))
+        .unwrap()
+        .strip_prefix(builder.sysroot_libdir_relative(compiler))
+        .unwrap();
+    // Don't use custom libdir here because ^lib/ will be resolved again with installer
+    let backends_dst = PathBuf::from("lib").join(backends_rel);
+
+    let codegen_backend_dylib = get_codegen_backend_file(stamp);
+    tarball.add_renamed_file(
+        &codegen_backend_dylib,
+        &backends_dst,
+        &normalize_codegen_backend_name(builder, &codegen_backend_dylib),
+        FileType::NativeLibrary,
+    );
+}
+
 #[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub struct Rustfmt {
     pub compilers: RustcPrivateCompilers,

src/bootstrap/src/core/builder/mod.rs

@@ -967,6 +967,7 @@ impl<'a> Builder<'a> {
                 dist::Mingw,
                 dist::Rustc,
                 dist::CraneliftCodegenBackend,
+                dist::GccCodegenBackend,
                 dist::Std,
                 dist::RustcDev,
                 dist::Analysis,

src/bootstrap/src/utils/tarball.rs

@@ -25,6 +25,7 @@ pub(crate) enum OverlayKind {
     Rustfmt,
     RustAnalyzer,
     RustcCodegenCranelift,
+    RustcCodegenGcc,
     LlvmBitcodeLinker,
 }
 
@@ -66,6 +67,11 @@ impl OverlayKind {
                 "compiler/rustc_codegen_cranelift/LICENSE-APACHE",
                 "compiler/rustc_codegen_cranelift/LICENSE-MIT",
             ],
+            OverlayKind::RustcCodegenGcc => &[
+                "compiler/rustc_codegen_gcc/Readme.md",
+                "compiler/rustc_codegen_gcc/LICENSE-APACHE",
+                "compiler/rustc_codegen_gcc/LICENSE-MIT",
+            ],
             OverlayKind::LlvmBitcodeLinker => &[
                 "COPYRIGHT",
                 "LICENSE-APACHE",
@@ -93,6 +99,7 @@ impl OverlayKind {
                 .rust_analyzer_info
                 .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")),
             OverlayKind::RustcCodegenCranelift => builder.rust_version(),
+            OverlayKind::RustcCodegenGcc => builder.rust_version(),
             OverlayKind::LlvmBitcodeLinker => builder.rust_version(),
         }
     }

src/ci/docker/host-x86_64/dist-x86_64-linux/dist.sh

@@ -7,7 +7,9 @@ python3 ../x.py build --set rust.debug=true opt-dist
 ./build/$HOSTS/stage1-tools-bin/opt-dist linux-ci -- python3 ../x.py dist \
     --host $HOSTS --target $HOSTS \
     --include-default-paths \
-    build-manifest bootstrap
+    build-manifest \
+    bootstrap \
+    rustc_codegen_gcc
 
 # Use GCC for building GCC components, as it seems to behave badly when built with Clang
 # Only build GCC on full builds, not try builds

src/tools/opt-dist/src/main.rs

@@ -456,6 +456,7 @@ fn main() -> anyhow::Result<()> {
             "rustfmt",
             "generate-copyright",
             "bootstrap",
+            "rustc_codegen_gcc",
         ] {
             build_args.extend(["--skip".to_string(), target.to_string()]);
         }