Auto merge of #146233 - jieyouxu:run-make-fission, r=Kobzol

Split `run-make` into two {`run-make`,`run-make-cargo`} test suites

## Summary

Split `tests/run-make` into two test suites, to make it faster and more convenient for contributors to run run-make tests that do not need in-tree `cargo`.

| New test suites        | Explanation                                                                                                                                                                                                                              |
| ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `tests/run-make`       | The "fast path" test suite intended for run-make tests that do not need in-tree `cargo`. These tests may not use `cargo`.                                                                                                                |
| `tests/run-make-cargo` | The "slow path" test suite that requires checking out `cargo` submodule and building in-tree `cargo`, and thus will have access to in-tree `cargo`. In practice, these constitute a very small portion of the original `run-make` tests. |

This PR carries out [MCP 847: Split run-make test suite into slower-building test suite with suitably-staged cargo and faster-building test suite without cargo](https://github.com/rust-lang/compiler-team/issues/847).
Fixes rust-lang/rust#135573 (for the tests that do not need in-tree `cargo`).
Fixes rust-lang/rust#134109.

## Remarks

- I considered if we want to split by in-tree tools previously. However, as discussed rust-lang/rust#134109, in practice `rustdoc` is not very slow to build, but `cargo` takes a good few minutes. So, the partition boundary was determined to be along in-tree `cargo` availability.
- The `run-make` tests previously that wanted to use `cargo` cannot just use the bootstrap `cargo`, otherwise they would run into situations where bootstrap `cargo` can significantly diverge from in-tree `cargo` (see https://github.com/rust-lang/rust/pull/130642).

---

try-job: aarch64-msvc-1
try-job: test-various
try-job: x86_64-gnu-debug
try-job: aarch64-gnu-debug
try-job: aarch64-apple
try-job: dist-various-1

compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh

@@ -121,7 +121,7 @@ rm tests/ui/abi/large-byval-align.rs # exceeds implementation limit of Cranelift
 # ============================================================
 rm -r tests/run-make/remap-path-prefix-dwarf # requires llvm-dwarfdump
 rm -r tests/run-make/strip # same
-rm -r tests/run-make/compiler-builtins # Expects lib/rustlib/src/rust to contains the standard library source
+rm -r tests/run-make-cargo/compiler-builtins # Expects lib/rustlib/src/rust to contains the standard library source
 rm -r tests/run-make/translation # same
 rm -r tests/run-make/missing-unstable-trait-bound # This disables support for unstable features, but running cg_clif needs some unstable features
 rm -r tests/run-make/const-trait-stable-toolchain # same
@@ -166,5 +166,5 @@ index 073116933bd..c3e4578204d 100644
 EOF
 
 echo "[TEST] rustc test suite"
-./x.py test --stage 0 --test-args=--no-capture tests/{codegen-units,run-make,ui,incremental}
+./x.py test --stage 0 --test-args=--no-capture tests/{codegen-units,run-make,run-make-cargo,ui,incremental}
 popd

compiler/rustc_codegen_gcc/build_system/src/test.rs

@@ -1083,11 +1083,12 @@ where
 
 fn test_rustc(env: &Env, args: &TestArg) -> Result<(), String> {
     test_rustc_inner(env, args, |_| Ok(false), false, "run-make")?;
+    test_rustc_inner(env, args, |_| Ok(false), false, "run-make-cargo")?;
     test_rustc_inner(env, args, |_| Ok(false), false, "ui")
 }
 
 fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> {
-    let result1 = test_rustc_inner(
+    let run_make_result = test_rustc_inner(
         env,
         args,
         retain_files_callback("tests/failing-run-make-tests.txt", "run-make"),
@@ -1095,7 +1096,15 @@ fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> {
         "run-make",
     );
 
-    let result2 = test_rustc_inner(
+    let run_make_cargo_result = test_rustc_inner(
+        env,
+        args,
+        retain_files_callback("tests/failing-run-make-tests.txt", "run-make-cargo"),
+        false,
+        "run-make",
+    );
+
+    let ui_result = test_rustc_inner(
         env,
         args,
         retain_files_callback("tests/failing-ui-tests.txt", "ui"),
@@ -1103,7 +1112,7 @@ fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> {
         "ui",
     );
 
-    result1.and(result2)
+    run_make_result.and(run_make_cargo_result).and(ui_result)
 }
 
 fn test_successful_rustc(env: &Env, args: &TestArg) -> Result<(), String> {
@@ -1120,6 +1129,13 @@ fn test_successful_rustc(env: &Env, args: &TestArg) -> Result<(), String> {
         remove_files_callback("tests/failing-run-make-tests.txt", "run-make"),
         false,
         "run-make",
+    )?;
+    test_rustc_inner(
+        env,
+        args,
+        remove_files_callback("tests/failing-run-make-tests.txt", "run-make-cargo"),
+        false,
+        "run-make-cargo",
     )
 }
 

src/bootstrap/src/core/build_steps/test.rs

@@ -1522,6 +1522,12 @@ test!(Pretty {
 });
 
 test!(RunMake { path: "tests/run-make", mode: "run-make", suite: "run-make", default: true });
+test!(RunMakeCargo {
+    path: "tests/run-make-cargo",
+    mode: "run-make",
+    suite: "run-make-cargo",
+    default: true
+});
 
 test!(AssemblyLlvm {
     path: "tests/assembly-llvm",
@@ -1773,7 +1779,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
                 target,
             });
         }
-        if suite == "run-make" {
+        if mode == "run-make" {
             builder.tool_exe(Tool::RunMakeSupport);
         }
 
@@ -1816,25 +1822,41 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
 
         let is_rustdoc = suite == "rustdoc-ui" || suite == "rustdoc-js";
 
+        // There are (potentially) 2 `cargo`s to consider:
+        //
+        // - A "bootstrap" cargo, which is the same cargo used to build bootstrap itself, and is
+        //   used to build the `run-make` test recipes and the `run-make-support` test library. All
+        //   of these may not use unstable rustc/cargo features.
+        // - An in-tree cargo, which should be considered as under test. The `run-make-cargo` test
+        //   suite is intended to support the use case of testing the "toolchain" (that is, at the
+        //   minimum the interaction between in-tree cargo + rustc) together.
+        //
+        // For build time and iteration purposes, we partition `run-make` tests which needs an
+        // in-tree cargo (a smaller subset) versus `run-make` tests that do not into two test
+        // suites, `run-make` and `run-make-cargo`. That way, contributors who do not need to run
+        // the `run-make` tests that need in-tree cargo do not need to spend time building in-tree
+        // cargo.
         if mode == "run-make" {
-            let cargo_path = if test_compiler.stage == 0 {
-                // If we're using `--stage 0`, we should provide the bootstrap cargo.
-                builder.initial_cargo.clone()
-            } else {
-                builder
-                    .ensure(tool::Cargo::from_build_compiler(
-                        builder.compiler(test_compiler.stage - 1, test_compiler.host),
-                        test_compiler.host,
-                    ))
-                    .tool_path
-            };
-
-            cmd.arg("--cargo-path").arg(cargo_path);
-
             // We need to pass the compiler that was used to compile run-make-support,
             // because we have to use the same compiler to compile rmake.rs recipes.
             let stage0_rustc_path = builder.compiler(0, test_compiler.host);
             cmd.arg("--stage0-rustc-path").arg(builder.rustc(stage0_rustc_path));
+
+            if suite == "run-make-cargo" {
+                let cargo_path = if test_compiler.stage == 0 {
+                    // If we're using `--stage 0`, we should provide the bootstrap cargo.
+                    builder.initial_cargo.clone()
+                } else {
+                    builder
+                        .ensure(tool::Cargo::from_build_compiler(
+                            builder.compiler(test_compiler.stage - 1, test_compiler.host),
+                            test_compiler.host,
+                        ))
+                        .tool_path
+                };
+
+                cmd.arg("--cargo-path").arg(cargo_path);
+            }
         }
 
         // Avoid depending on rustdoc when we don't need it.

src/bootstrap/src/core/builder/mod.rs

@@ -444,6 +444,7 @@ const PATH_REMAP: &[(&str, &[&str])] = &[
             "tests/mir-opt",
             "tests/pretty",
             "tests/run-make",
+            "tests/run-make-cargo",
             "tests/rustdoc",
             "tests/rustdoc-gui",
             "tests/rustdoc-js",
@@ -1128,8 +1129,8 @@ impl<'a> Builder<'a> {
                 test::RustInstaller,
                 test::TestFloatParse,
                 test::CollectLicenseMetadata,
-                // Run run-make last, since these won't pass without make on Windows
                 test::RunMake,
+                test::RunMakeCargo,
             ),
             Kind::Miri => describe!(test::Crate),
             Kind::Bench => describe!(test::Crate, test::CrateLibrustc),

src/bootstrap/src/core/builder/tests.rs

@@ -2152,8 +2152,9 @@ mod snapshot {
         [build] rustc 0 <host> -> HtmlChecker 1 <host>
         [test] html-check <host>
         [build] rustc 0 <host> -> RunMakeSupport 1 <host>
-        [build] rustc 0 <host> -> cargo 1 <host>
         [test] compiletest-run-make 1 <host>
+        [build] rustc 0 <host> -> cargo 1 <host>
+        [test] compiletest-run-make-cargo 1 <host>
         ");
     }
 
@@ -2171,7 +2172,6 @@ mod snapshot {
         [test] compiletest-ui 1 <host>
         [test] compiletest-ui-fulldeps 1 <host>
         [build] rustc 0 <host> -> RunMakeSupport 1 <host>
-        [build] rustc 0 <host> -> cargo 1 <host>
         [build] rustdoc 1 <host>
         [test] compiletest-run-make 1 <host>
         [test] compiletest-rustdoc 1 <host>
@@ -2200,7 +2200,6 @@ mod snapshot {
         [build] rustc 2 <host> -> rustc 3 <host>
         [test] compiletest-ui-fulldeps 2 <host>
         [build] rustc 0 <host> -> RunMakeSupport 1 <host>
-        [build] rustc 1 <host> -> cargo 2 <host>
         [build] rustdoc 2 <host>
         [test] compiletest-run-make 2 <host>
         [test] compiletest-rustdoc 2 <host>
@@ -2234,7 +2233,6 @@ mod snapshot {
         [build] rustc 2 <host> -> rustc 3 <target1>
         [test] compiletest-ui-fulldeps 2 <target1>
         [build] rustc 0 <host> -> RunMakeSupport 1 <host>
-        [build] rustc 1 <host> -> cargo 2 <host>
         [build] rustdoc 2 <host>
         [test] compiletest-run-make 2 <target1>
         [test] compiletest-rustdoc 2 <target1>
@@ -2329,8 +2327,9 @@ mod snapshot {
         [build] rustc 0 <host> -> HtmlChecker 1 <host>
         [test] html-check <host>
         [build] rustc 0 <host> -> RunMakeSupport 1 <host>
-        [build] rustc 1 <host> -> cargo 2 <host>
         [test] compiletest-run-make 2 <host>
+        [build] rustc 1 <host> -> cargo 2 <host>
+        [test] compiletest-run-make-cargo 2 <host>
         ");
     }
 
@@ -2464,6 +2463,43 @@ mod snapshot {
         ");
     }
 
+    // Differential snapshots for `./x test run-make` run `./x test run-make-cargo`: only
+    // `run-make-cargo` should build an in-tree cargo, running `./x test run-make` should not.
+    #[test]
+    fn test_run_make_no_cargo() {
+        let ctx = TestCtx::new();
+        insta::assert_snapshot!(
+            ctx.config("test")
+                .path("run-make")
+                .render_steps(), @r"
+        [build] llvm <host>
+        [build] rustc 0 <host> -> rustc 1 <host>
+        [build] rustc 0 <host> -> RunMakeSupport 1 <host>
+        [build] rustc 1 <host> -> std 1 <host>
+        [build] rustc 0 <host> -> Compiletest 1 <host>
+        [build] rustdoc 1 <host>
+        [test] compiletest-run-make 1 <host>
+        ");
+    }
+
+    #[test]
+    fn test_run_make_cargo_builds_cargo() {
+        let ctx = TestCtx::new();
+        insta::assert_snapshot!(
+            ctx.config("test")
+                .path("run-make-cargo")
+                .render_steps(), @r"
+        [build] llvm <host>
+        [build] rustc 0 <host> -> rustc 1 <host>
+        [build] rustc 0 <host> -> RunMakeSupport 1 <host>
+        [build] rustc 1 <host> -> std 1 <host>
+        [build] rustc 0 <host> -> Compiletest 1 <host>
+        [build] rustc 0 <host> -> cargo 1 <host>
+        [build] rustdoc 1 <host>
+        [test] compiletest-run-make-cargo 1 <host>
+        ");
+    }
+
     #[test]
     fn doc_all() {
         let ctx = TestCtx::new();

src/ci/docker/host-aarch64/aarch64-gnu-debug/Dockerfile

@@ -54,4 +54,4 @@ ENV RUST_CONFIGURE_ARGS \
 
 ENV SCRIPT \
   python3 ../x.py --stage 2 build && \
-  python3 ../x.py --stage 2 test tests/run-make
+  python3 ../x.py --stage 2 test tests/run-make tests/run-make-cargo

src/ci/docker/host-x86_64/dist-various-1/Dockerfile

@@ -158,7 +158,7 @@ ENV RUST_CONFIGURE_ARGS \
       --disable-docs
 
 ENV SCRIPT \
-      python3 ../x.py --stage 2 test --host='' --target $RUN_MAKE_TARGETS tests/run-make && \
+      python3 ../x.py --stage 2 test --host='' --target $RUN_MAKE_TARGETS tests/run-make tests/run-make-cargo && \
       python3 ../x.py dist --host='' --target $TARGETS
 
 # sccache

src/ci/docker/host-x86_64/test-various/Dockerfile

@@ -60,9 +60,10 @@ RUN curl -L https://github.com/bytecodealliance/wasmtime/releases/download/v19.0
   tar -xJ
 ENV PATH "$PATH:/wasmtime-v19.0.0-x86_64-linux"
 
-ENV WASM_WASIP_TARGET=wasm32-wasip1 
+ENV WASM_WASIP_TARGET=wasm32-wasip1
 ENV WASM_WASIP_SCRIPT python3 /checkout/x.py --stage 2 test --host='' --target $WASM_WASIP_TARGET \
   tests/run-make \
+  tests/run-make-cargo \
   tests/ui \
   tests/mir-opt \
   tests/codegen-units \
@@ -73,6 +74,7 @@ ENV WASM_WASIP_SCRIPT python3 /checkout/x.py --stage 2 test --host='' --target $
 ENV NVPTX_TARGETS=nvptx64-nvidia-cuda
 ENV NVPTX_SCRIPT python3 /checkout/x.py --stage 2 test --host='' --target $NVPTX_TARGETS \
   tests/run-make \
+  tests/run-make-cargo \
   tests/assembly-llvm
 
 ENV MUSL_TARGETS=x86_64-unknown-linux-musl \
@@ -88,8 +90,8 @@ ENV UEFI_TARGETS=aarch64-unknown-uefi,i686-unknown-uefi,x86_64-unknown-uefi \
     CC_x86_64_unknown_uefi=clang-11 \
     CXX_x86_64_unknown_uefi=clang++-11
 ENV UEFI_SCRIPT python3 /checkout/x.py --stage 2 build --host='' --target $UEFI_TARGETS && \
-  python3 /checkout/x.py --stage 2 test tests/run-make/uefi-qemu/rmake.rs --target aarch64-unknown-uefi && \
-  python3 /checkout/x.py --stage 2 test tests/run-make/uefi-qemu/rmake.rs --target i686-unknown-uefi && \
-  python3 /checkout/x.py --stage 2 test tests/run-make/uefi-qemu/rmake.rs --target x86_64-unknown-uefi
+  python3 /checkout/x.py --stage 2 test tests/run-make-cargo/uefi-qemu/rmake.rs --target aarch64-unknown-uefi && \
+  python3 /checkout/x.py --stage 2 test tests/run-make-cargo/uefi-qemu/rmake.rs --target i686-unknown-uefi && \
+  python3 /checkout/x.py --stage 2 test tests/run-make-cargo/uefi-qemu/rmake.rs --target x86_64-unknown-uefi
 
 ENV SCRIPT $WASM_WASIP_SCRIPT && $NVPTX_SCRIPT && $MUSL_SCRIPT && $UEFI_SCRIPT

src/ci/docker/host-x86_64/x86_64-gnu-debug/Dockerfile

@@ -56,4 +56,4 @@ ENV RUST_CONFIGURE_ARGS \
 ENV SCRIPT \
   python3 ../x.py --stage 2 build && \
   python3 ../x.py --stage 2 test tests/ui && \
-  python3 ../x.py --stage 2 test tests/run-make
+  python3 ../x.py --stage 2 test tests/run-make tests/run-make-cargo

src/doc/rustc-dev-guide/src/tests/adding.md

@@ -29,6 +29,8 @@ guidelines:
     suites.
   - Need to inspect the resulting binary in some way? Or if all the other test
     suites are too limited for your purposes? Then use `run-make`.
+    - Use `run-make-cargo` if you need to exercise in-tree `cargo` in conjunction
+      with in-tree `rustc`.
   - Check out the [compiletest] chapter for more specialized test suites.
 
 After deciding on which kind of test to add, see [best

src/doc/rustc-dev-guide/src/tests/best-practices.md

@@ -83,10 +83,10 @@ related tests.
     - E.g. for an implementation of RFC 2093 specifically, we can group a
       collection of tests under `tests/ui/rfc-2093-infer-outlives/`. For the
       directory name, include what the RFC is about.
-- For the [`run-make`] test suite, each `rmake.rs` must be contained within an
-  immediate subdirectory under `tests/run-make/`. Further nesting is not
-  presently supported. Avoid including issue number in the directory name too,
-  include that info in a comment inside `rmake.rs`.
+- For the [`run-make`]/`run-make-support` test suites, each `rmake.rs` must
+  be contained within an immediate subdirectory under `tests/run-make/` or
+  `tests/run-make-cargo/` respectively. Further nesting is not presently
+  supported. Avoid using _only_ an issue number for the test name as well.
 
 ## Test descriptions
 

src/doc/rustc-dev-guide/src/tests/compiletest.md

@@ -397,13 +397,19 @@ your test, causing separate files to be generated for 32bit and 64bit systems.
 
 ### `run-make` tests
 
-The tests in [`tests/run-make`] are general-purpose tests using Rust *recipes*,
-which are small programs (`rmake.rs`) allowing arbitrary Rust code such as
-`rustc` invocations, and is supported by a [`run_make_support`] library. Using
-Rust recipes provide the ultimate in flexibility.
+The tests in [`tests/run-make`] and [`tests/run-make-cargo`] are general-purpose
+tests using Rust *recipes*, which are small programs (`rmake.rs`) allowing
+arbitrary Rust code such as `rustc` invocations, and is supported by a
+[`run_make_support`] library. Using Rust recipes provide the ultimate in
+flexibility.
 
 `run-make` tests should be used if no other test suites better suit your needs.
 
+The `run-make-cargo` test suite additionally builds an in-tree `cargo` to support
+use cases that require testing in-tree `cargo` in conjunction with in-tree `rustc`.
+The `run-make` test suite does not have access to in-tree `cargo` (so it can be the
+faster-to-iterate test suite).
+
 #### Using Rust recipes
 
 Each test should be in a separate directory with a `rmake.rs` Rust program,
@@ -476,6 +482,7 @@ Then add a corresponding entry to `"rust-analyzer.linkedProjects"`
 ```
 
 [`tests/run-make`]: https://github.com/rust-lang/rust/tree/master/tests/run-make
+[`tests/run-make-cargo`]: https://github.com/rust-lang/rust/tree/master/tests/run-make-cargo
 [`run_make_support`]: https://github.com/rust-lang/rust/tree/master/src/tools/run-make-support
 
 ### Coverage tests

src/doc/rustc-dev-guide/src/tests/directives.md

@@ -52,14 +52,14 @@ not be exhaustive. Directives can generally be found by browsing the
 
 See [Building auxiliary crates](compiletest.html#building-auxiliary-crates)
 
-| Directive             | Explanation                                                                                           | Supported test suites | Possible values                               |
-|-----------------------|-------------------------------------------------------------------------------------------------------|-----------------------|-----------------------------------------------|
-| `aux-bin`             | Build a aux binary, made available in `auxiliary/bin` relative to test directory                      | All except `run-make` | Path to auxiliary `.rs` file                  |
-| `aux-build`           | Build a separate crate from the named source file                                                     | All except `run-make` | Path to auxiliary `.rs` file                  |
-| `aux-crate`           | Like `aux-build` but makes available as extern prelude                                                | All except `run-make` | `<extern_prelude_name>=<path/to/aux/file.rs>` |
-| `aux-codegen-backend` | Similar to `aux-build` but pass the compiled dylib to `-Zcodegen-backend` when building the main file | `ui-fulldeps`         | Path to codegen backend file                  |
-| `proc-macro`          | Similar to `aux-build`, but for aux forces host and don't use `-Cprefer-dynamic`[^pm].                | All except `run-make` | Path to auxiliary proc-macro `.rs` file       |
-| `build-aux-docs`      | Build docs for auxiliaries as well.  Note that this only works with `aux-build`, not `aux-crate`.     | All except `run-make` | N/A                                           |
+| Directive             | Explanation                                                                                           | Supported test suites                  | Possible values                               |
+|-----------------------|-------------------------------------------------------------------------------------------------------|----------------------------------------|-----------------------------------------------|
+| `aux-bin`             | Build a aux binary, made available in `auxiliary/bin` relative to test directory                      | All except `run-make`/`run-make-cargo` | Path to auxiliary `.rs` file                  |
+| `aux-build`           | Build a separate crate from the named source file                                                     | All except `run-make`/`run-make-cargo` | Path to auxiliary `.rs` file                  |
+| `aux-crate`           | Like `aux-build` but makes available as extern prelude                                                | All except `run-make`/`run-make-cargo` | `<extern_prelude_name>=<path/to/aux/file.rs>` |
+| `aux-codegen-backend` | Similar to `aux-build` but pass the compiled dylib to `-Zcodegen-backend` when building the main file | `ui-fulldeps`                          | Path to codegen backend file                  |
+| `proc-macro`          | Similar to `aux-build`, but for aux forces host and don't use `-Cprefer-dynamic`[^pm].                | All except `run-make`/`run-make-cargo` | Path to auxiliary proc-macro `.rs` file       |
+| `build-aux-docs`      | Build docs for auxiliaries as well.  Note that this only works with `aux-build`, not `aux-crate`.     | All except `run-make`/`run-make-cargo` | N/A                                           |
 
 [^pm]: please see the [Auxiliary proc-macro section](compiletest.html#auxiliary-proc-macro) in the compiletest chapter for specifics.
 
@@ -243,18 +243,18 @@ ignoring debuggers.
 
 ### Affecting how tests are built
 
-| Directive           | Explanation                                                                                  | Supported test suites     | Possible values                                                                            |
-|---------------------|----------------------------------------------------------------------------------------------|---------------------------|--------------------------------------------------------------------------------------------|
-| `compile-flags`     | Flags passed to `rustc` when building the test or aux file                                   | All except for `run-make` | Any valid `rustc` flags, e.g. `-Awarnings -Dfoo`. Cannot be `-Cincremental` or `--edition` |
-| `edition`           | The edition used to build the test                                                           | All except for `run-make` | Any valid `--edition` value                                                                |
-| `rustc-env`         | Env var to set when running `rustc`                                                          | All except for `run-make` | `<KEY>=<VALUE>`                                                                            |
-| `unset-rustc-env`   | Env var to unset when running `rustc`                                                        | All except for `run-make` | Any env var name                                                                           |
-| `incremental`       | Proper incremental support for tests outside of incremental test suite                       | `ui`, `crashes`           | N/A                                                                                        |
-| `no-prefer-dynamic` | Don't use `-C prefer-dynamic`, don't build as a dylib via a `--crate-type=dylib` preset flag | `ui`, `crashes`           | N/A                                                                                        |
+| Directive           | Explanation                                                                                  | Supported test suites                      | Possible values                                                                            |
+|---------------------|----------------------------------------------------------------------------------------------|--------------------------------------------|--------------------------------------------------------------------------------------------|
+| `compile-flags`     | Flags passed to `rustc` when building the test or aux file                                   | All except for `run-make`/`run-make-cargo` | Any valid `rustc` flags, e.g. `-Awarnings -Dfoo`. Cannot be `-Cincremental` or `--edition` |
+| `edition`           | The edition used to build the test                                                           | All except for `run-make`/`run-make-cargo` | Any valid `--edition` value                                                                |
+| `rustc-env`         | Env var to set when running `rustc`                                                          | All except for `run-make`/`run-make-cargo` | `<KEY>=<VALUE>`                                                                            |
+| `unset-rustc-env`   | Env var to unset when running `rustc`                                                        | All except for `run-make`/`run-make-cargo` | Any env var name                                                                           |
+| `incremental`       | Proper incremental support for tests outside of incremental test suite                       | `ui`, `crashes`                            | N/A                                                                                        |
+| `no-prefer-dynamic` | Don't use `-C prefer-dynamic`, don't build as a dylib via a `--crate-type=dylib` preset flag | `ui`, `crashes`                            | N/A                                                                                        |
 
 <div class="warning">
 
-Tests (outside of `run-make`) that want to use incremental tests not in the
+Tests (outside of `run-make`/`run-make-cargo`) that want to use incremental tests not in the
 incremental test-suite must not pass `-C incremental` via `compile-flags`, and
 must instead use the `//@ incremental` directive.
 
@@ -264,9 +264,9 @@ Consider writing the test as a proper incremental test instead.
 
 ### Rustdoc
 
-| Directive   | Explanation                                                  | Supported test suites                    | Possible values           |
-|-------------|--------------------------------------------------------------|------------------------------------------|---------------------------|
-| `doc-flags` | Flags passed to `rustdoc` when building the test or aux file | `rustdoc`, `rustdoc-js`, `rustdoc-json`  | Any valid `rustdoc` flags |
+| Directive   | Explanation                                                  | Supported test suites                   | Possible values           |
+|-------------|--------------------------------------------------------------|-----------------------------------------|---------------------------|
+| `doc-flags` | Flags passed to `rustdoc` when building the test or aux file | `rustdoc`, `rustdoc-js`, `rustdoc-json` | Any valid `rustdoc` flags |
 
 <!--
 **FIXME(rustdoc)**: what does `check-test-line-numbers-match` do?

src/doc/rustc-dev-guide/src/tests/misc.md

@@ -24,8 +24,8 @@ In `ui` tests and other test suites that support `//@ rustc-env`, you can specif
 //@ rustc-env:RUSTC_BOOTSTRAP=-1
 ```
 
-For `run-make` tests, `//@ rustc-env` is not supported. You can do something
-like the following for individual `rustc` invocations.
+For `run-make`/`run-make-cargo` tests, `//@ rustc-env` is not supported. You can do
+something like the following for individual `rustc` invocations.
 
 ```rust,ignore
 use run_make_support::rustc;

src/tools/compiletest/src/common.rs

@@ -67,6 +67,7 @@ string_enum! {
         MirOpt => "mir-opt",
         Pretty => "pretty",
         RunMake => "run-make",
+        RunMakeCargo => "run-make-cargo",
         Rustdoc => "rustdoc",
         RustdocGui => "rustdoc-gui",
         RustdocJs => "rustdoc-js",
@@ -269,9 +270,6 @@ pub struct Config {
     /// between e.g. beta `cargo` vs in-tree `cargo`.
     ///
     /// FIXME: maybe rename this to reflect that this is a *staged* host cargo.
-    ///
-    /// FIXME(#134109): split `run-make` into two test suites, a test suite *with* staged cargo, and
-    /// another test suite *without*.
     pub cargo_path: Option<Utf8PathBuf>,
 
     /// Path to the stage 0 `rustc` used to build `run-make` recipes. This must not be confused with

src/tools/compiletest/src/directives.rs

@@ -1012,7 +1012,7 @@ impl Config {
         if let Some(raw) = self.parse_name_value_directive(line, "revisions", testfile, line_number)
         {
             if self.mode == TestMode::RunMake {
-                panic!("`run-make` tests do not support revisions: {}", testfile);
+                panic!("`run-make` mode tests do not support revisions: {}", testfile);
             }
 
             let mut duplicates: HashSet<_> = existing.iter().cloned().collect();

src/tools/compiletest/src/runtest/run_make.rs

@@ -5,11 +5,12 @@ use build_helper::fs::{ignore_not_found, recursive_remove};
 use camino::{Utf8Path, Utf8PathBuf};
 
 use super::{ProcRes, TestCx, disable_error_reporting};
+use crate::common::TestSuite;
 use crate::util::{copy_dir_all, dylib_env_var};
 
 impl TestCx<'_> {
     pub(super) fn run_rmake_test(&self) {
-        // For `run-make` V2, we need to perform 2 steps to build and run a `run-make` V2 recipe
+        // For `run-make`, we need to perform 2 steps to build and run a `run-make` recipe
         // (`rmake.rs`) to run the actual tests. The support library is already built as a tool rust
         // library and is available under
         // `build/$HOST/bootstrap-tools/$TARGET/release/librun_make_support.rlib`.
@@ -189,8 +190,12 @@ impl TestCx<'_> {
             // through a specific CI runner).
             .env("LLVM_COMPONENTS", &self.config.llvm_components);
 
-        if let Some(ref cargo) = self.config.cargo_path {
-            cmd.env("CARGO", cargo);
+        // Only `run-make-cargo` test suite gets an in-tree `cargo`, not `run-make`.
+        if self.config.suite == TestSuite::RunMakeCargo {
+            cmd.env(
+                "CARGO",
+                self.config.cargo_path.as_ref().expect("cargo must be built and made available"),
+            );
         }
 
         if let Some(ref rustdoc) = self.config.rustdoc_path {

src/tools/run-make-support/src/external_deps/cargo.rs

@@ -1,11 +1,17 @@
 use crate::command::Command;
-use crate::env_var;
 use crate::util::set_host_compiler_dylib_path;
 
-/// Returns a command that can be used to invoke cargo. The cargo is provided by compiletest
-/// through the `CARGO` env var.
+/// Returns a command that can be used to invoke in-tree cargo. The cargo is provided by compiletest
+/// through the `CARGO` env var, and is **only** available for the `run-make-cargo` test suite.
 pub fn cargo() -> Command {
-    let mut cmd = Command::new(env_var("CARGO"));
+    let cargo_path = std::env::var("CARGO").unwrap_or_else(|e| {
+        panic!(
+            "in-tree `cargo` should be available for `run-make-cargo` test suite, but not \
+            `run-make` test suite: {e}"
+        )
+    });
+
+    let mut cmd = Command::new(cargo_path);
     set_host_compiler_dylib_path(&mut cmd);
     cmd
 }

src/tools/run-make-support/src/lib.rs

@@ -1,7 +1,8 @@
 //! `run-make-support` is a support library for run-make tests. It provides command wrappers and
 //! convenience utility functions to help test writers reduce duplication. The support library
-//! notably is built via cargo: this means that if your test wants some non-trivial utility, such
-//! as `object` or `wasmparser`, they can be re-exported and be made available through this library.
+//! notably is built via bootstrap cargo: this means that if your test wants some non-trivial
+//! utility, such as `object` or `wasmparser`, they can be re-exported and be made available through
+//! this library.
 
 #![warn(unreachable_pub)]
 

src/tools/tidy/src/deps.rs

@@ -80,7 +80,7 @@ pub(crate) const WORKSPACES: &[(&str, ExceptionList, Option<(&[&str], &[&str])>,
     ("src/tools/rustbook", EXCEPTIONS_RUSTBOOK, None, &["src/doc/book", "src/doc/reference"]),
     ("src/tools/rustc-perf", EXCEPTIONS_RUSTC_PERF, None, &["src/tools/rustc-perf"]),
     ("src/tools/test-float-parse", EXCEPTIONS, None, &[]),
-    ("tests/run-make/uefi-qemu/uefi_qemu_test", EXCEPTIONS_UEFI_QEMU_TEST, None, &[]),
+    ("tests/run-make-cargo/uefi-qemu/uefi_qemu_test", EXCEPTIONS_UEFI_QEMU_TEST, None, &[]),
     // tidy-alphabetical-end
 ];
 

tests/run-make-cargo/thumb-none-cortex-m/rmake.rs

@@ -4,7 +4,7 @@
 //!
 //! How to run this
 //! $ ./x.py clean
-//! $ ./x.py test --target thumbv6m-none-eabi,thumbv7m-none-eabi tests/run-make
+//! $ ./x.py test --target thumbv6m-none-eabi,thumbv7m-none-eabi tests/run-make-cargo
 //!
 //! Supported targets:
 //! - thumbv6m-none-eabi (Bare Cortex-M0, M0+, M1)

triagebot.toml

@@ -336,6 +336,7 @@ trigger_files = [
     "tests/mir-opt",
     "tests/pretty",
     "tests/run-make",
+    "tests/run-make-cargo",
     "tests/ui",
     "tests/ui-fulldeps",
 ]
@@ -593,6 +594,7 @@ trigger_files = [
 [autolabel."A-run-make"]
 trigger_files = [
     "tests/run-make",
+    "tests/run-make-cargo",
     "src/tools/run-make-support"
 ]
 
@@ -1543,6 +1545,7 @@ dep-bumps = [
 "/src/rustdoc-json-types" =                              ["rustdoc"]
 "/src/stage0" =                                          ["bootstrap"]
 "/tests/run-make" =                                      ["@jieyouxu"]
+"/tests/run-make-cargo" =                                ["@jieyouxu"]
 "/tests/rustdoc" =                                       ["rustdoc"]
 "/tests/rustdoc-gui" =                                   ["rustdoc"]
 "/tests/rustdoc-js-std" =                                ["rustdoc"]