user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

use rustc-build-sysroot instead of xargo

Browse source
This commit is contained in:
Ralf Jung 2022-09-24 19:57:33 +02:00
parent e8683f50fb
commit 5f18674c31
10 changed files with 166 additions and 219 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/tools/miri/.github/workflows/ci.yml vendored
View file

@ -57,15 +57,14 @@ jobs:
# contains package information of crates installed via `cargo install`.
~/.cargo/.crates.toml
~/.cargo/.crates2.json
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', 'cargo-miri/src/version.rs') }}
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
restore-keys: ${{ runner.os }}-cargo
- name: Install rustup-toolchain-install-master and xargo
- name: Install rustup-toolchain-install-master
if: ${{ steps.cache.outputs.cache-hit == 'false' }}
shell: bash
run: |
cargo install -f rustup-toolchain-install-master
cargo install -f xargo
- name: Install "master" toolchain
shell: bash

2
src/tools/miri/CONTRIBUTING.md
View file

@ -38,7 +38,7 @@ for you. If you don't want all of these to happen, you can add individual `.auto
## Building and testing Miri
Invoking Miri requires getting a bunch of flags right and setting up a custom
sysroot with xargo. The `miri` script takes care of that for you. With the
sysroot. The `miri` script takes care of that for you. With the
build environment prepared, compiling Miri is just one command away:
```

2
src/tools/miri/README.md
View file

@ -447,7 +447,7 @@ binaries, and as such worth documenting:
some compiler flags to prepare the code for interpretation; with `host`, this is not done.
This environment variable is useful to be sure that the compiled `rlib`s are compatible
with Miri.
* `MIRI_CALLED_FROM_XARGO` is set during the Miri-induced `xargo` sysroot build,
* `MIRI_CALLED_FROM_SETUP` is set during the Miri sysroot build,
which will re-invoke `cargo-miri` as the `rustc` to use for this build.
* `MIRI_CALLED_FROM_RUSTDOC` when set to any value tells `cargo-miri` that it is
running as a child process of `rustdoc`, which invokes it twice for each doc-test

74
src/tools/miri/cargo-miri/Cargo.lock
View file

@ -35,6 +35,7 @@ version = "0.1.0"
dependencies = [
"cargo_metadata",
"directories",
"rustc-build-sysroot",
"rustc-workspace-hack",
"rustc_version",
"serde",
@ -142,6 +143,12 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "fuchsia-cprng"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
[[package]]
name = "getrandom"
version = "0.2.3"
@ -322,6 +329,43 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "rand"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293"
dependencies = [
"fuchsia-cprng",
"libc",
"rand_core 0.3.1",
"rdrand",
"winapi",
]
[[package]]
name = "rand_core"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
dependencies = [
"rand_core 0.4.2",
]
[[package]]
name = "rand_core"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
[[package]]
name = "rdrand"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
dependencies = [
"rand_core 0.3.1",
]
[[package]]
name = "redox_syscall"
version = "0.2.10"
@ -341,6 +385,26 @@ dependencies = [
"redox_syscall",
]
[[package]]
name = "remove_dir_all"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
dependencies = [
"winapi",
]
[[package]]
name = "rustc-build-sysroot"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea9e30aa5a34196fe1b2899704f1e1dccbc91fa0981f6c36b749899f924fcadd"
dependencies = [
"anyhow",
"rustc_version",
"tempdir",
]
[[package]]
name = "rustc-workspace-hack"
version = "1.0.0"
@ -419,6 +483,16 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "tempdir"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
dependencies = [
"rand",
"remove_dir_all",
]
[[package]]
name = "thiserror"
version = "1.0.30"

1
src/tools/miri/cargo-miri/Cargo.toml
View file

@ -18,6 +18,7 @@ directories = "3"
rustc_version = "0.4"
serde_json = "1.0.40"
cargo_metadata = "0.15.0"
rustc-build-sysroot = "0.2.1"
# A noop dependency that changes in the Rust repository, it's a bit of a hack.
# See the `src/tools/rustc-workspace-hack/README.md` file in `rust-lang/rust`

7
src/tools/miri/cargo-miri/src/main.rs
View file

@ -6,7 +6,6 @@ mod util;
mod arg;
mod phases;
mod setup;
mod version;
use std::{env, iter};
@ -22,8 +21,8 @@ fn main() {
// Dispatch to `cargo-miri` phase. Here is a rough idea of "who calls who".
//
// Initially, we are invoked as `cargo-miri miri run/test`. We first run the setup phase:
// - We call `xargo`, and set `RUSTC` back to us, together with `MIRI_CALLED_FROM_XARGO`,
// so that xargo's rustc invocations end up in `phase_rustc` with `RustcPhase::Setup`.
// - We use `rustc-build-sysroot`, and set `RUSTC` back to us, together with `MIRI_CALLED_FROM_SETUP`,
// so that the sysroot build rustc invocations end up in `phase_rustc` with `RustcPhase::Setup`.
// There we then call the Miri driver with `MIRI_BE_RUSTC` to perform the actual build.
//
// Then we call `cargo run/test`, exactly forwarding all user flags, plus some configuration so
@ -52,7 +51,7 @@ fn main() {
// the Miri driver for interpretation.
// Dispatch running as part of sysroot compilation.
if env::var_os("MIRI_CALLED_FROM_XARGO").is_some() {
if env::var_os("MIRI_CALLED_FROM_SETUP").is_some() {
phase_rustc(args, RustcPhase::Setup);
return;
}

16
src/tools/miri/cargo-miri/src/phases.rs
View file

@ -7,6 +7,8 @@ use std::io::BufReader;
use std::path::PathBuf;
use std::process::Command;
use rustc_version::VersionMeta;
use crate::{setup::*, util::*};
const CARGO_MIRI_HELP: &str = r#"Runs binary crates and tests in Miri
@ -90,12 +92,14 @@ pub fn phase_cargo_miri(mut args: impl Iterator<Item = String>) {
let verbose = num_arg_flag("-v");
// Determine the involved architectures.
let host = version_info().host;
let rustc_version = VersionMeta::for_command(miri_for_host())
.expect("failed to determine underlying rustc version of Miri");
let host = &rustc_version.host;
let target = get_arg_flag_value("--target");
let target = target.as_ref().unwrap_or(&host);
let target = target.as_ref().unwrap_or(host);
// We always setup.
setup(&subcommand, &host, target);
setup(&subcommand, target, &rustc_version);
// Invoke actual cargo for the job, but with different flags.
// We re-use `cargo test` and `cargo run`, which makes target and binary handling very easy but
@ -146,7 +150,7 @@ pub fn phase_cargo_miri(mut args: impl Iterator<Item = String>) {
if get_arg_flag_value("--target").is_none() {
// No target given. Explicitly pick the host.
cmd.arg("--target");
cmd.arg(&host);
cmd.arg(host);
}
// Set ourselves as runner for al binaries invoked by cargo.
@ -204,7 +208,7 @@ pub fn phase_cargo_miri(mut args: impl Iterator<Item = String>) {
#[derive(Debug, Copy, Clone, PartialEq)]
pub enum RustcPhase {
/// `rustc` called via `xargo` for sysroot build.
/// `rustc` called during sysroot build.
Setup,
/// `rustc` called by `cargo` for regular build.
Build,
@ -264,7 +268,7 @@ pub fn phase_rustc(mut args: impl Iterator<Item = String>, phase: RustcPhase) {
let verbose = std::env::var("MIRI_VERBOSE")
.map_or(0, |verbose| verbose.parse().expect("verbosity flag must be an integer"));
let target_crate = is_target_crate();
// Determine whether this is cargo/xargo invoking rustc to get some infos.
// Determine whether this is cargo invoking rustc to get some infos.
let info_query = get_arg_flag_value("--print").is_some() || has_arg_flag("-vV");
let store_json = |info: CrateRunInfo| {

247
src/tools/miri/cargo-miri/src/setup.rs
View file

@ -1,64 +1,19 @@
//! Implements `cargo miri setup` via xargo
//! Implements `cargo miri setup`.
use std::env;
use std::ffi::OsStr;
use std::fs::{self};
use std::io::BufRead;
use std::ops::Not;
use std::path::{Path, PathBuf};
use std::path::PathBuf;
use std::process::{self, Command};
use crate::{util::*, version::*};
use rustc_build_sysroot::{BuildMode, Sysroot, SysrootConfig};
use rustc_version::VersionMeta;
fn xargo_version() -> Option<(u32, u32, u32)> {
let out = xargo_check().arg("--version").output().ok()?;
if !out.status.success() {
return None;
}
// Parse output. The first line looks like "xargo 0.3.12 (b004f1c 2018-12-13)".
let line = out
.stderr
.lines()
.next()
.expect("malformed `xargo --version` output: not at least one line")
.expect("malformed `xargo --version` output: error reading first line");
let (name, version) = {
let mut split = line.split(' ');
(
split.next().expect("malformed `xargo --version` output: empty"),
split.next().expect("malformed `xargo --version` output: not at least two words"),
)
};
if name != "xargo" {
// This is some fork of xargo
return None;
}
let mut version_pieces = version.split('.');
let major = version_pieces
.next()
.expect("malformed `xargo --version` output: not a major version piece")
.parse()
.expect("malformed `xargo --version` output: major version is not an integer");
let minor = version_pieces
.next()
.expect("malformed `xargo --version` output: not a minor version piece")
.parse()
.expect("malformed `xargo --version` output: minor version is not an integer");
let patch = version_pieces
.next()
.expect("malformed `xargo --version` output: not a patch version piece")
.parse()
.expect("malformed `xargo --version` output: patch version is not an integer");
if version_pieces.next().is_some() {
panic!("malformed `xargo --version` output: more than three pieces in version");
}
Some((major, minor, patch))
}
use crate::util::*;
/// Performs the setup required to make `cargo miri` work: Getting a custom-built libstd. Then sets
/// `MIRI_SYSROOT`. Skipped if `MIRI_SYSROOT` is already set, in which case we expect the user has
/// done all this already.
pub fn setup(subcommand: &MiriCommand, host: &str, target: &str) {
pub fn setup(subcommand: &MiriCommand, target: &str, rustc_version: &VersionMeta) {
let only_setup = matches!(subcommand, MiriCommand::Setup);
let ask_user = !only_setup;
let print_sysroot = only_setup && has_arg_flag("--print-sysroot"); // whether we just print the sysroot path
@ -69,21 +24,8 @@ pub fn setup(subcommand: &MiriCommand, host: &str, target: &str) {
return;
}
// First, we need xargo.
if xargo_version().map_or(true, |v| v < XARGO_MIN_VERSION) {
if std::env::var_os("XARGO_CHECK").is_some() {
// The user manually gave us a xargo binary; don't do anything automatically.
show_error!("xargo is too old; please upgrade to the latest version")
}
let mut cmd = cargo();
cmd.args(["install", "xargo"]);
ask_to_run(cmd, ask_user, "install a recent enough xargo");
}
// Determine where the rust sources are located. The env vars manually setting the source
// (`MIRI_LIB_SRC`, `XARGO_RUST_SRC`) trump auto-detection.
let rust_src_env_var =
std::env::var_os("MIRI_LIB_SRC").or_else(|| std::env::var_os("XARGO_RUST_SRC"));
// Determine where the rust sources are located. The env var trumps auto-detection.
let rust_src_env_var = std::env::var_os("MIRI_LIB_SRC");
let rust_src = match rust_src_env_var {
Some(path) => {
let path = PathBuf::from(path);
@ -92,22 +34,9 @@ pub fn setup(subcommand: &MiriCommand, host: &str, target: &str) {
}
None => {
// Check for `rust-src` rustup component.
let output = miri_for_host()
.args(["--print", "sysroot"])
.output()
.expect("failed to determine sysroot");
if !output.status.success() {
show_error!(
"Failed to determine sysroot; Miri said:\n{}",
String::from_utf8_lossy(&output.stderr).trim_end()
);
}
let sysroot = std::str::from_utf8(&output.stdout).unwrap();
let sysroot = Path::new(sysroot.trim_end_matches('\n'));
// Check for `$SYSROOT/lib/rustlib/src/rust/library`; test if that contains `std/Cargo.toml`.
let rustup_src =
sysroot.join("lib").join("rustlib").join("src").join("rust").join("library");
if !rustup_src.join("std").join("Cargo.toml").exists() {
let rustup_src = rustc_build_sysroot::rustc_sysroot_src(miri_for_host())
.expect("could not determine sysroot source directory");
if !rustup_src.exists() {
// Ask the user to install the `rust-src` component, and use that.
let mut cmd = Command::new("rustup");
cmd.args(["component", "add", "rust-src"]);
@ -131,115 +60,85 @@ pub fn setup(subcommand: &MiriCommand, host: &str, target: &str) {
);
}
// Next, we need our own libstd. Prepare a xargo project for that purpose.
// We will do this work in whatever is a good cache dir for this platform.
let dirs = directories::ProjectDirs::from("org", "rust-lang", "miri").unwrap();
let dir = dirs.cache_dir();
if !dir.exists() {
fs::create_dir_all(dir).unwrap();
}
// The interesting bit: Xargo.toml (only needs content if we actually need std)
let xargo_toml = if std::env::var_os("MIRI_NO_STD").is_some() {
""
// Determine where to put the sysroot.
let user_dirs = directories::ProjectDirs::from("org", "rust-lang", "miri").unwrap();
let sysroot_dir = user_dirs.cache_dir();
// Sysroot configuration and build details.
let sysroot_config = if std::env::var_os("MIRI_NO_STD").is_some() {
SysrootConfig::NoStd
} else {
r#"
[dependencies.std]
default_features = false
# We support unwinding, so enable that panic runtime.
features = ["panic_unwind", "backtrace"]
[dependencies.test]
"#
SysrootConfig::WithStd { std_features: &["panic_unwind", "backtrace"] }
};
write_to_file(&dir.join("Xargo.toml"), xargo_toml);
// The boring bits: a dummy project for xargo.
// FIXME: With xargo-check, can we avoid doing this?
write_to_file(
&dir.join("Cargo.toml"),
r#"
[package]
name = "miri-xargo"
description = "A dummy project for building libstd with xargo."
version = "0.0.0"
let cargo_cmd = || {
let mut command = cargo();
// Use Miri as rustc to build a libstd compatible with us (and use the right flags).
// However, when we are running in bootstrap, we cannot just overwrite `RUSTC`,
// because we still need bootstrap to distinguish between host and target crates.
// In that case we overwrite `RUSTC_REAL` instead which determines the rustc used
// for target crates.
// We set ourselves (`cargo-miri`) instead of Miri directly to be able to patch the flags
// for `libpanic_abort` (usually this is done by bootstrap but we have to do it ourselves).
// The `MIRI_CALLED_FROM_SETUP` will mean we dispatch to `phase_setup_rustc`.
let cargo_miri_path = std::env::current_exe().expect("current executable path invalid");
if env::var_os("RUSTC_STAGE").is_some() {
assert!(env::var_os("RUSTC").is_some());
command.env("RUSTC_REAL", &cargo_miri_path);
} else {
command.env("RUSTC", &cargo_miri_path);
}
command.env("MIRI_CALLED_FROM_SETUP", "1");
// Make sure there are no other wrappers getting in our way (Cc
// https://github.com/rust-lang/miri/issues/1421,
// https://github.com/rust-lang/miri/issues/2429). Looks like setting
// `RUSTC_WRAPPER` to the empty string overwrites `build.rustc-wrapper` set via
// `config.toml`.
command.env("RUSTC_WRAPPER", "");
[lib]
path = "lib.rs"
"#,
);
write_to_file(&dir.join("lib.rs"), "#![no_std]");
// Figure out where xargo will build its stuff.
// Unfortunately, it puts things into a different directory when the
// architecture matches the host.
let sysroot = if target == host { dir.join("HOST") } else { PathBuf::from(dir) };
if only_setup {
if print_sysroot {
// Be extra sure there is no noise on stdout.
command.stdout(process::Stdio::null());
}
} else {
command.stdout(process::Stdio::null());
command.stderr(process::Stdio::null());
}
// Disable debug assertions in the standard library -- Miri is already slow enough.
// But keep the overflow checks, they are cheap. This completely overwrites flags
// the user might have set, which is consistent with normal `cargo build` that does
// not apply `RUSTFLAGS` to the sysroot either.
let rustflags = vec!["-Cdebug-assertions=off".into(), "-Coverflow-checks=on".into()];
(command, rustflags)
};
// Make sure all target-level Miri invocations know their sysroot.
std::env::set_var("MIRI_SYSROOT", &sysroot);
std::env::set_var("MIRI_SYSROOT", sysroot_dir);
// Now invoke xargo.
let mut command = xargo_check();
command.arg("check").arg("-q");
command.current_dir(dir);
command.env("XARGO_HOME", dir);
command.env("XARGO_RUST_SRC", &rust_src);
// We always need to set a target so rustc bootstrap can tell apart host from target crates.
command.arg("--target").arg(target);
// Use Miri as rustc to build a libstd compatible with us (and use the right flags).
// However, when we are running in bootstrap, we cannot just overwrite `RUSTC`,
// because we still need bootstrap to distinguish between host and target crates.
// In that case we overwrite `RUSTC_REAL` instead which determines the rustc used
// for target crates.
// We set ourselves (`cargo-miri`) instead of Miri directly to be able to patch the flags
// for `libpanic_abort` (usually this is done by bootstrap but we have to do it ourselves).
// The `MIRI_CALLED_FROM_XARGO` will mean we dispatch to `phase_setup_rustc`.
let cargo_miri_path = std::env::current_exe().expect("current executable path invalid");
if env::var_os("RUSTC_STAGE").is_some() {
assert!(env::var_os("RUSTC").is_some());
command.env("RUSTC_REAL", &cargo_miri_path);
} else {
command.env("RUSTC", &cargo_miri_path);
}
command.env("MIRI_CALLED_FROM_XARGO", "1");
// Make sure there are no other wrappers getting in our way
// (Cc https://github.com/rust-lang/miri/issues/1421, https://github.com/rust-lang/miri/issues/2429).
// Looks like setting `RUSTC_WRAPPER` to the empty string overwrites `build.rustc-wrapper` set via `config.toml`.
command.env("RUSTC_WRAPPER", "");
// Disable debug assertions in the standard library -- Miri is already slow enough. But keep the
// overflow checks, they are cheap. This completely overwrites flags the user might have set,
// which is consistent with normal `cargo build` that does not apply `RUSTFLAGS` to the sysroot
// either.
command.env("RUSTFLAGS", "-Cdebug-assertions=off -Coverflow-checks=on");
// Manage the output the user sees.
// Do the build.
if only_setup {
// We want to be explicit.
eprintln!("Preparing a sysroot for Miri (target: {target})...");
if print_sysroot {
// Be extra sure there is no noise on stdout.
command.stdout(process::Stdio::null());
}
} else {
// We want to be quiet, but still let the user know that something is happening.
eprint!("Preparing a sysroot for Miri (target: {target})... ");
command.stdout(process::Stdio::null());
command.stderr(process::Stdio::null());
}
// Finally run it!
if command.status().expect("failed to run xargo").success().not() {
if only_setup {
show_error!("failed to run xargo, see error details above")
} else {
show_error!("failed to run xargo; run `cargo miri setup` to see the error details")
}
}
// Figure out what to print.
Sysroot::new(sysroot_dir, target)
.build_from_source(&rust_src, BuildMode::Check, sysroot_config, rustc_version, cargo_cmd)
.unwrap_or_else(|_| {
if only_setup {
show_error!("failed to build sysroot, see error details above")
} else {
show_error!(
"failed to build sysroot; run `cargo miri setup` to see the error details"
)
}
});
if only_setup {
eprintln!("A sysroot for Miri is now available in `{}`.", sysroot.display());
eprintln!("A sysroot for Miri is now available in `{}`.", sysroot_dir.display());
} else {
eprintln!("done");
}
if print_sysroot {
// Print just the sysroot and nothing else to stdout; this way we do not need any escaping.
println!("{}", sysroot.display());
println!("{}", sysroot_dir.display());
}
}

29
src/tools/miri/cargo-miri/src/util.rs
View file

@ -2,14 +2,13 @@ use std::collections::HashMap;
use std::env;
use std::ffi::OsString;
use std::fmt::Write as _;
use std::fs::{self, File};
use std::fs::File;
use std::io::{self, BufWriter, Read, Write};
use std::ops::Not;
use std::path::{Path, PathBuf};
use std::process::Command;
use cargo_metadata::{Metadata, MetadataCommand};
use rustc_version::VersionMeta;
use serde::{Deserialize, Serialize};
pub use crate::arg::*;
@ -111,19 +110,10 @@ pub fn miri_for_host() -> Command {
cmd
}
pub fn version_info() -> VersionMeta {
VersionMeta::for_command(miri_for_host())
.expect("failed to determine underlying rustc version of Miri")
}
pub fn cargo() -> Command {
Command::new(env::var_os("CARGO").unwrap_or_else(|| OsString::from("cargo")))
}
pub fn xargo_check() -> Command {
Command::new(env::var_os("XARGO_CHECK").unwrap_or_else(|| OsString::from("xargo-check")))
}
/// Execute the `Command`, where possible by replacing the current process with a new process
/// described by the `Command`. Then exit this process with the exit code of the new process.
pub fn exec(mut cmd: Command) -> ! {
@ -203,23 +193,6 @@ pub fn ask_to_run(mut cmd: Command, ask: bool, text: &str) {
}
}
/// Writes the given content to the given file *cross-process atomically*, in the sense that another
/// process concurrently reading that file will see either the old content or the new content, but
/// not some intermediate (e.g., empty) state.
///
/// We assume no other parts of this same process are trying to read or write that file.
pub fn write_to_file(filename: &Path, content: &str) {
// Create a temporary file with the desired contents.
let mut temp_filename = filename.as_os_str().to_os_string();
temp_filename.push(&format!(".{}", std::process::id()));
let mut temp_file = File::create(&temp_filename).unwrap();
temp_file.write_all(content.as_bytes()).unwrap();
drop(temp_file);
// Move file to the desired location.
fs::rename(temp_filename, filename).unwrap();
}
// Computes the extra flags that need to be passed to cargo to make it behave like the current
// cargo invocation.
fn cargo_extra_flags() -> Vec<String> {

2
src/tools/miri/cargo-miri/src/version.rs
View file

@ -1,2 +0,0 @@
// We put this in a separate file so that it can be hashed for GHA caching.
pub const XARGO_MIN_VERSION: (u32, u32, u32) = (0, 3, 26);