From 04f32f2016495ec8aabf6ba00b47a5665811eae6 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Mon, 13 Nov 2023 16:05:35 +0100 Subject: [PATCH 01/34] Allow rustfmt to run on `build_system` --- .rustfmt.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.rustfmt.toml b/.rustfmt.toml index c7ad93bafe36..87f034950e3b 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -1 +1 @@ -disable_all_formatting = true +ignore = ["/src", "/tests"] From 7b76ac4eb74f3cf8e0d616b82f8135fd7fd7ccab Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 8 Nov 2023 17:31:56 +0100 Subject: [PATCH 02/34] Rustify `test.sh` --- build_system/src/build.rs | 128 ++-- build_system/src/config.rs | 330 ++++++---- build_system/src/prepare.rs | 18 +- build_system/src/test.rs | 1143 ++++++++++++++++++++++++++++++++++- build_system/src/utils.rs | 104 +++- 5 files changed, 1518 insertions(+), 205 deletions(-) diff --git a/build_system/src/build.rs b/build_system/src/build.rs index f1c3701a946e..6390458d4fd0 100644 --- a/build_system/src/build.rs +++ b/build_system/src/build.rs @@ -1,7 +1,5 @@ -use crate::config::{set_config, ConfigInfo}; -use crate::utils::{ - get_gcc_path, run_command, run_command_with_output_and_env, walk_dir, -}; +use crate::config::ConfigInfo; +use crate::utils::{get_gcc_path, run_command, run_command_with_output_and_env, walk_dir}; use std::collections::HashMap; use std::ffi::OsStr; use std::fs; @@ -10,10 +8,9 @@ use std::path::Path; #[derive(Default)] struct BuildArg { codegen_release_channel: bool, - sysroot_release_channel: bool, - sysroot_panic_abort: bool, flags: Vec, gcc_path: String, + config_info: ConfigInfo, } impl BuildArg { @@ -29,13 +26,9 @@ impl BuildArg { while let Some(arg) = args.next() { match arg.as_str() { "--release" => build_arg.codegen_release_channel = true, - "--release-sysroot" => build_arg.sysroot_release_channel = true, "--no-default-features" => { build_arg.flags.push("--no-default-features".to_string()); } - "--sysroot-panic-abort" => { - build_arg.sysroot_panic_abort = true; - }, "--features" => { if let Some(arg) = args.next() { build_arg.flags.push("--features".to_string()); @@ -63,12 +56,14 @@ impl BuildArg { if args.next().is_some() { // Handled in config.rs. } else { - return Err( - "Expected a value after `--target`, found nothing".to_string() - ); + return Err("Expected a value after `--target`, found nothing".to_string()); + } + } + arg => { + if !build_arg.config_info.parse_argument(arg, &mut args)? { + return Err(format!("Unknown argument `{}`", arg)); } } - arg => return Err(format!("Unknown argument `{}`", arg)), } } Ok(Some(build_arg)) @@ -80,28 +75,26 @@ impl BuildArg { `build` command help: --release : Build codegen in release mode - --release-sysroot : Build sysroot in release mode - --sysroot-panic-abort : Build the sysroot without unwinding support. --no-default-features : Add `--no-default-features` flag - --features [arg] : Add a new feature [arg] - --target-triple [arg] : Set the target triple to [arg] - --help : Show this help -"# - ) + --features [arg] : Add a new feature [arg]"# + ); + ConfigInfo::show_usage(); + println!(" --help : Show this help"); } } -fn build_sysroot( - env: &mut HashMap, - args: &BuildArg, +fn build_sysroot_inner( + env: &HashMap, + sysroot_panic_abort: bool, + sysroot_release_channel: bool, config: &ConfigInfo, + start_dir: Option<&Path>, ) -> Result<(), String> { - std::env::set_current_dir("build_sysroot") - .map_err(|error| format!("Failed to go to `build_sysroot` directory: {:?}", error))?; + let start_dir = start_dir.unwrap_or_else(|| Path::new(".")); // Cleanup for previous run // Clean target dir except for build scripts and incremental cache let _ = walk_dir( - "target", + start_dir.join("target"), |dir: &Path| { for top in &["debug", "release"] { let _ = fs::remove_dir_all(dir.join(top).join("build")); @@ -138,23 +131,22 @@ fn build_sysroot( |_| Ok(()), ); - let _ = fs::remove_file("Cargo.lock"); - let _ = fs::remove_file("test_target/Cargo.lock"); - let _ = fs::remove_dir_all("sysroot"); + let _ = fs::remove_file(start_dir.join("Cargo.lock")); + let _ = fs::remove_file(start_dir.join("test_target/Cargo.lock")); + let _ = fs::remove_dir_all(start_dir.join("sysroot")); // Builds libs - let mut rustflags = env - .get("RUSTFLAGS") - .cloned() - .unwrap_or_default(); - if args.sysroot_panic_abort { + let mut rustflags = env.get("RUSTFLAGS").cloned().unwrap_or_default(); + if sysroot_panic_abort { rustflags.push_str(" -Cpanic=abort -Zpanic-abort-tests"); } - env.insert( - "RUSTFLAGS".to_string(), - format!("{} -Zmir-opt-level=3", rustflags), - ); - let channel = if args.sysroot_release_channel { + rustflags.push_str(" -Z force-unstable-if-unmarked"); + let channel = if sysroot_release_channel { + let mut env = env.clone(); + env.insert( + "RUSTFLAGS".to_string(), + format!("{} -Zmir-opt-level=3", rustflags), + ); run_command_with_output_and_env( &[ &"cargo", @@ -163,33 +155,34 @@ fn build_sysroot( &config.target, &"--release", ], - None, + Some(start_dir), Some(&env), )?; "release" } else { run_command_with_output_and_env( - &[ - &"cargo", - &"build", - &"--target", - &config.target, - ], - None, + &[&"cargo", &"build", &"--target", &config.target], + Some(start_dir), Some(env), )?; "debug" }; // Copy files to sysroot - let sysroot_path = format!("sysroot/lib/rustlib/{}/lib/", config.target_triple); - fs::create_dir_all(&sysroot_path) - .map_err(|error| format!("Failed to create directory `{}`: {:?}", sysroot_path, error))?; + let sysroot_path = start_dir.join(format!("sysroot/lib/rustlib/{}/lib/", config.target_triple)); + fs::create_dir_all(&sysroot_path).map_err(|error| { + format!( + "Failed to create directory `{}`: {:?}", + sysroot_path.display(), + error + ) + })?; let copier = |dir_to_copy: &Path| { + // FIXME: should not use shell command! run_command(&[&"cp", &"-r", &dir_to_copy, &sysroot_path], None).map(|_| ()) }; walk_dir( - &format!("target/{}/{}/deps", config.target_triple, channel), + start_dir.join(&format!("target/{}/{}/deps", config.target_triple, channel)), copier, copier, )?; @@ -203,7 +196,22 @@ fn build_sysroot( Ok(()) } -fn build_codegen(args: &BuildArg) -> Result<(), String> { +pub fn build_sysroot( + env: &HashMap, + sysroot_panic_abort: bool, + sysroot_release_channel: bool, + config: &ConfigInfo, +) -> Result<(), String> { + build_sysroot_inner( + env, + sysroot_panic_abort, + sysroot_release_channel, + config, + Some(Path::new("build_sysroot")), + ) +} + +fn build_codegen(args: &mut BuildArg) -> Result<(), String> { let mut env = HashMap::new(); env.insert("LD_LIBRARY_PATH".to_string(), args.gcc_path.clone()); @@ -223,7 +231,8 @@ fn build_codegen(args: &BuildArg) -> Result<(), String> { } run_command_with_output_and_env(&command, None, Some(&env))?; - let config = set_config(&mut env, &[], Some(&args.gcc_path))?; + args.config_info + .setup(&mut env, &[], Some(&args.gcc_path))?; // We voluntarily ignore the error. let _ = fs::remove_dir_all("target/out"); @@ -237,18 +246,19 @@ fn build_codegen(args: &BuildArg) -> Result<(), String> { println!("[BUILD] sysroot"); build_sysroot( - &mut env, - args, - &config, + &env, + args.config_info.sysroot_panic_abort, + args.config_info.sysroot_release_channel, + &args.config_info, )?; Ok(()) } pub fn run() -> Result<(), String> { - let args = match BuildArg::new()? { + let mut args = match BuildArg::new()? { Some(args) => args, None => return Ok(()), }; - build_codegen(&args)?; + build_codegen(&mut args)?; Ok(()) } diff --git a/build_system/src/config.rs b/build_system/src/config.rs index 64d9bd73e01a..763cac8edb66 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -1,149 +1,229 @@ -use crate::utils::{get_gcc_path, get_os_name, get_rustc_host_triple}; +use crate::utils::{get_gcc_path, get_os_name, rustc_version_info, split_args}; use std::collections::HashMap; use std::env as std_env; +#[derive(Default)] pub struct ConfigInfo { pub target: String, pub target_triple: String, + pub host_triple: String, pub rustc_command: Vec, + pub run_in_vm: bool, + pub cargo_target_dir: String, + pub dylib_ext: String, + pub sysroot_release_channel: bool, + pub sysroot_panic_abort: bool, } -// Returns the beginning for the command line of rustc. -pub fn set_config( - env: &mut HashMap, - test_flags: &[String], - gcc_path: Option<&str>, -) -> Result { - env.insert("CARGO_INCREMENTAL".to_string(), "0".to_string()); - - let gcc_path = match gcc_path { - Some(path) => path.to_string(), - None => get_gcc_path()?, - }; - env.insert("GCC_PATH".to_string(), gcc_path.clone()); - - let os_name = get_os_name()?; - let dylib_ext = match os_name.as_str() { - "Linux" => "so", - "Darwin" => "dylib", - os => return Err(format!("unsupported OS `{}`", os)), - }; - let host_triple = get_rustc_host_triple()?; - let mut linker = None; - let mut target_triple = host_triple.clone(); - let mut target = target_triple.clone(); - - // We skip binary name and the command. - let mut args = std::env::args().skip(2); - - let mut set_target_triple = false; - let mut set_target = false; - while let Some(arg) = args.next() { - match arg.as_str() { - "--target-triple" => { - if let Some(arg) = args.next() { - target_triple = arg; - set_target_triple = true; - } else { +impl ConfigInfo { + /// Returns `true` if the argument was taken into account. + pub fn parse_argument( + &mut self, + arg: &str, + args: &mut impl Iterator, + ) -> Result { + match arg { + "--target-triple" => match args.next() { + Some(arg) if !arg.is_empty() => self.target_triple = arg.to_string(), + _ => { return Err( "Expected a value after `--target-triple`, found nothing".to_string() - ); + ) } }, - "--target" => { - if let Some(arg) = args.next() { - target = arg; - set_target = true; - } else { - return Err( - "Expected a value after `--target`, found nothing".to_string() - ); - } + "--target" => match args.next() { + Some(arg) if !arg.is_empty() => self.target = arg.to_string(), + _ => return Err("Expected a value after `--target`, found nothing".to_string()), }, - _ => (), + "--out-dir" => match args.next() { + Some(arg) if !arg.is_empty() => { + // env.insert("CARGO_TARGET_DIR".to_string(), arg.to_string()); + self.cargo_target_dir = arg.to_string(); + } + _ => return Err("Expected a value after `--out-dir`, found nothing".to_string()), + }, + "--release-sysroot" => self.sysroot_release_channel = true, + "--sysroot-panic-abort" => self.sysroot_panic_abort = true, + _ => return Ok(false), } + Ok(true) } - if set_target_triple && !set_target { - target = target_triple.clone(); - } + pub fn setup( + &mut self, + env: &mut HashMap, + test_flags: &[String], + gcc_path: Option<&str>, + ) -> Result<(), String> { + env.insert("CARGO_INCREMENTAL".to_string(), "0".to_string()); - if host_triple != target_triple { - linker = Some(format!("-Clinker={}-gcc", target_triple)); - } - let current_dir = - std_env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?; - let channel = if let Some(channel) = env.get("CHANNEL") { - channel.as_str() - } else { - "debug" - }; - let cg_backend_path = current_dir - .join("target") - .join(channel) - .join(&format!("librustc_codegen_gcc.{}", dylib_ext)); - let sysroot_path = current_dir.join("build_sysroot/sysroot"); - let mut rustflags = Vec::new(); - if let Some(cg_rustflags) = env.get("CG_RUSTFLAGS") { - rustflags.push(cg_rustflags.clone()); - } - if let Some(linker) = linker { - rustflags.push(linker.to_string()); - } - rustflags.extend_from_slice(&[ - "-Csymbol-mangling-version=v0".to_string(), - "-Cdebuginfo=2".to_string(), - format!("-Zcodegen-backend={}", cg_backend_path.display()), - "--sysroot".to_string(), - sysroot_path.display().to_string(), - ]); + let gcc_path = match gcc_path { + Some(path) => path.to_string(), + None => get_gcc_path()?, + }; + env.insert("GCC_PATH".to_string(), gcc_path.clone()); - // Since we don't support ThinLTO, disable LTO completely when not trying to do LTO. - // TODO(antoyo): remove when we can handle ThinLTO. - if !env.contains_key(&"FAT_LTO".to_string()) { - rustflags.push("-Clto=off".to_string()); - } - rustflags.extend_from_slice(test_flags); - // FIXME(antoyo): remove once the atomic shim is gone - if os_name == "Darwin" { + if self.cargo_target_dir.is_empty() { + match env.get("CARGO_TARGET_DIR").filter(|dir| !dir.is_empty()) { + Some(cargo_target_dir) => self.cargo_target_dir = cargo_target_dir.clone(), + None => self.cargo_target_dir = "target/out".to_string(), + } + } + + let os_name = get_os_name()?; + self.dylib_ext = match os_name.as_str() { + "Linux" => "so", + "Darwin" => "dylib", + os => return Err(format!("unsupported OS `{}`", os)), + } + .to_string(); + let rustc = match env.get("RUSTC") { + Some(r) if !r.is_empty() => r.to_string(), + _ => "rustc".to_string(), + }; + self.host_triple = rustc_version_info(Some(&rustc))?.host.unwrap_or_default(); + + if !self.target_triple.is_empty() && self.target.is_empty() { + self.target = self.target_triple.clone(); + } + if self.target.is_empty() { + self.target = self.host_triple.clone(); + } + if self.target_triple.is_empty() { + self.target_triple = self.host_triple.clone(); + } + + let mut linker = None; + + if self.host_triple != self.target_triple { + if self.target_triple == "m68k-unknown-linux-gnu" { + linker = Some("-Clinker=m68k-unknown-linux-gnu-gcc".to_string()); + } else if self.target_triple == "aarch64-unknown-linux-gnu" { + // We are cross-compiling for aarch64. Use the correct linker and run tests in qemu. + linker = Some("-Clinker=aarch64-linux-gnu-gcc".to_string()); + } else { + return Err("Unknown non-native platform".to_string()); + } + + self.run_in_vm = true; + } + + let current_dir = + std_env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?; + let channel = if let Some(channel) = env.get("CHANNEL") { + channel.as_str() + } else { + "debug" + }; + + let has_builtin_backend = env + .get("BUILTIN_BACKEND") + .map(|backend| !backend.is_empty()) + .unwrap_or(false); + let cg_backend_path; + + let mut rustflags = Vec::new(); + if has_builtin_backend { + // It means we're building inside the rustc testsuite, so some options need to be handled + // a bit differently. + cg_backend_path = "gcc".to_string(); + + match env.get("RUSTC_SYSROOT") { + Some(rustc_sysroot) if !rustc_sysroot.is_empty() => { + rustflags.extend_from_slice(&["--sysroot".to_string(), rustc_sysroot.clone()]); + } + _ => {} + } + rustflags.push("-Cpanic=abort".to_string()); + } else { + cg_backend_path = current_dir + .join("target") + .join(channel) + .join(&format!("librustc_codegen_gcc.{}", self.dylib_ext)) + .display() + .to_string(); + let sysroot_path = current_dir.join("build_sysroot/sysroot"); + rustflags + .extend_from_slice(&["--sysroot".to_string(), sysroot_path.display().to_string()]); + }; + + if let Some(cg_rustflags) = env.get("CG_RUSTFLAGS") { + rustflags.extend_from_slice(&split_args(&cg_rustflags)); + } + if let Some(linker) = linker { + rustflags.push(linker.to_string()); + } rustflags.extend_from_slice(&[ - "-Clink-arg=-undefined".to_string(), - "-Clink-arg=dynamic_lookup".to_string(), + "-Csymbol-mangling-version=v0".to_string(), + "-Cdebuginfo=2".to_string(), + format!("-Zcodegen-backend={}", cg_backend_path), ]); + + // Since we don't support ThinLTO, disable LTO completely when not trying to do LTO. + // TODO(antoyo): remove when we can handle ThinLTO. + if !env.contains_key(&"FAT_LTO".to_string()) { + rustflags.push("-Clto=off".to_string()); + } + rustflags.extend_from_slice(test_flags); + // FIXME(antoyo): remove once the atomic shim is gone + if os_name == "Darwin" { + rustflags.extend_from_slice(&[ + "-Clink-arg=-undefined".to_string(), + "-Clink-arg=dynamic_lookup".to_string(), + ]); + } + env.insert("RUSTFLAGS".to_string(), rustflags.join(" ")); + // display metadata load errors + env.insert("RUSTC_LOG".to_string(), "warn".to_string()); + + let sysroot = current_dir.join(&format!( + "build_sysroot/sysroot/lib/rustlib/{}/lib", + self.target_triple, + )); + let ld_library_path = format!( + "{target}:{sysroot}:{gcc_path}", + target = current_dir.join("target/out").display(), + sysroot = sysroot.display(), + ); + env.insert("LD_LIBRARY_PATH".to_string(), ld_library_path.clone()); + env.insert("DYLD_LIBRARY_PATH".to_string(), ld_library_path); + + // NOTE: To avoid the -fno-inline errors, use /opt/gcc/bin/gcc instead of cc. + // To do so, add a symlink for cc to /opt/gcc/bin/gcc in our PATH. + // Another option would be to add the following Rust flag: -Clinker=/opt/gcc/bin/gcc + let path = std::env::var("PATH").unwrap_or_default(); + env.insert( + "PATH".to_string(), + format!( + "/opt/gcc/bin:/opt/m68k-unknown-linux-gnu/bin{}{}", + if path.is_empty() { "" } else { ":" }, + path + ), + ); + + self.rustc_command = vec![rustc]; + self.rustc_command.extend_from_slice(&rustflags); + self.rustc_command.extend_from_slice(&[ + "-L".to_string(), + "crate=target/out".to_string(), + "--out-dir".to_string(), + self.cargo_target_dir.clone(), + ]); + + if !env.contains_key("RUSTC_LOG") { + env.insert("RUSTC_LOG".to_string(), "warn".to_string()); + } + Ok(()) } - env.insert("RUSTFLAGS".to_string(), rustflags.join(" ")); - // display metadata load errors - env.insert("RUSTC_LOG".to_string(), "warn".to_string()); - let sysroot = current_dir.join(&format!( - "build_sysroot/sysroot/lib/rustlib/{}/lib", - target_triple - )); - let ld_library_path = format!( - "{target}:{sysroot}:{gcc_path}", - target = current_dir.join("target/out").display(), - sysroot = sysroot.display(), - ); - env.insert("LD_LIBRARY_PATH".to_string(), ld_library_path.clone()); - env.insert("DYLD_LIBRARY_PATH".to_string(), ld_library_path); - - // NOTE: To avoid the -fno-inline errors, use /opt/gcc/bin/gcc instead of cc. - // To do so, add a symlink for cc to /opt/gcc/bin/gcc in our PATH. - // Another option would be to add the following Rust flag: -Clinker=/opt/gcc/bin/gcc - let path = std::env::var("PATH").unwrap_or_default(); - env.insert("PATH".to_string(), format!("/opt/gcc/bin:{}", path)); - - let mut rustc_command = vec!["rustc".to_string()]; - rustc_command.extend_from_slice(&rustflags); - rustc_command.extend_from_slice(&[ - "-L".to_string(), - "crate=target/out".to_string(), - "--out-dir".to_string(), - "target/out".to_string(), - ]); - Ok(ConfigInfo { - target, - target_triple, - rustc_command, - }) + pub fn show_usage() { + println!( + "\ + --target [arg] : Set the target to [arg] + --target-triple [arg] : Set the target triple to [arg] + --out-dir : Location where the files will be generated + --release-sysroot : Build sysroot in release mode + --sysroot-panic-abort : Build the sysroot without unwinding support." + ); + } } diff --git a/build_system/src/prepare.rs b/build_system/src/prepare.rs index 6c7c85868345..da9f8953ec3c 100644 --- a/build_system/src/prepare.rs +++ b/build_system/src/prepare.rs @@ -4,7 +4,11 @@ use crate::utils::{cargo_install, git_clone, run_command, run_command_with_outpu use std::fs; use std::path::Path; -fn prepare_libcore(sysroot_path: &Path, libgccjit12_patches: bool, cross_compile: bool) -> Result<(), String> { +fn prepare_libcore( + sysroot_path: &Path, + libgccjit12_patches: bool, + cross_compile: bool, +) -> Result<(), String> { let rustc_path = match get_rustc_path() { Some(path) => path, None => return Err("`rustc` path not found".to_string()), @@ -88,10 +92,14 @@ fn prepare_libcore(sysroot_path: &Path, libgccjit12_patches: bool, cross_compile }, )?; if cross_compile { - walk_dir("cross_patches", |_| Ok(()), |file_path: &Path| { - patches.push(file_path.to_path_buf()); - Ok(()) - })?; + walk_dir( + "cross_patches", + |_| Ok(()), + |file_path: &Path| { + patches.push(file_path.to_path_buf()); + Ok(()) + }, + )?; } if libgccjit12_patches { walk_dir( diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 4c8c63e59ab7..fb2b24da9a23 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -1,15 +1,1138 @@ -use crate::utils::run_command_with_output; +use crate::build; +use crate::config::ConfigInfo; +use crate::utils::{ + get_gcc_path, get_toolchain, run_command, run_command_with_env, + run_command_with_output_and_env, rustc_version_info, split_args, walk_dir, +}; -fn get_args<'a>(args: &mut Vec<&'a dyn AsRef>, extra_args: &'a Vec) { - for extra_arg in extra_args { - args.push(extra_arg); +use std::collections::{BTreeSet, HashMap}; +use std::ffi::OsStr; +use std::fs::remove_dir_all; +use std::path::{Path, PathBuf}; +use std::str::FromStr; + +type Env = HashMap; +type Runner = &'static dyn Fn(&Env, &TestArg) -> Result<(), String>; +type Runners = HashMap<&'static str, (&'static str, Runner)>; + +fn get_runners() -> Runners { + let mut runners = HashMap::new(); + + runners.insert( + "--test-rustc", + ("Run all rustc tests", &test_rustc as Runner), + ); + runners.insert( + "--test-successful-rustc", + ("Run successful rustc tests", &test_successful_rustc), + ); + runners.insert( + "--test-failing-rustc", + ("Run failing rustc tests", &test_failing_rustc), + ); + runners.insert("--test-libcore", ("Run libcore tests", &test_libcore)); + runners.insert("--clean-ui-tests", ("Clean ui tests", &clean_ui_tests)); + runners.insert("--clean", ("Empty cargo target directory", &clean)); + runners.insert("--std-tests", ("Run std tests", &std_tests)); + runners.insert("--asm-tests", ("Run asm tests", &asm_tests)); + runners.insert( + "--extended-tests", + ("Run extended sysroot tests", &extended_sysroot_tests), + ); + runners.insert( + "--extended-rand-tests", + ("Run extended rand tests", &extended_rand_tests), + ); + runners.insert( + "--extended-regex-example-tests", + ( + "Run extended regex example tests", + &extended_regex_example_tests, + ), + ); + runners.insert( + "--extended-regex-tests", + ("Run extended regex tests", &extended_regex_tests), + ); + runners.insert("--mini-tests", ("Run mini tests", &mini_tests)); + + runners +} + +fn get_number_after_arg( + args: &mut impl Iterator, + option: &str, +) -> Result { + match args.next() { + Some(nb) if !nb.is_empty() => match usize::from_str(&nb) { + Ok(nb) => Ok(nb), + Err(_) => Err(format!( + "Expected a number after `{}`, found `{}`", + option, nb + )), + }, + _ => Err(format!( + "Expected a number after `{}`, found nothing", + option + )), } } -pub fn run() -> Result<(), String> { - let mut args: Vec<&dyn AsRef> = vec![&"bash", &"test.sh"]; - let extra_args = std::env::args().skip(2).collect::>(); - get_args(&mut args, &extra_args); - let current_dir = std::env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?; - run_command_with_output(args.as_slice(), Some(¤t_dir)) +fn show_usage() { + println!( + r#" +`test` command help: + + --release : Build codegen in release mode + --release-sysroot : Build sysroot in release mode + --sysroot-panic-abort : Build the sysroot without unwinding support. + --no-default-features : Add `--no-default-features` flag + --features [arg] : Add a new feature [arg] + --use-system-gcc : Use system installed libgccjit + --build-only : Only build rustc_codegen_gcc then exits + --use-backend : Useful only for rustc testsuite + --nb-parts : Used to split rustc_tests (for CI needs) + --current-part : Used with `--nb-parts`, allows you to specify which parts to test"# + ); + ConfigInfo::show_usage(); + for (option, (doc, _)) in get_runners() { + let needed_spaces = 23_usize.saturating_sub(option.len()); + let spaces: String = std::iter::repeat(' ').take(needed_spaces).collect(); + println!(" {}{}: {}", option, spaces, doc); + } + println!(" --help : Show this help"); +} + +#[derive(Default, PartialEq, Eq, Clone, Copy)] +enum Channel { + #[default] + Debug, + Release, +} + +impl Channel { + pub fn as_str(self) -> &'static str { + match self { + Self::Debug => "debug", + Self::Release => "release", + } + } +} + +#[derive(Default)] +struct TestArg { + no_default_features: bool, + build_only: bool, + gcc_path: String, + channel: Channel, + sysroot_channel: Channel, + use_backend: bool, + runners: BTreeSet, + flags: Vec, + backend: Option, + nb_parts: Option, + current_part: Option, + sysroot_panic_abort: bool, + config_info: ConfigInfo, +} + +impl TestArg { + fn new() -> Result, String> { + let mut use_system_gcc = false; + let mut test_arg = Self::default(); + + // We skip binary name and the `test` command. + let mut args = std::env::args().skip(2); + let runners = get_runners(); + + while let Some(arg) = args.next() { + match arg.as_str() { + "--release" => test_arg.channel = Channel::Release, + "--release-sysroot" => test_arg.sysroot_channel = Channel::Release, + "--no-default-features" => { + // To prevent adding it more than once. + if !test_arg.no_default_features { + test_arg.flags.push("--no-default-features".into()); + } + test_arg.no_default_features = true; + } + "--features" => match args.next() { + Some(feature) if !feature.is_empty() => { + test_arg + .flags + .extend_from_slice(&["--features".into(), feature]); + } + _ => { + return Err("Expected an argument after `--features`, found nothing".into()) + } + }, + "--use-system-gcc" => use_system_gcc = true, + "--build-only" => test_arg.build_only = true, + "--use-backend" => match args.next() { + Some(backend) if !backend.is_empty() => test_arg.backend = Some(backend), + _ => { + return Err( + "Expected an argument after `--use-backend`, found nothing".into() + ) + } + }, + "--nb-parts" => { + test_arg.nb_parts = Some(get_number_after_arg(&mut args, "--nb-parts")?); + } + "--current-part" => { + test_arg.current_part = + Some(get_number_after_arg(&mut args, "--current-part")?); + } + "--sysroot-panic-abort" => { + test_arg.sysroot_panic_abort = true; + } + "--help" => { + show_usage(); + return Ok(None); + } + x if runners.contains_key(x) => { + test_arg.runners.insert(x.into()); + } + arg => { + if !test_arg.config_info.parse_argument(arg, &mut args)? { + return Err(format!("Unknown option {}", arg)); + } + } + } + + test_arg.gcc_path = if use_system_gcc { + println!("Using system GCC"); + "gcc".to_string() + } else { + get_gcc_path()? + }; + } + Ok(Some(test_arg)) + } +} + +fn build_if_no_backend(env: &Env, args: &TestArg) -> Result<(), String> { + if args.use_backend { + return Ok(()); + } + let mut command: Vec<&dyn AsRef> = vec![&"cargo", &"rustc"]; + if args.channel == Channel::Release { + let mut env = env.clone(); + env.insert("CARGO_INCREMENTAL".to_string(), "1".to_string()); + command.push(&"--release"); + for flag in args.flags.iter() { + command.push(flag); + } + run_command_with_output_and_env(&command, None, Some(&env)) + } else { + for flag in args.flags.iter() { + command.push(flag); + } + run_command_with_output_and_env(&command, None, Some(&env)) + } +} + +fn clean(_env: &Env, args: &TestArg) -> Result<(), String> { + let _ = std::fs::remove_dir_all(&args.config_info.cargo_target_dir); + let path = Path::new(&args.config_info.cargo_target_dir).join("gccjit"); + std::fs::create_dir_all(&path) + .map_err(|error| format!("failed to create folder `{}`: {:?}", path.display(), error)) +} + +fn mini_tests(env: &Env, args: &TestArg) -> Result<(), String> { + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[BUILD] mini_core"); + let crate_types = if args.config_info.host_triple != args.config_info.target_triple { + "lib" + } else { + "lib,dylib" + } + .to_string(); + let mut command: Vec<&dyn AsRef> = Vec::new(); + for arg in args.config_info.rustc_command.iter() { + command.push(arg); + } + command.extend_from_slice(&[ + &"example/mini_core.rs", + &"--crate-name", + &"mini_core", + &"--crate-type", + &crate_types, + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_output_and_env(&command, None, Some(&env))?; + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[BUILD] example"); + command.clear(); + for arg in args.config_info.rustc_command.iter() { + command.push(arg); + } + command.extend_from_slice(&[ + &"example/example.rs", + &"--crate-type", + &"lib", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_output_and_env(&command, None, Some(&env))?; + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] mini_core_hello_world"); + command.clear(); + for arg in args.config_info.rustc_command.iter() { + command.push(arg); + } + command.extend_from_slice(&[ + &"example/mini_core_hello_world.rs", + &"--crate-name", + &"mini_core_hello_world", + &"--crate-type", + &"bin", + &"-g", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_output_and_env(&command, None, Some(&env))?; + + let command: &[&dyn AsRef] = &[ + &Path::new(&args.config_info.cargo_target_dir).join("mini_core_hello_world"), + &"abc", + &"bcd", + ]; + run_command_in_vm(&command, env, args)?; + Ok(()) +} + +fn build_sysroot(env: &Env, args: &TestArg) -> Result<(), String> { + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[BUILD] sysroot"); + build::build_sysroot( + env, + args.config_info.sysroot_panic_abort, + args.config_info.sysroot_release_channel, + &args.config_info, + )?; + Ok(()) +} + +// TODO(GuillaumeGomez): when rewriting in Rust, refactor with the code in tests/lang_tests_common.rs if possible. +fn run_command_in_vm( + command: &[&dyn AsRef], + env: &Env, + args: &TestArg, +) -> Result<(), String> { + if !args.config_info.run_in_vm { + run_command_with_env(command, None, Some(env))?; + return Ok(()); + } + let vm_parent_dir = match env.get("CG_GCC_VM_DIR") { + Some(dir) if !dir.is_empty() => PathBuf::from(dir.clone()), + _ => std::env::current_dir().unwrap(), + }; + let vm_dir = "vm"; + let exe_to_run = command.first().unwrap(); + let exe = Path::new(&exe_to_run); + let exe_filename = exe.file_name().unwrap(); + let vm_home_dir = vm_parent_dir.join(vm_dir).join("home"); + let vm_exe_path = vm_home_dir.join(exe_filename); + let inside_vm_exe_path = Path::new("/home").join(exe_filename); + + let sudo_command: &[&dyn AsRef] = &[&"sudo", &"cp", &exe, &vm_exe_path]; + run_command_with_env(sudo_command, None, Some(env))?; + + let mut vm_command: Vec<&dyn AsRef> = + vec![&"sudo", &"chroot", &"qemu-m68k-static", &inside_vm_exe_path]; + vm_command.extend_from_slice(command); + run_command_with_env(&vm_command, Some(&vm_parent_dir), Some(env))?; + Ok(()) +} + +fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] arbitrary_self_types_pointers_and_wrappers"); + let mut command: Vec<&dyn AsRef> = Vec::new(); + for arg in args.config_info.rustc_command.iter() { + command.push(arg); + } + command.extend_from_slice(&[ + &"example/arbitrary_self_types_pointers_and_wrappers.rs", + &"--crate-name", + &"arbitrary_self_types_pointers_and_wrappers", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_env(&command, None, Some(env))?; + run_command_in_vm( + &[&Path::new(&args.config_info.cargo_target_dir) + .join("arbitrary_self_types_pointers_and_wrappers")], + env, + args, + )?; + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] alloc_system"); + let mut command: Vec<&dyn AsRef> = Vec::new(); + for arg in args.config_info.rustc_command.iter() { + command.push(arg); + } + command.extend_from_slice(&[ + &"example/alloc_system.rs", + &"--crate-type", + &"lib", + &"--target", + &args.config_info.target_triple, + ]); + if !args.no_default_features { + command.push(&"--cfg feature=\"master\""); + } + run_command_with_env(&command, None, Some(env))?; + + // FIXME: doesn't work on m68k. + if args.config_info.host_triple != args.config_info.target_triple { + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] alloc_example"); + let mut command: Vec<&dyn AsRef> = Vec::new(); + for arg in args.config_info.rustc_command.iter() { + command.push(arg); + } + command.extend_from_slice(&[ + &"example/alloc_example.rs", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_env(&command, None, Some(env))?; + run_command_in_vm( + &[&Path::new(&args.config_info.cargo_target_dir).join("alloc_example")], + env, + args, + )?; + } + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] dst_field_align"); + // FIXME(antoyo): Re-add -Zmir-opt-level=2 once rust-lang/rust#67529 is fixed. + let mut command: Vec<&dyn AsRef> = Vec::new(); + for arg in args.config_info.rustc_command.iter() { + command.push(arg); + } + command.extend_from_slice(&[ + &"example/dst-field-align.rs", + &"--crate-name", + &"dst_field_align", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_env(&command, None, Some(env))?; + run_command_in_vm( + &[&Path::new(&args.config_info.cargo_target_dir).join("dst_field_align")], + env, + args, + )?; + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] std_example"); + let mut command: Vec<&dyn AsRef> = Vec::new(); + for arg in args.config_info.rustc_command.iter() { + command.push(arg); + } + command.extend_from_slice(&[ + &"example/std_example.rs", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + if !args.no_default_features { + command.push(&"--cfg feature=\"master\""); + } + run_command_with_env(&command, None, Some(env))?; + run_command_in_vm( + &[ + &Path::new(&args.config_info.cargo_target_dir).join("std_example"), + &"--target", + &args.config_info.target_triple, + ], + env, + args, + )?; + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] subslice-patterns-const-eval"); + let mut command: Vec<&dyn AsRef> = Vec::new(); + for arg in args.config_info.rustc_command.iter() { + command.push(arg); + } + command.extend_from_slice(&[ + &"example/subslice-patterns-const-eval.rs", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_env(&command, None, Some(env))?; + run_command_in_vm( + &[&Path::new(&args.config_info.cargo_target_dir).join("subslice-patterns-const-eval")], + env, + args, + )?; + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] track-caller-attribute"); + let mut command: Vec<&dyn AsRef> = Vec::new(); + for arg in args.config_info.rustc_command.iter() { + command.push(arg); + } + command.extend_from_slice(&[ + &"example/track-caller-attribute.rs", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_env(&command, None, Some(env))?; + run_command_in_vm( + &[&Path::new(&args.config_info.cargo_target_dir).join("track-caller-attribute")], + env, + args, + )?; + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[AOT] mod_bench"); + let mut command: Vec<&dyn AsRef> = Vec::new(); + for arg in args.config_info.rustc_command.iter() { + command.push(arg); + } + command.extend_from_slice(&[ + &"example/mod_bench.rs", + &"--crate-type", + &"bin", + &"--target", + &args.config_info.target_triple, + ]); + run_command_with_env(&command, None, Some(env))?; + // FIXME: the compiled binary is not run. Is it normal? + + Ok(()) +} + +fn setup_rustc(env: &mut Env, args: &TestArg) -> Result<(), String> { + let toolchain = get_toolchain()?; + + let rust_dir = Some(Path::new("rust")); + // If the repository was already cloned, command will fail, so doesn't matter. + let _ = run_command_with_output_and_env( + &[&"git", &"clone", &"https://github.com/rust-lang/rust.git"], + None, + Some(env), + ); + run_command_with_output_and_env(&[&"git", &"fetch"], rust_dir, Some(env))?; + let rustc_commit = match rustc_version_info(env.get("RUSTC").map(|s| s.as_str()))?.commit_hash { + Some(commit_hash) => commit_hash, + None => return Err("Couldn't retrieve rustc commit hash".to_string()), + }; + run_command_with_output_and_env(&[&"git", &"checkout", &rustc_commit], rust_dir, Some(env))?; + env.insert("RUSTFLAGS".to_string(), String::new()); + let cargo = String::from_utf8( + run_command_with_env(&[&"rustup", &"which", &"cargo"], rust_dir, Some(env))?.stdout, + ) + .map_err(|error| format!("Failed to retrieve cargo path: {:?}", error)) + .and_then(|cargo| { + let cargo = cargo.trim().to_owned(); + if cargo.is_empty() { + Err(format!("`cargo` path is empty")) + } else { + Ok(cargo) + } + })?; + let llvm_filecheck = String::from_utf8( + run_command_with_env( + &[ + &"bash", + &"-c", + &"which FileCheck-10 || \ + which FileCheck-11 || \ + which FileCheck-12 || \ + which FileCheck-13 || \ + which FileCheck-14", + ], + rust_dir, + Some(env), + )? + .stdout, + ) + .map_err(|error| format!("Failed to retrieve LLVM FileCheck: {:?}", error))?; + std::fs::write( + "rust/config.toml", + &format!( + r#"change-id = 115898 + +[rust] +codegen-backends = [] +deny-warnings = false +verbose-tests = true + +[build] +cargo = "{cargo}" +local-rebuild = true +rustc = "{home}/.rustup/toolchains/{toolchain}-{host_triple}/bin/rustc" + +[target.x86_64-unknown-linux-gnu] +llvm-filecheck = "{llvm_filecheck}" + +[llvm] +download-ci-llvm = false +"#, + cargo = cargo.trim(), + home = env.get("HOME").unwrap(), + toolchain = toolchain, + host_triple = args.config_info.host_triple, + llvm_filecheck = llvm_filecheck, + ), + ) + .map_err(|error| format!("Failed to write into `rust/config.toml`: {:?}", error))?; + + let rustc_commit = match rustc_version_info(env.get("RUSTC").map(|s| s.as_str()))?.commit_hash { + Some(commit_hash) => commit_hash, + None => return Err("Couldn't retrieve rustc commit hash".to_string()), + }; + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("commit: {:?}", rustc_commit); + let command: &[&dyn AsRef] = &[&"git", &"checkout", &rustc_commit, &"tests"]; + run_command_with_output_and_env(command, rust_dir, Some(env))?; + Ok(()) +} + +fn asm_tests(env: &Env, args: &TestArg) -> Result<(), String> { + let mut env = env.clone(); + setup_rustc(&mut env, args)?; + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] rustc asm test suite"); + + env.insert("COMPILETEST_FORCE_STAGE0".to_string(), "1".to_string()); + + run_command_with_env( + &[ + &"./x.py", + &"test", + &"--run", + &"always", + &"--stage", + &"0", + &"tests/assembly/asm", + &"--rustc-args", + &format!( + r#"-Zpanic-abort-tests -Csymbol-mangling-version=v0 \ + -Zcodegen-backend="{pwd}/target/{channel}/librustc_codegen_gcc.{dylib_ext}" \ + --sysroot "{pwd}/build_sysroot/sysroot" -Cpanic=abort"#, + pwd = std::env::current_dir() + .map_err(|error| format!("`current_dir` failed: {:?}", error))? + .display(), + channel = args.channel.as_str(), + dylib_ext = args.config_info.dylib_ext, + ) + .as_str(), + ], + Some(Path::new("rust")), + Some(&env), + )?; + Ok(()) +} + +fn run_cargo_command( + command: &[&dyn AsRef], + cwd: Option<&Path>, + env: &Env, + args: &TestArg, +) -> Result<(), String> { + run_cargo_command_with_callback(command, cwd, env, args, |cargo_command, cwd, env| { + run_command_with_output_and_env(&cargo_command, cwd, Some(env))?; + Ok(()) + }) +} + +fn run_cargo_command_with_callback( + command: &[&dyn AsRef], + cwd: Option<&Path>, + env: &Env, + args: &TestArg, + callback: F, +) -> Result<(), String> +where + F: Fn(&[&dyn AsRef], Option<&Path>, &Env) -> Result<(), String>, +{ + let toolchain = get_toolchain()?; + let rustc_version = String::from_utf8( + run_command_with_env(&[&args.config_info.rustc_command[0], &"-V"], cwd, Some(env))?.stdout, + ) + .map_err(|error| format!("Failed to retrieve rustc version: {:?}", error))?; + let rustc_toolchain_version = String::from_utf8( + run_command_with_env( + &[ + &args.config_info.rustc_command[0], + &format!("+{}", toolchain), + &"-V", + ], + cwd, + Some(env), + )? + .stdout, + ) + .map_err(|error| format!("Failed to retrieve rustc +toolchain version: {:?}", error))?; + + if rustc_version != rustc_toolchain_version { + eprintln!( + "rustc_codegen_gcc is built for `{}` but the default rustc version is `{}`.", + rustc_toolchain_version, rustc_version, + ); + eprintln!("Using `{}`.", rustc_toolchain_version); + } + let mut cargo_command: Vec<&dyn AsRef> = vec![&"cargo", &toolchain]; + cargo_command.extend_from_slice(&command); + callback(&cargo_command, cwd, env) +} + +// FIXME(antoyo): linker gives multiple definitions error on Linux +// echo "[BUILD] sysroot in release mode" +// ./build_sysroot/build_sysroot.sh --release + +fn test_libcore(env: &Env, args: &TestArg) -> Result<(), String> { + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] libcore"); + let path = Path::new("build_sysroot/sysroot_src/library/core/tests"); + let _ = remove_dir_all(path.join("target")); + run_cargo_command(&[&"test"], Some(path), env, args)?; + Ok(()) +} + +// echo "[BENCH COMPILE] mod_bench" +// +// COMPILE_MOD_BENCH_INLINE="$RUSTC example/mod_bench.rs --crate-type bin -Zmir-opt-level=3 -O --crate-name mod_bench_inline" +// COMPILE_MOD_BENCH_LLVM_0="rustc example/mod_bench.rs --crate-type bin -Copt-level=0 -o $cargo_target_dir/mod_bench_llvm_0 -Cpanic=abort" +// COMPILE_MOD_BENCH_LLVM_1="rustc example/mod_bench.rs --crate-type bin -Copt-level=1 -o $cargo_target_dir/mod_bench_llvm_1 -Cpanic=abort" +// COMPILE_MOD_BENCH_LLVM_2="rustc example/mod_bench.rs --crate-type bin -Copt-level=2 -o $cargo_target_dir/mod_bench_llvm_2 -Cpanic=abort" +// COMPILE_MOD_BENCH_LLVM_3="rustc example/mod_bench.rs --crate-type bin -Copt-level=3 -o $cargo_target_dir/mod_bench_llvm_3 -Cpanic=abort" +// +// Use 100 runs, because a single compilations doesn't take more than ~150ms, so it isn't very slow +// hyperfine --runs ${COMPILE_RUNS:-100} "$COMPILE_MOD_BENCH_INLINE" "$COMPILE_MOD_BENCH_LLVM_0" "$COMPILE_MOD_BENCH_LLVM_1" "$COMPILE_MOD_BENCH_LLVM_2" "$COMPILE_MOD_BENCH_LLVM_3" +// echo "[BENCH RUN] mod_bench" +// hyperfine --runs ${RUN_RUNS:-10} $cargo_target_dir/mod_bench{,_inline} $cargo_target_dir/mod_bench_llvm_* + +fn extended_rand_tests(env: &Env, args: &TestArg) -> Result<(), String> { + if args.no_default_features { + return Ok(()); + } + let path = Path::new("rand"); + run_cargo_command(&[&"clean"], Some(path), env, args)?; + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] rust-random/rand"); + run_cargo_command(&[&"test", &"--workspace"], Some(path), env, args)?; + Ok(()) +} + +fn extended_regex_example_tests(env: &Env, args: &TestArg) -> Result<(), String> { + if args.no_default_features { + return Ok(()); + } + let path = Path::new("regex"); + run_cargo_command(&[&"clean"], Some(path), env, args)?; + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] rust-lang/regex example shootout-regex-dna"); + let mut env = env.clone(); + // newer aho_corasick versions throw a deprecation warning + env.insert("CG_RUSTFLAGS".to_string(), "--cap-lints warn".to_string()); + // Make sure `[codegen mono items] start` doesn't poison the diff + run_cargo_command( + &[&"build", &"--example", &"shootout-regex-dna"], + Some(path), + &env, + args, + )?; + + run_cargo_command_with_callback( + &[&"run", &"--example", &"shootout-regex-dna"], + Some(path), + &env, + args, + |cargo_command, cwd, env| { + // FIXME: rewrite this with `child.stdin.write_all()` because + // `examples/regexdna-input.txt` is very small. + let mut command: Vec<&dyn AsRef> = vec![&"bash", &"-c"]; + let cargo_args = cargo_command + .iter() + .map(|s| s.as_ref().to_str().unwrap()) + .collect::>(); + let bash_command = format!( + "cat examples/regexdna-input.txt | {} | grep -v 'Spawned thread' > res.txt", + cargo_args.join(" "), + ); + command.push(&bash_command); + run_command_with_output_and_env(&command, cwd, Some(env))?; + run_command_with_output_and_env( + &[&"diff", &"-u", &"res.txt", &"examples/regexdna-output.txt"], + cwd, + Some(env), + )?; + Ok(()) + }, + )?; + + Ok(()) +} + +fn extended_regex_tests(env: &Env, args: &TestArg) -> Result<(), String> { + if args.no_default_features { + return Ok(()); + } + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] rust-lang/regex tests"); + let mut env = env.clone(); + env.insert("CG_RUSTFLAGS".to_string(), "--cap-lints warn".to_string()); + run_cargo_command( + &[ + &"test", + &"--tests", + &"--", + &"--exclude-should-panic", + &"--test-threads", + &"1", + &"-Zunstable-options", + &"-q", + ], + Some(Path::new("regex")), + &env, + args, + )?; + Ok(()) +} + +fn extended_sysroot_tests(env: &Env, args: &TestArg) -> Result<(), String> { + // pushd simple-raytracer + // echo "[BENCH COMPILE] ebobby/simple-raytracer" + // hyperfine --runs "${RUN_RUNS:-10}" --warmup 1 --prepare "cargo clean" \ + // "RUSTC=rustc RUSTFLAGS='' cargo build" \ + // "../cargo.sh build" + + // echo "[BENCH RUN] ebobby/simple-raytracer" + // cp ./target/debug/main ./raytracer_cg_gcc + // hyperfine --runs "${RUN_RUNS:-10}" ./raytracer_cg_llvm ./raytracer_cg_gcc + // popd + extended_rand_tests(env, args)?; + extended_regex_example_tests(env, args)?; + extended_regex_tests(env, args)?; + + Ok(()) +} + +fn should_remove_ui_test(content: &str) -> bool { + for line in content + .lines() + .map(|line| line.trim()) + .filter(|line| !line.is_empty()) + { + if [ + "// error-pattern:", + "// build-fail", + "// run-fail", + "-Cllvm-args", + "//~", + "// ~", + ] + .iter() + .any(|check| line.contains(check)) + { + return true; + } + } + false +} + +fn should_not_remove_test(file: &str) -> bool { + // contains //~ERROR, but shouldn't be removed + [ + "issues/auxiliary/issue-3136-a.rs", + "type-alias-impl-trait/auxiliary/cross_crate_ice.rs", + "type-alias-impl-trait/auxiliary/cross_crate_ice2.rs", + "macros/rfc-2011-nicer-assert-messages/auxiliary/common.rs", + "imports/ambiguous-1.rs", + "imports/ambiguous-4-extern.rs", + "entry-point/auxiliary/bad_main_functions.rs", + ] + .iter() + .any(|to_ignore| file.ends_with(to_ignore)) +} + +fn should_remove_test(path: &Path, path_str: &str) -> bool { + // Tests generating errors. + path.file_name() + .and_then(|name| name.to_str()) + .map(|name| name.contains("thread")) + .unwrap_or(false) + || [ + "consts/issue-miri-1910.rs", + // Tests generating errors. + "consts/issue-94675.rs", + // this test is oom-killed in the CI. + "mir/mir_heavy/issue-miri-1910.rs", + ] + .iter() + .any(|to_ignore| path_str.ends_with(to_ignore)) +} + +fn test_rustc_inner(env: &Env, args: &TestArg, callback: F) -> Result<(), String> +where + F: Fn() -> Result, +{ + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] rust-lang/rust"); + walk_dir( + "rust/tests/ui", + |dir| { + let dir_name = dir.file_name().and_then(|name| name.to_str()).unwrap_or(""); + if [ + "abi", + "extern", + "unsized-locals", + "proc-macro", + "threads-sendsync", + "borrowck", + "test-attrs", + ] + .iter() + .any(|name| *name == dir_name) + { + std::fs::remove_dir_all(dir).map_err(|error| { + format!("Failed to remove folder `{}`: {:?}", dir.display(), error) + })?; + } + Ok(()) + }, + |_| Ok(()), + )?; + + fn dir_handling(dir: &Path) -> Result<(), String> { + walk_dir(dir, dir_handling, file_handling) + } + fn file_handling(file: &Path) -> Result<(), String> { + let path_str = file.display().to_string().replace("\\", "/"); + if should_not_remove_test(&path_str) { + return Ok(()); + } else if should_remove_test(file, &path_str) { + return std::fs::remove_file(file) + .map_err(|error| format!("Failed to remove `{}`: {:?}", file.display(), error)); + } + let file_content = std::fs::read_to_string(file) + .map_err(|error| format!("Failed to read `{}`: {:?}", file.display(), error))?; + if should_remove_ui_test(&file_content) { + std::fs::remove_file(file) + .map_err(|error| format!("Failed to remove `{}`: {:?}", file.display(), error))?; + } + Ok(()) + } + + walk_dir("rust/tests/ui", dir_handling, file_handling)?; + let file = "rust/tests/ui/consts/const_cmp_type_id.rs"; + std::fs::remove_file(file) + .map_err(|error| format!("Failed to remove `{}`: {:?}", file, error))?; + let file = "rust/tests/ui/consts/issue-73976-monomorphic.rs"; + std::fs::remove_file(file) + .map_err(|error| format!("Failed to remove `{}`: {:?}", file, error))?; + + let mut env = env.clone(); + setup_rustc(&mut env, args)?; + if !callback()? { + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("Keeping all UI tests"); + } + + let nb_parts = args.nb_parts.unwrap_or(0); + if nb_parts > 0 { + let current_part = args.current_part.unwrap_or(0); + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!( + "Splitting ui_test into {} parts (and running part {})", + nb_parts, current_part + ); + let out = String::from_utf8( + run_command( + &[ + &"find", + &"tests/ui", + &"-type", + &"f", + &"-name", + &"*.rs", + &"-not", + &"-path", + &"*/auxiliary/*", + ], + Some(Path::new("rust")), + )? + .stdout, + ) + .map_err(|error| format!("Failed to retrieve output of find command: {:?}", error))?; + let mut files = out + .split('\n') + .map(|line| line.trim()) + .filter(|line| !line.is_empty()) + .collect::>(); + // To ensure it'll be always the same sub files, we sort the content. + files.sort(); + // We increment the number of tests by one because if this is an odd number, we would skip + // one test. + let count = files.len() / nb_parts + 1; + let start = nb_parts * count; + let end = start + count; + for (pos, path) in files.iter().enumerate() { + if pos >= start && pos <= end { + continue; + } + std::fs::remove_file(path) + .map_err(|error| format!("Failed to remove `{}`: {:?}", path, error))?; + } + } + + // FIXME: create a function "display_if_not_quiet" or something along the line. + println!("[TEST] rustc test suite"); + env.insert("COMPILETEST_FORCE_STAGE0".to_string(), "1".to_string()); + let rustc_args = env + .get("RUSTFLAGS") + .expect("RUSTFLAGS should not be empty at this stage"); + run_command_with_output_and_env( + &[ + &"./x.py", + &"test", + &"--run", + &"always", + &"--stage", + &"0", + &"tests/ui", + &"--rustc-args", + &rustc_args, + ], + Some(Path::new("rust")), + Some(&env), + )?; + Ok(()) +} + +fn test_rustc(env: &Env, args: &TestArg) -> Result<(), String> { + test_rustc_inner(env, args, || Ok(false)) +} + +fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> { + test_rustc_inner(env, args, || { + // Removing all tests. + run_command( + &[ + &"find", + &"tests/ui", + &"-type", + &"f", + &"-name", + &"*.rs", + &"-not", + &"-path", + &"*/auxiliary/*", + &"-delete", + ], + Some(Path::new("rust")), + )?; + // Putting back only the failing ones. + run_command( + &[ + &"xargs", + &"-a", + &"../failing-ui-tests.txt", + &"-d'\n'", + &"git", + &"checkout", + &"--", + ], + Some(Path::new("rust")), + )?; + Ok(true) + }) +} + +fn test_successful_rustc(env: &Env, args: &TestArg) -> Result<(), String> { + test_rustc_inner(env, args, || { + // Removing the failing tests. + run_command( + &[ + &"xargs", + &"-a", + &"../failing-ui-tests.txt", + &"-d'\n'", + &"rm", + ], + Some(Path::new("rust")), + )?; + Ok(true) + }) +} + +fn clean_ui_tests(_env: &Env, _args: &TestArg) -> Result<(), String> { + run_command( + &[ + &"find", + &"rust/build/x86_64-unknown-linux-gnu/test/ui/", + &"-name", + &"stamp", + &"-delete", + ], + None, + )?; + Ok(()) +} + +fn run_all(env: &Env, args: &TestArg) -> Result<(), String> { + clean(env, args)?; + mini_tests(env, args)?; + build_sysroot(env, args)?; + std_tests(env, args)?; + // asm_tests(env, args)?; + test_libcore(env, args)?; + extended_sysroot_tests(env, args)?; + test_rustc(env, args)?; + Ok(()) +} + +pub fn run() -> Result<(), String> { + let mut args = match TestArg::new()? { + Some(args) => args, + None => return Ok(()), + }; + let mut env: HashMap = std::env::vars().collect(); + + env.insert("LD_LIBRARY_PATH".to_string(), args.gcc_path.clone()); + env.insert("LIBRARY_PATH".to_string(), args.gcc_path.clone()); + + build_if_no_backend(&env, &args)?; + if args.build_only { + println!("Since it's build only, exiting..."); + return Ok(()); + } + + let test_flags = split_args(env.get("TEST_FLAGS").unwrap_or(&String::new())); + args.config_info + .setup(&mut env, &test_flags, Some(&args.gcc_path))?; + + if args.runners.is_empty() { + run_all(&env, &args)?; + } else { + let runners = get_runners(); + for runner in args.runners.iter() { + runners.get(runner.as_str()).unwrap().1(&env, &args)?; + } + } + + Ok(()) } diff --git a/build_system/src/utils.rs b/build_system/src/utils.rs index 536f33a80293..ba1e040cb20f 100644 --- a/build_system/src/utils.rs +++ b/build_system/src/utils.rs @@ -143,17 +143,56 @@ pub fn get_os_name() -> Result { } } -pub fn get_rustc_host_triple() -> Result { - let output = run_command(&[&"rustc", &"-vV"], None)?; +#[derive(Default)] +pub struct RustcVersionInfo { + pub version: String, + pub host: Option, + pub commit_hash: Option, + pub commit_date: Option, +} + +pub fn rustc_version_info(rustc: Option<&str>) -> Result { + let output = run_command(&[&rustc.unwrap_or("rustc"), &"-vV"], None)?; let content = std::str::from_utf8(&output.stdout).unwrap_or(""); + let mut info = RustcVersionInfo::default(); + for line in content.split('\n').map(|line| line.trim()) { - if !line.starts_with("host:") { - continue; + match line.split_once(':') { + Some(("host", data)) => info.host = Some(data.trim().to_string()), + Some(("release", data)) => info.version = data.trim().to_string(), + Some(("commit-hash", data)) => info.commit_hash = Some(data.trim().to_string()), + Some(("commit-date", data)) => info.commit_date = Some(data.trim().to_string()), + _ => {} } - return Ok(line.split(':').nth(1).unwrap().trim().to_string()); } - Err("Cannot find host triple".to_string()) + if info.version.is_empty() { + Err("failed to retrieve rustc version".to_string()) + } else { + Ok(info) + } +} + +pub fn get_toolchain() -> Result { + let content = match fs::read_to_string("rust-toolchain") { + Ok(content) => content, + Err(_) => return Err("No `rust-toolchain` file found".to_string()), + }; + match content + .split('\n') + .map(|line| line.trim()) + .filter(|line| !line.is_empty()) + .filter_map(|line| { + if !line.starts_with("channel") { + return None; + } + line.split('"').skip(1).next() + }) + .next() + { + Some(toolchain) => Ok(toolchain.to_string()), + None => Err("Couldn't find `channel` in `rust-toolchain` file".to_string()), + } } pub fn get_gcc_path() -> Result { @@ -238,3 +277,56 @@ where } Ok(()) } + +pub fn split_args(args: &str) -> Vec { + let mut out = Vec::new(); + let mut start = 0; + let mut iter = args.char_indices().peekable(); + + while iter.peek().is_some() { + while let Some((pos, c)) = iter.next() { + if c == ' ' { + if pos != 0 { + out.push(args[start..pos].to_string()); + } + let mut found_start = false; + while let Some((pos, c)) = iter.peek() { + if *c != ' ' { + start = *pos; + found_start = true; + break; + } else { + iter.next(); + } + } + if !found_start { + return out; + } + } else if c == '"' || c == '\'' { + let end = c; + let mut found_end = false; + while let Some((_, c)) = iter.next() { + if c == end { + found_end = true; + break; + } else if c == '\\' { + // We skip the escaped character. + iter.next(); + } + } + if !found_end { + out.push(args[start..].to_string()); + return out; + } + } else if c == '\\' { + // We skip the escaped character. + iter.next(); + } + } + } + let s = args[start..].trim(); + if !s.is_empty() { + out.push(s.to_string()); + } + out +} From 84ca4f59c21425b02ba2042297187953d9aa283e Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 16 Nov 2023 22:51:51 +0100 Subject: [PATCH 03/34] Remove `test.sh`, `config.sh` and all calls and documentation pointing to it --- .github/workflows/ci.yml | 2 +- .github/workflows/failures.yml | 2 +- .github/workflows/gcc12.yml | 2 +- .github/workflows/m68k.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/stdarch.yml | 2 +- Readme.md | 4 +- config.sh | 85 ------ test.sh | 479 --------------------------------- 9 files changed, 8 insertions(+), 572 deletions(-) delete mode 100644 config.sh delete mode 100755 test.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 308bc55ead71..8e361bf617b1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -124,7 +124,7 @@ jobs: - name: Run tests run: | # TODO: remove --features master when it is back to the default. - ./test.sh --features master --release --clean --build-sysroot ${{ matrix.commands }} + ./y.sh test --features master --release --clean --build-sysroot ${{ matrix.commands }} duplicates: runs-on: ubuntu-latest diff --git a/.github/workflows/failures.yml b/.github/workflows/failures.yml index ae8de79b773d..b411b9a17846 100644 --- a/.github/workflows/failures.yml +++ b/.github/workflows/failures.yml @@ -128,5 +128,5 @@ jobs: - name: Run tests id: tests run: | - ${{ matrix.libgccjit_version.env_extra }} ./test.sh --release --clean --build-sysroot --test-failing-rustc ${{ matrix.libgccjit_version.extra }} | tee output_log + ${{ matrix.libgccjit_version.env_extra }} ./y.sh test --release --clean --build-sysroot --test-failing-rustc ${{ matrix.libgccjit_version.extra }} | tee output_log rg --text "test result" output_log >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/gcc12.yml b/.github/workflows/gcc12.yml index a0d363cf1fbd..1a17b936c743 100644 --- a/.github/workflows/gcc12.yml +++ b/.github/workflows/gcc12.yml @@ -112,4 +112,4 @@ jobs: - name: Run tests run: | - ./test.sh --release --clean --build-sysroot ${{ matrix.commands }} --no-default-features + ./y.sh test --release --clean --build-sysroot ${{ matrix.commands }} --no-default-features diff --git a/.github/workflows/m68k.yml b/.github/workflows/m68k.yml index 4d9d7e23dc2b..ac141e062479 100644 --- a/.github/workflows/m68k.yml +++ b/.github/workflows/m68k.yml @@ -139,4 +139,4 @@ jobs: - name: Run tests run: | # TODO: remove --features master when it is back to the default. - ./test.sh --release --features master --clean --build-sysroot ${{ matrix.commands }} + ./y.sh test --release --features master --clean --build-sysroot ${{ matrix.commands }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 43b90fcec933..9798bc338f3d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -105,4 +105,4 @@ jobs: - name: Run tests run: | # TODO: remove --features master when it is back to the default. - EMBED_LTO_BITCODE=1 ./test.sh --release --clean --release-sysroot --build-sysroot ${{ matrix.commands }} --features master + EMBED_LTO_BITCODE=1 ./y.sh test --release --clean --release-sysroot --build-sysroot ${{ matrix.commands }} --features master diff --git a/.github/workflows/stdarch.yml b/.github/workflows/stdarch.yml index 42109ba3e024..d290f1d05628 100644 --- a/.github/workflows/stdarch.yml +++ b/.github/workflows/stdarch.yml @@ -120,7 +120,7 @@ jobs: if: ${{ !matrix.cargo_runner }} run: | # TODO: remove `--features master` when it is back to the default. - ./test.sh --release --clean --release-sysroot --build-sysroot --mini-tests --std-tests --test-libcore --features master + ./y.sh test --release --clean --release-sysroot --build-sysroot --mini-tests --std-tests --test-libcore --features master - name: Run stdarch tests if: ${{ !matrix.cargo_runner }} diff --git a/Readme.md b/Readme.md index 9db1dec10323..68effb2bf78f 100644 --- a/Readme.md +++ b/Readme.md @@ -65,7 +65,7 @@ $ LIBRARY_PATH=$(cat gcc_path) LD_LIBRARY_PATH=$(cat gcc_path) ./y.sh build --re To run the tests: ```bash -$ ./test.sh --release --features master +$ ./y.sh test --release --features master ``` ## Usage @@ -82,7 +82,7 @@ export CG_GCCJIT_DIR=[the full path to rustc_codegen_gcc] $ CHANNEL="release" $CG_GCCJIT_DIR/cargo.sh run ``` -If you compiled cg_gccjit in debug mode (aka you didn't pass `--release` to `./test.sh`) you should use `CHANNEL="debug"` instead or omit `CHANNEL="release"` completely. +If you compiled cg_gccjit in debug mode (aka you didn't pass `--release` to `./y.sh test`) you should use `CHANNEL="debug"` instead or omit `CHANNEL="release"` completely. ### LTO diff --git a/config.sh b/config.sh deleted file mode 100644 index 7ae2175d41de..000000000000 --- a/config.sh +++ /dev/null @@ -1,85 +0,0 @@ -set -e - -export CARGO_INCREMENTAL=0 - -if [ -f ./gcc_path ]; then - export GCC_PATH=$(cat gcc_path) -elif (( $use_system_gcc == 1 )); then - echo 'Using system GCC' -else - echo 'Please put the path to your custom build of libgccjit in the file `gcc_path`, see Readme.md for details' - exit 1 -fi - -if [[ -z "$RUSTC" ]]; then - export RUSTC="rustc" -fi - -unamestr=`uname` -if [[ "$unamestr" == 'Linux' ]]; then - dylib_ext='so' -elif [[ "$unamestr" == 'Darwin' ]]; then - dylib_ext='dylib' -else - echo "Unsupported os" - exit 1 -fi - -HOST_TRIPLE=$($RUSTC -vV | grep host | cut -d: -f2 | tr -d " ") -# TODO: remove $OVERWRITE_TARGET_TRIPLE when config.sh is removed. -TARGET_TRIPLE="${OVERWRITE_TARGET_TRIPLE:-$HOST_TRIPLE}" - -linker='' -RUN_WRAPPER='' -if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then - RUN_WRAPPER=run_in_vm - if [[ "$TARGET_TRIPLE" == "m68k-unknown-linux-gnu" ]]; then - linker='-Clinker=m68k-unknown-linux-gnu-gcc' - elif [[ "$TARGET_TRIPLE" == "aarch64-unknown-linux-gnu" ]]; then - # We are cross-compiling for aarch64. Use the correct linker and run tests in qemu. - linker='-Clinker=aarch64-linux-gnu-gcc' - else - echo "Unknown non-native platform" - fi -fi - -# Since we don't support ThinLTO, disable LTO completely when not trying to do LTO. -# TODO(antoyo): remove when we can handle ThinLTO. -disable_lto_flags='' -if [[ ! -v FAT_LTO ]]; then - disable_lto_flags='-Clto=off' -fi - -if [[ -z "$BUILTIN_BACKEND" ]]; then - export RUSTFLAGS="$CG_RUSTFLAGS $linker -Csymbol-mangling-version=v0 -Cdebuginfo=2 $disable_lto_flags -Zcodegen-backend=$(pwd)/target/${CHANNEL:-debug}/librustc_codegen_gcc.$dylib_ext --sysroot $(pwd)/build_sysroot/sysroot $TEST_FLAGS" -else - export RUSTFLAGS="$CG_RUSTFLAGS $linker -Csymbol-mangling-version=v0 -Cdebuginfo=2 $disable_lto_flags -Zcodegen-backend=gcc $TEST_FLAGS -Cpanic=abort" - - if [[ ! -z "$RUSTC_SYSROOT" ]]; then - export RUSTFLAGS="$RUSTFLAGS --sysroot $RUSTC_SYSROOT" - fi -fi - -# FIXME(antoyo): remove once the atomic shim is gone -if [[ unamestr == 'Darwin' ]]; then - export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup" -fi - -if [[ -z "$cargo_target_dir" ]]; then - RUST_CMD="$RUSTC $RUSTFLAGS -L crate=target/out --out-dir target/out" - cargo_target_dir="target/out" -else - RUST_CMD="$RUSTC $RUSTFLAGS -L crate=$cargo_target_dir --out-dir $cargo_target_dir" -fi -export RUSTC_LOG=warn # display metadata load errors - -export LD_LIBRARY_PATH="$(pwd)/target/out:$(pwd)/build_sysroot/sysroot/lib/rustlib/$TARGET_TRIPLE/lib" -if [[ ! -z "$:$GCC_PATH" ]]; then - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GCC_PATH" -fi - -export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH -# NOTE: To avoid the -fno-inline errors, use /opt/gcc/bin/gcc instead of cc. -# To do so, add a symlink for cc to /opt/gcc/bin/gcc in our PATH. -# Another option would be to add the following Rust flag: -Clinker=/opt/gcc/bin/gcc -export PATH="/opt/gcc/bin:/opt/m68k-unknown-linux-gnu/bin:$PATH" diff --git a/test.sh b/test.sh deleted file mode 100755 index e896237a1ea4..000000000000 --- a/test.sh +++ /dev/null @@ -1,479 +0,0 @@ -#!/usr/bin/env bash - -# TODO(antoyo): rewrite to cargo-make (or just) or something like that to only rebuild the sysroot when needed? - -set -e -#set -x - -flags= -gcc_master_branch=1 -channel="debug" -funcs=() -build_only=0 -nb_parts=0 -current_part=0 -use_system_gcc=0 -use_backend=0 -cargo_target_dir="" - -export CHANNEL='debug' - -while [[ $# -gt 0 ]]; do - case $1 in - --release) - codegen_channel=release - channel="release" - export CHANNEL='release' - shift - ;; - --release-sysroot) - sysroot_channel="--release" - shift - ;; - --no-default-features) - gcc_master_branch=0 - flags="$flags --no-default-features" - shift - ;; - --features) - shift - flags="$flags --features $1" - shift - ;; - "--test-rustc") - funcs+=(test_rustc) - shift - ;; - "--test-successful-rustc") - funcs+=(test_successful_rustc) - shift - ;; - "--test-failing-rustc") - funcs+=(test_failing_rustc) - shift - ;; - - "--test-libcore") - funcs+=(test_libcore) - shift - ;; - - "--clean-ui-tests") - funcs+=(clean_ui_tests) - shift - ;; - "--clean") - funcs+=(clean) - shift - ;; - - "--std-tests") - funcs+=(std_tests) - shift - ;; - - "--asm-tests") - funcs+=(asm_tests) - shift - ;; - - "--extended-tests") - funcs+=(extended_sysroot_tests) - shift - ;; - "--extended-rand-tests") - funcs+=(extended_rand_tests) - shift - ;; - "--extended-regex-example-tests") - funcs+=(extended_regex_example_tests) - shift - ;; - "--extended-regex-tests") - funcs+=(extended_regex_tests) - shift - ;; - - "--mini-tests") - funcs+=(mini_tests) - shift - ;; - - "--build-sysroot") - funcs+=(build_sysroot) - shift - ;; - "--build") - build_only=1 - shift - ;; - "--use-system-gcc") - use_system_gcc=1 - shift - ;; - "--use-backend") - use_backend=1 - shift - export BUILTIN_BACKEND=$1 - shift - ;; - "--out-dir") - shift - export CARGO_TARGET_DIR=$1 - cargo_target_dir=$1 - shift - ;; - "--nb-parts") - shift - nb_parts=$1 - shift - ;; - "--current-part") - shift - current_part=$1 - shift - ;; - *) - echo "Unknown option $1" - exit 1 - ;; - esac -done - -if [ -f ./gcc_path ]; then - export GCC_PATH=$(cat gcc_path) -elif (( $use_system_gcc == 1 )); then - echo 'Using system GCC' -else - echo 'Please put the path to your custom build of libgccjit in the file `gcc_path`, see Readme.md for details' - exit 1 -fi - -export LD_LIBRARY_PATH="$GCC_PATH" -export LIBRARY_PATH="$GCC_PATH" - -if [[ $use_backend == 0 ]]; then - if [[ $channel == "release" ]]; then - CARGO_INCREMENTAL=1 cargo rustc --release $flags - else - echo $LD_LIBRARY_PATH - cargo rustc $flags - fi -fi - -if (( $build_only == 1 )); then - echo "Since it's 'build-only', exiting..." - exit -fi - -source config.sh - -function clean() { - rm -r $cargo_target_dir || true - mkdir -p $cargo_target_dir/gccjit -} - -function mini_tests() { - echo "[BUILD] mini_core" - crate_types="lib,dylib" - - if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then - crate_types="lib" - fi - - $RUST_CMD example/mini_core.rs --crate-name mini_core --crate-type $crate_types --target $TARGET_TRIPLE - - echo "[BUILD] example" - $RUST_CMD example/example.rs --crate-type lib --target $TARGET_TRIPLE - - echo "[AOT] mini_core_hello_world" - $RUST_CMD example/mini_core_hello_world.rs --crate-name mini_core_hello_world --crate-type bin -g --target $TARGET_TRIPLE - $RUN_WRAPPER $cargo_target_dir/mini_core_hello_world abc bcd -} - -function build_sysroot() { - echo "[BUILD] sysroot" - time ./build_sysroot/build_sysroot.sh $sysroot_channel -} - -# TODO(GuillaumeGomez): when rewriting in Rust, refactor with the code in tests/lang_tests_common.rs if possible. -function run_in_vm() { - vm_parent_dir=${CG_GCC_VM_DIR:-$(pwd)} - vm_dir=vm - exe=$1 - exe_filename=$(basename $exe) - vm_home_dir=$vm_parent_dir/$vm_dir/home - vm_exe_path=$vm_home_dir/$exe_filename - inside_vm_exe_path=/home/$exe_filename - sudo cp $exe $vm_exe_path - - shift - pushd $vm_parent_dir - sudo chroot $vm_dir qemu-m68k-static $inside_vm_exe_path $@ - popd -} - -function std_tests() { - echo "[AOT] arbitrary_self_types_pointers_and_wrappers" - $RUST_CMD example/arbitrary_self_types_pointers_and_wrappers.rs --crate-name arbitrary_self_types_pointers_and_wrappers --crate-type bin --target $TARGET_TRIPLE - $RUN_WRAPPER $cargo_target_dir/arbitrary_self_types_pointers_and_wrappers - - echo "[AOT] alloc_system" - $RUST_CMD example/alloc_system.rs --crate-type lib --target "$TARGET_TRIPLE" - - # FIXME: doesn't work on m68k. - if [[ "$HOST_TRIPLE" == "$TARGET_TRIPLE" ]]; then - echo "[AOT] alloc_example" - $RUST_CMD example/alloc_example.rs --crate-type bin --target $TARGET_TRIPLE - $RUN_WRAPPER $cargo_target_dir/alloc_example - fi - - echo "[AOT] dst_field_align" - # FIXME(antoyo): Re-add -Zmir-opt-level=2 once rust-lang/rust#67529 is fixed. - $RUST_CMD example/dst-field-align.rs --crate-name dst_field_align --crate-type bin --target $TARGET_TRIPLE - $RUN_WRAPPER $cargo_target_dir/dst_field_align || (echo $?; false) - - echo "[AOT] std_example" - std_flags="--cfg feature=\"master\"" - if (( $gcc_master_branch == 0 )); then - std_flags="" - fi - $RUST_CMD example/std_example.rs --crate-type bin --target $TARGET_TRIPLE $std_flags - $RUN_WRAPPER $cargo_target_dir/std_example --target $TARGET_TRIPLE - - echo "[AOT] subslice-patterns-const-eval" - $RUST_CMD example/subslice-patterns-const-eval.rs --crate-type bin $TEST_FLAGS --target $TARGET_TRIPLE - $RUN_WRAPPER $cargo_target_dir/subslice-patterns-const-eval - - echo "[AOT] track-caller-attribute" - $RUST_CMD example/track-caller-attribute.rs --crate-type bin $TEST_FLAGS --target $TARGET_TRIPLE - $RUN_WRAPPER $cargo_target_dir/track-caller-attribute - - echo "[BUILD] mod_bench" - $RUST_CMD example/mod_bench.rs --crate-type bin --target $TARGET_TRIPLE -} - -function setup_rustc() { - rust_toolchain=$(cat rust-toolchain | grep channel | sed 's/channel = "\(.*\)"/\1/') - - git clone https://github.com/rust-lang/rust.git || true - cd rust - git fetch - git checkout $($RUSTC -V | cut -d' ' -f3 | tr -d '(') - export RUSTFLAGS= - - rm config.toml || true - - cat > config.toml < res.txt - diff -u res.txt examples/regexdna-output.txt - popd -} - -function extended_regex_tests() { - if (( $gcc_master_branch == 0 )); then - return - fi - - pushd regex - echo "[TEST] rust-lang/regex tests" - export CG_RUSTFLAGS="--cap-lints warn" # newer aho_corasick versions throw a deprecation warning - ../cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q - popd -} - -function extended_sysroot_tests() { - #pushd simple-raytracer - #echo "[BENCH COMPILE] ebobby/simple-raytracer" - #hyperfine --runs "${RUN_RUNS:-10}" --warmup 1 --prepare "cargo clean" \ - #"RUSTC=rustc RUSTFLAGS='' cargo build" \ - #"../cargo.sh build" - - #echo "[BENCH RUN] ebobby/simple-raytracer" - #cp ./target/debug/main ./raytracer_cg_gcc - #hyperfine --runs "${RUN_RUNS:-10}" ./raytracer_cg_llvm ./raytracer_cg_gcc - #popd - - extended_rand_tests - extended_regex_example_tests - extended_regex_tests -} - -function test_rustc() { - echo - echo "[TEST] rust-lang/rust" - - setup_rustc - - for test in $(rg -i --files-with-matches "//(\[\w+\])?~|// error-pattern:|// build-fail|// run-fail|-Cllvm-args" tests/ui); do - rm $test - done - rm tests/ui/consts/const_cmp_type_id.rs - rm tests/ui/consts/issue-73976-monomorphic.rs - - git checkout -- tests/ui/issues/auxiliary/issue-3136-a.rs # contains //~ERROR, but shouldn't be removed - - rm -r tests/ui/{abi*,extern/,unsized-locals/,proc-macro/,threads-sendsync/,borrowck/,test*,consts/issue-miri-1910.rs} || true - rm tests/ui/mir/mir_heavy_promoted.rs # this test is oom-killed in the CI. - # Tests generating errors. - rm tests/ui/consts/issue-94675.rs - for test in $(rg --files-with-matches "thread" tests/ui); do - rm $test - done - git checkout tests/ui/type-alias-impl-trait/auxiliary/cross_crate_ice.rs - git checkout tests/ui/type-alias-impl-trait/auxiliary/cross_crate_ice2.rs - git checkout tests/ui/macros/rfc-2011-nicer-assert-messages/auxiliary/common.rs - git checkout tests/ui/imports/ambiguous-1.rs - git checkout tests/ui/imports/ambiguous-4-extern.rs - git checkout tests/ui/entry-point/auxiliary/bad_main_functions.rs - - RUSTC_ARGS="$TEST_FLAGS -Csymbol-mangling-version=v0 -Zcodegen-backend="$(pwd)"/../target/"$CHANNEL"/librustc_codegen_gcc."$dylib_ext" --sysroot "$(pwd)"/../build_sysroot/sysroot" - - if [ $# -eq 0 ]; then - # No argument supplied to the function. Doing nothing. - echo "No argument provided. Keeping all UI tests" - elif [ $1 = "0" ]; then - # Removing the failing tests. - xargs -a ../failing-ui-tests.txt -d'\n' rm - else - # Removing all tests. - find tests/ui -type f -name '*.rs' -not -path '*/auxiliary/*' -delete - # Putting back only the failing ones. - xargs -a ../failing-ui-tests.txt -d'\n' git checkout -- - fi - - if [ $nb_parts -gt 0 ]; then - echo "Splitting ui_test into $nb_parts parts (and running part $current_part)" - find tests/ui -type f -name '*.rs' -not -path "*/auxiliary/*" > ui_tests - # To ensure it'll be always the same sub files, we sort the content. - sort ui_tests -o ui_tests - count=$((`wc -l < ui_tests` / $nb_parts)) - # We increment the number of tests by one because if this is an odd number, we would skip - # one test. - count=$((count + 1)) - split -d -l $count -a 1 ui_tests ui_tests.split - # Removing all tests. - find tests/ui -type f -name '*.rs' -not -path "*/auxiliary/*" -delete - # Putting back only the ones we want to test. - xargs -a "ui_tests.split$current_part" -d'\n' git checkout -- - fi - - echo "[TEST] rustc test suite" - COMPILETEST_FORCE_STAGE0=1 ./x.py test --run always --stage 0 tests/ui/ --rustc-args "$RUSTC_ARGS" # --target $TARGET_TRIPLE -} - -function test_failing_rustc() { - test_rustc "1" -} - -function test_successful_rustc() { - test_rustc "0" -} - -function clean_ui_tests() { - find rust/build/x86_64-unknown-linux-gnu/test/ui/ -name stamp -delete -} - -function all() { - clean - mini_tests - build_sysroot - std_tests - #asm_tests - test_libcore - extended_sysroot_tests - test_rustc -} - -if [ ${#funcs[@]} -eq 0 ]; then - echo "No command passed, running '--all'..." - all -else - for t in ${funcs[@]}; do - $t - done -fi From d3e14a49c9710a93ebd315b1b54a596496531de7 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 21 Nov 2023 16:29:32 +0100 Subject: [PATCH 04/34] Display stdout and stderr if a command failed to run --- build_system/src/build.rs | 13 ++++++++--- build_system/src/main.rs | 2 +- build_system/src/utils.rs | 47 ++++++++++++++++++++++++++------------- 3 files changed, 42 insertions(+), 20 deletions(-) diff --git a/build_system/src/build.rs b/build_system/src/build.rs index 6390458d4fd0..43fa442bf5b2 100644 --- a/build_system/src/build.rs +++ b/build_system/src/build.rs @@ -189,9 +189,16 @@ fn build_sysroot_inner( // Copy the source files to the sysroot (Rust for Linux needs this). let sysroot_src_path = "sysroot/lib/rustlib/src/rust"; - fs::create_dir_all(&sysroot_src_path) - .map_err(|error| format!("Failed to create directory `{}`: {:?}", sysroot_src_path, error))?; - run_command(&[&"cp", &"-r", &"sysroot_src/library/", &sysroot_src_path], None)?; + fs::create_dir_all(&sysroot_src_path).map_err(|error| { + format!( + "Failed to create directory `{}`: {:?}", + sysroot_src_path, error + ) + })?; + run_command( + &[&"cp", &"-r", &"sysroot_src/library/", &sysroot_src_path], + None, + )?; Ok(()) } diff --git a/build_system/src/main.rs b/build_system/src/main.rs index bff82b6e3e57..e0091ff69773 100644 --- a/build_system/src/main.rs +++ b/build_system/src/main.rs @@ -61,7 +61,7 @@ fn main() { Command::Build => build::run(), Command::Test => test::run(), } { - eprintln!("Command failed to run: {e:?}"); + eprintln!("Command failed to run: {e}"); process::exit(1); } } diff --git a/build_system/src/utils.rs b/build_system/src/utils.rs index ba1e040cb20f..6dfc6a6506aa 100644 --- a/build_system/src/utils.rs +++ b/build_system/src/utils.rs @@ -29,22 +29,37 @@ fn check_exit_status( input: &[&dyn AsRef], cwd: Option<&Path>, exit_status: ExitStatus, + output: Option<&Output>, ) -> Result<(), String> { if exit_status.success() { - Ok(()) - } else { - Err(format!( - "Command `{}`{} exited with status {:?}", - input - .iter() - .map(|s| s.as_ref().to_str().unwrap()) - .collect::>() - .join(" "), - cwd.map(|cwd| format!(" (running in folder `{}`)", cwd.display())) - .unwrap_or_default(), - exit_status.code(), - )) + return Ok(()); } + let mut error = format!( + "Command `{}`{} exited with status {:?}", + input + .iter() + .map(|s| s.as_ref().to_str().unwrap()) + .collect::>() + .join(" "), + cwd.map(|cwd| format!(" (running in folder `{}`)", cwd.display())) + .unwrap_or_default(), + exit_status.code() + ); + if let Some(output) = output { + unsafe { + let stdout = std::str::from_utf8_unchecked(&output.stdout); + if !stdout.is_empty() { + error.push_str("\n==== STDOUT ====\n"); + error.push_str(stdout); + } + let stderr = std::str::from_utf8_unchecked(&output.stderr); + if !stderr.is_empty() { + error.push_str("\n==== STDERR ====\n"); + error.push_str(stderr); + } + } + } + Err(error) } fn command_error(input: &[&dyn AsRef], cwd: &Option<&Path>, error: D) -> String { @@ -73,7 +88,7 @@ pub fn run_command_with_env( let output = get_command_inner(input, cwd, env) .output() .map_err(|e| command_error(input, &cwd, e))?; - check_exit_status(input, cwd, output.status)?; + check_exit_status(input, cwd, output.status, Some(&output))?; Ok(output) } @@ -86,7 +101,7 @@ pub fn run_command_with_output( .map_err(|e| command_error(input, &cwd, e))? .wait() .map_err(|e| command_error(input, &cwd, e))?; - check_exit_status(input, cwd, exit_status)?; + check_exit_status(input, cwd, exit_status, None)?; Ok(()) } @@ -100,7 +115,7 @@ pub fn run_command_with_output_and_env( .map_err(|e| command_error(input, &cwd, e))? .wait() .map_err(|e| command_error(input, &cwd, e))?; - check_exit_status(input, cwd, exit_status)?; + check_exit_status(input, cwd, exit_status, None)?; Ok(()) } From 8cc024c84dc76cb081bbaf8ae2eef4647ed8d78b Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 21 Nov 2023 18:38:30 +0100 Subject: [PATCH 05/34] Fix invalid path in `build_sysroot_inner` --- build_system/src/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_system/src/build.rs b/build_system/src/build.rs index 43fa442bf5b2..8f5c113fe319 100644 --- a/build_system/src/build.rs +++ b/build_system/src/build.rs @@ -196,7 +196,7 @@ fn build_sysroot_inner( ) })?; run_command( - &[&"cp", &"-r", &"sysroot_src/library/", &sysroot_src_path], + &[&"cp", &"-r", &start_dir.join("sysroot_src/library/"), &sysroot_src_path], None, )?; From 694a80d3724cdf7138df550d64fcaceb5d1404b0 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 21 Nov 2023 21:38:16 +0100 Subject: [PATCH 06/34] Add missing `--build-sysroot` option --- build_system/src/test.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/build_system/src/test.rs b/build_system/src/test.rs index fb2b24da9a23..2619fbc2f646 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -33,6 +33,7 @@ fn get_runners() -> Runners { runners.insert("--test-libcore", ("Run libcore tests", &test_libcore)); runners.insert("--clean-ui-tests", ("Clean ui tests", &clean_ui_tests)); runners.insert("--clean", ("Empty cargo target directory", &clean)); + runners.insert("--build-sysroot", ("Build sysroot", &build_sysroot)); runners.insert("--std-tests", ("Run std tests", &std_tests)); runners.insert("--asm-tests", ("Run asm tests", &asm_tests)); runners.insert( From c27fe3e0366313933daf6f9a6b3df578bde2682b Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 22 Nov 2023 15:17:48 +0100 Subject: [PATCH 07/34] Correctly handle channel in config --- build_system/src/config.rs | 4 +++- build_system/src/test.rs | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/build_system/src/config.rs b/build_system/src/config.rs index 763cac8edb66..8bab64f121ac 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -110,7 +110,9 @@ impl ConfigInfo { let current_dir = std_env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?; - let channel = if let Some(channel) = env.get("CHANNEL") { + let channel = if self.sysroot_release_channel { + "release" + } else if let Some(channel) = env.get("CHANNEL") { channel.as_str() } else { "debug" diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 2619fbc2f646..06a5c3157bb2 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -126,7 +126,6 @@ struct TestArg { build_only: bool, gcc_path: String, channel: Channel, - sysroot_channel: Channel, use_backend: bool, runners: BTreeSet, flags: Vec, @@ -148,8 +147,11 @@ impl TestArg { while let Some(arg) = args.next() { match arg.as_str() { - "--release" => test_arg.channel = Channel::Release, - "--release-sysroot" => test_arg.sysroot_channel = Channel::Release, + "--release" => { + test_arg.channel = Channel::Release; + test_arg.config_info.sysroot_release_channel = true; + } + "--release-sysroot" => test_arg.config_info.sysroot_release_channel = true, "--no-default-features" => { // To prevent adding it more than once. if !test_arg.no_default_features { From 9d104a0cbf0abbf5d2f5176298373510392e5a36 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 22 Nov 2023 17:01:05 +0100 Subject: [PATCH 08/34] Clone rust repository before modifying it --- build_system/src/test.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 06a5c3157bb2..7785957541f5 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -895,6 +895,9 @@ where { // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[TEST] rust-lang/rust"); + let mut env = env.clone(); + setup_rustc(&mut env, args)?; + walk_dir( "rust/tests/ui", |dir| { @@ -948,8 +951,6 @@ where std::fs::remove_file(file) .map_err(|error| format!("Failed to remove `{}`: {:?}", file, error))?; - let mut env = env.clone(); - setup_rustc(&mut env, args)?; if !callback()? { // FIXME: create a function "display_if_not_quiet" or something along the line. println!("Keeping all UI tests"); From 87c284c9bc05e290cb0ee577717a94de9e853c89 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 22 Nov 2023 17:20:14 +0100 Subject: [PATCH 09/34] Only read rust test files --- build_system/src/test.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 7785957541f5..bbe2322f93d3 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -928,7 +928,9 @@ where } fn file_handling(file: &Path) -> Result<(), String> { let path_str = file.display().to_string().replace("\\", "/"); - if should_not_remove_test(&path_str) { + if !path_str.ends_with(".rs") { + return Ok(()) + } else if should_not_remove_test(&path_str) { return Ok(()); } else if should_remove_test(file, &path_str) { return std::fs::remove_file(file) From 23c97b545dd7d8dbee80e491269f95308707c750 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 22 Nov 2023 17:35:49 +0100 Subject: [PATCH 10/34] Replace `xargs` command with pure Rust --- build_system/src/build.rs | 7 ++++- build_system/src/test.rs | 59 ++++++++++++++++++++++++--------------- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/build_system/src/build.rs b/build_system/src/build.rs index 8f5c113fe319..3087a5d79e01 100644 --- a/build_system/src/build.rs +++ b/build_system/src/build.rs @@ -196,7 +196,12 @@ fn build_sysroot_inner( ) })?; run_command( - &[&"cp", &"-r", &start_dir.join("sysroot_src/library/"), &sysroot_src_path], + &[ + &"cp", + &"-r", + &start_dir.join("sysroot_src/library/"), + &sysroot_src_path, + ], None, )?; diff --git a/build_system/src/test.rs b/build_system/src/test.rs index bbe2322f93d3..12927c5d13ca 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -929,7 +929,7 @@ where fn file_handling(file: &Path) -> Result<(), String> { let path_str = file.display().to_string().replace("\\", "/"); if !path_str.ends_with(".rs") { - return Ok(()) + return Ok(()); } else if should_not_remove_test(&path_str) { return Ok(()); } else if should_remove_test(file, &path_str) { @@ -1052,18 +1052,24 @@ fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> { Some(Path::new("rust")), )?; // Putting back only the failing ones. - run_command( - &[ - &"xargs", - &"-a", - &"../failing-ui-tests.txt", - &"-d'\n'", - &"git", - &"checkout", - &"--", - ], - Some(Path::new("rust")), - )?; + let path = "failing-ui-tests.txt"; + if let Ok(files) = std::fs::read_to_string(path) { + for file in files + .split('\n') + .map(|line| line.trim()) + .filter(|line| !line.is_empty()) + { + run_command( + &[&"git", &"checkout", &"--", &file], + Some(Path::new("rust")), + )?; + } + } else { + println!( + "Failed to read `{}`, not putting back failing ui tests", + path + ); + } Ok(true) }) } @@ -1071,16 +1077,23 @@ fn test_failing_rustc(env: &Env, args: &TestArg) -> Result<(), String> { fn test_successful_rustc(env: &Env, args: &TestArg) -> Result<(), String> { test_rustc_inner(env, args, || { // Removing the failing tests. - run_command( - &[ - &"xargs", - &"-a", - &"../failing-ui-tests.txt", - &"-d'\n'", - &"rm", - ], - Some(Path::new("rust")), - )?; + let path = "failing-ui-tests.txt"; + if let Ok(files) = std::fs::read_to_string(path) { + for file in files + .split('\n') + .map(|line| line.trim()) + .filter(|line| !line.is_empty()) + { + let path = Path::new("rust").join(file); + std::fs::remove_file(&path) + .map_err(|error| format!("failed to remove `{}`: {:?}", path.display(), error))?; + } + } else { + println!( + "Failed to read `{}`, not putting back failing ui tests", + path + ); + } Ok(true) }) } From 673661db8b35fbbdc71b91af9b122a7eaf7f7bf3 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 22 Nov 2023 17:54:58 +0100 Subject: [PATCH 11/34] Remove newline for llvm FileCheck binary path --- build_system/src/test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 12927c5d13ca..e175b62625cb 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -597,7 +597,7 @@ download-ci-llvm = false home = env.get("HOME").unwrap(), toolchain = toolchain, host_triple = args.config_info.host_triple, - llvm_filecheck = llvm_filecheck, + llvm_filecheck = llvm_filecheck.trim(), ), ) .map_err(|error| format!("Failed to write into `rust/config.toml`: {:?}", error))?; From d793f80bd4ceb85c032e18c1d0badac9a8664bba Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 23 Nov 2023 16:09:11 +0100 Subject: [PATCH 12/34] Correctly pass `cfg` option --- build_system/src/test.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/build_system/src/test.rs b/build_system/src/test.rs index e175b62625cb..614042b36c20 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -390,7 +390,7 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { &args.config_info.target_triple, ]); if !args.no_default_features { - command.push(&"--cfg feature=\"master\""); + command.extend_from_slice(&[&"--cfg", &"feature=\"master\""]); } run_command_with_env(&command, None, Some(env))?; @@ -454,7 +454,7 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { &args.config_info.target_triple, ]); if !args.no_default_features { - command.push(&"--cfg feature=\"master\""); + command.extend_from_slice(&[&"--cfg", &"feature=\"master\""]); } run_command_with_env(&command, None, Some(env))?; run_command_in_vm( @@ -1085,8 +1085,9 @@ fn test_successful_rustc(env: &Env, args: &TestArg) -> Result<(), String> { .filter(|line| !line.is_empty()) { let path = Path::new("rust").join(file); - std::fs::remove_file(&path) - .map_err(|error| format!("failed to remove `{}`: {:?}", path.display(), error))?; + std::fs::remove_file(&path).map_err(|error| { + format!("failed to remove `{}`: {:?}", path.display(), error) + })?; } } else { println!( From 4bed89f79bfd40fdfe615334c38e2427319db34f Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 23 Nov 2023 16:25:52 +0100 Subject: [PATCH 13/34] Correctly pass toolchain to cargo command --- build_system/src/test.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 614042b36c20..114aa6dc7207 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -672,17 +672,14 @@ where F: Fn(&[&dyn AsRef], Option<&Path>, &Env) -> Result<(), String>, { let toolchain = get_toolchain()?; + let toolchain_arg = format!("+{}", toolchain); let rustc_version = String::from_utf8( run_command_with_env(&[&args.config_info.rustc_command[0], &"-V"], cwd, Some(env))?.stdout, ) .map_err(|error| format!("Failed to retrieve rustc version: {:?}", error))?; let rustc_toolchain_version = String::from_utf8( run_command_with_env( - &[ - &args.config_info.rustc_command[0], - &format!("+{}", toolchain), - &"-V", - ], + &[&args.config_info.rustc_command[0], &toolchain_arg, &"-V"], cwd, Some(env), )? @@ -697,7 +694,7 @@ where ); eprintln!("Using `{}`.", rustc_toolchain_version); } - let mut cargo_command: Vec<&dyn AsRef> = vec![&"cargo", &toolchain]; + let mut cargo_command: Vec<&dyn AsRef> = vec![&"cargo", &toolchain_arg]; cargo_command.extend_from_slice(&command); callback(&cargo_command, cwd, env) } From 3c6bae7fa88d700a34b57f43e55749c510c57b72 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 23 Nov 2023 16:41:16 +0100 Subject: [PATCH 14/34] Use the correct folder when deleting rust UI tests --- build_system/src/test.rs | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 114aa6dc7207..16f01a1ba2de 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -942,13 +942,15 @@ where Ok(()) } - walk_dir("rust/tests/ui", dir_handling, file_handling)?; - let file = "rust/tests/ui/consts/const_cmp_type_id.rs"; - std::fs::remove_file(file) - .map_err(|error| format!("Failed to remove `{}`: {:?}", file, error))?; - let file = "rust/tests/ui/consts/issue-73976-monomorphic.rs"; - std::fs::remove_file(file) - .map_err(|error| format!("Failed to remove `{}`: {:?}", file, error))?; + let rust_path = Path::new("rust"); + + walk_dir(rust_path.join("tests/ui"), dir_handling, file_handling)?; + let file = rust_path.join("tests/ui/consts/const_cmp_type_id.rs"); + std::fs::remove_file(&file) + .map_err(|error| format!("Failed to remove `{}`: {:?}", file.display(), error))?; + let file = rust_path.join("tests/ui/consts/issue-73976-monomorphic.rs"); + std::fs::remove_file(&file) + .map_err(|error| format!("Failed to remove `{}`: {:?}", file.display(), error))?; if !callback()? { // FIXME: create a function "display_if_not_quiet" or something along the line. @@ -976,7 +978,7 @@ where &"-path", &"*/auxiliary/*", ], - Some(Path::new("rust")), + Some(rust_path), )? .stdout, ) @@ -997,8 +999,10 @@ where if pos >= start && pos <= end { continue; } - std::fs::remove_file(path) - .map_err(|error| format!("Failed to remove `{}`: {:?}", path, error))?; + let test_path = rust_path.join(path); + std::fs::remove_file(&test_path).map_err(|error| { + format!("Failed to remove `{}`: {:?}", test_path.display(), error) + })?; } } @@ -1020,7 +1024,7 @@ where &"--rustc-args", &rustc_args, ], - Some(Path::new("rust")), + Some(rust_path), Some(&env), )?; Ok(()) From 7013eccc052228044faa0103ca596ed75ea2a70e Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 23 Nov 2023 17:05:33 +0100 Subject: [PATCH 15/34] Add missing code comment --- build_system/src/test.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 16f01a1ba2de..7ab4767e405d 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -794,6 +794,7 @@ fn extended_regex_tests(env: &Env, args: &TestArg) -> Result<(), String> { // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[TEST] rust-lang/regex tests"); let mut env = env.clone(); + // newer aho_corasick versions throw a deprecation warning env.insert("CG_RUSTFLAGS".to_string(), "--cap-lints warn".to_string()); run_cargo_command( &[ From ad1d5417e705c8d915b718b8fc29e58c49b8defa Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 23 Nov 2023 17:11:07 +0100 Subject: [PATCH 16/34] Set RUSTDOCFLAGS environment variable in `run_cargo_command_with_callback` function --- build_system/src/test.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 7ab4767e405d..02e6309bd338 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -694,9 +694,12 @@ where ); eprintln!("Using `{}`.", rustc_toolchain_version); } + let mut env = env.clone(); + let rustflags = env.get("RUSTFLAGS").cloned().unwrap_or_default(); + env.insert("RUSTDOCFLAGS".to_string(), rustflags); let mut cargo_command: Vec<&dyn AsRef> = vec![&"cargo", &toolchain_arg]; cargo_command.extend_from_slice(&command); - callback(&cargo_command, cwd, env) + callback(&cargo_command, cwd, &env) } // FIXME(antoyo): linker gives multiple definitions error on Linux From 7d71b87691c22d54a1119a491647db63d28b55d3 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Thu, 23 Nov 2023 18:10:01 +0100 Subject: [PATCH 17/34] Correctly set `--cap-lints` when running regex tests --- build_system/src/config.rs | 2 ++ build_system/src/test.rs | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/build_system/src/config.rs b/build_system/src/config.rs index 8bab64f121ac..b31a728c6807 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -149,9 +149,11 @@ impl ConfigInfo { .extend_from_slice(&["--sysroot".to_string(), sysroot_path.display().to_string()]); }; + // This environment variable is useful in case we want to change options of rustc commands. if let Some(cg_rustflags) = env.get("CG_RUSTFLAGS") { rustflags.extend_from_slice(&split_args(&cg_rustflags)); } + if let Some(linker) = linker { rustflags.push(linker.to_string()); } diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 02e6309bd338..4e8c176db737 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -750,7 +750,8 @@ fn extended_regex_example_tests(env: &Env, args: &TestArg) -> Result<(), String> println!("[TEST] rust-lang/regex example shootout-regex-dna"); let mut env = env.clone(); // newer aho_corasick versions throw a deprecation warning - env.insert("CG_RUSTFLAGS".to_string(), "--cap-lints warn".to_string()); + let rustflags = format!("{} --cap-lints warn", env.get("RUSTFLAGS").cloned().unwrap_or_default()); + env.insert("RUSTFLAGS".to_string(), rustflags); // Make sure `[codegen mono items] start` doesn't poison the diff run_cargo_command( &[&"build", &"--example", &"shootout-regex-dna"], @@ -798,7 +799,8 @@ fn extended_regex_tests(env: &Env, args: &TestArg) -> Result<(), String> { println!("[TEST] rust-lang/regex tests"); let mut env = env.clone(); // newer aho_corasick versions throw a deprecation warning - env.insert("CG_RUSTFLAGS".to_string(), "--cap-lints warn".to_string()); + let rustflags = format!("{} --cap-lints warn", env.get("RUSTFLAGS").cloned().unwrap_or_default()); + env.insert("RUSTFLAGS".to_string(), rustflags); run_cargo_command( &[ &"test", From 970b2c770010c044abe5e3a8433fffc4ab3b8d52 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 24 Nov 2023 21:34:29 +0100 Subject: [PATCH 18/34] Fix `build_sysroot` by adding missing `RUSTFLAGS` environment variable --- build_system/src/build.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/build_system/src/build.rs b/build_system/src/build.rs index 3087a5d79e01..da29e87c33cb 100644 --- a/build_system/src/build.rs +++ b/build_system/src/build.rs @@ -141,8 +141,8 @@ fn build_sysroot_inner( rustflags.push_str(" -Cpanic=abort -Zpanic-abort-tests"); } rustflags.push_str(" -Z force-unstable-if-unmarked"); + let mut env = env.clone(); let channel = if sysroot_release_channel { - let mut env = env.clone(); env.insert( "RUSTFLAGS".to_string(), format!("{} -Zmir-opt-level=3", rustflags), @@ -160,10 +160,15 @@ fn build_sysroot_inner( )?; "release" } else { + env.insert( + "RUSTFLAGS".to_string(), + rustflags, + ); + run_command_with_output_and_env( &[&"cargo", &"build", &"--target", &config.target], Some(start_dir), - Some(env), + Some(&env), )?; "debug" }; From ff043162432a33fb7d2c675f05aef2803b39d387 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 24 Nov 2023 22:05:08 +0100 Subject: [PATCH 19/34] Remove `--target` option --- build_system/src/build.rs | 25 +++---------------------- build_system/src/config.rs | 12 ------------ build_system/src/test.rs | 10 ++++++++-- 3 files changed, 11 insertions(+), 36 deletions(-) diff --git a/build_system/src/build.rs b/build_system/src/build.rs index da29e87c33cb..189c393019fd 100644 --- a/build_system/src/build.rs +++ b/build_system/src/build.rs @@ -43,22 +43,6 @@ impl BuildArg { Self::usage(); return Ok(None); } - "--target-triple" => { - if args.next().is_some() { - // Handled in config.rs. - } else { - return Err( - "Expected a value after `--target-triple`, found nothing".to_string() - ); - } - } - "--target" => { - if args.next().is_some() { - // Handled in config.rs. - } else { - return Err("Expected a value after `--target`, found nothing".to_string()); - } - } arg => { if !build_arg.config_info.parse_argument(arg, &mut args)? { return Err(format!("Unknown argument `{}`", arg)); @@ -152,7 +136,7 @@ fn build_sysroot_inner( &"cargo", &"build", &"--target", - &config.target, + &config.target_triple, &"--release", ], Some(start_dir), @@ -160,13 +144,10 @@ fn build_sysroot_inner( )?; "release" } else { - env.insert( - "RUSTFLAGS".to_string(), - rustflags, - ); + env.insert("RUSTFLAGS".to_string(), rustflags); run_command_with_output_and_env( - &[&"cargo", &"build", &"--target", &config.target], + &[&"cargo", &"build", &"--target", &config.target_triple], Some(start_dir), Some(&env), )?; diff --git a/build_system/src/config.rs b/build_system/src/config.rs index b31a728c6807..267f45464423 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -4,7 +4,6 @@ use std::env as std_env; #[derive(Default)] pub struct ConfigInfo { - pub target: String, pub target_triple: String, pub host_triple: String, pub rustc_command: Vec, @@ -31,10 +30,6 @@ impl ConfigInfo { ) } }, - "--target" => match args.next() { - Some(arg) if !arg.is_empty() => self.target = arg.to_string(), - _ => return Err("Expected a value after `--target`, found nothing".to_string()), - }, "--out-dir" => match args.next() { Some(arg) if !arg.is_empty() => { // env.insert("CARGO_TARGET_DIR".to_string(), arg.to_string()); @@ -83,12 +78,6 @@ impl ConfigInfo { }; self.host_triple = rustc_version_info(Some(&rustc))?.host.unwrap_or_default(); - if !self.target_triple.is_empty() && self.target.is_empty() { - self.target = self.target_triple.clone(); - } - if self.target.is_empty() { - self.target = self.host_triple.clone(); - } if self.target_triple.is_empty() { self.target_triple = self.host_triple.clone(); } @@ -223,7 +212,6 @@ impl ConfigInfo { pub fn show_usage() { println!( "\ - --target [arg] : Set the target to [arg] --target-triple [arg] : Set the target triple to [arg] --out-dir : Location where the files will be generated --release-sysroot : Build sysroot in release mode diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 4e8c176db737..1e012798cba1 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -750,7 +750,10 @@ fn extended_regex_example_tests(env: &Env, args: &TestArg) -> Result<(), String> println!("[TEST] rust-lang/regex example shootout-regex-dna"); let mut env = env.clone(); // newer aho_corasick versions throw a deprecation warning - let rustflags = format!("{} --cap-lints warn", env.get("RUSTFLAGS").cloned().unwrap_or_default()); + let rustflags = format!( + "{} --cap-lints warn", + env.get("RUSTFLAGS").cloned().unwrap_or_default() + ); env.insert("RUSTFLAGS".to_string(), rustflags); // Make sure `[codegen mono items] start` doesn't poison the diff run_cargo_command( @@ -799,7 +802,10 @@ fn extended_regex_tests(env: &Env, args: &TestArg) -> Result<(), String> { println!("[TEST] rust-lang/regex tests"); let mut env = env.clone(); // newer aho_corasick versions throw a deprecation warning - let rustflags = format!("{} --cap-lints warn", env.get("RUSTFLAGS").cloned().unwrap_or_default()); + let rustflags = format!( + "{} --cap-lints warn", + env.get("RUSTFLAGS").cloned().unwrap_or_default() + ); env.insert("RUSTFLAGS".to_string(), rustflags); run_cargo_command( &[ From 53b2759bef791bad51bfcef022f2da432e6a4269 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 24 Nov 2023 22:23:08 +0100 Subject: [PATCH 20/34] Show command which failed --- build_system/src/utils.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build_system/src/utils.rs b/build_system/src/utils.rs index 6dfc6a6506aa..88fce2fcbce1 100644 --- a/build_system/src/utils.rs +++ b/build_system/src/utils.rs @@ -45,6 +45,8 @@ fn check_exit_status( .unwrap_or_default(), exit_status.code() ); + let input = input.iter().map(|i| i.as_ref()).collect::>(); + eprintln!("Command `{:?}` failed", input); if let Some(output) = output { unsafe { let stdout = std::str::from_utf8_unchecked(&output.stdout); From 2ec8d46dd1b6adc116a5efba46b7aad4a5315f86 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 24 Nov 2023 23:54:18 +0100 Subject: [PATCH 21/34] Correctly handle `OVERWRITE_TARGET_TRIPLE` env variable --- build_system/src/config.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build_system/src/config.rs b/build_system/src/config.rs index 267f45464423..d602cec9f9f5 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -78,6 +78,11 @@ impl ConfigInfo { }; self.host_triple = rustc_version_info(Some(&rustc))?.host.unwrap_or_default(); + if self.target_triple.is_empty() { + if let Some(overwrite) = env.get("OVERWRITE_TARGET_TRIPLE") { + self.target_triple = overwrite.clone(); + } + } if self.target_triple.is_empty() { self.target_triple = self.host_triple.clone(); } From 996635bad689a90ba90a8dbf887611f6fa287bab Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sat, 25 Nov 2023 00:04:08 +0100 Subject: [PATCH 22/34] Fix chroot command --- build_system/src/test.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 1e012798cba1..db3e4d9894d8 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -345,8 +345,13 @@ fn run_command_in_vm( let sudo_command: &[&dyn AsRef] = &[&"sudo", &"cp", &exe, &vm_exe_path]; run_command_with_env(sudo_command, None, Some(env))?; - let mut vm_command: Vec<&dyn AsRef> = - vec![&"sudo", &"chroot", &"qemu-m68k-static", &inside_vm_exe_path]; + let mut vm_command: Vec<&dyn AsRef> = vec![ + &"sudo", + &"chroot", + &vm_dir, + &"qemu-m68k-static", + &inside_vm_exe_path, + ]; vm_command.extend_from_slice(command); run_command_with_env(&vm_command, Some(&vm_parent_dir), Some(env))?; Ok(()) From ebb7aa0b8575a27e2768c42f1472d3523925357c Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Fri, 1 Dec 2023 23:57:16 +0100 Subject: [PATCH 23/34] Apply suggestions --- build_system/src/build.rs | 40 +------ build_system/src/config.rs | 25 ++-- build_system/src/prepare.rs | 7 +- build_system/src/test.rs | 227 ++++++++++++++++-------------------- build_system/src/utils.rs | 47 +++++--- 5 files changed, 156 insertions(+), 190 deletions(-) diff --git a/build_system/src/build.rs b/build_system/src/build.rs index 189c393019fd..618e74be2c0c 100644 --- a/build_system/src/build.rs +++ b/build_system/src/build.rs @@ -67,14 +67,8 @@ impl BuildArg { } } -fn build_sysroot_inner( - env: &HashMap, - sysroot_panic_abort: bool, - sysroot_release_channel: bool, - config: &ConfigInfo, - start_dir: Option<&Path>, -) -> Result<(), String> { - let start_dir = start_dir.unwrap_or_else(|| Path::new(".")); +pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Result<(), String> { + let start_dir = Path::new("build_sysroot"); // Cleanup for previous run // Clean target dir except for build scripts and incremental cache let _ = walk_dir( @@ -121,12 +115,11 @@ fn build_sysroot_inner( // Builds libs let mut rustflags = env.get("RUSTFLAGS").cloned().unwrap_or_default(); - if sysroot_panic_abort { + if config.sysroot_panic_abort { rustflags.push_str(" -Cpanic=abort -Zpanic-abort-tests"); } - rustflags.push_str(" -Z force-unstable-if-unmarked"); let mut env = env.clone(); - let channel = if sysroot_release_channel { + let channel = if config.sysroot_release_channel { env.insert( "RUSTFLAGS".to_string(), format!("{} -Zmir-opt-level=3", rustflags), @@ -194,21 +187,6 @@ fn build_sysroot_inner( Ok(()) } -pub fn build_sysroot( - env: &HashMap, - sysroot_panic_abort: bool, - sysroot_release_channel: bool, - config: &ConfigInfo, -) -> Result<(), String> { - build_sysroot_inner( - env, - sysroot_panic_abort, - sysroot_release_channel, - config, - Some(Path::new("build_sysroot")), - ) -} - fn build_codegen(args: &mut BuildArg) -> Result<(), String> { let mut env = HashMap::new(); @@ -229,8 +207,7 @@ fn build_codegen(args: &mut BuildArg) -> Result<(), String> { } run_command_with_output_and_env(&command, None, Some(&env))?; - args.config_info - .setup(&mut env, &[], Some(&args.gcc_path))?; + args.config_info.setup(&mut env, Some(&args.gcc_path))?; // We voluntarily ignore the error. let _ = fs::remove_dir_all("target/out"); @@ -243,12 +220,7 @@ fn build_codegen(args: &mut BuildArg) -> Result<(), String> { })?; println!("[BUILD] sysroot"); - build_sysroot( - &env, - args.config_info.sysroot_panic_abort, - args.config_info.sysroot_release_channel, - &args.config_info, - )?; + build_sysroot(&env, &args.config_info)?; Ok(()) } diff --git a/build_system/src/config.rs b/build_system/src/config.rs index d602cec9f9f5..8396681b292a 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -1,8 +1,9 @@ use crate::utils::{get_gcc_path, get_os_name, rustc_version_info, split_args}; use std::collections::HashMap; use std::env as std_env; +use std::ffi::OsStr; -#[derive(Default)] +#[derive(Default, Debug)] pub struct ConfigInfo { pub target_triple: String, pub host_triple: String, @@ -32,7 +33,6 @@ impl ConfigInfo { }, "--out-dir" => match args.next() { Some(arg) if !arg.is_empty() => { - // env.insert("CARGO_TARGET_DIR".to_string(), arg.to_string()); self.cargo_target_dir = arg.to_string(); } _ => return Err("Expected a value after `--out-dir`, found nothing".to_string()), @@ -44,10 +44,17 @@ impl ConfigInfo { Ok(true) } + pub fn rustc_command_vec(&self) -> Vec<&dyn AsRef> { + let mut command: Vec<&dyn AsRef> = Vec::with_capacity(self.rustc_command.len()); + for arg in self.rustc_command.iter() { + command.push(arg); + } + command + } + pub fn setup( &mut self, env: &mut HashMap, - test_flags: &[String], gcc_path: Option<&str>, ) -> Result<(), String> { env.insert("CARGO_INCREMENTAL".to_string(), "0".to_string()); @@ -90,15 +97,10 @@ impl ConfigInfo { let mut linker = None; if self.host_triple != self.target_triple { - if self.target_triple == "m68k-unknown-linux-gnu" { - linker = Some("-Clinker=m68k-unknown-linux-gnu-gcc".to_string()); - } else if self.target_triple == "aarch64-unknown-linux-gnu" { - // We are cross-compiling for aarch64. Use the correct linker and run tests in qemu. - linker = Some("-Clinker=aarch64-linux-gnu-gcc".to_string()); - } else { + if self.target_triple.is_empty() { return Err("Unknown non-native platform".to_string()); } - + linker = Some(format!("-Clinker={}-gcc", self.target_triple)); self.run_in_vm = true; } @@ -145,7 +147,7 @@ impl ConfigInfo { // This environment variable is useful in case we want to change options of rustc commands. if let Some(cg_rustflags) = env.get("CG_RUSTFLAGS") { - rustflags.extend_from_slice(&split_args(&cg_rustflags)); + rustflags.extend_from_slice(&split_args(&cg_rustflags)?); } if let Some(linker) = linker { @@ -162,7 +164,6 @@ impl ConfigInfo { if !env.contains_key(&"FAT_LTO".to_string()) { rustflags.push("-Clto=off".to_string()); } - rustflags.extend_from_slice(test_flags); // FIXME(antoyo): remove once the atomic shim is gone if os_name == "Darwin" { rustflags.extend_from_slice(&[ diff --git a/build_system/src/prepare.rs b/build_system/src/prepare.rs index da9f8953ec3c..ce9b440be053 100644 --- a/build_system/src/prepare.rs +++ b/build_system/src/prepare.rs @@ -1,5 +1,7 @@ use crate::rustc_info::get_rustc_path; -use crate::utils::{cargo_install, git_clone, run_command, run_command_with_output, walk_dir}; +use crate::utils::{ + cargo_install, git_clone, remove_file, run_command, run_command_with_output, walk_dir, +}; use std::fs; use std::path::Path; @@ -137,8 +139,7 @@ fn build_raytracer(repo_dir: &Path) -> Result<(), String> { run_command(&[&"cargo", &"build"], Some(repo_dir))?; let mv_target = repo_dir.join("raytracer_cg_llvm"); if mv_target.is_file() { - std::fs::remove_file(&mv_target) - .map_err(|e| format!("Failed to remove file `{}`: {e:?}", mv_target.display()))?; + remove_file(&mv_target)?; } run_command( &[&"mv", &"target/debug/main", &"raytracer_cg_llvm"], diff --git a/build_system/src/test.rs b/build_system/src/test.rs index db3e4d9894d8..af2367e668eb 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -3,11 +3,13 @@ use crate::config::ConfigInfo; use crate::utils::{ get_gcc_path, get_toolchain, run_command, run_command_with_env, run_command_with_output_and_env, rustc_version_info, split_args, walk_dir, + remove_file, }; use std::collections::{BTreeSet, HashMap}; use std::ffi::OsStr; -use std::fs::remove_dir_all; +use std::fs::{File, remove_dir_all}; +use std::io::{BufRead, BufReader}; use std::path::{Path, PathBuf}; use std::str::FromStr; @@ -85,7 +87,6 @@ fn show_usage() { `test` command help: --release : Build codegen in release mode - --release-sysroot : Build sysroot in release mode --sysroot-panic-abort : Build the sysroot without unwinding support. --no-default-features : Add `--no-default-features` flag --features [arg] : Add a new feature [arg] @@ -104,7 +105,7 @@ fn show_usage() { println!(" --help : Show this help"); } -#[derive(Default, PartialEq, Eq, Clone, Copy)] +#[derive(Default, PartialEq, Eq, Clone, Copy, Debug)] enum Channel { #[default] Debug, @@ -120,7 +121,7 @@ impl Channel { } } -#[derive(Default)] +#[derive(Default, Debug)] struct TestArg { no_default_features: bool, build_only: bool, @@ -151,7 +152,6 @@ impl TestArg { test_arg.channel = Channel::Release; test_arg.config_info.sysroot_release_channel = true; } - "--release-sysroot" => test_arg.config_info.sysroot_release_channel = true, "--no-default-features" => { // To prevent adding it more than once. if !test_arg.no_default_features { @@ -210,8 +210,19 @@ impl TestArg { get_gcc_path()? }; } + match (test_arg.current_part, test_arg.nb_parts) { + (Some(_), Some(_)) | (None, None) => {} + _ => { + return Err("If either `--current-part` or `--nb-parts` is specified, the other one \ + needs to be specified as well!".to_string()); + } + } Ok(Some(test_arg)) } + + pub fn is_using_gcc_master_branch(&self) -> bool { + !self.no_default_features + } } fn build_if_no_backend(env: &Env, args: &TestArg) -> Result<(), String> { @@ -251,10 +262,7 @@ fn mini_tests(env: &Env, args: &TestArg) -> Result<(), String> { "lib,dylib" } .to_string(); - let mut command: Vec<&dyn AsRef> = Vec::new(); - for arg in args.config_info.rustc_command.iter() { - command.push(arg); - } + let mut command = args.config_info.rustc_command_vec(); command.extend_from_slice(&[ &"example/mini_core.rs", &"--crate-name", @@ -268,10 +276,7 @@ fn mini_tests(env: &Env, args: &TestArg) -> Result<(), String> { // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[BUILD] example"); - command.clear(); - for arg in args.config_info.rustc_command.iter() { - command.push(arg); - } + let mut command = args.config_info.rustc_command_vec(); command.extend_from_slice(&[ &"example/example.rs", &"--crate-type", @@ -283,10 +288,7 @@ fn mini_tests(env: &Env, args: &TestArg) -> Result<(), String> { // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[AOT] mini_core_hello_world"); - command.clear(); - for arg in args.config_info.rustc_command.iter() { - command.push(arg); - } + let mut command = args.config_info.rustc_command_vec(); command.extend_from_slice(&[ &"example/mini_core_hello_world.rs", &"--crate-name", @@ -304,24 +306,19 @@ fn mini_tests(env: &Env, args: &TestArg) -> Result<(), String> { &"abc", &"bcd", ]; - run_command_in_vm(&command, env, args)?; + maybe_run_command_in_vm(&command, env, args)?; Ok(()) } fn build_sysroot(env: &Env, args: &TestArg) -> Result<(), String> { // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[BUILD] sysroot"); - build::build_sysroot( - env, - args.config_info.sysroot_panic_abort, - args.config_info.sysroot_release_channel, - &args.config_info, - )?; + build::build_sysroot(env, &args.config_info)?; Ok(()) } // TODO(GuillaumeGomez): when rewriting in Rust, refactor with the code in tests/lang_tests_common.rs if possible. -fn run_command_in_vm( +fn maybe_run_command_in_vm( command: &[&dyn AsRef], env: &Env, args: &TestArg, @@ -358,12 +355,10 @@ fn run_command_in_vm( } fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { + let cargo_target_dir = Path::new(&args.config_info.cargo_target_dir); // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[AOT] arbitrary_self_types_pointers_and_wrappers"); - let mut command: Vec<&dyn AsRef> = Vec::new(); - for arg in args.config_info.rustc_command.iter() { - command.push(arg); - } + let mut command = args.config_info.rustc_command_vec(); command.extend_from_slice(&[ &"example/arbitrary_self_types_pointers_and_wrappers.rs", &"--crate-name", @@ -374,19 +369,15 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { &args.config_info.target_triple, ]); run_command_with_env(&command, None, Some(env))?; - run_command_in_vm( - &[&Path::new(&args.config_info.cargo_target_dir) - .join("arbitrary_self_types_pointers_and_wrappers")], + maybe_run_command_in_vm( + &[&cargo_target_dir.join("arbitrary_self_types_pointers_and_wrappers")], env, args, )?; // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[AOT] alloc_system"); - let mut command: Vec<&dyn AsRef> = Vec::new(); - for arg in args.config_info.rustc_command.iter() { - command.push(arg); - } + let mut command = args.config_info.rustc_command_vec(); command.extend_from_slice(&[ &"example/alloc_system.rs", &"--crate-type", @@ -394,19 +385,16 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { &"--target", &args.config_info.target_triple, ]); - if !args.no_default_features { + if args.is_using_gcc_master_branch() { command.extend_from_slice(&[&"--cfg", &"feature=\"master\""]); } run_command_with_env(&command, None, Some(env))?; // FIXME: doesn't work on m68k. - if args.config_info.host_triple != args.config_info.target_triple { + if args.config_info.host_triple == args.config_info.target_triple { // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[AOT] alloc_example"); - let mut command: Vec<&dyn AsRef> = Vec::new(); - for arg in args.config_info.rustc_command.iter() { - command.push(arg); - } + let mut command = args.config_info.rustc_command_vec(); command.extend_from_slice(&[ &"example/alloc_example.rs", &"--crate-type", @@ -415,8 +403,8 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { &args.config_info.target_triple, ]); run_command_with_env(&command, None, Some(env))?; - run_command_in_vm( - &[&Path::new(&args.config_info.cargo_target_dir).join("alloc_example")], + maybe_run_command_in_vm( + &[&cargo_target_dir.join("alloc_example")], env, args, )?; @@ -425,10 +413,7 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[AOT] dst_field_align"); // FIXME(antoyo): Re-add -Zmir-opt-level=2 once rust-lang/rust#67529 is fixed. - let mut command: Vec<&dyn AsRef> = Vec::new(); - for arg in args.config_info.rustc_command.iter() { - command.push(arg); - } + let mut command = args.config_info.rustc_command_vec(); command.extend_from_slice(&[ &"example/dst-field-align.rs", &"--crate-name", @@ -439,18 +424,15 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { &args.config_info.target_triple, ]); run_command_with_env(&command, None, Some(env))?; - run_command_in_vm( - &[&Path::new(&args.config_info.cargo_target_dir).join("dst_field_align")], + maybe_run_command_in_vm( + &[&cargo_target_dir.join("dst_field_align")], env, args, )?; // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[AOT] std_example"); - let mut command: Vec<&dyn AsRef> = Vec::new(); - for arg in args.config_info.rustc_command.iter() { - command.push(arg); - } + let mut command = args.config_info.rustc_command_vec(); command.extend_from_slice(&[ &"example/std_example.rs", &"--crate-type", @@ -458,13 +440,13 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { &"--target", &args.config_info.target_triple, ]); - if !args.no_default_features { + if args.is_using_gcc_master_branch() { command.extend_from_slice(&[&"--cfg", &"feature=\"master\""]); } run_command_with_env(&command, None, Some(env))?; - run_command_in_vm( + maybe_run_command_in_vm( &[ - &Path::new(&args.config_info.cargo_target_dir).join("std_example"), + &cargo_target_dir.join("std_example"), &"--target", &args.config_info.target_triple, ], @@ -472,12 +454,14 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { args, )?; + let test_flags = if let Some(test_flags) = env.get("TEST_FLAGS") { + split_args(test_flags)? + } else { + Vec::new() + }; // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[AOT] subslice-patterns-const-eval"); - let mut command: Vec<&dyn AsRef> = Vec::new(); - for arg in args.config_info.rustc_command.iter() { - command.push(arg); - } + let mut command = args.config_info.rustc_command_vec(); command.extend_from_slice(&[ &"example/subslice-patterns-const-eval.rs", &"--crate-type", @@ -485,19 +469,19 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { &"--target", &args.config_info.target_triple, ]); + for test_flag in &test_flags { + command.push(test_flag); + } run_command_with_env(&command, None, Some(env))?; - run_command_in_vm( - &[&Path::new(&args.config_info.cargo_target_dir).join("subslice-patterns-const-eval")], + maybe_run_command_in_vm( + &[&cargo_target_dir.join("subslice-patterns-const-eval")], env, args, )?; // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[AOT] track-caller-attribute"); - let mut command: Vec<&dyn AsRef> = Vec::new(); - for arg in args.config_info.rustc_command.iter() { - command.push(arg); - } + let mut command = args.config_info.rustc_command_vec(); command.extend_from_slice(&[ &"example/track-caller-attribute.rs", &"--crate-type", @@ -505,19 +489,19 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { &"--target", &args.config_info.target_triple, ]); + for test_flag in &test_flags { + command.push(test_flag); + } run_command_with_env(&command, None, Some(env))?; - run_command_in_vm( - &[&Path::new(&args.config_info.cargo_target_dir).join("track-caller-attribute")], + maybe_run_command_in_vm( + &[&cargo_target_dir.join("track-caller-attribute")], env, args, )?; // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[AOT] mod_bench"); - let mut command: Vec<&dyn AsRef> = Vec::new(); - for arg in args.config_info.rustc_command.iter() { - command.push(arg); - } + let mut command = args.config_info.rustc_command_vec(); command.extend_from_slice(&[ &"example/mod_bench.rs", &"--crate-type", @@ -547,6 +531,7 @@ fn setup_rustc(env: &mut Env, args: &TestArg) -> Result<(), String> { None => return Err("Couldn't retrieve rustc commit hash".to_string()), }; run_command_with_output_and_env(&[&"git", &"checkout", &rustc_commit], rust_dir, Some(env))?; + // FIXME: Is it really needed to empty `RUSTFLAGS` here? env.insert("RUSTFLAGS".to_string(), String::new()); let cargo = String::from_utf8( run_command_with_env(&[&"rustup", &"which", &"cargo"], rust_dir, Some(env))?.stdout, @@ -661,7 +646,7 @@ fn run_cargo_command( args: &TestArg, ) -> Result<(), String> { run_cargo_command_with_callback(command, cwd, env, args, |cargo_command, cwd, env| { - run_command_with_output_and_env(&cargo_command, cwd, Some(env))?; + run_command_with_output_and_env(cargo_command, cwd, Some(env))?; Ok(()) }) } @@ -734,7 +719,7 @@ fn test_libcore(env: &Env, args: &TestArg) -> Result<(), String> { // hyperfine --runs ${RUN_RUNS:-10} $cargo_target_dir/mod_bench{,_inline} $cargo_target_dir/mod_bench_llvm_* fn extended_rand_tests(env: &Env, args: &TestArg) -> Result<(), String> { - if args.no_default_features { + if !args.is_using_gcc_master_branch() { return Ok(()); } let path = Path::new("rand"); @@ -746,7 +731,7 @@ fn extended_rand_tests(env: &Env, args: &TestArg) -> Result<(), String> { } fn extended_regex_example_tests(env: &Env, args: &TestArg) -> Result<(), String> { - if args.no_default_features { + if !args.is_using_gcc_master_branch() { return Ok(()); } let path = Path::new("regex"); @@ -800,7 +785,7 @@ fn extended_regex_example_tests(env: &Env, args: &TestArg) -> Result<(), String> } fn extended_regex_tests(env: &Env, args: &TestArg) -> Result<(), String> { - if args.no_default_features { + if !args.is_using_gcc_master_branch() { return Ok(()); } // FIXME: create a function "display_if_not_quiet" or something along the line. @@ -817,6 +802,7 @@ fn extended_regex_tests(env: &Env, args: &TestArg) -> Result<(), String> { &"test", &"--tests", &"--", + // FIXME: try removing `--exclude-should-panic` argument &"--exclude-should-panic", &"--test-threads", &"1", @@ -848,24 +834,22 @@ fn extended_sysroot_tests(env: &Env, args: &TestArg) -> Result<(), String> { Ok(()) } -fn should_remove_ui_test(content: &str) -> bool { - for line in content - .lines() - .map(|line| line.trim()) - .filter(|line| !line.is_empty()) - { - if [ - "// error-pattern:", - "// build-fail", - "// run-fail", - "-Cllvm-args", - "//~", - "// ~", - ] - .iter() - .any(|check| line.contains(check)) - { - return true; +fn should_remove_ui_test(file: File) -> bool { + for line in BufReader::new(file).lines() { + if let Ok(line) = line { + if [ + "// error-pattern:", + "// build-fail", + "// run-fail", + "-Cllvm-args", + "//~", + "// ~", + ] + .iter() + .any(|check| line.contains(check)) + { + return true; + } } } false @@ -903,7 +887,7 @@ fn should_remove_test(path: &Path, path_str: &str) -> bool { .any(|to_ignore| path_str.ends_with(to_ignore)) } -fn test_rustc_inner(env: &Env, args: &TestArg, callback: F) -> Result<(), String> +fn test_rustc_inner(env: &Env, args: &TestArg, prepare_files_callback: F) -> Result<(), String> where F: Fn() -> Result, { @@ -937,24 +921,24 @@ where |_| Ok(()), )?; + // These two functions are used to remove files that are known to not be working currently + // with the GCC backend to reduce noise. fn dir_handling(dir: &Path) -> Result<(), String> { walk_dir(dir, dir_handling, file_handling) } - fn file_handling(file: &Path) -> Result<(), String> { - let path_str = file.display().to_string().replace("\\", "/"); + fn file_handling(file_path: &Path) -> Result<(), String> { + let path_str = file_path.display().to_string().replace("\\", "/"); if !path_str.ends_with(".rs") { return Ok(()); } else if should_not_remove_test(&path_str) { return Ok(()); - } else if should_remove_test(file, &path_str) { - return std::fs::remove_file(file) - .map_err(|error| format!("Failed to remove `{}`: {:?}", file.display(), error)); + } else if should_remove_test(file_path, &path_str) { + return remove_file(&file_path); } - let file_content = std::fs::read_to_string(file) - .map_err(|error| format!("Failed to read `{}`: {:?}", file.display(), error))?; - if should_remove_ui_test(&file_content) { - std::fs::remove_file(file) - .map_err(|error| format!("Failed to remove `{}`: {:?}", file.display(), error))?; + let file = File::open(file_path) + .map_err(|error| format!("Failed to read `{}`: {:?}", file_path.display(), error))?; + if should_remove_ui_test(file) { + remove_file(&file_path)?; } Ok(()) } @@ -963,20 +947,18 @@ where walk_dir(rust_path.join("tests/ui"), dir_handling, file_handling)?; let file = rust_path.join("tests/ui/consts/const_cmp_type_id.rs"); - std::fs::remove_file(&file) - .map_err(|error| format!("Failed to remove `{}`: {:?}", file.display(), error))?; + remove_file(&file)?; let file = rust_path.join("tests/ui/consts/issue-73976-monomorphic.rs"); - std::fs::remove_file(&file) - .map_err(|error| format!("Failed to remove `{}`: {:?}", file.display(), error))?; + remove_file(&file)?; - if !callback()? { + if !prepare_files_callback()? { // FIXME: create a function "display_if_not_quiet" or something along the line. println!("Keeping all UI tests"); } let nb_parts = args.nb_parts.unwrap_or(0); if nb_parts > 0 { - let current_part = args.current_part.unwrap_or(0); + let current_part = args.current_part.unwrap(); // FIXME: create a function "display_if_not_quiet" or something along the line. println!( "Splitting ui_test into {} parts (and running part {})", @@ -1017,18 +999,19 @@ where continue; } let test_path = rust_path.join(path); - std::fs::remove_file(&test_path).map_err(|error| { - format!("Failed to remove `{}`: {:?}", test_path.display(), error) - })?; + remove_file(&test_path)?; } } // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[TEST] rustc test suite"); env.insert("COMPILETEST_FORCE_STAGE0".to_string(), "1".to_string()); - let rustc_args = env - .get("RUSTFLAGS") - .expect("RUSTFLAGS should not be empty at this stage"); + let rustc_args = format!( + "{} {}", + env.get("RUSTFLAGS") + .expect("RUSTFLAGS should not be empty at this stage"), + env.get("TEST_FLAGS").unwrap_or(&String::new()), + ); run_command_with_output_and_env( &[ &"./x.py", @@ -1103,9 +1086,7 @@ fn test_successful_rustc(env: &Env, args: &TestArg) -> Result<(), String> { .filter(|line| !line.is_empty()) { let path = Path::new("rust").join(file); - std::fs::remove_file(&path).map_err(|error| { - format!("failed to remove `{}`: {:?}", path.display(), error) - })?; + remove_file(&path)?; } } else { println!( @@ -1159,9 +1140,7 @@ pub fn run() -> Result<(), String> { return Ok(()); } - let test_flags = split_args(env.get("TEST_FLAGS").unwrap_or(&String::new())); - args.config_info - .setup(&mut env, &test_flags, Some(&args.gcc_path))?; + args.config_info.setup(&mut env, Some(&args.gcc_path))?; if args.runners.is_empty() { run_all(&env, &args)?; diff --git a/build_system/src/utils.rs b/build_system/src/utils.rs index 88fce2fcbce1..59863fcfd90a 100644 --- a/build_system/src/utils.rs +++ b/build_system/src/utils.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::collections::HashMap; use std::ffi::OsStr; use std::fmt::Debug; @@ -48,16 +49,20 @@ fn check_exit_status( let input = input.iter().map(|i| i.as_ref()).collect::>(); eprintln!("Command `{:?}` failed", input); if let Some(output) = output { - unsafe { - let stdout = std::str::from_utf8_unchecked(&output.stdout); - if !stdout.is_empty() { - error.push_str("\n==== STDOUT ====\n"); - error.push_str(stdout); + let stdout = String::from_utf8_lossy(&output.stdout); + if !stdout.is_empty() { + error.push_str("\n==== STDOUT ====\n"); + match stdout { + Cow::Owned(s) => error.push_str(&s), + Cow::Borrowed(s) => error.push_str(s), } - let stderr = std::str::from_utf8_unchecked(&output.stderr); - if !stderr.is_empty() { - error.push_str("\n==== STDERR ====\n"); - error.push_str(stderr); + } + let stderr = String::from_utf8_lossy(&output.stderr); + if !stderr.is_empty() { + error.push_str("\n==== STDERR ====\n"); + match stderr { + Cow::Owned(s) => error.push_str(&s), + Cow::Borrowed(s) => error.push_str(s), } } } @@ -295,17 +300,16 @@ where Ok(()) } -pub fn split_args(args: &str) -> Vec { +pub fn split_args(args: &str) -> Result, String> { let mut out = Vec::new(); let mut start = 0; + let args = args.trim(); let mut iter = args.char_indices().peekable(); while iter.peek().is_some() { while let Some((pos, c)) = iter.next() { if c == ' ' { - if pos != 0 { - out.push(args[start..pos].to_string()); - } + out.push(args[start..pos].to_string()); let mut found_start = false; while let Some((pos, c)) = iter.peek() { if *c != ' ' { @@ -317,7 +321,7 @@ pub fn split_args(args: &str) -> Vec { } } if !found_start { - return out; + return Ok(out); } } else if c == '"' || c == '\'' { let end = c; @@ -332,8 +336,7 @@ pub fn split_args(args: &str) -> Vec { } } if !found_end { - out.push(args[start..].to_string()); - return out; + return Err(format!("Didn't find `{}` at the end of `{}`", end, &args[start..])); } } else if c == '\\' { // We skip the escaped character. @@ -345,5 +348,15 @@ pub fn split_args(args: &str) -> Vec { if !s.is_empty() { out.push(s.to_string()); } - out + Ok(out) +} + +pub fn remove_file>(file_path: &P) -> Result<(), String> { + std::fs::remove_file(file_path).map_err(|error| { + format!( + "Failed to remove `{}`: {:?}", + file_path.as_ref().display(), + error + ) + }) } From 867ea124884cabf81462e152fcfa2cdccd3ed1aa Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 5 Dec 2023 21:09:13 +0100 Subject: [PATCH 24/34] Fix non-running rustc ui tests --- build_system/src/build.rs | 2 +- build_system/src/config.rs | 17 ++-- build_system/src/test.rs | 188 ++++++++++++++++++------------------- build_system/src/utils.rs | 6 +- 4 files changed, 109 insertions(+), 104 deletions(-) diff --git a/build_system/src/build.rs b/build_system/src/build.rs index 618e74be2c0c..370d8436e3de 100644 --- a/build_system/src/build.rs +++ b/build_system/src/build.rs @@ -128,9 +128,9 @@ pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Resu &[ &"cargo", &"build", + &"--release", &"--target", &config.target_triple, - &"--release", ], Some(start_dir), Some(&env), diff --git a/build_system/src/config.rs b/build_system/src/config.rs index 8396681b292a..091186b90660 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -13,6 +13,8 @@ pub struct ConfigInfo { pub dylib_ext: String, pub sysroot_release_channel: bool, pub sysroot_panic_abort: bool, + pub cg_backend_path: String, + pub sysroot_path: String, } impl ConfigInfo { @@ -118,13 +120,12 @@ impl ConfigInfo { .get("BUILTIN_BACKEND") .map(|backend| !backend.is_empty()) .unwrap_or(false); - let cg_backend_path; let mut rustflags = Vec::new(); if has_builtin_backend { // It means we're building inside the rustc testsuite, so some options need to be handled // a bit differently. - cg_backend_path = "gcc".to_string(); + self.cg_backend_path = "gcc".to_string(); match env.get("RUSTC_SYSROOT") { Some(rustc_sysroot) if !rustc_sysroot.is_empty() => { @@ -134,15 +135,17 @@ impl ConfigInfo { } rustflags.push("-Cpanic=abort".to_string()); } else { - cg_backend_path = current_dir + self.cg_backend_path = current_dir .join("target") .join(channel) .join(&format!("librustc_codegen_gcc.{}", self.dylib_ext)) .display() .to_string(); - let sysroot_path = current_dir.join("build_sysroot/sysroot"); - rustflags - .extend_from_slice(&["--sysroot".to_string(), sysroot_path.display().to_string()]); + self.sysroot_path = current_dir + .join("build_sysroot/sysroot") + .display() + .to_string(); + rustflags.extend_from_slice(&["--sysroot".to_string(), self.sysroot_path.clone()]); }; // This environment variable is useful in case we want to change options of rustc commands. @@ -156,7 +159,7 @@ impl ConfigInfo { rustflags.extend_from_slice(&[ "-Csymbol-mangling-version=v0".to_string(), "-Cdebuginfo=2".to_string(), - format!("-Zcodegen-backend={}", cg_backend_path), + format!("-Zcodegen-backend={}", self.cg_backend_path), ]); // Since we don't support ThinLTO, disable LTO completely when not trying to do LTO. diff --git a/build_system/src/test.rs b/build_system/src/test.rs index af2367e668eb..efd8ebdd52d9 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -1,14 +1,13 @@ use crate::build; use crate::config::ConfigInfo; use crate::utils::{ - get_gcc_path, get_toolchain, run_command, run_command_with_env, + get_gcc_path, get_toolchain, remove_file, run_command, run_command_with_env, run_command_with_output_and_env, rustc_version_info, split_args, walk_dir, - remove_file, }; use std::collections::{BTreeSet, HashMap}; use std::ffi::OsStr; -use std::fs::{File, remove_dir_all}; +use std::fs::{remove_dir_all, File}; use std::io::{BufRead, BufReader}; use std::path::{Path, PathBuf}; use std::str::FromStr; @@ -213,8 +212,11 @@ impl TestArg { match (test_arg.current_part, test_arg.nb_parts) { (Some(_), Some(_)) | (None, None) => {} _ => { - return Err("If either `--current-part` or `--nb-parts` is specified, the other one \ - needs to be specified as well!".to_string()); + return Err( + "If either `--current-part` or `--nb-parts` is specified, the other one \ + needs to be specified as well!" + .to_string(), + ); } } Ok(Some(test_arg)) @@ -230,20 +232,19 @@ fn build_if_no_backend(env: &Env, args: &TestArg) -> Result<(), String> { return Ok(()); } let mut command: Vec<&dyn AsRef> = vec![&"cargo", &"rustc"]; - if args.channel == Channel::Release { - let mut env = env.clone(); - env.insert("CARGO_INCREMENTAL".to_string(), "1".to_string()); + let mut tmp_env; + let env = if args.channel == Channel::Release { + tmp_env = env.clone(); + tmp_env.insert("CARGO_INCREMENTAL".to_string(), "1".to_string()); command.push(&"--release"); - for flag in args.flags.iter() { - command.push(flag); - } - run_command_with_output_and_env(&command, None, Some(&env)) + &tmp_env } else { - for flag in args.flags.iter() { - command.push(flag); - } - run_command_with_output_and_env(&command, None, Some(&env)) + &env + }; + for flag in args.flags.iter() { + command.push(flag); } + run_command_with_output_and_env(&command, None, Some(env)) } fn clean(_env: &Env, args: &TestArg) -> Result<(), String> { @@ -403,11 +404,7 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { &args.config_info.target_triple, ]); run_command_with_env(&command, None, Some(env))?; - maybe_run_command_in_vm( - &[&cargo_target_dir.join("alloc_example")], - env, - args, - )?; + maybe_run_command_in_vm(&[&cargo_target_dir.join("alloc_example")], env, args)?; } // FIXME: create a function "display_if_not_quiet" or something along the line. @@ -424,11 +421,7 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { &args.config_info.target_triple, ]); run_command_with_env(&command, None, Some(env))?; - maybe_run_command_in_vm( - &[&cargo_target_dir.join("dst_field_align")], - env, - args, - )?; + maybe_run_command_in_vm(&[&cargo_target_dir.join("dst_field_align")], env, args)?; // FIXME: create a function "display_if_not_quiet" or something along the line. println!("[AOT] std_example"); @@ -525,6 +518,7 @@ fn setup_rustc(env: &mut Env, args: &TestArg) -> Result<(), String> { None, Some(env), ); + run_command(&[&"git", &"checkout", &"--", &"tests/"], rust_dir)?; run_command_with_output_and_env(&[&"git", &"fetch"], rust_dir, Some(env))?; let rustc_commit = match rustc_version_info(env.get("RUSTC").map(|s| s.as_str()))?.commit_hash { Some(commit_hash) => commit_hash, @@ -532,7 +526,7 @@ fn setup_rustc(env: &mut Env, args: &TestArg) -> Result<(), String> { }; run_command_with_output_and_env(&[&"git", &"checkout", &rustc_commit], rust_dir, Some(env))?; // FIXME: Is it really needed to empty `RUSTFLAGS` here? - env.insert("RUSTFLAGS".to_string(), String::new()); + // env.insert("RUSTFLAGS".to_string(), String::new()); let cargo = String::from_utf8( run_command_with_env(&[&"rustup", &"which", &"cargo"], rust_dir, Some(env))?.stdout, ) @@ -591,15 +585,6 @@ download-ci-llvm = false ), ) .map_err(|error| format!("Failed to write into `rust/config.toml`: {:?}", error))?; - - let rustc_commit = match rustc_version_info(env.get("RUSTC").map(|s| s.as_str()))?.commit_hash { - Some(commit_hash) => commit_hash, - None => return Err("Couldn't retrieve rustc commit hash".to_string()), - }; - // FIXME: create a function "display_if_not_quiet" or something along the line. - println!("commit: {:?}", rustc_commit); - let command: &[&dyn AsRef] = &[&"git", &"checkout", &rustc_commit, &"tests"]; - run_command_with_output_and_env(command, rust_dir, Some(env))?; Ok(()) } @@ -834,27 +819,6 @@ fn extended_sysroot_tests(env: &Env, args: &TestArg) -> Result<(), String> { Ok(()) } -fn should_remove_ui_test(file: File) -> bool { - for line in BufReader::new(file).lines() { - if let Ok(line) = line { - if [ - "// error-pattern:", - "// build-fail", - "// run-fail", - "-Cllvm-args", - "//~", - "// ~", - ] - .iter() - .any(|check| line.contains(check)) - { - return true; - } - } - } - false -} - fn should_not_remove_test(file: &str) -> bool { // contains //~ERROR, but shouldn't be removed [ @@ -870,21 +834,40 @@ fn should_not_remove_test(file: &str) -> bool { .any(|to_ignore| file.ends_with(to_ignore)) } -fn should_remove_test(path: &Path, path_str: &str) -> bool { +fn should_remove_test(file_path: &Path) -> Result { // Tests generating errors. - path.file_name() - .and_then(|name| name.to_str()) - .map(|name| name.contains("thread")) - .unwrap_or(false) - || [ - "consts/issue-miri-1910.rs", - // Tests generating errors. - "consts/issue-94675.rs", - // this test is oom-killed in the CI. - "mir/mir_heavy/issue-miri-1910.rs", + let file = File::open(file_path) + .map_err(|error| format!("Failed to read `{}`: {:?}", file_path.display(), error))?; + for line in BufReader::new(file).lines().filter_map(|line| line.ok()) { + let line = line.trim(); + if line.is_empty() { + continue; + } + if [ + "// error-pattern:", + "// build-fail", + "// run-fail", + "-Cllvm-args", + "//~", + "thread", ] .iter() - .any(|to_ignore| path_str.ends_with(to_ignore)) + .any(|check| line.contains(check)) + { + return Ok(true); + } + if line.contains("//[") && line.contains("]~") { + return Ok(true); + } + } + if file_path + .display() + .to_string() + .contains("ambiguous-4-extern.rs") + { + eprintln!("nothing found for {file_path:?}"); + } + Ok(false) } fn test_rustc_inner(env: &Env, args: &TestArg, prepare_files_callback: F) -> Result<(), String> @@ -896,6 +879,8 @@ where let mut env = env.clone(); setup_rustc(&mut env, args)?; + let rust_path = Path::new("rust"); + walk_dir( "rust/tests/ui", |dir| { @@ -924,32 +909,41 @@ where // These two functions are used to remove files that are known to not be working currently // with the GCC backend to reduce noise. fn dir_handling(dir: &Path) -> Result<(), String> { + if dir + .file_name() + .map(|name| name == "auxiliary") + .unwrap_or(true) + { + return Ok(()); + } walk_dir(dir, dir_handling, file_handling) } fn file_handling(file_path: &Path) -> Result<(), String> { - let path_str = file_path.display().to_string().replace("\\", "/"); - if !path_str.ends_with(".rs") { + if !file_path + .extension() + .map(|extension| extension == "rs") + .unwrap_or(false) + { return Ok(()); - } else if should_not_remove_test(&path_str) { - return Ok(()); - } else if should_remove_test(file_path, &path_str) { - return remove_file(&file_path); } - let file = File::open(file_path) - .map_err(|error| format!("Failed to read `{}`: {:?}", file_path.display(), error))?; - if should_remove_ui_test(file) { - remove_file(&file_path)?; + let path_str = file_path.display().to_string().replace("\\", "/"); + if should_not_remove_test(&path_str) { + return Ok(()); + } else if should_remove_test(file_path)? { + return remove_file(&file_path); } Ok(()) } - let rust_path = Path::new("rust"); + remove_file(&rust_path.join("tests/ui/consts/const_cmp_type_id.rs"))?; + remove_file(&rust_path.join("tests/ui/consts/issue-73976-monomorphic.rs"))?; + // this test is oom-killed in the CI. + remove_file(&rust_path.join("tests/ui/consts/issue-miri-1910.rs"))?; + // Tests generating errors. + remove_file(&rust_path.join("tests/ui/consts/issue-94675.rs"))?; + remove_file(&rust_path.join("tests/ui/mir/mir_heavy_promoted.rs"))?; walk_dir(rust_path.join("tests/ui"), dir_handling, file_handling)?; - let file = rust_path.join("tests/ui/consts/const_cmp_type_id.rs"); - remove_file(&file)?; - let file = rust_path.join("tests/ui/consts/issue-73976-monomorphic.rs"); - remove_file(&file)?; if !prepare_files_callback()? { // FIXME: create a function "display_if_not_quiet" or something along the line. @@ -992,14 +986,16 @@ where // We increment the number of tests by one because if this is an odd number, we would skip // one test. let count = files.len() / nb_parts + 1; - let start = nb_parts * count; - let end = start + count; - for (pos, path) in files.iter().enumerate() { - if pos >= start && pos <= end { - continue; - } - let test_path = rust_path.join(path); - remove_file(&test_path)?; + let start = current_part * count; + let end = current_part * count + count; + // We remove the files we don't want to test. + for path in files + .iter() + .enumerate() + .filter(|(pos, _)| *pos < start || *pos >= end) + .map(|(_, path)| path) + { + remove_file(&rust_path.join(path))?; } } @@ -1007,11 +1003,13 @@ where println!("[TEST] rustc test suite"); env.insert("COMPILETEST_FORCE_STAGE0".to_string(), "1".to_string()); let rustc_args = format!( - "{} {}", - env.get("RUSTFLAGS") - .expect("RUSTFLAGS should not be empty at this stage"), + "{} -Csymbol-mangling-version=v0 -Zcodegen-backend={} --sysroot {}", env.get("TEST_FLAGS").unwrap_or(&String::new()), + args.config_info.cg_backend_path, + args.config_info.sysroot_path, ); + + env.get_mut("RUSTFLAGS").unwrap().clear(); run_command_with_output_and_env( &[ &"./x.py", diff --git a/build_system/src/utils.rs b/build_system/src/utils.rs index 59863fcfd90a..9d785e7f57cf 100644 --- a/build_system/src/utils.rs +++ b/build_system/src/utils.rs @@ -336,7 +336,11 @@ pub fn split_args(args: &str) -> Result, String> { } } if !found_end { - return Err(format!("Didn't find `{}` at the end of `{}`", end, &args[start..])); + return Err(format!( + "Didn't find `{}` at the end of `{}`", + end, + &args[start..] + )); } } else if c == '\\' { // We skip the escaped character. From db9b932314023318f49b0b5941d09f034a12b31e Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Wed, 13 Dec 2023 21:35:05 +0100 Subject: [PATCH 25/34] Fix sysroot build --- build_system/src/build.rs | 8 +++----- build_system/src/config.rs | 24 +++++++++++++++++++++++- build_system/src/test.rs | 27 +++------------------------ 3 files changed, 29 insertions(+), 30 deletions(-) diff --git a/build_system/src/build.rs b/build_system/src/build.rs index 370d8436e3de..9fb47195aeed 100644 --- a/build_system/src/build.rs +++ b/build_system/src/build.rs @@ -1,4 +1,4 @@ -use crate::config::ConfigInfo; +use crate::config::{Channel, ConfigInfo}; use crate::utils::{get_gcc_path, run_command, run_command_with_output_and_env, walk_dir}; use std::collections::HashMap; use std::ffi::OsStr; @@ -7,7 +7,6 @@ use std::path::Path; #[derive(Default)] struct BuildArg { - codegen_release_channel: bool, flags: Vec, gcc_path: String, config_info: ConfigInfo, @@ -25,7 +24,6 @@ impl BuildArg { while let Some(arg) = args.next() { match arg.as_str() { - "--release" => build_arg.codegen_release_channel = true, "--no-default-features" => { build_arg.flags.push("--no-default-features".to_string()); } @@ -58,7 +56,6 @@ impl BuildArg { r#" `build` command help: - --release : Build codegen in release mode --no-default-features : Add `--no-default-features` flag --features [arg] : Add a new feature [arg]"# ); @@ -118,6 +115,7 @@ pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Resu if config.sysroot_panic_abort { rustflags.push_str(" -Cpanic=abort -Zpanic-abort-tests"); } + rustflags.push_str(" -Z force-unstable-if-unmarked"); let mut env = env.clone(); let channel = if config.sysroot_release_channel { env.insert( @@ -194,7 +192,7 @@ fn build_codegen(args: &mut BuildArg) -> Result<(), String> { env.insert("LIBRARY_PATH".to_string(), args.gcc_path.clone()); let mut command: Vec<&dyn AsRef> = vec![&"cargo", &"rustc"]; - if args.codegen_release_channel { + if args.config_info.channel == Channel::Release { command.push(&"--release"); env.insert("CHANNEL".to_string(), "release".to_string()); env.insert("CARGO_INCREMENTAL".to_string(), "1".to_string()); diff --git a/build_system/src/config.rs b/build_system/src/config.rs index 091186b90660..09375791aa31 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -3,6 +3,22 @@ use std::collections::HashMap; use std::env as std_env; use std::ffi::OsStr; +#[derive(Default, PartialEq, Eq, Clone, Copy, Debug)] +pub enum Channel { + #[default] + Debug, + Release, +} + +impl Channel { + pub fn as_str(self) -> &'static str { + match self { + Self::Debug => "debug", + Self::Release => "release", + } + } +} + #[derive(Default, Debug)] pub struct ConfigInfo { pub target_triple: String, @@ -12,6 +28,7 @@ pub struct ConfigInfo { pub cargo_target_dir: String, pub dylib_ext: String, pub sysroot_release_channel: bool, + pub channel: Channel, pub sysroot_panic_abort: bool, pub cg_backend_path: String, pub sysroot_path: String, @@ -40,6 +57,7 @@ impl ConfigInfo { _ => return Err("Expected a value after `--out-dir`, found nothing".to_string()), }, "--release-sysroot" => self.sysroot_release_channel = true, + "--release" => self.channel = Channel::Release, "--sysroot-panic-abort" => self.sysroot_panic_abort = true, _ => return Ok(false), } @@ -108,7 +126,7 @@ impl ConfigInfo { let current_dir = std_env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?; - let channel = if self.sysroot_release_channel { + let channel = if self.channel == Channel::Release { "release" } else if let Some(channel) = env.get("CHANNEL") { channel.as_str() @@ -152,6 +170,9 @@ impl ConfigInfo { if let Some(cg_rustflags) = env.get("CG_RUSTFLAGS") { rustflags.extend_from_slice(&split_args(&cg_rustflags)?); } + if let Some(test_flags) = env.get("TEST_FLAGS") { + rustflags.extend_from_slice(&split_args(&test_flags)?); + } if let Some(linker) = linker { rustflags.push(linker.to_string()); @@ -223,6 +244,7 @@ impl ConfigInfo { "\ --target-triple [arg] : Set the target triple to [arg] --out-dir : Location where the files will be generated + --release : Build in release mode --release-sysroot : Build sysroot in release mode --sysroot-panic-abort : Build the sysroot without unwinding support." ); diff --git a/build_system/src/test.rs b/build_system/src/test.rs index efd8ebdd52d9..1e9652d28220 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -1,5 +1,5 @@ use crate::build; -use crate::config::ConfigInfo; +use crate::config::{Channel, ConfigInfo}; use crate::utils::{ get_gcc_path, get_toolchain, remove_file, run_command, run_command_with_env, run_command_with_output_and_env, rustc_version_info, split_args, walk_dir, @@ -104,28 +104,11 @@ fn show_usage() { println!(" --help : Show this help"); } -#[derive(Default, PartialEq, Eq, Clone, Copy, Debug)] -enum Channel { - #[default] - Debug, - Release, -} - -impl Channel { - pub fn as_str(self) -> &'static str { - match self { - Self::Debug => "debug", - Self::Release => "release", - } - } -} - #[derive(Default, Debug)] struct TestArg { no_default_features: bool, build_only: bool, gcc_path: String, - channel: Channel, use_backend: bool, runners: BTreeSet, flags: Vec, @@ -147,10 +130,6 @@ impl TestArg { while let Some(arg) = args.next() { match arg.as_str() { - "--release" => { - test_arg.channel = Channel::Release; - test_arg.config_info.sysroot_release_channel = true; - } "--no-default-features" => { // To prevent adding it more than once. if !test_arg.no_default_features { @@ -233,7 +212,7 @@ fn build_if_no_backend(env: &Env, args: &TestArg) -> Result<(), String> { } let mut command: Vec<&dyn AsRef> = vec![&"cargo", &"rustc"]; let mut tmp_env; - let env = if args.channel == Channel::Release { + let env = if args.config_info.channel == Channel::Release { tmp_env = env.clone(); tmp_env.insert("CARGO_INCREMENTAL".to_string(), "1".to_string()); command.push(&"--release"); @@ -613,7 +592,7 @@ fn asm_tests(env: &Env, args: &TestArg) -> Result<(), String> { pwd = std::env::current_dir() .map_err(|error| format!("`current_dir` failed: {:?}", error))? .display(), - channel = args.channel.as_str(), + channel = args.config_info.channel.as_str(), dylib_ext = args.config_info.dylib_ext, ) .as_str(), From 95dfe5ec9040bcba53a8dd61d3593d182defab71 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sat, 16 Dec 2023 17:39:58 +0100 Subject: [PATCH 26/34] Simplify `split_args` code, add a unit test for it and run it into CI --- .github/workflows/ci.yml | 9 ++++ build_system/src/utils.rs | 97 ++++++++++++++++++++++++--------------- 2 files changed, 68 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e361bf617b1..b04ea1550ba2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -131,3 +131,12 @@ jobs: steps: - uses: actions/checkout@v3 - run: python tools/check_intrinsics_duplicates.py + + build_system: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Test build system + run: | + cd build_system + cargo test diff --git a/build_system/src/utils.rs b/build_system/src/utils.rs index 9d785e7f57cf..ebfa41c761ce 100644 --- a/build_system/src/utils.rs +++ b/build_system/src/utils.rs @@ -306,46 +306,44 @@ pub fn split_args(args: &str) -> Result, String> { let args = args.trim(); let mut iter = args.char_indices().peekable(); - while iter.peek().is_some() { - while let Some((pos, c)) = iter.next() { - if c == ' ' { - out.push(args[start..pos].to_string()); - let mut found_start = false; - while let Some((pos, c)) = iter.peek() { - if *c != ' ' { - start = *pos; - found_start = true; - break; - } else { - iter.next(); - } + while let Some((pos, c)) = iter.next() { + if c == ' ' { + out.push(args[start..pos].to_string()); + let mut found_start = false; + while let Some((pos, c)) = iter.peek() { + if *c != ' ' { + start = *pos; + found_start = true; + break; + } else { + iter.next(); } - if !found_start { - return Ok(out); - } - } else if c == '"' || c == '\'' { - let end = c; - let mut found_end = false; - while let Some((_, c)) = iter.next() { - if c == end { - found_end = true; - break; - } else if c == '\\' { - // We skip the escaped character. - iter.next(); - } - } - if !found_end { - return Err(format!( - "Didn't find `{}` at the end of `{}`", - end, - &args[start..] - )); - } - } else if c == '\\' { - // We skip the escaped character. - iter.next(); } + if !found_start { + return Ok(out); + } + } else if c == '"' || c == '\'' { + let end = c; + let mut found_end = false; + while let Some((_, c)) = iter.next() { + if c == end { + found_end = true; + break; + } else if c == '\\' { + // We skip the escaped character. + iter.next(); + } + } + if !found_end { + return Err(format!( + "Didn't find `{}` at the end of `{}`", + end, + &args[start..] + )); + } + } else if c == '\\' { + // We skip the escaped character. + iter.next(); } } let s = args[start..].trim(); @@ -364,3 +362,26 @@ pub fn remove_file>(file_path: &P) -> Result<(), String> { ) }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_split_args() { + // Missing `"` at the end. + assert!(split_args("\"tada").is_err()); + // Missing `'` at the end. + assert!(split_args("\'tada").is_err()); + + assert_eq!( + split_args("a \"b\" c"), + Ok(vec!["a".to_string(), "\"b\"".to_string(), "c".to_string()]) + ); + // Trailing whitespace characters. + assert_eq!( + split_args(" a \"b\" c "), + Ok(vec!["a".to_string(), "\"b\"".to_string(), "c".to_string()]) + ); + } +} From 9882d7c511fcfed404c547a64cbc42b6cf3fc17c Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Sat, 16 Dec 2023 17:55:53 +0100 Subject: [PATCH 27/34] Apply suggestions --- build_system/src/build.rs | 4 ++-- build_system/src/config.rs | 12 ++++++++++++ build_system/src/test.rs | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/build_system/src/build.rs b/build_system/src/build.rs index 9fb47195aeed..d264aac7effb 100644 --- a/build_system/src/build.rs +++ b/build_system/src/build.rs @@ -128,7 +128,7 @@ pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Resu &"build", &"--release", &"--target", - &config.target_triple, + &config.target, ], Some(start_dir), Some(&env), @@ -138,7 +138,7 @@ pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Resu env.insert("RUSTFLAGS".to_string(), rustflags); run_command_with_output_and_env( - &[&"cargo", &"build", &"--target", &config.target_triple], + &[&"cargo", &"build", &"--target", &config.target], Some(start_dir), Some(&env), )?; diff --git a/build_system/src/config.rs b/build_system/src/config.rs index 09375791aa31..d948572bda56 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -21,6 +21,7 @@ impl Channel { #[derive(Default, Debug)] pub struct ConfigInfo { + pub target: String, pub target_triple: String, pub host_triple: String, pub rustc_command: Vec, @@ -42,6 +43,13 @@ impl ConfigInfo { args: &mut impl Iterator, ) -> Result { match arg { + "--target" => { + if let Some(arg) = args.next() { + self.target = arg; + } else { + return Err("Expected a value after `--target`, found nothing".to_string()); + } + } "--target-triple" => match args.next() { Some(arg) if !arg.is_empty() => self.target_triple = arg.to_string(), _ => { @@ -113,6 +121,9 @@ impl ConfigInfo { if self.target_triple.is_empty() { self.target_triple = self.host_triple.clone(); } + if self.target.is_empty() && !self.target_triple.is_empty() { + self.target = self.target_triple.clone(); + } let mut linker = None; @@ -243,6 +254,7 @@ impl ConfigInfo { println!( "\ --target-triple [arg] : Set the target triple to [arg] + --target [arg] : Set the target to [arg] --out-dir : Location where the files will be generated --release : Build in release mode --release-sysroot : Build sysroot in release mode diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 1e9652d28220..a926ee4c79eb 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -482,7 +482,7 @@ fn std_tests(env: &Env, args: &TestArg) -> Result<(), String> { &args.config_info.target_triple, ]); run_command_with_env(&command, None, Some(env))?; - // FIXME: the compiled binary is not run. Is it normal? + // FIXME: the compiled binary is not run. Ok(()) } From a8b0e30a8b5c9b97b83bdabd700c0f6e92e0f96d Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Mon, 18 Dec 2023 23:25:23 +0100 Subject: [PATCH 28/34] Error earlier if the rustc host cannot be found --- build_system/src/config.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/build_system/src/config.rs b/build_system/src/config.rs index d948572bda56..b61c65f7b022 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -111,7 +111,10 @@ impl ConfigInfo { Some(r) if !r.is_empty() => r.to_string(), _ => "rustc".to_string(), }; - self.host_triple = rustc_version_info(Some(&rustc))?.host.unwrap_or_default(); + self.host_triple = match rustc_version_info(Some(&rustc))?.host { + Some(host) => host, + None => return Err("no host found".to_string()), + }; if self.target_triple.is_empty() { if let Some(overwrite) = env.get("OVERWRITE_TARGET_TRIPLE") { @@ -216,6 +219,8 @@ impl ConfigInfo { )); let ld_library_path = format!( "{target}:{sysroot}:{gcc_path}", + // FIXME: It's possible to pick another out directory. Would be nice to have a command + // line option to change it. target = current_dir.join("target/out").display(), sysroot = sysroot.display(), ); From f516c9681133788c6ff20932e443226ffef98d5c Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 19 Dec 2023 01:07:01 +0100 Subject: [PATCH 29/34] Add comment about why `-Cpanic=abort` option is needed --- build_system/src/config.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build_system/src/config.rs b/build_system/src/config.rs index b61c65f7b022..1824bdd292ff 100644 --- a/build_system/src/config.rs +++ b/build_system/src/config.rs @@ -165,6 +165,8 @@ impl ConfigInfo { } _ => {} } + // This should not be needed, but is necessary for the CI in the rust repository. + // FIXME: Remove when the rust CI switches to the master version of libgccjit. rustflags.push("-Cpanic=abort".to_string()); } else { self.cg_backend_path = current_dir From bb4fd2c638fa31c0121255e8548488381ab39041 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 19 Dec 2023 01:16:29 +0100 Subject: [PATCH 30/34] Simplify code by removing unneeded pattern matching --- build_system/src/utils.rs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/build_system/src/utils.rs b/build_system/src/utils.rs index ebfa41c761ce..9c3a86ad68a2 100644 --- a/build_system/src/utils.rs +++ b/build_system/src/utils.rs @@ -1,4 +1,3 @@ -use std::borrow::Cow; use std::collections::HashMap; use std::ffi::OsStr; use std::fmt::Debug; @@ -52,18 +51,12 @@ fn check_exit_status( let stdout = String::from_utf8_lossy(&output.stdout); if !stdout.is_empty() { error.push_str("\n==== STDOUT ====\n"); - match stdout { - Cow::Owned(s) => error.push_str(&s), - Cow::Borrowed(s) => error.push_str(s), - } + error.push_str(&*stdout); } let stderr = String::from_utf8_lossy(&output.stderr); if !stderr.is_empty() { error.push_str("\n==== STDERR ====\n"); - match stderr { - Cow::Owned(s) => error.push_str(&s), - Cow::Borrowed(s) => error.push_str(s), - } + error.push_str(&*stderr); } } Err(error) From 984e045848ff28294d86471478286307942680da Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 19 Dec 2023 16:54:02 +0100 Subject: [PATCH 31/34] Show output of `--mini-tests` and `--std-tests` commands --- build_system/src/test.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_system/src/test.rs b/build_system/src/test.rs index a926ee4c79eb..13828e461913 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -304,7 +304,7 @@ fn maybe_run_command_in_vm( args: &TestArg, ) -> Result<(), String> { if !args.config_info.run_in_vm { - run_command_with_env(command, None, Some(env))?; + run_command_with_output_and_env(command, None, Some(env))?; return Ok(()); } let vm_parent_dir = match env.get("CG_GCC_VM_DIR") { @@ -330,7 +330,7 @@ fn maybe_run_command_in_vm( &inside_vm_exe_path, ]; vm_command.extend_from_slice(command); - run_command_with_env(&vm_command, Some(&vm_parent_dir), Some(env))?; + run_command_with_output_and_env(&vm_command, Some(&vm_parent_dir), Some(env))?; Ok(()) } From a46066ca230a421ef40bf1136ec030cc10afab84 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 19 Dec 2023 16:55:22 +0100 Subject: [PATCH 32/34] Remove ignored commands from gcc12 CI --- .github/workflows/gcc12.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/gcc12.yml b/.github/workflows/gcc12.yml index 1a17b936c743..a27ef913c21d 100644 --- a/.github/workflows/gcc12.yml +++ b/.github/workflows/gcc12.yml @@ -28,9 +28,6 @@ jobs: # FIXME: re-enable asm tests when GCC can emit in the right syntax. # "--asm-tests", "--test-libcore", - "--extended-rand-tests", - "--extended-regex-example-tests", - "--extended-regex-tests", "--test-successful-rustc --nb-parts 2 --current-part 0", "--test-successful-rustc --nb-parts 2 --current-part 1", ] From 6e53832eda190141516e0faeb4651eecfe3710be Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 19 Dec 2023 21:15:28 +0100 Subject: [PATCH 33/34] Simplify `Runner` type alias --- build_system/src/test.rs | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/build_system/src/test.rs b/build_system/src/test.rs index 13828e461913..f368e5b420e6 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -13,7 +13,7 @@ use std::path::{Path, PathBuf}; use std::str::FromStr; type Env = HashMap; -type Runner = &'static dyn Fn(&Env, &TestArg) -> Result<(), String>; +type Runner = fn(&Env, &TestArg) -> Result<(), String>; type Runners = HashMap<&'static str, (&'static str, Runner)>; fn get_runners() -> Runners { @@ -21,42 +21,42 @@ fn get_runners() -> Runners { runners.insert( "--test-rustc", - ("Run all rustc tests", &test_rustc as Runner), + ("Run all rustc tests", test_rustc as Runner), ); runners.insert( "--test-successful-rustc", - ("Run successful rustc tests", &test_successful_rustc), + ("Run successful rustc tests", test_successful_rustc), ); runners.insert( "--test-failing-rustc", - ("Run failing rustc tests", &test_failing_rustc), + ("Run failing rustc tests", test_failing_rustc), ); - runners.insert("--test-libcore", ("Run libcore tests", &test_libcore)); - runners.insert("--clean-ui-tests", ("Clean ui tests", &clean_ui_tests)); - runners.insert("--clean", ("Empty cargo target directory", &clean)); - runners.insert("--build-sysroot", ("Build sysroot", &build_sysroot)); - runners.insert("--std-tests", ("Run std tests", &std_tests)); - runners.insert("--asm-tests", ("Run asm tests", &asm_tests)); + runners.insert("--test-libcore", ("Run libcore tests", test_libcore)); + runners.insert("--clean-ui-tests", ("Clean ui tests", clean_ui_tests)); + runners.insert("--clean", ("Empty cargo target directory", clean)); + runners.insert("--build-sysroot", ("Build sysroot", build_sysroot)); + runners.insert("--std-tests", ("Run std tests", std_tests)); + runners.insert("--asm-tests", ("Run asm tests", asm_tests)); runners.insert( "--extended-tests", - ("Run extended sysroot tests", &extended_sysroot_tests), + ("Run extended sysroot tests", extended_sysroot_tests), ); runners.insert( "--extended-rand-tests", - ("Run extended rand tests", &extended_rand_tests), + ("Run extended rand tests", extended_rand_tests), ); runners.insert( "--extended-regex-example-tests", ( "Run extended regex example tests", - &extended_regex_example_tests, + extended_regex_example_tests, ), ); runners.insert( "--extended-regex-tests", - ("Run extended regex tests", &extended_regex_tests), + ("Run extended regex tests", extended_regex_tests), ); - runners.insert("--mini-tests", ("Run mini tests", &mini_tests)); + runners.insert("--mini-tests", ("Run mini tests", mini_tests)); runners } From 8e870c75d991b06a19a6ae8af45cef39a70ffae9 Mon Sep 17 00:00:00 2001 From: Guillaume Gomez Date: Tue, 19 Dec 2023 21:22:32 +0100 Subject: [PATCH 34/34] Remove unused `TestArgs::use_backend` and display messages in case a test is not run --- build_system/src/test.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/build_system/src/test.rs b/build_system/src/test.rs index f368e5b420e6..f66b16c31496 100644 --- a/build_system/src/test.rs +++ b/build_system/src/test.rs @@ -97,6 +97,7 @@ fn show_usage() { ); ConfigInfo::show_usage(); for (option, (doc, _)) in get_runners() { + // FIXME: Instead of using the hard-coded `23` value, better to compute it instead. let needed_spaces = 23_usize.saturating_sub(option.len()); let spaces: String = std::iter::repeat(' ').take(needed_spaces).collect(); println!(" {}{}: {}", option, spaces, doc); @@ -109,7 +110,6 @@ struct TestArg { no_default_features: bool, build_only: bool, gcc_path: String, - use_backend: bool, runners: BTreeSet, flags: Vec, backend: Option, @@ -207,7 +207,7 @@ impl TestArg { } fn build_if_no_backend(env: &Env, args: &TestArg) -> Result<(), String> { - if args.use_backend { + if args.backend.is_some() { return Ok(()); } let mut command: Vec<&dyn AsRef> = vec![&"cargo", &"rustc"]; @@ -504,8 +504,6 @@ fn setup_rustc(env: &mut Env, args: &TestArg) -> Result<(), String> { None => return Err("Couldn't retrieve rustc commit hash".to_string()), }; run_command_with_output_and_env(&[&"git", &"checkout", &rustc_commit], rust_dir, Some(env))?; - // FIXME: Is it really needed to empty `RUSTFLAGS` here? - // env.insert("RUSTFLAGS".to_string(), String::new()); let cargo = String::from_utf8( run_command_with_env(&[&"rustup", &"which", &"cargo"], rust_dir, Some(env))?.stdout, ) @@ -684,6 +682,7 @@ fn test_libcore(env: &Env, args: &TestArg) -> Result<(), String> { fn extended_rand_tests(env: &Env, args: &TestArg) -> Result<(), String> { if !args.is_using_gcc_master_branch() { + println!("Not using GCC master branch. Skipping `extended_rand_tests`."); return Ok(()); } let path = Path::new("rand"); @@ -696,6 +695,7 @@ fn extended_rand_tests(env: &Env, args: &TestArg) -> Result<(), String> { fn extended_regex_example_tests(env: &Env, args: &TestArg) -> Result<(), String> { if !args.is_using_gcc_master_branch() { + println!("Not using GCC master branch. Skipping `extended_regex_example_tests`."); return Ok(()); } let path = Path::new("regex"); @@ -750,6 +750,7 @@ fn extended_regex_example_tests(env: &Env, args: &TestArg) -> Result<(), String> fn extended_regex_tests(env: &Env, args: &TestArg) -> Result<(), String> { if !args.is_using_gcc_master_branch() { + println!("Not using GCC master branch. Skipping `extended_regex_tests`."); return Ok(()); } // FIXME: create a function "display_if_not_quiet" or something along the line.