user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Auto merge of #53245 - michaelwoerister:thinlto-rust-llvm, r=alexcrichton

Browse source 
[experimental]: Build LLVM with ThinLTO enabled (2nd attempt)

This is https://github.com/rust-lang/rust/pull/51207 revived. This time, I'd like to run actual performance tests to see if it improves compile times.
This commit is contained in:
bors 2018-08-29 10:55:27 +00:00
commit ba48850409
11 changed files with 144 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

10
config.toml.example
View file

@ -21,6 +21,12 @@
# Indicates whether the LLVM build is a Release or Debug build
#optimize = true
# Indicates whether LLVM should be built with ThinLTO. Note that this will
# only succeed if you use clang, lld, llvm-ar, and llvm-ranlib in your C/C++
# toolchain (see the `cc`, `cxx`, `linker`, `ar`, and `ranlib` options below).
# More info at: https://clang.llvm.org/docs/ThinLTO.html#clang-bootstrap
#thin-lto = false
# Indicates whether an LLVM Release build should include debug info
#release-debuginfo = false
@ -388,6 +394,10 @@
# Note: an absolute path should be used, otherwise LLVM build will break.
#ar = "ar"
# Ranlib to be used to assemble static libraries compiled from C/C++ code.
# Note: an absolute path should be used, otherwise LLVM build will break.
#ranlib = "ranlib"
# Linker to be used to link Rust code. Note that the
# default value is platform specific, and if not specified it may also depend on
# what platform is crossing to what platform.

4
src/bootstrap/builder.rs
View file

@ -827,7 +827,7 @@ impl<'a> Builder<'a> {
if let Some(ref error_format) = self.config.rustc_error_format {
cargo.env("RUSTC_ERROR_FORMAT", error_format);
}
if cmd != "build" && cmd != "check" && want_rustdoc {
if cmd != "build" && cmd != "check" && cmd != "rustc" && want_rustdoc {
cargo.env("RUSTDOC_LIBDIR", self.sysroot_libdir(compiler, self.config.build));
}
@ -988,7 +988,7 @@ impl<'a> Builder<'a> {
}
}
if cmd == "build"
if (cmd == "build" || cmd == "rustc")
&& mode == Mode::Std
&& self.config.extended
&& compiler.is_final_stage(self)

5
src/bootstrap/check.rs
View file

@ -50,6 +50,7 @@ impl Step for Std {
println!("Checking std artifacts ({} -> {})", &compiler.host, target);
run_cargo(builder,
&mut cargo,
vec![],
&libstd_stamp(builder, compiler, target),
true);
@ -98,6 +99,7 @@ impl Step for Rustc {
println!("Checking compiler artifacts ({} -> {})", &compiler.host, target);
run_cargo(builder,
&mut cargo,
vec![],
&librustc_stamp(builder, compiler, target),
true);
@ -149,6 +151,7 @@ impl Step for CodegenBackend {
let _folder = builder.fold_output(|| format!("stage{}-rustc_codegen_llvm", compiler.stage));
run_cargo(builder,
&mut cargo,
vec![],
&codegen_backend_stamp(builder, compiler, target, backend),
true);
}
@ -187,6 +190,7 @@ impl Step for Test {
println!("Checking test artifacts ({} -> {})", &compiler.host, target);
run_cargo(builder,
&mut cargo,
vec![],
&libtest_stamp(builder, compiler, target),
true);
@ -236,6 +240,7 @@ impl Step for Rustdoc {
println!("Checking rustdoc artifacts ({} -> {})", &compiler.host, target);
run_cargo(builder,
&mut cargo,
vec![],
&rustdoc_stamp(builder, compiler, target),
true);

47
src/bootstrap/compile.rs
View file

@ -117,6 +117,7 @@ impl Step for Std {
&compiler.host, target));
run_cargo(builder,
&mut cargo,
vec![],
&libstd_stamp(builder, compiler, target),
false);
@ -396,6 +397,7 @@ impl Step for Test {
&compiler.host, target));
run_cargo(builder,
&mut cargo,
vec![],
&libtest_stamp(builder, compiler, target),
false);
@ -529,6 +531,7 @@ impl Step for Rustc {
compiler.stage, &compiler.host, target));
run_cargo(builder,
&mut cargo,
vec![],
&librustc_stamp(builder, compiler, target),
false);
@ -673,18 +676,47 @@ impl Step for CodegenBackend {
let out_dir = builder.cargo_out(compiler, Mode::Codegen, target);
builder.clear_if_dirty(&out_dir, &librustc_stamp(builder, compiler, target));
let mut cargo = builder.cargo(compiler, Mode::Codegen, target, "build");
let mut cargo = builder.cargo(compiler, Mode::Codegen, target, "rustc");
cargo.arg("--manifest-path")
.arg(builder.src.join("src/librustc_codegen_llvm/Cargo.toml"));
rustc_cargo_env(builder, &mut cargo);
let features = build_codegen_backend(&builder, &mut cargo, &compiler, target, backend);
let mut cargo_tails_args = vec![];
if builder.config.llvm_thin_lto {
cargo_tails_args.push("--".to_string());
let num_jobs = builder.jobs();
if !target.contains("msvc") {
// Here we assume that the linker is clang. If it's not, there'll
// be linker errors.
cargo_tails_args.push("-Clink-arg=-fuse-ld=lld".to_string());
cargo_tails_args.push("-Clink-arg=-flto=thin".to_string());
if builder.config.llvm_optimize {
cargo_tails_args.push("-Clink-arg=-O2".to_string());
}
// Let's make LLD respect the `-j` option.
let num_jobs_arg = format!("-Clink-arg=-Wl,--thinlto-jobs={}", num_jobs);
cargo_tails_args.push(num_jobs_arg);
} else {
// Here we assume that the linker is lld-link.exe. lld-link.exe
// does not need the extra arguments except for num_jobs
let num_jobs_arg = format!("-Clink-arg=/opt:lldltojobs={}", num_jobs);
cargo_tails_args.push(num_jobs_arg);
}
}
let tmp_stamp = out_dir.join(".tmp.stamp");
let _folder = builder.fold_output(|| format!("stage{}-rustc_codegen_llvm", compiler.stage));
let files = run_cargo(builder,
cargo.arg("--features").arg(features),
cargo_tails_args,
&tmp_stamp,
false);
if builder.config.dry_run {
@ -1045,7 +1077,11 @@ fn stderr_isatty() -> bool {
}
}
pub fn run_cargo(builder: &Builder, cargo: &mut Command, stamp: &Path, is_check: bool)
pub fn run_cargo(builder: &Builder,
cargo: &mut Command,
tail_args: Vec<String>,
stamp: &Path,
is_check: bool)
-> Vec<PathBuf>
{
if builder.config.dry_run {
@ -1066,7 +1102,7 @@ pub fn run_cargo(builder: &Builder, cargo: &mut Command, stamp: &Path, is_check:
// files we need to probe for later.
let mut deps = Vec::new();
let mut toplevel = Vec::new();
let ok = stream_cargo(builder, cargo, &mut |msg| {
let ok = stream_cargo(builder, cargo, tail_args, &mut |msg| {
let filenames = match msg {
CargoMessage::CompilerArtifact { filenames, .. } => filenames,
_ => return,
@ -1191,6 +1227,7 @@ pub fn run_cargo(builder: &Builder, cargo: &mut Command, stamp: &Path, is_check:
pub fn stream_cargo(
builder: &Builder,
cargo: &mut Command,
tail_args: Vec<String>,
cb: &mut dyn FnMut(CargoMessage),
) -> bool {
if builder.config.dry_run {
@ -1210,6 +1247,10 @@ pub fn stream_cargo(
cargo.env("RUSTC_COLOR", "1");
}
for arg in tail_args {
cargo.arg(arg);
}
builder.verbose(&format!("running: {:?}", cargo));
let mut child = match cargo.spawn() {
Ok(child) => child,

6
src/bootstrap/config.rs
View file

@ -77,6 +77,7 @@ pub struct Config {
pub llvm_enabled: bool,
pub llvm_assertions: bool,
pub llvm_optimize: bool,
pub llvm_thin_lto: bool,
pub llvm_release_debuginfo: bool,
pub llvm_version_check: bool,
pub llvm_static_stdcpp: bool,
@ -163,6 +164,7 @@ pub struct Target {
pub cc: Option<PathBuf>,
pub cxx: Option<PathBuf>,
pub ar: Option<PathBuf>,
pub ranlib: Option<PathBuf>,
pub linker: Option<PathBuf>,
pub ndk: Option<PathBuf>,
pub crt_static: Option<bool>,
@ -246,6 +248,7 @@ struct Llvm {
ninja: Option<bool>,
assertions: Option<bool>,
optimize: Option<bool>,
thin_lto: Option<bool>,
release_debuginfo: Option<bool>,
version_check: Option<bool>,
static_libstdcpp: Option<bool>,
@ -327,6 +330,7 @@ struct TomlTarget {
cc: Option<String>,
cxx: Option<String>,
ar: Option<String>,
ranlib: Option<String>,
linker: Option<String>,
android_ndk: Option<String>,
crt_static: Option<bool>,
@ -503,6 +507,7 @@ impl Config {
set(&mut config.llvm_enabled, llvm.enabled);
llvm_assertions = llvm.assertions;
set(&mut config.llvm_optimize, llvm.optimize);
set(&mut config.llvm_thin_lto, llvm.thin_lto);
set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo);
set(&mut config.llvm_version_check, llvm.version_check);
set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp);
@ -581,6 +586,7 @@ impl Config {
target.cc = cfg.cc.clone().map(PathBuf::from);
target.cxx = cfg.cxx.clone().map(PathBuf::from);
target.ar = cfg.ar.clone().map(PathBuf::from);
target.ranlib = cfg.ranlib.clone().map(PathBuf::from);
target.linker = cfg.linker.clone().map(PathBuf::from);
target.crt_static = cfg.crt_static.clone();
target.musl_root = cfg.musl_root.clone().map(PathBuf::from);

46
src/bootstrap/dist.rs
View file

@ -1885,6 +1885,34 @@ impl Step for HashSign {
}
}
// Maybe add libLLVM.so to the lib-dir. It will only have been built if
// LLVM tools are linked dynamically.
// Note: This function does no yet support Windows but we also don't support
// linking LLVM tools dynamically on Windows yet.
fn maybe_install_llvm_dylib(builder: &Builder,
target: Interned<String>,
image: &Path) {
let src_libdir = builder
.llvm_out(target)
.join("lib");
// Usually libLLVM.so is a symlink to something like libLLVM-6.0.so.
// Since tools link to the latter rather than the former, we have to
// follow the symlink to find out what to distribute.
let llvm_dylib_path = src_libdir.join("libLLVM.so");
if llvm_dylib_path.exists() {
let llvm_dylib_path = llvm_dylib_path.canonicalize().unwrap_or_else(|e| {
panic!("dist: Error calling canonicalize path `{}`: {}",
llvm_dylib_path.display(), e);
});
let dst_libdir = image.join("lib");
t!(fs::create_dir_all(&dst_libdir));
builder.install(&llvm_dylib_path, &dst_libdir, 0o644);
}
}
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct LlvmTools {
pub stage: u32,
@ -1929,18 +1957,18 @@ impl Step for LlvmTools {
drop(fs::remove_dir_all(&image));
// Prepare the image directory
let bindir = builder
let src_bindir = builder
.llvm_out(target)
.join("bin");
let dst = image.join("lib/rustlib")
.join(target)
.join("bin");
t!(fs::create_dir_all(&dst));
let dst_bindir = image.join("bin");
t!(fs::create_dir_all(&dst_bindir));
for tool in LLVM_TOOLS {
let exe = bindir.join(exe(tool, &target));
builder.install(&exe, &dst, 0o755);
let exe = src_bindir.join(exe(tool, &target));
builder.install(&exe, &dst_bindir, 0o755);
}
maybe_install_llvm_dylib(builder, target, &image);
// Prepare the overlay
let overlay = tmp.join("llvm-tools-overlay");
drop(fs::remove_dir_all(&overlay));
@ -2025,7 +2053,6 @@ impl Step for Lldb {
let dst = image.join("lib");
t!(fs::create_dir_all(&dst));
for entry in t!(fs::read_dir(&libdir)) {
// let entry = t!(entry);
let entry = entry.unwrap();
if let Ok(name) = entry.file_name().into_string() {
if name.starts_with("liblldb.") && !name.ends_with(".a") {
@ -2060,6 +2087,9 @@ impl Step for Lldb {
}
}
// Copy libLLVM.so to the lib dir as well, if needed.
maybe_install_llvm_dylib(builder, target, &image);
// Prepare the overlay
let overlay = tmp.join("lldb-overlay");
drop(fs::remove_dir_all(&overlay));

11
src/bootstrap/lib.rs
View file

@ -281,6 +281,7 @@ pub struct Build {
cc: HashMap<Interned<String>, cc::Tool>,
cxx: HashMap<Interned<String>, cc::Tool>,
ar: HashMap<Interned<String>, PathBuf>,
ranlib: HashMap<Interned<String>, PathBuf>,
// Misc
crates: HashMap<Interned<String>, Crate>,
is_sudo: bool,
@ -406,6 +407,7 @@ impl Build {
cc: HashMap::new(),
cxx: HashMap::new(),
ar: HashMap::new(),
ranlib: HashMap::new(),
crates: HashMap::new(),
lldb_version: None,
lldb_python_dir: None,
@ -772,6 +774,11 @@ impl Build {
self.ar.get(&target).map(|p| &**p)
}
/// Returns the path to the `ranlib` utility for the target specified.
fn ranlib(&self, target: Interned<String>) -> Option<&Path> {
self.ranlib.get(&target).map(|p| &**p)
}
/// Returns the path to the C++ compiler for the target specified.
fn cxx(&self, target: Interned<String>) -> Result<&Path, String> {
match self.cxx.get(&target) {
@ -1018,6 +1025,10 @@ impl Build {
self.rust_version()
}
fn llvm_link_tools_dynamically(&self, target: Interned<String>) -> bool {
(target.contains("linux-gnu") || target.contains("apple-darwin"))
}
/// Returns the `version` string associated with this compiler for Rust
/// itself.
///

25
src/bootstrap/native.rs
View file

@ -153,6 +153,11 @@ impl Step for Llvm {
.define("LLVM_TARGET_ARCH", target.split('-').next().unwrap())
.define("LLVM_DEFAULT_TARGET_TRIPLE", target);
if builder.config.llvm_thin_lto && !emscripten {
cfg.define("LLVM_ENABLE_LTO", "Thin")
.define("LLVM_ENABLE_LLD", "ON");
}
// By default, LLVM will automatically find OCaml and, if it finds it,
// install the LLVM bindings in LLVM_OCAML_INSTALL_PATH, which defaults
// to /usr/bin/ocaml.
@ -166,14 +171,10 @@ impl Step for Llvm {
// This setting makes the LLVM tools link to the dynamic LLVM library,
// which saves both memory during parallel links and overall disk space
// for the tools. We don't distribute any of those tools, so this is
// just a local concern. However, it doesn't work well everywhere.
//
// If we are shipping llvm tools then we statically link them LLVM
if (target.contains("linux-gnu") || target.contains("apple-darwin")) &&
!builder.config.llvm_tools_enabled &&
!want_lldb {
cfg.define("LLVM_LINK_LLVM_DYLIB", "ON");
// for the tools. We don't do this on every platform as it doesn't work
// equally well everywhere.
if builder.llvm_link_tools_dynamically(target) && !emscripten {
cfg.define("LLVM_LINK_LLVM_DYLIB", "ON");
}
// For distribution we want the LLVM tools to be *statically* linked to libstdc++
@ -379,6 +380,14 @@ fn configure_cmake(builder: &Builder,
}
}
if let Some(ranlib) = builder.ranlib(target) {
if ranlib.is_absolute() {
// LLVM build breaks if `CMAKE_RANLIB` is a relative path, for some reason it
// tries to resolve this path in the LLVM build directory.
cfg.define("CMAKE_RANLIB", sanitize_cc(ranlib));
}
}
if env::var_os("SCCACHE_ERROR_LOG").is_some() {
cfg.env("RUST_LOG", "sccache=warn");
}

2
src/bootstrap/tool.rs
View file

@ -136,7 +136,7 @@ impl Step for ToolBuild {
let _folder = builder.fold_output(|| format!("stage{}-{}", compiler.stage, tool));
builder.info(&format!("Building stage{} tool {} ({})", compiler.stage, tool, target));
let mut duplicates = Vec::new();
let is_expected = compile::stream_cargo(builder, &mut cargo, &mut |msg| {
let is_expected = compile::stream_cargo(builder, &mut cargo, vec![], &mut |msg| {
// Only care about big things like the RLS/Cargo for now
match tool {
| "rls"

5
src/ci/docker/dist-x86_64-linux/Dockerfile
View file

@ -93,7 +93,10 @@ ENV RUST_CONFIGURE_ARGS \
--enable-sanitizers \
--enable-profiler \
--enable-compiler-docs \
--set target.x86_64-unknown-linux-gnu.linker=clang
--set target.x86_64-unknown-linux-gnu.linker=clang \
--set target.x86_64-unknown-linux-gnu.ar=/rustroot/bin/llvm-ar \
--set target.x86_64-unknown-linux-gnu.ranlib=/rustroot/bin/llvm-ranlib \
--set llvm.thin-lto=true
ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS
ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang

6
src/ci/docker/dist-x86_64-linux/build-clang.sh
View file

@ -30,6 +30,12 @@ curl https://releases.llvm.org/$LLVM/cfe-$LLVM.src.tar.xz | \
xz -d | \
tar xf - -C tools/clang --strip-components=1
mkdir -p tools/lld
curl https://releases.llvm.org/$LLVM/lld-$LLVM.src.tar.xz | \
xz -d | \
tar xf - -C tools/lld --strip-components=1
mkdir ../clang-build
cd ../clang-build