Auto merge of #144062 - bjorn3:lto_refactors2, r=davidtwco
Various refactors to the LTO handling code (part 2) Continuing from https://github.com/rust-lang/rust/pull/143388 this removes a bit of dead code and moves the LTO symbol export calculation from individual backends to cg_ssa.
This commit is contained in:
commit
5d22242a3a
13 changed files with 257 additions and 331 deletions
|
|
@ -3,12 +3,4 @@ codegen_gcc_unwinding_inline_asm =
|
|||
|
||||
codegen_gcc_copy_bitcode = failed to copy bitcode to object file: {$err}
|
||||
|
||||
codegen_gcc_dynamic_linking_with_lto =
|
||||
cannot prefer dynamic linking when performing LTO
|
||||
.note = only 'staticlib', 'bin', and 'cdylib' outputs are supported with LTO
|
||||
|
||||
codegen_gcc_lto_disallowed = lto can only be run for executables, cdylibs and static library outputs
|
||||
|
||||
codegen_gcc_lto_dylib = lto cannot be used for `dylib` crate type without `-Zdylib-lto`
|
||||
|
||||
codegen_gcc_lto_bitcode_from_rlib = failed to get bitcode from object file for LTO ({$gcc_err})
|
||||
|
|
|
|||
|
|
@ -25,35 +25,21 @@ use std::sync::Arc;
|
|||
use gccjit::{Context, OutputKind};
|
||||
use object::read::archive::ArchiveFile;
|
||||
use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule, ThinShared};
|
||||
use rustc_codegen_ssa::back::symbol_export;
|
||||
use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput};
|
||||
use rustc_codegen_ssa::traits::*;
|
||||
use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, looks_like_rust_object_file};
|
||||
use rustc_data_structures::memmap::Mmap;
|
||||
use rustc_errors::{DiagCtxtHandle, FatalError};
|
||||
use rustc_hir::def_id::LOCAL_CRATE;
|
||||
use rustc_middle::bug;
|
||||
use rustc_middle::dep_graph::WorkProduct;
|
||||
use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel};
|
||||
use rustc_session::config::{CrateType, Lto};
|
||||
use rustc_session::config::Lto;
|
||||
use rustc_target::spec::RelocModel;
|
||||
use tempfile::{TempDir, tempdir};
|
||||
|
||||
use crate::back::write::save_temp_bitcode;
|
||||
use crate::errors::{DynamicLinkingWithLTO, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib};
|
||||
use crate::errors::LtoBitcodeFromRlib;
|
||||
use crate::{GccCodegenBackend, GccContext, SyncContext, to_gcc_opt_level};
|
||||
|
||||
pub fn crate_type_allows_lto(crate_type: CrateType) -> bool {
|
||||
match crate_type {
|
||||
CrateType::Executable
|
||||
| CrateType::Dylib
|
||||
| CrateType::Staticlib
|
||||
| CrateType::Cdylib
|
||||
| CrateType::Sdylib => true,
|
||||
CrateType::Rlib | CrateType::ProcMacro => false,
|
||||
}
|
||||
}
|
||||
|
||||
struct LtoData {
|
||||
// TODO(antoyo): use symbols_below_threshold.
|
||||
//symbols_below_threshold: Vec<String>,
|
||||
|
|
@ -63,18 +49,9 @@ struct LtoData {
|
|||
|
||||
fn prepare_lto(
|
||||
cgcx: &CodegenContext<GccCodegenBackend>,
|
||||
each_linked_rlib_for_lto: &[PathBuf],
|
||||
dcx: DiagCtxtHandle<'_>,
|
||||
) -> Result<LtoData, FatalError> {
|
||||
let export_threshold = match cgcx.lto {
|
||||
// We're just doing LTO for our one crate
|
||||
Lto::ThinLocal => SymbolExportLevel::Rust,
|
||||
|
||||
// We're doing LTO for the entire crate graph
|
||||
Lto::Fat | Lto::Thin => symbol_export::crates_export_threshold(&cgcx.crate_types),
|
||||
|
||||
Lto::No => panic!("didn't request LTO but we're doing LTO"),
|
||||
};
|
||||
|
||||
let tmp_path = match tempdir() {
|
||||
Ok(tmp_path) => tmp_path,
|
||||
Err(error) => {
|
||||
|
|
@ -83,20 +60,6 @@ fn prepare_lto(
|
|||
}
|
||||
};
|
||||
|
||||
let symbol_filter = &|&(ref name, info): &(String, SymbolExportInfo)| {
|
||||
if info.level.is_below_threshold(export_threshold) || info.used {
|
||||
Some(name.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
let exported_symbols = cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
|
||||
let mut symbols_below_threshold = {
|
||||
let _timer = cgcx.prof.generic_activity("GCC_lto_generate_symbols_below_threshold");
|
||||
exported_symbols[&LOCAL_CRATE].iter().filter_map(symbol_filter).collect::<Vec<String>>()
|
||||
};
|
||||
info!("{} symbols to preserve in this crate", symbols_below_threshold.len());
|
||||
|
||||
// If we're performing LTO for the entire crate graph, then for each of our
|
||||
// upstream dependencies, find the corresponding rlib and load the bitcode
|
||||
// from the archive.
|
||||
|
|
@ -105,32 +68,7 @@ fn prepare_lto(
|
|||
// with either fat or thin LTO
|
||||
let mut upstream_modules = Vec::new();
|
||||
if cgcx.lto != Lto::ThinLocal {
|
||||
// Make sure we actually can run LTO
|
||||
for crate_type in cgcx.crate_types.iter() {
|
||||
if !crate_type_allows_lto(*crate_type) {
|
||||
dcx.emit_err(LtoDisallowed);
|
||||
return Err(FatalError);
|
||||
}
|
||||
if *crate_type == CrateType::Dylib && !cgcx.opts.unstable_opts.dylib_lto {
|
||||
dcx.emit_err(LtoDylib);
|
||||
return Err(FatalError);
|
||||
}
|
||||
}
|
||||
|
||||
if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto {
|
||||
dcx.emit_err(DynamicLinkingWithLTO);
|
||||
return Err(FatalError);
|
||||
}
|
||||
|
||||
for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() {
|
||||
let exported_symbols =
|
||||
cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
|
||||
{
|
||||
let _timer = cgcx.prof.generic_activity("GCC_lto_generate_symbols_below_threshold");
|
||||
symbols_below_threshold
|
||||
.extend(exported_symbols[&cnum].iter().filter_map(symbol_filter));
|
||||
}
|
||||
|
||||
for path in each_linked_rlib_for_lto {
|
||||
let archive_data = unsafe {
|
||||
Mmap::map(File::open(path).expect("couldn't open rlib")).expect("couldn't map rlib")
|
||||
};
|
||||
|
|
@ -174,19 +112,18 @@ fn save_as_file(obj: &[u8], path: &Path) -> Result<(), LtoBitcodeFromRlib> {
|
|||
/// for further optimization.
|
||||
pub(crate) fn run_fat(
|
||||
cgcx: &CodegenContext<GccCodegenBackend>,
|
||||
each_linked_rlib_for_lto: &[PathBuf],
|
||||
modules: Vec<FatLtoInput<GccCodegenBackend>>,
|
||||
cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
|
||||
) -> Result<ModuleCodegen<GccContext>, FatalError> {
|
||||
let dcx = cgcx.create_dcx();
|
||||
let dcx = dcx.handle();
|
||||
let lto_data = prepare_lto(cgcx, dcx)?;
|
||||
let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx)?;
|
||||
/*let symbols_below_threshold =
|
||||
lto_data.symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();*/
|
||||
fat_lto(
|
||||
cgcx,
|
||||
dcx,
|
||||
modules,
|
||||
cached_modules,
|
||||
lto_data.upstream_modules,
|
||||
lto_data.tmp_path,
|
||||
//<o_data.symbols_below_threshold,
|
||||
|
|
@ -197,7 +134,6 @@ fn fat_lto(
|
|||
cgcx: &CodegenContext<GccCodegenBackend>,
|
||||
_dcx: DiagCtxtHandle<'_>,
|
||||
modules: Vec<FatLtoInput<GccCodegenBackend>>,
|
||||
cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
|
||||
mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
|
||||
tmp_path: TempDir,
|
||||
//symbols_below_threshold: &[String],
|
||||
|
|
@ -211,21 +147,12 @@ fn fat_lto(
|
|||
// modules that are serialized in-memory.
|
||||
// * `in_memory` contains modules which are already parsed and in-memory,
|
||||
// such as from multi-CGU builds.
|
||||
//
|
||||
// All of `cached_modules` (cached from previous incremental builds) can
|
||||
// immediately go onto the `serialized_modules` modules list and then we can
|
||||
// split the `modules` array into these two lists.
|
||||
let mut in_memory = Vec::new();
|
||||
serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| {
|
||||
info!("pushing cached module {:?}", wp.cgu_name);
|
||||
(buffer, CString::new(wp.cgu_name).unwrap())
|
||||
}));
|
||||
for module in modules {
|
||||
match module {
|
||||
FatLtoInput::InMemory(m) => in_memory.push(m),
|
||||
FatLtoInput::Serialized { name, buffer } => {
|
||||
info!("pushing serialized module {:?}", name);
|
||||
let buffer = SerializedModule::Local(buffer);
|
||||
serialized_modules.push((buffer, CString::new(name).unwrap()));
|
||||
}
|
||||
}
|
||||
|
|
@ -356,12 +283,13 @@ impl ModuleBufferMethods for ModuleBuffer {
|
|||
/// can simply be copied over from the incr. comp. cache.
|
||||
pub(crate) fn run_thin(
|
||||
cgcx: &CodegenContext<GccCodegenBackend>,
|
||||
each_linked_rlib_for_lto: &[PathBuf],
|
||||
modules: Vec<(String, ThinBuffer)>,
|
||||
cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
|
||||
) -> Result<(Vec<ThinModule<GccCodegenBackend>>, Vec<WorkProduct>), FatalError> {
|
||||
let dcx = cgcx.create_dcx();
|
||||
let dcx = dcx.handle();
|
||||
let lto_data = prepare_lto(cgcx, dcx)?;
|
||||
let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx)?;
|
||||
if cgcx.opts.cg.linker_plugin_lto.enabled() {
|
||||
unreachable!(
|
||||
"We should never reach this case if the LTO step \
|
||||
|
|
|
|||
|
|
@ -14,19 +14,6 @@ pub(crate) struct CopyBitcode {
|
|||
pub err: std::io::Error,
|
||||
}
|
||||
|
||||
#[derive(Diagnostic)]
|
||||
#[diag(codegen_gcc_dynamic_linking_with_lto)]
|
||||
#[note]
|
||||
pub(crate) struct DynamicLinkingWithLTO;
|
||||
|
||||
#[derive(Diagnostic)]
|
||||
#[diag(codegen_gcc_lto_disallowed)]
|
||||
pub(crate) struct LtoDisallowed;
|
||||
|
||||
#[derive(Diagnostic)]
|
||||
#[diag(codegen_gcc_lto_dylib)]
|
||||
pub(crate) struct LtoDylib;
|
||||
|
||||
#[derive(Diagnostic)]
|
||||
#[diag(codegen_gcc_lto_bitcode_from_rlib)]
|
||||
pub(crate) struct LtoBitcodeFromRlib {
|
||||
|
|
|
|||
|
|
@ -81,6 +81,7 @@ mod type_of;
|
|||
use std::any::Any;
|
||||
use std::fmt::Debug;
|
||||
use std::ops::Deref;
|
||||
use std::path::PathBuf;
|
||||
#[cfg(not(feature = "master"))]
|
||||
use std::sync::atomic::AtomicBool;
|
||||
#[cfg(not(feature = "master"))]
|
||||
|
|
@ -358,23 +359,28 @@ impl WriteBackendMethods for GccCodegenBackend {
|
|||
|
||||
fn run_and_optimize_fat_lto(
|
||||
cgcx: &CodegenContext<Self>,
|
||||
// FIXME(bjorn3): Limit LTO exports to these symbols
|
||||
_exported_symbols_for_lto: &[String],
|
||||
each_linked_rlib_for_lto: &[PathBuf],
|
||||
modules: Vec<FatLtoInput<Self>>,
|
||||
cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
|
||||
diff_fncs: Vec<AutoDiffItem>,
|
||||
) -> Result<ModuleCodegen<Self::Module>, FatalError> {
|
||||
if !diff_fncs.is_empty() {
|
||||
unimplemented!();
|
||||
}
|
||||
|
||||
back::lto::run_fat(cgcx, modules, cached_modules)
|
||||
back::lto::run_fat(cgcx, each_linked_rlib_for_lto, modules)
|
||||
}
|
||||
|
||||
fn run_thin_lto(
|
||||
cgcx: &CodegenContext<Self>,
|
||||
// FIXME(bjorn3): Limit LTO exports to these symbols
|
||||
_exported_symbols_for_lto: &[String],
|
||||
each_linked_rlib_for_lto: &[PathBuf],
|
||||
modules: Vec<(String, Self::ThinBuffer)>,
|
||||
cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
|
||||
) -> Result<(Vec<ThinModule<Self>>, Vec<WorkProduct>), FatalError> {
|
||||
back::lto::run_thin(cgcx, modules, cached_modules)
|
||||
back::lto::run_thin(cgcx, each_linked_rlib_for_lto, modules, cached_modules)
|
||||
}
|
||||
|
||||
fn print_pass_timings(&self) {
|
||||
|
|
|
|||
|
|
@ -2,10 +2,6 @@ codegen_llvm_autodiff_without_enable = using the autodiff feature requires -Z au
|
|||
|
||||
codegen_llvm_copy_bitcode = failed to copy bitcode to object file: {$err}
|
||||
|
||||
codegen_llvm_dynamic_linking_with_lto =
|
||||
cannot prefer dynamic linking when performing LTO
|
||||
.note = only 'staticlib', 'bin', and 'cdylib' outputs are supported with LTO
|
||||
|
||||
|
||||
codegen_llvm_fixed_x18_invalid_arch = the `-Zfixed-x18` flag is not supported on the `{$arch}` architecture
|
||||
|
||||
|
|
@ -18,12 +14,6 @@ codegen_llvm_load_bitcode_with_llvm_err = failed to load bitcode of module "{$na
|
|||
|
||||
codegen_llvm_lto_bitcode_from_rlib = failed to get bitcode from object file for LTO ({$llvm_err})
|
||||
|
||||
codegen_llvm_lto_disallowed = lto can only be run for executables, cdylibs and static library outputs
|
||||
|
||||
codegen_llvm_lto_dylib = lto cannot be used for `dylib` crate type without `-Zdylib-lto`
|
||||
|
||||
codegen_llvm_lto_proc_macro = lto cannot be used for `proc-macro` crate type without `-Zdylib-lto`
|
||||
|
||||
codegen_llvm_mismatch_data_layout =
|
||||
data-layout for target `{$rustc_target}`, `{$rustc_layout}`, differs from LLVM target's `{$llvm_target}` default layout, `{$llvm_layout}`
|
||||
|
||||
|
|
|
|||
|
|
@ -1,33 +1,28 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::ffi::{CStr, CString};
|
||||
use std::fs::File;
|
||||
use std::path::Path;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::ptr::NonNull;
|
||||
use std::sync::Arc;
|
||||
use std::{io, iter, slice};
|
||||
|
||||
use object::read::archive::ArchiveFile;
|
||||
use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule, ThinShared};
|
||||
use rustc_codegen_ssa::back::symbol_export;
|
||||
use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput};
|
||||
use rustc_codegen_ssa::traits::*;
|
||||
use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, looks_like_rust_object_file};
|
||||
use rustc_data_structures::fx::FxHashMap;
|
||||
use rustc_data_structures::memmap::Mmap;
|
||||
use rustc_errors::{DiagCtxtHandle, FatalError};
|
||||
use rustc_hir::def_id::LOCAL_CRATE;
|
||||
use rustc_middle::bug;
|
||||
use rustc_middle::dep_graph::WorkProduct;
|
||||
use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel};
|
||||
use rustc_session::config::{self, CrateType, Lto};
|
||||
use rustc_session::config::{self, Lto};
|
||||
use tracing::{debug, info};
|
||||
|
||||
use crate::back::write::{
|
||||
self, CodegenDiagnosticsStage, DiagnosticHandlers, bitcode_section_name, save_temp_bitcode,
|
||||
};
|
||||
use crate::errors::{
|
||||
DynamicLinkingWithLTO, LlvmError, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib, LtoProcMacro,
|
||||
};
|
||||
use crate::errors::{LlvmError, LtoBitcodeFromRlib};
|
||||
use crate::llvm::AttributePlace::Function;
|
||||
use crate::llvm::{self, build_string};
|
||||
use crate::{LlvmCodegenBackend, ModuleLlvm, SimpleCx, attributes};
|
||||
|
|
@ -36,45 +31,21 @@ use crate::{LlvmCodegenBackend, ModuleLlvm, SimpleCx, attributes};
|
|||
/// session to determine which CGUs we can reuse.
|
||||
const THIN_LTO_KEYS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-keys.bin";
|
||||
|
||||
fn crate_type_allows_lto(crate_type: CrateType) -> bool {
|
||||
match crate_type {
|
||||
CrateType::Executable
|
||||
| CrateType::Dylib
|
||||
| CrateType::Staticlib
|
||||
| CrateType::Cdylib
|
||||
| CrateType::ProcMacro
|
||||
| CrateType::Sdylib => true,
|
||||
CrateType::Rlib => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn prepare_lto(
|
||||
cgcx: &CodegenContext<LlvmCodegenBackend>,
|
||||
exported_symbols_for_lto: &[String],
|
||||
each_linked_rlib_for_lto: &[PathBuf],
|
||||
dcx: DiagCtxtHandle<'_>,
|
||||
) -> Result<(Vec<CString>, Vec<(SerializedModule<ModuleBuffer>, CString)>), FatalError> {
|
||||
let export_threshold = match cgcx.lto {
|
||||
// We're just doing LTO for our one crate
|
||||
Lto::ThinLocal => SymbolExportLevel::Rust,
|
||||
let mut symbols_below_threshold = exported_symbols_for_lto
|
||||
.iter()
|
||||
.map(|symbol| CString::new(symbol.to_owned()).unwrap())
|
||||
.collect::<Vec<CString>>();
|
||||
|
||||
// We're doing LTO for the entire crate graph
|
||||
Lto::Fat | Lto::Thin => symbol_export::crates_export_threshold(&cgcx.crate_types),
|
||||
|
||||
Lto::No => panic!("didn't request LTO but we're doing LTO"),
|
||||
};
|
||||
|
||||
let symbol_filter = &|&(ref name, info): &(String, SymbolExportInfo)| {
|
||||
if info.level.is_below_threshold(export_threshold) || info.used {
|
||||
Some(CString::new(name.as_str()).unwrap())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
let exported_symbols = cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
|
||||
let mut symbols_below_threshold = {
|
||||
let _timer = cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold");
|
||||
exported_symbols[&LOCAL_CRATE].iter().filter_map(symbol_filter).collect::<Vec<CString>>()
|
||||
};
|
||||
info!("{} symbols to preserve in this crate", symbols_below_threshold.len());
|
||||
// __llvm_profile_counter_bias is pulled in at link time by an undefined reference to
|
||||
// __llvm_profile_runtime, therefore we won't know until link time if this symbol
|
||||
// should have default visibility.
|
||||
symbols_below_threshold.push(c"__llvm_profile_counter_bias".to_owned());
|
||||
|
||||
// If we're performing LTO for the entire crate graph, then for each of our
|
||||
// upstream dependencies, find the corresponding rlib and load the bitcode
|
||||
|
|
@ -84,37 +55,7 @@ fn prepare_lto(
|
|||
// with either fat or thin LTO
|
||||
let mut upstream_modules = Vec::new();
|
||||
if cgcx.lto != Lto::ThinLocal {
|
||||
// Make sure we actually can run LTO
|
||||
for crate_type in cgcx.crate_types.iter() {
|
||||
if !crate_type_allows_lto(*crate_type) {
|
||||
dcx.emit_err(LtoDisallowed);
|
||||
return Err(FatalError);
|
||||
} else if *crate_type == CrateType::Dylib {
|
||||
if !cgcx.opts.unstable_opts.dylib_lto {
|
||||
dcx.emit_err(LtoDylib);
|
||||
return Err(FatalError);
|
||||
}
|
||||
} else if *crate_type == CrateType::ProcMacro && !cgcx.opts.unstable_opts.dylib_lto {
|
||||
dcx.emit_err(LtoProcMacro);
|
||||
return Err(FatalError);
|
||||
}
|
||||
}
|
||||
|
||||
if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto {
|
||||
dcx.emit_err(DynamicLinkingWithLTO);
|
||||
return Err(FatalError);
|
||||
}
|
||||
|
||||
for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() {
|
||||
let exported_symbols =
|
||||
cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
|
||||
{
|
||||
let _timer =
|
||||
cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold");
|
||||
symbols_below_threshold
|
||||
.extend(exported_symbols[&cnum].iter().filter_map(symbol_filter));
|
||||
}
|
||||
|
||||
for path in each_linked_rlib_for_lto {
|
||||
let archive_data = unsafe {
|
||||
Mmap::map(std::fs::File::open(&path).expect("couldn't open rlib"))
|
||||
.expect("couldn't map rlib")
|
||||
|
|
@ -147,10 +88,6 @@ fn prepare_lto(
|
|||
}
|
||||
}
|
||||
|
||||
// __llvm_profile_counter_bias is pulled in at link time by an undefined reference to
|
||||
// __llvm_profile_runtime, therefore we won't know until link time if this symbol
|
||||
// should have default visibility.
|
||||
symbols_below_threshold.push(c"__llvm_profile_counter_bias".to_owned());
|
||||
Ok((symbols_below_threshold, upstream_modules))
|
||||
}
|
||||
|
||||
|
|
@ -199,15 +136,17 @@ fn get_bitcode_slice_from_object_data<'a>(
|
|||
/// for further optimization.
|
||||
pub(crate) fn run_fat(
|
||||
cgcx: &CodegenContext<LlvmCodegenBackend>,
|
||||
exported_symbols_for_lto: &[String],
|
||||
each_linked_rlib_for_lto: &[PathBuf],
|
||||
modules: Vec<FatLtoInput<LlvmCodegenBackend>>,
|
||||
cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
|
||||
) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> {
|
||||
let dcx = cgcx.create_dcx();
|
||||
let dcx = dcx.handle();
|
||||
let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, dcx)?;
|
||||
let (symbols_below_threshold, upstream_modules) =
|
||||
prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx)?;
|
||||
let symbols_below_threshold =
|
||||
symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();
|
||||
fat_lto(cgcx, dcx, modules, cached_modules, upstream_modules, &symbols_below_threshold)
|
||||
fat_lto(cgcx, dcx, modules, upstream_modules, &symbols_below_threshold)
|
||||
}
|
||||
|
||||
/// Performs thin LTO by performing necessary global analysis and returning two
|
||||
|
|
@ -215,12 +154,15 @@ pub(crate) fn run_fat(
|
|||
/// can simply be copied over from the incr. comp. cache.
|
||||
pub(crate) fn run_thin(
|
||||
cgcx: &CodegenContext<LlvmCodegenBackend>,
|
||||
exported_symbols_for_lto: &[String],
|
||||
each_linked_rlib_for_lto: &[PathBuf],
|
||||
modules: Vec<(String, ThinBuffer)>,
|
||||
cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
|
||||
) -> Result<(Vec<ThinModule<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> {
|
||||
let dcx = cgcx.create_dcx();
|
||||
let dcx = dcx.handle();
|
||||
let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, dcx)?;
|
||||
let (symbols_below_threshold, upstream_modules) =
|
||||
prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx)?;
|
||||
let symbols_below_threshold =
|
||||
symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();
|
||||
if cgcx.opts.cg.linker_plugin_lto.enabled() {
|
||||
|
|
@ -245,7 +187,6 @@ fn fat_lto(
|
|||
cgcx: &CodegenContext<LlvmCodegenBackend>,
|
||||
dcx: DiagCtxtHandle<'_>,
|
||||
modules: Vec<FatLtoInput<LlvmCodegenBackend>>,
|
||||
cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
|
||||
mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
|
||||
symbols_below_threshold: &[*const libc::c_char],
|
||||
) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> {
|
||||
|
|
@ -258,21 +199,12 @@ fn fat_lto(
|
|||
// modules that are serialized in-memory.
|
||||
// * `in_memory` contains modules which are already parsed and in-memory,
|
||||
// such as from multi-CGU builds.
|
||||
//
|
||||
// All of `cached_modules` (cached from previous incremental builds) can
|
||||
// immediately go onto the `serialized_modules` modules list and then we can
|
||||
// split the `modules` array into these two lists.
|
||||
let mut in_memory = Vec::new();
|
||||
serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| {
|
||||
info!("pushing cached module {:?}", wp.cgu_name);
|
||||
(buffer, CString::new(wp.cgu_name).unwrap())
|
||||
}));
|
||||
for module in modules {
|
||||
match module {
|
||||
FatLtoInput::InMemory(m) => in_memory.push(m),
|
||||
FatLtoInput::Serialized { name, buffer } => {
|
||||
info!("pushing serialized module {:?}", name);
|
||||
let buffer = SerializedModule::Local(buffer);
|
||||
serialized_modules.push((buffer, CString::new(name).unwrap()));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,11 +20,6 @@ pub(crate) struct SymbolAlreadyDefined<'a> {
|
|||
#[diag(codegen_llvm_sanitizer_memtag_requires_mte)]
|
||||
pub(crate) struct SanitizerMemtagRequiresMte;
|
||||
|
||||
#[derive(Diagnostic)]
|
||||
#[diag(codegen_llvm_dynamic_linking_with_lto)]
|
||||
#[note]
|
||||
pub(crate) struct DynamicLinkingWithLTO;
|
||||
|
||||
pub(crate) struct ParseTargetMachineConfig<'a>(pub LlvmError<'a>);
|
||||
|
||||
impl<G: EmissionGuarantee> Diagnostic<'_, G> for ParseTargetMachineConfig<'_> {
|
||||
|
|
@ -41,18 +36,6 @@ impl<G: EmissionGuarantee> Diagnostic<'_, G> for ParseTargetMachineConfig<'_> {
|
|||
#[diag(codegen_llvm_autodiff_without_enable)]
|
||||
pub(crate) struct AutoDiffWithoutEnable;
|
||||
|
||||
#[derive(Diagnostic)]
|
||||
#[diag(codegen_llvm_lto_disallowed)]
|
||||
pub(crate) struct LtoDisallowed;
|
||||
|
||||
#[derive(Diagnostic)]
|
||||
#[diag(codegen_llvm_lto_dylib)]
|
||||
pub(crate) struct LtoDylib;
|
||||
|
||||
#[derive(Diagnostic)]
|
||||
#[diag(codegen_llvm_lto_proc_macro)]
|
||||
pub(crate) struct LtoProcMacro;
|
||||
|
||||
#[derive(Diagnostic)]
|
||||
#[diag(codegen_llvm_lto_bitcode_from_rlib)]
|
||||
pub(crate) struct LtoBitcodeFromRlib {
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@
|
|||
use std::any::Any;
|
||||
use std::ffi::CStr;
|
||||
use std::mem::ManuallyDrop;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use back::owned_target_machine::OwnedTargetMachine;
|
||||
use back::write::{create_informational_target_machine, create_target_machine};
|
||||
|
|
@ -176,11 +177,13 @@ impl WriteBackendMethods for LlvmCodegenBackend {
|
|||
}
|
||||
fn run_and_optimize_fat_lto(
|
||||
cgcx: &CodegenContext<Self>,
|
||||
exported_symbols_for_lto: &[String],
|
||||
each_linked_rlib_for_lto: &[PathBuf],
|
||||
modules: Vec<FatLtoInput<Self>>,
|
||||
cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
|
||||
diff_fncs: Vec<AutoDiffItem>,
|
||||
) -> Result<ModuleCodegen<Self::Module>, FatalError> {
|
||||
let mut module = back::lto::run_fat(cgcx, modules, cached_modules)?;
|
||||
let mut module =
|
||||
back::lto::run_fat(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, modules)?;
|
||||
|
||||
if !diff_fncs.is_empty() {
|
||||
builder::autodiff::differentiate(&module, cgcx, diff_fncs)?;
|
||||
|
|
@ -194,10 +197,18 @@ impl WriteBackendMethods for LlvmCodegenBackend {
|
|||
}
|
||||
fn run_thin_lto(
|
||||
cgcx: &CodegenContext<Self>,
|
||||
exported_symbols_for_lto: &[String],
|
||||
each_linked_rlib_for_lto: &[PathBuf],
|
||||
modules: Vec<(String, Self::ThinBuffer)>,
|
||||
cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
|
||||
) -> Result<(Vec<ThinModule<Self>>, Vec<WorkProduct>), FatalError> {
|
||||
back::lto::run_thin(cgcx, modules, cached_modules)
|
||||
back::lto::run_thin(
|
||||
cgcx,
|
||||
exported_symbols_for_lto,
|
||||
each_linked_rlib_for_lto,
|
||||
modules,
|
||||
cached_modules,
|
||||
)
|
||||
}
|
||||
fn optimize(
|
||||
cgcx: &CodegenContext<Self>,
|
||||
|
|
|
|||
|
|
@ -35,6 +35,10 @@ codegen_ssa_dlltool_fail_import_library =
|
|||
{$stdout}
|
||||
{$stderr}
|
||||
|
||||
codegen_ssa_dynamic_linking_with_lto =
|
||||
cannot prefer dynamic linking when performing LTO
|
||||
.note = only 'staticlib', 'bin', and 'cdylib' outputs are supported with LTO
|
||||
|
||||
codegen_ssa_error_calling_dlltool =
|
||||
Error calling dlltool '{$dlltool_path}': {$error}
|
||||
|
||||
|
|
@ -191,6 +195,12 @@ codegen_ssa_linker_unsupported_modifier = `as-needed` modifier not supported for
|
|||
|
||||
codegen_ssa_linking_failed = linking with `{$linker_path}` failed: {$exit_status}
|
||||
|
||||
codegen_ssa_lto_disallowed = lto can only be run for executables, cdylibs and static library outputs
|
||||
|
||||
codegen_ssa_lto_dylib = lto cannot be used for `dylib` crate type without `-Zdylib-lto`
|
||||
|
||||
codegen_ssa_lto_proc_macro = lto cannot be used for `proc-macro` crate type without `-Zdylib-lto`
|
||||
|
||||
codegen_ssa_malformed_cgu_name =
|
||||
found malformed codegen unit name `{$user_path}`. codegen units names must always start with the name of the crate (`{$crate_name}` in this case).
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,15 @@ use std::ffi::CString;
|
|||
use std::sync::Arc;
|
||||
|
||||
use rustc_data_structures::memmap::Mmap;
|
||||
use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
|
||||
use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo, SymbolExportLevel};
|
||||
use rustc_middle::ty::TyCtxt;
|
||||
use rustc_session::config::{CrateType, Lto};
|
||||
use tracing::info;
|
||||
|
||||
use crate::back::symbol_export::{self, symbol_name_for_instance_in_crate};
|
||||
use crate::back::write::CodegenContext;
|
||||
use crate::errors::{DynamicLinkingWithLTO, LtoDisallowed, LtoDylib, LtoProcMacro};
|
||||
use crate::traits::*;
|
||||
|
||||
pub struct ThinModule<B: WriteBackendMethods> {
|
||||
|
|
@ -52,3 +60,86 @@ impl<M: ModuleBufferMethods> SerializedModule<M> {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn crate_type_allows_lto(crate_type: CrateType) -> bool {
|
||||
match crate_type {
|
||||
CrateType::Executable
|
||||
| CrateType::Dylib
|
||||
| CrateType::Staticlib
|
||||
| CrateType::Cdylib
|
||||
| CrateType::ProcMacro
|
||||
| CrateType::Sdylib => true,
|
||||
CrateType::Rlib => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn exported_symbols_for_lto(
|
||||
tcx: TyCtxt<'_>,
|
||||
each_linked_rlib_for_lto: &[CrateNum],
|
||||
) -> Vec<String> {
|
||||
let export_threshold = match tcx.sess.lto() {
|
||||
// We're just doing LTO for our one crate
|
||||
Lto::ThinLocal => SymbolExportLevel::Rust,
|
||||
|
||||
// We're doing LTO for the entire crate graph
|
||||
Lto::Fat | Lto::Thin => symbol_export::crates_export_threshold(&tcx.crate_types()),
|
||||
|
||||
Lto::No => return vec![],
|
||||
};
|
||||
|
||||
let copy_symbols = |cnum| {
|
||||
tcx.exported_non_generic_symbols(cnum)
|
||||
.iter()
|
||||
.chain(tcx.exported_generic_symbols(cnum))
|
||||
.filter_map(|&(s, info): &(ExportedSymbol<'_>, SymbolExportInfo)| {
|
||||
if info.level.is_below_threshold(export_threshold) || info.used {
|
||||
Some(symbol_name_for_instance_in_crate(tcx, s, cnum))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
let mut symbols_below_threshold = {
|
||||
let _timer = tcx.prof.generic_activity("lto_generate_symbols_below_threshold");
|
||||
copy_symbols(LOCAL_CRATE)
|
||||
};
|
||||
info!("{} symbols to preserve in this crate", symbols_below_threshold.len());
|
||||
|
||||
// If we're performing LTO for the entire crate graph, then for each of our
|
||||
// upstream dependencies, include their exported symbols.
|
||||
if tcx.sess.lto() != Lto::ThinLocal {
|
||||
for &cnum in each_linked_rlib_for_lto {
|
||||
let _timer = tcx.prof.generic_activity("lto_generate_symbols_below_threshold");
|
||||
symbols_below_threshold.extend(copy_symbols(cnum));
|
||||
}
|
||||
}
|
||||
|
||||
symbols_below_threshold
|
||||
}
|
||||
|
||||
pub(super) fn check_lto_allowed<B: WriteBackendMethods>(cgcx: &CodegenContext<B>) {
|
||||
if cgcx.lto == Lto::ThinLocal {
|
||||
// Crate local LTO is always allowed
|
||||
return;
|
||||
}
|
||||
|
||||
let dcx = cgcx.create_dcx();
|
||||
|
||||
// Make sure we actually can run LTO
|
||||
for crate_type in cgcx.crate_types.iter() {
|
||||
if !crate_type_allows_lto(*crate_type) {
|
||||
dcx.handle().emit_fatal(LtoDisallowed);
|
||||
} else if *crate_type == CrateType::Dylib {
|
||||
if !cgcx.opts.unstable_opts.dylib_lto {
|
||||
dcx.handle().emit_fatal(LtoDylib);
|
||||
}
|
||||
} else if *crate_type == CrateType::ProcMacro && !cgcx.opts.unstable_opts.dylib_lto {
|
||||
dcx.handle().emit_fatal(LtoProcMacro);
|
||||
}
|
||||
}
|
||||
|
||||
if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto {
|
||||
dcx.handle().emit_fatal(DynamicLinkingWithLTO);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ use std::{fs, io, mem, str, thread};
|
|||
use rustc_abi::Size;
|
||||
use rustc_ast::attr;
|
||||
use rustc_ast::expand::autodiff_attrs::AutoDiffItem;
|
||||
use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
|
||||
use rustc_data_structures::fx::FxIndexMap;
|
||||
use rustc_data_structures::jobserver::{self, Acquired};
|
||||
use rustc_data_structures::memmap::Mmap;
|
||||
use rustc_data_structures::profiling::{SelfProfilerRef, VerboseTimingGuard};
|
||||
|
|
@ -20,14 +20,12 @@ use rustc_errors::{
|
|||
Suggestions,
|
||||
};
|
||||
use rustc_fs_util::link_or_copy;
|
||||
use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
|
||||
use rustc_incremental::{
|
||||
copy_cgu_workproduct_to_incr_comp_cache_dir, in_incr_comp_dir, in_incr_comp_dir_sess,
|
||||
};
|
||||
use rustc_metadata::fs::copy_to_stdout;
|
||||
use rustc_middle::bug;
|
||||
use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
|
||||
use rustc_middle::middle::exported_symbols::SymbolExportInfo;
|
||||
use rustc_middle::ty::TyCtxt;
|
||||
use rustc_session::Session;
|
||||
use rustc_session::config::{
|
||||
|
|
@ -40,7 +38,7 @@ use tracing::debug;
|
|||
|
||||
use super::link::{self, ensure_removed};
|
||||
use super::lto::{self, SerializedModule};
|
||||
use super::symbol_export::symbol_name_for_instance_in_crate;
|
||||
use crate::back::lto::check_lto_allowed;
|
||||
use crate::errors::{AutodiffWithoutLto, ErrorCreatingRemarkDir};
|
||||
use crate::traits::*;
|
||||
use crate::{
|
||||
|
|
@ -332,8 +330,6 @@ pub type TargetMachineFactoryFn<B> = Arc<
|
|||
+ Sync,
|
||||
>;
|
||||
|
||||
type ExportedSymbols = FxHashMap<CrateNum, Arc<Vec<(String, SymbolExportInfo)>>>;
|
||||
|
||||
/// Additional resources used by optimize_and_codegen (not module specific)
|
||||
#[derive(Clone)]
|
||||
pub struct CodegenContext<B: WriteBackendMethods> {
|
||||
|
|
@ -343,10 +339,8 @@ pub struct CodegenContext<B: WriteBackendMethods> {
|
|||
pub save_temps: bool,
|
||||
pub fewer_names: bool,
|
||||
pub time_trace: bool,
|
||||
pub exported_symbols: Option<Arc<ExportedSymbols>>,
|
||||
pub opts: Arc<config::Options>,
|
||||
pub crate_types: Vec<CrateType>,
|
||||
pub each_linked_rlib_for_lto: Vec<(CrateNum, PathBuf)>,
|
||||
pub output_filenames: Arc<OutputFilenames>,
|
||||
pub invocation_temp: Option<String>,
|
||||
pub regular_module_config: Arc<ModuleConfig>,
|
||||
|
|
@ -401,13 +395,21 @@ impl<B: WriteBackendMethods> CodegenContext<B> {
|
|||
|
||||
fn generate_thin_lto_work<B: ExtraBackendMethods>(
|
||||
cgcx: &CodegenContext<B>,
|
||||
exported_symbols_for_lto: &[String],
|
||||
each_linked_rlib_for_lto: &[PathBuf],
|
||||
needs_thin_lto: Vec<(String, B::ThinBuffer)>,
|
||||
import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>,
|
||||
) -> Vec<(WorkItem<B>, u64)> {
|
||||
let _prof_timer = cgcx.prof.generic_activity("codegen_thin_generate_lto_work");
|
||||
|
||||
let (lto_modules, copy_jobs) =
|
||||
B::run_thin_lto(cgcx, needs_thin_lto, import_only_modules).unwrap_or_else(|e| e.raise());
|
||||
let (lto_modules, copy_jobs) = B::run_thin_lto(
|
||||
cgcx,
|
||||
exported_symbols_for_lto,
|
||||
each_linked_rlib_for_lto,
|
||||
needs_thin_lto,
|
||||
import_only_modules,
|
||||
)
|
||||
.unwrap_or_else(|e| e.raise());
|
||||
lto_modules
|
||||
.into_iter()
|
||||
.map(|module| {
|
||||
|
|
@ -723,6 +725,8 @@ pub(crate) enum WorkItem<B: WriteBackendMethods> {
|
|||
CopyPostLtoArtifacts(CachedModuleCodegen),
|
||||
/// Performs fat LTO on the given module.
|
||||
FatLto {
|
||||
exported_symbols_for_lto: Arc<Vec<String>>,
|
||||
each_linked_rlib_for_lto: Vec<PathBuf>,
|
||||
needs_fat_lto: Vec<FatLtoInput<B>>,
|
||||
import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>,
|
||||
autodiff: Vec<AutoDiffItem>,
|
||||
|
|
@ -810,7 +814,7 @@ pub(crate) enum WorkItemResult<B: WriteBackendMethods> {
|
|||
}
|
||||
|
||||
pub enum FatLtoInput<B: WriteBackendMethods> {
|
||||
Serialized { name: String, buffer: B::ModuleBuffer },
|
||||
Serialized { name: String, buffer: SerializedModule<B::ModuleBuffer> },
|
||||
InMemory(ModuleCodegen<B::Module>),
|
||||
}
|
||||
|
||||
|
|
@ -899,7 +903,10 @@ fn execute_optimize_work_item<B: ExtraBackendMethods>(
|
|||
fs::write(&path, buffer.data()).unwrap_or_else(|e| {
|
||||
panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
|
||||
});
|
||||
Ok(WorkItemResult::NeedsFatLto(FatLtoInput::Serialized { name, buffer }))
|
||||
Ok(WorkItemResult::NeedsFatLto(FatLtoInput::Serialized {
|
||||
name,
|
||||
buffer: SerializedModule::Local(buffer),
|
||||
}))
|
||||
}
|
||||
None => Ok(WorkItemResult::NeedsFatLto(FatLtoInput::InMemory(module))),
|
||||
},
|
||||
|
|
@ -992,12 +999,24 @@ fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
|
|||
|
||||
fn execute_fat_lto_work_item<B: ExtraBackendMethods>(
|
||||
cgcx: &CodegenContext<B>,
|
||||
needs_fat_lto: Vec<FatLtoInput<B>>,
|
||||
exported_symbols_for_lto: &[String],
|
||||
each_linked_rlib_for_lto: &[PathBuf],
|
||||
mut needs_fat_lto: Vec<FatLtoInput<B>>,
|
||||
import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>,
|
||||
autodiff: Vec<AutoDiffItem>,
|
||||
module_config: &ModuleConfig,
|
||||
) -> Result<WorkItemResult<B>, FatalError> {
|
||||
let module = B::run_and_optimize_fat_lto(cgcx, needs_fat_lto, import_only_modules, autodiff)?;
|
||||
for (module, wp) in import_only_modules {
|
||||
needs_fat_lto.push(FatLtoInput::Serialized { name: wp.cgu_name, buffer: module })
|
||||
}
|
||||
|
||||
let module = B::run_and_optimize_fat_lto(
|
||||
cgcx,
|
||||
exported_symbols_for_lto,
|
||||
each_linked_rlib_for_lto,
|
||||
needs_fat_lto,
|
||||
autodiff,
|
||||
)?;
|
||||
let module = B::codegen(cgcx, module, module_config)?;
|
||||
Ok(WorkItemResult::Finished(module))
|
||||
}
|
||||
|
|
@ -1032,7 +1051,7 @@ pub(crate) enum Message<B: WriteBackendMethods> {
|
|||
|
||||
/// The backend has finished processing a work item for a codegen unit.
|
||||
/// Sent from a backend worker thread.
|
||||
WorkItem { result: Result<WorkItemResult<B>, Option<WorkerFatalError>>, worker_id: usize },
|
||||
WorkItem { result: Result<WorkItemResult<B>, Option<WorkerFatalError>> },
|
||||
|
||||
/// The frontend has finished generating something (backend IR or a
|
||||
/// post-LTO artifact) for a codegen unit, and it should be passed to the
|
||||
|
|
@ -1113,42 +1132,18 @@ fn start_executing_work<B: ExtraBackendMethods>(
|
|||
let autodiff_items = autodiff_items.to_vec();
|
||||
|
||||
let mut each_linked_rlib_for_lto = Vec::new();
|
||||
let mut each_linked_rlib_file_for_lto = Vec::new();
|
||||
drop(link::each_linked_rlib(crate_info, None, &mut |cnum, path| {
|
||||
if link::ignored_for_lto(sess, crate_info, cnum) {
|
||||
return;
|
||||
}
|
||||
each_linked_rlib_for_lto.push((cnum, path.to_path_buf()));
|
||||
each_linked_rlib_for_lto.push(cnum);
|
||||
each_linked_rlib_file_for_lto.push(path.to_path_buf());
|
||||
}));
|
||||
|
||||
// Compute the set of symbols we need to retain when doing LTO (if we need to)
|
||||
let exported_symbols = {
|
||||
let mut exported_symbols = FxHashMap::default();
|
||||
|
||||
let copy_symbols = |cnum| {
|
||||
let symbols = tcx
|
||||
.exported_non_generic_symbols(cnum)
|
||||
.iter()
|
||||
.chain(tcx.exported_generic_symbols(cnum))
|
||||
.map(|&(s, lvl)| (symbol_name_for_instance_in_crate(tcx, s, cnum), lvl))
|
||||
.collect();
|
||||
Arc::new(symbols)
|
||||
};
|
||||
|
||||
match sess.lto() {
|
||||
Lto::No => None,
|
||||
Lto::ThinLocal => {
|
||||
exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
|
||||
Some(Arc::new(exported_symbols))
|
||||
}
|
||||
Lto::Fat | Lto::Thin => {
|
||||
exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
|
||||
for &(cnum, ref _path) in &each_linked_rlib_for_lto {
|
||||
exported_symbols.insert(cnum, copy_symbols(cnum));
|
||||
}
|
||||
Some(Arc::new(exported_symbols))
|
||||
}
|
||||
}
|
||||
};
|
||||
let exported_symbols_for_lto =
|
||||
Arc::new(lto::exported_symbols_for_lto(tcx, &each_linked_rlib_for_lto));
|
||||
|
||||
// First up, convert our jobserver into a helper thread so we can use normal
|
||||
// mpsc channels to manage our messages and such.
|
||||
|
|
@ -1183,14 +1178,12 @@ fn start_executing_work<B: ExtraBackendMethods>(
|
|||
|
||||
let cgcx = CodegenContext::<B> {
|
||||
crate_types: tcx.crate_types().to_vec(),
|
||||
each_linked_rlib_for_lto,
|
||||
lto: sess.lto(),
|
||||
fewer_names: sess.fewer_names(),
|
||||
save_temps: sess.opts.cg.save_temps,
|
||||
time_trace: sess.opts.unstable_opts.llvm_time_trace,
|
||||
opts: Arc::new(sess.opts.clone()),
|
||||
prof: sess.prof.clone(),
|
||||
exported_symbols,
|
||||
remark: sess.opts.cg.remark.clone(),
|
||||
remark_dir,
|
||||
incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()),
|
||||
|
|
@ -1350,18 +1343,6 @@ fn start_executing_work<B: ExtraBackendMethods>(
|
|||
// necessary. There's already optimizations in place to avoid sending work
|
||||
// back to the coordinator if LTO isn't requested.
|
||||
return B::spawn_named_thread(cgcx.time_trace, "coordinator".to_string(), move || {
|
||||
let mut worker_id_counter = 0;
|
||||
let mut free_worker_ids = Vec::new();
|
||||
let mut get_worker_id = |free_worker_ids: &mut Vec<usize>| {
|
||||
if let Some(id) = free_worker_ids.pop() {
|
||||
id
|
||||
} else {
|
||||
let id = worker_id_counter;
|
||||
worker_id_counter += 1;
|
||||
id
|
||||
}
|
||||
};
|
||||
|
||||
// This is where we collect codegen units that have gone all the way
|
||||
// through codegen and LLVM.
|
||||
let mut compiled_modules = vec![];
|
||||
|
|
@ -1442,12 +1423,7 @@ fn start_executing_work<B: ExtraBackendMethods>(
|
|||
let (item, _) =
|
||||
work_items.pop().expect("queue empty - queue_full_enough() broken?");
|
||||
main_thread_state = MainThreadState::Lending;
|
||||
spawn_work(
|
||||
&cgcx,
|
||||
&mut llvm_start_time,
|
||||
get_worker_id(&mut free_worker_ids),
|
||||
item,
|
||||
);
|
||||
spawn_work(&cgcx, &mut llvm_start_time, item);
|
||||
}
|
||||
}
|
||||
} else if codegen_state == Completed {
|
||||
|
|
@ -1474,12 +1450,18 @@ fn start_executing_work<B: ExtraBackendMethods>(
|
|||
let needs_fat_lto = mem::take(&mut needs_fat_lto);
|
||||
let needs_thin_lto = mem::take(&mut needs_thin_lto);
|
||||
let import_only_modules = mem::take(&mut lto_import_only_modules);
|
||||
let each_linked_rlib_file_for_lto =
|
||||
mem::take(&mut each_linked_rlib_file_for_lto);
|
||||
|
||||
check_lto_allowed(&cgcx);
|
||||
|
||||
if !needs_fat_lto.is_empty() {
|
||||
assert!(needs_thin_lto.is_empty());
|
||||
|
||||
work_items.push((
|
||||
WorkItem::FatLto {
|
||||
exported_symbols_for_lto: Arc::clone(&exported_symbols_for_lto),
|
||||
each_linked_rlib_for_lto: each_linked_rlib_file_for_lto,
|
||||
needs_fat_lto,
|
||||
import_only_modules,
|
||||
autodiff: autodiff_items.clone(),
|
||||
|
|
@ -1495,9 +1477,13 @@ fn start_executing_work<B: ExtraBackendMethods>(
|
|||
dcx.handle().emit_fatal(AutodiffWithoutLto {});
|
||||
}
|
||||
|
||||
for (work, cost) in
|
||||
generate_thin_lto_work(&cgcx, needs_thin_lto, import_only_modules)
|
||||
{
|
||||
for (work, cost) in generate_thin_lto_work(
|
||||
&cgcx,
|
||||
&exported_symbols_for_lto,
|
||||
&each_linked_rlib_file_for_lto,
|
||||
needs_thin_lto,
|
||||
import_only_modules,
|
||||
) {
|
||||
let insertion_index = work_items
|
||||
.binary_search_by_key(&cost, |&(_, cost)| cost)
|
||||
.unwrap_or_else(|e| e);
|
||||
|
|
@ -1516,12 +1502,7 @@ fn start_executing_work<B: ExtraBackendMethods>(
|
|||
MainThreadState::Idle => {
|
||||
if let Some((item, _)) = work_items.pop() {
|
||||
main_thread_state = MainThreadState::Lending;
|
||||
spawn_work(
|
||||
&cgcx,
|
||||
&mut llvm_start_time,
|
||||
get_worker_id(&mut free_worker_ids),
|
||||
item,
|
||||
);
|
||||
spawn_work(&cgcx, &mut llvm_start_time, item);
|
||||
} else {
|
||||
// There is no unstarted work, so let the main thread
|
||||
// take over for a running worker. Otherwise the
|
||||
|
|
@ -1557,12 +1538,7 @@ fn start_executing_work<B: ExtraBackendMethods>(
|
|||
while running_with_own_token < tokens.len()
|
||||
&& let Some((item, _)) = work_items.pop()
|
||||
{
|
||||
spawn_work(
|
||||
&cgcx,
|
||||
&mut llvm_start_time,
|
||||
get_worker_id(&mut free_worker_ids),
|
||||
item,
|
||||
);
|
||||
spawn_work(&cgcx, &mut llvm_start_time, item);
|
||||
running_with_own_token += 1;
|
||||
}
|
||||
}
|
||||
|
|
@ -1570,21 +1546,6 @@ fn start_executing_work<B: ExtraBackendMethods>(
|
|||
// Relinquish accidentally acquired extra tokens.
|
||||
tokens.truncate(running_with_own_token);
|
||||
|
||||
// If a thread exits successfully then we drop a token associated
|
||||
// with that worker and update our `running_with_own_token` count.
|
||||
// We may later re-acquire a token to continue running more work.
|
||||
// We may also not actually drop a token here if the worker was
|
||||
// running with an "ephemeral token".
|
||||
let mut free_worker = |worker_id| {
|
||||
if main_thread_state == MainThreadState::Lending {
|
||||
main_thread_state = MainThreadState::Idle;
|
||||
} else {
|
||||
running_with_own_token -= 1;
|
||||
}
|
||||
|
||||
free_worker_ids.push(worker_id);
|
||||
};
|
||||
|
||||
let msg = coordinator_receive.recv().unwrap();
|
||||
match *msg.downcast::<Message<B>>().ok().unwrap() {
|
||||
// Save the token locally and the next turn of the loop will use
|
||||
|
|
@ -1653,8 +1614,17 @@ fn start_executing_work<B: ExtraBackendMethods>(
|
|||
codegen_state = Aborted;
|
||||
}
|
||||
|
||||
Message::WorkItem { result, worker_id } => {
|
||||
free_worker(worker_id);
|
||||
Message::WorkItem { result } => {
|
||||
// If a thread exits successfully then we drop a token associated
|
||||
// with that worker and update our `running_with_own_token` count.
|
||||
// We may later re-acquire a token to continue running more work.
|
||||
// We may also not actually drop a token here if the worker was
|
||||
// running with an "ephemeral token".
|
||||
if main_thread_state == MainThreadState::Lending {
|
||||
main_thread_state = MainThreadState::Idle;
|
||||
} else {
|
||||
running_with_own_token -= 1;
|
||||
}
|
||||
|
||||
match result {
|
||||
Ok(WorkItemResult::Finished(compiled_module)) => {
|
||||
|
|
@ -1800,7 +1770,6 @@ pub(crate) struct WorkerFatalError;
|
|||
fn spawn_work<'a, B: ExtraBackendMethods>(
|
||||
cgcx: &'a CodegenContext<B>,
|
||||
llvm_start_time: &mut Option<VerboseTimingGuard<'a>>,
|
||||
worker_id: usize,
|
||||
work: WorkItem<B>,
|
||||
) {
|
||||
if cgcx.config(work.module_kind()).time_module && llvm_start_time.is_none() {
|
||||
|
|
@ -1815,24 +1784,21 @@ fn spawn_work<'a, B: ExtraBackendMethods>(
|
|||
struct Bomb<B: ExtraBackendMethods> {
|
||||
coordinator_send: Sender<Box<dyn Any + Send>>,
|
||||
result: Option<Result<WorkItemResult<B>, FatalError>>,
|
||||
worker_id: usize,
|
||||
}
|
||||
impl<B: ExtraBackendMethods> Drop for Bomb<B> {
|
||||
fn drop(&mut self) {
|
||||
let worker_id = self.worker_id;
|
||||
let msg = match self.result.take() {
|
||||
Some(Ok(result)) => Message::WorkItem::<B> { result: Ok(result), worker_id },
|
||||
Some(Ok(result)) => Message::WorkItem::<B> { result: Ok(result) },
|
||||
Some(Err(FatalError)) => {
|
||||
Message::WorkItem::<B> { result: Err(Some(WorkerFatalError)), worker_id }
|
||||
Message::WorkItem::<B> { result: Err(Some(WorkerFatalError)) }
|
||||
}
|
||||
None => Message::WorkItem::<B> { result: Err(None), worker_id },
|
||||
None => Message::WorkItem::<B> { result: Err(None) },
|
||||
};
|
||||
drop(self.coordinator_send.send(Box::new(msg)));
|
||||
}
|
||||
}
|
||||
|
||||
let mut bomb =
|
||||
Bomb::<B> { coordinator_send: cgcx.coordinator_send.clone(), result: None, worker_id };
|
||||
let mut bomb = Bomb::<B> { coordinator_send: cgcx.coordinator_send.clone(), result: None };
|
||||
|
||||
// Execute the work itself, and if it finishes successfully then flag
|
||||
// ourselves as a success as well.
|
||||
|
|
@ -1856,12 +1822,20 @@ fn spawn_work<'a, B: ExtraBackendMethods>(
|
|||
);
|
||||
Ok(execute_copy_from_cache_work_item(&cgcx, m, module_config))
|
||||
}
|
||||
WorkItem::FatLto { needs_fat_lto, import_only_modules, autodiff } => {
|
||||
WorkItem::FatLto {
|
||||
exported_symbols_for_lto,
|
||||
each_linked_rlib_for_lto,
|
||||
needs_fat_lto,
|
||||
import_only_modules,
|
||||
autodiff,
|
||||
} => {
|
||||
let _timer = cgcx
|
||||
.prof
|
||||
.generic_activity_with_arg("codegen_module_perform_lto", "everything");
|
||||
execute_fat_lto_work_item(
|
||||
&cgcx,
|
||||
&exported_symbols_for_lto,
|
||||
&each_linked_rlib_for_lto,
|
||||
needs_fat_lto,
|
||||
import_only_modules,
|
||||
autodiff,
|
||||
|
|
|
|||
|
|
@ -1294,3 +1294,20 @@ pub(crate) struct FeatureNotValid<'a> {
|
|||
#[help]
|
||||
pub plus_hint: bool,
|
||||
}
|
||||
|
||||
#[derive(Diagnostic)]
|
||||
#[diag(codegen_ssa_lto_disallowed)]
|
||||
pub(crate) struct LtoDisallowed;
|
||||
|
||||
#[derive(Diagnostic)]
|
||||
#[diag(codegen_ssa_lto_dylib)]
|
||||
pub(crate) struct LtoDylib;
|
||||
|
||||
#[derive(Diagnostic)]
|
||||
#[diag(codegen_ssa_lto_proc_macro)]
|
||||
pub(crate) struct LtoProcMacro;
|
||||
|
||||
#[derive(Diagnostic)]
|
||||
#[diag(codegen_ssa_dynamic_linking_with_lto)]
|
||||
#[note]
|
||||
pub(crate) struct DynamicLinkingWithLTO;
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
use std::path::PathBuf;
|
||||
|
||||
use rustc_ast::expand::autodiff_attrs::AutoDiffItem;
|
||||
use rustc_errors::{DiagCtxtHandle, FatalError};
|
||||
use rustc_middle::dep_graph::WorkProduct;
|
||||
|
|
@ -24,8 +26,9 @@ pub trait WriteBackendMethods: Clone + 'static {
|
|||
/// if necessary and running any further optimizations
|
||||
fn run_and_optimize_fat_lto(
|
||||
cgcx: &CodegenContext<Self>,
|
||||
exported_symbols_for_lto: &[String],
|
||||
each_linked_rlib_for_lto: &[PathBuf],
|
||||
modules: Vec<FatLtoInput<Self>>,
|
||||
cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
|
||||
diff_fncs: Vec<AutoDiffItem>,
|
||||
) -> Result<ModuleCodegen<Self::Module>, FatalError>;
|
||||
/// Performs thin LTO by performing necessary global analysis and returning two
|
||||
|
|
@ -33,6 +36,8 @@ pub trait WriteBackendMethods: Clone + 'static {
|
|||
/// can simply be copied over from the incr. comp. cache.
|
||||
fn run_thin_lto(
|
||||
cgcx: &CodegenContext<Self>,
|
||||
exported_symbols_for_lto: &[String],
|
||||
each_linked_rlib_for_lto: &[PathBuf],
|
||||
modules: Vec<(String, Self::ThinBuffer)>,
|
||||
cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
|
||||
) -> Result<(Vec<ThinModule<Self>>, Vec<WorkProduct>), FatalError>;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue