From d97d1e192b37557f760f41cbc9826bb431b14d1a Mon Sep 17 00:00:00 2001 From: Michael Woerister Date: Fri, 17 Aug 2018 16:07:23 +0200 Subject: [PATCH] Persist ThinLTO import data in incr. comp. session directory. --- src/librustc_codegen_llvm/back/lto.rs | 119 +++++++++++++++++++++++- src/librustc_codegen_llvm/base.rs | 20 ++++ src/librustc_codegen_llvm/lib.rs | 2 +- src/librustc_incremental/lib.rs | 1 + src/librustc_incremental/persist/mod.rs | 1 + 5 files changed, 139 insertions(+), 4 deletions(-) diff --git a/src/librustc_codegen_llvm/back/lto.rs b/src/librustc_codegen_llvm/back/lto.rs index 56858a31efd2..22d862f4ad5d 100644 --- a/src/librustc_codegen_llvm/back/lto.rs +++ b/src/librustc_codegen_llvm/back/lto.rs @@ -20,16 +20,23 @@ use rustc::hir::def_id::LOCAL_CRATE; use rustc::middle::exported_symbols::SymbolExportLevel; use rustc::session::config::{self, Lto}; use rustc::util::common::time_ext; +use rustc_data_structures::fx::FxHashMap; use time_graph::Timeline; use {ModuleCodegen, ModuleLlvm, ModuleKind, ModuleSource}; use libc; -use std::ffi::CString; +use std::ffi::{CStr, CString}; +use std::fs::File; +use std::io; +use std::mem; +use std::path::Path; use std::ptr; use std::slice; use std::sync::Arc; +pub const THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME: &str = "thin-lto-imports.bin"; + pub fn crate_type_allows_lto(crate_type: config::CrateType) -> bool { match crate_type { config::CrateType::Executable | @@ -199,7 +206,7 @@ pub(crate) fn run(cgcx: &CodegenContext, unreachable!("We should never reach this case if the LTO step \ is deferred to the linker"); } - thin_lto(&diag_handler, modules, upstream_modules, &arr, timeline) + thin_lto(cgcx, &diag_handler, modules, upstream_modules, &arr, timeline) } Lto::No => unreachable!(), } @@ -362,7 +369,8 @@ impl Drop for Linker<'a> { /// calculating the *index* for ThinLTO. This index will then be shared amongst /// all of the `LtoModuleCodegen` units returned below and destroyed once /// they all go out of scope. -fn thin_lto(diag_handler: &Handler, +fn thin_lto(cgcx: &CodegenContext, + diag_handler: &Handler, modules: Vec, serialized_modules: Vec<(SerializedModule, CString)>, symbol_white_list: &[*const libc::c_char], @@ -439,6 +447,17 @@ fn thin_lto(diag_handler: &Handler, write::llvm_err(&diag_handler, "failed to prepare thin LTO context".to_string()) })?; + // Save the ThinLTO import information for incremental compilation. + if let Some(ref incr_comp_session_dir) = cgcx.incr_comp_session_dir { + let path = incr_comp_session_dir.join(THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME); + let imports = ThinLTOImports::from_thin_lto_data(data); + if let Err(err) = imports.save_to_file(&path) { + let msg = format!("Error while writing ThinLTO import data: {}", + err); + return Err(write::llvm_err(&diag_handler, msg)); + } + } + let data = ThinData(data); info!("thin LTO data created"); timeline.record("data"); @@ -776,3 +795,97 @@ impl ThinModule { Ok(module) } } + +#[derive(Debug)] +pub struct ThinLTOImports { + // key = llvm name of importing module, value = list of modules it imports from + imports: FxHashMap>, +} + +impl ThinLTOImports { + pub fn new() -> ThinLTOImports { + ThinLTOImports { + imports: FxHashMap(), + } + } + + /// Load the ThinLTO import map from ThinLTOData. + unsafe fn from_thin_lto_data(data: *const llvm::ThinLTOData) -> ThinLTOImports { + fn module_name_to_str(c_str: &CStr) -> &str { + match c_str.to_str() { + Ok(s) => s, + Err(e) => { + bug!("Encountered non-utf8 LLVM module name `{}`: {}", + c_str.to_string_lossy(), + e) + } + } + } + unsafe extern "C" fn imported_module_callback(payload: *mut libc::c_void, + importing_module_name: *const libc::c_char, + imported_module_name: *const libc::c_char) { + let map = &mut* (payload as *mut ThinLTOImports); + let importing_module_name = CStr::from_ptr(importing_module_name); + let importing_module_name = module_name_to_str(&importing_module_name); + let imported_module_name = CStr::from_ptr(imported_module_name); + let imported_module_name = module_name_to_str(&imported_module_name); + if !map.imports.contains_key(importing_module_name) { + map.imports.insert(importing_module_name.to_owned(), vec![]); + } + map.imports + .get_mut(importing_module_name) + .unwrap() + .push(imported_module_name.to_owned()); + } + let mut map = ThinLTOImports { + imports: FxHashMap(), + }; + llvm::LLVMRustGetThinLTOModuleImports(data, + imported_module_callback, + &mut map as *mut _ as *mut libc::c_void); + map + } + + pub fn save_to_file(&self, path: &Path) -> io::Result<()> { + use std::io::Write; + let file = File::create(path)?; + let mut writer = io::BufWriter::new(file); + for (importing_module_name, imported_modules) in &self.imports { + writeln!(writer, "{}", importing_module_name)?; + for imported_module in imported_modules { + writeln!(writer, " {}", imported_module)?; + } + writeln!(writer)?; + } + Ok(()) + } + + pub fn load_from_file(path: &Path) -> io::Result { + use std::io::BufRead; + let mut imports = FxHashMap(); + let mut current_module = None; + let mut current_imports = vec![]; + let file = File::open(path)?; + for line in io::BufReader::new(file).lines() { + let line = line?; + if line.is_empty() { + let importing_module = current_module + .take() + .expect("Importing module not set"); + imports.insert(importing_module, + mem::replace(&mut current_imports, vec![])); + } else if line.starts_with(" ") { + // This is an imported module + assert_ne!(current_module, None); + current_imports.push(line.trim().to_string()); + } else { + // This is the beginning of a new module + assert_eq!(current_module, None); + current_module = Some(line.trim().to_string()); + } + } + Ok(ThinLTOImports { + imports + }) + } +} \ No newline at end of file diff --git a/src/librustc_codegen_llvm/base.rs b/src/librustc_codegen_llvm/base.rs index 009c6da9d8d1..32f88f867431 100644 --- a/src/librustc_codegen_llvm/base.rs +++ b/src/librustc_codegen_llvm/base.rs @@ -29,6 +29,7 @@ use super::ModuleCodegen; use super::ModuleKind; use abi; +use back::lto; use back::write::{self, OngoingCodegen}; use llvm::{self, TypeKind, get_param}; use metadata; @@ -1314,6 +1315,25 @@ pub fn visibility_to_llvm(linkage: Visibility) -> llvm::Visibility { } } +#[allow(unused)] +fn load_thin_lto_imports(sess: &Session) -> lto::ThinLTOImports { + let path = rustc_incremental::in_incr_comp_dir_sess( + sess, + lto::THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME + ); + if !path.exists() { + return lto::ThinLTOImports::new(); + } + match lto::ThinLTOImports::load_from_file(&path) { + Ok(imports) => imports, + Err(e) => { + let msg = format!("Error while trying to load ThinLTO import data \ + for incremental compilation: {}", e); + sess.fatal(&msg) + } + } +} + // FIXME(mw): Anything that is produced via DepGraph::with_task() must implement // the HashStable trait. Normally DepGraph::with_task() calls are // hidden behind queries, but CGU creation is a special case in two diff --git a/src/librustc_codegen_llvm/lib.rs b/src/librustc_codegen_llvm/lib.rs index 31eeb5633fba..e02fd2cae203 100644 --- a/src/librustc_codegen_llvm/lib.rs +++ b/src/librustc_codegen_llvm/lib.rs @@ -100,7 +100,7 @@ mod back { mod command; pub mod linker; pub mod link; - mod lto; + pub mod lto; pub mod symbol_export; pub mod write; mod rpath; diff --git a/src/librustc_incremental/lib.rs b/src/librustc_incremental/lib.rs index e100b49c7f24..4ffd726c1d47 100644 --- a/src/librustc_incremental/lib.rs +++ b/src/librustc_incremental/lib.rs @@ -44,6 +44,7 @@ pub use persist::copy_cgu_workproducts_to_incr_comp_cache_dir; pub use persist::save_dep_graph; pub use persist::save_work_product_index; pub use persist::in_incr_comp_dir; +pub use persist::in_incr_comp_dir_sess; pub use persist::prepare_session_directory; pub use persist::finalize_session_directory; pub use persist::delete_workproduct_files; diff --git a/src/librustc_incremental/persist/mod.rs b/src/librustc_incremental/persist/mod.rs index e1f00db56d5c..17d36ba3fa7f 100644 --- a/src/librustc_incremental/persist/mod.rs +++ b/src/librustc_incremental/persist/mod.rs @@ -23,6 +23,7 @@ mod file_format; pub use self::fs::finalize_session_directory; pub use self::fs::garbage_collect_session_directories; pub use self::fs::in_incr_comp_dir; +pub use self::fs::in_incr_comp_dir_sess; pub use self::fs::prepare_session_directory; pub use self::load::dep_graph_tcx_init; pub use self::load::load_dep_graph;