user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

incr.comp.: Serialize and deserialize new DepGraph

Browse source
This commit is contained in:
Michael Woerister 2017-09-22 13:00:42 +02:00
parent a7428da415
commit 5974ec745e
11 changed files with 264 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

128
src/librustc/dep_graph/graph.rs
View file

@ -25,6 +25,8 @@ use super::query::DepGraphQuery;
use super::raii;
use super::safe::DepGraphSafe;
use super::edges::{self, DepGraphEdges};
use super::serialized::{SerializedDepGraph, SerializedDepNodeIndex};
use super::prev::PreviousDepGraph;
#[derive(Clone)]
pub struct DepGraph {
@ -68,6 +70,10 @@ struct DepGraphData {
/// current one anymore.
current: RefCell<CurrentDepGraph>,
/// The dep-graph from the previous compilation session. It contains all
/// nodes and edges as well as all fingerprints of nodes that have them.
previous: PreviousDepGraph,
/// When we load, there may be `.o` files, cached mir, or other such
/// things available to us. If we find that they are not dirty, we
/// load the path to the file storing those work-products here into
@ -81,19 +87,24 @@ struct DepGraphData {
}
impl DepGraph {
pub fn new(enabled: bool) -> DepGraph {
pub fn new(prev_graph: PreviousDepGraph) -> DepGraph {
DepGraph {
data: if enabled {
Some(Rc::new(DepGraphData {
previous_work_products: RefCell::new(FxHashMap()),
work_products: RefCell::new(FxHashMap()),
edges: RefCell::new(DepGraphEdges::new()),
dep_node_debug: RefCell::new(FxHashMap()),
current: RefCell::new(CurrentDepGraph::new()),
}))
} else {
None
},
data: Some(Rc::new(DepGraphData {
previous_work_products: RefCell::new(FxHashMap()),
work_products: RefCell::new(FxHashMap()),
edges: RefCell::new(DepGraphEdges::new()),
dep_node_debug: RefCell::new(FxHashMap()),
current: RefCell::new(CurrentDepGraph::new()),
previous: prev_graph,
})),
fingerprints: Rc::new(RefCell::new(FxHashMap())),
}
}
pub fn new_disabled() -> DepGraph {
DepGraph {
data: None,
fingerprints: Rc::new(RefCell::new(FxHashMap())),
}
}
@ -231,7 +242,16 @@ impl DepGraph {
pub fn read(&self, v: DepNode) {
if let Some(ref data) = self.data {
data.edges.borrow_mut().read(v);
data.current.borrow_mut().read(v);
let mut current = data.current.borrow_mut();
debug_assert!(current.node_to_node_index.contains_key(&v),
"DepKind {:?} should be pre-allocated but isn't.",
v.kind);
if let Some(&dep_node_index_new) = current.node_to_node_index.get(&v) {
current.read_index(dep_node_index_new);
} else {
bug!("DepKind {:?} should be pre-allocated but isn't.", v.kind)
}
}
}
@ -254,22 +274,12 @@ impl DepGraph {
self.data.as_ref().unwrap().edges.borrow_mut().add_node(node);
}
pub fn alloc_input_node(&self, node: DepNode) -> DepNodeIndex {
if let Some(ref data) = self.data {
let dep_node_index_legacy = data.edges.borrow_mut().add_node(node);
let dep_node_index_new = data.current.borrow_mut()
.alloc_node(node, Vec::new());
DepNodeIndex {
legacy: dep_node_index_legacy,
new: dep_node_index_new,
}
} else {
DepNodeIndex::INVALID
}
pub fn fingerprint_of(&self, dep_node: &DepNode) -> Fingerprint {
self.fingerprints.borrow()[dep_node]
}
pub fn fingerprint_of(&self, dep_node: &DepNode) -> Option<Fingerprint> {
self.fingerprints.borrow().get(dep_node).cloned()
pub fn prev_fingerprint_of(&self, dep_node: &DepNode) -> Fingerprint {
self.data.as_ref().unwrap().previous.fingerprint_of(dep_node)
}
/// Indicates that a previous work product exists for `v`. This is
@ -338,6 +348,44 @@ impl DepGraph {
pub(super) fn dep_node_debug_str(&self, dep_node: DepNode) -> Option<String> {
self.data.as_ref().and_then(|t| t.dep_node_debug.borrow().get(&dep_node).cloned())
}
pub fn serialize(&self) -> SerializedDepGraph {
let fingerprints = self.fingerprints.borrow();
let current_dep_graph = self.data.as_ref().unwrap().current.borrow();
let nodes: IndexVec<_, _> = current_dep_graph.nodes.iter().map(|dep_node| {
let fingerprint = fingerprints.get(dep_node)
.cloned()
.unwrap_or(Fingerprint::zero());
(*dep_node, fingerprint)
}).collect();
let total_edge_count: usize = current_dep_graph.edges.iter()
.map(|v| v.len())
.sum();
let mut edge_list_indices = IndexVec::with_capacity(nodes.len());
let mut edge_list_data = Vec::with_capacity(total_edge_count);
for (current_dep_node_index, edges) in current_dep_graph.edges.iter_enumerated() {
let start = edge_list_data.len() as u32;
// This should really just be a memcpy :/
edge_list_data.extend(edges.iter().map(|i| SerializedDepNodeIndex(i.index)));
let end = edge_list_data.len() as u32;
debug_assert_eq!(current_dep_node_index.index(), edge_list_indices.len());
edge_list_indices.push((start, end));
}
debug_assert!(edge_list_data.len() <= ::std::u32::MAX as usize);
debug_assert_eq!(edge_list_data.len(), total_edge_count);
SerializedDepGraph {
nodes,
edge_list_indices,
edge_list_data,
}
}
}
/// A "work product" is an intermediate result that we save into the
@ -478,11 +526,6 @@ impl CurrentDepGraph {
}
}
fn read(&mut self, source: DepNode) {
let dep_node_index = self.maybe_alloc_node(source);
self.read_index(dep_node_index);
}
fn read_index(&mut self, source: DepNodeIndexNew) {
match self.task_stack.last_mut() {
Some(&mut OpenTask::Regular {
@ -521,27 +564,6 @@ impl CurrentDepGraph {
self.edges.push(edges);
dep_node_index
}
fn maybe_alloc_node(&mut self,
dep_node: DepNode)
-> DepNodeIndexNew {
debug_assert_eq!(self.edges.len(), self.nodes.len());
debug_assert_eq!(self.node_to_node_index.len(), self.nodes.len());
let CurrentDepGraph {
ref mut node_to_node_index,
ref mut nodes,
ref mut edges,
..
} = *self;
*node_to_node_index.entry(dep_node).or_insert_with(|| {
let next_id = nodes.len();
nodes.push(dep_node);
edges.push(Vec::new());
DepNodeIndexNew::new(next_id)
})
}
}
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]

13
src/librustc/dep_graph/mod.rs
View file

@ -13,18 +13,17 @@ mod dep_node;
mod dep_tracking_map;
mod edges;
mod graph;
mod prev;
mod query;
mod raii;
mod safe;
mod serialized;
pub use self::dep_tracking_map::{DepTrackingMap, DepTrackingMapConfig};
pub use self::dep_node::DepNode;
pub use self::dep_node::WorkProductId;
pub use self::graph::DepGraph;
pub use self::graph::WorkProduct;
pub use self::graph::DepNodeIndex;
pub use self::dep_node::{DepNode, DepKind, DepConstructor, WorkProductId};
pub use self::graph::{DepGraph, WorkProduct, DepNodeIndex};
pub use self::prev::PreviousDepGraph;
pub use self::query::DepGraphQuery;
pub use self::safe::AssertDepGraphSafe;
pub use self::safe::DepGraphSafe;
pub use self::dep_node::{DepKind, DepConstructor};
pub use self::serialized::SerializedDepGraph;

46
src/librustc/dep_graph/prev.rs Normal file
View file

@ -0,0 +1,46 @@
// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
use ich::Fingerprint;
use rustc_data_structures::fx::FxHashMap;
use super::dep_node::DepNode;
use super::serialized::{SerializedDepGraph, SerializedDepNodeIndex};
#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct PreviousDepGraph {
data: SerializedDepGraph,
index: FxHashMap<DepNode, SerializedDepNodeIndex>,
}
impl PreviousDepGraph {
pub fn new(data: SerializedDepGraph) -> PreviousDepGraph {
let index: FxHashMap<_, _> = data.nodes
.iter_enumerated()
.map(|(idx, &(dep_node, _))| (dep_node, idx))
.collect();
PreviousDepGraph { data, index }
}
pub fn with_edges_from<F>(&self, dep_node: &DepNode, mut f: F)
where
F: FnMut(&(DepNode, Fingerprint)),
{
let node_index = self.index[dep_node];
self.data
.edge_targets_from(node_index)
.into_iter()
.for_each(|&index| f(&self.data.nodes[index]));
}
pub fn fingerprint_of(&self, dep_node: &DepNode) -> Fingerprint {
let node_index = self.index[dep_node];
self.data.nodes[node_index].1
}
}

71
src/librustc/dep_graph/serialized.rs Normal file
View file

@ -0,0 +1,71 @@
// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
//! The data that we will serialize and deserialize.
use dep_graph::DepNode;
use ich::Fingerprint;
use rustc_data_structures::indexed_vec::{IndexVec, Idx};
/// The index of a DepNode in the SerializedDepGraph::nodes array.
#[derive(Copy, Clone, Hash, Eq, PartialEq, Ord, PartialOrd, Debug,
RustcEncodable, RustcDecodable)]
pub struct SerializedDepNodeIndex(pub u32);
impl SerializedDepNodeIndex {
#[inline]
pub fn new(idx: usize) -> SerializedDepNodeIndex {
assert!(idx <= ::std::u32::MAX as usize);
SerializedDepNodeIndex(idx as u32)
}
}
impl Idx for SerializedDepNodeIndex {
#[inline]
fn new(idx: usize) -> Self {
assert!(idx <= ::std::u32::MAX as usize);
SerializedDepNodeIndex(idx as u32)
}
#[inline]
fn index(self) -> usize {
self.0 as usize
}
}
/// Data for use when recompiling the **current crate**.
#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct SerializedDepGraph {
/// The set of all DepNodes in the graph
pub nodes: IndexVec<SerializedDepNodeIndex, (DepNode, Fingerprint)>,
/// For each DepNode, stores the list of edges originating from that
/// DepNode. Encoded as a [start, end) pair indexing into edge_list_data,
/// which holds the actual DepNodeIndices of the target nodes.
pub edge_list_indices: IndexVec<SerializedDepNodeIndex, (u32, u32)>,
/// A flattened list of all edge targets in the graph. Edge sources are
/// implicit in edge_list_indices.
pub edge_list_data: Vec<SerializedDepNodeIndex>,
}
impl SerializedDepGraph {
pub fn new() -> SerializedDepGraph {
SerializedDepGraph {
nodes: IndexVec::new(),
edge_list_indices: IndexVec::new(),
edge_list_data: Vec::new(),
}
}
pub fn edge_targets_from(&self, source: SerializedDepNodeIndex) -> &[SerializedDepNodeIndex] {
let targets = self.edge_list_indices[source];
&self.edge_list_data[targets.0 as usize..targets.1 as usize]
}
}

12
src/librustc_driver/driver.rs
View file

@ -643,7 +643,16 @@ pub fn phase_2_configure_and_expand<F>(sess: &Session,
&crate_name,
&disambiguator.as_str(),
);
let dep_graph = DepGraph::new(sess.opts.build_dep_graph());
let dep_graph = if sess.opts.build_dep_graph() {
let prev_dep_graph = time(time_passes, "load prev dep-graph (new)", || {
rustc_incremental::load_dep_graph_new(sess)
});
DepGraph::new(prev_dep_graph)
} else {
DepGraph::new_disabled()
};
time(time_passes, "recursion limit", || {
middle::recursion_limit::update_limits(sess, &krate);
@ -713,7 +722,6 @@ pub fn phase_2_configure_and_expand<F>(sess: &Session,
// item, much like we do for macro expansion. In other words, the hash reflects not just
// its contents but the results of name resolution on those contents. Hopefully we'll push
// this back at some point.
let _ignore = dep_graph.in_ignore();
let mut crate_loader = CrateLoader::new(sess, &cstore, crate_name);
let resolver_arenas = Resolver::arenas();
let mut resolver = Resolver::new(sess,

1
src/librustc_incremental/lib.rs
View file

@ -32,6 +32,7 @@ mod persist;
pub use assert_dep_graph::assert_dep_graph;
pub use persist::load_dep_graph;
pub use persist::load_dep_graph_new;
pub use persist::save_dep_graph;
pub use persist::save_trans_partition;
pub use persist::save_work_products;

5
src/librustc_incremental/persist/fs.rs
View file

@ -132,6 +132,7 @@ use std::__rand::{thread_rng, Rng};
const LOCK_FILE_EXT: &'static str = ".lock";
const DEP_GRAPH_FILENAME: &'static str = "dep-graph.bin";
const DEP_GRAPH_NEW_FILENAME: &'static str = "dep-graph-new.bin";
const WORK_PRODUCTS_FILENAME: &'static str = "work-products.bin";
const METADATA_HASHES_FILENAME: &'static str = "metadata.bin";
@ -145,6 +146,10 @@ pub fn dep_graph_path(sess: &Session) -> PathBuf {
in_incr_comp_dir_sess(sess, DEP_GRAPH_FILENAME)
}
pub fn dep_graph_path_new(sess: &Session) -> PathBuf {
in_incr_comp_dir_sess(sess, DEP_GRAPH_NEW_FILENAME)
}
pub fn work_products_path(sess: &Session) -> PathBuf {
in_incr_comp_dir_sess(sess, WORK_PRODUCTS_FILENAME)
}

35
src/librustc_incremental/persist/load.rs
View file

@ -433,3 +433,38 @@ fn process_edge<'a, 'tcx, 'edges>(
}
}
}
pub fn load_dep_graph_new(sess: &Session) -> PreviousDepGraph {
use rustc::dep_graph::SerializedDepGraph as SerializedDepGraphNew;
let empty = PreviousDepGraph::new(SerializedDepGraphNew::new());
if sess.opts.incremental.is_none() {
return empty
}
if let Some(bytes) = load_data(sess, &dep_graph_path_new(sess)) {
let mut decoder = Decoder::new(&bytes, 0);
let prev_commandline_args_hash = u64::decode(&mut decoder)
.expect("Error reading commandline arg hash from cached dep-graph");
if prev_commandline_args_hash != sess.opts.dep_tracking_hash() {
if sess.opts.debugging_opts.incremental_info {
eprintln!("incremental: completely ignoring cache because of \
differing commandline arguments");
}
// We can't reuse the cache, purge it.
debug!("load_dep_graph_new: differing commandline arg hashes");
// No need to do any further work
return empty
}
let dep_graph = SerializedDepGraphNew::decode(&mut decoder)
.expect("Error reading cached dep-graph");
PreviousDepGraph::new(dep_graph)
} else {
empty
}
}

1
src/librustc_incremental/persist/mod.rs
View file

@ -26,6 +26,7 @@ pub use self::fs::prepare_session_directory;
pub use self::fs::finalize_session_directory;
pub use self::fs::in_incr_comp_dir;
pub use self::load::load_dep_graph;
pub use self::load::load_dep_graph_new;
pub use self::save::save_dep_graph;
pub use self::save::save_work_products;
pub use self::work_product::save_trans_partition;

13
src/librustc_incremental/persist/save.rs
View file

@ -174,6 +174,19 @@ fn save_in<F>(sess: &Session, path_buf: PathBuf, encode: F)
}
}
fn encode_dep_graph_new(tcx: TyCtxt,
encoder: &mut Encoder)
-> io::Result<()> {
// First encode the commandline arguments hash
tcx.sess.opts.dep_tracking_hash().encode(encoder)?;
// Encode the graph data.
let serialized_graph = tcx.dep_graph.serialize();
serialized_graph.encode(encoder)?;
Ok(())
}
pub fn encode_dep_graph(tcx: TyCtxt,
preds: &Predecessors,
encoder: &mut Encoder)

3
src/librustc_trans/base.rs
View file

@ -942,8 +942,7 @@ pub fn trans_crate<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
let crate_hash = tcx.dep_graph
.fingerprint_of(&DepNode::new_no_params(DepKind::Krate))
.unwrap();
.fingerprint_of(&DepNode::new_no_params(DepKind::Krate));
let link_meta = link::build_link_meta(crate_hash);
let exported_symbol_node_ids = find_exported_symbols(tcx);