user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

incr.comp.: Use a more efficient encoding for the on-disk dependency graph.

Browse source
This commit is contained in:
Michael Woerister 2017-06-01 15:23:21 +02:00
parent 0cbb8b58e8
commit a3417bf302
3 changed files with 115 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

41
src/librustc_incremental/persist/data.rs
View file

@ -17,11 +17,20 @@ use rustc::ich::Fingerprint;
use rustc::middle::cstore::EncodedMetadataHash;
use std::sync::Arc;
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::indexed_vec::{IndexVec, Idx};
/// Data for use when recompiling the **current crate**.
#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct SerializedDepGraph {
pub edges: Vec<SerializedEdgeSet>,
/// The set of all DepNodes in the graph
pub nodes: IndexVec<DepNodeIndex, DepNode<DefPathHash>>,
/// For each DepNode, stores the list of edges originating from that
/// DepNode. Encoded as a [start, end) pair indexing into edge_list_data,
/// which holds the actual DepNodeIndices of the target nodes.
pub edge_list_indices: Vec<(u32, u32)>,
/// A flattened list of all edge targets in the graph. Edge sources are
/// implicit in edge_list_indices.
pub edge_list_data: Vec<DepNodeIndex>,
/// These are output nodes that have no incoming edges. We track
/// these separately so that when we reload all edges, we don't
@ -50,12 +59,30 @@ pub struct SerializedDepGraph {
pub hashes: Vec<SerializedHash>,
}
/// Represents a set of "reduced" dependency edge. We group the
/// outgoing edges from a single source together.
#[derive(Debug, RustcEncodable, RustcDecodable)]
pub struct SerializedEdgeSet {
pub source: DepNode<DefPathHash>,
pub targets: Vec<DepNode<DefPathHash>>
/// The index of a DepNode in the SerializedDepGraph::nodes array.
#[derive(Copy, Clone, Hash, Eq, PartialEq, Ord, PartialOrd, Debug,
RustcEncodable, RustcDecodable)]
pub struct DepNodeIndex(pub u32);
impl DepNodeIndex {
#[inline]
pub fn new(idx: usize) -> DepNodeIndex {
assert!(idx <= ::std::u32::MAX as usize);
DepNodeIndex(idx as u32)
}
}
impl Idx for DepNodeIndex {
#[inline]
fn new(idx: usize) -> Self {
assert!(idx <= ::std::u32::MAX as usize);
DepNodeIndex(idx as u32)
}
#[inline]
fn index(self) -> usize {
self.0 as usize
}
}
#[derive(Debug, RustcEncodable, RustcDecodable)]

22
src/librustc_incremental/persist/load.rs
View file

@ -20,6 +20,7 @@ use rustc::ty::TyCtxt;
use rustc_data_structures::fx::{FxHashSet, FxHashMap};
use rustc_serialize::Decodable as RustcDecodable;
use rustc_serialize::opaque::Decoder;
use std::default::Default;
use std::path::{Path};
use std::sync::Arc;
@ -161,10 +162,23 @@ pub fn decode_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
let serialized_dep_graph = SerializedDepGraph::decode(&mut dep_graph_decoder)?;
let edge_map: FxHashMap<_, _> = serialized_dep_graph.edges
.into_iter()
.map(|s| (s.source, s.targets))
.collect();
let edge_map: FxHashMap<DepNode<DefPathHash>, Vec<DepNode<DefPathHash>>> = {
let capacity = serialized_dep_graph.edge_list_data.len();
let mut edge_map = FxHashMap::with_capacity_and_hasher(capacity, Default::default());
for (node_index, source) in serialized_dep_graph.nodes.iter().enumerate() {
let (start, end) = serialized_dep_graph.edge_list_indices[node_index];
let targets =
(&serialized_dep_graph.edge_list_data[start as usize .. end as usize])
.into_iter()
.map(|&node_index| serialized_dep_graph.nodes[node_index].clone())
.collect();
edge_map.insert(source.clone(), targets);
}
edge_map
};
// Compute the set of nodes from the old graph where some input
// has changed or been removed. These are "raw" source nodes,

115
src/librustc_incremental/persist/save.rs
View file

@ -11,11 +11,14 @@
use rustc::dep_graph::DepNode;
use rustc::hir::def_id::DefId;
use rustc::hir::svh::Svh;
use rustc::hir::map::DefPathHash;
use rustc::ich::Fingerprint;
use rustc::middle::cstore::EncodedMetadataHashes;
use rustc::session::Session;
use rustc::ty::TyCtxt;
use rustc_data_structures::fx::FxHashMap;
use rustc_data_structures::graph;
use rustc_data_structures::indexed_vec::IndexVec;
use rustc_serialize::Encodable as RustcEncodable;
use rustc_serialize::opaque::Encoder;
use std::io::{self, Cursor, Write};
@ -175,66 +178,74 @@ pub fn encode_dep_graph(tcx: TyCtxt,
dep_node.map_def(|&def_id| Some(tcx.def_path_hash(def_id))).unwrap()
};
// Create a flat list of (Input, WorkProduct) edges for
// serialization.
let mut edges = FxHashMap();
for edge in preds.reduced_graph.all_edges() {
let source = *preds.reduced_graph.node_data(edge.source());
let target = *preds.reduced_graph.node_data(edge.target());
match *target {
DepNode::MetaData(ref def_id) => {
// Metadata *targets* are always local metadata nodes. We have
// already handled those in `encode_metadata_hashes`.
assert!(def_id.is_local());
continue;
}
_ => (),
// NB: We rely on this Vec being indexable by reduced_graph's NodeIndex.
let nodes: IndexVec<DepNodeIndex, DepNode<DefPathHash>> = preds
.reduced_graph
.all_nodes()
.iter()
.map(|node| to_hash_based_node(node.data))
.collect();
let mut edge_list_indices = Vec::with_capacity(nodes.len());
let mut edge_list_data = Vec::with_capacity(preds.reduced_graph.len_edges());
for node_index in 0 .. nodes.len() {
let start = edge_list_data.len() as u32;
for target in preds.reduced_graph.successor_nodes(graph::NodeIndex(node_index)) {
edge_list_data.push(DepNodeIndex::new(target.node_id()));
}
debug!("serialize edge: {:?} -> {:?}", source, target);
let source = to_hash_based_node(source);
let target = to_hash_based_node(target);
edges.entry(source).or_insert(vec![]).push(target);
let end = edge_list_data.len() as u32;
debug_assert_eq!(node_index, edge_list_indices.len());
edge_list_indices.push((start, end));
}
// Let's make we had no overflow there.
assert!(edge_list_data.len() <= ::std::u32::MAX as usize);
// Check that we have a consistent number of edges.
assert_eq!(edge_list_data.len(), preds.reduced_graph.len_edges());
let bootstrap_outputs = preds
.bootstrap_outputs
.iter()
.map(|n| to_hash_based_node(n))
.collect();
let hashes = preds
.hashes
.iter()
.map(|(&dep_node, &hash)| {
SerializedHash {
dep_node: to_hash_based_node(dep_node),
hash: hash,
}
})
.collect();
let graph = SerializedDepGraph {
nodes,
edge_list_indices,
edge_list_data,
bootstrap_outputs,
hashes,
};
// Encode the graph data.
graph.encode(encoder)?;
if tcx.sess.opts.debugging_opts.incremental_info {
println!("incremental: {} nodes in reduced dep-graph", graph.nodes.len());
println!("incremental: {} edges in serialized dep-graph", graph.edge_list_data.len());
println!("incremental: {} hashes in serialized dep-graph", graph.hashes.len());
}
if tcx.sess.opts.debugging_opts.incremental_dump_hash {
for (dep_node, hash) in &preds.hashes {
println!("HIR hash for {:?} is {}", dep_node, hash);
println!("ICH for {:?} is {}", dep_node, hash);
}
}
// Create the serialized dep-graph.
let bootstrap_outputs = preds.bootstrap_outputs
.iter()
.map(|n| to_hash_based_node(n))
.collect();
let edges = edges.into_iter()
.map(|(k, v)| SerializedEdgeSet { source: k, targets: v })
.collect();
let graph = SerializedDepGraph {
bootstrap_outputs,
edges,
hashes: preds.hashes
.iter()
.map(|(&dep_node, &hash)| {
SerializedHash {
dep_node: to_hash_based_node(dep_node),
hash: hash,
}
})
.collect(),
};
if tcx.sess.opts.debugging_opts.incremental_info {
println!("incremental: {} nodes in reduced dep-graph", preds.reduced_graph.len_nodes());
println!("incremental: {} edges in serialized dep-graph", graph.edges.len());
println!("incremental: {} hashes in serialized dep-graph", graph.hashes.len());
}
debug!("graph = {:#?}", graph);
// Encode the graph data.
graph.encode(encoder)?;
Ok(())
}