Rollup merge of #37233 - michaelwoerister:blake2-for-ich, r=nikomatsakis
ICH: Use 128-bit Blake2b hash instead of 64-bit SipHash for incr. comp. fingerprints This PR makes incr. comp. hashes 128 bits wide in order to push collision probability below a threshold that we need to worry about. It also replaces SipHash, which has been mentioned multiple times as not being built for fingerprinting, with the [BLAKE2b hash function](https://blake2.net/), an improved version of the BLAKE sha-3 finalist. I was worried that using a cryptographic hash function would make ICH computation noticeably slower, but after doing some performance tests, I'm not any more. Most of the time BLAKE2b is actually faster than using two SipHashes (in order to get 128 bits): ``` SipHash libcore: 0.199 seconds libstd: 0.090 seconds BLAKE2b libcore: 0.162 seconds libstd: 0.078 seconds ``` If someone can prove that something like MetroHash128 provides a comparably low collision probability as BLAKE2, I'm happy to switch. But for now we are at least not taking a performance hit. I also suggest that we throw out the sha-256 implementation in the compiler and replace it with BLAKE2, since our sha-256 implementation is two to three times slower than the BLAKE2 implementation in this PR (cc @alexcrichton @eddyb @brson) r? @nikomatsakis (although there's not much incr. comp. specific in here, so feel free to re-assign)
This commit is contained in:
Eduard-Mihai Burtescu
GitHub
commit
7343291ac3
14 changed files with 460 additions and 38 deletions
286
src/librustc_data_structures/blake2b.rs
Normal file
286
src/librustc_data_structures/blake2b.rs
Normal file
|
|
@ -0,0 +1,286 @@
|
|||
// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
|
||||
// file at the top-level directory of this distribution and at
|
||||
// http://rust-lang.org/COPYRIGHT.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
||||
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
||||
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
||||
// option. This file may not be copied, modified, or distributed
|
||||
// except according to those terms.
|
||||
|
||||
|
||||
// An implementation of the Blake2b cryptographic hash function.
|
||||
// The implementation closely follows: https://tools.ietf.org/html/rfc7693
|
||||
//
|
||||
// "BLAKE2 is a cryptographic hash function faster than MD5, SHA-1, SHA-2, and
|
||||
// SHA-3, yet is at least as secure as the latest standard SHA-3."
|
||||
// according to their own website :)
|
||||
//
|
||||
// Indeed this implementation is two to three times as fast as our SHA-256
|
||||
// implementation. If you have the luxury of being able to use crates from
|
||||
// crates.io, you can go there and find still faster implementations.
|
||||
|
||||
pub struct Blake2bCtx {
|
||||
b: [u8; 128],
|
||||
h: [u64; 8],
|
||||
t: [u64; 2],
|
||||
c: usize,
|
||||
outlen: usize,
|
||||
}
|
||||
|
||||
impl ::std::fmt::Debug for Blake2bCtx {
|
||||
fn fmt(&self, fmt: &mut ::std::fmt::Formatter) -> Result<(), ::std::fmt::Error> {
|
||||
write!(fmt, "{:?}", self.h)
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn b2b_g(v: &mut [u64; 16],
|
||||
a: usize,
|
||||
b: usize,
|
||||
c: usize,
|
||||
d: usize,
|
||||
x: u64,
|
||||
y: u64)
|
||||
{
|
||||
v[a] = v[a].wrapping_add(v[b]).wrapping_add(x);
|
||||
v[d] = (v[d] ^ v[a]).rotate_right(32);
|
||||
v[c] = v[c].wrapping_add(v[d]);
|
||||
v[b] = (v[b] ^ v[c]).rotate_right(24);
|
||||
v[a] = v[a].wrapping_add(v[b]).wrapping_add(y);
|
||||
v[d] = (v[d] ^ v[a]).rotate_right(16);
|
||||
v[c] = v[c].wrapping_add(v[d]);
|
||||
v[b] = (v[b] ^ v[c]).rotate_right(63);
|
||||
}
|
||||
|
||||
// Initialization vector
|
||||
const BLAKE2B_IV: [u64; 8] = [
|
||||
0x6A09E667F3BCC908, 0xBB67AE8584CAA73B,
|
||||
0x3C6EF372FE94F82B, 0xA54FF53A5F1D36F1,
|
||||
0x510E527FADE682D1, 0x9B05688C2B3E6C1F,
|
||||
0x1F83D9ABFB41BD6B, 0x5BE0CD19137E2179
|
||||
];
|
||||
|
||||
fn blake2b_compress(ctx: &mut Blake2bCtx, last: bool) {
|
||||
|
||||
const SIGMA: [[usize; 16]; 12] = [
|
||||
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ],
|
||||
[14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 ],
|
||||
[11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 ],
|
||||
[7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 ],
|
||||
[9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 ],
|
||||
[2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 ],
|
||||
[12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 ],
|
||||
[13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 ],
|
||||
[6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 ],
|
||||
[10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 ],
|
||||
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 ],
|
||||
[14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 ]
|
||||
];
|
||||
|
||||
let mut v: [u64; 16] = [
|
||||
ctx.h[0],
|
||||
ctx.h[1],
|
||||
ctx.h[2],
|
||||
ctx.h[3],
|
||||
ctx.h[4],
|
||||
ctx.h[5],
|
||||
ctx.h[6],
|
||||
ctx.h[7],
|
||||
|
||||
BLAKE2B_IV[0],
|
||||
BLAKE2B_IV[1],
|
||||
BLAKE2B_IV[2],
|
||||
BLAKE2B_IV[3],
|
||||
BLAKE2B_IV[4],
|
||||
BLAKE2B_IV[5],
|
||||
BLAKE2B_IV[6],
|
||||
BLAKE2B_IV[7],
|
||||
];
|
||||
|
||||
v[12] ^= ctx.t[0]; // low 64 bits of offset
|
||||
v[13] ^= ctx.t[1]; // high 64 bits
|
||||
if last {
|
||||
v[14] = !v[14];
|
||||
}
|
||||
|
||||
{
|
||||
// Re-interpret the input buffer in the state as u64s
|
||||
let m: &mut [u64; 16] = unsafe {
|
||||
let b: &mut [u8; 128] = &mut ctx.b;
|
||||
::std::mem::transmute(b)
|
||||
};
|
||||
|
||||
// It's OK to modify the buffer in place since this is the last time
|
||||
// this data will be accessed before it's overwritten
|
||||
if cfg!(target_endian = "big") {
|
||||
for word in &mut m[..] {
|
||||
*word = word.to_be();
|
||||
}
|
||||
}
|
||||
|
||||
for i in 0 .. 12 {
|
||||
b2b_g(&mut v, 0, 4, 8, 12, m[SIGMA[i][ 0]], m[SIGMA[i][ 1]]);
|
||||
b2b_g(&mut v, 1, 5, 9, 13, m[SIGMA[i][ 2]], m[SIGMA[i][ 3]]);
|
||||
b2b_g(&mut v, 2, 6, 10, 14, m[SIGMA[i][ 4]], m[SIGMA[i][ 5]]);
|
||||
b2b_g(&mut v, 3, 7, 11, 15, m[SIGMA[i][ 6]], m[SIGMA[i][ 7]]);
|
||||
b2b_g(&mut v, 0, 5, 10, 15, m[SIGMA[i][ 8]], m[SIGMA[i][ 9]]);
|
||||
b2b_g(&mut v, 1, 6, 11, 12, m[SIGMA[i][10]], m[SIGMA[i][11]]);
|
||||
b2b_g(&mut v, 2, 7, 8, 13, m[SIGMA[i][12]], m[SIGMA[i][13]]);
|
||||
b2b_g(&mut v, 3, 4, 9, 14, m[SIGMA[i][14]], m[SIGMA[i][15]]);
|
||||
}
|
||||
}
|
||||
|
||||
for i in 0 .. 8 {
|
||||
ctx.h[i] ^= v[i] ^ v[i + 8];
|
||||
}
|
||||
}
|
||||
|
||||
pub fn blake2b_new(outlen: usize, key: &[u8]) -> Blake2bCtx {
|
||||
assert!(outlen > 0 && outlen <= 64 && key.len() <= 64);
|
||||
|
||||
let mut ctx = Blake2bCtx {
|
||||
b: [0; 128],
|
||||
h: BLAKE2B_IV,
|
||||
t: [0; 2],
|
||||
c: 0,
|
||||
outlen: outlen,
|
||||
};
|
||||
|
||||
ctx.h[0] ^= 0x01010000 ^ ((key.len() << 8) as u64) ^ (outlen as u64);
|
||||
|
||||
if key.len() > 0 {
|
||||
blake2b_update(&mut ctx, key);
|
||||
ctx.c = ctx.b.len();
|
||||
}
|
||||
|
||||
ctx
|
||||
}
|
||||
|
||||
pub fn blake2b_update(ctx: &mut Blake2bCtx, mut data: &[u8])
|
||||
{
|
||||
let mut bytes_to_copy = data.len();
|
||||
let mut space_in_buffer = ctx.b.len() - ctx.c;
|
||||
|
||||
while bytes_to_copy > space_in_buffer {
|
||||
checked_mem_copy(data, &mut ctx.b[ctx.c .. ], space_in_buffer);
|
||||
|
||||
ctx.t[0] = ctx.t[0].wrapping_add(ctx.b.len() as u64);
|
||||
if ctx.t[0] < (ctx.b.len() as u64) {
|
||||
ctx.t[1] += 1;
|
||||
}
|
||||
blake2b_compress(ctx, false);
|
||||
ctx.c = 0;
|
||||
|
||||
data = &data[space_in_buffer .. ];
|
||||
bytes_to_copy -= space_in_buffer;
|
||||
space_in_buffer = ctx.b.len();
|
||||
}
|
||||
|
||||
if bytes_to_copy > 0 {
|
||||
checked_mem_copy(data, &mut ctx.b[ctx.c .. ], bytes_to_copy);
|
||||
ctx.c += bytes_to_copy;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn blake2b_final(mut ctx: Blake2bCtx, out: &mut [u8])
|
||||
{
|
||||
ctx.t[0] = ctx.t[0].wrapping_add(ctx.c as u64);
|
||||
if ctx.t[0] < ctx.c as u64 {
|
||||
ctx.t[1] += 1;
|
||||
}
|
||||
|
||||
while ctx.c < 128 {
|
||||
ctx.b[ctx.c] = 0;
|
||||
ctx.c += 1;
|
||||
}
|
||||
|
||||
blake2b_compress(&mut ctx, true);
|
||||
|
||||
if cfg!(target_endian = "big") {
|
||||
// Make sure that the data is in memory in little endian format, as is
|
||||
// demanded by BLAKE2
|
||||
for word in &mut ctx.h {
|
||||
*word = word.to_le();
|
||||
}
|
||||
}
|
||||
|
||||
checked_mem_copy(&ctx.h, out, ctx.outlen);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn checked_mem_copy<T1, T2>(from: &[T1], to: &mut [T2], byte_count: usize) {
|
||||
let from_size = from.len() * ::std::mem::size_of::<T1>();
|
||||
let to_size = to.len() * ::std::mem::size_of::<T2>();
|
||||
assert!(from_size >= byte_count);
|
||||
assert!(to_size >= byte_count);
|
||||
let from_byte_ptr = from.as_ptr() as * const u8;
|
||||
let to_byte_ptr = to.as_mut_ptr() as * mut u8;
|
||||
unsafe {
|
||||
::std::ptr::copy_nonoverlapping(from_byte_ptr, to_byte_ptr, byte_count);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn blake2b(out: &mut [u8], key: &[u8], data: &[u8])
|
||||
{
|
||||
let mut ctx = blake2b_new(out.len(), key);
|
||||
blake2b_update(&mut ctx, data);
|
||||
blake2b_final(ctx, out);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn selftest_seq(out: &mut [u8], seed: u32)
|
||||
{
|
||||
let mut a: u32 = 0xDEAD4BADu32.wrapping_mul(seed);
|
||||
let mut b: u32 = 1;
|
||||
|
||||
for i in 0 .. out.len() {
|
||||
let t: u32 = a.wrapping_add(b);
|
||||
a = b;
|
||||
b = t;
|
||||
out[i] = ((t >> 24) & 0xFF) as u8;
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blake2b_selftest()
|
||||
{
|
||||
// grand hash of hash results
|
||||
const BLAKE2B_RES: [u8; 32] = [
|
||||
0xC2, 0x3A, 0x78, 0x00, 0xD9, 0x81, 0x23, 0xBD,
|
||||
0x10, 0xF5, 0x06, 0xC6, 0x1E, 0x29, 0xDA, 0x56,
|
||||
0x03, 0xD7, 0x63, 0xB8, 0xBB, 0xAD, 0x2E, 0x73,
|
||||
0x7F, 0x5E, 0x76, 0x5A, 0x7B, 0xCC, 0xD4, 0x75
|
||||
];
|
||||
|
||||
// parameter sets
|
||||
const B2B_MD_LEN: [usize; 4] = [20, 32, 48, 64];
|
||||
const B2B_IN_LEN: [usize; 6] = [0, 3, 128, 129, 255, 1024];
|
||||
|
||||
let mut data = [0u8; 1024];
|
||||
let mut md = [0u8; 64];
|
||||
let mut key = [0u8; 64];
|
||||
|
||||
let mut ctx = blake2b_new(32, &[]);
|
||||
|
||||
for i in 0 .. 4 {
|
||||
let outlen = B2B_MD_LEN[i];
|
||||
for j in 0 .. 6 {
|
||||
let inlen = B2B_IN_LEN[j];
|
||||
|
||||
selftest_seq(&mut data[.. inlen], inlen as u32); // unkeyed hash
|
||||
blake2b(&mut md[.. outlen], &[], &data[.. inlen]);
|
||||
blake2b_update(&mut ctx, &md[.. outlen]); // hash the hash
|
||||
|
||||
selftest_seq(&mut key[0 .. outlen], outlen as u32); // keyed hash
|
||||
blake2b(&mut md[.. outlen], &key[.. outlen], &data[.. inlen]);
|
||||
blake2b_update(&mut ctx, &md[.. outlen]); // hash the hash
|
||||
}
|
||||
}
|
||||
|
||||
// compute and compare the hash of hashes
|
||||
blake2b_final(ctx, &mut md[..]);
|
||||
for i in 0 .. 32 {
|
||||
assert_eq!(md[i], BLAKE2B_RES[i]);
|
||||
}
|
||||
}
|
||||
|
|
@ -42,6 +42,7 @@ extern crate serialize as rustc_serialize; // used by deriving
|
|||
extern crate libc;
|
||||
|
||||
pub mod bitslice;
|
||||
pub mod blake2b;
|
||||
pub mod bitvec;
|
||||
pub mod graph;
|
||||
pub mod ivar;
|
||||
|
|
|
|||
|
|
@ -9,20 +9,20 @@
|
|||
// except according to those terms.
|
||||
|
||||
use std::hash::Hasher;
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
use std::mem;
|
||||
use rustc_data_structures::blake2b;
|
||||
use ich::Fingerprint;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct IchHasher {
|
||||
// FIXME: this should use SHA1, not DefaultHasher. DefaultHasher is not
|
||||
// built to avoid collisions.
|
||||
state: DefaultHasher,
|
||||
state: blake2b::Blake2bCtx,
|
||||
bytes_hashed: u64,
|
||||
}
|
||||
|
||||
impl IchHasher {
|
||||
pub fn new() -> IchHasher {
|
||||
IchHasher {
|
||||
state: DefaultHasher::new(),
|
||||
state: blake2b::blake2b_new(mem::size_of::<Fingerprint>(), &[]),
|
||||
bytes_hashed: 0
|
||||
}
|
||||
}
|
||||
|
|
@ -30,17 +30,43 @@ impl IchHasher {
|
|||
pub fn bytes_hashed(&self) -> u64 {
|
||||
self.bytes_hashed
|
||||
}
|
||||
|
||||
pub fn finish(self) -> Fingerprint {
|
||||
let mut fingerprint = Fingerprint::zero();
|
||||
blake2b::blake2b_final(self.state, &mut fingerprint.0);
|
||||
fingerprint
|
||||
}
|
||||
}
|
||||
|
||||
impl Hasher for IchHasher {
|
||||
#[inline]
|
||||
fn finish(&self) -> u64 {
|
||||
self.state.finish()
|
||||
bug!("Use other finish() implementation to get the full 128-bit hash.");
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn write(&mut self, bytes: &[u8]) {
|
||||
self.state.write(bytes);
|
||||
blake2b::blake2b_update(&mut self.state, bytes);
|
||||
self.bytes_hashed += bytes.len() as u64;
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn write_u16(&mut self, i: u16) {
|
||||
self.write(&unsafe { mem::transmute::<_, [u8; 2]>(i.to_le()) })
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn write_u32(&mut self, i: u32) {
|
||||
self.write(&unsafe { mem::transmute::<_, [u8; 4]>(i.to_le()) })
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn write_u64(&mut self, i: u64) {
|
||||
self.write(&unsafe { mem::transmute::<_, [u8; 8]>(i.to_le()) })
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn write_usize(&mut self, i: usize) {
|
||||
// always hash as u64, so we don't depend on the size of `usize`
|
||||
self.write_u64(i as u64);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@
|
|||
|
||||
use syntax::ast;
|
||||
use std::cell::RefCell;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::hash::Hash;
|
||||
use rustc::dep_graph::DepNode;
|
||||
use rustc::hir;
|
||||
use rustc::hir::def_id::{CRATE_DEF_INDEX, DefId};
|
||||
|
|
@ -43,21 +43,22 @@ use self::def_path_hash::DefPathHashes;
|
|||
use self::svh_visitor::StrictVersionHashVisitor;
|
||||
use self::caching_codemap_view::CachingCodemapView;
|
||||
use self::hasher::IchHasher;
|
||||
use ich::Fingerprint;
|
||||
|
||||
mod def_path_hash;
|
||||
mod svh_visitor;
|
||||
mod caching_codemap_view;
|
||||
mod hasher;
|
||||
pub mod hasher;
|
||||
|
||||
pub struct IncrementalHashesMap {
|
||||
hashes: FnvHashMap<DepNode<DefId>, u64>,
|
||||
hashes: FnvHashMap<DepNode<DefId>, Fingerprint>,
|
||||
|
||||
// These are the metadata hashes for the current crate as they were stored
|
||||
// during the last compilation session. They are only loaded if
|
||||
// -Z query-dep-graph was specified and are needed for auto-tests using
|
||||
// the #[rustc_metadata_dirty] and #[rustc_metadata_clean] attributes to
|
||||
// check whether some metadata hash has changed in between two revisions.
|
||||
pub prev_metadata_hashes: RefCell<FnvHashMap<DefId, u64>>,
|
||||
pub prev_metadata_hashes: RefCell<FnvHashMap<DefId, Fingerprint>>,
|
||||
}
|
||||
|
||||
impl IncrementalHashesMap {
|
||||
|
|
@ -68,11 +69,12 @@ impl IncrementalHashesMap {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn insert(&mut self, k: DepNode<DefId>, v: u64) -> Option<u64> {
|
||||
pub fn insert(&mut self, k: DepNode<DefId>, v: Fingerprint) -> Option<Fingerprint> {
|
||||
self.hashes.insert(k, v)
|
||||
}
|
||||
|
||||
pub fn iter<'a>(&'a self) -> ::std::collections::hash_map::Iter<'a, DepNode<DefId>, u64> {
|
||||
pub fn iter<'a>(&'a self)
|
||||
-> ::std::collections::hash_map::Iter<'a, DepNode<DefId>, Fingerprint> {
|
||||
self.hashes.iter()
|
||||
}
|
||||
|
||||
|
|
@ -82,9 +84,9 @@ impl IncrementalHashesMap {
|
|||
}
|
||||
|
||||
impl<'a> ::std::ops::Index<&'a DepNode<DefId>> for IncrementalHashesMap {
|
||||
type Output = u64;
|
||||
type Output = Fingerprint;
|
||||
|
||||
fn index(&self, index: &'a DepNode<DefId>) -> &u64 {
|
||||
fn index(&self, index: &'a DepNode<DefId>) -> &Fingerprint {
|
||||
&self.hashes[index]
|
||||
}
|
||||
}
|
||||
|
|
@ -141,12 +143,13 @@ impl<'a, 'tcx> HashItemsVisitor<'a, 'tcx> {
|
|||
&mut self.def_path_hashes,
|
||||
&mut self.codemap,
|
||||
self.hash_spans));
|
||||
let bytes_hashed = state.bytes_hashed();
|
||||
let item_hash = state.finish();
|
||||
self.hashes.insert(DepNode::Hir(def_id), item_hash);
|
||||
debug!("calculate_item_hash: def_id={:?} hash={:?}", def_id, item_hash);
|
||||
|
||||
let bytes_hashed = self.tcx.sess.perf_stats.incr_comp_bytes_hashed.get() +
|
||||
state.bytes_hashed();
|
||||
bytes_hashed;
|
||||
self.tcx.sess.perf_stats.incr_comp_bytes_hashed.set(bytes_hashed);
|
||||
}
|
||||
|
||||
|
|
|
|||
81
src/librustc_incremental/ich/fingerprint.rs
Normal file
81
src/librustc_incremental/ich/fingerprint.rs
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
|
||||
// file at the top-level directory of this distribution and at
|
||||
// http://rust-lang.org/COPYRIGHT.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
||||
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
||||
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
||||
// option. This file may not be copied, modified, or distributed
|
||||
// except according to those terms.
|
||||
|
||||
use rustc_serialize::{Encodable, Decodable, Encoder, Decoder};
|
||||
|
||||
const FINGERPRINT_LENGTH: usize = 16;
|
||||
|
||||
#[derive(Eq, PartialEq, Ord, PartialOrd, Hash, Debug, Clone, Copy)]
|
||||
pub struct Fingerprint(pub [u8; FINGERPRINT_LENGTH]);
|
||||
|
||||
impl Fingerprint {
|
||||
#[inline]
|
||||
pub fn zero() -> Fingerprint {
|
||||
Fingerprint([0; FINGERPRINT_LENGTH])
|
||||
}
|
||||
|
||||
pub fn from_smaller_hash(hash: u64) -> Fingerprint {
|
||||
let mut result = Fingerprint::zero();
|
||||
result.0[0] = (hash >> 0) as u8;
|
||||
result.0[1] = (hash >> 8) as u8;
|
||||
result.0[2] = (hash >> 16) as u8;
|
||||
result.0[3] = (hash >> 24) as u8;
|
||||
result.0[4] = (hash >> 32) as u8;
|
||||
result.0[5] = (hash >> 40) as u8;
|
||||
result.0[6] = (hash >> 48) as u8;
|
||||
result.0[7] = (hash >> 56) as u8;
|
||||
result
|
||||
}
|
||||
|
||||
pub fn to_smaller_hash(&self) -> u64 {
|
||||
((self.0[0] as u64) << 0) |
|
||||
((self.0[1] as u64) << 8) |
|
||||
((self.0[2] as u64) << 16) |
|
||||
((self.0[3] as u64) << 24) |
|
||||
((self.0[4] as u64) << 32) |
|
||||
((self.0[5] as u64) << 40) |
|
||||
((self.0[6] as u64) << 48) |
|
||||
((self.0[7] as u64) << 56)
|
||||
}
|
||||
}
|
||||
|
||||
impl Encodable for Fingerprint {
|
||||
#[inline]
|
||||
fn encode<S: Encoder>(&self, s: &mut S) -> Result<(), S::Error> {
|
||||
for &byte in &self.0[..] {
|
||||
s.emit_u8(byte)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Decodable for Fingerprint {
|
||||
#[inline]
|
||||
fn decode<D: Decoder>(d: &mut D) -> Result<Fingerprint, D::Error> {
|
||||
let mut result = Fingerprint([0u8; FINGERPRINT_LENGTH]);
|
||||
for byte in &mut result.0[..] {
|
||||
*byte = d.read_u8()?;
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
impl ::std::fmt::Display for Fingerprint {
|
||||
fn fmt(&self, formatter: &mut ::std::fmt::Formatter) -> Result<(), ::std::fmt::Error> {
|
||||
for i in 0 .. self.0.len() {
|
||||
if i > 0 {
|
||||
write!(formatter, "::")?;
|
||||
}
|
||||
|
||||
write!(formatter, "{}", self.0[i])?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
13
src/librustc_incremental/ich/mod.rs
Normal file
13
src/librustc_incremental/ich/mod.rs
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
|
||||
// file at the top-level directory of this distribution and at
|
||||
// http://rust-lang.org/COPYRIGHT.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
|
||||
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
|
||||
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
|
||||
// option. This file may not be copied, modified, or distributed
|
||||
// except according to those terms.
|
||||
|
||||
pub use self::fingerprint::Fingerprint;
|
||||
|
||||
mod fingerprint;
|
||||
|
|
@ -45,6 +45,7 @@ const ATTR_THEN_THIS_WOULD_NEED: &'static str = "rustc_then_this_would_need";
|
|||
mod assert_dep_graph;
|
||||
mod calculate_svh;
|
||||
mod persist;
|
||||
pub mod ich;
|
||||
|
||||
pub use assert_dep_graph::assert_dep_graph;
|
||||
pub use calculate_svh::compute_incremental_hashes_map;
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ use rustc::dep_graph::{DepNode, WorkProduct, WorkProductId};
|
|||
use rustc::hir::def_id::DefIndex;
|
||||
use std::sync::Arc;
|
||||
use rustc_data_structures::fnv::FnvHashMap;
|
||||
use ich::Fingerprint;
|
||||
|
||||
use super::directory::DefPathIndex;
|
||||
|
||||
|
|
@ -60,7 +61,7 @@ pub struct SerializedHash {
|
|||
|
||||
/// the hash as of previous compilation, computed by code in
|
||||
/// `hash` module
|
||||
pub hash: u64,
|
||||
pub hash: Fingerprint,
|
||||
}
|
||||
|
||||
#[derive(Debug, RustcEncodable, RustcDecodable)]
|
||||
|
|
@ -116,5 +117,5 @@ pub struct SerializedMetadataHash {
|
|||
pub def_index: DefIndex,
|
||||
|
||||
/// the hash itself, computed by `calculate_item_hash`
|
||||
pub hash: u64,
|
||||
pub hash: Fingerprint,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@ use rustc_data_structures::fnv::{FnvHashSet, FnvHashMap};
|
|||
use syntax::parse::token::InternedString;
|
||||
use syntax_pos::Span;
|
||||
use rustc::ty::TyCtxt;
|
||||
use ich::Fingerprint;
|
||||
|
||||
use {ATTR_DIRTY, ATTR_CLEAN, ATTR_DIRTY_METADATA, ATTR_CLEAN_METADATA};
|
||||
|
||||
|
|
@ -186,8 +187,8 @@ impl<'a, 'tcx> Visitor<'tcx> for DirtyCleanVisitor<'a, 'tcx> {
|
|||
}
|
||||
|
||||
pub fn check_dirty_clean_metadata<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
|
||||
prev_metadata_hashes: &FnvHashMap<DefId, u64>,
|
||||
current_metadata_hashes: &FnvHashMap<DefId, u64>) {
|
||||
prev_metadata_hashes: &FnvHashMap<DefId, Fingerprint>,
|
||||
current_metadata_hashes: &FnvHashMap<DefId, Fingerprint>) {
|
||||
if !tcx.sess.opts.debugging_opts.query_dep_graph {
|
||||
return;
|
||||
}
|
||||
|
|
@ -204,8 +205,8 @@ pub fn check_dirty_clean_metadata<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
|
|||
|
||||
pub struct DirtyCleanMetadataVisitor<'a, 'tcx:'a, 'm> {
|
||||
tcx: TyCtxt<'a, 'tcx, 'tcx>,
|
||||
prev_metadata_hashes: &'m FnvHashMap<DefId, u64>,
|
||||
current_metadata_hashes: &'m FnvHashMap<DefId, u64>,
|
||||
prev_metadata_hashes: &'m FnvHashMap<DefId, Fingerprint>,
|
||||
current_metadata_hashes: &'m FnvHashMap<DefId, Fingerprint>,
|
||||
}
|
||||
|
||||
impl<'a, 'tcx, 'm> Visitor<'tcx> for DirtyCleanMetadataVisitor<'a, 'tcx, 'm> {
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ use rustc_serialize::Decodable;
|
|||
use rustc_serialize::opaque::Decoder;
|
||||
|
||||
use IncrementalHashesMap;
|
||||
use ich::Fingerprint;
|
||||
use super::data::*;
|
||||
use super::fs::*;
|
||||
use super::file_format;
|
||||
|
|
@ -25,7 +26,7 @@ use super::file_format;
|
|||
pub struct HashContext<'a, 'tcx: 'a> {
|
||||
pub tcx: TyCtxt<'a, 'tcx, 'tcx>,
|
||||
incremental_hashes_map: &'a IncrementalHashesMap,
|
||||
item_metadata_hashes: FnvHashMap<DefId, u64>,
|
||||
item_metadata_hashes: FnvHashMap<DefId, Fingerprint>,
|
||||
crate_hashes: FnvHashMap<CrateNum, Svh>,
|
||||
}
|
||||
|
||||
|
|
@ -50,7 +51,7 @@ impl<'a, 'tcx> HashContext<'a, 'tcx> {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn hash(&mut self, dep_node: &DepNode<DefId>) -> Option<u64> {
|
||||
pub fn hash(&mut self, dep_node: &DepNode<DefId>) -> Option<Fingerprint> {
|
||||
match *dep_node {
|
||||
DepNode::Krate => {
|
||||
Some(self.incremental_hashes_map[dep_node])
|
||||
|
|
@ -89,7 +90,7 @@ impl<'a, 'tcx> HashContext<'a, 'tcx> {
|
|||
}
|
||||
}
|
||||
|
||||
fn metadata_hash(&mut self, def_id: DefId) -> u64 {
|
||||
fn metadata_hash(&mut self, def_id: DefId) -> Fingerprint {
|
||||
debug!("metadata_hash(def_id={:?})", def_id);
|
||||
|
||||
assert!(!def_id.is_local());
|
||||
|
|
@ -102,14 +103,15 @@ impl<'a, 'tcx> HashContext<'a, 'tcx> {
|
|||
|
||||
// check whether we did not find detailed metadata for this
|
||||
// krate; in that case, we just use the krate's overall hash
|
||||
if let Some(&hash) = self.crate_hashes.get(&def_id.krate) {
|
||||
debug!("metadata_hash: def_id={:?} crate_hash={:?}", def_id, hash);
|
||||
if let Some(&svh) = self.crate_hashes.get(&def_id.krate) {
|
||||
debug!("metadata_hash: def_id={:?} crate_hash={:?}", def_id, svh);
|
||||
|
||||
// micro-"optimization": avoid a cache miss if we ask
|
||||
// for metadata from this particular def-id again.
|
||||
self.item_metadata_hashes.insert(def_id, hash.as_u64());
|
||||
let fingerprint = svh_to_fingerprint(svh);
|
||||
self.item_metadata_hashes.insert(def_id, fingerprint);
|
||||
|
||||
return hash.as_u64();
|
||||
return fingerprint;
|
||||
}
|
||||
|
||||
// otherwise, load the data and repeat.
|
||||
|
|
@ -206,3 +208,7 @@ impl<'a, 'tcx> HashContext<'a, 'tcx> {
|
|||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn svh_to_fingerprint(svh: Svh) -> Fingerprint {
|
||||
Fingerprint::from_smaller_hash(svh.as_u64())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ use std::fs;
|
|||
use std::path::{Path};
|
||||
|
||||
use IncrementalHashesMap;
|
||||
use ich::Fingerprint;
|
||||
use super::data::*;
|
||||
use super::directory::*;
|
||||
use super::dirty_clean;
|
||||
|
|
@ -315,7 +316,7 @@ fn delete_dirty_work_product(tcx: TyCtxt,
|
|||
|
||||
fn load_prev_metadata_hashes(tcx: TyCtxt,
|
||||
retraced: &RetracedDefIdDirectory,
|
||||
output: &mut FnvHashMap<DefId, u64>) {
|
||||
output: &mut FnvHashMap<DefId, Fingerprint>) {
|
||||
if !tcx.sess.opts.debugging_opts.query_dep_graph {
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ use rustc_data_structures::fnv::FnvHashMap;
|
|||
use rustc_data_structures::graph::{DepthFirstTraversal, INCOMING, NodeIndex};
|
||||
|
||||
use super::hash::*;
|
||||
use ich::Fingerprint;
|
||||
|
||||
/// A data-structure that makes it easy to enumerate the hashable
|
||||
/// predecessors of any given dep-node.
|
||||
|
|
@ -26,7 +27,7 @@ pub struct Predecessors<'query> {
|
|||
|
||||
// - Keys: some hashable node
|
||||
// - Values: the hash thereof
|
||||
pub hashes: FnvHashMap<&'query DepNode<DefId>, u64>,
|
||||
pub hashes: FnvHashMap<&'query DepNode<DefId>, Fingerprint>,
|
||||
}
|
||||
|
||||
impl<'q> Predecessors<'q> {
|
||||
|
|
|
|||
|
|
@ -16,13 +16,13 @@ use rustc::ty::TyCtxt;
|
|||
use rustc_data_structures::fnv::FnvHashMap;
|
||||
use rustc_serialize::Encodable as RustcEncodable;
|
||||
use rustc_serialize::opaque::Encoder;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::hash::Hash;
|
||||
use std::io::{self, Cursor, Write};
|
||||
use std::fs::{self, File};
|
||||
use std::path::PathBuf;
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
|
||||
use IncrementalHashesMap;
|
||||
use ich::Fingerprint;
|
||||
use super::data::*;
|
||||
use super::directory::*;
|
||||
use super::hash::*;
|
||||
|
|
@ -30,6 +30,7 @@ use super::preds::*;
|
|||
use super::fs::*;
|
||||
use super::dirty_clean;
|
||||
use super::file_format;
|
||||
use calculate_svh::hasher::IchHasher;
|
||||
|
||||
pub fn save_dep_graph<'a, 'tcx>(tcx: TyCtxt<'a, 'tcx, 'tcx>,
|
||||
incremental_hashes_map: &IncrementalHashesMap,
|
||||
|
|
@ -185,7 +186,7 @@ pub fn encode_metadata_hashes(tcx: TyCtxt,
|
|||
svh: Svh,
|
||||
preds: &Predecessors,
|
||||
builder: &mut DefIdDirectoryBuilder,
|
||||
current_metadata_hashes: &mut FnvHashMap<DefId, u64>,
|
||||
current_metadata_hashes: &mut FnvHashMap<DefId, Fingerprint>,
|
||||
encoder: &mut Encoder)
|
||||
-> io::Result<()> {
|
||||
// For each `MetaData(X)` node where `X` is local, accumulate a
|
||||
|
|
@ -233,7 +234,7 @@ pub fn encode_metadata_hashes(tcx: TyCtxt,
|
|||
// is the det. hash of the def-path. This is convenient
|
||||
// because we can sort this to get a stable ordering across
|
||||
// compilations, even if the def-ids themselves have changed.
|
||||
let mut hashes: Vec<(DepNode<u64>, u64)> = sources.iter()
|
||||
let mut hashes: Vec<(DepNode<u64>, Fingerprint)> = sources.iter()
|
||||
.map(|dep_node| {
|
||||
let hash_dep_node = dep_node.map_def(|&def_id| Some(def_id_hash(def_id))).unwrap();
|
||||
let hash = preds.hashes[dep_node];
|
||||
|
|
@ -242,7 +243,7 @@ pub fn encode_metadata_hashes(tcx: TyCtxt,
|
|||
.collect();
|
||||
|
||||
hashes.sort();
|
||||
let mut state = DefaultHasher::new();
|
||||
let mut state = IchHasher::new();
|
||||
hashes.hash(&mut state);
|
||||
let hash = state.finish();
|
||||
|
||||
|
|
|
|||
|
|
@ -130,7 +130,7 @@ pub fn build_link_meta(incremental_hashes_map: &IncrementalHashesMap,
|
|||
-> LinkMeta {
|
||||
let r = LinkMeta {
|
||||
crate_name: name.to_owned(),
|
||||
crate_hash: Svh::new(incremental_hashes_map[&DepNode::Krate]),
|
||||
crate_hash: Svh::new(incremental_hashes_map[&DepNode::Krate].to_smaller_hash()),
|
||||
};
|
||||
info!("{:?}", r);
|
||||
return r;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue