From 6ace43cf77803ccb854fdcd31bde70860cb84613 Mon Sep 17 00:00:00 2001 From: Niko Matsakis Date: Wed, 26 Sep 2018 15:49:41 -0400 Subject: [PATCH] hide `SmallCanonicalVarValues` in `OriginalQueryValues` struct --- src/librustc/infer/canonical/canonicalizer.rs | 24 +++++----- src/librustc/infer/canonical/mod.rs | 13 ++++-- .../infer/canonical/query_response.rs | 46 +++++++++++-------- src/librustc/traits/query/dropck_outlives.rs | 4 +- .../traits/query/evaluate_obligation.rs | 4 +- src/librustc/traits/query/normalize.rs | 4 +- src/librustc/traits/query/outlives_bounds.rs | 4 +- src/librustc/traits/query/type_op/mod.rs | 6 +-- src/librustc_traits/chalk_context.rs | 7 +-- 9 files changed, 66 insertions(+), 46 deletions(-) diff --git a/src/librustc/infer/canonical/canonicalizer.rs b/src/librustc/infer/canonical/canonicalizer.rs index 95822eb18552..8c87c2a01c04 100644 --- a/src/librustc/infer/canonical/canonicalizer.rs +++ b/src/librustc/infer/canonical/canonicalizer.rs @@ -17,7 +17,7 @@ use infer::canonical::{ Canonical, CanonicalTyVarKind, CanonicalVarInfo, CanonicalVarKind, Canonicalized, - SmallCanonicalVarValues, + OriginalQueryValues, }; use infer::InferCtxt; use std::sync::atomic::Ordering; @@ -48,7 +48,7 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> { pub fn canonicalize_query( &self, value: &V, - var_values: &mut SmallCanonicalVarValues<'tcx>, + query_state: &mut OriginalQueryValues<'tcx>, ) -> Canonicalized<'gcx, V> where V: TypeFoldable<'tcx> + Lift<'gcx>, @@ -64,7 +64,7 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> { Some(self), self.tcx, &CanonicalizeAllFreeRegions, - var_values, + query_state, ) } @@ -97,13 +97,13 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> { where V: TypeFoldable<'tcx> + Lift<'gcx>, { - let mut var_values = SmallVec::new(); + let mut query_state = OriginalQueryValues::default(); Canonicalizer::canonicalize( value, Some(self), self.tcx, &CanonicalizeQueryResponse, - &mut var_values, + &mut query_state, ) } @@ -119,7 +119,7 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> { pub fn canonicalize_hr_query_hack( &self, value: &V, - var_values: &mut SmallCanonicalVarValues<'tcx>, + query_state: &mut OriginalQueryValues<'tcx>, ) -> Canonicalized<'gcx, V> where V: TypeFoldable<'tcx> + Lift<'gcx>, @@ -135,7 +135,7 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> { Some(self), self.tcx, &CanonicalizeFreeRegionsOtherThanStatic, - var_values, + query_state, ) } } @@ -222,7 +222,7 @@ struct Canonicalizer<'cx, 'gcx: 'tcx, 'tcx: 'cx> { infcx: Option<&'cx InferCtxt<'cx, 'gcx, 'tcx>>, tcx: TyCtxt<'cx, 'gcx, 'tcx>, variables: SmallVec<[CanonicalVarInfo; 8]>, - var_values: &'cx mut SmallCanonicalVarValues<'tcx>, + query_state: &'cx mut OriginalQueryValues<'tcx>, // Note that indices is only used once `var_values` is big enough to be // heap-allocated. indices: FxHashMap, CanonicalVar>, @@ -330,7 +330,7 @@ impl<'cx, 'gcx, 'tcx> Canonicalizer<'cx, 'gcx, 'tcx> { infcx: Option<&InferCtxt<'_, 'gcx, 'tcx>>, tcx: TyCtxt<'_, 'gcx, 'tcx>, canonicalize_region_mode: &dyn CanonicalizeRegionMode, - var_values: &mut SmallCanonicalVarValues<'tcx>, + query_state: &mut OriginalQueryValues<'tcx>, ) -> Canonicalized<'gcx, V> where V: TypeFoldable<'tcx> + Lift<'gcx>, @@ -365,7 +365,7 @@ impl<'cx, 'gcx, 'tcx> Canonicalizer<'cx, 'gcx, 'tcx> { canonicalize_region_mode, needs_canonical_flags, variables: SmallVec::new(), - var_values, + query_state, indices: FxHashMap::default(), }; let out_value = value.fold_with(&mut canonicalizer); @@ -396,11 +396,13 @@ impl<'cx, 'gcx, 'tcx> Canonicalizer<'cx, 'gcx, 'tcx> { fn canonical_var(&mut self, info: CanonicalVarInfo, kind: Kind<'tcx>) -> CanonicalVar { let Canonicalizer { variables, - var_values, + query_state, indices, .. } = self; + let var_values = &mut query_state.var_values; + // This code is hot. `variables` and `var_values` are usually small // (fewer than 8 elements ~95% of the time). They are SmallVec's to // avoid allocations in those cases. We also don't use `indices` to diff --git a/src/librustc/infer/canonical/mod.rs b/src/librustc/infer/canonical/mod.rs index c937f80ea099..a78b5b7d0726 100644 --- a/src/librustc/infer/canonical/mod.rs +++ b/src/librustc/infer/canonical/mod.rs @@ -75,9 +75,16 @@ pub struct CanonicalVarValues<'tcx> { pub var_values: IndexVec>, } -/// Like CanonicalVarValues, but for use in places where a SmallVec is -/// appropriate. -pub type SmallCanonicalVarValues<'tcx> = SmallVec<[Kind<'tcx>; 8]>; +/// When we canonicalize a value to form a query, we wind up replacing +/// various parts of it with canonical variables. This struct stores +/// those replaced bits to remember for when we process the query +/// result. +#[derive(Clone, Debug, Default, PartialEq, Eq, Hash, RustcDecodable, RustcEncodable)] +pub struct OriginalQueryValues<'tcx> { + /// This is equivalent to `CanonicalVarValues`, but using a + /// `SmallVec` yields a significant performance win. + pub var_values: SmallVec<[Kind<'tcx>; 8]>, +} /// Information about a canonical variable that is included with the /// canonical value. This is sufficient information for code to create diff --git a/src/librustc/infer/canonical/query_response.rs b/src/librustc/infer/canonical/query_response.rs index 07421f0bd455..c29a75c34cf9 100644 --- a/src/librustc/infer/canonical/query_response.rs +++ b/src/librustc/infer/canonical/query_response.rs @@ -20,7 +20,7 @@ use infer::canonical::substitute::substitute_value; use infer::canonical::{ Canonical, CanonicalVarKind, CanonicalVarValues, CanonicalizedQueryResponse, Certainty, - QueryRegionConstraint, QueryResponse, SmallCanonicalVarValues, + OriginalQueryValues, QueryRegionConstraint, QueryResponse, }; use infer::region_constraints::{Constraint, RegionConstraintData}; use infer::InferCtxtBuilder; @@ -64,11 +64,15 @@ impl<'cx, 'gcx, 'tcx> InferCtxtBuilder<'cx, 'gcx, 'tcx> { K: TypeFoldable<'tcx>, R: Debug + Lift<'gcx> + TypeFoldable<'tcx>, { - self.enter_with_canonical(DUMMY_SP, canonical_key, |ref infcx, key, canonical_inference_vars| { - let fulfill_cx = &mut FulfillmentContext::new(); - let value = operation(infcx, fulfill_cx, key)?; - infcx.make_canonicalized_query_response(canonical_inference_vars, value, fulfill_cx) - }) + self.enter_with_canonical( + DUMMY_SP, + canonical_key, + |ref infcx, key, canonical_inference_vars| { + let fulfill_cx = &mut FulfillmentContext::new(); + let value = operation(infcx, fulfill_cx, key)?; + infcx.make_canonicalized_query_response(canonical_inference_vars, value, fulfill_cx) + }, + ) } } @@ -153,7 +157,8 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> { region_obligations .iter() .map(|(_, r_o)| (r_o.sup_type, r_o.sub_region)), - region_constraints) + region_constraints, + ) }); let certainty = if ambig_errors.is_empty() { @@ -184,7 +189,7 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> { &self, cause: &ObligationCause<'tcx>, param_env: ty::ParamEnv<'tcx>, - original_values: &SmallCanonicalVarValues<'tcx>, + original_values: &OriginalQueryValues<'tcx>, query_response: &Canonical<'tcx, QueryResponse<'tcx, R>>, ) -> InferResult<'tcx, R> where @@ -250,7 +255,7 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> { &self, cause: &ObligationCause<'tcx>, param_env: ty::ParamEnv<'tcx>, - original_values: &SmallCanonicalVarValues<'tcx>, + original_values: &OriginalQueryValues<'tcx>, query_response: &Canonical<'tcx, QueryResponse<'tcx, R>>, output_query_region_constraints: &mut Vec>, ) -> InferResult<'tcx, R> @@ -272,7 +277,7 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> { // variable... let mut obligations = vec![]; - for (index, original_value) in original_values.iter().enumerate() { + for (index, original_value) in original_values.var_values.iter().enumerate() { // ...with the value `v_r` of that variable from the query. let result_value = query_response.substitute_projected(self.tcx, &result_subst, |v| { &v.var_values[CanonicalVar::new(index)] @@ -344,7 +349,7 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> { &self, cause: &ObligationCause<'tcx>, param_env: ty::ParamEnv<'tcx>, - original_values: &SmallCanonicalVarValues<'tcx>, + original_values: &OriginalQueryValues<'tcx>, query_response: &Canonical<'tcx, QueryResponse<'tcx, R>>, ) -> InferResult<'tcx, CanonicalVarValues<'tcx>> where @@ -385,7 +390,7 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> { fn query_response_substitution_guess( &self, cause: &ObligationCause<'tcx>, - original_values: &SmallCanonicalVarValues<'tcx>, + original_values: &OriginalQueryValues<'tcx>, query_response: &Canonical<'tcx, QueryResponse<'tcx, R>>, ) -> CanonicalVarValues<'tcx> where @@ -401,7 +406,7 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> { // these values with the original inputs that were // canonicalized. let result_values = &query_response.value.var_values; - assert_eq!(original_values.len(), result_values.len()); + assert_eq!(original_values.var_values.len(), result_values.len()); // Quickly try to find initial values for the canonical // variables in the result in terms of the query. We do this @@ -415,7 +420,7 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> { // In terms of our example above, we are iterating over pairs like: // [(?A, Vec), ('static, '?1), (?B, ?0)] - for (original_value, result_value) in original_values.iter().zip(result_values) { + for (original_value, result_value) in original_values.var_values.iter().zip(result_values) { match result_value.unpack() { UnpackedKind::Type(result_value) => { // e.g., here `result_value` might be `?0` in the example above... @@ -461,7 +466,7 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> { &self, cause: &ObligationCause<'tcx>, param_env: ty::ParamEnv<'tcx>, - original_values: &SmallCanonicalVarValues<'tcx>, + original_values: &OriginalQueryValues<'tcx>, result_subst: &CanonicalVarValues<'tcx>, query_response: &Canonical<'tcx, QueryResponse<'tcx, R>>, ) -> InferResult<'tcx, ()> @@ -478,7 +483,12 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> { // Unify the original value for each variable with the value // taken from `query_response` (after applying `result_subst`). - Ok(self.unify_canonical_vars(cause, param_env, original_values, substituted_query_response)?) + Ok(self.unify_canonical_vars( + cause, + param_env, + original_values, + substituted_query_response, + )?) } /// Converts the region constraints resulting from a query into an @@ -522,12 +532,12 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> { &self, cause: &ObligationCause<'tcx>, param_env: ty::ParamEnv<'tcx>, - variables1: &SmallCanonicalVarValues<'tcx>, + variables1: &OriginalQueryValues<'tcx>, variables2: impl Fn(CanonicalVar) -> Kind<'tcx>, ) -> InferResult<'tcx, ()> { self.commit_if_ok(|_| { let mut obligations = vec![]; - for (index, value1) in variables1.iter().enumerate() { + for (index, value1) in variables1.var_values.iter().enumerate() { let value2 = variables2(CanonicalVar::new(index)); match (value1.unpack(), value2.unpack()) { diff --git a/src/librustc/traits/query/dropck_outlives.rs b/src/librustc/traits/query/dropck_outlives.rs index c2091c182d65..8f7b0df8b95a 100644 --- a/src/librustc/traits/query/dropck_outlives.rs +++ b/src/librustc/traits/query/dropck_outlives.rs @@ -10,7 +10,7 @@ use infer::at::At; use infer::InferOk; -use smallvec::SmallVec; +use infer::canonical::OriginalQueryValues; use std::iter::FromIterator; use syntax::source_map::Span; use ty::subst::Kind; @@ -51,7 +51,7 @@ impl<'cx, 'gcx, 'tcx> At<'cx, 'gcx, 'tcx> { } let gcx = tcx.global_tcx(); - let mut orig_values = SmallVec::new(); + let mut orig_values = OriginalQueryValues::default(); let c_ty = self.infcx.canonicalize_query(&self.param_env.and(ty), &mut orig_values); let span = self.cause.span; debug!("c_ty = {:?}", c_ty); diff --git a/src/librustc/traits/query/evaluate_obligation.rs b/src/librustc/traits/query/evaluate_obligation.rs index f573b1ef45e9..ca11c5f5a087 100644 --- a/src/librustc/traits/query/evaluate_obligation.rs +++ b/src/librustc/traits/query/evaluate_obligation.rs @@ -9,7 +9,7 @@ // except according to those terms. use infer::InferCtxt; -use smallvec::SmallVec; +use infer::canonical::OriginalQueryValues; use traits::{EvaluationResult, PredicateObligation, SelectionContext, TraitQueryMode, OverflowError}; @@ -38,7 +38,7 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> { &self, obligation: &PredicateObligation<'tcx>, ) -> Result { - let mut _orig_values = SmallVec::new(); + let mut _orig_values = OriginalQueryValues::default(); let c_pred = self.canonicalize_query(&obligation.param_env.and(obligation.predicate), &mut _orig_values); // Run canonical query. If overflow occurs, rerun from scratch but this time diff --git a/src/librustc/traits/query/normalize.rs b/src/librustc/traits/query/normalize.rs index 833c650680ea..4adb65dc58d9 100644 --- a/src/librustc/traits/query/normalize.rs +++ b/src/librustc/traits/query/normalize.rs @@ -13,9 +13,9 @@ //! `normalize_projection_ty` query when it encounters projections. use infer::at::At; +use infer::canonical::OriginalQueryValues; use infer::{InferCtxt, InferOk}; use mir::interpret::{ConstValue, GlobalId}; -use smallvec::SmallVec; use traits::project::Normalized; use traits::{Obligation, ObligationCause, PredicateObligation, Reveal}; use ty::fold::{TypeFoldable, TypeFolder}; @@ -154,7 +154,7 @@ impl<'cx, 'gcx, 'tcx> TypeFolder<'gcx, 'tcx> for QueryNormalizer<'cx, 'gcx, 'tcx let gcx = self.infcx.tcx.global_tcx(); - let mut orig_values = SmallVec::new(); + let mut orig_values = OriginalQueryValues::default(); let c_data = self.infcx.canonicalize_query( &self.param_env.and(*data), &mut orig_values); debug!("QueryNormalizer: c_data = {:#?}", c_data); diff --git a/src/librustc/traits/query/outlives_bounds.rs b/src/librustc/traits/query/outlives_bounds.rs index 5415efe1514d..99f557d44d9a 100644 --- a/src/librustc/traits/query/outlives_bounds.rs +++ b/src/librustc/traits/query/outlives_bounds.rs @@ -9,9 +9,9 @@ // except according to those terms. use infer::InferCtxt; +use infer::canonical::OriginalQueryValues; use syntax::ast; use syntax::source_map::Span; -use smallvec::SmallVec; use traits::{FulfillmentContext, ObligationCause, TraitEngine, TraitEngineExt}; use traits::query::NoSolution; use ty::{self, Ty, TyCtxt}; @@ -105,7 +105,7 @@ impl<'cx, 'gcx, 'tcx> InferCtxt<'cx, 'gcx, 'tcx> { ) -> Vec> { debug!("implied_outlives_bounds(ty = {:?})", ty); - let mut orig_values = SmallVec::new(); + let mut orig_values = OriginalQueryValues::default(); let key = self.canonicalize_query(¶m_env.and(ty), &mut orig_values); let result = match self.tcx.global_tcx().implied_outlives_bounds(key) { Ok(r) => r, diff --git a/src/librustc/traits/query/type_op/mod.rs b/src/librustc/traits/query/type_op/mod.rs index c2135dc12a7e..b292df758eeb 100644 --- a/src/librustc/traits/query/type_op/mod.rs +++ b/src/librustc/traits/query/type_op/mod.rs @@ -9,10 +9,10 @@ // except according to those terms. use infer::canonical::{ - Canonical, Canonicalized, CanonicalizedQueryResponse, QueryRegionConstraint, QueryResponse, + Canonical, Canonicalized, CanonicalizedQueryResponse, OriginalQueryValues, + QueryRegionConstraint, QueryResponse, }; use infer::{InferCtxt, InferOk}; -use smallvec::SmallVec; use std::fmt; use std::rc::Rc; use traits::query::Fallible; @@ -106,7 +106,7 @@ pub trait QueryTypeOp<'gcx: 'tcx, 'tcx>: // `canonicalize_hr_query_hack` here because of things // like the subtype query, which go awry around // `'static` otherwise. - let mut canonical_var_values = SmallVec::new(); + let mut canonical_var_values = OriginalQueryValues::default(); let canonical_self = infcx.canonicalize_hr_query_hack(&query_key, &mut canonical_var_values); let canonical_result = Self::perform_query(infcx.tcx, canonical_self)?; diff --git a/src/librustc_traits/chalk_context.rs b/src/librustc_traits/chalk_context.rs index 5a0ab8422003..536c15234064 100644 --- a/src/librustc_traits/chalk_context.rs +++ b/src/librustc_traits/chalk_context.rs @@ -10,7 +10,9 @@ use chalk_engine::fallible::Fallible as ChalkEngineFallible; use chalk_engine::{context, hh::HhGoal, DelayedLiteral, ExClause}; -use rustc::infer::canonical::{Canonical, CanonicalVarValues, QueryRegionConstraint, QueryResponse}; +use rustc::infer::canonical::{ + Canonical, CanonicalVarValues, OriginalQueryValues, QueryRegionConstraint, QueryResponse, +}; use rustc::infer::{InferCtxt, InferOk, LateBoundRegionConversionTime}; use rustc::traits::{ WellFormed, @@ -26,7 +28,6 @@ use rustc::traits::{ use rustc::ty::fold::{TypeFoldable, TypeFolder, TypeVisitor}; use rustc::ty::subst::Kind; use rustc::ty::{self, TyCtxt}; -use smallvec::SmallVec; use std::fmt::{self, Debug}; use std::marker::PhantomData; @@ -390,7 +391,7 @@ impl context::UnificationOps, ChalkArenas<'tcx>> &mut self, value: &ty::ParamEnvAnd<'tcx, Goal<'tcx>>, ) -> Canonical<'gcx, ty::ParamEnvAnd<'gcx, Goal<'gcx>>> { - let mut _orig_values = SmallVec::new(); + let mut _orig_values = OriginalQueryValues::default(); self.infcx.canonicalize_query(value, &mut _orig_values) }