user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Auto merge of #152160 - nnethercote:start-cutting-down-rustc_query_system, r=Zalathar

Browse source 
Start cutting down `rustc_query_system`

The query system is implemented in `rustc_query_system`, `rustc_middle`, and `rustc_query_impl`. `rustc_query_system` is hamstrung by not having access to `TyCtxt`, and there seems to be consensus to eliminate it. It's contents can be moved into the other two crates. Moving as much stuff as possible to `rustc_query_impl` is preferred, because `rustc_middle` is already so big.

This PR starts this process. It moves one small function to `rustc_middle` and a good chunk of code to `rustc_query_impl`.

Once `rustc_query_system` is gone (or at least shrunk down a lot more) some of the traits like `DepContext`, `QueryContext`, and `QueryDispatcher` will be removable.

r? @Zalathar
This commit is contained in:
bors 2026-02-08 11:41:28 +00:00
commit 286fbe5d84
10 changed files with 823 additions and 847 deletions
Download patch file Download diff file Expand all files Collapse all files

21
compiler/rustc_middle/src/query/inner.rs
View file

@ -2,7 +2,7 @@
//! `tcx.$query(..)` and its variations.
use rustc_query_system::dep_graph::{DepKind, DepNodeKey};
use rustc_query_system::query::{QueryCache, QueryMode, try_get_cached};
use rustc_query_system::query::{QueryCache, QueryMode};
use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span};
use crate::dep_graph;
@ -10,6 +10,25 @@ use crate::query::erase::{self, Erasable, Erased};
use crate::query::plumbing::QueryVTable;
use crate::ty::TyCtxt;
/// Checks whether there is already a value for this key in the in-memory
/// query cache, returning that value if present.
///
/// (Also performs some associated bookkeeping, if a value was found.)
#[inline(always)]
fn try_get_cached<'tcx, C>(tcx: TyCtxt<'tcx>, cache: &C, key: &C::Key) -> Option<C::Value>
where
C: QueryCache,
{
match cache.lookup(key) {
Some((value, index)) => {
tcx.prof.query_cache_hit(index.into());
tcx.dep_graph.read_index(index);
Some(value)
}
None => None,
}
}
/// Shared implementation of `tcx.$query(..)` and `tcx.at(span).$query(..)`
/// for all queries.
#[inline(always)]

708
compiler/rustc_query_impl/src/execution.rs Normal file
View file

@ -0,0 +1,708 @@
use std::hash::Hash;
use std::mem;
use rustc_data_structures::hash_table::{Entry, HashTable};
use rustc_data_structures::stack::ensure_sufficient_stack;
use rustc_data_structures::{outline, sharded, sync};
use rustc_errors::{Diag, FatalError, StashKey};
use rustc_middle::dep_graph::DepsType;
use rustc_middle::ty::TyCtxt;
use rustc_query_system::dep_graph::{DepGraphData, DepNodeKey, HasDepContext};
use rustc_query_system::query::{
ActiveKeyStatus, CycleError, CycleErrorHandling, QueryCache, QueryContext, QueryDispatcher,
QueryJob, QueryJobId, QueryJobInfo, QueryLatch, QueryMap, QueryMode, QueryStackDeferred,
QueryStackFrame, QueryState, incremental_verify_ich, report_cycle,
};
use rustc_span::{DUMMY_SP, Span};
use crate::dep_graph::{DepContext, DepNode, DepNodeIndex};
use crate::plumbing::QueryCtxt;
#[inline]
fn equivalent_key<K: Eq, V>(k: &K) -> impl Fn(&(K, V)) -> bool + '_ {
move |x| x.0 == *k
}
/// Obtains the enclosed [`QueryJob`], or panics if this query evaluation
/// was poisoned by a panic.
fn expect_job<'tcx>(status: ActiveKeyStatus<'tcx>) -> QueryJob<'tcx> {
match status {
ActiveKeyStatus::Started(job) => job,
ActiveKeyStatus::Poisoned => {
panic!("job for query failed to start and was poisoned")
}
}
}
pub(crate) fn all_inactive<'tcx, K>(state: &QueryState<'tcx, K>) -> bool {
state.active.lock_shards().all(|shard| shard.is_empty())
}
/// Internal plumbing for collecting the set of active jobs for this query.
///
/// Should only be called from `gather_active_jobs`.
pub(crate) fn gather_active_jobs_inner<'tcx, K: Copy>(
state: &QueryState<'tcx, K>,
tcx: TyCtxt<'tcx>,
make_frame: fn(TyCtxt<'tcx>, K) -> QueryStackFrame<QueryStackDeferred<'tcx>>,
jobs: &mut QueryMap<'tcx>,
require_complete: bool,
) -> Option<()> {
let mut active = Vec::new();
// Helper to gather active jobs from a single shard.
let mut gather_shard_jobs = |shard: &HashTable<(K, ActiveKeyStatus<'tcx>)>| {
for (k, v) in shard.iter() {
if let ActiveKeyStatus::Started(ref job) = *v {
active.push((*k, job.clone()));
}
}
};
// Lock shards and gather jobs from each shard.
if require_complete {
for shard in state.active.lock_shards() {
gather_shard_jobs(&shard);
}
} else {
// We use try_lock_shards here since we are called from the
// deadlock handler, and this shouldn't be locked.
for shard in state.active.try_lock_shards() {
let shard = shard?;
gather_shard_jobs(&shard);
}
}
// Call `make_frame` while we're not holding a `state.active` lock as `make_frame` may call
// queries leading to a deadlock.
for (key, job) in active {
let frame = make_frame(tcx, key);
jobs.insert(job.id, QueryJobInfo { frame, job });
}
Some(())
}
/// A type representing the responsibility to execute the job in the `job` field.
/// This will poison the relevant query if dropped.
struct JobOwner<'tcx, K>
where
K: Eq + Hash + Copy,
{
state: &'tcx QueryState<'tcx, K>,
key: K,
}
#[cold]
#[inline(never)]
fn mk_cycle<'tcx, Q>(query: Q, qcx: QueryCtxt<'tcx>, cycle_error: CycleError) -> Q::Value
where
Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
{
let error = report_cycle(qcx.tcx.sess, &cycle_error);
handle_cycle_error(query, qcx, &cycle_error, error)
}
fn handle_cycle_error<'tcx, Q>(
query: Q,
qcx: QueryCtxt<'tcx>,
cycle_error: &CycleError,
error: Diag<'_>,
) -> Q::Value
where
Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
{
match query.cycle_error_handling() {
CycleErrorHandling::Error => {
let guar = error.emit();
query.value_from_cycle_error(qcx.tcx, cycle_error, guar)
}
CycleErrorHandling::Fatal => {
error.emit();
qcx.tcx.dcx().abort_if_errors();
unreachable!()
}
CycleErrorHandling::DelayBug => {
let guar = error.delay_as_bug();
query.value_from_cycle_error(qcx.tcx, cycle_error, guar)
}
CycleErrorHandling::Stash => {
let guar = if let Some(root) = cycle_error.cycle.first()
&& let Some(span) = root.frame.info.span
{
error.stash(span, StashKey::Cycle).unwrap()
} else {
error.emit()
};
query.value_from_cycle_error(qcx.tcx, cycle_error, guar)
}
}
}
impl<'tcx, K> JobOwner<'tcx, K>
where
K: Eq + Hash + Copy,
{
/// Completes the query by updating the query cache with the `result`,
/// signals the waiter and forgets the JobOwner, so it won't poison the query
fn complete<C>(self, cache: &C, key_hash: u64, result: C::Value, dep_node_index: DepNodeIndex)
where
C: QueryCache<Key = K>,
{
let key = self.key;
let state = self.state;
// Forget ourself so our destructor won't poison the query
mem::forget(self);
// Mark as complete before we remove the job from the active state
// so no other thread can re-execute this query.
cache.complete(key, result, dep_node_index);
let job = {
// don't keep the lock during the `unwrap()` of the retrieved value, or we taint the
// underlying shard.
// since unwinding also wants to look at this map, this can also prevent a double
// panic.
let mut shard = state.active.lock_shard_by_hash(key_hash);
match shard.find_entry(key_hash, equivalent_key(&key)) {
Err(_) => None,
Ok(occupied) => Some(occupied.remove().0.1),
}
};
let job = expect_job(job.expect("active query job entry"));
job.signal_complete();
}
}
impl<'tcx, K> Drop for JobOwner<'tcx, K>
where
K: Eq + Hash + Copy,
{
#[inline(never)]
#[cold]
fn drop(&mut self) {
// Poison the query so jobs waiting on it panic.
let state = self.state;
let job = {
let key_hash = sharded::make_hash(&self.key);
let mut shard = state.active.lock_shard_by_hash(key_hash);
match shard.find_entry(key_hash, equivalent_key(&self.key)) {
Err(_) => panic!(),
Ok(occupied) => {
let ((key, value), vacant) = occupied.remove();
vacant.insert((key, ActiveKeyStatus::Poisoned));
expect_job(value)
}
}
};
// Also signal the completion of the job, so waiters
// will continue execution.
job.signal_complete();
}
}
#[cold]
#[inline(never)]
fn cycle_error<'tcx, Q>(
query: Q,
qcx: QueryCtxt<'tcx>,
try_execute: QueryJobId,
span: Span,
) -> (Q::Value, Option<DepNodeIndex>)
where
Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
{
// Ensure there was no errors collecting all active jobs.
// We need the complete map to ensure we find a cycle to break.
let query_map = qcx
.collect_active_jobs_from_all_queries(false)
.ok()
.expect("failed to collect active queries");
let error = try_execute.find_cycle_in_stack(query_map, &qcx.current_query_job(), span);
(mk_cycle(query, qcx, error.lift()), None)
}
#[inline(always)]
fn wait_for_query<'tcx, Q>(
query: Q,
qcx: QueryCtxt<'tcx>,
span: Span,
key: Q::Key,
latch: QueryLatch<'tcx>,
current: Option<QueryJobId>,
) -> (Q::Value, Option<DepNodeIndex>)
where
Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
{
// For parallel queries, we'll block and wait until the query running
// in another thread has completed. Record how long we wait in the
// self-profiler.
let query_blocked_prof_timer = qcx.tcx.prof.query_blocked();
// With parallel queries we might just have to wait on some other
// thread.
let result = latch.wait_on(qcx, current, span);
match result {
Ok(()) => {
let Some((v, index)) = query.query_cache(qcx).lookup(&key) else {
outline(|| {
// We didn't find the query result in the query cache. Check if it was
// poisoned due to a panic instead.
let key_hash = sharded::make_hash(&key);
let shard = query.query_state(qcx).active.lock_shard_by_hash(key_hash);
match shard.find(key_hash, equivalent_key(&key)) {
// The query we waited on panicked. Continue unwinding here.
Some((_, ActiveKeyStatus::Poisoned)) => FatalError.raise(),
_ => panic!(
"query '{}' result must be in the cache or the query must be poisoned after a wait",
query.name()
),
}
})
};
qcx.tcx.prof.query_cache_hit(index.into());
query_blocked_prof_timer.finish_with_query_invocation_id(index.into());
(v, Some(index))
}
Err(cycle) => (mk_cycle(query, qcx, cycle.lift()), None),
}
}
#[inline(never)]
fn try_execute_query<'tcx, Q, const INCR: bool>(
query: Q,
qcx: QueryCtxt<'tcx>,
span: Span,
key: Q::Key,
dep_node: Option<DepNode>,
) -> (Q::Value, Option<DepNodeIndex>)
where
Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
{
let state = query.query_state(qcx);
let key_hash = sharded::make_hash(&key);
let mut state_lock = state.active.lock_shard_by_hash(key_hash);
// For the parallel compiler we need to check both the query cache and query state structures
// while holding the state lock to ensure that 1) the query has not yet completed and 2) the
// query is not still executing. Without checking the query cache here, we can end up
// re-executing the query since `try_start` only checks that the query is not currently
// executing, but another thread may have already completed the query and stores it result
// in the query cache.
if qcx.tcx.sess.threads() > 1 {
if let Some((value, index)) = query.query_cache(qcx).lookup(&key) {
qcx.tcx.prof.query_cache_hit(index.into());
return (value, Some(index));
}
}
let current_job_id = qcx.current_query_job();
match state_lock.entry(key_hash, equivalent_key(&key), |(k, _)| sharded::make_hash(k)) {
Entry::Vacant(entry) => {
// Nothing has computed or is computing the query, so we start a new job and insert it in the
// state map.
let id = qcx.next_job_id();
let job = QueryJob::new(id, span, current_job_id);
entry.insert((key, ActiveKeyStatus::Started(job)));
// Drop the lock before we start executing the query
drop(state_lock);
execute_job::<Q, INCR>(query, qcx, state, key, key_hash, id, dep_node)
}
Entry::Occupied(mut entry) => {
match &mut entry.get_mut().1 {
ActiveKeyStatus::Started(job) => {
if sync::is_dyn_thread_safe() {
// Get the latch out
let latch = job.latch();
drop(state_lock);
// Only call `wait_for_query` if we're using a Rayon thread pool
// as it will attempt to mark the worker thread as blocked.
return wait_for_query(query, qcx, span, key, latch, current_job_id);
}
let id = job.id;
drop(state_lock);
// If we are single-threaded we know that we have cycle error,
// so we just return the error.
cycle_error(query, qcx, id, span)
}
ActiveKeyStatus::Poisoned => FatalError.raise(),
}
}
}
}
#[inline(always)]
fn execute_job<'tcx, Q, const INCR: bool>(
query: Q,
qcx: QueryCtxt<'tcx>,
state: &'tcx QueryState<'tcx, Q::Key>,
key: Q::Key,
key_hash: u64,
id: QueryJobId,
dep_node: Option<DepNode>,
) -> (Q::Value, Option<DepNodeIndex>)
where
Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
{
// Use `JobOwner` so the query will be poisoned if executing it panics.
let job_owner = JobOwner { state, key };
debug_assert_eq!(qcx.tcx.dep_graph.is_fully_enabled(), INCR);
let (result, dep_node_index) = if INCR {
execute_job_incr(query, qcx, qcx.tcx.dep_graph.data().unwrap(), key, dep_node, id)
} else {
execute_job_non_incr(query, qcx, key, id)
};
let cache = query.query_cache(qcx);
if query.feedable() {
// We should not compute queries that also got a value via feeding.
// This can't happen, as query feeding adds the very dependencies to the fed query
// as its feeding query had. So if the fed query is red, so is its feeder, which will
// get evaluated first, and re-feed the query.
if let Some((cached_result, _)) = cache.lookup(&key) {
let Some(hasher) = query.hash_result() else {
panic!(
"no_hash fed query later has its value computed.\n\
Remove `no_hash` modifier to allow recomputation.\n\
The already cached value: {}",
(query.format_value())(&cached_result)
);
};
let (old_hash, new_hash) = qcx.dep_context().with_stable_hashing_context(|mut hcx| {
(hasher(&mut hcx, &cached_result), hasher(&mut hcx, &result))
});
let formatter = query.format_value();
if old_hash != new_hash {
// We have an inconsistency. This can happen if one of the two
// results is tainted by errors.
assert!(
qcx.tcx.dcx().has_errors().is_some(),
"Computed query value for {:?}({:?}) is inconsistent with fed value,\n\
computed={:#?}\nfed={:#?}",
query.dep_kind(),
key,
formatter(&result),
formatter(&cached_result),
);
}
}
}
job_owner.complete(cache, key_hash, result, dep_node_index);
(result, Some(dep_node_index))
}
// Fast path for when incr. comp. is off.
#[inline(always)]
fn execute_job_non_incr<'tcx, Q>(
query: Q,
qcx: QueryCtxt<'tcx>,
key: Q::Key,
job_id: QueryJobId,
) -> (Q::Value, DepNodeIndex)
where
Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
{
debug_assert!(!qcx.tcx.dep_graph.is_fully_enabled());
// Fingerprint the key, just to assert that it doesn't
// have anything we don't consider hashable
if cfg!(debug_assertions) {
let _ = key.to_fingerprint(qcx.tcx);
}
let prof_timer = qcx.tcx.prof.query_provider();
let result = qcx.start_query(job_id, query.depth_limit(), || query.compute(qcx, key));
let dep_node_index = qcx.tcx.dep_graph.next_virtual_depnode_index();
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
// Similarly, fingerprint the result to assert that
// it doesn't have anything not considered hashable.
if cfg!(debug_assertions)
&& let Some(hash_result) = query.hash_result()
{
qcx.dep_context().with_stable_hashing_context(|mut hcx| {
hash_result(&mut hcx, &result);
});
}
(result, dep_node_index)
}
#[inline(always)]
fn execute_job_incr<'tcx, Q>(
query: Q,
qcx: QueryCtxt<'tcx>,
dep_graph_data: &DepGraphData<DepsType>,
key: Q::Key,
mut dep_node_opt: Option<DepNode>,
job_id: QueryJobId,
) -> (Q::Value, DepNodeIndex)
where
Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
{
if !query.anon() && !query.eval_always() {
// `to_dep_node` is expensive for some `DepKind`s.
let dep_node = dep_node_opt.get_or_insert_with(|| query.construct_dep_node(qcx.tcx, &key));
// The diagnostics for this query will be promoted to the current session during
// `try_mark_green()`, so we can ignore them here.
if let Some(ret) = qcx.start_query(job_id, false, || {
try_load_from_disk_and_cache_in_memory(query, dep_graph_data, qcx, &key, dep_node)
}) {
return ret;
}
}
let prof_timer = qcx.tcx.prof.query_provider();
let (result, dep_node_index) = qcx.start_query(job_id, query.depth_limit(), || {
if query.anon() {
return dep_graph_data
.with_anon_task_inner(qcx.tcx, query.dep_kind(), || query.compute(qcx, key));
}
// `to_dep_node` is expensive for some `DepKind`s.
let dep_node = dep_node_opt.unwrap_or_else(|| query.construct_dep_node(qcx.tcx, &key));
dep_graph_data.with_task(
dep_node,
(qcx, query),
key,
|(qcx, query), key| query.compute(qcx, key),
query.hash_result(),
)
});
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
(result, dep_node_index)
}
#[inline(always)]
fn try_load_from_disk_and_cache_in_memory<'tcx, Q>(
query: Q,
dep_graph_data: &DepGraphData<DepsType>,
qcx: QueryCtxt<'tcx>,
key: &Q::Key,
dep_node: &DepNode,
) -> Option<(Q::Value, DepNodeIndex)>
where
Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
{
// Note this function can be called concurrently from the same query
// We must ensure that this is handled correctly.
let (prev_dep_node_index, dep_node_index) = dep_graph_data.try_mark_green(qcx, dep_node)?;
debug_assert!(dep_graph_data.is_index_green(prev_dep_node_index));
// First we try to load the result from the on-disk cache.
// Some things are never cached on disk.
if let Some(result) = query.try_load_from_disk(qcx, key, prev_dep_node_index, dep_node_index) {
if std::intrinsics::unlikely(qcx.tcx.sess.opts.unstable_opts.query_dep_graph) {
dep_graph_data.mark_debug_loaded_from_disk(*dep_node)
}
let prev_fingerprint = dep_graph_data.prev_fingerprint_of(prev_dep_node_index);
// If `-Zincremental-verify-ich` is specified, re-hash results from
// the cache and make sure that they have the expected fingerprint.
//
// If not, we still seek to verify a subset of fingerprints loaded
// from disk. Re-hashing results is fairly expensive, so we can't
// currently afford to verify every hash. This subset should still
// give us some coverage of potential bugs though.
let try_verify = prev_fingerprint.split().1.as_u64().is_multiple_of(32);
if std::intrinsics::unlikely(
try_verify || qcx.tcx.sess.opts.unstable_opts.incremental_verify_ich,
) {
incremental_verify_ich(
qcx.tcx,
dep_graph_data,
&result,
prev_dep_node_index,
query.hash_result(),
query.format_value(),
);
}
return Some((result, dep_node_index));
}
// We always expect to find a cached result for things that
// can be forced from `DepNode`.
debug_assert!(
!query.will_cache_on_disk_for_key(qcx.tcx, key)
|| !qcx.dep_context().fingerprint_style(dep_node.kind).reconstructible(),
"missing on-disk cache entry for {dep_node:?}"
);
// Sanity check for the logic in `ensure`: if the node is green and the result loadable,
// we should actually be able to load it.
debug_assert!(
!query.is_loadable_from_disk(qcx, key, prev_dep_node_index),
"missing on-disk cache entry for loadable {dep_node:?}"
);
// We could not load a result from the on-disk cache, so
// recompute.
let prof_timer = qcx.tcx.prof.query_provider();
// The dep-graph for this computation is already in-place.
let result = qcx.tcx.dep_graph.with_ignore(|| query.compute(qcx, *key));
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
// Verify that re-running the query produced a result with the expected hash
// This catches bugs in query implementations, turning them into ICEs.
// For example, a query might sort its result by `DefId` - since `DefId`s are
// not stable across compilation sessions, the result could get up getting sorted
// in a different order when the query is re-run, even though all of the inputs
// (e.g. `DefPathHash` values) were green.
//
// See issue #82920 for an example of a miscompilation that would get turned into
// an ICE by this check
incremental_verify_ich(
qcx.tcx,
dep_graph_data,
&result,
prev_dep_node_index,
query.hash_result(),
query.format_value(),
);
Some((result, dep_node_index))
}
/// Ensure that either this query has all green inputs or been executed.
/// Executing `query::ensure(D)` is considered a read of the dep-node `D`.
/// Returns true if the query should still run.
///
/// This function is particularly useful when executing passes for their
/// side-effects -- e.g., in order to report errors for erroneous programs.
///
/// Note: The optimization is only available during incr. comp.
#[inline(never)]
fn ensure_must_run<'tcx, Q>(
query: Q,
qcx: QueryCtxt<'tcx>,
key: &Q::Key,
check_cache: bool,
) -> (bool, Option<DepNode>)
where
Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
{
if query.eval_always() {
return (true, None);
}
// Ensuring an anonymous query makes no sense
assert!(!query.anon());
let dep_node = query.construct_dep_node(qcx.tcx, key);
let dep_graph = &qcx.tcx.dep_graph;
let serialized_dep_node_index = match dep_graph.try_mark_green(qcx, &dep_node) {
None => {
// A None return from `try_mark_green` means that this is either
// a new dep node or that the dep node has already been marked red.
// Either way, we can't call `dep_graph.read()` as we don't have the
// DepNodeIndex. We must invoke the query itself. The performance cost
// this introduces should be negligible as we'll immediately hit the
// in-memory cache, or another query down the line will.
return (true, Some(dep_node));
}
Some((serialized_dep_node_index, dep_node_index)) => {
dep_graph.read_index(dep_node_index);
qcx.tcx.prof.query_cache_hit(dep_node_index.into());
serialized_dep_node_index
}
};
// We do not need the value at all, so do not check the cache.
if !check_cache {
return (false, None);
}
let loadable = query.is_loadable_from_disk(qcx, key, serialized_dep_node_index);
(!loadable, Some(dep_node))
}
#[inline(always)]
pub(super) fn get_query_non_incr<'tcx, Q>(
query: Q,
qcx: QueryCtxt<'tcx>,
span: Span,
key: Q::Key,
) -> Q::Value
where
Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
{
debug_assert!(!qcx.tcx.dep_graph.is_fully_enabled());
ensure_sufficient_stack(|| try_execute_query::<Q, false>(query, qcx, span, key, None).0)
}
#[inline(always)]
pub(super) fn get_query_incr<'tcx, Q>(
query: Q,
qcx: QueryCtxt<'tcx>,
span: Span,
key: Q::Key,
mode: QueryMode,
) -> Option<Q::Value>
where
Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
{
debug_assert!(qcx.tcx.dep_graph.is_fully_enabled());
let dep_node = if let QueryMode::Ensure { check_cache } = mode {
let (must_run, dep_node) = ensure_must_run(query, qcx, &key, check_cache);
if !must_run {
return None;
}
dep_node
} else {
None
};
let (result, dep_node_index) =
ensure_sufficient_stack(|| try_execute_query::<Q, true>(query, qcx, span, key, dep_node));
if let Some(dep_node_index) = dep_node_index {
qcx.tcx.dep_graph.read_index(dep_node_index)
}
Some(result)
}
pub(super) fn force_query<'tcx, Q>(query: Q, qcx: QueryCtxt<'tcx>, key: Q::Key, dep_node: DepNode)
where
Q: QueryDispatcher<'tcx, Qcx = QueryCtxt<'tcx>>,
{
// We may be concurrently trying both execute and force a query.
// Ensure that only one of them runs the query.
if let Some((_, index)) = query.query_cache(qcx).lookup(&key) {
qcx.tcx.prof.query_cache_hit(index.into());
return;
}
debug_assert!(!query.anon());
ensure_sufficient_stack(|| {
try_execute_query::<Q, true>(query, qcx, DUMMY_SP, key, Some(dep_node))
});
}

16
compiler/rustc_query_impl/src/lib.rs
View file

@ -3,6 +3,7 @@
// tidy-alphabetical-start
#![allow(internal_features)]
#![feature(adt_const_params)]
#![feature(core_intrinsics)]
#![feature(min_specialization)]
#![feature(rustc_attrs)]
// tidy-alphabetical-end
@ -25,21 +26,20 @@ use rustc_query_system::dep_graph::SerializedDepNodeIndex;
use rustc_query_system::ich::StableHashingContext;
use rustc_query_system::query::{
CycleError, CycleErrorHandling, HashResult, QueryCache, QueryDispatcher, QueryMap, QueryMode,
QueryState, get_query_incr, get_query_non_incr,
QueryState,
};
use rustc_span::{ErrorGuaranteed, Span};
pub use crate::plumbing::{QueryCtxt, query_key_hash_verify_all};
use crate::plumbing::{encode_all_query_results, try_mark_green};
use crate::profiling_support::QueryKeyStringCache;
#[macro_use]
mod plumbing;
pub use crate::plumbing::{QueryCtxt, query_key_hash_verify_all};
mod profiling_support;
pub use self::profiling_support::alloc_self_profile_query_strings;
pub use crate::profiling_support::alloc_self_profile_query_strings;
mod error;
mod execution;
#[macro_use]
mod plumbing;
mod profiling_support;
#[derive(ConstParamTy)] // Allow this struct to be used for const-generic values.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]

110
compiler/rustc_query_impl/src/plumbing.rs
View file

@ -14,7 +14,8 @@ use rustc_hir::limit::Limit;
use rustc_index::Idx;
use rustc_middle::bug;
use rustc_middle::dep_graph::{
self, DepContext, DepKindVTable, DepNode, DepNodeIndex, SerializedDepNodeIndex, dep_kinds,
self, DepContext, DepKindVTable, DepNode, DepNodeIndex, DepsType, SerializedDepNodeIndex,
dep_kinds,
};
use rustc_middle::query::Key;
use rustc_middle::query::on_disk_cache::{
@ -29,13 +30,14 @@ use rustc_query_system::dep_graph::{DepNodeKey, FingerprintStyle, HasDepContext}
use rustc_query_system::ich::StableHashingContext;
use rustc_query_system::query::{
QueryCache, QueryContext, QueryDispatcher, QueryJobId, QueryMap, QuerySideEffect,
QueryStackDeferred, QueryStackFrame, QueryStackFrameExtra, force_query,
QueryStackDeferred, QueryStackFrame, QueryStackFrameExtra,
};
use rustc_serialize::{Decodable, Encodable};
use rustc_span::def_id::LOCAL_CRATE;
use crate::QueryDispatcherUnerased;
use crate::error::{QueryOverflow, QueryOverflowNote};
use crate::execution::{all_inactive, force_query};
/// Implements [`QueryContext`] for use by [`rustc_query_system`], since that
/// crate does not have direct access to [`TyCtxt`].
@ -68,10 +70,57 @@ impl<'tcx> QueryCtxt<'tcx> {
crate_name: self.tcx.crate_name(LOCAL_CRATE),
});
}
#[inline]
pub(crate) fn next_job_id(self) -> QueryJobId {
QueryJobId(
NonZero::new(
self.tcx.query_system.jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
)
.unwrap(),
)
}
#[inline]
pub(crate) fn current_query_job(self) -> Option<QueryJobId> {
tls::with_related_context(self.tcx, |icx| icx.query)
}
/// Executes a job by changing the `ImplicitCtxt` to point to the
/// new query job while it executes.
#[inline(always)]
pub(crate) fn start_query<R>(
self,
token: QueryJobId,
depth_limit: bool,
compute: impl FnOnce() -> R,
) -> R {
// The `TyCtxt` stored in TLS has the same global interner lifetime
// as `self`, so we use `with_related_context` to relate the 'tcx lifetimes
// when accessing the `ImplicitCtxt`.
tls::with_related_context(self.tcx, move |current_icx| {
if depth_limit
&& !self.tcx.recursion_limit().value_within_limit(current_icx.query_depth)
{
self.depth_limit_error(token);
}
// Update the `ImplicitCtxt` to point to our new query job.
let new_icx = ImplicitCtxt {
tcx: self.tcx,
query: Some(token),
query_depth: current_icx.query_depth + depth_limit as usize,
task_deps: current_icx.task_deps,
};
// Use the `ImplicitCtxt` while we execute the query.
tls::enter_context(&new_icx, compute)
})
}
}
impl<'tcx> HasDepContext for QueryCtxt<'tcx> {
type Deps = rustc_middle::dep_graph::DepsType;
type Deps = DepsType;
type DepContext = TyCtxt<'tcx>;
#[inline]
@ -86,21 +135,6 @@ impl<'tcx> QueryContext<'tcx> for QueryCtxt<'tcx> {
&self.tcx.jobserver_proxy
}
#[inline]
fn next_job_id(self) -> QueryJobId {
QueryJobId(
NonZero::new(
self.tcx.query_system.jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
)
.unwrap(),
)
}
#[inline]
fn current_query_job(self) -> Option<QueryJobId> {
tls::with_related_context(self.tcx, |icx| icx.query)
}
/// Returns a map of currently active query jobs, collected from all queries.
///
/// If `require_complete` is `true`, this function locks all shards of the
@ -146,38 +180,6 @@ impl<'tcx> QueryContext<'tcx> for QueryCtxt<'tcx> {
c.store_side_effect(dep_node_index, side_effect)
}
}
/// Executes a job by changing the `ImplicitCtxt` to point to the
/// new query job while it executes.
#[inline(always)]
fn start_query<R>(
self,
token: QueryJobId,
depth_limit: bool,
compute: impl FnOnce() -> R,
) -> R {
// The `TyCtxt` stored in TLS has the same global interner lifetime
// as `self`, so we use `with_related_context` to relate the 'tcx lifetimes
// when accessing the `ImplicitCtxt`.
tls::with_related_context(self.tcx, move |current_icx| {
if depth_limit
&& !self.tcx.recursion_limit().value_within_limit(current_icx.query_depth)
{
self.depth_limit_error(token);
}
// Update the `ImplicitCtxt` to point to our new query job.
let new_icx = ImplicitCtxt {
tcx: self.tcx,
query: Some(token),
query_depth: current_icx.query_depth + depth_limit as usize,
task_deps: current_icx.task_deps,
};
// Use the `ImplicitCtxt` while we execute the query.
tls::enter_context(&new_icx, compute)
})
}
}
pub(super) fn try_mark_green<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool {
@ -387,7 +389,7 @@ pub(crate) fn encode_query_results<'a, 'tcx, Q>(
{
let _timer = qcx.tcx.prof.generic_activity_with_arg("encode_query_results_for", query.name());
assert!(query.query_state(qcx).all_inactive());
assert!(all_inactive(query.query_state(qcx)));
let cache = query.query_cache(qcx);
cache.iter(&mut |key, value, dep_node| {
if query.will_cache_on_disk_for_key(qcx.tcx, key) {
@ -594,7 +596,7 @@ macro_rules! define_queries {
) -> Option<Erased<queries::$name::Value<'tcx>>> {
#[cfg(debug_assertions)]
let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered();
get_query_incr(
execution::get_query_incr(
QueryType::query_dispatcher(tcx),
QueryCtxt::new(tcx),
span,
@ -614,7 +616,7 @@ macro_rules! define_queries {
key: queries::$name::Key<'tcx>,
__mode: QueryMode,
) -> Option<Erased<queries::$name::Value<'tcx>>> {
Some(get_query_non_incr(
Some(execution::get_query_non_incr(
QueryType::query_dispatcher(tcx),
QueryCtxt::new(tcx),
span,
@ -748,7 +750,7 @@ macro_rules! define_queries {
};
// Call `gather_active_jobs_inner` to do the actual work.
let res = tcx.query_system.states.$name.gather_active_jobs_inner(
let res = crate::execution::gather_active_jobs_inner(&tcx.query_system.states.$name,
tcx,
make_frame,
qmap,

18
compiler/rustc_query_system/src/dep_graph/graph.rs
View file

@ -82,7 +82,7 @@ pub(super) enum DepNodeColor {
Unknown,
}
pub(crate) struct DepGraphData<D: Deps> {
pub struct DepGraphData<D: Deps> {
/// The new encoding of the dependency graph, optimized for red/green
/// tracking. The `current` field is the dependency graph of only the
/// current compilation session: We don't merge the previous dep-graph into
@ -171,7 +171,7 @@ impl<D: Deps> DepGraph<D> {
}
#[inline]
pub(crate) fn data(&self) -> Option<&DepGraphData<D>> {
pub fn data(&self) -> Option<&DepGraphData<D>> {
self.data.as_deref()
}
@ -323,7 +323,7 @@ impl<D: Deps> DepGraphData<D> {
///
/// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/queries/incremental-compilation.html
#[inline(always)]
pub(crate) fn with_task<Ctxt: HasDepContext<Deps = D>, A: Debug, R>(
pub fn with_task<Ctxt: HasDepContext<Deps = D>, A: Debug, R>(
&self,
key: DepNode,
cx: Ctxt,
@ -377,7 +377,7 @@ impl<D: Deps> DepGraphData<D> {
/// FIXME: This could perhaps return a `WithDepNode` to ensure that the
/// user of this function actually performs the read; we'll have to see
/// how to make that work with `anon` in `execute_job_incr`, though.
pub(crate) fn with_anon_task_inner<Tcx: DepContext<Deps = D>, OP, R>(
pub fn with_anon_task_inner<Tcx: DepContext<Deps = D>, OP, R>(
&self,
cx: Tcx,
dep_kind: DepKind,
@ -665,12 +665,12 @@ impl<D: Deps> DepGraphData<D> {
/// Returns true if the given node has been marked as green during the
/// current compilation session. Used in various assertions
#[inline]
pub(crate) fn is_index_green(&self, prev_index: SerializedDepNodeIndex) -> bool {
pub fn is_index_green(&self, prev_index: SerializedDepNodeIndex) -> bool {
matches!(self.colors.get(prev_index), DepNodeColor::Green(_))
}
#[inline]
pub(crate) fn prev_fingerprint_of(&self, prev_index: SerializedDepNodeIndex) -> Fingerprint {
pub fn prev_fingerprint_of(&self, prev_index: SerializedDepNodeIndex) -> Fingerprint {
self.previous.fingerprint_by_index(prev_index)
}
@ -679,7 +679,7 @@ impl<D: Deps> DepGraphData<D> {
self.previous.index_to_node(prev_index)
}
pub(crate) fn mark_debug_loaded_from_disk(&self, dep_node: DepNode) {
pub fn mark_debug_loaded_from_disk(&self, dep_node: DepNode) {
self.debug_loaded_from_disk.lock().insert(dep_node);
}
@ -875,7 +875,7 @@ impl<D: Deps> DepGraphData<D> {
/// A node will have an index, when it's already been marked green, or when we can mark it
/// green. This function will mark the current task as a reader of the specified node, when
/// a node index can be found for that node.
pub(crate) fn try_mark_green<'tcx, Qcx: QueryContext<'tcx, Deps = D>>(
pub fn try_mark_green<'tcx, Qcx: QueryContext<'tcx, Deps = D>>(
&self,
qcx: Qcx,
dep_node: &DepNode,
@ -1087,7 +1087,7 @@ impl<D: Deps> DepGraph<D> {
if let Some(data) = &self.data { data.current.encoder.finish(&data.current) } else { Ok(0) }
}
pub(crate) fn next_virtual_depnode_index(&self) -> DepNodeIndex {
pub fn next_virtual_depnode_index(&self) -> DepNodeIndex {
debug_assert!(self.data.is_none());
let index = self.virtual_dep_node_index.fetch_add(1, Ordering::Relaxed);
DepNodeIndex::from_u32(index)

5
compiler/rustc_query_system/src/dep_graph/mod.rs
View file

@ -8,8 +8,9 @@ mod serialized;
use std::panic;
pub use dep_node::{DepKind, DepKindVTable, DepNode, DepNodeKey, WorkProductId};
pub(crate) use graph::DepGraphData;
pub use graph::{DepGraph, DepNodeIndex, TaskDepsRef, WorkProduct, WorkProductMap, hash_result};
pub use graph::{
DepGraph, DepGraphData, DepNodeIndex, TaskDepsRef, WorkProduct, WorkProductMap, hash_result,
};
pub use query::DepGraphQuery;
use rustc_data_structures::profiling::SelfProfilerRef;
use rustc_data_structures::sync::DynSync;

1
compiler/rustc_query_system/src/lib.rs
View file

@ -1,7 +1,6 @@
// tidy-alphabetical-start
#![allow(internal_features)]
#![feature(assert_matches)]
#![feature(core_intrinsics)]
#![feature(min_specialization)]
// tidy-alphabetical-end

16
compiler/rustc_query_system/src/query/job.rs
View file

@ -65,7 +65,7 @@ pub struct QueryJobInfo<'tcx> {
}
/// Represents an active query job.
#[derive(Debug)]
#[derive(Clone, Debug)]
pub struct QueryJob<'tcx> {
pub id: QueryJobId,
@ -79,12 +79,6 @@ pub struct QueryJob<'tcx> {
latch: Option<QueryLatch<'tcx>>,
}
impl<'tcx> Clone for QueryJob<'tcx> {
fn clone(&self) -> Self {
Self { id: self.id, span: self.span, parent: self.parent, latch: self.latch.clone() }
}
}
impl<'tcx> QueryJob<'tcx> {
/// Creates a new query job.
#[inline]
@ -92,7 +86,7 @@ impl<'tcx> QueryJob<'tcx> {
QueryJob { id, span, parent, latch: None }
}
pub(super) fn latch(&mut self) -> QueryLatch<'tcx> {
pub fn latch(&mut self) -> QueryLatch<'tcx> {
if self.latch.is_none() {
self.latch = Some(QueryLatch::new());
}
@ -112,7 +106,7 @@ impl<'tcx> QueryJob<'tcx> {
}
impl QueryJobId {
pub(super) fn find_cycle_in_stack<'tcx>(
pub fn find_cycle_in_stack<'tcx>(
&self,
query_map: QueryMap<'tcx>,
current_job: &Option<QueryJobId>,
@ -188,7 +182,7 @@ struct QueryLatchInfo<'tcx> {
}
#[derive(Debug)]
pub(super) struct QueryLatch<'tcx> {
pub struct QueryLatch<'tcx> {
info: Arc<Mutex<QueryLatchInfo<'tcx>>>,
}
@ -206,7 +200,7 @@ impl<'tcx> QueryLatch<'tcx> {
}
/// Awaits for the query job to complete.
pub(super) fn wait_on(
pub fn wait_on(
&self,
qcx: impl QueryContext<'tcx>,
query: Option<QueryJobId>,

16
compiler/rustc_query_system/src/query/mod.rs
View file

@ -15,8 +15,8 @@ use rustc_span::def_id::DefId;
pub use self::caches::{DefIdCache, DefaultCache, QueryCache, SingleCache, VecCache};
pub use self::dispatcher::{HashResult, QueryDispatcher};
pub use self::job::{
QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryMap, break_query_cycles, print_query_stack,
report_cycle,
QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryLatch, QueryMap, break_query_cycles,
print_query_stack, report_cycle,
};
pub use self::plumbing::*;
use crate::dep_graph::{DepKind, DepNodeIndex, HasDepContext, SerializedDepNodeIndex};
@ -84,7 +84,7 @@ impl<'tcx> QueryStackFrame<QueryStackDeferred<'tcx>> {
#[derive(Clone, Debug)]
pub struct QueryStackFrameExtra {
pub description: String,
span: Option<Span>,
pub span: Option<Span>,
pub def_kind: Option<DefKind>,
}
@ -161,11 +161,6 @@ pub trait QueryContext<'tcx>: HasDepContext {
/// a token while waiting on a query.
fn jobserver_proxy(&self) -> &Proxy;
fn next_job_id(self) -> QueryJobId;
/// Get the query information from the TLS context.
fn current_query_job(self) -> Option<QueryJobId>;
fn collect_active_jobs_from_all_queries(
self,
require_complete: bool,
@ -179,9 +174,4 @@ pub trait QueryContext<'tcx>: HasDepContext {
/// Register a side effect for the given node, for use in next session.
fn store_side_effect(self, dep_node_index: DepNodeIndex, side_effect: QuerySideEffect);
/// Executes a job by changing the `ImplicitCtxt` to point to the
/// new query job while it executes.
fn start_query<R>(self, token: QueryJobId, depth_limit: bool, compute: impl FnOnce() -> R)
-> R;
}

759
compiler/rustc_query_system/src/query/plumbing.rs
View file

@ -1,36 +1,17 @@
//! The implementation of the query system itself. This defines the macros that
//! generate the actual methods on tcx which find and execute the provider,
//! manage the caches, and so forth.
use std::cell::Cell;
use std::fmt::Debug;
use std::hash::Hash;
use std::mem;
use rustc_data_structures::fingerprint::Fingerprint;
use rustc_data_structures::hash_table::{self, Entry, HashTable};
use rustc_data_structures::sharded::{self, Sharded};
use rustc_data_structures::stack::ensure_sufficient_stack;
use rustc_data_structures::{outline, sync};
use rustc_errors::{Diag, FatalError, StashKey};
use rustc_span::{DUMMY_SP, Span};
use rustc_data_structures::hash_table::HashTable;
use rustc_data_structures::sharded::Sharded;
use rustc_span::Span;
use tracing::instrument;
use super::{QueryDispatcher, QueryStackDeferred, QueryStackFrameExtra};
use crate::dep_graph::{
DepContext, DepGraphData, DepNode, DepNodeIndex, DepNodeKey, HasDepContext,
};
use super::{QueryStackDeferred, QueryStackFrameExtra};
use crate::dep_graph::{DepContext, DepGraphData};
use crate::ich::StableHashingContext;
use crate::query::caches::QueryCache;
use crate::query::job::{QueryInfo, QueryJob, QueryJobId, QueryJobInfo, QueryLatch, report_cycle};
use crate::query::{
CycleErrorHandling, QueryContext, QueryMap, QueryStackFrame, SerializedDepNodeIndex,
};
#[inline]
fn equivalent_key<K: Eq, V>(k: &K) -> impl Fn(&(K, V)) -> bool + '_ {
move |x| x.0 == *k
}
use crate::query::job::{QueryInfo, QueryJob};
use crate::query::{QueryStackFrame, SerializedDepNodeIndex};
/// For a particular query, keeps track of "active" keys, i.e. keys whose
/// evaluation has started but has not yet finished successfully.
@ -38,7 +19,7 @@ fn equivalent_key<K: Eq, V>(k: &K) -> impl Fn(&(K, V)) -> bool + '_ {
/// (Successful query evaluation for a key is represented by an entry in the
/// query's in-memory cache.)
pub struct QueryState<'tcx, K> {
active: Sharded<hash_table::HashTable<(K, ActiveKeyStatus<'tcx>)>>,
pub active: Sharded<HashTable<(K, ActiveKeyStatus<'tcx>)>>,
}
/// For a particular query and key, tracks the status of a query evaluation
@ -46,7 +27,7 @@ pub struct QueryState<'tcx, K> {
///
/// (Successful query evaluation for a key is represented by an entry in the
/// query's in-memory cache.)
enum ActiveKeyStatus<'tcx> {
pub enum ActiveKeyStatus<'tcx> {
/// Some thread is already evaluating the query for this key.
///
/// The enclosed [`QueryJob`] can be used to wait for it to finish.
@ -57,199 +38,12 @@ enum ActiveKeyStatus<'tcx> {
Poisoned,
}
impl<'tcx> ActiveKeyStatus<'tcx> {
/// Obtains the enclosed [`QueryJob`], or panics if this query evaluation
/// was poisoned by a panic.
fn expect_job(self) -> QueryJob<'tcx> {
match self {
Self::Started(job) => job,
Self::Poisoned => {
panic!("job for query failed to start and was poisoned")
}
}
}
}
impl<'tcx, K> QueryState<'tcx, K>
where
K: Eq + Hash + Copy + Debug,
{
pub fn all_inactive(&self) -> bool {
self.active.lock_shards().all(|shard| shard.is_empty())
}
/// Internal plumbing for collecting the set of active jobs for this query.
///
/// Should only be called from `gather_active_jobs`.
pub fn gather_active_jobs_inner<Qcx: Copy>(
&self,
qcx: Qcx,
make_frame: fn(Qcx, K) -> QueryStackFrame<QueryStackDeferred<'tcx>>,
jobs: &mut QueryMap<'tcx>,
require_complete: bool,
) -> Option<()> {
let mut active = Vec::new();
// Helper to gather active jobs from a single shard.
let mut gather_shard_jobs = |shard: &HashTable<(K, ActiveKeyStatus<'tcx>)>| {
for (k, v) in shard.iter() {
if let ActiveKeyStatus::Started(ref job) = *v {
active.push((*k, job.clone()));
}
}
};
// Lock shards and gather jobs from each shard.
if require_complete {
for shard in self.active.lock_shards() {
gather_shard_jobs(&shard);
}
} else {
// We use try_lock_shards here since we are called from the
// deadlock handler, and this shouldn't be locked.
for shard in self.active.try_lock_shards() {
let shard = shard?;
gather_shard_jobs(&shard);
}
}
// Call `make_frame` while we're not holding a `self.active` lock as `make_frame` may call
// queries leading to a deadlock.
for (key, job) in active {
let frame = make_frame(qcx, key);
jobs.insert(job.id, QueryJobInfo { frame, job });
}
Some(())
}
}
impl<'tcx, K> Default for QueryState<'tcx, K> {
fn default() -> QueryState<'tcx, K> {
QueryState { active: Default::default() }
}
}
/// A type representing the responsibility to execute the job in the `job` field.
/// This will poison the relevant query if dropped.
struct JobOwner<'tcx, K>
where
K: Eq + Hash + Copy,
{
state: &'tcx QueryState<'tcx, K>,
key: K,
}
#[cold]
#[inline(never)]
fn mk_cycle<'tcx, Q>(query: Q, qcx: Q::Qcx, cycle_error: CycleError) -> Q::Value
where
Q: QueryDispatcher<'tcx>,
{
let error = report_cycle(qcx.dep_context().sess(), &cycle_error);
handle_cycle_error(query, qcx, &cycle_error, error)
}
fn handle_cycle_error<'tcx, Q>(
query: Q,
qcx: Q::Qcx,
cycle_error: &CycleError,
error: Diag<'_>,
) -> Q::Value
where
Q: QueryDispatcher<'tcx>,
{
match query.cycle_error_handling() {
CycleErrorHandling::Error => {
let guar = error.emit();
query.value_from_cycle_error(*qcx.dep_context(), cycle_error, guar)
}
CycleErrorHandling::Fatal => {
error.emit();
qcx.dep_context().sess().dcx().abort_if_errors();
unreachable!()
}
CycleErrorHandling::DelayBug => {
let guar = error.delay_as_bug();
query.value_from_cycle_error(*qcx.dep_context(), cycle_error, guar)
}
CycleErrorHandling::Stash => {
let guar = if let Some(root) = cycle_error.cycle.first()
&& let Some(span) = root.frame.info.span
{
error.stash(span, StashKey::Cycle).unwrap()
} else {
error.emit()
};
query.value_from_cycle_error(*qcx.dep_context(), cycle_error, guar)
}
}
}
impl<'tcx, K> JobOwner<'tcx, K>
where
K: Eq + Hash + Copy,
{
/// Completes the query by updating the query cache with the `result`,
/// signals the waiter and forgets the JobOwner, so it won't poison the query
fn complete<C>(self, cache: &C, key_hash: u64, result: C::Value, dep_node_index: DepNodeIndex)
where
C: QueryCache<Key = K>,
{
let key = self.key;
let state = self.state;
// Forget ourself so our destructor won't poison the query
mem::forget(self);
// Mark as complete before we remove the job from the active state
// so no other thread can re-execute this query.
cache.complete(key, result, dep_node_index);
let job = {
// don't keep the lock during the `unwrap()` of the retrieved value, or we taint the
// underlying shard.
// since unwinding also wants to look at this map, this can also prevent a double
// panic.
let mut shard = state.active.lock_shard_by_hash(key_hash);
match shard.find_entry(key_hash, equivalent_key(&key)) {
Err(_) => None,
Ok(occupied) => Some(occupied.remove().0.1),
}
};
let job = job.expect("active query job entry").expect_job();
job.signal_complete();
}
}
impl<'tcx, K> Drop for JobOwner<'tcx, K>
where
K: Eq + Hash + Copy,
{
#[inline(never)]
#[cold]
fn drop(&mut self) {
// Poison the query so jobs waiting on it panic.
let state = self.state;
let job = {
let key_hash = sharded::make_hash(&self.key);
let mut shard = state.active.lock_shard_by_hash(key_hash);
match shard.find_entry(key_hash, equivalent_key(&self.key)) {
Err(_) => panic!(),
Ok(occupied) => {
let ((key, value), vacant) = occupied.remove();
vacant.insert((key, ActiveKeyStatus::Poisoned));
value.expect_job()
}
}
};
// Also signal the completion of the job, so waiters
// will continue execution.
job.signal_complete();
}
}
#[derive(Clone, Debug)]
pub struct CycleError<I = QueryStackFrameExtra> {
/// The query and related span that uses the cycle.
@ -258,7 +52,7 @@ pub struct CycleError<I = QueryStackFrameExtra> {
}
impl<'tcx> CycleError<QueryStackDeferred<'tcx>> {
fn lift(&self) -> CycleError<QueryStackFrameExtra> {
pub fn lift(&self) -> CycleError<QueryStackFrameExtra> {
CycleError {
usage: self.usage.as_ref().map(|(span, frame)| (*span, frame.lift())),
cycle: self.cycle.iter().map(|info| info.lift()).collect(),
@ -266,427 +60,9 @@ impl<'tcx> CycleError<QueryStackDeferred<'tcx>> {
}
}
/// Checks whether there is already a value for this key in the in-memory
/// query cache, returning that value if present.
///
/// (Also performs some associated bookkeeping, if a value was found.)
#[inline(always)]
pub fn try_get_cached<Tcx, C>(tcx: Tcx, cache: &C, key: &C::Key) -> Option<C::Value>
where
C: QueryCache,
Tcx: DepContext,
{
match cache.lookup(key) {
Some((value, index)) => {
tcx.profiler().query_cache_hit(index.into());
tcx.dep_graph().read_index(index);
Some(value)
}
None => None,
}
}
#[cold]
#[inline(never)]
fn cycle_error<'tcx, Q>(
query: Q,
qcx: Q::Qcx,
try_execute: QueryJobId,
span: Span,
) -> (Q::Value, Option<DepNodeIndex>)
where
Q: QueryDispatcher<'tcx>,
{
// Ensure there was no errors collecting all active jobs.
// We need the complete map to ensure we find a cycle to break.
let query_map = qcx
.collect_active_jobs_from_all_queries(false)
.ok()
.expect("failed to collect active queries");
let error = try_execute.find_cycle_in_stack(query_map, &qcx.current_query_job(), span);
(mk_cycle(query, qcx, error.lift()), None)
}
#[inline(always)]
fn wait_for_query<'tcx, Q>(
query: Q,
qcx: Q::Qcx,
span: Span,
key: Q::Key,
latch: QueryLatch<'tcx>,
current: Option<QueryJobId>,
) -> (Q::Value, Option<DepNodeIndex>)
where
Q: QueryDispatcher<'tcx>,
{
// For parallel queries, we'll block and wait until the query running
// in another thread has completed. Record how long we wait in the
// self-profiler.
let query_blocked_prof_timer = qcx.dep_context().profiler().query_blocked();
// With parallel queries we might just have to wait on some other
// thread.
let result = latch.wait_on(qcx, current, span);
match result {
Ok(()) => {
let Some((v, index)) = query.query_cache(qcx).lookup(&key) else {
outline(|| {
// We didn't find the query result in the query cache. Check if it was
// poisoned due to a panic instead.
let key_hash = sharded::make_hash(&key);
let shard = query.query_state(qcx).active.lock_shard_by_hash(key_hash);
match shard.find(key_hash, equivalent_key(&key)) {
// The query we waited on panicked. Continue unwinding here.
Some((_, ActiveKeyStatus::Poisoned)) => FatalError.raise(),
_ => panic!(
"query '{}' result must be in the cache or the query must be poisoned after a wait",
query.name()
),
}
})
};
qcx.dep_context().profiler().query_cache_hit(index.into());
query_blocked_prof_timer.finish_with_query_invocation_id(index.into());
(v, Some(index))
}
Err(cycle) => (mk_cycle(query, qcx, cycle.lift()), None),
}
}
#[inline(never)]
fn try_execute_query<'tcx, Q, const INCR: bool>(
query: Q,
qcx: Q::Qcx,
span: Span,
key: Q::Key,
dep_node: Option<DepNode>,
) -> (Q::Value, Option<DepNodeIndex>)
where
Q: QueryDispatcher<'tcx>,
{
let state = query.query_state(qcx);
let key_hash = sharded::make_hash(&key);
let mut state_lock = state.active.lock_shard_by_hash(key_hash);
// For the parallel compiler we need to check both the query cache and query state structures
// while holding the state lock to ensure that 1) the query has not yet completed and 2) the
// query is not still executing. Without checking the query cache here, we can end up
// re-executing the query since `try_start` only checks that the query is not currently
// executing, but another thread may have already completed the query and stores it result
// in the query cache.
if qcx.dep_context().sess().threads() > 1 {
if let Some((value, index)) = query.query_cache(qcx).lookup(&key) {
qcx.dep_context().profiler().query_cache_hit(index.into());
return (value, Some(index));
}
}
let current_job_id = qcx.current_query_job();
match state_lock.entry(key_hash, equivalent_key(&key), |(k, _)| sharded::make_hash(k)) {
Entry::Vacant(entry) => {
// Nothing has computed or is computing the query, so we start a new job and insert it in the
// state map.
let id = qcx.next_job_id();
let job = QueryJob::new(id, span, current_job_id);
entry.insert((key, ActiveKeyStatus::Started(job)));
// Drop the lock before we start executing the query
drop(state_lock);
execute_job::<Q, INCR>(query, qcx, state, key, key_hash, id, dep_node)
}
Entry::Occupied(mut entry) => {
match &mut entry.get_mut().1 {
ActiveKeyStatus::Started(job) => {
if sync::is_dyn_thread_safe() {
// Get the latch out
let latch = job.latch();
drop(state_lock);
// Only call `wait_for_query` if we're using a Rayon thread pool
// as it will attempt to mark the worker thread as blocked.
return wait_for_query(query, qcx, span, key, latch, current_job_id);
}
let id = job.id;
drop(state_lock);
// If we are single-threaded we know that we have cycle error,
// so we just return the error.
cycle_error(query, qcx, id, span)
}
ActiveKeyStatus::Poisoned => FatalError.raise(),
}
}
}
}
#[inline(always)]
fn execute_job<'tcx, Q, const INCR: bool>(
query: Q,
qcx: Q::Qcx,
state: &'tcx QueryState<'tcx, Q::Key>,
key: Q::Key,
key_hash: u64,
id: QueryJobId,
dep_node: Option<DepNode>,
) -> (Q::Value, Option<DepNodeIndex>)
where
Q: QueryDispatcher<'tcx>,
{
// Use `JobOwner` so the query will be poisoned if executing it panics.
let job_owner = JobOwner { state, key };
debug_assert_eq!(qcx.dep_context().dep_graph().is_fully_enabled(), INCR);
let (result, dep_node_index) = if INCR {
execute_job_incr(
query,
qcx,
qcx.dep_context().dep_graph().data().unwrap(),
key,
dep_node,
id,
)
} else {
execute_job_non_incr(query, qcx, key, id)
};
let cache = query.query_cache(qcx);
if query.feedable() {
// We should not compute queries that also got a value via feeding.
// This can't happen, as query feeding adds the very dependencies to the fed query
// as its feeding query had. So if the fed query is red, so is its feeder, which will
// get evaluated first, and re-feed the query.
if let Some((cached_result, _)) = cache.lookup(&key) {
let Some(hasher) = query.hash_result() else {
panic!(
"no_hash fed query later has its value computed.\n\
Remove `no_hash` modifier to allow recomputation.\n\
The already cached value: {}",
(query.format_value())(&cached_result)
);
};
let (old_hash, new_hash) = qcx.dep_context().with_stable_hashing_context(|mut hcx| {
(hasher(&mut hcx, &cached_result), hasher(&mut hcx, &result))
});
let formatter = query.format_value();
if old_hash != new_hash {
// We have an inconsistency. This can happen if one of the two
// results is tainted by errors.
assert!(
qcx.dep_context().sess().dcx().has_errors().is_some(),
"Computed query value for {:?}({:?}) is inconsistent with fed value,\n\
computed={:#?}\nfed={:#?}",
query.dep_kind(),
key,
formatter(&result),
formatter(&cached_result),
);
}
}
}
job_owner.complete(cache, key_hash, result, dep_node_index);
(result, Some(dep_node_index))
}
// Fast path for when incr. comp. is off.
#[inline(always)]
fn execute_job_non_incr<'tcx, Q>(
query: Q,
qcx: Q::Qcx,
key: Q::Key,
job_id: QueryJobId,
) -> (Q::Value, DepNodeIndex)
where
Q: QueryDispatcher<'tcx>,
{
debug_assert!(!qcx.dep_context().dep_graph().is_fully_enabled());
// Fingerprint the key, just to assert that it doesn't
// have anything we don't consider hashable
if cfg!(debug_assertions) {
let _ = key.to_fingerprint(*qcx.dep_context());
}
let prof_timer = qcx.dep_context().profiler().query_provider();
let result = qcx.start_query(job_id, query.depth_limit(), || query.compute(qcx, key));
let dep_node_index = qcx.dep_context().dep_graph().next_virtual_depnode_index();
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
// Similarly, fingerprint the result to assert that
// it doesn't have anything not considered hashable.
if cfg!(debug_assertions)
&& let Some(hash_result) = query.hash_result()
{
qcx.dep_context().with_stable_hashing_context(|mut hcx| {
hash_result(&mut hcx, &result);
});
}
(result, dep_node_index)
}
#[inline(always)]
fn execute_job_incr<'tcx, Q>(
query: Q,
qcx: Q::Qcx,
dep_graph_data: &DepGraphData<<Q::Qcx as HasDepContext>::Deps>,
key: Q::Key,
mut dep_node_opt: Option<DepNode>,
job_id: QueryJobId,
) -> (Q::Value, DepNodeIndex)
where
Q: QueryDispatcher<'tcx>,
{
if !query.anon() && !query.eval_always() {
// `to_dep_node` is expensive for some `DepKind`s.
let dep_node =
dep_node_opt.get_or_insert_with(|| query.construct_dep_node(*qcx.dep_context(), &key));
// The diagnostics for this query will be promoted to the current session during
// `try_mark_green()`, so we can ignore them here.
if let Some(ret) = qcx.start_query(job_id, false, || {
try_load_from_disk_and_cache_in_memory(query, dep_graph_data, qcx, &key, dep_node)
}) {
return ret;
}
}
let prof_timer = qcx.dep_context().profiler().query_provider();
let (result, dep_node_index) = qcx.start_query(job_id, query.depth_limit(), || {
if query.anon() {
return dep_graph_data.with_anon_task_inner(
*qcx.dep_context(),
query.dep_kind(),
|| query.compute(qcx, key),
);
}
// `to_dep_node` is expensive for some `DepKind`s.
let dep_node =
dep_node_opt.unwrap_or_else(|| query.construct_dep_node(*qcx.dep_context(), &key));
dep_graph_data.with_task(
dep_node,
(qcx, query),
key,
|(qcx, query), key| query.compute(qcx, key),
query.hash_result(),
)
});
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
(result, dep_node_index)
}
#[inline(always)]
fn try_load_from_disk_and_cache_in_memory<'tcx, Q>(
query: Q,
dep_graph_data: &DepGraphData<<Q::Qcx as HasDepContext>::Deps>,
qcx: Q::Qcx,
key: &Q::Key,
dep_node: &DepNode,
) -> Option<(Q::Value, DepNodeIndex)>
where
Q: QueryDispatcher<'tcx>,
{
// Note this function can be called concurrently from the same query
// We must ensure that this is handled correctly.
let (prev_dep_node_index, dep_node_index) = dep_graph_data.try_mark_green(qcx, dep_node)?;
debug_assert!(dep_graph_data.is_index_green(prev_dep_node_index));
// First we try to load the result from the on-disk cache.
// Some things are never cached on disk.
if let Some(result) = query.try_load_from_disk(qcx, key, prev_dep_node_index, dep_node_index) {
if std::intrinsics::unlikely(qcx.dep_context().sess().opts.unstable_opts.query_dep_graph) {
dep_graph_data.mark_debug_loaded_from_disk(*dep_node)
}
let prev_fingerprint = dep_graph_data.prev_fingerprint_of(prev_dep_node_index);
// If `-Zincremental-verify-ich` is specified, re-hash results from
// the cache and make sure that they have the expected fingerprint.
//
// If not, we still seek to verify a subset of fingerprints loaded
// from disk. Re-hashing results is fairly expensive, so we can't
// currently afford to verify every hash. This subset should still
// give us some coverage of potential bugs though.
let try_verify = prev_fingerprint.split().1.as_u64().is_multiple_of(32);
if std::intrinsics::unlikely(
try_verify || qcx.dep_context().sess().opts.unstable_opts.incremental_verify_ich,
) {
incremental_verify_ich(
*qcx.dep_context(),
dep_graph_data,
&result,
prev_dep_node_index,
query.hash_result(),
query.format_value(),
);
}
return Some((result, dep_node_index));
}
// We always expect to find a cached result for things that
// can be forced from `DepNode`.
debug_assert!(
!query.will_cache_on_disk_for_key(*qcx.dep_context(), key)
|| !qcx.dep_context().fingerprint_style(dep_node.kind).reconstructible(),
"missing on-disk cache entry for {dep_node:?}"
);
// Sanity check for the logic in `ensure`: if the node is green and the result loadable,
// we should actually be able to load it.
debug_assert!(
!query.is_loadable_from_disk(qcx, key, prev_dep_node_index),
"missing on-disk cache entry for loadable {dep_node:?}"
);
// We could not load a result from the on-disk cache, so
// recompute.
let prof_timer = qcx.dep_context().profiler().query_provider();
// The dep-graph for this computation is already in-place.
let result = qcx.dep_context().dep_graph().with_ignore(|| query.compute(qcx, *key));
prof_timer.finish_with_query_invocation_id(dep_node_index.into());
// Verify that re-running the query produced a result with the expected hash
// This catches bugs in query implementations, turning them into ICEs.
// For example, a query might sort its result by `DefId` - since `DefId`s are
// not stable across compilation sessions, the result could get up getting sorted
// in a different order when the query is re-run, even though all of the inputs
// (e.g. `DefPathHash` values) were green.
//
// See issue #82920 for an example of a miscompilation that would get turned into
// an ICE by this check
incremental_verify_ich(
*qcx.dep_context(),
dep_graph_data,
&result,
prev_dep_node_index,
query.hash_result(),
query.format_value(),
);
Some((result, dep_node_index))
}
#[inline]
#[instrument(skip(tcx, dep_graph_data, result, hash_result, format_value), level = "debug")]
pub(crate) fn incremental_verify_ich<Tcx, V>(
pub fn incremental_verify_ich<Tcx, V>(
tcx: Tcx,
dep_graph_data: &DepGraphData<Tcx::Deps>,
result: &V,
@ -767,121 +143,8 @@ fn incremental_verify_ich_failed<Tcx>(
INSIDE_VERIFY_PANIC.set(old_in_panic);
}
/// Ensure that either this query has all green inputs or been executed.
/// Executing `query::ensure(D)` is considered a read of the dep-node `D`.
/// Returns true if the query should still run.
///
/// This function is particularly useful when executing passes for their
/// side-effects -- e.g., in order to report errors for erroneous programs.
///
/// Note: The optimization is only available during incr. comp.
#[inline(never)]
fn ensure_must_run<'tcx, Q>(
query: Q,
qcx: Q::Qcx,
key: &Q::Key,
check_cache: bool,
) -> (bool, Option<DepNode>)
where
Q: QueryDispatcher<'tcx>,
{
if query.eval_always() {
return (true, None);
}
// Ensuring an anonymous query makes no sense
assert!(!query.anon());
let dep_node = query.construct_dep_node(*qcx.dep_context(), key);
let dep_graph = qcx.dep_context().dep_graph();
let serialized_dep_node_index = match dep_graph.try_mark_green(qcx, &dep_node) {
None => {
// A None return from `try_mark_green` means that this is either
// a new dep node or that the dep node has already been marked red.
// Either way, we can't call `dep_graph.read()` as we don't have the
// DepNodeIndex. We must invoke the query itself. The performance cost
// this introduces should be negligible as we'll immediately hit the
// in-memory cache, or another query down the line will.
return (true, Some(dep_node));
}
Some((serialized_dep_node_index, dep_node_index)) => {
dep_graph.read_index(dep_node_index);
qcx.dep_context().profiler().query_cache_hit(dep_node_index.into());
serialized_dep_node_index
}
};
// We do not need the value at all, so do not check the cache.
if !check_cache {
return (false, None);
}
let loadable = query.is_loadable_from_disk(qcx, key, serialized_dep_node_index);
(!loadable, Some(dep_node))
}
#[derive(Debug)]
pub enum QueryMode {
Get,
Ensure { check_cache: bool },
}
#[inline(always)]
pub fn get_query_non_incr<'tcx, Q>(query: Q, qcx: Q::Qcx, span: Span, key: Q::Key) -> Q::Value
where
Q: QueryDispatcher<'tcx>,
{
debug_assert!(!qcx.dep_context().dep_graph().is_fully_enabled());
ensure_sufficient_stack(|| try_execute_query::<Q, false>(query, qcx, span, key, None).0)
}
#[inline(always)]
pub fn get_query_incr<'tcx, Q>(
query: Q,
qcx: Q::Qcx,
span: Span,
key: Q::Key,
mode: QueryMode,
) -> Option<Q::Value>
where
Q: QueryDispatcher<'tcx>,
{
debug_assert!(qcx.dep_context().dep_graph().is_fully_enabled());
let dep_node = if let QueryMode::Ensure { check_cache } = mode {
let (must_run, dep_node) = ensure_must_run(query, qcx, &key, check_cache);
if !must_run {
return None;
}
dep_node
} else {
None
};
let (result, dep_node_index) =
ensure_sufficient_stack(|| try_execute_query::<Q, true>(query, qcx, span, key, dep_node));
if let Some(dep_node_index) = dep_node_index {
qcx.dep_context().dep_graph().read_index(dep_node_index)
}
Some(result)
}
pub fn force_query<'tcx, Q>(query: Q, qcx: Q::Qcx, key: Q::Key, dep_node: DepNode)
where
Q: QueryDispatcher<'tcx>,
{
// We may be concurrently trying both execute and force a query.
// Ensure that only one of them runs the query.
if let Some((_, index)) = query.query_cache(qcx).lookup(&key) {
qcx.dep_context().profiler().query_cache_hit(index.into());
return;
}
debug_assert!(!query.anon());
ensure_sufficient_stack(|| {
try_execute_query::<Q, true>(query, qcx, DUMMY_SP, key, Some(dep_node))
});
}