user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove wrapper struct QueryCtxt

Browse source 
This struct was only wrapping `TyCtxt` in order to implement traits that
were removed by RUST-152636.

This commit also slightly simplifies the signature of `execute_job_incr`, by
having it call `tcx.dep_graph.data()` internally.
This commit is contained in:
Zalathar 2026-02-15 18:42:52 +11:00
parent d1a11b670b
commit fbb34d8c75
6 changed files with 203 additions and 223 deletions
Download patch file Download diff file Expand all files Collapse all files

4
compiler/rustc_interface/src/interface.rs
View file

@ -16,7 +16,7 @@ use rustc_parse::lexer::StripTokens;
use rustc_parse::new_parser_from_source_str; use rustc_parse::new_parser_from_source_str;
use rustc_parse::parser::Recovery; use rustc_parse::parser::Recovery;
use rustc_parse::parser::attr::AllowLeadingUnsafe; use rustc_parse::parser::attr::AllowLeadingUnsafe;
use rustc_query_impl::{QueryCtxt, print_query_stack}; use rustc_query_impl::print_query_stack;
use rustc_session::config::{self, Cfg, CheckCfg, ExpectedValues, Input, OutFileName}; use rustc_session::config::{self, Cfg, CheckCfg, ExpectedValues, Input, OutFileName};
use rustc_session::parse::ParseSess; use rustc_session::parse::ParseSess;
use rustc_session::{CompilerIO, EarlyDiagCtxt, Session, lint}; use rustc_session::{CompilerIO, EarlyDiagCtxt, Session, lint};
@ -556,7 +556,7 @@ pub fn try_print_query_stack(
let all_frames = ty::tls::with_context_opt(|icx| { let all_frames = ty::tls::with_context_opt(|icx| {
if let Some(icx) = icx { if let Some(icx) = icx {
ty::print::with_no_queries!(print_query_stack( ty::print::with_no_queries!(print_query_stack(
QueryCtxt::new(icx.tcx), icx.tcx,
icx.query, icx.query,
dcx, dcx,
limit_frames, limit_frames,

5
compiler/rustc_interface/src/util.rs
View file

@ -18,6 +18,7 @@ use rustc_data_structures::sync;
use rustc_metadata::{DylibError, EncodedMetadata, load_symbol_from_dylib}; use rustc_metadata::{DylibError, EncodedMetadata, load_symbol_from_dylib};
use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
use rustc_middle::ty::{CurrentGcx, TyCtxt}; use rustc_middle::ty::{CurrentGcx, TyCtxt};
use rustc_query_impl::collect_active_jobs_from_all_queries;
use rustc_session::config::{ use rustc_session::config::{
Cfg, CrateType, OutFileName, OutputFilenames, OutputTypes, Sysroot, host_tuple, Cfg, CrateType, OutFileName, OutputFilenames, OutputTypes, Sysroot, host_tuple,
}; };
@ -184,7 +185,7 @@ pub(crate) fn run_in_thread_pool_with_globals<
use rustc_data_structures::defer; use rustc_data_structures::defer;
use rustc_data_structures::sync::FromDyn; use rustc_data_structures::sync::FromDyn;
use rustc_middle::ty::tls; use rustc_middle::ty::tls;
use rustc_query_impl::{QueryCtxt, break_query_cycles}; use rustc_query_impl::break_query_cycles;
let thread_stack_size = init_stack_size(thread_builder_diag); let thread_stack_size = init_stack_size(thread_builder_diag);
@ -253,7 +254,7 @@ internal compiler error: query cycle handler thread panicked, aborting process";
|| { || {
// Ensure there were no errors collecting all active jobs. // Ensure there were no errors collecting all active jobs.
// We need the complete map to ensure we find a cycle to break. // We need the complete map to ensure we find a cycle to break.
QueryCtxt::new(tcx).collect_active_jobs_from_all_queries(false).expect( collect_active_jobs_from_all_queries(tcx, false).expect(
"failed to collect active queries in deadlock handler", "failed to collect active queries in deadlock handler",
) )
}, },

181
compiler/rustc_query_impl/src/execution.rs
View file

@ -5,7 +5,7 @@ use rustc_data_structures::hash_table::{Entry, HashTable};
use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_data_structures::stack::ensure_sufficient_stack;
use rustc_data_structures::{outline, sharded, sync}; use rustc_data_structures::{outline, sharded, sync};
use rustc_errors::{Diag, FatalError, StashKey}; use rustc_errors::{Diag, FatalError, StashKey};
use rustc_middle::dep_graph::{DepGraphData, DepNodeKey, HasDepContext}; use rustc_middle::dep_graph::{DepGraphData, DepNodeKey};
use rustc_middle::query::{ use rustc_middle::query::{
ActiveKeyStatus, CycleError, CycleErrorHandling, QueryCache, QueryJob, QueryJobId, QueryLatch, ActiveKeyStatus, CycleError, CycleErrorHandling, QueryCache, QueryJob, QueryJobId, QueryLatch,
QueryMode, QueryStackDeferred, QueryStackFrame, QueryState, QueryMode, QueryStackDeferred, QueryStackFrame, QueryState,
@ -16,7 +16,10 @@ use rustc_span::{DUMMY_SP, Span};
use crate::dep_graph::{DepNode, DepNodeIndex}; use crate::dep_graph::{DepNode, DepNodeIndex};
use crate::job::{QueryJobInfo, QueryJobMap, find_cycle_in_stack, report_cycle}; use crate::job::{QueryJobInfo, QueryJobMap, find_cycle_in_stack, report_cycle};
use crate::{QueryCtxt, QueryFlags, SemiDynamicQueryDispatcher}; use crate::plumbing::{
collect_active_jobs_from_all_queries, current_query_job, next_job_id, start_query,
};
use crate::{QueryFlags, SemiDynamicQueryDispatcher};
#[inline] #[inline]
fn equivalent_key<K: Eq, V>(k: &K) -> impl Fn(&(K, V)) -> bool + '_ { fn equivalent_key<K: Eq, V>(k: &K) -> impl Fn(&(K, V)) -> bool + '_ {
@ -101,32 +104,32 @@ where
#[inline(never)] #[inline(never)]
fn mk_cycle<'tcx, C: QueryCache, const FLAGS: QueryFlags>( fn mk_cycle<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
qcx: QueryCtxt<'tcx>, tcx: TyCtxt<'tcx>,
cycle_error: CycleError, cycle_error: CycleError,
) -> C::Value { ) -> C::Value {
let error = report_cycle(qcx.tcx.sess, &cycle_error); let error = report_cycle(tcx.sess, &cycle_error);
handle_cycle_error(query, qcx, &cycle_error, error) handle_cycle_error(query, tcx, &cycle_error, error)
} }
fn handle_cycle_error<'tcx, C: QueryCache, const FLAGS: QueryFlags>( fn handle_cycle_error<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
qcx: QueryCtxt<'tcx>, tcx: TyCtxt<'tcx>,
cycle_error: &CycleError, cycle_error: &CycleError,
error: Diag<'_>, error: Diag<'_>,
) -> C::Value { ) -> C::Value {
match query.cycle_error_handling() { match query.cycle_error_handling() {
CycleErrorHandling::Error => { CycleErrorHandling::Error => {
let guar = error.emit(); let guar = error.emit();
query.value_from_cycle_error(qcx.tcx, cycle_error, guar) query.value_from_cycle_error(tcx, cycle_error, guar)
} }
CycleErrorHandling::Fatal => { CycleErrorHandling::Fatal => {
error.emit(); error.emit();
qcx.tcx.dcx().abort_if_errors(); tcx.dcx().abort_if_errors();
unreachable!() unreachable!()
} }
CycleErrorHandling::DelayBug => { CycleErrorHandling::DelayBug => {
let guar = error.delay_as_bug(); let guar = error.delay_as_bug();
query.value_from_cycle_error(qcx.tcx, cycle_error, guar) query.value_from_cycle_error(tcx, cycle_error, guar)
} }
CycleErrorHandling::Stash => { CycleErrorHandling::Stash => {
let guar = if let Some(root) = cycle_error.cycle.first() let guar = if let Some(root) = cycle_error.cycle.first()
@ -136,7 +139,7 @@ fn handle_cycle_error<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
} else { } else {
error.emit() error.emit()
}; };
query.value_from_cycle_error(qcx.tcx, cycle_error, guar) query.value_from_cycle_error(tcx, cycle_error, guar)
} }
} }
} }
@ -207,25 +210,24 @@ where
#[inline(never)] #[inline(never)]
fn cycle_error<'tcx, C: QueryCache, const FLAGS: QueryFlags>( fn cycle_error<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
qcx: QueryCtxt<'tcx>, tcx: TyCtxt<'tcx>,
try_execute: QueryJobId, try_execute: QueryJobId,
span: Span, span: Span,
) -> (C::Value, Option<DepNodeIndex>) { ) -> (C::Value, Option<DepNodeIndex>) {
// Ensure there was no errors collecting all active jobs. // Ensure there was no errors collecting all active jobs.
// We need the complete map to ensure we find a cycle to break. // We need the complete map to ensure we find a cycle to break.
let job_map = qcx let job_map = collect_active_jobs_from_all_queries(tcx, false)
.collect_active_jobs_from_all_queries(false)
.ok() .ok()
.expect("failed to collect active queries"); .expect("failed to collect active queries");
let error = find_cycle_in_stack(try_execute, job_map, &qcx.current_query_job(), span); let error = find_cycle_in_stack(try_execute, job_map, &current_query_job(tcx), span);
(mk_cycle(query, qcx, error.lift()), None) (mk_cycle(query, tcx, error.lift()), None)
} }
#[inline(always)] #[inline(always)]
fn wait_for_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>( fn wait_for_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
qcx: QueryCtxt<'tcx>, tcx: TyCtxt<'tcx>,
span: Span, span: Span,
key: C::Key, key: C::Key,
latch: QueryLatch<'tcx>, latch: QueryLatch<'tcx>,
@ -234,20 +236,20 @@ fn wait_for_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
// For parallel queries, we'll block and wait until the query running // For parallel queries, we'll block and wait until the query running
// in another thread has completed. Record how long we wait in the // in another thread has completed. Record how long we wait in the
// self-profiler. // self-profiler.
let query_blocked_prof_timer = qcx.tcx.prof.query_blocked(); let query_blocked_prof_timer = tcx.prof.query_blocked();
// With parallel queries we might just have to wait on some other // With parallel queries we might just have to wait on some other
// thread. // thread.
let result = latch.wait_on(qcx.tcx, current, span); let result = latch.wait_on(tcx, current, span);
match result { match result {
Ok(()) => { Ok(()) => {
let Some((v, index)) = query.query_cache(qcx).lookup(&key) else { let Some((v, index)) = query.query_cache(tcx).lookup(&key) else {
outline(|| { outline(|| {
// We didn't find the query result in the query cache. Check if it was // We didn't find the query result in the query cache. Check if it was
// poisoned due to a panic instead. // poisoned due to a panic instead.
let key_hash = sharded::make_hash(&key); let key_hash = sharded::make_hash(&key);
let shard = query.query_state(qcx).active.lock_shard_by_hash(key_hash); let shard = query.query_state(tcx).active.lock_shard_by_hash(key_hash);
match shard.find(key_hash, equivalent_key(&key)) { match shard.find(key_hash, equivalent_key(&key)) {
// The query we waited on panicked. Continue unwinding here. // The query we waited on panicked. Continue unwinding here.
Some((_, ActiveKeyStatus::Poisoned)) => FatalError.raise(), Some((_, ActiveKeyStatus::Poisoned)) => FatalError.raise(),
@ -259,24 +261,24 @@ fn wait_for_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
}) })
}; };
qcx.tcx.prof.query_cache_hit(index.into()); tcx.prof.query_cache_hit(index.into());
query_blocked_prof_timer.finish_with_query_invocation_id(index.into()); query_blocked_prof_timer.finish_with_query_invocation_id(index.into());
(v, Some(index)) (v, Some(index))
} }
Err(cycle) => (mk_cycle(query, qcx, cycle.lift()), None), Err(cycle) => (mk_cycle(query, tcx, cycle.lift()), None),
} }
} }
#[inline(never)] #[inline(never)]
fn try_execute_query<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>( fn try_execute_query<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>(
query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
qcx: QueryCtxt<'tcx>, tcx: TyCtxt<'tcx>,
span: Span, span: Span,
key: C::Key, key: C::Key,
dep_node: Option<DepNode>, dep_node: Option<DepNode>,
) -> (C::Value, Option<DepNodeIndex>) { ) -> (C::Value, Option<DepNodeIndex>) {
let state = query.query_state(qcx); let state = query.query_state(tcx);
let key_hash = sharded::make_hash(&key); let key_hash = sharded::make_hash(&key);
let mut state_lock = state.active.lock_shard_by_hash(key_hash); let mut state_lock = state.active.lock_shard_by_hash(key_hash);
@ -286,27 +288,27 @@ fn try_execute_query<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: b
// re-executing the query since `try_start` only checks that the query is not currently // re-executing the query since `try_start` only checks that the query is not currently
// executing, but another thread may have already completed the query and stores it result // executing, but another thread may have already completed the query and stores it result
// in the query cache. // in the query cache.
if qcx.tcx.sess.threads() > 1 { if tcx.sess.threads() > 1 {
if let Some((value, index)) = query.query_cache(qcx).lookup(&key) { if let Some((value, index)) = query.query_cache(tcx).lookup(&key) {
qcx.tcx.prof.query_cache_hit(index.into()); tcx.prof.query_cache_hit(index.into());
return (value, Some(index)); return (value, Some(index));
} }
} }
let current_job_id = qcx.current_query_job(); let current_job_id = current_query_job(tcx);
match state_lock.entry(key_hash, equivalent_key(&key), |(k, _)| sharded::make_hash(k)) { match state_lock.entry(key_hash, equivalent_key(&key), |(k, _)| sharded::make_hash(k)) {
Entry::Vacant(entry) => { Entry::Vacant(entry) => {
// Nothing has computed or is computing the query, so we start a new job and insert it in the // Nothing has computed or is computing the query, so we start a new job and insert it in the
// state map. // state map.
let id = qcx.next_job_id(); let id = next_job_id(tcx);
let job = QueryJob::new(id, span, current_job_id); let job = QueryJob::new(id, span, current_job_id);
entry.insert((key, ActiveKeyStatus::Started(job))); entry.insert((key, ActiveKeyStatus::Started(job)));
// Drop the lock before we start executing the query // Drop the lock before we start executing the query
drop(state_lock); drop(state_lock);
execute_job::<C, FLAGS, INCR>(query, qcx, state, key, key_hash, id, dep_node) execute_job::<C, FLAGS, INCR>(query, tcx, state, key, key_hash, id, dep_node)
} }
Entry::Occupied(mut entry) => { Entry::Occupied(mut entry) => {
match &mut entry.get_mut().1 { match &mut entry.get_mut().1 {
@ -318,7 +320,7 @@ fn try_execute_query<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: b
// Only call `wait_for_query` if we're using a Rayon thread pool // Only call `wait_for_query` if we're using a Rayon thread pool
// as it will attempt to mark the worker thread as blocked. // as it will attempt to mark the worker thread as blocked.
return wait_for_query(query, qcx, span, key, latch, current_job_id); return wait_for_query(query, tcx, span, key, latch, current_job_id);
} }
let id = job.id; let id = job.id;
@ -326,7 +328,7 @@ fn try_execute_query<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: b
// If we are single-threaded we know that we have cycle error, // If we are single-threaded we know that we have cycle error,
// so we just return the error. // so we just return the error.
cycle_error(query, qcx, id, span) cycle_error(query, tcx, id, span)
} }
ActiveKeyStatus::Poisoned => FatalError.raise(), ActiveKeyStatus::Poisoned => FatalError.raise(),
} }
@ -337,7 +339,7 @@ fn try_execute_query<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: b
#[inline(always)] #[inline(always)]
fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>( fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>(
query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
qcx: QueryCtxt<'tcx>, tcx: TyCtxt<'tcx>,
state: &'tcx QueryState<'tcx, C::Key>, state: &'tcx QueryState<'tcx, C::Key>,
key: C::Key, key: C::Key,
key_hash: u64, key_hash: u64,
@ -348,16 +350,16 @@ fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>(
// panic occurs while executing the query (or any intermediate plumbing). // panic occurs while executing the query (or any intermediate plumbing).
let job_guard = ActiveJobGuard { state, key, key_hash }; let job_guard = ActiveJobGuard { state, key, key_hash };
debug_assert_eq!(qcx.tcx.dep_graph.is_fully_enabled(), INCR); debug_assert_eq!(tcx.dep_graph.is_fully_enabled(), INCR);
// Delegate to another function to actually execute the query job. // Delegate to another function to actually execute the query job.
let (result, dep_node_index) = if INCR { let (result, dep_node_index) = if INCR {
execute_job_incr(query, qcx, qcx.tcx.dep_graph.data().unwrap(), key, dep_node, id) execute_job_incr(query, tcx, key, dep_node, id)
} else { } else {
execute_job_non_incr(query, qcx, key, id) execute_job_non_incr(query, tcx, key, id)
}; };
let cache = query.query_cache(qcx); let cache = query.query_cache(tcx);
if query.feedable() { if query.feedable() {
// We should not compute queries that also got a value via feeding. // We should not compute queries that also got a value via feeding.
// This can't happen, as query feeding adds the very dependencies to the fed query // This can't happen, as query feeding adds the very dependencies to the fed query
@ -373,7 +375,7 @@ fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>(
); );
}; };
let (old_hash, new_hash) = qcx.dep_context().with_stable_hashing_context(|mut hcx| { let (old_hash, new_hash) = tcx.with_stable_hashing_context(|mut hcx| {
(hasher(&mut hcx, &cached_result), hasher(&mut hcx, &result)) (hasher(&mut hcx, &cached_result), hasher(&mut hcx, &result))
}); });
let formatter = query.format_value(); let formatter = query.format_value();
@ -381,7 +383,7 @@ fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>(
// We have an inconsistency. This can happen if one of the two // We have an inconsistency. This can happen if one of the two
// results is tainted by errors. // results is tainted by errors.
assert!( assert!(
qcx.tcx.dcx().has_errors().is_some(), tcx.dcx().has_errors().is_some(),
"Computed query value for {:?}({:?}) is inconsistent with fed value,\n\ "Computed query value for {:?}({:?}) is inconsistent with fed value,\n\
computed={:#?}\nfed={:#?}", computed={:#?}\nfed={:#?}",
query.dep_kind(), query.dep_kind(),
@ -403,22 +405,22 @@ fn execute_job<'tcx, C: QueryCache, const FLAGS: QueryFlags, const INCR: bool>(
#[inline(always)] #[inline(always)]
fn execute_job_non_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>( fn execute_job_non_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
qcx: QueryCtxt<'tcx>, tcx: TyCtxt<'tcx>,
key: C::Key, key: C::Key,
job_id: QueryJobId, job_id: QueryJobId,
) -> (C::Value, DepNodeIndex) { ) -> (C::Value, DepNodeIndex) {
debug_assert!(!qcx.tcx.dep_graph.is_fully_enabled()); debug_assert!(!tcx.dep_graph.is_fully_enabled());
// Fingerprint the key, just to assert that it doesn't // Fingerprint the key, just to assert that it doesn't
// have anything we don't consider hashable // have anything we don't consider hashable
if cfg!(debug_assertions) { if cfg!(debug_assertions) {
let _ = key.to_fingerprint(qcx.tcx); let _ = key.to_fingerprint(tcx);
} }
let prof_timer = qcx.tcx.prof.query_provider(); let prof_timer = tcx.prof.query_provider();
// Call the query provider. // Call the query provider.
let result = qcx.start_query(job_id, query.depth_limit(), || query.invoke_provider(qcx, key)); let result = start_query(tcx, job_id, query.depth_limit(), || query.invoke_provider(tcx, key));
let dep_node_index = qcx.tcx.dep_graph.next_virtual_depnode_index(); let dep_node_index = tcx.dep_graph.next_virtual_depnode_index();
prof_timer.finish_with_query_invocation_id(dep_node_index.into()); prof_timer.finish_with_query_invocation_id(dep_node_index.into());
// Similarly, fingerprint the result to assert that // Similarly, fingerprint the result to assert that
@ -426,7 +428,7 @@ fn execute_job_non_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
if cfg!(debug_assertions) if cfg!(debug_assertions)
&& let Some(hash_result) = query.hash_result() && let Some(hash_result) = query.hash_result()
{ {
qcx.dep_context().with_stable_hashing_context(|mut hcx| { tcx.with_stable_hashing_context(|mut hcx| {
hash_result(&mut hcx, &result); hash_result(&mut hcx, &result);
}); });
} }
@ -437,44 +439,45 @@ fn execute_job_non_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
#[inline(always)] #[inline(always)]
fn execute_job_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>( fn execute_job_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
qcx: QueryCtxt<'tcx>, tcx: TyCtxt<'tcx>,
dep_graph_data: &DepGraphData,
key: C::Key, key: C::Key,
mut dep_node_opt: Option<DepNode>, mut dep_node_opt: Option<DepNode>,
job_id: QueryJobId, job_id: QueryJobId,
) -> (C::Value, DepNodeIndex) { ) -> (C::Value, DepNodeIndex) {
let dep_graph_data =
tcx.dep_graph.data().expect("should always be present in incremental mode");
if !query.anon() && !query.eval_always() { if !query.anon() && !query.eval_always() {
// `to_dep_node` is expensive for some `DepKind`s. // `to_dep_node` is expensive for some `DepKind`s.
let dep_node = dep_node_opt.get_or_insert_with(|| query.construct_dep_node(qcx.tcx, &key)); let dep_node = dep_node_opt.get_or_insert_with(|| query.construct_dep_node(tcx, &key));
// The diagnostics for this query will be promoted to the current session during // The diagnostics for this query will be promoted to the current session during
// `try_mark_green()`, so we can ignore them here. // `try_mark_green()`, so we can ignore them here.
if let Some(ret) = qcx.start_query(job_id, false, || { if let Some(ret) = start_query(tcx, job_id, false, || {
try_load_from_disk_and_cache_in_memory(query, dep_graph_data, qcx, &key, dep_node) try_load_from_disk_and_cache_in_memory(query, dep_graph_data, tcx, &key, dep_node)
}) { }) {
return ret; return ret;
} }
} }
let prof_timer = qcx.tcx.prof.query_provider(); let prof_timer = tcx.prof.query_provider();
let (result, dep_node_index) = qcx.start_query(job_id, query.depth_limit(), || { let (result, dep_node_index) = start_query(tcx, job_id, query.depth_limit(), || {
if query.anon() { if query.anon() {
// Call the query provider inside an anon task. // Call the query provider inside an anon task.
return dep_graph_data.with_anon_task_inner(qcx.tcx, query.dep_kind(), || { return dep_graph_data
query.invoke_provider(qcx, key) .with_anon_task_inner(tcx, query.dep_kind(), || query.invoke_provider(tcx, key));
});
} }
// `to_dep_node` is expensive for some `DepKind`s. // `to_dep_node` is expensive for some `DepKind`s.
let dep_node = dep_node_opt.unwrap_or_else(|| query.construct_dep_node(qcx.tcx, &key)); let dep_node = dep_node_opt.unwrap_or_else(|| query.construct_dep_node(tcx, &key));
// Call the query provider. // Call the query provider.
dep_graph_data.with_task( dep_graph_data.with_task(
dep_node, dep_node,
(qcx, query), (tcx, query),
key, key,
|(qcx, query), key| query.invoke_provider(qcx, key), |(tcx, query), key| query.invoke_provider(tcx, key),
query.hash_result(), query.hash_result(),
) )
}); });
@ -488,21 +491,21 @@ fn execute_job_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
fn try_load_from_disk_and_cache_in_memory<'tcx, C: QueryCache, const FLAGS: QueryFlags>( fn try_load_from_disk_and_cache_in_memory<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
dep_graph_data: &DepGraphData, dep_graph_data: &DepGraphData,
qcx: QueryCtxt<'tcx>, tcx: TyCtxt<'tcx>,
key: &C::Key, key: &C::Key,
dep_node: &DepNode, dep_node: &DepNode,
) -> Option<(C::Value, DepNodeIndex)> { ) -> Option<(C::Value, DepNodeIndex)> {
// Note this function can be called concurrently from the same query // Note this function can be called concurrently from the same query
// We must ensure that this is handled correctly. // We must ensure that this is handled correctly.
let (prev_dep_node_index, dep_node_index) = dep_graph_data.try_mark_green(qcx.tcx, dep_node)?; let (prev_dep_node_index, dep_node_index) = dep_graph_data.try_mark_green(tcx, dep_node)?;
debug_assert!(dep_graph_data.is_index_green(prev_dep_node_index)); debug_assert!(dep_graph_data.is_index_green(prev_dep_node_index));
// First we try to load the result from the on-disk cache. // First we try to load the result from the on-disk cache.
// Some things are never cached on disk. // Some things are never cached on disk.
if let Some(result) = query.try_load_from_disk(qcx, key, prev_dep_node_index, dep_node_index) { if let Some(result) = query.try_load_from_disk(tcx, key, prev_dep_node_index, dep_node_index) {
if std::intrinsics::unlikely(qcx.tcx.sess.opts.unstable_opts.query_dep_graph) { if std::intrinsics::unlikely(tcx.sess.opts.unstable_opts.query_dep_graph) {
dep_graph_data.mark_debug_loaded_from_disk(*dep_node) dep_graph_data.mark_debug_loaded_from_disk(*dep_node)
} }
@ -516,10 +519,10 @@ fn try_load_from_disk_and_cache_in_memory<'tcx, C: QueryCache, const FLAGS: Quer
// give us some coverage of potential bugs though. // give us some coverage of potential bugs though.
let try_verify = prev_fingerprint.split().1.as_u64().is_multiple_of(32); let try_verify = prev_fingerprint.split().1.as_u64().is_multiple_of(32);
if std::intrinsics::unlikely( if std::intrinsics::unlikely(
try_verify || qcx.tcx.sess.opts.unstable_opts.incremental_verify_ich, try_verify || tcx.sess.opts.unstable_opts.incremental_verify_ich,
) { ) {
incremental_verify_ich( incremental_verify_ich(
qcx.tcx, tcx,
dep_graph_data, dep_graph_data,
&result, &result,
prev_dep_node_index, prev_dep_node_index,
@ -534,25 +537,25 @@ fn try_load_from_disk_and_cache_in_memory<'tcx, C: QueryCache, const FLAGS: Quer
// We always expect to find a cached result for things that // We always expect to find a cached result for things that
// can be forced from `DepNode`. // can be forced from `DepNode`.
debug_assert!( debug_assert!(
!query.will_cache_on_disk_for_key(qcx.tcx, key) !query.will_cache_on_disk_for_key(tcx, key)
|| !qcx.tcx.fingerprint_style(dep_node.kind).reconstructible(), || !tcx.fingerprint_style(dep_node.kind).reconstructible(),
"missing on-disk cache entry for {dep_node:?}" "missing on-disk cache entry for {dep_node:?}"
); );
// Sanity check for the logic in `ensure`: if the node is green and the result loadable, // Sanity check for the logic in `ensure`: if the node is green and the result loadable,
// we should actually be able to load it. // we should actually be able to load it.
debug_assert!( debug_assert!(
!query.is_loadable_from_disk(qcx, key, prev_dep_node_index), !query.is_loadable_from_disk(tcx, key, prev_dep_node_index),
"missing on-disk cache entry for loadable {dep_node:?}" "missing on-disk cache entry for loadable {dep_node:?}"
); );
// We could not load a result from the on-disk cache, so // We could not load a result from the on-disk cache, so
// recompute. // recompute.
let prof_timer = qcx.tcx.prof.query_provider(); let prof_timer = tcx.prof.query_provider();
// The dep-graph for this computation is already in-place. // The dep-graph for this computation is already in-place.
// Call the query provider. // Call the query provider.
let result = qcx.tcx.dep_graph.with_ignore(|| query.invoke_provider(qcx, *key)); let result = tcx.dep_graph.with_ignore(|| query.invoke_provider(tcx, *key));
prof_timer.finish_with_query_invocation_id(dep_node_index.into()); prof_timer.finish_with_query_invocation_id(dep_node_index.into());
@ -566,7 +569,7 @@ fn try_load_from_disk_and_cache_in_memory<'tcx, C: QueryCache, const FLAGS: Quer
// See issue #82920 for an example of a miscompilation that would get turned into // See issue #82920 for an example of a miscompilation that would get turned into
// an ICE by this check // an ICE by this check
incremental_verify_ich( incremental_verify_ich(
qcx.tcx, tcx,
dep_graph_data, dep_graph_data,
&result, &result,
prev_dep_node_index, prev_dep_node_index,
@ -588,7 +591,7 @@ fn try_load_from_disk_and_cache_in_memory<'tcx, C: QueryCache, const FLAGS: Quer
#[inline(never)] #[inline(never)]
fn ensure_must_run<'tcx, C: QueryCache, const FLAGS: QueryFlags>( fn ensure_must_run<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
qcx: QueryCtxt<'tcx>, tcx: TyCtxt<'tcx>,
key: &C::Key, key: &C::Key,
check_cache: bool, check_cache: bool,
) -> (bool, Option<DepNode>) { ) -> (bool, Option<DepNode>) {
@ -599,10 +602,10 @@ fn ensure_must_run<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
// Ensuring an anonymous query makes no sense // Ensuring an anonymous query makes no sense
assert!(!query.anon()); assert!(!query.anon());
let dep_node = query.construct_dep_node(qcx.tcx, key); let dep_node = query.construct_dep_node(tcx, key);
let dep_graph = &qcx.tcx.dep_graph; let dep_graph = &tcx.dep_graph;
let serialized_dep_node_index = match dep_graph.try_mark_green(qcx.tcx, &dep_node) { let serialized_dep_node_index = match dep_graph.try_mark_green(tcx, &dep_node) {
None => { None => {
// A None return from `try_mark_green` means that this is either // A None return from `try_mark_green` means that this is either
// a new dep node or that the dep node has already been marked red. // a new dep node or that the dep node has already been marked red.
@ -614,7 +617,7 @@ fn ensure_must_run<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
} }
Some((serialized_dep_node_index, dep_node_index)) => { Some((serialized_dep_node_index, dep_node_index)) => {
dep_graph.read_index(dep_node_index); dep_graph.read_index(dep_node_index);
qcx.tcx.prof.query_cache_hit(dep_node_index.into()); tcx.prof.query_cache_hit(dep_node_index.into());
serialized_dep_node_index serialized_dep_node_index
} }
}; };
@ -624,34 +627,34 @@ fn ensure_must_run<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
return (false, None); return (false, None);
} }
let loadable = query.is_loadable_from_disk(qcx, key, serialized_dep_node_index); let loadable = query.is_loadable_from_disk(tcx, key, serialized_dep_node_index);
(!loadable, Some(dep_node)) (!loadable, Some(dep_node))
} }
#[inline(always)] #[inline(always)]
pub(super) fn get_query_non_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>( pub(super) fn get_query_non_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
qcx: QueryCtxt<'tcx>, tcx: TyCtxt<'tcx>,
span: Span, span: Span,
key: C::Key, key: C::Key,
) -> C::Value { ) -> C::Value {
debug_assert!(!qcx.tcx.dep_graph.is_fully_enabled()); debug_assert!(!tcx.dep_graph.is_fully_enabled());
ensure_sufficient_stack(|| try_execute_query::<C, FLAGS, false>(query, qcx, span, key, None).0) ensure_sufficient_stack(|| try_execute_query::<C, FLAGS, false>(query, tcx, span, key, None).0)
} }
#[inline(always)] #[inline(always)]
pub(super) fn get_query_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>( pub(super) fn get_query_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
qcx: QueryCtxt<'tcx>, tcx: TyCtxt<'tcx>,
span: Span, span: Span,
key: C::Key, key: C::Key,
mode: QueryMode, mode: QueryMode,
) -> Option<C::Value> { ) -> Option<C::Value> {
debug_assert!(qcx.tcx.dep_graph.is_fully_enabled()); debug_assert!(tcx.dep_graph.is_fully_enabled());
let dep_node = if let QueryMode::Ensure { check_cache } = mode { let dep_node = if let QueryMode::Ensure { check_cache } = mode {
let (must_run, dep_node) = ensure_must_run(query, qcx, &key, check_cache); let (must_run, dep_node) = ensure_must_run(query, tcx, &key, check_cache);
if !must_run { if !must_run {
return None; return None;
} }
@ -661,30 +664,30 @@ pub(super) fn get_query_incr<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
}; };
let (result, dep_node_index) = ensure_sufficient_stack(|| { let (result, dep_node_index) = ensure_sufficient_stack(|| {
try_execute_query::<C, FLAGS, true>(query, qcx, span, key, dep_node) try_execute_query::<C, FLAGS, true>(query, tcx, span, key, dep_node)
}); });
if let Some(dep_node_index) = dep_node_index { if let Some(dep_node_index) = dep_node_index {
qcx.tcx.dep_graph.read_index(dep_node_index) tcx.dep_graph.read_index(dep_node_index)
} }
Some(result) Some(result)
} }
pub(crate) fn force_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>( pub(crate) fn force_query<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
qcx: QueryCtxt<'tcx>, tcx: TyCtxt<'tcx>,
key: C::Key, key: C::Key,
dep_node: DepNode, dep_node: DepNode,
) { ) {
// We may be concurrently trying both execute and force a query. // We may be concurrently trying both execute and force a query.
// Ensure that only one of them runs the query. // Ensure that only one of them runs the query.
if let Some((_, index)) = query.query_cache(qcx).lookup(&key) { if let Some((_, index)) = query.query_cache(tcx).lookup(&key) {
qcx.tcx.prof.query_cache_hit(index.into()); tcx.prof.query_cache_hit(index.into());
return; return;
} }
debug_assert!(!query.anon()); debug_assert!(!query.anon());
ensure_sufficient_stack(|| { ensure_sufficient_stack(|| {
try_execute_query::<C, FLAGS, true>(query, qcx, DUMMY_SP, key, Some(dep_node)) try_execute_query::<C, FLAGS, true>(query, tcx, DUMMY_SP, key, Some(dep_node))
}); });
} }

10
compiler/rustc_query_impl/src/job.rs
View file

@ -10,10 +10,11 @@ use rustc_middle::query::{
CycleError, QueryInfo, QueryJob, QueryJobId, QueryLatch, QueryStackDeferred, QueryStackFrame, CycleError, QueryInfo, QueryJob, QueryJobId, QueryLatch, QueryStackDeferred, QueryStackFrame,
QueryWaiter, QueryWaiter,
}; };
use rustc_middle::ty::TyCtxt;
use rustc_session::Session; use rustc_session::Session;
use rustc_span::{DUMMY_SP, Span}; use rustc_span::{DUMMY_SP, Span};
use crate::QueryCtxt; use crate::plumbing::collect_active_jobs_from_all_queries;
/// Map from query job IDs to job information collected by /// Map from query job IDs to job information collected by
/// `collect_active_jobs_from_all_queries`. /// `collect_active_jobs_from_all_queries`.
@ -384,7 +385,7 @@ pub fn break_query_cycles<'tcx>(
} }
pub fn print_query_stack<'tcx>( pub fn print_query_stack<'tcx>(
qcx: QueryCtxt<'tcx>, tcx: TyCtxt<'tcx>,
mut current_query: Option<QueryJobId>, mut current_query: Option<QueryJobId>,
dcx: DiagCtxtHandle<'_>, dcx: DiagCtxtHandle<'_>,
limit_frames: Option<usize>, limit_frames: Option<usize>,
@ -397,8 +398,7 @@ pub fn print_query_stack<'tcx>(
let mut count_total = 0; let mut count_total = 0;
// Make use of a partial query job map if we fail to take locks collecting active queries. // Make use of a partial query job map if we fail to take locks collecting active queries.
let job_map: QueryJobMap<'_> = qcx let job_map: QueryJobMap<'_> = collect_active_jobs_from_all_queries(tcx, false)
.collect_active_jobs_from_all_queries(false)
.unwrap_or_else(|partial_job_map| partial_job_map); .unwrap_or_else(|partial_job_map| partial_job_map);
if let Some(ref mut file) = file { if let Some(ref mut file) = file {
@ -425,7 +425,7 @@ pub fn print_query_stack<'tcx>(
file, file,
"#{} [{}] {}", "#{} [{}] {}",
count_total, count_total,
qcx.tcx.dep_kind_vtable(query_info.frame.dep_kind).name, tcx.dep_kind_vtable(query_info.frame.dep_kind).name,
query_extra.description query_extra.description
); );
} }

22
compiler/rustc_query_impl/src/lib.rs
View file

@ -26,7 +26,7 @@ use rustc_span::{ErrorGuaranteed, Span};
pub use crate::dep_kind_vtables::make_dep_kind_vtables; pub use crate::dep_kind_vtables::make_dep_kind_vtables;
pub use crate::job::{QueryJobMap, break_query_cycles, print_query_stack}; pub use crate::job::{QueryJobMap, break_query_cycles, print_query_stack};
pub use crate::plumbing::{QueryCtxt, query_key_hash_verify_all}; pub use crate::plumbing::{collect_active_jobs_from_all_queries, query_key_hash_verify_all};
use crate::plumbing::{encode_all_query_results, try_mark_green}; use crate::plumbing::{encode_all_query_results, try_mark_green};
use crate::profiling_support::QueryKeyStringCache; use crate::profiling_support::QueryKeyStringCache;
pub use crate::profiling_support::alloc_self_profile_query_strings; pub use crate::profiling_support::alloc_self_profile_query_strings;
@ -89,11 +89,11 @@ impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> SemiDynamicQueryDispatcher<'t
// Don't use this method to access query results, instead use the methods on TyCtxt. // Don't use this method to access query results, instead use the methods on TyCtxt.
#[inline(always)] #[inline(always)]
fn query_state(self, qcx: QueryCtxt<'tcx>) -> &'tcx QueryState<'tcx, C::Key> { fn query_state(self, tcx: TyCtxt<'tcx>) -> &'tcx QueryState<'tcx, C::Key> {
// Safety: // Safety:
// This is just manually doing the subfield referencing through pointer math. // This is just manually doing the subfield referencing through pointer math.
unsafe { unsafe {
&*(&qcx.tcx.query_system.states as *const QueryStates<'tcx>) &*(&tcx.query_system.states as *const QueryStates<'tcx>)
.byte_add(self.vtable.query_state) .byte_add(self.vtable.query_state)
.cast::<QueryState<'tcx, C::Key>>() .cast::<QueryState<'tcx, C::Key>>()
} }
@ -101,11 +101,11 @@ impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> SemiDynamicQueryDispatcher<'t
// Don't use this method to access query results, instead use the methods on TyCtxt. // Don't use this method to access query results, instead use the methods on TyCtxt.
#[inline(always)] #[inline(always)]
fn query_cache(self, qcx: QueryCtxt<'tcx>) -> &'tcx C { fn query_cache(self, tcx: TyCtxt<'tcx>) -> &'tcx C {
// Safety: // Safety:
// This is just manually doing the subfield referencing through pointer math. // This is just manually doing the subfield referencing through pointer math.
unsafe { unsafe {
&*(&qcx.tcx.query_system.caches as *const QueryCaches<'tcx>) &*(&tcx.query_system.caches as *const QueryCaches<'tcx>)
.byte_add(self.vtable.query_cache) .byte_add(self.vtable.query_cache)
.cast::<C>() .cast::<C>()
} }
@ -121,30 +121,30 @@ impl<'tcx, C: QueryCache, const FLAGS: QueryFlags> SemiDynamicQueryDispatcher<'t
/// Calls the actual provider function for this query. /// Calls the actual provider function for this query.
/// See [`QueryVTable::invoke_provider_fn`] for more details. /// See [`QueryVTable::invoke_provider_fn`] for more details.
#[inline(always)] #[inline(always)]
fn invoke_provider(self, qcx: QueryCtxt<'tcx>, key: C::Key) -> C::Value { fn invoke_provider(self, tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value {
(self.vtable.invoke_provider_fn)(qcx.tcx, key) (self.vtable.invoke_provider_fn)(tcx, key)
} }
#[inline(always)] #[inline(always)]
fn try_load_from_disk( fn try_load_from_disk(
self, self,
qcx: QueryCtxt<'tcx>, tcx: TyCtxt<'tcx>,
key: &C::Key, key: &C::Key,
prev_index: SerializedDepNodeIndex, prev_index: SerializedDepNodeIndex,
index: DepNodeIndex, index: DepNodeIndex,
) -> Option<C::Value> { ) -> Option<C::Value> {
// `?` will return None immediately for queries that never cache to disk. // `?` will return None immediately for queries that never cache to disk.
self.vtable.try_load_from_disk_fn?(qcx.tcx, key, prev_index, index) self.vtable.try_load_from_disk_fn?(tcx, key, prev_index, index)
} }
#[inline] #[inline]
fn is_loadable_from_disk( fn is_loadable_from_disk(
self, self,
qcx: QueryCtxt<'tcx>, tcx: TyCtxt<'tcx>,
key: &C::Key, key: &C::Key,
index: SerializedDepNodeIndex, index: SerializedDepNodeIndex,
) -> bool { ) -> bool {
self.vtable.is_loadable_from_disk_fn.map_or(false, |f| f(qcx.tcx, key, index)) self.vtable.is_loadable_from_disk_fn.map_or(false, |f| f(tcx, key, index))
} }
/// Synthesize an error value to let compilation continue after a cycle. /// Synthesize an error value to let compilation continue after a cycle.

204
compiler/rustc_query_impl/src/plumbing.rs
View file

@ -15,7 +15,7 @@ use rustc_middle::bug;
#[expect(unused_imports, reason = "used by doc comments")] #[expect(unused_imports, reason = "used by doc comments")]
use rustc_middle::dep_graph::DepKindVTable; use rustc_middle::dep_graph::DepKindVTable;
use rustc_middle::dep_graph::{ use rustc_middle::dep_graph::{
self, DepNode, DepNodeIndex, DepNodeKey, HasDepContext, SerializedDepNodeIndex, dep_kinds, self, DepNode, DepNodeIndex, DepNodeKey, SerializedDepNodeIndex, dep_kinds,
}; };
use rustc_middle::query::on_disk_cache::{ use rustc_middle::query::on_disk_cache::{
AbsoluteBytePos, CacheDecoder, CacheEncoder, EncodedDepNodeIndex, AbsoluteBytePos, CacheDecoder, CacheEncoder, EncodedDepNodeIndex,
@ -36,115 +36,91 @@ use crate::execution::{all_inactive, force_query};
use crate::job::{QueryJobMap, find_dep_kind_root}; use crate::job::{QueryJobMap, find_dep_kind_root};
use crate::{QueryDispatcherUnerased, QueryFlags, SemiDynamicQueryDispatcher}; use crate::{QueryDispatcherUnerased, QueryFlags, SemiDynamicQueryDispatcher};
#[derive(Copy, Clone)] fn depth_limit_error<'tcx>(tcx: TyCtxt<'tcx>, job: QueryJobId) {
pub struct QueryCtxt<'tcx> { let job_map =
pub tcx: TyCtxt<'tcx>, collect_active_jobs_from_all_queries(tcx, true).expect("failed to collect active queries");
let (info, depth) = find_dep_kind_root(job, job_map);
let suggested_limit = match tcx.recursion_limit() {
Limit(0) => Limit(2),
limit => limit * 2,
};
tcx.sess.dcx().emit_fatal(QueryOverflow {
span: info.job.span,
note: QueryOverflowNote { desc: info.frame.info.extract().description, depth },
suggested_limit,
crate_name: tcx.crate_name(LOCAL_CRATE),
});
} }
impl<'tcx> QueryCtxt<'tcx> { #[inline]
#[inline] pub(crate) fn next_job_id<'tcx>(tcx: TyCtxt<'tcx>) -> QueryJobId {
pub fn new(tcx: TyCtxt<'tcx>) -> Self { QueryJobId(
QueryCtxt { tcx } NonZero::new(tcx.query_system.jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed))
}
fn depth_limit_error(self, job: QueryJobId) {
let job_map = self
.collect_active_jobs_from_all_queries(true)
.expect("failed to collect active queries");
let (info, depth) = find_dep_kind_root(job, job_map);
let suggested_limit = match self.tcx.recursion_limit() {
Limit(0) => Limit(2),
limit => limit * 2,
};
self.tcx.sess.dcx().emit_fatal(QueryOverflow {
span: info.job.span,
note: QueryOverflowNote { desc: info.frame.info.extract().description, depth },
suggested_limit,
crate_name: self.tcx.crate_name(LOCAL_CRATE),
});
}
#[inline]
pub(crate) fn next_job_id(self) -> QueryJobId {
QueryJobId(
NonZero::new(
self.tcx.query_system.jobs.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
)
.unwrap(), .unwrap(),
) )
} }
#[inline] #[inline]
pub(crate) fn current_query_job(self) -> Option<QueryJobId> { pub(crate) fn current_query_job<'tcx>(tcx: TyCtxt<'tcx>) -> Option<QueryJobId> {
tls::with_related_context(self.tcx, |icx| icx.query) tls::with_related_context(tcx, |icx| icx.query)
} }
/// Executes a job by changing the `ImplicitCtxt` to point to the /// Executes a job by changing the `ImplicitCtxt` to point to the
/// new query job while it executes. /// new query job while it executes.
#[inline(always)] #[inline(always)]
pub(crate) fn start_query<R>( pub(crate) fn start_query<'tcx, R>(
self, tcx: TyCtxt<'tcx>,
token: QueryJobId, token: QueryJobId,
depth_limit: bool, depth_limit: bool,
compute: impl FnOnce() -> R, compute: impl FnOnce() -> R,
) -> R { ) -> R {
// The `TyCtxt` stored in TLS has the same global interner lifetime // The `TyCtxt` stored in TLS has the same global interner lifetime
// as `self`, so we use `with_related_context` to relate the 'tcx lifetimes // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes
// when accessing the `ImplicitCtxt`. // when accessing the `ImplicitCtxt`.
tls::with_related_context(self.tcx, move |current_icx| { tls::with_related_context(tcx, move |current_icx| {
if depth_limit if depth_limit && !tcx.recursion_limit().value_within_limit(current_icx.query_depth) {
&& !self.tcx.recursion_limit().value_within_limit(current_icx.query_depth) depth_limit_error(tcx, token);
{
self.depth_limit_error(token);
}
// Update the `ImplicitCtxt` to point to our new query job.
let new_icx = ImplicitCtxt {
tcx: self.tcx,
query: Some(token),
query_depth: current_icx.query_depth + depth_limit as usize,
task_deps: current_icx.task_deps,
};
// Use the `ImplicitCtxt` while we execute the query.
tls::enter_context(&new_icx, compute)
})
}
/// Returns a map of currently active query jobs, collected from all queries.
///
/// If `require_complete` is `true`, this function locks all shards of the
/// query results to produce a complete map, which always returns `Ok`.
/// Otherwise, it may return an incomplete map as an error if any shard
/// lock cannot be acquired.
///
/// Prefer passing `false` to `require_complete` to avoid potential deadlocks,
/// especially when called from within a deadlock handler, unless a
/// complete map is needed and no deadlock is possible at this call site.
pub fn collect_active_jobs_from_all_queries(
self,
require_complete: bool,
) -> Result<QueryJobMap<'tcx>, QueryJobMap<'tcx>> {
let mut job_map_out = QueryJobMap::default();
let mut complete = true;
for gather_fn in crate::PER_QUERY_GATHER_ACTIVE_JOBS_FNS.iter() {
if gather_fn(self.tcx, require_complete, &mut job_map_out).is_none() {
complete = false;
}
} }
if complete { Ok(job_map_out) } else { Err(job_map_out) } // Update the `ImplicitCtxt` to point to our new query job.
} let new_icx = ImplicitCtxt {
tcx,
query: Some(token),
query_depth: current_icx.query_depth + depth_limit as usize,
task_deps: current_icx.task_deps,
};
// Use the `ImplicitCtxt` while we execute the query.
tls::enter_context(&new_icx, compute)
})
} }
impl<'tcx> HasDepContext<'tcx> for QueryCtxt<'tcx> { /// Returns a map of currently active query jobs, collected from all queries.
#[inline] ///
fn dep_context(&self) -> TyCtxt<'tcx> { /// If `require_complete` is `true`, this function locks all shards of the
self.tcx /// query results to produce a complete map, which always returns `Ok`.
/// Otherwise, it may return an incomplete map as an error if any shard
/// lock cannot be acquired.
///
/// Prefer passing `false` to `require_complete` to avoid potential deadlocks,
/// especially when called from within a deadlock handler, unless a
/// complete map is needed and no deadlock is possible at this call site.
pub fn collect_active_jobs_from_all_queries<'tcx>(
tcx: TyCtxt<'tcx>,
require_complete: bool,
) -> Result<QueryJobMap<'tcx>, QueryJobMap<'tcx>> {
let mut job_map_out = QueryJobMap::default();
let mut complete = true;
for gather_fn in crate::PER_QUERY_GATHER_ACTIVE_JOBS_FNS.iter() {
if gather_fn(tcx, require_complete, &mut job_map_out).is_none() {
complete = false;
}
} }
if complete { Ok(job_map_out) } else { Err(job_map_out) }
} }
pub(super) fn try_mark_green<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool { pub(super) fn try_mark_green<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool {
@ -361,19 +337,19 @@ where
pub(crate) fn encode_query_results<'a, 'tcx, Q, C: QueryCache, const FLAGS: QueryFlags>( pub(crate) fn encode_query_results<'a, 'tcx, Q, C: QueryCache, const FLAGS: QueryFlags>(
query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
qcx: QueryCtxt<'tcx>, tcx: TyCtxt<'tcx>,
encoder: &mut CacheEncoder<'a, 'tcx>, encoder: &mut CacheEncoder<'a, 'tcx>,
query_result_index: &mut EncodedDepNodeIndex, query_result_index: &mut EncodedDepNodeIndex,
) where ) where
Q: QueryDispatcherUnerased<'tcx, C, FLAGS>, Q: QueryDispatcherUnerased<'tcx, C, FLAGS>,
Q::UnerasedValue: Encodable<CacheEncoder<'a, 'tcx>>, Q::UnerasedValue: Encodable<CacheEncoder<'a, 'tcx>>,
{ {
let _timer = qcx.tcx.prof.generic_activity_with_arg("encode_query_results_for", query.name()); let _timer = tcx.prof.generic_activity_with_arg("encode_query_results_for", query.name());
assert!(all_inactive(query.query_state(qcx))); assert!(all_inactive(query.query_state(tcx)));
let cache = query.query_cache(qcx); let cache = query.query_cache(tcx);
cache.iter(&mut |key, value, dep_node| { cache.iter(&mut |key, value, dep_node| {
if query.will_cache_on_disk_for_key(qcx.tcx, key) { if query.will_cache_on_disk_for_key(tcx, key) {
let dep_node = SerializedDepNodeIndex::new(dep_node.index()); let dep_node = SerializedDepNodeIndex::new(dep_node.index());
// Record position of the cache entry. // Record position of the cache entry.
@ -388,14 +364,14 @@ pub(crate) fn encode_query_results<'a, 'tcx, Q, C: QueryCache, const FLAGS: Quer
pub(crate) fn query_key_hash_verify<'tcx, C: QueryCache, const FLAGS: QueryFlags>( pub(crate) fn query_key_hash_verify<'tcx, C: QueryCache, const FLAGS: QueryFlags>(
query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>, query: SemiDynamicQueryDispatcher<'tcx, C, FLAGS>,
qcx: QueryCtxt<'tcx>, tcx: TyCtxt<'tcx>,
) { ) {
let _timer = qcx.tcx.prof.generic_activity_with_arg("query_key_hash_verify_for", query.name()); let _timer = tcx.prof.generic_activity_with_arg("query_key_hash_verify_for", query.name());
let cache = query.query_cache(qcx); let cache = query.query_cache(tcx);
let mut map = UnordMap::with_capacity(cache.len()); let mut map = UnordMap::with_capacity(cache.len());
cache.iter(&mut |key, _, _| { cache.iter(&mut |key, _, _| {
let node = DepNode::construct(qcx.tcx, query.dep_kind(), key); let node = DepNode::construct(tcx, query.dep_kind(), key);
if let Some(other_key) = map.insert(node, *key) { if let Some(other_key) = map.insert(node, *key) {
bug!( bug!(
"query key:\n\ "query key:\n\
@ -487,7 +463,7 @@ pub(crate) fn force_from_dep_node_inner<'tcx, C: QueryCache, const FLAGS: QueryF
); );
if let Some(key) = C::Key::recover(tcx, &dep_node) { if let Some(key) = C::Key::recover(tcx, &dep_node) {
force_query(query, QueryCtxt::new(tcx), key, dep_node); force_query(query, tcx, key, dep_node);
true true
} else { } else {
false false
@ -525,7 +501,7 @@ macro_rules! define_queries {
let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered(); let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered();
execution::get_query_incr( execution::get_query_incr(
QueryType::query_dispatcher(tcx), QueryType::query_dispatcher(tcx),
QueryCtxt::new(tcx), tcx,
span, span,
key, key,
mode mode
@ -545,7 +521,7 @@ macro_rules! define_queries {
) -> Option<Erased<queries::$name::Value<'tcx>>> { ) -> Option<Erased<queries::$name::Value<'tcx>>> {
Some(execution::get_query_non_incr( Some(execution::get_query_non_incr(
QueryType::query_dispatcher(tcx), QueryType::query_dispatcher(tcx),
QueryCtxt::new(tcx), tcx,
span, span,
key, key,
)) ))
@ -729,7 +705,7 @@ macro_rules! define_queries {
_ _
> ( > (
query_impl::$name::QueryType::query_dispatcher(tcx), query_impl::$name::QueryType::query_dispatcher(tcx),
QueryCtxt::new(tcx), tcx,
encoder, encoder,
query_result_index, query_result_index,
) )
@ -739,7 +715,7 @@ macro_rules! define_queries {
pub(crate) fn query_key_hash_verify<'tcx>(tcx: TyCtxt<'tcx>) { pub(crate) fn query_key_hash_verify<'tcx>(tcx: TyCtxt<'tcx>) {
$crate::plumbing::query_key_hash_verify( $crate::plumbing::query_key_hash_verify(
query_impl::$name::QueryType::query_dispatcher(tcx), query_impl::$name::QueryType::query_dispatcher(tcx),
QueryCtxt::new(tcx), tcx,
) )
} }
})*} })*}