user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rollup merge of #150551 - lqd:lazy-polonius, r=jackh726

Browse source 
Compute localized outlives constraints lazily

This PR rewrites the loan liveness algorithm to compute localized constraints lazily during traversal, to avoid the sometimes costly conversion and indexing happening in the current eager algorithm (or for now as well, the need to reshape the liveness data to better list regions live at a given point).

It thus greatly reduces the current alpha overhead, although it wasn't entirely removed of course (we're obviously doing more work to improve precision): the worst offending benchmark has a +5-6% wall-time regression — but icounts are worse looking (+13%) rn.

Best reviewed per-commit, as steps are in sequence to simplify the process or unify some things.

r? @jackh726
This commit is contained in:
Stuart Cook 2026-02-13 15:19:07 +11:00 • committed by GitHub
commit 5df4f59fe3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 477 additions and 712 deletions
Download patch file Download diff file Expand all files Collapse all files

4
compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs
View file

@ -649,8 +649,8 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> {
// We want to focus on relevant live locals in diagnostics, so when polonius is enabled, we
// ensure that we don't emit live boring locals as explanations.
let is_local_boring = |local| {
if let Some(polonius_diagnostics) = self.polonius_diagnostics {
polonius_diagnostics.boring_nll_locals.contains(&local)
if let Some(polonius_context) = self.polonius_context {
polonius_context.boring_nll_locals.contains(&local)
} else {
assert!(!tcx.sess.opts.unstable_opts.polonius.is_next_enabled());

12
compiler/rustc_borrowck/src/lib.rs
View file

@ -63,10 +63,10 @@ use crate::diagnostics::{
use crate::path_utils::*;
use crate::place_ext::PlaceExt;
use crate::places_conflict::{PlaceConflictBias, places_conflict};
use crate::polonius::PoloniusContext;
use crate::polonius::legacy::{
PoloniusFacts, PoloniusFactsExt, PoloniusLocationTable, PoloniusOutput,
};
use crate::polonius::{PoloniusContext, PoloniusDiagnosticsContext};
use crate::prefixes::PrefixSet;
use crate::region_infer::RegionInferenceContext;
use crate::region_infer::opaque_types::DeferredOpaqueTypeError;
@ -424,7 +424,7 @@ fn borrowck_check_region_constraints<'tcx>(
polonius_output,
opt_closure_req,
nll_errors,
polonius_diagnostics,
polonius_context,
} = nll::compute_regions(
root_cx,
&infcx,
@ -448,7 +448,7 @@ fn borrowck_check_region_constraints<'tcx>(
&regioncx,
&opt_closure_req,
&borrow_set,
polonius_diagnostics.as_ref(),
polonius_context.as_ref(),
);
// We also have a `#[rustc_regions]` annotation that causes us to dump
@ -490,7 +490,7 @@ fn borrowck_check_region_constraints<'tcx>(
polonius_output: None,
move_errors: Vec::new(),
diags_buffer,
polonius_diagnostics: polonius_diagnostics.as_ref(),
polonius_context: polonius_context.as_ref(),
};
struct MoveVisitor<'a, 'b, 'infcx, 'tcx> {
ctxt: &'a mut MirBorrowckCtxt<'b, 'infcx, 'tcx>,
@ -529,7 +529,7 @@ fn borrowck_check_region_constraints<'tcx>(
move_errors: Vec::new(),
diags_buffer,
polonius_output: polonius_output.as_deref(),
polonius_diagnostics: polonius_diagnostics.as_ref(),
polonius_context: polonius_context.as_ref(),
};
// Compute and report region errors, if any.
@ -779,7 +779,7 @@ struct MirBorrowckCtxt<'a, 'infcx, 'tcx> {
/// Results of Polonius analysis.
polonius_output: Option<&'a PoloniusOutput>,
/// When using `-Zpolonius=next`: the data used to compute errors and diagnostics.
polonius_diagnostics: Option<&'a PoloniusDiagnosticsContext>,
polonius_context: Option<&'a PoloniusContext>,
}
// Check that:

14
compiler/rustc_borrowck/src/nll.rs
View file

@ -23,10 +23,10 @@ use crate::borrow_set::BorrowSet;
use crate::consumers::RustcFacts;
use crate::diagnostics::RegionErrors;
use crate::handle_placeholders::compute_sccs_applying_placeholder_outlives_constraints;
use crate::polonius::PoloniusContext;
use crate::polonius::legacy::{
PoloniusFacts, PoloniusFactsExt, PoloniusLocationTable, PoloniusOutput,
};
use crate::polonius::{PoloniusContext, PoloniusDiagnosticsContext};
use crate::region_infer::RegionInferenceContext;
use crate::type_check::MirTypeckRegionConstraints;
use crate::type_check::free_region_relations::UniversalRegionRelations;
@ -47,7 +47,7 @@ pub(crate) struct NllOutput<'tcx> {
/// When using `-Zpolonius=next`: the data used to compute errors and diagnostics, e.g.
/// localized typeck and liveness constraints.
pub polonius_diagnostics: Option<PoloniusDiagnosticsContext>,
pub polonius_context: Option<PoloniusContext>,
}
/// Rewrites the regions in the MIR to use NLL variables, also scraping out the set of universal
@ -122,7 +122,7 @@ pub(crate) fn compute_regions<'tcx>(
universal_region_relations: Frozen<UniversalRegionRelations<'tcx>>,
constraints: MirTypeckRegionConstraints<'tcx>,
mut polonius_facts: Option<AllFacts<RustcFacts>>,
polonius_context: Option<PoloniusContext>,
mut polonius_context: Option<PoloniusContext>,
) -> NllOutput<'tcx> {
let polonius_output = root_cx.consumer.as_ref().map_or(false, |c| c.polonius_output())
|| infcx.tcx.sess.opts.unstable_opts.polonius.is_legacy_enabled();
@ -154,9 +154,9 @@ pub(crate) fn compute_regions<'tcx>(
// If requested for `-Zpolonius=next`, convert NLL constraints to localized outlives constraints
// and use them to compute loan liveness.
let polonius_diagnostics = polonius_context.map(|polonius_context| {
polonius_context.compute_loan_liveness(infcx.tcx, &mut regioncx, body, borrow_set)
});
if let Some(polonius_context) = polonius_context.as_mut() {
polonius_context.compute_loan_liveness(&mut regioncx, body, borrow_set)
}
// If requested: dump NLL facts, and run legacy polonius analysis.
let polonius_output = polonius_facts.as_ref().and_then(|polonius_facts| {
@ -189,7 +189,7 @@ pub(crate) fn compute_regions<'tcx>(
polonius_output,
opt_closure_req: closure_region_requirements,
nll_errors,
polonius_diagnostics,
polonius_context,
}
}

321
compiler/rustc_borrowck/src/polonius/constraints.rs
View file

@ -1,6 +1,19 @@
use std::collections::BTreeMap;
use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
use rustc_index::interval::SparseIntervalMatrix;
use rustc_middle::mir::{Body, Location};
use rustc_middle::ty::RegionVid;
use rustc_mir_dataflow::points::PointIndex;
use crate::BorrowSet;
use crate::constraints::OutlivesConstraint;
use crate::dataflow::BorrowIndex;
use crate::polonius::ConstraintDirection;
use crate::region_infer::values::LivenessValues;
use crate::type_check::Locations;
use crate::universal_regions::UniversalRegions;
/// A localized outlives constraint reifies the CFG location where the outlives constraint holds,
/// within the origins themselves as if they were different from point to point: from `a: b`
/// outlives constraints to `a@p: b@p`, where `p` is the point in the CFG.
@ -12,32 +25,300 @@ use rustc_mir_dataflow::points::PointIndex;
/// of `q`. These depend on the liveness of the regions at these points, as well as their
/// variance.
///
/// The `source` origin at `from` flows into the `target` origin at `to`.
///
/// This dual of NLL's [crate::constraints::OutlivesConstraint] therefore encodes the
/// position-dependent outlives constraints used by Polonius, to model the flow-sensitive loan
/// propagation via reachability within a graph of localized constraints.
#[derive(Copy, Clone, PartialEq, Eq, Debug, Hash)]
pub(crate) struct LocalizedOutlivesConstraint {
pub source: RegionVid,
pub from: PointIndex,
pub target: RegionVid,
pub to: PointIndex,
///
/// That `LocalizedConstraintGraph` can create these edges on-demand during traversal, and we
/// therefore model them as a pair of `LocalizedNode` vertices.
///
#[derive(Copy, Clone, PartialEq, Eq, Hash)]
pub(super) struct LocalizedNode {
pub region: RegionVid,
pub point: PointIndex,
}
/// A container of [LocalizedOutlivesConstraint]s that can be turned into a traversable
/// `rustc_data_structures` graph.
#[derive(Clone, Default, Debug)]
pub(crate) struct LocalizedOutlivesConstraintSet {
pub outlives: Vec<LocalizedOutlivesConstraint>,
/// The localized constraint graph indexes the physical and logical edges to lazily compute a given
/// node's successors during traversal.
pub(super) struct LocalizedConstraintGraph {
/// The actual, physical, edges we have recorded for a given node. We localize them on-demand
/// when traversing from the node to the successor region.
edges: FxHashMap<LocalizedNode, FxIndexSet<RegionVid>>,
/// The logical edges representing the outlives constraints that hold at all points in the CFG,
/// which we don't localize to avoid creating a lot of unnecessary edges in the graph. Some CFGs
/// can be big, and we don't need to create such a physical edge for every point in the CFG.
logical_edges: FxHashMap<RegionVid, FxIndexSet<RegionVid>>,
}
impl LocalizedOutlivesConstraintSet {
pub(crate) fn push(&mut self, constraint: LocalizedOutlivesConstraint) {
if constraint.source == constraint.target && constraint.from == constraint.to {
// 'a@p: 'a@p is pretty uninteresting
return;
}
self.outlives.push(constraint);
/// The visitor interface when traversing a `LocalizedConstraintGraph`.
pub(super) trait LocalizedConstraintGraphVisitor {
/// Callback called when traversing a given `loan` encounters a localized `node` it hasn't
/// visited before.
fn on_node_traversed(&mut self, _loan: BorrowIndex, _node: LocalizedNode) {}
/// Callback called when discovering a new `successor` node for the `current_node`.
fn on_successor_discovered(&mut self, _current_node: LocalizedNode, _successor: LocalizedNode) {
}
}
impl LocalizedConstraintGraph {
/// Traverses the constraints and returns the indexed graph of edges per node.
pub(super) fn new<'tcx>(
liveness: &LivenessValues,
outlives_constraints: impl Iterator<Item = OutlivesConstraint<'tcx>>,
) -> Self {
let mut edges: FxHashMap<_, FxIndexSet<_>> = FxHashMap::default();
let mut logical_edges: FxHashMap<_, FxIndexSet<_>> = FxHashMap::default();
for outlives_constraint in outlives_constraints {
match outlives_constraint.locations {
Locations::All(_) => {
logical_edges
.entry(outlives_constraint.sup)
.or_default()
.insert(outlives_constraint.sub);
}
Locations::Single(location) => {
let node = LocalizedNode {
region: outlives_constraint.sup,
point: liveness.point_from_location(location),
};
edges.entry(node).or_default().insert(outlives_constraint.sub);
}
}
}
LocalizedConstraintGraph { edges, logical_edges }
}
/// Traverses the localized constraint graph per-loan, and notifies the `visitor` of discovered
/// nodes and successors.
pub(super) fn traverse<'tcx>(
&self,
body: &Body<'tcx>,
liveness: &LivenessValues,
live_region_variances: &BTreeMap<RegionVid, ConstraintDirection>,
universal_regions: &UniversalRegions<'tcx>,
borrow_set: &BorrowSet<'tcx>,
visitor: &mut impl LocalizedConstraintGraphVisitor,
) {
let live_regions = liveness.points();
let mut visited = FxHashSet::default();
let mut stack = Vec::new();
// Compute reachability per loan by traversing each loan's subgraph starting from where it
// is introduced.
for (loan_idx, loan) in borrow_set.iter_enumerated() {
visited.clear();
stack.clear();
let start_node = LocalizedNode {
region: loan.region,
point: liveness.point_from_location(loan.reserve_location),
};
stack.push(start_node);
while let Some(node) = stack.pop() {
if !visited.insert(node) {
continue;
}
// We've reached a node we haven't visited before.
let location = liveness.location_from_point(node.point);
visitor.on_node_traversed(loan_idx, node);
// When we find a _new_ successor, we'd like to
// - visit it eventually,
// - and let the generic visitor know about it.
let mut successor_found = |succ| {
if !visited.contains(&succ) {
stack.push(succ);
visitor.on_successor_discovered(node, succ);
}
};
// Then, we propagate the loan along the localized constraint graph. The outgoing
// edges are computed lazily, from:
// - the various physical edges present at this node,
// - the materialized logical edges that exist virtually at all points for this
// node's region, localized at this point.
// Universal regions propagate loans along the CFG, i.e. forwards only.
let is_universal_region = universal_regions.is_universal_region(node.region);
// The physical edges present at this node are:
//
// 1. the typeck edges that flow from region to region *at this point*.
for &succ in self.edges.get(&node).into_iter().flatten() {
let succ = LocalizedNode { region: succ, point: node.point };
successor_found(succ);
}
// 2a. the liveness edges that flow *forward*, from this node's point to its
// successors in the CFG.
if body[location.block].statements.get(location.statement_index).is_some() {
// Intra-block edges, straight line constraints from each point to its successor
// within the same block.
let next_point = node.point + 1;
if let Some(succ) = compute_forward_successor(
node.region,
next_point,
live_regions,
live_region_variances,
is_universal_region,
) {
successor_found(succ);
}
} else {
// Inter-block edges, from the block's terminator to each successor block's
// entry point.
for successor_block in body[location.block].terminator().successors() {
let next_location = Location { block: successor_block, statement_index: 0 };
let next_point = liveness.point_from_location(next_location);
if let Some(succ) = compute_forward_successor(
node.region,
next_point,
live_regions,
live_region_variances,
is_universal_region,
) {
successor_found(succ);
}
}
}
// 2b. the liveness edges that flow *backward*, from this node's point to its
// predecessors in the CFG.
if !is_universal_region {
if location.statement_index > 0 {
// Backward edges to the predecessor point in the same block.
let previous_point = PointIndex::from(node.point.as_usize() - 1);
if let Some(succ) = compute_backward_successor(
node.region,
node.point,
previous_point,
live_regions,
live_region_variances,
) {
successor_found(succ);
}
} else {
// Backward edges from the block entry point to the terminator of the
// predecessor blocks.
let predecessors = body.basic_blocks.predecessors();
for &pred_block in &predecessors[location.block] {
let previous_location = Location {
block: pred_block,
statement_index: body[pred_block].statements.len(),
};
let previous_point = liveness.point_from_location(previous_location);
if let Some(succ) = compute_backward_successor(
node.region,
node.point,
previous_point,
live_regions,
live_region_variances,
) {
successor_found(succ);
}
}
}
}
// And finally, we have the logical edges, materialized at this point.
for &logical_succ in self.logical_edges.get(&node.region).into_iter().flatten() {
let succ = LocalizedNode { region: logical_succ, point: node.point };
successor_found(succ);
}
}
}
}
}
/// Returns the successor for the current region/point node when propagating a loan through forward
/// edges, if applicable, according to liveness and variance.
fn compute_forward_successor(
region: RegionVid,
next_point: PointIndex,
live_regions: &SparseIntervalMatrix<RegionVid, PointIndex>,
live_region_variances: &BTreeMap<RegionVid, ConstraintDirection>,
is_universal_region: bool,
) -> Option<LocalizedNode> {
// 1. Universal regions are semantically live at all points.
if is_universal_region {
let succ = LocalizedNode { region, point: next_point };
return Some(succ);
}
// 2. Otherwise, gather the edges due to explicit region liveness, when applicable.
if !live_regions.contains(region, next_point) {
return None;
}
// Here, `region` could be live at the current point, and is live at the next point: add a
// constraint between them, according to variance.
// Note: there currently are cases related to promoted and const generics, where we don't yet
// have variance information (possibly about temporary regions created when typeck sanitizes the
// promoteds). Until that is done, we conservatively fallback to maximizing reachability by
// adding a bidirectional edge here. This will not limit traversal whatsoever, and thus
// propagate liveness when needed.
//
// FIXME: add the missing variance information and remove this fallback bidirectional edge.
let direction =
live_region_variances.get(&region).unwrap_or(&ConstraintDirection::Bidirectional);
match direction {
ConstraintDirection::Backward => {
// Contravariant cases: loans flow in the inverse direction, but we're only interested
// in forward successors and there are none here.
None
}
ConstraintDirection::Forward | ConstraintDirection::Bidirectional => {
// 1. For covariant cases: loans flow in the regular direction, from the current point
// to the next point.
// 2. For invariant cases, loans can flow in both directions, but here as well, we only
// want the forward path of the bidirectional edge.
Some(LocalizedNode { region, point: next_point })
}
}
}
/// Returns the successor for the current region/point node when propagating a loan through backward
/// edges, if applicable, according to liveness and variance.
fn compute_backward_successor(
region: RegionVid,
current_point: PointIndex,
previous_point: PointIndex,
live_regions: &SparseIntervalMatrix<RegionVid, PointIndex>,
live_region_variances: &BTreeMap<RegionVid, ConstraintDirection>,
) -> Option<LocalizedNode> {
// Liveness flows into the regions live at the next point. So, in a backwards view, we'll link
// the region from the current point, if it's live there, to the previous point.
if !live_regions.contains(region, current_point) {
return None;
}
// FIXME: add the missing variance information and remove this fallback bidirectional edge. See
// the same comment in `compute_forward_successor`.
let direction =
live_region_variances.get(&region).unwrap_or(&ConstraintDirection::Bidirectional);
match direction {
ConstraintDirection::Forward => {
// Covariant cases: loans flow in the regular direction, but we're only interested in
// backward successors and there are none here.
None
}
ConstraintDirection::Backward | ConstraintDirection::Bidirectional => {
// 1. For contravariant cases: loans flow in the inverse direction, from the current
// point to the previous point.
// 2. For invariant cases, loans can flow in both directions, but here as well, we only
// want the backward path of the bidirectional edge.
Some(LocalizedNode { region, point: previous_point })
}
}
}

75
compiler/rustc_borrowck/src/polonius/dump.rs
View file

@ -10,9 +10,7 @@ use rustc_session::config::MirIncludeSpans;
use crate::borrow_set::BorrowSet;
use crate::constraints::OutlivesConstraint;
use crate::polonius::{
LocalizedOutlivesConstraint, LocalizedOutlivesConstraintSet, PoloniusDiagnosticsContext,
};
use crate::polonius::{LocalizedConstraintGraphVisitor, LocalizedNode, PoloniusContext};
use crate::region_infer::values::LivenessValues;
use crate::type_check::Locations;
use crate::{BorrowckInferCtxt, ClosureRegionRequirements, RegionInferenceContext};
@ -24,7 +22,7 @@ pub(crate) fn dump_polonius_mir<'tcx>(
regioncx: &RegionInferenceContext<'tcx>,
closure_region_requirements: &Option<ClosureRegionRequirements<'tcx>>,
borrow_set: &BorrowSet<'tcx>,
polonius_diagnostics: Option<&PoloniusDiagnosticsContext>,
polonius_context: Option<&PoloniusContext>,
) {
let tcx = infcx.tcx;
if !tcx.sess.opts.unstable_opts.polonius.is_next_enabled() {
@ -33,8 +31,22 @@ pub(crate) fn dump_polonius_mir<'tcx>(
let Some(dumper) = MirDumper::new(tcx, "polonius", body) else { return };
let polonius_diagnostics =
polonius_diagnostics.expect("missing diagnostics context with `-Zpolonius=next`");
let polonius_context =
polonius_context.expect("missing polonius context with `-Zpolonius=next`");
// If we have a polonius graph to dump along the rest of the MIR and NLL info, we extract its
// constraints here.
let mut collector = LocalizedOutlivesConstraintCollector { constraints: Vec::new() };
if let Some(graph) = &polonius_context.graph {
graph.traverse(
body,
regioncx.liveness_constraints(),
&polonius_context.live_region_variances,
regioncx.universal_regions(),
borrow_set,
&mut collector,
);
}
let extra_data = &|pass_where, out: &mut dyn io::Write| {
emit_polonius_mir(
@ -42,7 +54,7 @@ pub(crate) fn dump_polonius_mir<'tcx>(
regioncx,
closure_region_requirements,
borrow_set,
&polonius_diagnostics.localized_outlives_constraints,
&collector.constraints,
pass_where,
out,
)
@ -60,17 +72,34 @@ pub(crate) fn dump_polonius_mir<'tcx>(
let _ = try {
let mut file = dumper.create_dump_file("html", body)?;
emit_polonius_dump(
&dumper,
body,
regioncx,
borrow_set,
&polonius_diagnostics.localized_outlives_constraints,
&mut file,
)?;
emit_polonius_dump(&dumper, body, regioncx, borrow_set, &collector.constraints, &mut file)?;
};
}
/// The constraints we'll dump as text or a mermaid graph.
struct LocalizedOutlivesConstraint {
source: RegionVid,
from: PointIndex,
target: RegionVid,
to: PointIndex,
}
/// Visitor to record constraints encountered when traversing the localized constraint graph.
struct LocalizedOutlivesConstraintCollector {
constraints: Vec<LocalizedOutlivesConstraint>,
}
impl LocalizedConstraintGraphVisitor for LocalizedOutlivesConstraintCollector {
fn on_successor_discovered(&mut self, current_node: LocalizedNode, successor: LocalizedNode) {
self.constraints.push(LocalizedOutlivesConstraint {
source: current_node.region,
from: current_node.point,
target: successor.region,
to: successor.point,
});
}
}
/// The polonius dump consists of:
/// - the NLL MIR
/// - the list of polonius localized constraints
@ -82,7 +111,7 @@ fn emit_polonius_dump<'tcx>(
body: &Body<'tcx>,
regioncx: &RegionInferenceContext<'tcx>,
borrow_set: &BorrowSet<'tcx>,
localized_outlives_constraints: &LocalizedOutlivesConstraintSet,
localized_outlives_constraints: &[LocalizedOutlivesConstraint],
out: &mut dyn io::Write,
) -> io::Result<()> {
// Prepare the HTML dump file prologue.
@ -193,7 +222,7 @@ fn emit_polonius_mir<'tcx>(
regioncx: &RegionInferenceContext<'tcx>,
closure_region_requirements: &Option<ClosureRegionRequirements<'tcx>>,
borrow_set: &BorrowSet<'tcx>,
localized_outlives_constraints: &LocalizedOutlivesConstraintSet,
localized_outlives_constraints: &[LocalizedOutlivesConstraint],
pass_where: PassWhere,
out: &mut dyn io::Write,
) -> io::Result<()> {
@ -212,10 +241,10 @@ fn emit_polonius_mir<'tcx>(
// Add localized outlives constraints
match pass_where {
PassWhere::BeforeCFG => {
if localized_outlives_constraints.outlives.len() > 0 {
if localized_outlives_constraints.len() > 0 {
writeln!(out, "| Localized constraints")?;
for constraint in &localized_outlives_constraints.outlives {
for constraint in localized_outlives_constraints {
let LocalizedOutlivesConstraint { source, from, target, to } = constraint;
let from = liveness.location_from_point(*from);
let to = liveness.location_from_point(*to);
@ -399,7 +428,7 @@ fn emit_mermaid_nll_sccs<'tcx>(
fn emit_mermaid_constraint_graph<'tcx>(
borrow_set: &BorrowSet<'tcx>,
liveness: &LivenessValues,
localized_outlives_constraints: &LocalizedOutlivesConstraintSet,
localized_outlives_constraints: &[LocalizedOutlivesConstraint],
out: &mut dyn io::Write,
) -> io::Result<usize> {
let location_name = |location: Location| {
@ -438,7 +467,7 @@ fn emit_mermaid_constraint_graph<'tcx>(
// The regions subgraphs containing the region/point nodes.
let mut points_per_region: FxIndexMap<RegionVid, FxIndexSet<PointIndex>> =
FxIndexMap::default();
for constraint in &localized_outlives_constraints.outlives {
for constraint in localized_outlives_constraints {
points_per_region.entry(constraint.source).or_default().insert(constraint.from);
points_per_region.entry(constraint.target).or_default().insert(constraint.to);
}
@ -451,7 +480,7 @@ fn emit_mermaid_constraint_graph<'tcx>(
}
// The constraint graph edges.
for constraint in &localized_outlives_constraints.outlives {
for constraint in localized_outlives_constraints {
// FIXME: add killed loans and constraint kind as edge labels.
writeln!(
out,
@ -463,6 +492,6 @@ fn emit_mermaid_constraint_graph<'tcx>(
// Return the number of edges: this is the biggest graph in the dump and its edge count will be
// mermaid's max edge count to support.
let edge_count = borrow_set.len() + localized_outlives_constraints.outlives.len();
let edge_count = borrow_set.len() + localized_outlives_constraints.len();
Ok(edge_count)
}

170
compiler/rustc_borrowck/src/polonius/liveness_constraints.rs
View file

@ -1,22 +1,15 @@
use std::collections::BTreeMap;
use rustc_hir::def_id::DefId;
use rustc_index::bit_set::SparseBitMatrix;
use rustc_middle::mir::{Body, Location};
use rustc_middle::ty::relate::{
self, Relate, RelateResult, TypeRelation, relate_args_with_variances,
};
use rustc_middle::ty::{self, RegionVid, Ty, TyCtxt, TypeVisitable};
use rustc_mir_dataflow::points::PointIndex;
use super::{
ConstraintDirection, LocalizedOutlivesConstraint, LocalizedOutlivesConstraintSet,
PoloniusLivenessContext,
};
use crate::region_infer::values::LivenessValues;
use super::{ConstraintDirection, PoloniusContext};
use crate::universal_regions::UniversalRegions;
impl PoloniusLivenessContext {
impl PoloniusContext {
/// Record the variance of each region contained within the given value.
pub(crate) fn record_live_region_variance<'tcx>(
&mut self,
@ -34,165 +27,6 @@ impl PoloniusLivenessContext {
}
}
/// Propagate loans throughout the CFG: for each statement in the MIR, create localized outlives
/// constraints for loans that are propagated to the next statements.
pub(super) fn create_liveness_constraints<'tcx>(
body: &Body<'tcx>,
liveness: &LivenessValues,
live_regions: &SparseBitMatrix<PointIndex, RegionVid>,
live_region_variances: &BTreeMap<RegionVid, ConstraintDirection>,
universal_regions: &UniversalRegions<'tcx>,
localized_outlives_constraints: &mut LocalizedOutlivesConstraintSet,
) {
for (block, bb) in body.basic_blocks.iter_enumerated() {
let statement_count = bb.statements.len();
for statement_index in 0..=statement_count {
let current_location = Location { block, statement_index };
let current_point = liveness.point_from_location(current_location);
if statement_index < statement_count {
// Intra-block edges, straight line constraints from each point to its successor
// within the same block.
let next_location = Location { block, statement_index: statement_index + 1 };
let next_point = liveness.point_from_location(next_location);
propagate_loans_between_points(
current_point,
next_point,
live_regions,
live_region_variances,
universal_regions,
localized_outlives_constraints,
);
} else {
// Inter-block edges, from the block's terminator to each successor block's entry
// point.
for successor_block in bb.terminator().successors() {
let next_location = Location { block: successor_block, statement_index: 0 };
let next_point = liveness.point_from_location(next_location);
propagate_loans_between_points(
current_point,
next_point,
live_regions,
live_region_variances,
universal_regions,
localized_outlives_constraints,
);
}
}
}
}
}
/// Propagate loans within a region between two points in the CFG, if that region is live at both
/// the source and target points.
fn propagate_loans_between_points(
current_point: PointIndex,
next_point: PointIndex,
live_regions: &SparseBitMatrix<PointIndex, RegionVid>,
live_region_variances: &BTreeMap<RegionVid, ConstraintDirection>,
universal_regions: &UniversalRegions<'_>,
localized_outlives_constraints: &mut LocalizedOutlivesConstraintSet,
) {
// Universal regions are semantically live at all points.
// Note: we always have universal regions but they're not always (or often) involved in the
// subset graph. For now, we emit all their edges unconditionally, but some of these subgraphs
// will be disconnected from the rest of the graph and thus, unnecessary.
//
// FIXME: only emit the edges of universal regions that existential regions can reach.
for region in universal_regions.universal_regions_iter() {
localized_outlives_constraints.push(LocalizedOutlivesConstraint {
source: region,
from: current_point,
target: region,
to: next_point,
});
}
let Some(next_live_regions) = live_regions.row(next_point) else {
// There are no constraints to add: there are no live regions at the next point.
return;
};
for region in next_live_regions.iter() {
// `region` could be live at the current point, and is live at the next point: add a
// constraint between them, according to variance.
if let Some(&direction) = live_region_variances.get(&region) {
add_liveness_constraint(
region,
current_point,
next_point,
direction,
localized_outlives_constraints,
);
} else {
// Note: there currently are cases related to promoted and const generics, where we
// don't yet have variance information (possibly about temporary regions created when
// typeck sanitizes the promoteds). Until that is done, we conservatively fallback to
// maximizing reachability by adding a bidirectional edge here. This will not limit
// traversal whatsoever, and thus propagate liveness when needed.
//
// FIXME: add the missing variance information and remove this fallback bidirectional
// edge.
let fallback = ConstraintDirection::Bidirectional;
add_liveness_constraint(
region,
current_point,
next_point,
fallback,
localized_outlives_constraints,
);
}
}
}
/// Adds `LocalizedOutlivesConstraint`s between two connected points, according to the given edge
/// direction.
fn add_liveness_constraint(
region: RegionVid,
current_point: PointIndex,
next_point: PointIndex,
direction: ConstraintDirection,
localized_outlives_constraints: &mut LocalizedOutlivesConstraintSet,
) {
match direction {
ConstraintDirection::Forward => {
// Covariant cases: loans flow in the regular direction, from the current point to the
// next point.
localized_outlives_constraints.push(LocalizedOutlivesConstraint {
source: region,
from: current_point,
target: region,
to: next_point,
});
}
ConstraintDirection::Backward => {
// Contravariant cases: loans flow in the inverse direction, from the next point to the
// current point.
localized_outlives_constraints.push(LocalizedOutlivesConstraint {
source: region,
from: next_point,
target: region,
to: current_point,
});
}
ConstraintDirection::Bidirectional => {
// For invariant cases, loans can flow in both directions: we add both edges.
localized_outlives_constraints.push(LocalizedOutlivesConstraint {
source: region,
from: current_point,
target: region,
to: next_point,
});
localized_outlives_constraints.push(LocalizedOutlivesConstraint {
source: region,
from: next_point,
target: region,
to: current_point,
});
}
}
}
/// Extracts variances for regions contained within types. Follows the same structure as
/// `rustc_infer`'s `Generalizer`: we try to relate a type with itself to track and extract the
/// variances of regions.

160
compiler/rustc_borrowck/src/polonius/loan_liveness.rs
View file

@ -1,160 +0,0 @@
use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
use rustc_middle::ty::RegionVid;
use rustc_mir_dataflow::points::PointIndex;
use super::{LiveLoans, LocalizedOutlivesConstraintSet};
use crate::BorrowSet;
use crate::constraints::OutlivesConstraint;
use crate::region_infer::values::LivenessValues;
use crate::type_check::Locations;
/// Compute loan reachability to approximately trace loan liveness throughout the CFG, by
/// traversing the full graph of constraints that combines:
/// - the localized constraints (the physical edges),
/// - with the constraints that hold at all points (the logical edges).
pub(super) fn compute_loan_liveness<'tcx>(
liveness: &LivenessValues,
outlives_constraints: impl Iterator<Item = OutlivesConstraint<'tcx>>,
borrow_set: &BorrowSet<'tcx>,
localized_outlives_constraints: &LocalizedOutlivesConstraintSet,
) -> LiveLoans {
let mut live_loans = LiveLoans::new(borrow_set.len());
// Create the full graph with the physical edges we've localized earlier, and the logical edges
// of constraints that hold at all points.
let logical_constraints =
outlives_constraints.filter(|c| matches!(c.locations, Locations::All(_)));
let graph = LocalizedConstraintGraph::new(&localized_outlives_constraints, logical_constraints);
let mut visited = FxHashSet::default();
let mut stack = Vec::new();
// Compute reachability per loan by traversing each loan's subgraph starting from where it is
// introduced.
for (loan_idx, loan) in borrow_set.iter_enumerated() {
visited.clear();
stack.clear();
let start_node = LocalizedNode {
region: loan.region,
point: liveness.point_from_location(loan.reserve_location),
};
stack.push(start_node);
while let Some(node) = stack.pop() {
if !visited.insert(node) {
continue;
}
// Record the loan as being live on entry to this point if it reaches a live region
// there.
//
// This is an approximation of liveness (which is the thing we want), in that we're
// using a single notion of reachability to represent what used to be _two_ different
// transitive closures. It didn't seem impactful when coming up with the single-graph
// and reachability through space (regions) + time (CFG) concepts, but in practice the
// combination of time-traveling with kills is more impactful than initially
// anticipated.
//
// Kills should prevent a loan from reaching its successor points in the CFG, but not
// while time-traveling: we're not actually at that CFG point, but looking for
// predecessor regions that contain the loan. One of the two TCs we had pushed the
// transitive subset edges to each point instead of having backward edges, and the
// problem didn't exist before. In the abstract, naive reachability is not enough to
// model this, we'd need a slightly different solution. For example, maybe with a
// two-step traversal:
// - at each point we first traverse the subgraph (and possibly time-travel) looking for
// exit nodes while ignoring kills,
// - and then when we're back at the current point, we continue normally.
//
// Another (less annoying) subtlety is that kills and the loan use-map are
// flow-insensitive. Kills can actually appear in places before a loan is introduced, or
// at a location that is actually unreachable in the CFG from the introduction point,
// and these can also be encountered during time-traveling.
//
// The simplest change that made sense to "fix" the issues above is taking into
// account kills that are:
// - reachable from the introduction point
// - encountered during forward traversal. Note that this is not transitive like the
// two-step traversal described above: only kills encountered on exit via a backward
// edge are ignored.
//
// This version of the analysis, however, is enough in practice to pass the tests that
// we care about and NLLs reject, without regressions on crater, and is an actionable
// subset of the full analysis. It also naturally points to areas of improvement that we
// wish to explore later, namely handling kills appropriately during traversal, instead
// of continuing traversal to all the reachable nodes.
//
// FIXME: analyze potential unsoundness, possibly in concert with a borrowck
// implementation in a-mir-formality, fuzzing, or manually crafting counter-examples.
if liveness.is_live_at(node.region, liveness.location_from_point(node.point)) {
live_loans.insert(node.point, loan_idx);
}
for succ in graph.outgoing_edges(node) {
stack.push(succ);
}
}
}
live_loans
}
/// The localized constraint graph indexes the physical and logical edges to compute a given node's
/// successors during traversal.
struct LocalizedConstraintGraph {
/// The actual, physical, edges we have recorded for a given node.
edges: FxHashMap<LocalizedNode, FxIndexSet<LocalizedNode>>,
/// The logical edges representing the outlives constraints that hold at all points in the CFG,
/// which we don't localize to avoid creating a lot of unnecessary edges in the graph. Some CFGs
/// can be big, and we don't need to create such a physical edge for every point in the CFG.
logical_edges: FxHashMap<RegionVid, FxIndexSet<RegionVid>>,
}
/// A node in the graph to be traversed, one of the two vertices of a localized outlives constraint.
#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)]
struct LocalizedNode {
region: RegionVid,
point: PointIndex,
}
impl LocalizedConstraintGraph {
/// Traverses the constraints and returns the indexed graph of edges per node.
fn new<'tcx>(
constraints: &LocalizedOutlivesConstraintSet,
logical_constraints: impl Iterator<Item = OutlivesConstraint<'tcx>>,
) -> Self {
let mut edges: FxHashMap<_, FxIndexSet<_>> = FxHashMap::default();
for constraint in &constraints.outlives {
let source = LocalizedNode { region: constraint.source, point: constraint.from };
let target = LocalizedNode { region: constraint.target, point: constraint.to };
edges.entry(source).or_default().insert(target);
}
let mut logical_edges: FxHashMap<_, FxIndexSet<_>> = FxHashMap::default();
for constraint in logical_constraints {
logical_edges.entry(constraint.sup).or_default().insert(constraint.sub);
}
LocalizedConstraintGraph { edges, logical_edges }
}
/// Returns the outgoing edges of a given node, not its transitive closure.
fn outgoing_edges(&self, node: LocalizedNode) -> impl Iterator<Item = LocalizedNode> {
// The outgoing edges are:
// - the physical edges present at this node,
// - the materialized logical edges that exist virtually at all points for this node's
// region, localized at this point.
let physical_edges =
self.edges.get(&node).into_iter().flat_map(|targets| targets.iter().copied());
let materialized_edges =
self.logical_edges.get(&node.region).into_iter().flat_map(move |targets| {
targets
.iter()
.copied()
.map(move |target| LocalizedNode { point: node.point, region: target })
});
physical_edges.chain(materialized_edges)
}
}

186
compiler/rustc_borrowck/src/polonius/mod.rs
View file

@ -32,47 +32,37 @@
//! - <https://smallcultfollowing.com/babysteps/blog/2023/09/22/polonius-part-1/>
//! - <https://smallcultfollowing.com/babysteps/blog/2023/09/29/polonius-part-2/>
//!
//!
//! Data flows like this:
//! 1) during MIR typeck, record liveness data needed later: live region variances, as well as the
//! usual NLL liveness data (just computed on more locals). That's the [PoloniusLivenessContext].
//! 2) once that is done, variance data is transferred, and the NLL region liveness is converted to
//! the polonius shape. That's the main [PoloniusContext].
//! 3) during region inference, that data and the NLL outlives constraints are used to create the
//! localized outlives constraints, as described above. That's the [PoloniusDiagnosticsContext].
//! 4) transfer this back to the main borrowck procedure: it handles computing errors and
//! diagnostics, debugging and MIR dumping concerns.
mod constraints;
mod dump;
pub(crate) mod legacy;
mod liveness_constraints;
mod loan_liveness;
mod typeck_constraints;
use std::collections::BTreeMap;
use rustc_data_structures::fx::FxHashSet;
use rustc_index::bit_set::SparseBitMatrix;
use rustc_index::interval::SparseIntervalMatrix;
use rustc_middle::mir::{Body, Local};
use rustc_middle::ty::{RegionVid, TyCtxt};
use rustc_middle::ty::RegionVid;
use rustc_mir_dataflow::points::PointIndex;
pub(crate) use self::constraints::*;
pub(self) use self::constraints::*;
pub(crate) use self::dump::dump_polonius_mir;
use self::liveness_constraints::create_liveness_constraints;
use self::loan_liveness::compute_loan_liveness;
use self::typeck_constraints::convert_typeck_constraints;
use crate::dataflow::BorrowIndex;
use crate::region_infer::values::LivenessValues;
use crate::{BorrowSet, RegionInferenceContext};
pub(crate) type LiveLoans = SparseBitMatrix<PointIndex, BorrowIndex>;
/// This struct holds the liveness data created during MIR typeck, and which will be used later in
/// the process, to compute the polonius localized constraints.
/// This struct holds the necessary
/// - liveness data, created during MIR typeck, and which will be used to lazily compute the
/// polonius localized constraints, during NLL region inference as well as MIR dumping,
/// - data needed by the borrowck error computation and diagnostics.
#[derive(Default)]
pub(crate) struct PoloniusLivenessContext {
pub(crate) struct PoloniusContext {
/// The graph from which we extract the localized outlives constraints.
graph: Option<LocalizedConstraintGraph>,
/// The expected edge direction per live region: the kind of directed edge we'll create as
/// liveness constraints depends on the variance of types with respect to each contained region.
live_region_variances: BTreeMap<RegionVid, ConstraintDirection>,
@ -84,27 +74,6 @@ pub(crate) struct PoloniusLivenessContext {
pub(crate) boring_nll_locals: FxHashSet<Local>,
}
/// This struct holds the data needed to create the Polonius localized constraints. Its data is
/// transferred and converted from the [PoloniusLivenessContext] at the end of MIR typeck.
pub(crate) struct PoloniusContext {
/// The liveness data we recorded during MIR typeck.
liveness_context: PoloniusLivenessContext,
/// The set of regions that are live at a given point in the CFG, used to create localized
/// outlives constraints between regions that are live at connected points in the CFG.
live_regions: SparseBitMatrix<PointIndex, RegionVid>,
}
/// This struct holds the data needed by the borrowck error computation and diagnostics. Its data is
/// computed from the [PoloniusContext] when computing NLL regions.
pub(crate) struct PoloniusDiagnosticsContext {
/// The localized outlives constraints that were computed in the main analysis.
localized_outlives_constraints: LocalizedOutlivesConstraintSet,
/// The liveness data computed during MIR typeck: [PoloniusLivenessContext::boring_nll_locals].
pub(crate) boring_nll_locals: FxHashSet<Local>,
}
/// The direction a constraint can flow into. Used to create liveness constraints according to
/// variance.
#[derive(Copy, Clone, PartialEq, Eq, Debug)]
@ -120,26 +89,6 @@ enum ConstraintDirection {
}
impl PoloniusContext {
/// Unlike NLLs, in polonius we traverse the cfg to look for regions live across an edge, so we
/// need to transpose the "points where each region is live" matrix to a "live regions per point"
/// matrix.
// FIXME: avoid this conversion by always storing liveness data in this shape in the rest of
// borrowck.
pub(crate) fn create_from_liveness(
liveness_context: PoloniusLivenessContext,
num_regions: usize,
points_per_live_region: &SparseIntervalMatrix<RegionVid, PointIndex>,
) -> PoloniusContext {
let mut live_regions_per_point = SparseBitMatrix::new(num_regions);
for region in points_per_live_region.rows() {
for point in points_per_live_region.row(region).unwrap().iter() {
live_regions_per_point.insert(point, region);
}
}
PoloniusContext { live_regions: live_regions_per_point, liveness_context }
}
/// Computes live loans using the set of loans model for `-Zpolonius=next`.
///
/// First, creates a constraint graph combining regions and CFG points, by:
@ -151,44 +100,91 @@ impl PoloniusContext {
///
/// The constraint data will be used to compute errors and diagnostics.
pub(crate) fn compute_loan_liveness<'tcx>(
self,
tcx: TyCtxt<'tcx>,
&mut self,
regioncx: &mut RegionInferenceContext<'tcx>,
body: &Body<'tcx>,
borrow_set: &BorrowSet<'tcx>,
) -> PoloniusDiagnosticsContext {
let PoloniusLivenessContext { live_region_variances, boring_nll_locals } =
self.liveness_context;
) {
let liveness = regioncx.liveness_constraints();
let mut localized_outlives_constraints = LocalizedOutlivesConstraintSet::default();
convert_typeck_constraints(
tcx,
body,
regioncx.liveness_constraints(),
regioncx.outlives_constraints(),
regioncx.universal_regions(),
&mut localized_outlives_constraints,
);
// We don't need to prepare the graph (index NLL constraints, etc.) if we have no loans to
// trace throughout localized constraints.
if borrow_set.len() > 0 {
// From the outlives constraints, liveness, and variances, we can compute reachability
// on the lazy localized constraint graph to trace the liveness of loans, for the next
// step in the chain (the NLL loan scope and active loans computations).
let graph = LocalizedConstraintGraph::new(liveness, regioncx.outlives_constraints());
create_liveness_constraints(
body,
regioncx.liveness_constraints(),
&self.live_regions,
&live_region_variances,
regioncx.universal_regions(),
&mut localized_outlives_constraints,
);
let mut live_loans = LiveLoans::new(borrow_set.len());
let mut visitor = LoanLivenessVisitor { liveness, live_loans: &mut live_loans };
graph.traverse(
body,
liveness,
&self.live_region_variances,
regioncx.universal_regions(),
borrow_set,
&mut visitor,
);
regioncx.record_live_loans(live_loans);
// Now that we have a complete graph, we can compute reachability to trace the liveness of
// loans for the next step in the chain, the NLL loan scope and active loans computations.
let live_loans = compute_loan_liveness(
regioncx.liveness_constraints(),
regioncx.outlives_constraints(),
borrow_set,
&localized_outlives_constraints,
);
regioncx.record_live_loans(live_loans);
PoloniusDiagnosticsContext { localized_outlives_constraints, boring_nll_locals }
// The graph can be traversed again during MIR dumping, so we store it here.
self.graph = Some(graph);
}
}
}
/// Visitor to record loan liveness when traversing the localized constraint graph.
struct LoanLivenessVisitor<'a> {
liveness: &'a LivenessValues,
live_loans: &'a mut LiveLoans,
}
impl LocalizedConstraintGraphVisitor for LoanLivenessVisitor<'_> {
fn on_node_traversed(&mut self, loan: BorrowIndex, node: LocalizedNode) {
// Record the loan as being live on entry to this point if it reaches a live region
// there.
//
// This is an approximation of liveness (which is the thing we want), in that we're
// using a single notion of reachability to represent what used to be _two_ different
// transitive closures. It didn't seem impactful when coming up with the single-graph
// and reachability through space (regions) + time (CFG) concepts, but in practice the
// combination of time-traveling with kills is more impactful than initially
// anticipated.
//
// Kills should prevent a loan from reaching its successor points in the CFG, but not
// while time-traveling: we're not actually at that CFG point, but looking for
// predecessor regions that contain the loan. One of the two TCs we had pushed the
// transitive subset edges to each point instead of having backward edges, and the
// problem didn't exist before. In the abstract, naive reachability is not enough to
// model this, we'd need a slightly different solution. For example, maybe with a
// two-step traversal:
// - at each point we first traverse the subgraph (and possibly time-travel) looking for
// exit nodes while ignoring kills,
// - and then when we're back at the current point, we continue normally.
//
// Another (less annoying) subtlety is that kills and the loan use-map are
// flow-insensitive. Kills can actually appear in places before a loan is introduced, or
// at a location that is actually unreachable in the CFG from the introduction point,
// and these can also be encountered during time-traveling.
//
// The simplest change that made sense to "fix" the issues above is taking into account
// kills that are:
// - reachable from the introduction point
// - encountered during forward traversal. Note that this is not transitive like the
// two-step traversal described above: only kills encountered on exit via a backward
// edge are ignored.
//
// This version of the analysis, however, is enough in practice to pass the tests that
// we care about and NLLs reject, without regressions on crater, and is an actionable
// subset of the full analysis. It also naturally points to areas of improvement that we
// wish to explore later, namely handling kills appropriately during traversal, instead
// of continuing traversal to all the reachable nodes.
//
// FIXME: analyze potential unsoundness, possibly in concert with a borrowck
// implementation in a-mir-formality, fuzzing, or manually crafting counter-examples.
let location = self.liveness.location_from_point(node.point);
if self.liveness.is_live_at(node.region, location) {
self.live_loans.insert(node.point, loan);
}
}
}

208
compiler/rustc_borrowck/src/polonius/typeck_constraints.rs
View file

@ -1,208 +0,0 @@
use rustc_data_structures::fx::FxHashSet;
use rustc_middle::mir::{Body, Location, Statement, StatementKind, Terminator, TerminatorKind};
use rustc_middle::ty::{TyCtxt, TypeVisitable};
use rustc_mir_dataflow::points::PointIndex;
use super::{LocalizedOutlivesConstraint, LocalizedOutlivesConstraintSet};
use crate::constraints::OutlivesConstraint;
use crate::region_infer::values::LivenessValues;
use crate::type_check::Locations;
use crate::universal_regions::UniversalRegions;
/// Propagate loans throughout the subset graph at a given point (with some subtleties around the
/// location where effects start to be visible).
pub(super) fn convert_typeck_constraints<'tcx>(
tcx: TyCtxt<'tcx>,
body: &Body<'tcx>,
liveness: &LivenessValues,
outlives_constraints: impl Iterator<Item = OutlivesConstraint<'tcx>>,
universal_regions: &UniversalRegions<'tcx>,
localized_outlives_constraints: &mut LocalizedOutlivesConstraintSet,
) {
for outlives_constraint in outlives_constraints {
match outlives_constraint.locations {
Locations::All(_) => {
// We don't turn constraints holding at all points into physical edges at every
// point in the graph. They are encoded into *traversal* instead: a given node's
// successors will combine these logical edges with the regular, physical, localized
// edges.
continue;
}
Locations::Single(location) => {
// This constraint is marked as holding at one location, we localize it to that
// location or its successor, depending on the corresponding MIR
// statement/terminator. Unfortunately, they all show up from typeck as coming "on
// entry", so for now we modify them to take effects that should apply "on exit"
// into account.
//
// FIXME: this approach is subtle, complicated, and hard to test, so we should track
// this information better in MIR typeck instead, for example with a new `Locations`
// variant that contains which node is crossing over between entry and exit.
let point = liveness.point_from_location(location);
let localized_constraint = if let Some(stmt) =
body[location.block].statements.get(location.statement_index)
{
localize_statement_constraint(
tcx,
stmt,
&outlives_constraint,
point,
universal_regions,
)
} else {
assert_eq!(location.statement_index, body[location.block].statements.len());
let terminator = body[location.block].terminator();
localize_terminator_constraint(
tcx,
body,
terminator,
liveness,
&outlives_constraint,
point,
universal_regions,
)
};
localized_outlives_constraints.push(localized_constraint);
}
}
}
}
/// For a given outlives constraint arising from a MIR statement, localize the constraint with the
/// needed CFG `from`-`to` intra-block nodes.
fn localize_statement_constraint<'tcx>(
tcx: TyCtxt<'tcx>,
stmt: &Statement<'tcx>,
outlives_constraint: &OutlivesConstraint<'tcx>,
current_point: PointIndex,
universal_regions: &UniversalRegions<'tcx>,
) -> LocalizedOutlivesConstraint {
match &stmt.kind {
StatementKind::Assign(box (lhs, rhs)) => {
// To create localized outlives constraints without midpoints, we rely on the property
// that no input regions from the RHS of the assignment will flow into themselves: they
// should not appear in the output regions in the LHS. We believe this to be true by
// construction of the MIR, via temporaries, and assert it here.
//
// We think we don't need midpoints because:
// - every LHS Place has a unique set of regions that don't appear elsewhere
// - this implies that for them to be part of the RHS, the same Place must be read and
// written
// - and that should be impossible in MIR
//
// When we have a more complete implementation in the future, tested with crater, etc,
// we can remove this assertion. It's a debug assert because it can be expensive.
debug_assert!(
{
let mut lhs_regions = FxHashSet::default();
tcx.for_each_free_region(lhs, |region| {
let region = universal_regions.to_region_vid(region);
lhs_regions.insert(region);
});
let mut rhs_regions = FxHashSet::default();
tcx.for_each_free_region(rhs, |region| {
let region = universal_regions.to_region_vid(region);
rhs_regions.insert(region);
});
// The intersection between LHS and RHS regions should be empty.
lhs_regions.is_disjoint(&rhs_regions)
},
"there should be no common regions between the LHS and RHS of an assignment"
);
}
_ => {
// Assignments should be the only statement that can both generate constraints that
// apply on entry (specific to the RHS place) *and* others that only apply on exit (the
// subset of RHS regions that actually flow into the LHS): i.e., where midpoints would
// be used to ensure the former happen before the latter, within the same MIR Location.
}
}
// We generally localize an outlives constraint to where it arises.
LocalizedOutlivesConstraint {
source: outlives_constraint.sup,
from: current_point,
target: outlives_constraint.sub,
to: current_point,
}
}
/// For a given outlives constraint arising from a MIR terminator, localize the constraint with the
/// needed CFG `from`-`to` inter-block nodes.
fn localize_terminator_constraint<'tcx>(
tcx: TyCtxt<'tcx>,
body: &Body<'tcx>,
terminator: &Terminator<'tcx>,
liveness: &LivenessValues,
outlives_constraint: &OutlivesConstraint<'tcx>,
current_point: PointIndex,
universal_regions: &UniversalRegions<'tcx>,
) -> LocalizedOutlivesConstraint {
// FIXME: check if other terminators need the same handling as `Call`s, in particular
// Assert/Yield/Drop.
match &terminator.kind {
// FIXME: also handle diverging calls.
TerminatorKind::Call { destination, target: Some(target), .. } => {
// If there is a target for the call we also relate what flows into the destination here
// to entry to that successor.
let destination_ty = destination.ty(&body.local_decls, tcx);
let successor_location = Location { block: *target, statement_index: 0 };
let successor_point = liveness.point_from_location(successor_location);
compute_constraint_direction(
tcx,
outlives_constraint,
&destination_ty,
current_point,
successor_point,
universal_regions,
)
}
_ => {
// Typeck constraints guide loans between regions at the current point, so we do that in
// the general case, and liveness will take care of making them flow to the terminator's
// successors.
LocalizedOutlivesConstraint {
source: outlives_constraint.sup,
from: current_point,
target: outlives_constraint.sub,
to: current_point,
}
}
}
}
/// For a given outlives constraint and CFG edge, returns the localized constraint with the
/// appropriate `from`-`to` direction. This is computed according to whether the constraint flows to
/// or from a free region in the given `value`, some kind of result for an effectful operation, like
/// the LHS of an assignment.
fn compute_constraint_direction<'tcx>(
tcx: TyCtxt<'tcx>,
outlives_constraint: &OutlivesConstraint<'tcx>,
value: &impl TypeVisitable<TyCtxt<'tcx>>,
current_point: PointIndex,
successor_point: PointIndex,
universal_regions: &UniversalRegions<'tcx>,
) -> LocalizedOutlivesConstraint {
let mut to = current_point;
let mut from = current_point;
tcx.for_each_free_region(value, |region| {
let region = universal_regions.to_region_vid(region);
if region == outlives_constraint.sub {
// This constraint flows into the result, its effects start becoming visible on exit.
to = successor_point;
} else if region == outlives_constraint.sup {
// This constraint flows from the result, its effects start becoming visible on exit.
from = successor_point;
}
});
LocalizedOutlivesConstraint {
source: outlives_constraint.sup,
from,
target: outlives_constraint.sub,
to,
}
}

16
compiler/rustc_borrowck/src/type_check/liveness/mod.rs
View file

@ -11,7 +11,7 @@ use tracing::debug;
use super::TypeChecker;
use crate::constraints::OutlivesConstraintSet;
use crate::polonius::PoloniusLivenessContext;
use crate::polonius::PoloniusContext;
use crate::region_infer::values::LivenessValues;
use crate::universal_regions::UniversalRegions;
@ -48,7 +48,7 @@ pub(super) fn generate<'tcx>(
if typeck.tcx().sess.opts.unstable_opts.polonius.is_next_enabled() {
let (_, boring_locals) =
compute_relevant_live_locals(typeck.tcx(), &free_regions, typeck.body);
typeck.polonius_liveness.as_mut().unwrap().boring_nll_locals =
typeck.polonius_context.as_mut().unwrap().boring_nll_locals =
boring_locals.into_iter().collect();
free_regions = typeck.universal_regions.universal_regions_iter().collect();
}
@ -63,7 +63,7 @@ pub(super) fn generate<'tcx>(
typeck.tcx(),
&mut typeck.constraints.liveness_constraints,
&typeck.universal_regions,
&mut typeck.polonius_liveness,
&mut typeck.polonius_context,
typeck.body,
);
}
@ -140,11 +140,11 @@ fn record_regular_live_regions<'tcx>(
tcx: TyCtxt<'tcx>,
liveness_constraints: &mut LivenessValues,
universal_regions: &UniversalRegions<'tcx>,
polonius_liveness: &mut Option<PoloniusLivenessContext>,
polonius_context: &mut Option<PoloniusContext>,
body: &Body<'tcx>,
) {
let mut visitor =
LiveVariablesVisitor { tcx, liveness_constraints, universal_regions, polonius_liveness };
LiveVariablesVisitor { tcx, liveness_constraints, universal_regions, polonius_context };
for (bb, data) in body.basic_blocks.iter_enumerated() {
visitor.visit_basic_block_data(bb, data);
}
@ -155,7 +155,7 @@ struct LiveVariablesVisitor<'a, 'tcx> {
tcx: TyCtxt<'tcx>,
liveness_constraints: &'a mut LivenessValues,
universal_regions: &'a UniversalRegions<'tcx>,
polonius_liveness: &'a mut Option<PoloniusLivenessContext>,
polonius_context: &'a mut Option<PoloniusContext>,
}
impl<'a, 'tcx> Visitor<'tcx> for LiveVariablesVisitor<'a, 'tcx> {
@ -207,8 +207,8 @@ impl<'a, 'tcx> LiveVariablesVisitor<'a, 'tcx> {
});
// When using `-Zpolonius=next`, we record the variance of each live region.
if let Some(polonius_liveness) = self.polonius_liveness {
polonius_liveness.record_live_region_variance(self.tcx, self.universal_regions, value);
if let Some(polonius_context) = self.polonius_context {
polonius_context.record_live_region_variance(self.tcx, self.universal_regions, value);
}
}
}

4
compiler/rustc_borrowck/src/type_check/liveness/trace.rs
View file

@ -622,8 +622,8 @@ impl<'tcx> LivenessContext<'_, '_, 'tcx> {
});
// When using `-Zpolonius=next`, we record the variance of each live region.
if let Some(polonius_liveness) = typeck.polonius_liveness.as_mut() {
polonius_liveness.record_live_region_variance(
if let Some(polonius_context) = typeck.polonius_context.as_mut() {
polonius_context.record_live_region_variance(
typeck.infcx.tcx,
typeck.universal_regions,
value,

19
compiler/rustc_borrowck/src/type_check/mod.rs
View file

@ -42,8 +42,8 @@ use tracing::{debug, instrument, trace};
use crate::borrow_set::BorrowSet;
use crate::constraints::{OutlivesConstraint, OutlivesConstraintSet};
use crate::diagnostics::UniverseInfo;
use crate::polonius::PoloniusContext;
use crate::polonius::legacy::{PoloniusFacts, PoloniusLocationTable};
use crate::polonius::{PoloniusContext, PoloniusLivenessContext};
use crate::region_infer::TypeTest;
use crate::region_infer::values::{LivenessValues, PlaceholderIndex, PlaceholderIndices};
use crate::session_diagnostics::{MoveUnsized, SimdIntrinsicArgConst};
@ -139,8 +139,8 @@ pub(crate) fn type_check<'tcx>(
debug!(?normalized_inputs_and_output);
let polonius_liveness = if infcx.tcx.sess.opts.unstable_opts.polonius.is_next_enabled() {
Some(PoloniusLivenessContext::default())
let polonius_context = if infcx.tcx.sess.opts.unstable_opts.polonius.is_next_enabled() {
Some(PoloniusContext::default())
} else {
None
};
@ -162,7 +162,7 @@ pub(crate) fn type_check<'tcx>(
borrow_set,
constraints: &mut constraints,
deferred_closure_requirements: &mut deferred_closure_requirements,
polonius_liveness,
polonius_context,
};
typeck.check_user_type_annotations();
@ -172,14 +172,7 @@ pub(crate) fn type_check<'tcx>(
liveness::generate(&mut typeck, &location_map, move_data);
// We're done with typeck, we can finalize the polonius liveness context for region inference.
let polonius_context = typeck.polonius_liveness.take().map(|liveness_context| {
PoloniusContext::create_from_liveness(
liveness_context,
infcx.num_region_vars(),
typeck.constraints.liveness_constraints.points(),
)
});
let polonius_context = typeck.polonius_context;
// In case type check encountered an error region, we suppress unhelpful extra
// errors in by clearing out all outlives bounds that we may end up checking.
@ -238,7 +231,7 @@ struct TypeChecker<'a, 'tcx> {
constraints: &'a mut MirTypeckRegionConstraints<'tcx>,
deferred_closure_requirements: &'a mut DeferredClosureRequirements<'tcx>,
/// When using `-Zpolonius=next`, the liveness helper data used to create polonius constraints.
polonius_liveness: Option<PoloniusLivenessContext>,
polonius_context: Option<PoloniusContext>,
}
/// Holder struct for passing results from MIR typeck to the rest of the non-lexical regions