Merge from rustc

Cargo.toml

@@ -2,12 +2,12 @@
 resolver = "2"
 members = [
   "compiler/rustc",
+  "src/build_helper",
   "src/etc/test-float-parse",
   "src/rustc-std-workspace/rustc-std-workspace-core",
   "src/rustc-std-workspace/rustc-std-workspace-alloc",
   "src/rustc-std-workspace/rustc-std-workspace-std",
   "src/rustdoc-json-types",
-  "src/tools/build_helper",
   "src/tools/cargotest",
   "src/tools/clippy",
   "src/tools/clippy/clippy_dev",

compiler/rustc/Cargo.toml

@@ -31,5 +31,4 @@ jemalloc = ['dep:jemalloc-sys']
 llvm = ['rustc_driver_impl/llvm']
 max_level_info = ['rustc_driver_impl/max_level_info']
 rustc_randomized_layouts = ['rustc_driver_impl/rustc_randomized_layouts']
-rustc_use_parallel_compiler = ['rustc_driver_impl/rustc_use_parallel_compiler']
 # tidy-alphabetical-end

compiler/rustc_ast/src/ast.rs

@@ -1194,7 +1194,7 @@ impl Expr {
     ///
     /// Does not ensure that the path resolves to a const param, the caller should check this.
     pub fn is_potential_trivial_const_arg(&self, strip_identity_block: bool) -> bool {
-        let this = if strip_identity_block { self.maybe_unwrap_block().1 } else { self };
+        let this = if strip_identity_block { self.maybe_unwrap_block() } else { self };
 
         if let ExprKind::Path(None, path) = &this.kind
             && path.is_potential_trivial_const_arg()
@@ -1206,14 +1206,41 @@ impl Expr {
     }
 
     /// Returns an expression with (when possible) *one* outter brace removed
-    pub fn maybe_unwrap_block(&self) -> (bool, &Expr) {
+    pub fn maybe_unwrap_block(&self) -> &Expr {
         if let ExprKind::Block(block, None) = &self.kind
             && let [stmt] = block.stmts.as_slice()
             && let StmtKind::Expr(expr) = &stmt.kind
         {
-            (true, expr)
+            expr
         } else {
-            (false, self)
+            self
+        }
+    }
+
+    /// Determines whether this expression is a macro call optionally wrapped in braces . If
+    /// `already_stripped_block` is set then we do not attempt to peel off a layer of braces.
+    ///
+    /// Returns the [`NodeId`] of the macro call and whether a layer of braces has been peeled
+    /// either before, or part of, this function.
+    pub fn optionally_braced_mac_call(
+        &self,
+        already_stripped_block: bool,
+    ) -> Option<(bool, NodeId)> {
+        match &self.kind {
+            ExprKind::Block(block, None)
+                if let [stmt] = &*block.stmts
+                    && !already_stripped_block =>
+            {
+                match &stmt.kind {
+                    StmtKind::MacCall(_) => Some((true, stmt.id)),
+                    StmtKind::Expr(expr) if let ExprKind::MacCall(_) = &expr.kind => {
+                        Some((true, expr.id))
+                    }
+                    _ => None,
+                }
+            }
+            ExprKind::MacCall(_) => Some((already_stripped_block, self.id)),
+            _ => None,
         }
     }
 

compiler/rustc_ast/src/tokenstream.rs

@@ -38,7 +38,6 @@ pub enum TokenTree {
 }
 
 // Ensure all fields of `TokenTree` are `DynSend` and `DynSync`.
-#[cfg(parallel_compiler)]
 fn _dummy()
 where
     Token: sync::DynSend + sync::DynSync,

compiler/rustc_ast/src/visit.rs

@@ -200,8 +200,8 @@ pub trait Visitor<'ast>: Sized {
     fn visit_param_bound(&mut self, bounds: &'ast GenericBound, _ctxt: BoundKind) -> Self::Result {
         walk_param_bound(self, bounds)
     }
-    fn visit_precise_capturing_arg(&mut self, arg: &'ast PreciseCapturingArg) {
-        walk_precise_capturing_arg(self, arg);
+    fn visit_precise_capturing_arg(&mut self, arg: &'ast PreciseCapturingArg) -> Self::Result {
+        walk_precise_capturing_arg(self, arg)
     }
     fn visit_poly_trait_ref(&mut self, t: &'ast PolyTraitRef) -> Self::Result {
         walk_poly_trait_ref(self, t)
@@ -730,14 +730,10 @@ pub fn walk_param_bound<'a, V: Visitor<'a>>(visitor: &mut V, bound: &'a GenericB
 pub fn walk_precise_capturing_arg<'a, V: Visitor<'a>>(
     visitor: &mut V,
     arg: &'a PreciseCapturingArg,
-) {
+) -> V::Result {
     match arg {
-        PreciseCapturingArg::Lifetime(lt) => {
-            visitor.visit_lifetime(lt, LifetimeCtxt::GenericArg);
-        }
-        PreciseCapturingArg::Arg(path, id) => {
-            visitor.visit_path(path, *id);
-        }
+        PreciseCapturingArg::Lifetime(lt) => visitor.visit_lifetime(lt, LifetimeCtxt::GenericArg),
+        PreciseCapturingArg::Arg(path, id) => visitor.visit_path(path, *id),
     }
 }
 

compiler/rustc_ast_lowering/src/asm.rs

@@ -45,6 +45,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                     | asm::InlineAsmArch::X86_64
                     | asm::InlineAsmArch::Arm
                     | asm::InlineAsmArch::AArch64
+                    | asm::InlineAsmArch::Arm64EC
                     | asm::InlineAsmArch::RiscV32
                     | asm::InlineAsmArch::RiscV64
                     | asm::InlineAsmArch::LoongArch64

compiler/rustc_ast_passes/src/feature_gate.rs

@@ -204,6 +204,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
                     "meant for internal use only" {
                         keyword => rustdoc_internals
                         fake_variadic => rustdoc_internals
+                        search_unbox => rustdoc_internals
                     }
                 );
             }
@@ -522,9 +523,18 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session, features: &Features) {
         "consider removing `for<...>`"
     );
     gate_all!(more_qualified_paths, "usage of qualified paths in this context is experimental");
-    for &span in spans.get(&sym::yield_expr).iter().copied().flatten() {
-        if !span.at_least_rust_2024() {
-            gate!(&visitor, coroutines, span, "yield syntax is experimental");
+    // yield can be enabled either by `coroutines` or `gen_blocks`
+    if let Some(spans) = spans.get(&sym::yield_expr) {
+        for span in spans {
+            if (!visitor.features.coroutines() && !span.allows_unstable(sym::coroutines))
+                && (!visitor.features.gen_blocks() && !span.allows_unstable(sym::gen_blocks))
+            {
+                #[allow(rustc::untranslatable_diagnostic)]
+                // Don't know which of the two features to include in the
+                // error message, so I am arbitrarily picking one.
+                feature_err(&visitor.sess, sym::coroutines, *span, "yield syntax is experimental")
+                    .emit();
+            }
         }
     }
     gate_all!(gen_blocks, "gen blocks are experimental");

compiler/rustc_attr/src/builtin.rs

@@ -16,9 +16,9 @@ use rustc_session::lint::BuiltinLintDiag;
 use rustc_session::lint::builtin::UNEXPECTED_CFGS;
 use rustc_session::parse::feature_err;
 use rustc_session::{RustcVersion, Session};
+use rustc_span::Span;
 use rustc_span::hygiene::Transparency;
 use rustc_span::symbol::{Symbol, kw, sym};
-use rustc_span::{DUMMY_SP, Span};
 
 use crate::fluent_generated;
 use crate::session_diagnostics::{self, IncorrectReprFormatGenericCause};
@@ -92,9 +92,7 @@ impl Stability {
 #[derive(HashStable_Generic)]
 pub struct ConstStability {
     pub level: StabilityLevel,
-    /// This can be `None` for functions that do not have an explicit const feature.
-    /// We still track them for recursive const stability checks.
-    pub feature: Option<Symbol>,
+    pub feature: Symbol,
     /// This is true iff the `const_stable_indirect` attribute is present.
     pub const_stable_indirect: bool,
     /// whether the function has a `#[rustc_promotable]` attribute
@@ -272,22 +270,19 @@ pub fn find_stability(
 
 /// Collects stability info from `rustc_const_stable`/`rustc_const_unstable`/`rustc_promotable`
 /// attributes in `attrs`. Returns `None` if no stability attributes are found.
-///
-/// `is_const_fn` indicates whether this is a function marked as `const`.
 pub fn find_const_stability(
     sess: &Session,
     attrs: &[Attribute],
     item_sp: Span,
-    is_const_fn: bool,
 ) -> Option<(ConstStability, Span)> {
     let mut const_stab: Option<(ConstStability, Span)> = None;
     let mut promotable = false;
-    let mut const_stable_indirect = None;
+    let mut const_stable_indirect = false;
 
     for attr in attrs {
         match attr.name_or_empty() {
             sym::rustc_promotable => promotable = true,
-            sym::rustc_const_stable_indirect => const_stable_indirect = Some(attr.span),
+            sym::rustc_const_stable_indirect => const_stable_indirect = true,
             sym::rustc_const_unstable => {
                 if const_stab.is_some() {
                     sess.dcx()
@@ -299,7 +294,7 @@ pub fn find_const_stability(
                     const_stab = Some((
                         ConstStability {
                             level,
-                            feature: Some(feature),
+                            feature,
                             const_stable_indirect: false,
                             promotable: false,
                         },
@@ -317,7 +312,7 @@ pub fn find_const_stability(
                     const_stab = Some((
                         ConstStability {
                             level,
-                            feature: Some(feature),
+                            feature,
                             const_stable_indirect: false,
                             promotable: false,
                         },
@@ -340,7 +335,7 @@ pub fn find_const_stability(
             }
         }
     }
-    if const_stable_indirect.is_some() {
+    if const_stable_indirect {
         match &mut const_stab {
             Some((stab, _)) => {
                 if stab.is_const_unstable() {
@@ -351,36 +346,37 @@ pub fn find_const_stability(
                     })
                 }
             }
-            _ => {}
+            _ => {
+                // This function has no const stability attribute, but has `const_stable_indirect`.
+                // We ignore that; unmarked functions are subject to recursive const stability
+                // checks by default so we do carry out the user's intent.
+            }
         }
     }
-    // Make sure if `const_stable_indirect` is present, that is recorded. Also make sure all `const
-    // fn` get *some* marker, since we are a staged_api crate and therefore will do recursive const
-    // stability checks for them. We need to do this because the default for whether an unmarked
-    // function enforces recursive stability differs between staged-api crates and force-unmarked
-    // crates: in force-unmarked crates, only functions *explicitly* marked `const_stable_indirect`
-    // enforce recursive stability. Therefore when `lookup_const_stability` is `None`, we have to
-    // assume the function does not have recursive stability. All functions that *do* have recursive
-    // stability must explicitly record this, and so that's what we do for all `const fn` in a
-    // staged_api crate.
-    if (is_const_fn || const_stable_indirect.is_some()) && const_stab.is_none() {
-        let c = ConstStability {
-            feature: None,
-            const_stable_indirect: const_stable_indirect.is_some(),
-            promotable: false,
-            level: StabilityLevel::Unstable {
-                reason: UnstableReason::Default,
-                issue: None,
-                is_soft: false,
-                implied_by: None,
-            },
-        };
-        const_stab = Some((c, const_stable_indirect.unwrap_or(DUMMY_SP)));
-    }
 
     const_stab
 }
 
+/// Calculates the const stability for a const function in a `-Zforce-unstable-if-unmarked` crate
+/// without the `staged_api` feature.
+pub fn unmarked_crate_const_stab(
+    _sess: &Session,
+    attrs: &[Attribute],
+    regular_stab: Stability,
+) -> ConstStability {
+    assert!(regular_stab.level.is_unstable());
+    // The only attribute that matters here is `rustc_const_stable_indirect`.
+    // We enforce recursive const stability rules for those functions.
+    let const_stable_indirect =
+        attrs.iter().any(|a| a.name_or_empty() == sym::rustc_const_stable_indirect);
+    ConstStability {
+        feature: regular_stab.feature,
+        const_stable_indirect,
+        promotable: false,
+        level: regular_stab.level,
+    }
+}
+
 /// Collects stability info from `rustc_default_body_unstable` attributes in `attrs`.
 /// Returns `None` if no stability attributes are found.
 pub fn find_body_stability(

compiler/rustc_baked_icu_data/Cargo.toml

@@ -8,11 +8,6 @@ edition = "2021"
 icu_list = "1.2"
 icu_locid = "1.2"
 icu_locid_transform = "1.3.2"
-icu_provider = "1.2"
+icu_provider = { version = "1.2", features = ["sync"] }
 zerovec = "0.10.0"
 # tidy-alphabetical-end
-
-[features]
-# tidy-alphabetical-start
-rustc_use_parallel_compiler = ['icu_provider/sync']
-# tidy-alphabetical-end

compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs

@@ -27,7 +27,7 @@ use rustc_middle::mir::{
 };
 use rustc_middle::ty::print::PrintTraitRefExt as _;
 use rustc_middle::ty::{
-    self, PredicateKind, Ty, TyCtxt, TypeSuperVisitable, TypeVisitor, Upcast,
+    self, ClauseKind, PredicateKind, Ty, TyCtxt, TypeSuperVisitable, TypeVisitor, Upcast,
     suggest_constraining_type_params,
 };
 use rustc_middle::util::CallKind;
@@ -39,6 +39,7 @@ use rustc_span::{BytePos, Span, Symbol};
 use rustc_trait_selection::error_reporting::InferCtxtErrorExt;
 use rustc_trait_selection::error_reporting::traits::FindExprBySpan;
 use rustc_trait_selection::infer::InferCtxtExt;
+use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt as _;
 use rustc_trait_selection::traits::{Obligation, ObligationCause, ObligationCtxt};
 use tracing::{debug, instrument};
 
@@ -201,16 +202,15 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
 
                 let mut has_suggest_reborrow = false;
                 if !seen_spans.contains(&move_span) {
-                    if !closure {
-                        self.suggest_ref_or_clone(
-                            mpi,
-                            &mut err,
-                            &mut in_pattern,
-                            move_spans,
-                            moved_place.as_ref(),
-                            &mut has_suggest_reborrow,
-                        );
-                    }
+                    self.suggest_ref_or_clone(
+                        mpi,
+                        &mut err,
+                        &mut in_pattern,
+                        move_spans,
+                        moved_place.as_ref(),
+                        &mut has_suggest_reborrow,
+                        closure,
+                    );
 
                     let msg_opt = CapturedMessageOpt {
                         is_partial_move,
@@ -266,27 +266,11 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 }
             }
 
-            let opt_name = self.describe_place_with_options(place.as_ref(), DescribePlaceOpt {
-                including_downcast: true,
-                including_tuple_field: true,
-            });
-            let note_msg = match opt_name {
-                Some(name) => format!("`{name}`"),
-                None => "value".to_owned(),
-            };
-            if self.suggest_borrow_fn_like(&mut err, ty, &move_site_vec, &note_msg)
-                || if let UseSpans::FnSelfUse { kind, .. } = use_spans
-                    && let CallKind::FnCall { fn_trait_id, self_ty } = kind
-                    && let ty::Param(_) = self_ty.kind()
-                    && ty == self_ty
-                    && self.infcx.tcx.is_lang_item(fn_trait_id, LangItem::FnOnce)
-                {
-                    // this is a type parameter `T: FnOnce()`, don't suggest `T: FnOnce() + Clone`.
-                    true
-                } else {
-                    false
-                }
-            {
+            if self.param_env.caller_bounds().iter().any(|c| {
+                c.as_trait_clause().is_some_and(|pred| {
+                    pred.skip_binder().self_ty() == ty && self.infcx.tcx.is_fn_trait(pred.def_id())
+                })
+            }) {
                 // Suppress the next suggestion since we don't want to put more bounds onto
                 // something that already has `Fn`-like bounds (or is a closure), so we can't
                 // restrict anyways.
@@ -295,6 +279,14 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 self.suggest_adding_bounds(&mut err, ty, copy_did, span);
             }
 
+            let opt_name = self.describe_place_with_options(place.as_ref(), DescribePlaceOpt {
+                including_downcast: true,
+                including_tuple_field: true,
+            });
+            let note_msg = match opt_name {
+                Some(name) => format!("`{name}`"),
+                None => "value".to_owned(),
+            };
             if needs_note {
                 if let Some(local) = place.as_local() {
                     let span = self.body.local_decls[local].source_info.span;
@@ -341,6 +333,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
         move_spans: UseSpans<'tcx>,
         moved_place: PlaceRef<'tcx>,
         has_suggest_reborrow: &mut bool,
+        moved_or_invoked_closure: bool,
     ) {
         let move_span = match move_spans {
             UseSpans::ClosureUse { capture_kind_span, .. } => capture_kind_span,
@@ -428,104 +421,76 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 }
                 let typeck = self.infcx.tcx.typeck(self.mir_def_id());
                 let parent = self.infcx.tcx.parent_hir_node(expr.hir_id);
-                let (def_id, args, offset) = if let hir::Node::Expr(parent_expr) = parent
+                let (def_id, call_id, args, offset) = if let hir::Node::Expr(parent_expr) = parent
                     && let hir::ExprKind::MethodCall(_, _, args, _) = parent_expr.kind
                 {
-                    (typeck.type_dependent_def_id(parent_expr.hir_id), args, 1)
+                    let def_id = typeck.type_dependent_def_id(parent_expr.hir_id);
+                    (def_id, Some(parent_expr.hir_id), args, 1)
                 } else if let hir::Node::Expr(parent_expr) = parent
                     && let hir::ExprKind::Call(call, args) = parent_expr.kind
                     && let ty::FnDef(def_id, _) = typeck.node_type(call.hir_id).kind()
                 {
-                    (Some(*def_id), args, 0)
+                    (Some(*def_id), Some(call.hir_id), args, 0)
                 } else {
-                    (None, &[][..], 0)
+                    (None, None, &[][..], 0)
                 };
+                let ty = place.ty(self.body, self.infcx.tcx).ty;
 
-                // If the moved value is a mut reference, it is used in a
-                // generic function and it's type is a generic param, it can be
-                // reborrowed to avoid moving.
-                // for example:
-                // struct Y(u32);
-                // x's type is '& mut Y' and it is used in `fn generic<T>(x: T) {}`.
+                let mut can_suggest_clone = true;
                 if let Some(def_id) = def_id
-                    && self.infcx.tcx.def_kind(def_id).is_fn_like()
                     && let Some(pos) = args.iter().position(|arg| arg.hir_id == expr.hir_id)
-                    && let Some(arg) = self
-                        .infcx
-                        .tcx
-                        .fn_sig(def_id)
-                        .skip_binder()
-                        .skip_binder()
-                        .inputs()
-                        .get(pos + offset)
-                    && let ty::Param(_) = arg.kind()
                 {
-                    let place = &self.move_data.move_paths[mpi].place;
-                    let ty = place.ty(self.body, self.infcx.tcx).ty;
-                    if let ty::Ref(_, _, hir::Mutability::Mut) = ty.kind() {
+                    // The move occurred as one of the arguments to a function call. Is that
+                    // argument generic? `def_id` can't be a closure here, so using `fn_sig` is fine
+                    let arg_param = if self.infcx.tcx.def_kind(def_id).is_fn_like()
+                        && let sig =
+                            self.infcx.tcx.fn_sig(def_id).instantiate_identity().skip_binder()
+                        && let Some(arg_ty) = sig.inputs().get(pos + offset)
+                        && let ty::Param(arg_param) = arg_ty.kind()
+                    {
+                        Some(arg_param)
+                    } else {
+                        None
+                    };
+
+                    // If the moved value is a mut reference, it is used in a
+                    // generic function and it's type is a generic param, it can be
+                    // reborrowed to avoid moving.
+                    // for example:
+                    // struct Y(u32);
+                    // x's type is '& mut Y' and it is used in `fn generic<T>(x: T) {}`.
+                    if let ty::Ref(_, _, hir::Mutability::Mut) = ty.kind()
+                        && arg_param.is_some()
+                    {
                         *has_suggest_reborrow = true;
                         self.suggest_reborrow(err, expr.span, moved_place);
                         return;
                     }
-                }
 
-                let mut can_suggest_clone = true;
-                if let Some(def_id) = def_id
-                    && let Some(local_def_id) = def_id.as_local()
-                    && let node = self.infcx.tcx.hir_node_by_def_id(local_def_id)
-                    && let Some(fn_sig) = node.fn_sig()
-                    && let Some(ident) = node.ident()
-                    && let Some(pos) = args.iter().position(|arg| arg.hir_id == expr.hir_id)
-                    && let Some(arg) = fn_sig.decl.inputs.get(pos + offset)
-                {
-                    let mut is_mut = false;
-                    if let hir::TyKind::Path(hir::QPath::Resolved(None, path)) = arg.kind
-                        && let Res::Def(DefKind::TyParam, param_def_id) = path.res
-                        && self
-                            .infcx
-                            .tcx
-                            .predicates_of(def_id)
-                            .instantiate_identity(self.infcx.tcx)
-                            .predicates
-                            .into_iter()
-                            .any(|pred| {
-                                if let ty::ClauseKind::Trait(predicate) = pred.kind().skip_binder()
-                                    && [
-                                        self.infcx.tcx.get_diagnostic_item(sym::AsRef),
-                                        self.infcx.tcx.get_diagnostic_item(sym::AsMut),
-                                        self.infcx.tcx.get_diagnostic_item(sym::Borrow),
-                                        self.infcx.tcx.get_diagnostic_item(sym::BorrowMut),
-                                    ]
-                                    .contains(&Some(predicate.def_id()))
-                                    && let ty::Param(param) = predicate.self_ty().kind()
-                                    && let generics = self.infcx.tcx.generics_of(def_id)
-                                    && let param = generics.type_param(*param, self.infcx.tcx)
-                                    && param.def_id == param_def_id
-                                {
-                                    if [
-                                        self.infcx.tcx.get_diagnostic_item(sym::AsMut),
-                                        self.infcx.tcx.get_diagnostic_item(sym::BorrowMut),
-                                    ]
-                                    .contains(&Some(predicate.def_id()))
-                                    {
-                                        is_mut = true;
-                                    }
-                                    true
-                                } else {
-                                    false
-                                }
-                            })
+                    // If the moved place is used generically by the callee and a reference to it
+                    // would still satisfy any bounds on its type, suggest borrowing.
+                    if let Some(&param) = arg_param
+                        && let Some(generic_args) = call_id.and_then(|id| typeck.node_args_opt(id))
+                        && let Some(ref_mutability) = self.suggest_borrow_generic_arg(
+                            err,
+                            def_id,
+                            generic_args,
+                            param,
+                            moved_place,
+                            pos + offset,
+                            ty,
+                            expr.span,
+                        )
                     {
-                        // The type of the argument corresponding to the expression that got moved
-                        // is a type parameter `T`, which is has a `T: AsRef` obligation.
-                        err.span_suggestion_verbose(
-                            expr.span.shrink_to_lo(),
-                            "borrow the value to avoid moving it",
-                            format!("&{}", if is_mut { "mut " } else { "" }),
-                            Applicability::MachineApplicable,
-                        );
-                        can_suggest_clone = is_mut;
-                    } else {
+                        can_suggest_clone = ref_mutability.is_mut();
+                    } else if let Some(local_def_id) = def_id.as_local()
+                        && let node = self.infcx.tcx.hir_node_by_def_id(local_def_id)
+                        && let Some(fn_decl) = node.fn_decl()
+                        && let Some(ident) = node.ident()
+                        && let Some(arg) = fn_decl.inputs.get(pos + offset)
+                    {
+                        // If we can't suggest borrowing in the call, but the function definition
+                        // is local, instead offer changing the function to borrow that argument.
                         let mut span: MultiSpan = arg.span.into();
                         span.push_span_label(
                             arg.span,
@@ -546,8 +511,6 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                         );
                     }
                 }
-                let place = &self.move_data.move_paths[mpi].place;
-                let ty = place.ty(self.body, self.infcx.tcx).ty;
                 if let hir::Node::Expr(parent_expr) = parent
                     && let hir::ExprKind::Call(call_expr, _) = parent_expr.kind
                     && let hir::ExprKind::Path(hir::QPath::LangItem(LangItem::IntoIterIntoIter, _)) =
@@ -557,6 +520,8 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 } else if let UseSpans::FnSelfUse { kind: CallKind::Normal { .. }, .. } = move_spans
                 {
                     // We already suggest cloning for these cases in `explain_captures`.
+                } else if moved_or_invoked_closure {
+                    // Do not suggest `closure.clone()()`.
                 } else if let UseSpans::ClosureUse {
                     closure_kind:
                         ClosureKind::Coroutine(CoroutineKind::Desugared(_, CoroutineSource::Block)),
@@ -665,6 +630,113 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
         );
     }
 
+    /// If a place is used after being moved as an argument to a function, the function is generic
+    /// in that argument, and a reference to the argument's type would still satisfy the function's
+    /// bounds, suggest borrowing. This covers, e.g., borrowing an `impl Fn()` argument being passed
+    /// in an `impl FnOnce()` position.
+    /// Returns `Some(mutability)` when suggesting to borrow with mutability `mutability`, or `None`
+    /// if no suggestion is made.
+    fn suggest_borrow_generic_arg(
+        &self,
+        err: &mut Diag<'_>,
+        callee_did: DefId,
+        generic_args: ty::GenericArgsRef<'tcx>,
+        param: ty::ParamTy,
+        moved_place: PlaceRef<'tcx>,
+        moved_arg_pos: usize,
+        moved_arg_ty: Ty<'tcx>,
+        place_span: Span,
+    ) -> Option<ty::Mutability> {
+        let tcx = self.infcx.tcx;
+        let sig = tcx.fn_sig(callee_did).instantiate_identity().skip_binder();
+        let clauses = tcx.predicates_of(callee_did).instantiate_identity(self.infcx.tcx).predicates;
+
+        // First, is there at least one method on one of `param`'s trait bounds?
+        // This keeps us from suggesting borrowing the argument to `mem::drop`, e.g.
+        if !clauses.iter().any(|clause| {
+            clause.as_trait_clause().is_some_and(|tc| {
+                tc.self_ty().skip_binder().is_param(param.index)
+                    && tc.polarity() == ty::PredicatePolarity::Positive
+                    && tcx
+                        .supertrait_def_ids(tc.def_id())
+                        .flat_map(|trait_did| tcx.associated_items(trait_did).in_definition_order())
+                        .any(|item| item.fn_has_self_parameter)
+            })
+        }) {
+            return None;
+        }
+
+        // Try borrowing a shared reference first, then mutably.
+        if let Some(mutbl) = [ty::Mutability::Not, ty::Mutability::Mut].into_iter().find(|&mutbl| {
+            let re = self.infcx.tcx.lifetimes.re_erased;
+            let ref_ty = Ty::new_ref(self.infcx.tcx, re, moved_arg_ty, mutbl);
+
+            // Ensure that substituting `ref_ty` in the callee's signature doesn't break
+            // other inputs or the return type.
+            let new_args = tcx.mk_args_from_iter(generic_args.iter().enumerate().map(
+                |(i, arg)| {
+                    if i == param.index as usize { ref_ty.into() } else { arg }
+                },
+            ));
+            let can_subst = |ty: Ty<'tcx>| {
+                // Normalize before comparing to see through type aliases and projections.
+                let old_ty = ty::EarlyBinder::bind(ty).instantiate(tcx, generic_args);
+                let new_ty = ty::EarlyBinder::bind(ty).instantiate(tcx, new_args);
+                if let Ok(old_ty) = tcx.try_normalize_erasing_regions(self.param_env, old_ty)
+                    && let Ok(new_ty) = tcx.try_normalize_erasing_regions(self.param_env, new_ty)
+                {
+                    old_ty == new_ty
+                } else {
+                    false
+                }
+            };
+            if !can_subst(sig.output())
+                || sig
+                    .inputs()
+                    .iter()
+                    .enumerate()
+                    .any(|(i, &input_ty)| i != moved_arg_pos && !can_subst(input_ty))
+            {
+                return false;
+            }
+
+            // Test the callee's predicates, substituting a reference in for the self ty
+            // in bounds on `param`.
+            clauses.iter().all(|&clause| {
+                let clause_for_ref = clause.kind().map_bound(|kind| match kind {
+                    ClauseKind::Trait(c) if c.self_ty().is_param(param.index) => {
+                        ClauseKind::Trait(c.with_self_ty(tcx, ref_ty))
+                    }
+                    ClauseKind::Projection(c) if c.self_ty().is_param(param.index) => {
+                        ClauseKind::Projection(c.with_self_ty(tcx, ref_ty))
+                    }
+                    _ => kind,
+                });
+                self.infcx.predicate_must_hold_modulo_regions(&Obligation::new(
+                    tcx,
+                    ObligationCause::dummy(),
+                    self.param_env,
+                    ty::EarlyBinder::bind(clause_for_ref).instantiate(tcx, generic_args),
+                ))
+            })
+        }) {
+            let place_desc = if let Some(desc) = self.describe_place(moved_place) {
+                format!("`{desc}`")
+            } else {
+                "here".to_owned()
+            };
+            err.span_suggestion_verbose(
+                place_span.shrink_to_lo(),
+                format!("consider {}borrowing {place_desc}", mutbl.mutably_str()),
+                mutbl.ref_prefix_str(),
+                Applicability::MaybeIncorrect,
+            );
+            Some(mutbl)
+        } else {
+            None
+        }
+    }
+
     fn report_use_of_uninitialized(
         &self,
         mpi: MovePathIndex,
@@ -845,74 +917,6 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
         );
     }
 
-    fn suggest_borrow_fn_like(
-        &self,
-        err: &mut Diag<'_>,
-        ty: Ty<'tcx>,
-        move_sites: &[MoveSite],
-        value_name: &str,
-    ) -> bool {
-        let tcx = self.infcx.tcx;
-
-        // Find out if the predicates show that the type is a Fn or FnMut
-        let find_fn_kind_from_did = |(pred, _): (ty::Clause<'tcx>, _)| {
-            if let ty::ClauseKind::Trait(pred) = pred.kind().skip_binder()
-                && pred.self_ty() == ty
-            {
-                if tcx.is_lang_item(pred.def_id(), LangItem::Fn) {
-                    return Some(hir::Mutability::Not);
-                } else if tcx.is_lang_item(pred.def_id(), LangItem::FnMut) {
-                    return Some(hir::Mutability::Mut);
-                }
-            }
-            None
-        };
-
-        // If the type is opaque/param/closure, and it is Fn or FnMut, let's suggest (mutably)
-        // borrowing the type, since `&mut F: FnMut` iff `F: FnMut` and similarly for `Fn`.
-        // These types seem reasonably opaque enough that they could be instantiated with their
-        // borrowed variants in a function body when we see a move error.
-        let borrow_level = match *ty.kind() {
-            ty::Param(_) => tcx
-                .explicit_predicates_of(self.mir_def_id().to_def_id())
-                .predicates
-                .iter()
-                .copied()
-                .find_map(find_fn_kind_from_did),
-            ty::Alias(ty::Opaque, ty::AliasTy { def_id, args, .. }) => tcx
-                .explicit_item_super_predicates(def_id)
-                .iter_instantiated_copied(tcx, args)
-                .find_map(|(clause, span)| find_fn_kind_from_did((clause, span))),
-            ty::Closure(_, args) => match args.as_closure().kind() {
-                ty::ClosureKind::Fn => Some(hir::Mutability::Not),
-                ty::ClosureKind::FnMut => Some(hir::Mutability::Mut),
-                _ => None,
-            },
-            _ => None,
-        };
-
-        let Some(borrow_level) = borrow_level else {
-            return false;
-        };
-        let sugg = move_sites
-            .iter()
-            .map(|move_site| {
-                let move_out = self.move_data.moves[(*move_site).moi];
-                let moved_place = &self.move_data.move_paths[move_out.path].place;
-                let move_spans = self.move_spans(moved_place.as_ref(), move_out.source);
-                let move_span = move_spans.args_or_use();
-                let suggestion = borrow_level.ref_prefix_str().to_owned();
-                (move_span.shrink_to_lo(), suggestion)
-            })
-            .collect();
-        err.multipart_suggestion_verbose(
-            format!("consider {}borrowing {value_name}", borrow_level.mutably_str()),
-            sugg,
-            Applicability::MaybeIncorrect,
-        );
-        true
-    }
-
     /// In a move error that occurs on a call within a loop, we try to identify cases where cloning
     /// the value would lead to a logic error. We infer these cases by seeing if the moved value is
     /// part of the logic to break the loop, either through an explicit `break` or if the expression

compiler/rustc_borrowck/src/diagnostics/opaque_suggestions.rs

@@ -4,15 +4,16 @@
 use std::ops::ControlFlow;
 
 use either::Either;
+use itertools::Itertools as _;
 use rustc_data_structures::fx::FxIndexSet;
-use rustc_errors::{Applicability, Diag};
+use rustc_errors::{Diag, Subdiagnostic};
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
 use rustc_middle::mir::{self, ConstraintCategory, Location};
 use rustc_middle::ty::{
     self, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor,
 };
-use rustc_span::Symbol;
+use rustc_trait_selection::errors::impl_trait_overcapture_suggestion;
 
 use crate::MirBorrowckCtxt;
 use crate::borrow_set::BorrowData;
@@ -61,6 +62,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
             // *does* mention. We'll use that for the `+ use<'a>` suggestion below.
             let mut visitor = CheckExplicitRegionMentionAndCollectGenerics {
                 tcx,
+                generics: tcx.generics_of(opaque_def_id),
                 offending_region_idx,
                 seen_opaques: [opaque_def_id].into_iter().collect(),
                 seen_lifetimes: Default::default(),
@@ -83,34 +85,50 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                         "this call may capture more lifetimes than intended, \
                         because Rust 2024 has adjusted the `impl Trait` lifetime capture rules",
                     );
-                    let mut seen_generics: Vec<_> =
-                        visitor.seen_lifetimes.iter().map(ToString::to_string).collect();
-                    // Capture all in-scope ty/const params.
-                    seen_generics.extend(
-                        ty::GenericArgs::identity_for_item(tcx, opaque_def_id)
-                            .iter()
-                            .filter(|arg| {
-                                matches!(
-                                    arg.unpack(),
-                                    ty::GenericArgKind::Type(_) | ty::GenericArgKind::Const(_)
-                                )
-                            })
-                            .map(|arg| arg.to_string()),
-                    );
-                    if opaque_def_id.is_local() {
-                        diag.span_suggestion_verbose(
-                            tcx.def_span(opaque_def_id).shrink_to_hi(),
-                            "add a precise capturing bound to avoid overcapturing",
-                            format!(" + use<{}>", seen_generics.join(", ")),
-                            Applicability::MaybeIncorrect,
-                        );
+                    let mut captured_args = visitor.seen_lifetimes;
+                    // Add in all of the type and const params, too.
+                    // Ordering here is kinda strange b/c we're walking backwards,
+                    // but we're trying to provide *a* suggestion, not a nice one.
+                    let mut next_generics = Some(visitor.generics);
+                    let mut any_synthetic = false;
+                    while let Some(generics) = next_generics {
+                        for param in &generics.own_params {
+                            if param.kind.is_ty_or_const() {
+                                captured_args.insert(param.def_id);
+                            }
+                            if param.kind.is_synthetic() {
+                                any_synthetic = true;
+                            }
+                        }
+                        next_generics = generics.parent.map(|def_id| tcx.generics_of(def_id));
+                    }
+
+                    if let Some(opaque_def_id) = opaque_def_id.as_local()
+                        && let hir::OpaqueTyOrigin::FnReturn { parent, .. } =
+                            tcx.hir().expect_opaque_ty(opaque_def_id).origin
+                    {
+                        if let Some(sugg) = impl_trait_overcapture_suggestion(
+                            tcx,
+                            opaque_def_id,
+                            parent,
+                            captured_args,
+                        ) {
+                            sugg.add_to_diag(diag);
+                        }
                     } else {
                         diag.span_help(
                             tcx.def_span(opaque_def_id),
                             format!(
                                 "if you can modify this crate, add a precise \
                                 capturing bound to avoid overcapturing: `+ use<{}>`",
-                                seen_generics.join(", ")
+                                if any_synthetic {
+                                    "/* Args */".to_string()
+                                } else {
+                                    captured_args
+                                        .into_iter()
+                                        .map(|def_id| tcx.item_name(def_id))
+                                        .join(", ")
+                                }
                             ),
                         );
                     }
@@ -182,9 +200,10 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for FindOpaqueRegion<'_, 'tcx> {
 
 struct CheckExplicitRegionMentionAndCollectGenerics<'tcx> {
     tcx: TyCtxt<'tcx>,
+    generics: &'tcx ty::Generics,
     offending_region_idx: usize,
     seen_opaques: FxIndexSet<DefId>,
-    seen_lifetimes: FxIndexSet<Symbol>,
+    seen_lifetimes: FxIndexSet<DefId>,
 }
 
 impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for CheckExplicitRegionMentionAndCollectGenerics<'tcx> {
@@ -214,7 +233,7 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for CheckExplicitRegionMentionAndCollectGen
                 if param.index as usize == self.offending_region_idx {
                     ControlFlow::Break(())
                 } else {
-                    self.seen_lifetimes.insert(param.name);
+                    self.seen_lifetimes.insert(self.generics.region_param(param, self.tcx).def_id);
                     ControlFlow::Continue(())
                 }
             }

compiler/rustc_builtin_macros/src/cfg_eval.rs

@@ -39,50 +39,10 @@ pub(crate) fn cfg_eval(
     let features = Some(features);
     CfgEval(StripUnconfigured { sess, features, config_tokens: true, lint_node_id })
         .configure_annotatable(annotatable)
-        // Since the item itself has already been configured by the `InvocationCollector`,
-        // we know that fold result vector will contain exactly one element.
-        .unwrap()
 }
 
 struct CfgEval<'a>(StripUnconfigured<'a>);
 
-fn flat_map_annotatable(
-    vis: &mut impl MutVisitor,
-    annotatable: Annotatable,
-) -> Option<Annotatable> {
-    match annotatable {
-        Annotatable::Item(item) => vis.flat_map_item(item).pop().map(Annotatable::Item),
-        Annotatable::AssocItem(item, ctxt) => {
-            Some(Annotatable::AssocItem(vis.flat_map_assoc_item(item, ctxt).pop()?, ctxt))
-        }
-        Annotatable::ForeignItem(item) => {
-            vis.flat_map_foreign_item(item).pop().map(Annotatable::ForeignItem)
-        }
-        Annotatable::Stmt(stmt) => {
-            vis.flat_map_stmt(stmt.into_inner()).pop().map(P).map(Annotatable::Stmt)
-        }
-        Annotatable::Expr(mut expr) => {
-            vis.visit_expr(&mut expr);
-            Some(Annotatable::Expr(expr))
-        }
-        Annotatable::Arm(arm) => vis.flat_map_arm(arm).pop().map(Annotatable::Arm),
-        Annotatable::ExprField(field) => {
-            vis.flat_map_expr_field(field).pop().map(Annotatable::ExprField)
-        }
-        Annotatable::PatField(fp) => vis.flat_map_pat_field(fp).pop().map(Annotatable::PatField),
-        Annotatable::GenericParam(param) => {
-            vis.flat_map_generic_param(param).pop().map(Annotatable::GenericParam)
-        }
-        Annotatable::Param(param) => vis.flat_map_param(param).pop().map(Annotatable::Param),
-        Annotatable::FieldDef(sf) => vis.flat_map_field_def(sf).pop().map(Annotatable::FieldDef),
-        Annotatable::Variant(v) => vis.flat_map_variant(v).pop().map(Annotatable::Variant),
-        Annotatable::Crate(mut krate) => {
-            vis.visit_crate(&mut krate);
-            Some(Annotatable::Crate(krate))
-        }
-    }
-}
-
 fn has_cfg_or_cfg_attr(annotatable: &Annotatable) -> bool {
     struct CfgFinder;
 
@@ -106,14 +66,7 @@ fn has_cfg_or_cfg_attr(annotatable: &Annotatable) -> bool {
         Annotatable::ForeignItem(item) => CfgFinder.visit_foreign_item(item),
         Annotatable::Stmt(stmt) => CfgFinder.visit_stmt(stmt),
         Annotatable::Expr(expr) => CfgFinder.visit_expr(expr),
-        Annotatable::Arm(arm) => CfgFinder.visit_arm(arm),
-        Annotatable::ExprField(field) => CfgFinder.visit_expr_field(field),
-        Annotatable::PatField(field) => CfgFinder.visit_pat_field(field),
-        Annotatable::GenericParam(param) => CfgFinder.visit_generic_param(param),
-        Annotatable::Param(param) => CfgFinder.visit_param(param),
-        Annotatable::FieldDef(field) => CfgFinder.visit_field_def(field),
-        Annotatable::Variant(variant) => CfgFinder.visit_variant(variant),
-        Annotatable::Crate(krate) => CfgFinder.visit_crate(krate),
+        _ => unreachable!(),
     };
     res.is_break()
 }
@@ -123,11 +76,11 @@ impl CfgEval<'_> {
         self.0.configure(node)
     }
 
-    fn configure_annotatable(&mut self, mut annotatable: Annotatable) -> Option<Annotatable> {
+    fn configure_annotatable(mut self, annotatable: Annotatable) -> Annotatable {
         // Tokenizing and re-parsing the `Annotatable` can have a significant
         // performance impact, so try to avoid it if possible
         if !has_cfg_or_cfg_attr(&annotatable) {
-            return Some(annotatable);
+            return annotatable;
         }
 
         // The majority of parsed attribute targets will never need to have early cfg-expansion
@@ -140,39 +93,6 @@ impl CfgEval<'_> {
         // the location of `#[cfg]` and `#[cfg_attr]` in the token stream. The tokenization
         // process is lossless, so this process is invisible to proc-macros.
 
-        let parse_annotatable_with: for<'a> fn(&mut Parser<'a>) -> PResult<'a, _> =
-            match annotatable {
-                Annotatable::Item(_) => {
-                    |parser| Ok(Annotatable::Item(parser.parse_item(ForceCollect::Yes)?.unwrap()))
-                }
-                Annotatable::AssocItem(_, AssocCtxt::Trait) => |parser| {
-                    Ok(Annotatable::AssocItem(
-                        parser.parse_trait_item(ForceCollect::Yes)?.unwrap().unwrap(),
-                        AssocCtxt::Trait,
-                    ))
-                },
-                Annotatable::AssocItem(_, AssocCtxt::Impl) => |parser| {
-                    Ok(Annotatable::AssocItem(
-                        parser.parse_impl_item(ForceCollect::Yes)?.unwrap().unwrap(),
-                        AssocCtxt::Impl,
-                    ))
-                },
-                Annotatable::ForeignItem(_) => |parser| {
-                    Ok(Annotatable::ForeignItem(
-                        parser.parse_foreign_item(ForceCollect::Yes)?.unwrap().unwrap(),
-                    ))
-                },
-                Annotatable::Stmt(_) => |parser| {
-                    Ok(Annotatable::Stmt(P(parser
-                        .parse_stmt_without_recovery(false, ForceCollect::Yes)?
-                        .unwrap())))
-                },
-                Annotatable::Expr(_) => {
-                    |parser| Ok(Annotatable::Expr(parser.parse_expr_force_collect()?))
-                }
-                _ => unreachable!(),
-            };
-
         // 'Flatten' all nonterminals (i.e. `TokenKind::Interpolated`)
         // to `None`-delimited groups containing the corresponding tokens. This
         // is normally delayed until the proc-macro server actually needs to
@@ -191,19 +111,56 @@ impl CfgEval<'_> {
         // Re-parse the tokens, setting the `capture_cfg` flag to save extra information
         // to the captured `AttrTokenStream` (specifically, we capture
         // `AttrTokenTree::AttrsTarget` for all occurrences of `#[cfg]` and `#[cfg_attr]`)
+        //
+        // After that we have our re-parsed `AttrTokenStream`, recursively configuring
+        // our attribute target will correctly configure the tokens as well.
         let mut parser = Parser::new(&self.0.sess.psess, orig_tokens, None);
         parser.capture_cfg = true;
-        match parse_annotatable_with(&mut parser) {
-            Ok(a) => annotatable = a,
+        let res: PResult<'_, Annotatable> = try {
+            match annotatable {
+                Annotatable::Item(_) => {
+                    let item = parser.parse_item(ForceCollect::Yes)?.unwrap();
+                    Annotatable::Item(self.flat_map_item(item).pop().unwrap())
+                }
+                Annotatable::AssocItem(_, AssocCtxt::Trait) => {
+                    let item = parser.parse_trait_item(ForceCollect::Yes)?.unwrap().unwrap();
+                    Annotatable::AssocItem(
+                        self.flat_map_assoc_item(item, AssocCtxt::Trait).pop().unwrap(),
+                        AssocCtxt::Trait,
+                    )
+                }
+                Annotatable::AssocItem(_, AssocCtxt::Impl) => {
+                    let item = parser.parse_impl_item(ForceCollect::Yes)?.unwrap().unwrap();
+                    Annotatable::AssocItem(
+                        self.flat_map_assoc_item(item, AssocCtxt::Impl).pop().unwrap(),
+                        AssocCtxt::Impl,
+                    )
+                }
+                Annotatable::ForeignItem(_) => {
+                    let item = parser.parse_foreign_item(ForceCollect::Yes)?.unwrap().unwrap();
+                    Annotatable::ForeignItem(self.flat_map_foreign_item(item).pop().unwrap())
+                }
+                Annotatable::Stmt(_) => {
+                    let stmt =
+                        parser.parse_stmt_without_recovery(false, ForceCollect::Yes)?.unwrap();
+                    Annotatable::Stmt(P(self.flat_map_stmt(stmt).pop().unwrap()))
+                }
+                Annotatable::Expr(_) => {
+                    let mut expr = parser.parse_expr_force_collect()?;
+                    self.visit_expr(&mut expr);
+                    Annotatable::Expr(expr)
+                }
+                _ => unreachable!(),
+            }
+        };
+
+        match res {
+            Ok(ann) => ann,
             Err(err) => {
                 err.emit();
-                return Some(annotatable);
+                annotatable
             }
         }
-
-        // Now that we have our re-parsed `AttrTokenStream`, recursively configuring
-        // our attribute target will correctly configure the tokens as well.
-        flat_map_annotatable(self, annotatable)
     }
 }
 

compiler/rustc_builtin_macros/src/deriving/generic/mod.rs

@@ -680,6 +680,12 @@ impl<'a> TraitDef<'a> {
                     param_clone
                 }
             })
+            .map(|mut param| {
+                // Remove all attributes, because there might be helper attributes
+                // from other macros that will not be valid in the expanded implementation.
+                param.attrs.clear();
+                param
+            })
             .collect();
 
         // and similarly for where clauses

compiler/rustc_codegen_cranelift/src/archive.rs

@@ -1,12 +0,0 @@
-use rustc_codegen_ssa::back::archive::{
-    ArArchiveBuilder, ArchiveBuilder, ArchiveBuilderBuilder, DEFAULT_OBJECT_READER,
-};
-use rustc_session::Session;
-
-pub(crate) struct ArArchiveBuilderBuilder;
-
-impl ArchiveBuilderBuilder for ArArchiveBuilderBuilder {
-    fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder + 'a> {
-        Box::new(ArArchiveBuilder::new(sess, &DEFAULT_OBJECT_READER))
-    }
-}

compiler/rustc_codegen_cranelift/src/lib.rs

@@ -43,7 +43,6 @@ use rustc_codegen_ssa::CodegenResults;
 use rustc_codegen_ssa::back::versioned_llvm_target;
 use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_data_structures::profiling::SelfProfilerRef;
-use rustc_errors::ErrorGuaranteed;
 use rustc_metadata::EncodedMetadata;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_session::Session;
@@ -56,7 +55,6 @@ use crate::prelude::*;
 mod abi;
 mod allocator;
 mod analyze;
-mod archive;
 mod base;
 mod cast;
 mod codegen_i128;
@@ -249,17 +247,6 @@ impl CodegenBackend for CraneliftCodegenBackend {
             self.config.borrow().as_ref().unwrap(),
         )
     }
-
-    fn link(
-        &self,
-        sess: &Session,
-        codegen_results: CodegenResults,
-        outputs: &OutputFilenames,
-    ) -> Result<(), ErrorGuaranteed> {
-        use rustc_codegen_ssa::back::link::link_binary;
-
-        link_binary(sess, &crate::archive::ArArchiveBuilderBuilder, &codegen_results, outputs)
-    }
 }
 
 fn target_triple(sess: &Session) -> target_lexicon::Triple {

compiler/rustc_codegen_gcc/src/archive.rs

@@ -1,25 +0,0 @@
-use std::path::Path;
-
-use rustc_codegen_ssa::back::archive::{
-    ArArchiveBuilder, ArchiveBuilder, ArchiveBuilderBuilder, DEFAULT_OBJECT_READER,
-    ImportLibraryItem,
-};
-use rustc_session::Session;
-
-pub(crate) struct ArArchiveBuilderBuilder;
-
-impl ArchiveBuilderBuilder for ArArchiveBuilderBuilder {
-    fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder + 'a> {
-        Box::new(ArArchiveBuilder::new(sess, &DEFAULT_OBJECT_READER))
-    }
-
-    fn create_dll_import_lib(
-        &self,
-        _sess: &Session,
-        _lib_name: &str,
-        _items: Vec<ImportLibraryItem>,
-        _output_path: &Path,
-    ) {
-        unimplemented!("creating dll imports is not yet supported");
-    }
-}

compiler/rustc_codegen_gcc/src/debuginfo.rs

@@ -52,6 +52,10 @@ impl<'a, 'gcc, 'tcx> DebugInfoBuilderMethods for Builder<'a, 'gcc, 'tcx> {
     fn clear_dbg_loc(&mut self) {
         self.location = None;
     }
+
+    fn get_dbg_loc(&self) -> Option<Self::DILocation> {
+        self.location
+    }
 }
 
 /// Generate the `debug_context` in an MIR Body.

compiler/rustc_codegen_gcc/src/lib.rs

@@ -58,7 +58,6 @@ extern crate rustc_driver;
 
 mod abi;
 mod allocator;
-mod archive;
 mod asm;
 mod attributes;
 mod back;
@@ -103,7 +102,7 @@ use rustc_codegen_ssa::traits::{CodegenBackend, ExtraBackendMethods, WriteBacken
 use rustc_codegen_ssa::{CodegenResults, CompiledModule, ModuleCodegen};
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::sync::IntoDynSyncSend;
-use rustc_errors::{DiagCtxtHandle, ErrorGuaranteed};
+use rustc_errors::DiagCtxtHandle;
 use rustc_metadata::EncodedMetadata;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::ty::TyCtxt;
@@ -261,17 +260,6 @@ impl CodegenBackend for GccCodegenBackend {
             .join(sess)
     }
 
-    fn link(
-        &self,
-        sess: &Session,
-        codegen_results: CodegenResults,
-        outputs: &OutputFilenames,
-    ) -> Result<(), ErrorGuaranteed> {
-        use rustc_codegen_ssa::back::link::link_binary;
-
-        link_binary(sess, &crate::archive::ArArchiveBuilderBuilder, &codegen_results, outputs)
-    }
-
     fn target_features(&self, sess: &Session, allow_unstable: bool) -> Vec<Symbol> {
         target_features(sess, allow_unstable, &self.target_info)
     }

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1574,6 +1574,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
                 cfi::typeid_for_fnabi(self.tcx, fn_abi, options)
             };
             let typeid_metadata = self.cx.typeid_metadata(typeid).unwrap();
+            let dbg_loc = self.get_dbg_loc();
 
             // Test whether the function pointer is associated with the type identifier.
             let cond = self.type_test(llfn, typeid_metadata);
@@ -1582,10 +1583,16 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
             self.cond_br(cond, bb_pass, bb_fail);
 
             self.switch_to_block(bb_fail);
+            if let Some(dbg_loc) = dbg_loc {
+                self.set_dbg_loc(dbg_loc);
+            }
             self.abort();
             self.unreachable();
 
             self.switch_to_block(bb_pass);
+            if let Some(dbg_loc) = dbg_loc {
+                self.set_dbg_loc(dbg_loc);
+            }
         }
     }
 

compiler/rustc_codegen_llvm/src/context.rs

@@ -274,8 +274,12 @@ pub(crate) unsafe fn create_module<'ll>(
         }
     }
 
-    // Control Flow Guard is currently only supported by the MSVC linker on Windows.
-    if sess.target.is_like_msvc {
+    // Control Flow Guard is currently only supported by MSVC and LLVM on Windows.
+    if sess.target.is_like_msvc
+        || (sess.target.options.os == "windows"
+            && sess.target.options.env == "gnu"
+            && sess.target.options.abi == "llvm")
+    {
         match sess.opts.cg.control_flow_guard {
             CFGuard::Disabled => {}
             CFGuard::NoChecks => {

compiler/rustc_codegen_llvm/src/coverageinfo/ffi.rs

@@ -1,5 +1,7 @@
 use rustc_middle::mir::coverage::{CounterId, CovTerm, ExpressionId, SourceRegion};
 
+use crate::coverageinfo::mapgen::LocalFileId;
+
 /// Must match the layout of `LLVMRustCounterKind`.
 #[derive(Copy, Clone, Debug)]
 #[repr(C)]
@@ -137,8 +139,12 @@ pub(crate) struct CoverageSpan {
 }
 
 impl CoverageSpan {
-    pub(crate) fn from_source_region(file_id: u32, code_region: &SourceRegion) -> Self {
-        let &SourceRegion { file_name: _, start_line, start_col, end_line, end_col } = code_region;
+    pub(crate) fn from_source_region(
+        local_file_id: LocalFileId,
+        code_region: &SourceRegion,
+    ) -> Self {
+        let file_id = local_file_id.as_u32();
+        let &SourceRegion { start_line, start_col, end_line, end_col } = code_region;
         // Internally, LLVM uses the high bit of `end_col` to distinguish between
         // code regions and gap regions, so it can't be used by the column number.
         assert!(end_col & (1u32 << 31) == 0, "high bit of `end_col` must be unset: {end_col:#X}");

compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs

@@ -6,7 +6,6 @@ use rustc_middle::mir::coverage::{
     SourceRegion,
 };
 use rustc_middle::ty::Instance;
-use rustc_span::Symbol;
 use tracing::{debug, instrument};
 
 use crate::coverageinfo::ffi::{Counter, CounterExpression, ExprKind};
@@ -180,7 +179,7 @@ impl<'tcx> FunctionCoverageCollector<'tcx> {
 }
 
 pub(crate) struct FunctionCoverage<'tcx> {
-    function_coverage_info: &'tcx FunctionCoverageInfo,
+    pub(crate) function_coverage_info: &'tcx FunctionCoverageInfo,
     is_used: bool,
 
     counters_seen: BitSet<CounterId>,
@@ -199,11 +198,6 @@ impl<'tcx> FunctionCoverage<'tcx> {
         if self.is_used { self.function_coverage_info.function_source_hash } else { 0 }
     }
 
-    /// Returns an iterator over all filenames used by this function's mappings.
-    pub(crate) fn all_file_names(&self) -> impl Iterator<Item = Symbol> + Captures<'_> {
-        self.function_coverage_info.mappings.iter().map(|mapping| mapping.source_region.file_name)
-    }
-
     /// Convert this function's coverage expression data into a form that can be
     /// passed through FFI to LLVM.
     pub(crate) fn counter_expressions(

compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs

@@ -12,8 +12,10 @@ use rustc_index::IndexVec;
 use rustc_middle::mir::coverage::MappingKind;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_middle::{bug, mir};
-use rustc_span::Symbol;
+use rustc_session::RemapFileNameExt;
+use rustc_session::config::RemapPathScopeComponents;
 use rustc_span::def_id::DefIdSet;
+use rustc_span::{Span, Symbol};
 use rustc_target::spec::HasTargetSpec;
 use tracing::debug;
 
@@ -70,8 +72,10 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) {
         .map(|(instance, function_coverage)| (instance, function_coverage.into_finished()))
         .collect::<Vec<_>>();
 
-    let all_file_names =
-        function_coverage_entries.iter().flat_map(|(_, fn_cov)| fn_cov.all_file_names());
+    let all_file_names = function_coverage_entries
+        .iter()
+        .map(|(_, fn_cov)| fn_cov.function_coverage_info.body_span)
+        .map(|span| span_file_name(tcx, span));
     let global_file_table = GlobalFileTable::new(all_file_names);
 
     // Encode all filenames referenced by coverage mappings in this CGU.
@@ -96,7 +100,7 @@ pub(crate) fn finalize(cx: &CodegenCx<'_, '_>) {
         let is_used = function_coverage.is_used();
 
         let coverage_mapping_buffer =
-            encode_mappings_for_function(&global_file_table, &function_coverage);
+            encode_mappings_for_function(tcx, &global_file_table, &function_coverage);
 
         if coverage_mapping_buffer.is_empty() {
             if function_coverage.is_used() {
@@ -164,13 +168,13 @@ impl GlobalFileTable {
         Self { raw_file_table }
     }
 
-    fn global_file_id_for_file_name(&self, file_name: Symbol) -> u32 {
+    fn global_file_id_for_file_name(&self, file_name: Symbol) -> GlobalFileId {
         let raw_id = self.raw_file_table.get_index_of(&file_name).unwrap_or_else(|| {
             bug!("file name not found in prepared global file table: {file_name}");
         });
         // The raw file table doesn't include an entry for the working dir
         // (which has ID 0), so add 1 to get the correct ID.
-        (raw_id + 1) as u32
+        GlobalFileId::from_usize(raw_id + 1)
     }
 
     fn make_filenames_buffer(&self, tcx: TyCtxt<'_>) -> Vec<u8> {
@@ -196,19 +200,27 @@ impl GlobalFileTable {
 }
 
 rustc_index::newtype_index! {
-    struct LocalFileId {}
+    /// An index into the CGU's overall list of file paths. The underlying paths
+    /// will be embedded in the `__llvm_covmap` linker section.
+    struct GlobalFileId {}
+}
+rustc_index::newtype_index! {
+    /// An index into a function's list of global file IDs. That underlying list
+    /// of local-to-global mappings will be embedded in the function's record in
+    /// the `__llvm_covfun` linker section.
+    pub(crate) struct LocalFileId {}
 }
 
 /// Holds a mapping from "local" (per-function) file IDs to "global" (per-CGU)
 /// file IDs.
 #[derive(Default)]
 struct VirtualFileMapping {
-    local_to_global: IndexVec<LocalFileId, u32>,
-    global_to_local: FxIndexMap<u32, LocalFileId>,
+    local_to_global: IndexVec<LocalFileId, GlobalFileId>,
+    global_to_local: FxIndexMap<GlobalFileId, LocalFileId>,
 }
 
 impl VirtualFileMapping {
-    fn local_id_for_global(&mut self, global_file_id: u32) -> LocalFileId {
+    fn local_id_for_global(&mut self, global_file_id: GlobalFileId) -> LocalFileId {
         *self
             .global_to_local
             .entry(global_file_id)
@@ -216,16 +228,26 @@ impl VirtualFileMapping {
     }
 
     fn into_vec(self) -> Vec<u32> {
-        self.local_to_global.raw
+        // This conversion should be optimized away to ~zero overhead.
+        // In any case, it's probably not hot enough to worry about.
+        self.local_to_global.into_iter().map(|global| global.as_u32()).collect()
     }
 }
 
+fn span_file_name(tcx: TyCtxt<'_>, span: Span) -> Symbol {
+    let source_file = tcx.sess.source_map().lookup_source_file(span.lo());
+    let name =
+        source_file.name.for_scope(tcx.sess, RemapPathScopeComponents::MACRO).to_string_lossy();
+    Symbol::intern(&name)
+}
+
 /// Using the expressions and counter regions collected for a single function,
 /// generate the variable-sized payload of its corresponding `__llvm_covfun`
 /// entry. The payload is returned as a vector of bytes.
 ///
 /// Newly-encountered filenames will be added to the global file table.
 fn encode_mappings_for_function(
+    tcx: TyCtxt<'_>,
     global_file_table: &GlobalFileTable,
     function_coverage: &FunctionCoverage<'_>,
 ) -> Vec<u8> {
@@ -242,53 +264,45 @@ fn encode_mappings_for_function(
     let mut mcdc_branch_regions = vec![];
     let mut mcdc_decision_regions = vec![];
 
-    // Group mappings into runs with the same filename, preserving the order
-    // yielded by `FunctionCoverage`.
-    // Prepare file IDs for each filename, and prepare the mapping data so that
-    // we can pass it through FFI to LLVM.
-    for (file_name, counter_regions_for_file) in
-        &counter_regions.group_by(|(_, region)| region.file_name)
-    {
-        // Look up the global file ID for this filename.
-        let global_file_id = global_file_table.global_file_id_for_file_name(file_name);
+    // Currently a function's mappings must all be in the same file as its body span.
+    let file_name = span_file_name(tcx, function_coverage.function_coverage_info.body_span);
 
-        // Associate that global file ID with a local file ID for this function.
-        let local_file_id = virtual_file_mapping.local_id_for_global(global_file_id);
-        debug!("  file id: {local_file_id:?} => global {global_file_id} = '{file_name:?}'");
+    // Look up the global file ID for that filename.
+    let global_file_id = global_file_table.global_file_id_for_file_name(file_name);
 
-        // For each counter/region pair in this function+file, convert it to a
-        // form suitable for FFI.
-        for (mapping_kind, region) in counter_regions_for_file {
-            debug!("Adding counter {mapping_kind:?} to map for {region:?}");
-            let span = ffi::CoverageSpan::from_source_region(local_file_id.as_u32(), region);
-            match mapping_kind {
-                MappingKind::Code(term) => {
-                    code_regions
-                        .push(ffi::CodeRegion { span, counter: ffi::Counter::from_term(term) });
-                }
-                MappingKind::Branch { true_term, false_term } => {
-                    branch_regions.push(ffi::BranchRegion {
-                        span,
-                        true_counter: ffi::Counter::from_term(true_term),
-                        false_counter: ffi::Counter::from_term(false_term),
-                    });
-                }
-                MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => {
-                    mcdc_branch_regions.push(ffi::MCDCBranchRegion {
-                        span,
-                        true_counter: ffi::Counter::from_term(true_term),
-                        false_counter: ffi::Counter::from_term(false_term),
-                        mcdc_branch_params: ffi::mcdc::BranchParameters::from(mcdc_params),
-                    });
-                }
-                MappingKind::MCDCDecision(mcdc_decision_params) => {
-                    mcdc_decision_regions.push(ffi::MCDCDecisionRegion {
-                        span,
-                        mcdc_decision_params: ffi::mcdc::DecisionParameters::from(
-                            mcdc_decision_params,
-                        ),
-                    });
-                }
+    // Associate that global file ID with a local file ID for this function.
+    let local_file_id = virtual_file_mapping.local_id_for_global(global_file_id);
+    debug!("  file id: {local_file_id:?} => {global_file_id:?} = '{file_name:?}'");
+
+    // For each counter/region pair in this function+file, convert it to a
+    // form suitable for FFI.
+    for (mapping_kind, region) in counter_regions {
+        debug!("Adding counter {mapping_kind:?} to map for {region:?}");
+        let span = ffi::CoverageSpan::from_source_region(local_file_id, region);
+        match mapping_kind {
+            MappingKind::Code(term) => {
+                code_regions.push(ffi::CodeRegion { span, counter: ffi::Counter::from_term(term) });
+            }
+            MappingKind::Branch { true_term, false_term } => {
+                branch_regions.push(ffi::BranchRegion {
+                    span,
+                    true_counter: ffi::Counter::from_term(true_term),
+                    false_counter: ffi::Counter::from_term(false_term),
+                });
+            }
+            MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => {
+                mcdc_branch_regions.push(ffi::MCDCBranchRegion {
+                    span,
+                    true_counter: ffi::Counter::from_term(true_term),
+                    false_counter: ffi::Counter::from_term(false_term),
+                    mcdc_branch_params: ffi::mcdc::BranchParameters::from(mcdc_params),
+                });
+            }
+            MappingKind::MCDCDecision(mcdc_decision_params) => {
+                mcdc_decision_regions.push(ffi::MCDCDecisionRegion {
+                    span,
+                    mcdc_decision_params: ffi::mcdc::DecisionParameters::from(mcdc_decision_params),
+                });
             }
         }
     }

compiler/rustc_codegen_llvm/src/debuginfo/mod.rs

@@ -206,6 +206,10 @@ impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> {
         }
     }
 
+    fn get_dbg_loc(&self) -> Option<&'ll DILocation> {
+        unsafe { llvm::LLVMGetCurrentDebugLocation2(self.llbuilder) }
+    }
+
     fn insert_reference_to_gdb_debug_scripts_section_global(&mut self) {
         gdb::insert_reference_to_gdb_debug_scripts_section_global(self)
     }

compiler/rustc_codegen_llvm/src/lib.rs

@@ -382,7 +382,7 @@ impl CodegenBackend for LlvmCodegenBackend {
 
         // Run the linker on any artifacts that resulted from the LLVM run.
         // This should produce either a finished executable or library.
-        link_binary(sess, &LlvmArchiveBuilderBuilder, &codegen_results, outputs)
+        link_binary(sess, &LlvmArchiveBuilderBuilder, codegen_results, outputs)
     }
 }
 

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -1063,6 +1063,7 @@ unsafe extern "C" {
 
     // Metadata
     pub fn LLVMSetCurrentDebugLocation2<'a>(Builder: &Builder<'a>, Loc: *const Metadata);
+    pub fn LLVMGetCurrentDebugLocation2<'a>(Builder: &Builder<'a>) -> Option<&'a Metadata>;
 
     // Terminators
     pub fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value;

compiler/rustc_codegen_llvm/src/llvm_util.rs

@@ -344,15 +344,23 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> {
         })
     {
         if enabled {
+            // Also add all transitively implied features.
             features.extend(sess.target.implied_target_features(std::iter::once(feature)));
         } else {
+            // Remove transitively reverse-implied features.
+
             // We don't care about the order in `features` since the only thing we use it for is the
             // `features.contains` below.
             #[allow(rustc::potential_query_instability)]
             features.retain(|f| {
-                // Keep a feature if it does not imply `feature`. Or, equivalently,
-                // remove the reverse-dependencies of `feature`.
-                !sess.target.implied_target_features(std::iter::once(*f)).contains(&feature)
+                if sess.target.implied_target_features(std::iter::once(*f)).contains(&feature) {
+                    // If `f` if implies `feature`, then `!feature` implies `!f`, so we have to
+                    // remove `f`. (This is the standard logical contraposition principle.)
+                    false
+                } else {
+                    // We can keep `f`.
+                    true
+                }
             });
         }
     }

compiler/rustc_codegen_ssa/src/back/archive.rs

@@ -304,6 +304,14 @@ pub trait ArchiveBuilder {
     fn build(self: Box<Self>, output: &Path) -> bool;
 }
 
+pub struct ArArchiveBuilderBuilder;
+
+impl ArchiveBuilderBuilder for ArArchiveBuilderBuilder {
+    fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box<dyn ArchiveBuilder + 'a> {
+        Box::new(ArArchiveBuilder::new(sess, &DEFAULT_OBJECT_READER))
+    }
+}
+
 #[must_use = "must call build() to finish building the archive"]
 pub struct ArArchiveBuilder<'a> {
     sess: &'a Session,

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -69,7 +69,7 @@ pub fn ensure_removed(dcx: DiagCtxtHandle<'_>, path: &Path) {
 pub fn link_binary(
     sess: &Session,
     archive_builder_builder: &dyn ArchiveBuilderBuilder,
-    codegen_results: &CodegenResults,
+    codegen_results: CodegenResults,
     outputs: &OutputFilenames,
 ) -> Result<(), ErrorGuaranteed> {
     let _timer = sess.timer("link_binary");
@@ -116,7 +116,7 @@ pub fn link_binary(
                     link_rlib(
                         sess,
                         archive_builder_builder,
-                        codegen_results,
+                        &codegen_results,
                         RlibFlavor::Normal,
                         &path,
                     )?
@@ -126,7 +126,7 @@ pub fn link_binary(
                     link_staticlib(
                         sess,
                         archive_builder_builder,
-                        codegen_results,
+                        &codegen_results,
                         &out_filename,
                         &path,
                     )?;
@@ -137,7 +137,7 @@ pub fn link_binary(
                         archive_builder_builder,
                         crate_type,
                         &out_filename,
-                        codegen_results,
+                        &codegen_results,
                         path.as_ref(),
                     )?;
                 }
@@ -1647,7 +1647,7 @@ fn get_object_file_path(sess: &Session, name: &str, self_contained: bool) -> Pat
             return file_path;
         }
     }
-    for search_path in sess.target_filesearch(PathKind::Native).search_paths() {
+    for search_path in sess.target_filesearch().search_paths(PathKind::Native) {
         let file_path = search_path.dir.join(name);
         if file_path.exists() {
             return file_path;

compiler/rustc_codegen_ssa/src/traits/backend.rs

@@ -16,6 +16,8 @@ use rustc_span::symbol::Symbol;
 
 use super::CodegenObject;
 use super::write::WriteBackendMethods;
+use crate::back::archive::ArArchiveBuilderBuilder;
+use crate::back::link::link_binary;
 use crate::back::write::TargetMachineFactoryFn;
 use crate::{CodegenResults, ModuleCodegen};
 
@@ -87,7 +89,9 @@ pub trait CodegenBackend {
         sess: &Session,
         codegen_results: CodegenResults,
         outputs: &OutputFilenames,
-    ) -> Result<(), ErrorGuaranteed>;
+    ) -> Result<(), ErrorGuaranteed> {
+        link_binary(sess, &ArArchiveBuilderBuilder, codegen_results, outputs)
+    }
 
     /// Returns `true` if this backend can be safely called from multiple threads.
     ///

compiler/rustc_codegen_ssa/src/traits/debuginfo.rs

@@ -81,6 +81,7 @@ pub trait DebugInfoBuilderMethods: BackendTypes {
     );
     fn set_dbg_loc(&mut self, dbg_loc: Self::DILocation);
     fn clear_dbg_loc(&mut self);
+    fn get_dbg_loc(&self) -> Option<Self::DILocation>;
     fn insert_reference_to_gdb_debug_scripts_section_global(&mut self);
     fn set_var_name(&mut self, value: Self::Value, name: &str);
 }

compiler/rustc_const_eval/src/check_consts/check.rs

@@ -3,6 +3,7 @@
 use std::assert_matches::assert_matches;
 use std::borrow::Cow;
 use std::mem;
+use std::num::NonZero;
 use std::ops::Deref;
 
 use rustc_attr::{ConstStability, StabilityLevel};
@@ -271,9 +272,18 @@ impl<'mir, 'tcx> Checker<'mir, 'tcx> {
     /// context.
     pub fn check_op_spanned<O: NonConstOp<'tcx>>(&mut self, op: O, span: Span) {
         let gate = match op.status_in_item(self.ccx) {
-            Status::Unstable { gate, safe_to_expose_on_stable, is_function_call }
-                if self.tcx.features().enabled(gate) =>
-            {
+            Status::Unstable {
+                gate,
+                safe_to_expose_on_stable,
+                is_function_call,
+                gate_already_checked,
+            } if gate_already_checked || self.tcx.features().enabled(gate) => {
+                if gate_already_checked {
+                    assert!(
+                        !safe_to_expose_on_stable,
+                        "setting `gate_already_checked` without `safe_to_expose_on_stable` makes no sense"
+                    );
+                }
                 // Generally this is allowed since the feature gate is enabled -- except
                 // if this function wants to be safe-to-expose-on-stable.
                 if !safe_to_expose_on_stable
@@ -709,6 +719,13 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
 
                 // Intrinsics are language primitives, not regular calls, so treat them separately.
                 if let Some(intrinsic) = tcx.intrinsic(callee) {
+                    if !tcx.is_const_fn(callee) {
+                        // Non-const intrinsic.
+                        self.check_op(ops::IntrinsicNonConst { name: intrinsic.name });
+                        // If we allowed this, we're in miri-unleashed mode, so we might
+                        // as well skip the remaining checks.
+                        return;
+                    }
                     // We use `intrinsic.const_stable` to determine if this can be safely exposed to
                     // stable code, rather than `const_stable_indirect`. This is to make
                     // `#[rustc_const_stable_indirect]` an attribute that is always safe to add.
@@ -716,17 +733,12 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                     // fallback body is safe to expose on stable.
                     let is_const_stable = intrinsic.const_stable
                         || (!intrinsic.must_be_overridden
-                            && tcx.is_const_fn(callee)
                             && is_safe_to_expose_on_stable_const_fn(tcx, callee));
                     match tcx.lookup_const_stability(callee) {
                         None => {
-                            // Non-const intrinsic.
-                            self.check_op(ops::IntrinsicNonConst { name: intrinsic.name });
-                        }
-                        Some(ConstStability { feature: None, .. }) => {
-                            // Intrinsic does not need a separate feature gate (we rely on the
-                            // regular stability checker). However, we have to worry about recursive
-                            // const stability.
+                            // This doesn't need a separate const-stability check -- const-stability equals
+                            // regular stability, and regular stability is checked separately.
+                            // However, we *do* have to worry about *recursive* const stability.
                             if !is_const_stable && self.enforce_recursive_const_stability() {
                                 self.dcx().emit_err(errors::UnmarkedIntrinsicExposed {
                                     span: self.span,
@@ -735,14 +747,14 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                             }
                         }
                         Some(ConstStability {
-                            feature: Some(feature),
                             level: StabilityLevel::Unstable { .. },
+                            feature,
                             ..
                         }) => {
                             self.check_op(ops::IntrinsicUnstable {
                                 name: intrinsic.name,
                                 feature,
-                                const_stable: is_const_stable,
+                                const_stable_indirect: is_const_stable,
                             });
                         }
                         Some(ConstStability { level: StabilityLevel::Stable { .. }, .. }) => {
@@ -773,7 +785,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                     Some(ConstStability { level: StabilityLevel::Stable { .. }, .. }) => {
                         // All good.
                     }
-                    None | Some(ConstStability { feature: None, .. }) => {
+                    None => {
                         // This doesn't need a separate const-stability check -- const-stability equals
                         // regular stability, and regular stability is checked separately.
                         // However, we *do* have to worry about *recursive* const stability.
@@ -787,8 +799,8 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                         }
                     }
                     Some(ConstStability {
-                        feature: Some(feature),
-                        level: StabilityLevel::Unstable { implied_by: implied_feature, .. },
+                        level: StabilityLevel::Unstable { implied_by: implied_feature, issue, .. },
+                        feature,
                         ..
                     }) => {
                         // An unstable const fn with a feature gate.
@@ -797,6 +809,12 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
 
                         // We only honor `span.allows_unstable` aka `#[allow_internal_unstable]` if
                         // the callee is safe to expose, to avoid bypassing recursive stability.
+                        // This is not ideal since it means the user sees an error, not the macro
+                        // author, but that's also the case if one forgets to set
+                        // `#[allow_internal_unstable]` in the first place. Note that this cannot be
+                        // integrated in the check below since we want to enforce
+                        // `callee_safe_to_expose_on_stable` even if
+                        // `!self.enforce_recursive_const_stability()`.
                         if (self.span.allows_unstable(feature)
                             || implied_feature.is_some_and(|f| self.span.allows_unstable(f)))
                             && callee_safe_to_expose_on_stable
@@ -810,16 +828,30 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                         // to allow this.
                         let feature_enabled = callee.is_local()
                             || tcx.features().enabled(feature)
-                            || implied_feature.is_some_and(|f| tcx.features().enabled(f));
-                        // We do *not* honor this if we are in the "danger zone": we have to enforce
-                        // recursive const-stability and the callee is not safe-to-expose. In that
-                        // case we need `check_op` to do the check.
-                        let danger_zone = !callee_safe_to_expose_on_stable
-                            && self.enforce_recursive_const_stability();
-                        if danger_zone || !feature_enabled {
+                            || implied_feature.is_some_and(|f| tcx.features().enabled(f))
+                            || {
+                                // When we're compiling the compiler itself we may pull in
+                                // crates from crates.io, but those crates may depend on other
+                                // crates also pulled in from crates.io. We want to ideally be
+                                // able to compile everything without requiring upstream
+                                // modifications, so in the case that this looks like a
+                                // `rustc_private` crate (e.g., a compiler crate) and we also have
+                                // the `-Z force-unstable-if-unmarked` flag present (we're
+                                // compiling a compiler crate), then let this missing feature
+                                // annotation slide.
+                                // This matches what we do in `eval_stability_allow_unstable` for
+                                // regular stability.
+                                feature == sym::rustc_private
+                                    && issue == NonZero::new(27812)
+                                    && self.tcx.sess.opts.unstable_opts.force_unstable_if_unmarked
+                            };
+                        // Even if the feature is enabled, we still need check_op to double-check
+                        // this if the callee is not safe to expose on stable.
+                        if !feature_enabled || !callee_safe_to_expose_on_stable {
                             self.check_op(ops::FnCallUnstable {
                                 def_id: callee,
                                 feature,
+                                feature_enabled,
                                 safe_to_expose_on_stable: callee_safe_to_expose_on_stable,
                             });
                         }

compiler/rustc_const_eval/src/check_consts/mod.rs

@@ -53,10 +53,11 @@ impl<'mir, 'tcx> ConstCx<'mir, 'tcx> {
     }
 
     pub fn enforce_recursive_const_stability(&self) -> bool {
-        // We can skip this if `staged_api` is not enabled, since in such crates
-        // `lookup_const_stability` will always be `None`.
+        // We can skip this if neither `staged_api` nor `-Zforce-unstable-if-unmarked` are enabled,
+        // since in such crates `lookup_const_stability` will always be `None`.
         self.const_kind == Some(hir::ConstContext::ConstFn)
-            && self.tcx.features().staged_api()
+            && (self.tcx.features().staged_api()
+                || self.tcx.sess.opts.unstable_opts.force_unstable_if_unmarked)
             && is_safe_to_expose_on_stable_const_fn(self.tcx, self.def_id().to_def_id())
     }
 
@@ -109,14 +110,15 @@ pub fn is_safe_to_expose_on_stable_const_fn(tcx: TyCtxt<'_>, def_id: DefId) -> b
 
     match tcx.lookup_const_stability(def_id) {
         None => {
-            // Only marked functions can be trusted. Note that this may be a function in a
-            // non-staged-API crate where no recursive checks were done!
-            false
+            // In a `staged_api` crate, we do enforce recursive const stability for all unmarked
+            // functions, so we can trust local functions. But in another crate we don't know which
+            // rules were applied, so we can't trust that.
+            def_id.is_local() && tcx.features().staged_api()
         }
         Some(stab) => {
-            // We consider things safe-to-expose if they are stable, if they don't have any explicit
-            // const stability attribute, or if they are marked as `const_stable_indirect`.
-            stab.is_const_stable() || stab.feature.is_none() || stab.const_stable_indirect
+            // We consider things safe-to-expose if they are stable or if they are marked as
+            // `const_stable_indirect`.
+            stab.is_const_stable() || stab.const_stable_indirect
         }
     }
 }

compiler/rustc_const_eval/src/check_consts/ops.rs

@@ -28,6 +28,9 @@ pub enum Status {
     Unstable {
         /// The feature that must be enabled to use this operation.
         gate: Symbol,
+        /// Whether the feature gate was already checked (because the logic is a bit more
+        /// complicated than just checking a single gate).
+        gate_already_checked: bool,
         /// Whether it is allowed to use this operation from stable `const fn`.
         /// This will usually be `false`.
         safe_to_expose_on_stable: bool,
@@ -82,6 +85,7 @@ impl<'tcx> NonConstOp<'tcx> for ConditionallyConstCall<'tcx> {
         // We use the `const_trait_impl` gate for all conditionally-const calls.
         Status::Unstable {
             gate: sym::const_trait_impl,
+            gate_already_checked: false,
             safe_to_expose_on_stable: false,
             // We don't want the "mark the callee as `#[rustc_const_stable_indirect]`" hint
             is_function_call: false,
@@ -330,6 +334,9 @@ impl<'tcx> NonConstOp<'tcx> for FnCallNonConst<'tcx> {
 pub(crate) struct FnCallUnstable {
     pub def_id: DefId,
     pub feature: Symbol,
+    /// If this is true, then the feature is enabled, but we need to still check if it is safe to
+    /// expose on stable.
+    pub feature_enabled: bool,
     pub safe_to_expose_on_stable: bool,
 }
 
@@ -337,12 +344,14 @@ impl<'tcx> NonConstOp<'tcx> for FnCallUnstable {
     fn status_in_item(&self, _ccx: &ConstCx<'_, 'tcx>) -> Status {
         Status::Unstable {
             gate: self.feature,
+            gate_already_checked: self.feature_enabled,
             safe_to_expose_on_stable: self.safe_to_expose_on_stable,
             is_function_call: true,
         }
     }
 
     fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> Diag<'tcx> {
+        assert!(!self.feature_enabled);
         let mut err = ccx.dcx().create_err(errors::UnstableConstFn {
             span,
             def_path: ccx.tcx.def_path_str(self.def_id),
@@ -376,14 +385,15 @@ impl<'tcx> NonConstOp<'tcx> for IntrinsicNonConst {
 pub(crate) struct IntrinsicUnstable {
     pub name: Symbol,
     pub feature: Symbol,
-    pub const_stable: bool,
+    pub const_stable_indirect: bool,
 }
 
 impl<'tcx> NonConstOp<'tcx> for IntrinsicUnstable {
     fn status_in_item(&self, _ccx: &ConstCx<'_, 'tcx>) -> Status {
         Status::Unstable {
             gate: self.feature,
-            safe_to_expose_on_stable: self.const_stable,
+            gate_already_checked: false,
+            safe_to_expose_on_stable: self.const_stable_indirect,
             // We do *not* want to suggest to mark the intrinsic as `const_stable_indirect`,
             // that's not a trivial change!
             is_function_call: false,
@@ -410,6 +420,7 @@ impl<'tcx> NonConstOp<'tcx> for Coroutine {
         {
             Status::Unstable {
                 gate: sym::const_async_blocks,
+                gate_already_checked: false,
                 safe_to_expose_on_stable: false,
                 is_function_call: false,
             }

compiler/rustc_data_structures/Cargo.toml

@@ -10,11 +10,11 @@ bitflags = "2.4.1"
 either = "1.0"
 elsa = "=1.7.1"
 ena = "0.14.3"
-indexmap = { version = "2.4.0" }
+indexmap = { version = "2.4.0", features = ["rustc-rayon"] }
 jobserver_crate = { version = "0.1.28", package = "jobserver" }
 measureme = "11"
 rustc-hash = "2.0.0"
-rustc-rayon = { version = "0.5.0", optional = true }
+rustc-rayon = "0.5.0"
 rustc-stable-hash = { version = "0.1.0", features = ["nightly"] }
 rustc_arena = { path = "../rustc_arena" }
 rustc_graphviz = { path = "../rustc_graphviz" }
@@ -53,8 +53,3 @@ memmap2 = "0.2.1"
 
 [target.'cfg(not(target_has_atomic = "64"))'.dependencies]
 portable-atomic = "1.5.1"
-
-[features]
-# tidy-alphabetical-start
-rustc_use_parallel_compiler = ["indexmap/rustc-rayon", "dep:rustc-rayon"]
-# tidy-alphabetical-end

compiler/rustc_data_structures/src/lib.rs

@@ -10,7 +10,6 @@
 #![allow(internal_features)]
 #![allow(rustc::default_hash_types)]
 #![allow(rustc::potential_query_instability)]
-#![cfg_attr(not(parallel_compiler), feature(cell_leak))]
 #![deny(unsafe_op_in_unsafe_fn)]
 #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
 #![doc(rust_logo)]

compiler/rustc_data_structures/src/marker.rs

@@ -1,194 +1,162 @@
-cfg_match! {
-    cfg(not(parallel_compiler)) => {
-        pub auto trait DynSend {}
-        pub auto trait DynSync {}
+#[rustc_on_unimplemented(message = "`{Self}` doesn't implement `DynSend`. \
+            Add it to `rustc_data_structures::marker` or use `IntoDynSyncSend` if it's already `Send`")]
+// This is an auto trait for types which can be sent across threads if `sync::is_dyn_thread_safe()`
+// is true. These types can be wrapped in a `FromDyn` to get a `Send` type. Wrapping a
+// `Send` type in `IntoDynSyncSend` will create a `DynSend` type.
+pub unsafe auto trait DynSend {}
 
-        impl<T> DynSend for T {}
-        impl<T> DynSync for T {}
-    }
-    _ => {
-        #[rustc_on_unimplemented(
-            message = "`{Self}` doesn't implement `DynSend`. \
-            Add it to `rustc_data_structures::marker` or use `IntoDynSyncSend` if it's already `Send`"
-        )]
-        // This is an auto trait for types which can be sent across threads if `sync::is_dyn_thread_safe()`
-        // is true. These types can be wrapped in a `FromDyn` to get a `Send` type. Wrapping a
-        // `Send` type in `IntoDynSyncSend` will create a `DynSend` type.
-        pub unsafe auto trait DynSend {}
+#[rustc_on_unimplemented(message = "`{Self}` doesn't implement `DynSync`. \
+            Add it to `rustc_data_structures::marker` or use `IntoDynSyncSend` if it's already `Sync`")]
+// This is an auto trait for types which can be shared across threads if `sync::is_dyn_thread_safe()`
+// is true. These types can be wrapped in a `FromDyn` to get a `Sync` type. Wrapping a
+// `Sync` type in `IntoDynSyncSend` will create a `DynSync` type.
+pub unsafe auto trait DynSync {}
 
-        #[rustc_on_unimplemented(
-            message = "`{Self}` doesn't implement `DynSync`. \
-            Add it to `rustc_data_structures::marker` or use `IntoDynSyncSend` if it's already `Sync`"
-        )]
-        // This is an auto trait for types which can be shared across threads if `sync::is_dyn_thread_safe()`
-        // is true. These types can be wrapped in a `FromDyn` to get a `Sync` type. Wrapping a
-        // `Sync` type in `IntoDynSyncSend` will create a `DynSync` type.
-        pub unsafe auto trait DynSync {}
+// Same with `Sync` and `Send`.
+unsafe impl<T: DynSync + ?Sized> DynSend for &T {}
 
-        // Same with `Sync` and `Send`.
-        unsafe impl<T: DynSync + ?Sized> DynSend for &T {}
-
-        macro_rules! impls_dyn_send_neg {
-            ($([$t1: ty $(where $($generics1: tt)*)?])*) => {
-                $(impl$(<$($generics1)*>)? !DynSend for $t1 {})*
-            };
-        }
-
-        // Consistent with `std`
-        impls_dyn_send_neg!(
-            [std::env::Args]
-            [std::env::ArgsOs]
-            [*const T where T: ?Sized]
-            [*mut T where T: ?Sized]
-            [std::ptr::NonNull<T> where T: ?Sized]
-            [std::rc::Rc<T> where T: ?Sized]
-            [std::rc::Weak<T> where T: ?Sized]
-            [std::sync::MutexGuard<'_, T> where T: ?Sized]
-            [std::sync::RwLockReadGuard<'_, T> where T: ?Sized]
-            [std::sync::RwLockWriteGuard<'_, T> where T: ?Sized]
-            [std::io::StdoutLock<'_>]
-            [std::io::StderrLock<'_>]
-        );
-
-        #[cfg(any(unix, target_os = "hermit", target_os = "wasi", target_os = "solid_asp3"))]
-        // Consistent with `std`, `os_imp::Env` is `!Sync` in these platforms
-        impl !DynSend for std::env::VarsOs {}
-
-        macro_rules! already_send {
-            ($([$ty: ty])*) => {
-                $(unsafe impl DynSend for $ty where $ty: Send {})*
-            };
-        }
-
-        // These structures are already `Send`.
-        already_send!(
-            [std::backtrace::Backtrace]
-            [std::io::Stdout]
-            [std::io::Stderr]
-            [std::io::Error]
-            [std::fs::File]
-            [rustc_arena::DroplessArena]
-            [crate::memmap::Mmap]
-            [crate::profiling::SelfProfiler]
-            [crate::owned_slice::OwnedSlice]
-        );
-
-        macro_rules! impl_dyn_send {
-            ($($($attr: meta)* [$ty: ty where $($generics2: tt)*])*) => {
-                $(unsafe impl<$($generics2)*> DynSend for $ty {})*
-            };
-        }
-
-        impl_dyn_send!(
-            [std::sync::atomic::AtomicPtr<T> where T]
-            [std::sync::Mutex<T> where T: ?Sized+ DynSend]
-            [std::sync::mpsc::Sender<T> where T: DynSend]
-            [std::sync::Arc<T> where T: ?Sized + DynSync + DynSend]
-            [std::sync::LazyLock<T, F> where T: DynSend, F: DynSend]
-            [std::collections::HashSet<K, S> where K: DynSend, S: DynSend]
-            [std::collections::HashMap<K, V, S> where K: DynSend, V: DynSend, S: DynSend]
-            [std::collections::BTreeMap<K, V, A> where K: DynSend, V: DynSend, A: std::alloc::Allocator + Clone + DynSend]
-            [Vec<T, A> where T: DynSend, A: std::alloc::Allocator + DynSend]
-            [Box<T, A> where T: ?Sized + DynSend, A: std::alloc::Allocator + DynSend]
-            [crate::sync::RwLock<T> where T: DynSend]
-            [crate::tagged_ptr::CopyTaggedPtr<P, T, CP> where P: Send + crate::tagged_ptr::Pointer, T: Send + crate::tagged_ptr::Tag, const CP: bool]
-            [rustc_arena::TypedArena<T> where T: DynSend]
-            [indexmap::IndexSet<V, S> where V: DynSend, S: DynSend]
-            [indexmap::IndexMap<K, V, S> where K: DynSend, V: DynSend, S: DynSend]
-            [thin_vec::ThinVec<T> where T: DynSend]
-            [smallvec::SmallVec<A> where A: smallvec::Array + DynSend]
-        );
-
-        macro_rules! impls_dyn_sync_neg {
-            ($([$t1: ty $(where $($generics1: tt)*)?])*) => {
-                $(impl$(<$($generics1)*>)? !DynSync for $t1 {})*
-            };
-        }
-
-        // Consistent with `std`
-        impls_dyn_sync_neg!(
-            [std::env::Args]
-            [std::env::ArgsOs]
-            [*const T where T: ?Sized]
-            [*mut T where T: ?Sized]
-            [std::cell::Cell<T> where T: ?Sized]
-            [std::cell::RefCell<T> where T: ?Sized]
-            [std::cell::UnsafeCell<T> where T: ?Sized]
-            [std::ptr::NonNull<T> where T: ?Sized]
-            [std::rc::Rc<T> where T: ?Sized]
-            [std::rc::Weak<T> where T: ?Sized]
-            [std::cell::OnceCell<T> where T]
-            [std::sync::mpsc::Receiver<T> where T]
-            [std::sync::mpsc::Sender<T> where T]
-        );
-
-        #[cfg(any(unix, target_os = "hermit", target_os = "wasi", target_os = "solid_asp3"))]
-        // Consistent with `std`, `os_imp::Env` is `!Sync` in these platforms
-        impl !DynSync for std::env::VarsOs {}
-
-        macro_rules! already_sync {
-            ($([$ty: ty])*) => {
-                $(unsafe impl DynSync for $ty where $ty: Sync {})*
-            };
-        }
-
-        // These structures are already `Sync`.
-        already_sync!(
-            [std::sync::atomic::AtomicBool]
-            [std::sync::atomic::AtomicUsize]
-            [std::sync::atomic::AtomicU8]
-            [std::sync::atomic::AtomicU32]
-            [std::backtrace::Backtrace]
-            [std::io::Error]
-            [std::fs::File]
-            [jobserver_crate::Client]
-            [crate::memmap::Mmap]
-            [crate::profiling::SelfProfiler]
-            [crate::owned_slice::OwnedSlice]
-        );
-
-        // Use portable AtomicU64 for targets without native 64-bit atomics
-        #[cfg(target_has_atomic = "64")]
-        already_sync!(
-            [std::sync::atomic::AtomicU64]
-        );
-
-        #[cfg(not(target_has_atomic = "64"))]
-        already_sync!(
-            [portable_atomic::AtomicU64]
-        );
-
-        macro_rules! impl_dyn_sync {
-            ($($($attr: meta)* [$ty: ty where $($generics2: tt)*])*) => {
-                $(unsafe impl<$($generics2)*> DynSync for $ty {})*
-            };
-        }
-
-        impl_dyn_sync!(
-            [std::sync::atomic::AtomicPtr<T> where T]
-            [std::sync::OnceLock<T> where T: DynSend + DynSync]
-            [std::sync::Mutex<T> where T: ?Sized + DynSend]
-            [std::sync::Arc<T> where T: ?Sized + DynSync + DynSend]
-            [std::sync::LazyLock<T, F> where T: DynSend + DynSync, F: DynSend]
-            [std::collections::HashSet<K, S> where K: DynSync, S: DynSync]
-            [std::collections::HashMap<K, V, S> where K: DynSync, V: DynSync, S: DynSync]
-            [std::collections::BTreeMap<K, V, A> where K: DynSync, V: DynSync, A: std::alloc::Allocator + Clone + DynSync]
-            [Vec<T, A> where T: DynSync, A: std::alloc::Allocator + DynSync]
-            [Box<T, A> where T: ?Sized + DynSync, A: std::alloc::Allocator + DynSync]
-            [crate::sync::RwLock<T> where T: DynSend + DynSync]
-            [crate::sync::WorkerLocal<T> where T: DynSend]
-            [crate::intern::Interned<'a, T> where 'a, T: DynSync]
-            [crate::tagged_ptr::CopyTaggedPtr<P, T, CP> where P: Sync + crate::tagged_ptr::Pointer, T: Sync + crate::tagged_ptr::Tag, const CP: bool]
-            [parking_lot::lock_api::Mutex<R, T> where R: DynSync, T: ?Sized + DynSend]
-            [parking_lot::lock_api::RwLock<R, T> where R: DynSync, T: ?Sized + DynSend + DynSync]
-            [indexmap::IndexSet<V, S> where V: DynSync, S: DynSync]
-            [indexmap::IndexMap<K, V, S> where K: DynSync, V: DynSync, S: DynSync]
-            [smallvec::SmallVec<A> where A: smallvec::Array + DynSync]
-            [thin_vec::ThinVec<T> where T: DynSync]
-        );
-    }
+macro_rules! impls_dyn_send_neg {
+    ($([$t1: ty $(where $($generics1: tt)*)?])*) => {
+        $(impl$(<$($generics1)*>)? !DynSend for $t1 {})*
+    };
 }
 
+// Consistent with `std`
+impls_dyn_send_neg!(
+    [std::env::Args]
+    [std::env::ArgsOs]
+    [*const T where T: ?Sized]
+    [*mut T where T: ?Sized]
+    [std::ptr::NonNull<T> where T: ?Sized]
+    [std::rc::Rc<T> where T: ?Sized]
+    [std::rc::Weak<T> where T: ?Sized]
+    [std::sync::MutexGuard<'_, T> where T: ?Sized]
+    [std::sync::RwLockReadGuard<'_, T> where T: ?Sized]
+    [std::sync::RwLockWriteGuard<'_, T> where T: ?Sized]
+    [std::io::StdoutLock<'_>]
+    [std::io::StderrLock<'_>]
+);
+
+#[cfg(any(unix, target_os = "hermit", target_os = "wasi", target_os = "solid_asp3"))]
+// Consistent with `std`, `os_imp::Env` is `!Sync` in these platforms
+impl !DynSend for std::env::VarsOs {}
+
+macro_rules! already_send {
+    ($([$ty: ty])*) => {
+        $(unsafe impl DynSend for $ty where $ty: Send {})*
+    };
+}
+
+// These structures are already `Send`.
+already_send!(
+    [std::backtrace::Backtrace][std::io::Stdout][std::io::Stderr][std::io::Error][std::fs::File]
+        [rustc_arena::DroplessArena][crate::memmap::Mmap][crate::profiling::SelfProfiler]
+        [crate::owned_slice::OwnedSlice]
+);
+
+macro_rules! impl_dyn_send {
+    ($($($attr: meta)* [$ty: ty where $($generics2: tt)*])*) => {
+        $(unsafe impl<$($generics2)*> DynSend for $ty {})*
+    };
+}
+
+impl_dyn_send!(
+    [std::sync::atomic::AtomicPtr<T> where T]
+    [std::sync::Mutex<T> where T: ?Sized+ DynSend]
+    [std::sync::mpsc::Sender<T> where T: DynSend]
+    [std::sync::Arc<T> where T: ?Sized + DynSync + DynSend]
+    [std::sync::LazyLock<T, F> where T: DynSend, F: DynSend]
+    [std::collections::HashSet<K, S> where K: DynSend, S: DynSend]
+    [std::collections::HashMap<K, V, S> where K: DynSend, V: DynSend, S: DynSend]
+    [std::collections::BTreeMap<K, V, A> where K: DynSend, V: DynSend, A: std::alloc::Allocator + Clone + DynSend]
+    [Vec<T, A> where T: DynSend, A: std::alloc::Allocator + DynSend]
+    [Box<T, A> where T: ?Sized + DynSend, A: std::alloc::Allocator + DynSend]
+    [crate::sync::RwLock<T> where T: DynSend]
+    [crate::tagged_ptr::CopyTaggedPtr<P, T, CP> where P: Send + crate::tagged_ptr::Pointer, T: Send + crate::tagged_ptr::Tag, const CP: bool]
+    [rustc_arena::TypedArena<T> where T: DynSend]
+    [indexmap::IndexSet<V, S> where V: DynSend, S: DynSend]
+    [indexmap::IndexMap<K, V, S> where K: DynSend, V: DynSend, S: DynSend]
+    [thin_vec::ThinVec<T> where T: DynSend]
+    [smallvec::SmallVec<A> where A: smallvec::Array + DynSend]
+);
+
+macro_rules! impls_dyn_sync_neg {
+    ($([$t1: ty $(where $($generics1: tt)*)?])*) => {
+        $(impl$(<$($generics1)*>)? !DynSync for $t1 {})*
+    };
+}
+
+// Consistent with `std`
+impls_dyn_sync_neg!(
+    [std::env::Args]
+    [std::env::ArgsOs]
+    [*const T where T: ?Sized]
+    [*mut T where T: ?Sized]
+    [std::cell::Cell<T> where T: ?Sized]
+    [std::cell::RefCell<T> where T: ?Sized]
+    [std::cell::UnsafeCell<T> where T: ?Sized]
+    [std::ptr::NonNull<T> where T: ?Sized]
+    [std::rc::Rc<T> where T: ?Sized]
+    [std::rc::Weak<T> where T: ?Sized]
+    [std::cell::OnceCell<T> where T]
+    [std::sync::mpsc::Receiver<T> where T]
+    [std::sync::mpsc::Sender<T> where T]
+);
+
+#[cfg(any(unix, target_os = "hermit", target_os = "wasi", target_os = "solid_asp3"))]
+// Consistent with `std`, `os_imp::Env` is `!Sync` in these platforms
+impl !DynSync for std::env::VarsOs {}
+
+macro_rules! already_sync {
+    ($([$ty: ty])*) => {
+        $(unsafe impl DynSync for $ty where $ty: Sync {})*
+    };
+}
+
+// These structures are already `Sync`.
+already_sync!(
+    [std::sync::atomic::AtomicBool][std::sync::atomic::AtomicUsize][std::sync::atomic::AtomicU8]
+        [std::sync::atomic::AtomicU32][std::backtrace::Backtrace][std::io::Error][std::fs::File]
+        [jobserver_crate::Client][crate::memmap::Mmap][crate::profiling::SelfProfiler]
+        [crate::owned_slice::OwnedSlice]
+);
+
+// Use portable AtomicU64 for targets without native 64-bit atomics
+#[cfg(target_has_atomic = "64")]
+already_sync!([std::sync::atomic::AtomicU64]);
+
+#[cfg(not(target_has_atomic = "64"))]
+already_sync!([portable_atomic::AtomicU64]);
+
+macro_rules! impl_dyn_sync {
+    ($($($attr: meta)* [$ty: ty where $($generics2: tt)*])*) => {
+        $(unsafe impl<$($generics2)*> DynSync for $ty {})*
+    };
+}
+
+impl_dyn_sync!(
+    [std::sync::atomic::AtomicPtr<T> where T]
+    [std::sync::OnceLock<T> where T: DynSend + DynSync]
+    [std::sync::Mutex<T> where T: ?Sized + DynSend]
+    [std::sync::Arc<T> where T: ?Sized + DynSync + DynSend]
+    [std::sync::LazyLock<T, F> where T: DynSend + DynSync, F: DynSend]
+    [std::collections::HashSet<K, S> where K: DynSync, S: DynSync]
+    [std::collections::HashMap<K, V, S> where K: DynSync, V: DynSync, S: DynSync]
+    [std::collections::BTreeMap<K, V, A> where K: DynSync, V: DynSync, A: std::alloc::Allocator + Clone + DynSync]
+    [Vec<T, A> where T: DynSync, A: std::alloc::Allocator + DynSync]
+    [Box<T, A> where T: ?Sized + DynSync, A: std::alloc::Allocator + DynSync]
+    [crate::sync::RwLock<T> where T: DynSend + DynSync]
+    [crate::sync::WorkerLocal<T> where T: DynSend]
+    [crate::intern::Interned<'a, T> where 'a, T: DynSync]
+    [crate::tagged_ptr::CopyTaggedPtr<P, T, CP> where P: Sync + crate::tagged_ptr::Pointer, T: Sync + crate::tagged_ptr::Tag, const CP: bool]
+    [parking_lot::lock_api::Mutex<R, T> where R: DynSync, T: ?Sized + DynSend]
+    [parking_lot::lock_api::RwLock<R, T> where R: DynSync, T: ?Sized + DynSend + DynSync]
+    [indexmap::IndexSet<V, S> where V: DynSync, S: DynSync]
+    [indexmap::IndexMap<K, V, S> where K: DynSync, V: DynSync, S: DynSync]
+    [smallvec::SmallVec<A> where A: smallvec::Array + DynSync]
+    [thin_vec::ThinVec<T> where T: DynSync]
+);
+
 pub fn assert_dyn_sync<T: ?Sized + DynSync>() {}
 pub fn assert_dyn_send<T: ?Sized + DynSend>() {}
 pub fn assert_dyn_send_val<T: ?Sized + DynSend>(_t: &T) {}
@@ -203,7 +171,6 @@ impl<T> FromDyn<T> {
         // Check that `sync::is_dyn_thread_safe()` is true on creation so we can
         // implement `Send` and `Sync` for this structure when `T`
         // implements `DynSend` and `DynSync` respectively.
-        #[cfg(parallel_compiler)]
         assert!(crate::sync::is_dyn_thread_safe());
         FromDyn(val)
     }
@@ -215,11 +182,9 @@ impl<T> FromDyn<T> {
 }
 
 // `FromDyn` is `Send` if `T` is `DynSend`, since it ensures that sync::is_dyn_thread_safe() is true.
-#[cfg(parallel_compiler)]
 unsafe impl<T: DynSend> Send for FromDyn<T> {}
 
 // `FromDyn` is `Sync` if `T` is `DynSync`, since it ensures that sync::is_dyn_thread_safe() is true.
-#[cfg(parallel_compiler)]
 unsafe impl<T: DynSync> Sync for FromDyn<T> {}
 
 impl<T> std::ops::Deref for FromDyn<T> {
@@ -237,9 +202,7 @@ impl<T> std::ops::Deref for FromDyn<T> {
 #[derive(Copy, Clone)]
 pub struct IntoDynSyncSend<T: ?Sized>(pub T);
 
-#[cfg(parallel_compiler)]
 unsafe impl<T: ?Sized + Send> DynSend for IntoDynSyncSend<T> {}
-#[cfg(parallel_compiler)]
 unsafe impl<T: ?Sized + Sync> DynSync for IntoDynSyncSend<T> {}
 
 impl<T> std::ops::Deref for IntoDynSyncSend<T> {

compiler/rustc_data_structures/src/owned_slice.rs

@@ -139,11 +139,9 @@ impl Borrow<[u8]> for OwnedSlice {
 }
 
 // Safety: `OwnedSlice` is conceptually `(&'self.1 [u8], Arc<dyn Send + Sync>)`, which is `Send`
-#[cfg(parallel_compiler)]
 unsafe impl sync::Send for OwnedSlice {}
 
 // Safety: `OwnedSlice` is conceptually `(&'self.1 [u8], Arc<dyn Send + Sync>)`, which is `Sync`
-#[cfg(parallel_compiler)]
 unsafe impl sync::Sync for OwnedSlice {}
 
 #[cfg(test)]

compiler/rustc_data_structures/src/sharded.rs

@@ -3,27 +3,22 @@ use std::collections::hash_map::RawEntryMut;
 use std::hash::{Hash, Hasher};
 use std::{iter, mem};
 
-#[cfg(parallel_compiler)]
 use either::Either;
 
 use crate::fx::{FxHashMap, FxHasher};
-#[cfg(parallel_compiler)]
-use crate::sync::{CacheAligned, is_dyn_thread_safe};
-use crate::sync::{Lock, LockGuard, Mode};
+use crate::sync::{CacheAligned, Lock, LockGuard, Mode, is_dyn_thread_safe};
 
 // 32 shards is sufficient to reduce contention on an 8-core Ryzen 7 1700,
 // but this should be tested on higher core count CPUs. How the `Sharded` type gets used
 // may also affect the ideal number of shards.
 const SHARD_BITS: usize = 5;
 
-#[cfg(parallel_compiler)]
 const SHARDS: usize = 1 << SHARD_BITS;
 
 /// An array of cache-line aligned inner locked structures with convenience methods.
 /// A single field is used when the compiler uses only one thread.
 pub enum Sharded<T> {
     Single(Lock<T>),
-    #[cfg(parallel_compiler)]
     Shards(Box<[CacheAligned<Lock<T>>; SHARDS]>),
 }
 
@@ -37,7 +32,6 @@ impl<T: Default> Default for Sharded<T> {
 impl<T> Sharded<T> {
     #[inline]
     pub fn new(mut value: impl FnMut() -> T) -> Self {
-        #[cfg(parallel_compiler)]
         if is_dyn_thread_safe() {
             return Sharded::Shards(Box::new(
                 [(); SHARDS].map(|()| CacheAligned(Lock::new(value()))),
@@ -52,7 +46,6 @@ impl<T> Sharded<T> {
     pub fn get_shard_by_value<K: Hash + ?Sized>(&self, _val: &K) -> &Lock<T> {
         match self {
             Self::Single(single) => single,
-            #[cfg(parallel_compiler)]
             Self::Shards(..) => self.get_shard_by_hash(make_hash(_val)),
         }
     }
@@ -66,7 +59,6 @@ impl<T> Sharded<T> {
     pub fn get_shard_by_index(&self, _i: usize) -> &Lock<T> {
         match self {
             Self::Single(single) => single,
-            #[cfg(parallel_compiler)]
             Self::Shards(shards) => {
                 // SAFETY: The index gets ANDed with the shard mask, ensuring it is always inbounds.
                 unsafe { &shards.get_unchecked(_i & (SHARDS - 1)).0 }
@@ -87,7 +79,6 @@ impl<T> Sharded<T> {
                 // `might_be_dyn_thread_safe` was also false.
                 unsafe { single.lock_assume(Mode::NoSync) }
             }
-            #[cfg(parallel_compiler)]
             Self::Shards(..) => self.lock_shard_by_hash(make_hash(_val)),
         }
     }
@@ -110,7 +101,6 @@ impl<T> Sharded<T> {
                 // `might_be_dyn_thread_safe` was also false.
                 unsafe { single.lock_assume(Mode::NoSync) }
             }
-            #[cfg(parallel_compiler)]
             Self::Shards(shards) => {
                 // Synchronization is enabled so use the `lock_assume_sync` method optimized
                 // for that case.
@@ -127,11 +117,7 @@ impl<T> Sharded<T> {
     #[inline]
     pub fn lock_shards(&self) -> impl Iterator<Item = LockGuard<'_, T>> {
         match self {
-            #[cfg(not(parallel_compiler))]
-            Self::Single(single) => iter::once(single.lock()),
-            #[cfg(parallel_compiler)]
             Self::Single(single) => Either::Left(iter::once(single.lock())),
-            #[cfg(parallel_compiler)]
             Self::Shards(shards) => Either::Right(shards.iter().map(|shard| shard.0.lock())),
         }
     }
@@ -139,11 +125,7 @@ impl<T> Sharded<T> {
     #[inline]
     pub fn try_lock_shards(&self) -> impl Iterator<Item = Option<LockGuard<'_, T>>> {
         match self {
-            #[cfg(not(parallel_compiler))]
-            Self::Single(single) => iter::once(single.try_lock()),
-            #[cfg(parallel_compiler)]
             Self::Single(single) => Either::Left(iter::once(single.try_lock())),
-            #[cfg(parallel_compiler)]
             Self::Shards(shards) => Either::Right(shards.iter().map(|shard| shard.0.try_lock())),
         }
     }
@@ -151,7 +133,6 @@ impl<T> Sharded<T> {
 
 #[inline]
 pub fn shards() -> usize {
-    #[cfg(parallel_compiler)]
     if is_dyn_thread_safe() {
         return SHARDS;
     }

compiler/rustc_data_structures/src/sync.rs

@@ -54,9 +54,7 @@ mod worker_local;
 pub use worker_local::{Registry, WorkerLocal};
 
 mod parallel;
-#[cfg(parallel_compiler)]
-pub use parallel::scope;
-pub use parallel::{join, par_for_each_in, par_map, parallel_guard, try_par_for_each_in};
+pub use parallel::{join, par_for_each_in, par_map, parallel_guard, scope, try_par_for_each_in};
 pub use vec::{AppendOnlyIndexVec, AppendOnlyVec};
 
 mod vec;
@@ -104,226 +102,66 @@ mod mode {
     }
 }
 
+// FIXME(parallel_compiler): Get rid of these aliases across the compiler.
+
+pub use std::marker::{Send, Sync};
+// Use portable AtomicU64 for targets without native 64-bit atomics
+#[cfg(target_has_atomic = "64")]
+pub use std::sync::atomic::AtomicU64;
+pub use std::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize};
+pub use std::sync::{Arc as Lrc, OnceLock, Weak};
+
 pub use mode::{is_dyn_thread_safe, set_dyn_thread_safe_mode};
+pub use parking_lot::{
+    MappedMutexGuard as MappedLockGuard, MappedRwLockReadGuard as MappedReadGuard,
+    MappedRwLockWriteGuard as MappedWriteGuard, RwLockReadGuard as ReadGuard,
+    RwLockWriteGuard as WriteGuard,
+};
+#[cfg(not(target_has_atomic = "64"))]
+pub use portable_atomic::AtomicU64;
 
-cfg_match! {
-    cfg(not(parallel_compiler)) => {
-        use std::ops::Add;
-        use std::cell::Cell;
-        use std::sync::atomic::Ordering;
+pub type LRef<'a, T> = &'a T;
 
-        pub unsafe auto trait Send {}
-        pub unsafe auto trait Sync {}
+#[derive(Debug, Default)]
+pub struct MTLock<T>(Lock<T>);
 
-        unsafe impl<T> Send for T {}
-        unsafe impl<T> Sync for T {}
-
-        /// This is a single threaded variant of `AtomicU64`, `AtomicUsize`, etc.
-        /// It has explicit ordering arguments and is only intended for use with
-        /// the native atomic types.
-        /// You should use this type through the `AtomicU64`, `AtomicUsize`, etc, type aliases
-        /// as it's not intended to be used separately.
-        #[derive(Debug, Default)]
-        pub struct Atomic<T: Copy>(Cell<T>);
-
-        impl<T: Copy> Atomic<T> {
-            #[inline]
-            pub fn new(v: T) -> Self {
-                Atomic(Cell::new(v))
-            }
-
-            #[inline]
-            pub fn into_inner(self) -> T {
-                self.0.into_inner()
-            }
-
-            #[inline]
-            pub fn load(&self, _: Ordering) -> T {
-                self.0.get()
-            }
-
-            #[inline]
-            pub fn store(&self, val: T, _: Ordering) {
-                self.0.set(val)
-            }
-
-            #[inline]
-            pub fn swap(&self, val: T, _: Ordering) -> T {
-                self.0.replace(val)
-            }
-        }
-
-        impl Atomic<bool> {
-            pub fn fetch_or(&self, val: bool, _: Ordering) -> bool {
-                let old = self.0.get();
-                self.0.set(val | old);
-                old
-            }
-            pub fn fetch_and(&self, val: bool, _: Ordering) -> bool {
-                let old = self.0.get();
-                self.0.set(val & old);
-                old
-            }
-        }
-
-        impl<T: Copy + PartialEq> Atomic<T> {
-            #[inline]
-            pub fn compare_exchange(&self,
-                                    current: T,
-                                    new: T,
-                                    _: Ordering,
-                                    _: Ordering)
-                                    -> Result<T, T> {
-                let read = self.0.get();
-                if read == current {
-                    self.0.set(new);
-                    Ok(read)
-                } else {
-                    Err(read)
-                }
-            }
-        }
-
-        impl<T: Add<Output=T> + Copy> Atomic<T> {
-            #[inline]
-            pub fn fetch_add(&self, val: T, _: Ordering) -> T {
-                let old = self.0.get();
-                self.0.set(old + val);
-                old
-            }
-        }
-
-        pub type AtomicUsize = Atomic<usize>;
-        pub type AtomicBool = Atomic<bool>;
-        pub type AtomicU32 = Atomic<u32>;
-        pub type AtomicU64 = Atomic<u64>;
-
-        pub use std::rc::Rc as Lrc;
-        pub use std::rc::Weak as Weak;
-        #[doc(no_inline)]
-        pub use std::cell::Ref as ReadGuard;
-        #[doc(no_inline)]
-        pub use std::cell::Ref as MappedReadGuard;
-        #[doc(no_inline)]
-        pub use std::cell::RefMut as WriteGuard;
-        #[doc(no_inline)]
-        pub use std::cell::RefMut as MappedWriteGuard;
-        #[doc(no_inline)]
-        pub use std::cell::RefMut as MappedLockGuard;
-
-        pub use std::cell::OnceCell as OnceLock;
-
-        use std::cell::RefCell as InnerRwLock;
-
-        pub type LRef<'a, T> = &'a mut T;
-
-        #[derive(Debug, Default)]
-        pub struct MTLock<T>(T);
-
-        impl<T> MTLock<T> {
-            #[inline(always)]
-            pub fn new(inner: T) -> Self {
-                MTLock(inner)
-            }
-
-            #[inline(always)]
-            pub fn into_inner(self) -> T {
-                self.0
-            }
-
-            #[inline(always)]
-            pub fn get_mut(&mut self) -> &mut T {
-                &mut self.0
-            }
-
-            #[inline(always)]
-            pub fn lock(&self) -> &T {
-                &self.0
-            }
-
-            #[inline(always)]
-            pub fn lock_mut(&mut self) -> &mut T {
-                &mut self.0
-            }
-        }
-
-        // FIXME: Probably a bad idea (in the threaded case)
-        impl<T: Clone> Clone for MTLock<T> {
-            #[inline]
-            fn clone(&self) -> Self {
-                MTLock(self.0.clone())
-            }
-        }
+impl<T> MTLock<T> {
+    #[inline(always)]
+    pub fn new(inner: T) -> Self {
+        MTLock(Lock::new(inner))
     }
-    _ => {
-        pub use std::marker::Send as Send;
-        pub use std::marker::Sync as Sync;
 
-        pub use parking_lot::RwLockReadGuard as ReadGuard;
-        pub use parking_lot::MappedRwLockReadGuard as MappedReadGuard;
-        pub use parking_lot::RwLockWriteGuard as WriteGuard;
-        pub use parking_lot::MappedRwLockWriteGuard as MappedWriteGuard;
+    #[inline(always)]
+    pub fn into_inner(self) -> T {
+        self.0.into_inner()
+    }
 
-        pub use parking_lot::MappedMutexGuard as MappedLockGuard;
+    #[inline(always)]
+    pub fn get_mut(&mut self) -> &mut T {
+        self.0.get_mut()
+    }
 
-        pub use std::sync::OnceLock;
+    #[inline(always)]
+    pub fn lock(&self) -> LockGuard<'_, T> {
+        self.0.lock()
+    }
 
-        pub use std::sync::atomic::{AtomicBool, AtomicUsize, AtomicU32};
-
-        // Use portable AtomicU64 for targets without native 64-bit atomics
-        #[cfg(target_has_atomic = "64")]
-        pub use std::sync::atomic::AtomicU64;
-
-        #[cfg(not(target_has_atomic = "64"))]
-        pub use portable_atomic::AtomicU64;
-
-        pub use std::sync::Arc as Lrc;
-        pub use std::sync::Weak as Weak;
-
-        pub type LRef<'a, T> = &'a T;
-
-        #[derive(Debug, Default)]
-        pub struct MTLock<T>(Lock<T>);
-
-        impl<T> MTLock<T> {
-            #[inline(always)]
-            pub fn new(inner: T) -> Self {
-                MTLock(Lock::new(inner))
-            }
-
-            #[inline(always)]
-            pub fn into_inner(self) -> T {
-                self.0.into_inner()
-            }
-
-            #[inline(always)]
-            pub fn get_mut(&mut self) -> &mut T {
-                self.0.get_mut()
-            }
-
-            #[inline(always)]
-            pub fn lock(&self) -> LockGuard<'_, T> {
-                self.0.lock()
-            }
-
-            #[inline(always)]
-            pub fn lock_mut(&self) -> LockGuard<'_, T> {
-                self.lock()
-            }
-        }
-
-        use parking_lot::RwLock as InnerRwLock;
-
-        /// This makes locks panic if they are already held.
-        /// It is only useful when you are running in a single thread
-        const ERROR_CHECKING: bool = false;
+    #[inline(always)]
+    pub fn lock_mut(&self) -> LockGuard<'_, T> {
+        self.lock()
     }
 }
 
+use parking_lot::RwLock as InnerRwLock;
+
+/// This makes locks panic if they are already held.
+/// It is only useful when you are running in a single thread
+const ERROR_CHECKING: bool = false;
+
 pub type MTLockRef<'a, T> = LRef<'a, MTLock<T>>;
 
 #[derive(Default)]
-#[cfg_attr(parallel_compiler, repr(align(64)))]
+#[repr(align(64))]
 pub struct CacheAligned<T>(pub T);
 
 pub trait HashMapExt<K, V> {
@@ -357,14 +195,6 @@ impl<T> RwLock<T> {
         self.0.get_mut()
     }
 
-    #[cfg(not(parallel_compiler))]
-    #[inline(always)]
-    #[track_caller]
-    pub fn read(&self) -> ReadGuard<'_, T> {
-        self.0.borrow()
-    }
-
-    #[cfg(parallel_compiler)]
     #[inline(always)]
     pub fn read(&self) -> ReadGuard<'_, T> {
         if ERROR_CHECKING {
@@ -380,26 +210,11 @@ impl<T> RwLock<T> {
         f(&*self.read())
     }
 
-    #[cfg(not(parallel_compiler))]
-    #[inline(always)]
-    pub fn try_write(&self) -> Result<WriteGuard<'_, T>, ()> {
-        self.0.try_borrow_mut().map_err(|_| ())
-    }
-
-    #[cfg(parallel_compiler)]
     #[inline(always)]
     pub fn try_write(&self) -> Result<WriteGuard<'_, T>, ()> {
         self.0.try_write().ok_or(())
     }
 
-    #[cfg(not(parallel_compiler))]
-    #[inline(always)]
-    #[track_caller]
-    pub fn write(&self) -> WriteGuard<'_, T> {
-        self.0.borrow_mut()
-    }
-
-    #[cfg(parallel_compiler)]
     #[inline(always)]
     pub fn write(&self) -> WriteGuard<'_, T> {
         if ERROR_CHECKING {
@@ -427,13 +242,6 @@ impl<T> RwLock<T> {
         self.write()
     }
 
-    #[cfg(not(parallel_compiler))]
-    #[inline(always)]
-    pub fn leak(&self) -> &T {
-        ReadGuard::leak(self.read())
-    }
-
-    #[cfg(parallel_compiler)]
     #[inline(always)]
     pub fn leak(&self) -> &T {
         let guard = self.read();

compiler/rustc_data_structures/src/sync/freeze.rs

@@ -5,9 +5,7 @@ use std::ops::{Deref, DerefMut};
 use std::ptr::NonNull;
 use std::sync::atomic::Ordering;
 
-use crate::sync::{AtomicBool, ReadGuard, RwLock, WriteGuard};
-#[cfg(parallel_compiler)]
-use crate::sync::{DynSend, DynSync};
+use crate::sync::{AtomicBool, DynSend, DynSync, ReadGuard, RwLock, WriteGuard};
 
 /// A type which allows mutation using a lock until
 /// the value is frozen and can be accessed lock-free.
@@ -22,7 +20,6 @@ pub struct FreezeLock<T> {
     lock: RwLock<()>,
 }
 
-#[cfg(parallel_compiler)]
 unsafe impl<T: DynSync + DynSend> DynSync for FreezeLock<T> {}
 
 impl<T> FreezeLock<T> {

compiler/rustc_data_structures/src/sync/lock.rs

@@ -1,236 +1,177 @@
 //! This module implements a lock which only uses synchronization if `might_be_dyn_thread_safe` is true.
 //! It implements `DynSend` and `DynSync` instead of the typical `Send` and `Sync` traits.
-//!
-//! When `cfg(parallel_compiler)` is not set, the lock is instead a wrapper around `RefCell`.
 
 #![allow(dead_code)]
 
 use std::fmt;
 
-#[cfg(parallel_compiler)]
-pub use maybe_sync::*;
-#[cfg(not(parallel_compiler))]
-pub use no_sync::*;
-
 #[derive(Clone, Copy, PartialEq)]
 pub enum Mode {
     NoSync,
     Sync,
 }
 
-mod maybe_sync {
-    use std::cell::{Cell, UnsafeCell};
-    use std::intrinsics::unlikely;
-    use std::marker::PhantomData;
-    use std::mem::ManuallyDrop;
-    use std::ops::{Deref, DerefMut};
+use std::cell::{Cell, UnsafeCell};
+use std::intrinsics::unlikely;
+use std::marker::PhantomData;
+use std::mem::ManuallyDrop;
+use std::ops::{Deref, DerefMut};
 
-    use parking_lot::RawMutex;
-    use parking_lot::lock_api::RawMutex as _;
+use parking_lot::RawMutex;
+use parking_lot::lock_api::RawMutex as _;
 
-    use super::Mode;
-    use crate::sync::mode;
-    #[cfg(parallel_compiler)]
-    use crate::sync::{DynSend, DynSync};
+use crate::sync::{DynSend, DynSync, mode};
 
-    /// A guard holding mutable access to a `Lock` which is in a locked state.
-    #[must_use = "if unused the Lock will immediately unlock"]
-    pub struct LockGuard<'a, T> {
-        lock: &'a Lock<T>,
-        marker: PhantomData<&'a mut T>,
+/// A guard holding mutable access to a `Lock` which is in a locked state.
+#[must_use = "if unused the Lock will immediately unlock"]
+pub struct LockGuard<'a, T> {
+    lock: &'a Lock<T>,
+    marker: PhantomData<&'a mut T>,
 
-        /// The synchronization mode of the lock. This is explicitly passed to let LLVM relate it
-        /// to the original lock operation.
-        mode: Mode,
+    /// The synchronization mode of the lock. This is explicitly passed to let LLVM relate it
+    /// to the original lock operation.
+    mode: Mode,
+}
+
+impl<'a, T: 'a> Deref for LockGuard<'a, T> {
+    type Target = T;
+    #[inline]
+    fn deref(&self) -> &T {
+        // SAFETY: We have shared access to the mutable access owned by this type,
+        // so we can give out a shared reference.
+        unsafe { &*self.lock.data.get() }
     }
+}
 
-    impl<'a, T: 'a> Deref for LockGuard<'a, T> {
-        type Target = T;
-        #[inline]
-        fn deref(&self) -> &T {
-            // SAFETY: We have shared access to the mutable access owned by this type,
-            // so we can give out a shared reference.
-            unsafe { &*self.lock.data.get() }
-        }
+impl<'a, T: 'a> DerefMut for LockGuard<'a, T> {
+    #[inline]
+    fn deref_mut(&mut self) -> &mut T {
+        // SAFETY: We have mutable access to the data so we can give out a mutable reference.
+        unsafe { &mut *self.lock.data.get() }
     }
+}
 
-    impl<'a, T: 'a> DerefMut for LockGuard<'a, T> {
-        #[inline]
-        fn deref_mut(&mut self) -> &mut T {
-            // SAFETY: We have mutable access to the data so we can give out a mutable reference.
-            unsafe { &mut *self.lock.data.get() }
-        }
-    }
-
-    impl<'a, T: 'a> Drop for LockGuard<'a, T> {
-        #[inline]
-        fn drop(&mut self) {
-            // SAFETY (union access): We get `self.mode` from the lock operation so it is consistent
-            // with the `lock.mode` state. This means we access the right union fields.
-            match self.mode {
-                Mode::NoSync => {
-                    let cell = unsafe { &self.lock.mode_union.no_sync };
-                    debug_assert!(cell.get());
-                    cell.set(false);
-                }
-                // SAFETY (unlock): We know that the lock is locked as this type is a proof of that.
-                Mode::Sync => unsafe { self.lock.mode_union.sync.unlock() },
+impl<'a, T: 'a> Drop for LockGuard<'a, T> {
+    #[inline]
+    fn drop(&mut self) {
+        // SAFETY (union access): We get `self.mode` from the lock operation so it is consistent
+        // with the `lock.mode` state. This means we access the right union fields.
+        match self.mode {
+            Mode::NoSync => {
+                let cell = unsafe { &self.lock.mode_union.no_sync };
+                debug_assert!(cell.get());
+                cell.set(false);
             }
+            // SAFETY (unlock): We know that the lock is locked as this type is a proof of that.
+            Mode::Sync => unsafe { self.lock.mode_union.sync.unlock() },
         }
     }
+}
 
-    union ModeUnion {
-        /// Indicates if the cell is locked. Only used if `Lock.mode` is `NoSync`.
-        no_sync: ManuallyDrop<Cell<bool>>,
+union ModeUnion {
+    /// Indicates if the cell is locked. Only used if `Lock.mode` is `NoSync`.
+    no_sync: ManuallyDrop<Cell<bool>>,
 
-        /// A lock implementation that's only used if `Lock.mode` is `Sync`.
-        sync: ManuallyDrop<RawMutex>,
+    /// A lock implementation that's only used if `Lock.mode` is `Sync`.
+    sync: ManuallyDrop<RawMutex>,
+}
+
+/// The value representing a locked state for the `Cell`.
+const LOCKED: bool = true;
+
+/// A lock which only uses synchronization if `might_be_dyn_thread_safe` is true.
+/// It implements `DynSend` and `DynSync` instead of the typical `Send` and `Sync`.
+pub struct Lock<T> {
+    /// Indicates if synchronization is used via `mode_union.sync` if it's `Sync`, or if a
+    /// not thread safe cell is used via `mode_union.no_sync` if it's `NoSync`.
+    /// This is set on initialization and never changed.
+    mode: Mode,
+
+    mode_union: ModeUnion,
+    data: UnsafeCell<T>,
+}
+
+impl<T> Lock<T> {
+    #[inline(always)]
+    pub fn new(inner: T) -> Self {
+        let (mode, mode_union) = if unlikely(mode::might_be_dyn_thread_safe()) {
+            // Create the lock with synchronization enabled using the `RawMutex` type.
+            (Mode::Sync, ModeUnion { sync: ManuallyDrop::new(RawMutex::INIT) })
+        } else {
+            // Create the lock with synchronization disabled.
+            (Mode::NoSync, ModeUnion { no_sync: ManuallyDrop::new(Cell::new(!LOCKED)) })
+        };
+        Lock { mode, mode_union, data: UnsafeCell::new(inner) }
     }
 
-    /// The value representing a locked state for the `Cell`.
-    const LOCKED: bool = true;
-
-    /// A lock which only uses synchronization if `might_be_dyn_thread_safe` is true.
-    /// It implements `DynSend` and `DynSync` instead of the typical `Send` and `Sync`.
-    pub struct Lock<T> {
-        /// Indicates if synchronization is used via `mode_union.sync` if it's `Sync`, or if a
-        /// not thread safe cell is used via `mode_union.no_sync` if it's `NoSync`.
-        /// This is set on initialization and never changed.
-        mode: Mode,
-
-        mode_union: ModeUnion,
-        data: UnsafeCell<T>,
+    #[inline(always)]
+    pub fn into_inner(self) -> T {
+        self.data.into_inner()
     }
 
-    impl<T> Lock<T> {
-        #[inline(always)]
-        pub fn new(inner: T) -> Self {
-            let (mode, mode_union) = if unlikely(mode::might_be_dyn_thread_safe()) {
-                // Create the lock with synchronization enabled using the `RawMutex` type.
-                (Mode::Sync, ModeUnion { sync: ManuallyDrop::new(RawMutex::INIT) })
-            } else {
-                // Create the lock with synchronization disabled.
-                (Mode::NoSync, ModeUnion { no_sync: ManuallyDrop::new(Cell::new(!LOCKED)) })
-            };
-            Lock { mode, mode_union, data: UnsafeCell::new(inner) }
+    #[inline(always)]
+    pub fn get_mut(&mut self) -> &mut T {
+        self.data.get_mut()
+    }
+
+    #[inline(always)]
+    pub fn try_lock(&self) -> Option<LockGuard<'_, T>> {
+        let mode = self.mode;
+        // SAFETY: This is safe since the union fields are used in accordance with `self.mode`.
+        match mode {
+            Mode::NoSync => {
+                let cell = unsafe { &self.mode_union.no_sync };
+                let was_unlocked = cell.get() != LOCKED;
+                if was_unlocked {
+                    cell.set(LOCKED);
+                }
+                was_unlocked
+            }
+            Mode::Sync => unsafe { self.mode_union.sync.try_lock() },
+        }
+        .then(|| LockGuard { lock: self, marker: PhantomData, mode })
+    }
+
+    /// This acquires the lock assuming synchronization is in a specific mode.
+    ///
+    /// Safety
+    /// This method must only be called with `Mode::Sync` if `might_be_dyn_thread_safe` was
+    /// true on lock creation.
+    #[inline(always)]
+    #[track_caller]
+    pub unsafe fn lock_assume(&self, mode: Mode) -> LockGuard<'_, T> {
+        #[inline(never)]
+        #[track_caller]
+        #[cold]
+        fn lock_held() -> ! {
+            panic!("lock was already held")
         }
 
-        #[inline(always)]
-        pub fn into_inner(self) -> T {
-            self.data.into_inner()
-        }
-
-        #[inline(always)]
-        pub fn get_mut(&mut self) -> &mut T {
-            self.data.get_mut()
-        }
-
-        #[inline(always)]
-        pub fn try_lock(&self) -> Option<LockGuard<'_, T>> {
-            let mode = self.mode;
-            // SAFETY: This is safe since the union fields are used in accordance with `self.mode`.
+        // SAFETY: This is safe since the union fields are used in accordance with `mode`
+        // which also must match `self.mode` due to the safety precondition.
+        unsafe {
             match mode {
                 Mode::NoSync => {
-                    let cell = unsafe { &self.mode_union.no_sync };
-                    let was_unlocked = cell.get() != LOCKED;
-                    if was_unlocked {
-                        cell.set(LOCKED);
+                    if unlikely(self.mode_union.no_sync.replace(LOCKED) == LOCKED) {
+                        lock_held()
                     }
-                    was_unlocked
                 }
-                Mode::Sync => unsafe { self.mode_union.sync.try_lock() },
+                Mode::Sync => self.mode_union.sync.lock(),
             }
-            .then(|| LockGuard { lock: self, marker: PhantomData, mode })
-        }
-
-        /// This acquires the lock assuming synchronization is in a specific mode.
-        ///
-        /// Safety
-        /// This method must only be called with `Mode::Sync` if `might_be_dyn_thread_safe` was
-        /// true on lock creation.
-        #[inline(always)]
-        #[track_caller]
-        pub unsafe fn lock_assume(&self, mode: Mode) -> LockGuard<'_, T> {
-            #[inline(never)]
-            #[track_caller]
-            #[cold]
-            fn lock_held() -> ! {
-                panic!("lock was already held")
-            }
-
-            // SAFETY: This is safe since the union fields are used in accordance with `mode`
-            // which also must match `self.mode` due to the safety precondition.
-            unsafe {
-                match mode {
-                    Mode::NoSync => {
-                        if unlikely(self.mode_union.no_sync.replace(LOCKED) == LOCKED) {
-                            lock_held()
-                        }
-                    }
-                    Mode::Sync => self.mode_union.sync.lock(),
-                }
-            }
-            LockGuard { lock: self, marker: PhantomData, mode }
-        }
-
-        #[inline(always)]
-        #[track_caller]
-        pub fn lock(&self) -> LockGuard<'_, T> {
-            unsafe { self.lock_assume(self.mode) }
         }
+        LockGuard { lock: self, marker: PhantomData, mode }
     }
 
-    #[cfg(parallel_compiler)]
-    unsafe impl<T: DynSend> DynSend for Lock<T> {}
-    #[cfg(parallel_compiler)]
-    unsafe impl<T: DynSend> DynSync for Lock<T> {}
-}
-
-mod no_sync {
-    use std::cell::RefCell;
-    #[doc(no_inline)]
-    pub use std::cell::RefMut as LockGuard;
-
-    use super::Mode;
-
-    pub struct Lock<T>(RefCell<T>);
-
-    impl<T> Lock<T> {
-        #[inline(always)]
-        pub fn new(inner: T) -> Self {
-            Lock(RefCell::new(inner))
-        }
-
-        #[inline(always)]
-        pub fn into_inner(self) -> T {
-            self.0.into_inner()
-        }
-
-        #[inline(always)]
-        pub fn get_mut(&mut self) -> &mut T {
-            self.0.get_mut()
-        }
-
-        #[inline(always)]
-        pub fn try_lock(&self) -> Option<LockGuard<'_, T>> {
-            self.0.try_borrow_mut().ok()
-        }
-
-        #[inline(always)]
-        #[track_caller]
-        // This is unsafe to match the API for the `parallel_compiler` case.
-        pub unsafe fn lock_assume(&self, _mode: Mode) -> LockGuard<'_, T> {
-            self.0.borrow_mut()
-        }
-
-        #[inline(always)]
-        #[track_caller]
-        pub fn lock(&self) -> LockGuard<'_, T> {
-            self.0.borrow_mut()
-        }
+    #[inline(always)]
+    #[track_caller]
+    pub fn lock(&self) -> LockGuard<'_, T> {
+        unsafe { self.lock_assume(self.mode) }
     }
 }
 
+unsafe impl<T: DynSend> DynSend for Lock<T> {}
+unsafe impl<T: DynSend> DynSync for Lock<T> {}
+
 impl<T> Lock<T> {
     #[inline(always)]
     #[track_caller]

compiler/rustc_data_structures/src/sync/parallel.rs

@@ -6,14 +6,11 @@
 use std::any::Any;
 use std::panic::{AssertUnwindSafe, catch_unwind, resume_unwind};
 
-#[cfg(not(parallel_compiler))]
-pub use disabled::*;
-#[cfg(parallel_compiler)]
-pub use enabled::*;
 use parking_lot::Mutex;
+use rayon::iter::{FromParallelIterator, IntoParallelIterator, ParallelIterator};
 
 use crate::FatalErrorMarker;
-use crate::sync::IntoDynSyncSend;
+use crate::sync::{DynSend, DynSync, FromDyn, IntoDynSyncSend, mode};
 
 /// A guard used to hold panics that occur during a parallel section to later by unwound.
 /// This is used for the parallel compiler to prevent fatal errors from non-deterministically
@@ -49,65 +46,23 @@ pub fn parallel_guard<R>(f: impl FnOnce(&ParallelGuard) -> R) -> R {
     ret
 }
 
-mod disabled {
-    use crate::sync::parallel_guard;
-
-    #[macro_export]
-    #[cfg(not(parallel_compiler))]
-    macro_rules! parallel {
-        ($($blocks:block),*) => {{
-            $crate::sync::parallel_guard(|guard| {
-                $(guard.run(|| $blocks);)*
-            });
-        }}
-    }
-
-    pub fn join<A, B, RA, RB>(oper_a: A, oper_b: B) -> (RA, RB)
-    where
-        A: FnOnce() -> RA,
-        B: FnOnce() -> RB,
-    {
-        let (a, b) = parallel_guard(|guard| {
-            let a = guard.run(oper_a);
-            let b = guard.run(oper_b);
-            (a, b)
-        });
-        (a.unwrap(), b.unwrap())
-    }
-
-    pub fn par_for_each_in<T: IntoIterator>(t: T, mut for_each: impl FnMut(T::Item)) {
-        parallel_guard(|guard| {
-            t.into_iter().for_each(|i| {
-                guard.run(|| for_each(i));
-            });
-        })
-    }
-
-    pub fn try_par_for_each_in<T: IntoIterator, E>(
-        t: T,
-        mut for_each: impl FnMut(T::Item) -> Result<(), E>,
-    ) -> Result<(), E> {
-        parallel_guard(|guard| {
-            t.into_iter().filter_map(|i| guard.run(|| for_each(i))).fold(Ok(()), Result::and)
-        })
-    }
-
-    pub fn par_map<T: IntoIterator, R, C: FromIterator<R>>(
-        t: T,
-        mut map: impl FnMut(<<T as IntoIterator>::IntoIter as Iterator>::Item) -> R,
-    ) -> C {
-        parallel_guard(|guard| t.into_iter().filter_map(|i| guard.run(|| map(i))).collect())
-    }
+pub fn serial_join<A, B, RA, RB>(oper_a: A, oper_b: B) -> (RA, RB)
+where
+    A: FnOnce() -> RA,
+    B: FnOnce() -> RB,
+{
+    let (a, b) = parallel_guard(|guard| {
+        let a = guard.run(oper_a);
+        let b = guard.run(oper_b);
+        (a, b)
+    });
+    (a.unwrap(), b.unwrap())
 }
 
-#[cfg(parallel_compiler)]
-mod enabled {
-    use crate::sync::{DynSend, DynSync, FromDyn, mode, parallel_guard};
-
-    /// Runs a list of blocks in parallel. The first block is executed immediately on
-    /// the current thread. Use that for the longest running block.
-    #[macro_export]
-    macro_rules! parallel {
+/// Runs a list of blocks in parallel. The first block is executed immediately on
+/// the current thread. Use that for the longest running block.
+#[macro_export]
+macro_rules! parallel {
         (impl $fblock:block [$($c:expr,)*] [$block:expr $(, $rest:expr)*]) => {
             parallel!(impl $fblock [$block, $($c,)*] [$($rest),*])
         };
@@ -139,92 +94,89 @@ mod enabled {
         };
     }
 
-    // This function only works when `mode::is_dyn_thread_safe()`.
-    pub fn scope<'scope, OP, R>(op: OP) -> R
-    where
-        OP: FnOnce(&rayon::Scope<'scope>) -> R + DynSend,
-        R: DynSend,
-    {
-        let op = FromDyn::from(op);
-        rayon::scope(|s| FromDyn::from(op.into_inner()(s))).into_inner()
-    }
+// This function only works when `mode::is_dyn_thread_safe()`.
+pub fn scope<'scope, OP, R>(op: OP) -> R
+where
+    OP: FnOnce(&rayon::Scope<'scope>) -> R + DynSend,
+    R: DynSend,
+{
+    let op = FromDyn::from(op);
+    rayon::scope(|s| FromDyn::from(op.into_inner()(s))).into_inner()
+}
 
-    #[inline]
-    pub fn join<A, B, RA: DynSend, RB: DynSend>(oper_a: A, oper_b: B) -> (RA, RB)
-    where
-        A: FnOnce() -> RA + DynSend,
-        B: FnOnce() -> RB + DynSend,
-    {
-        if mode::is_dyn_thread_safe() {
-            let oper_a = FromDyn::from(oper_a);
-            let oper_b = FromDyn::from(oper_b);
-            let (a, b) = parallel_guard(|guard| {
-                rayon::join(
-                    move || guard.run(move || FromDyn::from(oper_a.into_inner()())),
-                    move || guard.run(move || FromDyn::from(oper_b.into_inner()())),
-                )
-            });
-            (a.unwrap().into_inner(), b.unwrap().into_inner())
-        } else {
-            super::disabled::join(oper_a, oper_b)
-        }
-    }
-
-    use rayon::iter::{FromParallelIterator, IntoParallelIterator, ParallelIterator};
-
-    pub fn par_for_each_in<I, T: IntoIterator<Item = I> + IntoParallelIterator<Item = I>>(
-        t: T,
-        for_each: impl Fn(I) + DynSync + DynSend,
-    ) {
-        parallel_guard(|guard| {
-            if mode::is_dyn_thread_safe() {
-                let for_each = FromDyn::from(for_each);
-                t.into_par_iter().for_each(|i| {
-                    guard.run(|| for_each(i));
-                });
-            } else {
-                t.into_iter().for_each(|i| {
-                    guard.run(|| for_each(i));
-                });
-            }
+#[inline]
+pub fn join<A, B, RA: DynSend, RB: DynSend>(oper_a: A, oper_b: B) -> (RA, RB)
+where
+    A: FnOnce() -> RA + DynSend,
+    B: FnOnce() -> RB + DynSend,
+{
+    if mode::is_dyn_thread_safe() {
+        let oper_a = FromDyn::from(oper_a);
+        let oper_b = FromDyn::from(oper_b);
+        let (a, b) = parallel_guard(|guard| {
+            rayon::join(
+                move || guard.run(move || FromDyn::from(oper_a.into_inner()())),
+                move || guard.run(move || FromDyn::from(oper_b.into_inner()())),
+            )
         });
-    }
-
-    pub fn try_par_for_each_in<
-        T: IntoIterator + IntoParallelIterator<Item = <T as IntoIterator>::Item>,
-        E: Send,
-    >(
-        t: T,
-        for_each: impl Fn(<T as IntoIterator>::Item) -> Result<(), E> + DynSync + DynSend,
-    ) -> Result<(), E> {
-        parallel_guard(|guard| {
-            if mode::is_dyn_thread_safe() {
-                let for_each = FromDyn::from(for_each);
-                t.into_par_iter()
-                    .filter_map(|i| guard.run(|| for_each(i)))
-                    .reduce(|| Ok(()), Result::and)
-            } else {
-                t.into_iter().filter_map(|i| guard.run(|| for_each(i))).fold(Ok(()), Result::and)
-            }
-        })
-    }
-
-    pub fn par_map<
-        I,
-        T: IntoIterator<Item = I> + IntoParallelIterator<Item = I>,
-        R: std::marker::Send,
-        C: FromIterator<R> + FromParallelIterator<R>,
-    >(
-        t: T,
-        map: impl Fn(I) -> R + DynSync + DynSend,
-    ) -> C {
-        parallel_guard(|guard| {
-            if mode::is_dyn_thread_safe() {
-                let map = FromDyn::from(map);
-                t.into_par_iter().filter_map(|i| guard.run(|| map(i))).collect()
-            } else {
-                t.into_iter().filter_map(|i| guard.run(|| map(i))).collect()
-            }
-        })
+        (a.unwrap().into_inner(), b.unwrap().into_inner())
+    } else {
+        serial_join(oper_a, oper_b)
     }
 }
+
+pub fn par_for_each_in<I, T: IntoIterator<Item = I> + IntoParallelIterator<Item = I>>(
+    t: T,
+    for_each: impl Fn(I) + DynSync + DynSend,
+) {
+    parallel_guard(|guard| {
+        if mode::is_dyn_thread_safe() {
+            let for_each = FromDyn::from(for_each);
+            t.into_par_iter().for_each(|i| {
+                guard.run(|| for_each(i));
+            });
+        } else {
+            t.into_iter().for_each(|i| {
+                guard.run(|| for_each(i));
+            });
+        }
+    });
+}
+
+pub fn try_par_for_each_in<
+    T: IntoIterator + IntoParallelIterator<Item = <T as IntoIterator>::Item>,
+    E: Send,
+>(
+    t: T,
+    for_each: impl Fn(<T as IntoIterator>::Item) -> Result<(), E> + DynSync + DynSend,
+) -> Result<(), E> {
+    parallel_guard(|guard| {
+        if mode::is_dyn_thread_safe() {
+            let for_each = FromDyn::from(for_each);
+            t.into_par_iter()
+                .filter_map(|i| guard.run(|| for_each(i)))
+                .reduce(|| Ok(()), Result::and)
+        } else {
+            t.into_iter().filter_map(|i| guard.run(|| for_each(i))).fold(Ok(()), Result::and)
+        }
+    })
+}
+
+pub fn par_map<
+    I,
+    T: IntoIterator<Item = I> + IntoParallelIterator<Item = I>,
+    R: std::marker::Send,
+    C: FromIterator<R> + FromParallelIterator<R>,
+>(
+    t: T,
+    map: impl Fn(I) -> R + DynSync + DynSend,
+) -> C {
+    parallel_guard(|guard| {
+        if mode::is_dyn_thread_safe() {
+            let map = FromDyn::from(map);
+            t.into_par_iter().filter_map(|i| guard.run(|| map(i))).collect()
+        } else {
+            t.into_iter().filter_map(|i| guard.run(|| map(i))).collect()
+        }
+    })
+}

compiler/rustc_data_structures/src/sync/vec.rs

@@ -4,40 +4,23 @@ use rustc_index::Idx;
 
 #[derive(Default)]
 pub struct AppendOnlyIndexVec<I: Idx, T: Copy> {
-    #[cfg(not(parallel_compiler))]
-    vec: elsa::vec::FrozenVec<T>,
-    #[cfg(parallel_compiler)]
     vec: elsa::sync::LockFreeFrozenVec<T>,
     _marker: PhantomData<fn(&I)>,
 }
 
 impl<I: Idx, T: Copy> AppendOnlyIndexVec<I, T> {
     pub fn new() -> Self {
-        Self {
-            #[cfg(not(parallel_compiler))]
-            vec: elsa::vec::FrozenVec::new(),
-            #[cfg(parallel_compiler)]
-            vec: elsa::sync::LockFreeFrozenVec::new(),
-            _marker: PhantomData,
-        }
+        Self { vec: elsa::sync::LockFreeFrozenVec::new(), _marker: PhantomData }
     }
 
     pub fn push(&self, val: T) -> I {
-        #[cfg(not(parallel_compiler))]
-        let i = self.vec.len();
-        #[cfg(not(parallel_compiler))]
-        self.vec.push(val);
-        #[cfg(parallel_compiler)]
         let i = self.vec.push(val);
         I::new(i)
     }
 
     pub fn get(&self, i: I) -> Option<T> {
         let i = i.index();
-        #[cfg(not(parallel_compiler))]
-        return self.vec.get_copy(i);
-        #[cfg(parallel_compiler)]
-        return self.vec.get(i);
+        self.vec.get(i)
     }
 }
 

compiler/rustc_data_structures/src/sync/worker_local.rs

@@ -5,8 +5,9 @@ use std::ptr;
 use std::sync::Arc;
 
 use parking_lot::Mutex;
-#[cfg(parallel_compiler)]
-use {crate::outline, crate::sync::CacheAligned};
+
+use crate::outline;
+use crate::sync::CacheAligned;
 
 /// A pointer to the `RegistryData` which uniquely identifies a registry.
 /// This identifier can be reused if the registry gets freed.
@@ -21,7 +22,6 @@ impl RegistryId {
     ///
     /// Note that there's a race possible where the identifier in `THREAD_DATA` could be reused
     /// so this can succeed from a different registry.
-    #[cfg(parallel_compiler)]
     fn verify(self) -> usize {
         let (id, index) = THREAD_DATA.with(|data| (data.registry_id.get(), data.index.get()));
 
@@ -102,11 +102,7 @@ impl Registry {
 /// worker local value through the `Deref` impl on the registry associated with the thread it was
 /// created on. It will panic otherwise.
 pub struct WorkerLocal<T> {
-    #[cfg(not(parallel_compiler))]
-    local: T,
-    #[cfg(parallel_compiler)]
     locals: Box<[CacheAligned<T>]>,
-    #[cfg(parallel_compiler)]
     registry: Registry,
 }
 
@@ -114,7 +110,6 @@ pub struct WorkerLocal<T> {
 // or it will panic for threads without an associated local. So there isn't a need for `T` to do
 // it's own synchronization. The `verify` method on `RegistryId` has an issue where the id
 // can be reused, but `WorkerLocal` has a reference to `Registry` which will prevent any reuse.
-#[cfg(parallel_compiler)]
 unsafe impl<T: Send> Sync for WorkerLocal<T> {}
 
 impl<T> WorkerLocal<T> {
@@ -122,33 +117,17 @@ impl<T> WorkerLocal<T> {
     /// value this worker local should take for each thread in the registry.
     #[inline]
     pub fn new<F: FnMut(usize) -> T>(mut initial: F) -> WorkerLocal<T> {
-        #[cfg(parallel_compiler)]
-        {
-            let registry = Registry::current();
-            WorkerLocal {
-                locals: (0..registry.0.thread_limit.get())
-                    .map(|i| CacheAligned(initial(i)))
-                    .collect(),
-                registry,
-            }
-        }
-        #[cfg(not(parallel_compiler))]
-        {
-            WorkerLocal { local: initial(0) }
+        let registry = Registry::current();
+        WorkerLocal {
+            locals: (0..registry.0.thread_limit.get()).map(|i| CacheAligned(initial(i))).collect(),
+            registry,
         }
     }
 
     /// Returns the worker-local values for each thread
     #[inline]
     pub fn into_inner(self) -> impl Iterator<Item = T> {
-        #[cfg(parallel_compiler)]
-        {
-            self.locals.into_vec().into_iter().map(|local| local.0)
-        }
-        #[cfg(not(parallel_compiler))]
-        {
-            std::iter::once(self.local)
-        }
+        self.locals.into_vec().into_iter().map(|local| local.0)
     }
 }
 
@@ -156,13 +135,6 @@ impl<T> Deref for WorkerLocal<T> {
     type Target = T;
 
     #[inline(always)]
-    #[cfg(not(parallel_compiler))]
-    fn deref(&self) -> &T {
-        &self.local
-    }
-
-    #[inline(always)]
-    #[cfg(parallel_compiler)]
     fn deref(&self) -> &T {
         // This is safe because `verify` will only return values less than
         // `self.registry.thread_limit` which is the size of the `self.locals` array.

compiler/rustc_driver_impl/Cargo.toml

@@ -77,9 +77,4 @@ rustc_randomized_layouts = [
     'rustc_index/rustc_randomized_layouts',
     'rustc_middle/rustc_randomized_layouts'
 ]
-rustc_use_parallel_compiler = [
-    'rustc_data_structures/rustc_use_parallel_compiler',
-    'rustc_interface/rustc_use_parallel_compiler',
-    'rustc_middle/rustc_use_parallel_compiler'
-]
 # tidy-alphabetical-end

compiler/rustc_driver_impl/src/lib.rs

@@ -934,9 +934,12 @@ pub fn version_at_macro_invocation(
 }
 
 fn usage(verbose: bool, include_unstable_options: bool, nightly_build: bool) {
-    let groups = if verbose { config::rustc_optgroups() } else { config::rustc_short_optgroups() };
     let mut options = getopts::Options::new();
-    for option in groups.iter().filter(|x| include_unstable_options || x.is_stable()) {
+    for option in config::rustc_optgroups()
+        .iter()
+        .filter(|x| verbose || !x.is_verbose_help_only)
+        .filter(|x| include_unstable_options || x.is_stable())
+    {
         option.apply(&mut options);
     }
     let message = "Usage: rustc [OPTIONS] INPUT";

compiler/rustc_error_messages/Cargo.toml

@@ -19,8 +19,3 @@ rustc_span = { path = "../rustc_span" }
 tracing = "0.1"
 unic-langid = { version = "0.9.0", features = ["macros"] }
 # tidy-alphabetical-end
-
-[features]
-# tidy-alphabetical-start
-rustc_use_parallel_compiler = ['rustc_baked_icu_data/rustc_use_parallel_compiler']
-# tidy-alphabetical-end

compiler/rustc_error_messages/src/lib.rs

@@ -8,12 +8,9 @@
 // tidy-alphabetical-end
 
 use std::borrow::Cow;
-#[cfg(not(parallel_compiler))]
-use std::cell::LazyCell as Lazy;
 use std::error::Error;
 use std::path::{Path, PathBuf};
-#[cfg(parallel_compiler)]
-use std::sync::LazyLock as Lazy;
+use std::sync::LazyLock;
 use std::{fmt, fs, io};
 
 use fluent_bundle::FluentResource;
@@ -21,9 +18,6 @@ pub use fluent_bundle::types::FluentType;
 pub use fluent_bundle::{self, FluentArgs, FluentError, FluentValue};
 use fluent_syntax::parser::ParserError;
 use icu_provider_adapters::fallback::{LocaleFallbackProvider, LocaleFallbacker};
-#[cfg(not(parallel_compiler))]
-use intl_memoizer::IntlLangMemoizer;
-#[cfg(parallel_compiler)]
 use intl_memoizer::concurrent::IntlLangMemoizer;
 use rustc_data_structures::sync::{IntoDynSyncSend, Lrc};
 use rustc_macros::{Decodable, Encodable};
@@ -34,12 +28,6 @@ pub use unic_langid::{LanguageIdentifier, langid};
 pub type FluentBundle =
     IntoDynSyncSend<fluent_bundle::bundle::FluentBundle<FluentResource, IntlLangMemoizer>>;
 
-#[cfg(not(parallel_compiler))]
-fn new_bundle(locales: Vec<LanguageIdentifier>) -> FluentBundle {
-    IntoDynSyncSend(fluent_bundle::bundle::FluentBundle::new(locales))
-}
-
-#[cfg(parallel_compiler)]
 fn new_bundle(locales: Vec<LanguageIdentifier>) -> FluentBundle {
     IntoDynSyncSend(fluent_bundle::bundle::FluentBundle::new_concurrent(locales))
 }
@@ -217,7 +205,7 @@ fn register_functions(bundle: &mut FluentBundle) {
 
 /// Type alias for the result of `fallback_fluent_bundle` - a reference-counted pointer to a lazily
 /// evaluated fluent bundle.
-pub type LazyFallbackBundle = Lrc<Lazy<FluentBundle, impl FnOnce() -> FluentBundle>>;
+pub type LazyFallbackBundle = Lrc<LazyLock<FluentBundle, impl FnOnce() -> FluentBundle>>;
 
 /// Return the default `FluentBundle` with standard "en-US" diagnostic messages.
 #[instrument(level = "trace", skip(resources))]
@@ -225,7 +213,7 @@ pub fn fallback_fluent_bundle(
     resources: Vec<&'static str>,
     with_directionality_markers: bool,
 ) -> LazyFallbackBundle {
-    Lrc::new(Lazy::new(move || {
+    Lrc::new(LazyLock::new(move || {
         let mut fallback_bundle = new_bundle(vec![langid!("en-US")]);
 
         register_functions(&mut fallback_bundle);
@@ -548,15 +536,6 @@ pub fn fluent_value_from_str_list_sep_by_and(l: Vec<Cow<'_, str>>) -> FluentValu
             Cow::Owned(result)
         }
 
-        #[cfg(not(parallel_compiler))]
-        fn as_string_threadsafe(
-            &self,
-            _intls: &intl_memoizer::concurrent::IntlLangMemoizer,
-        ) -> Cow<'static, str> {
-            unreachable!("`as_string_threadsafe` is not used in non-parallel rustc")
-        }
-
-        #[cfg(parallel_compiler)]
         fn as_string_threadsafe(
             &self,
             intls: &intl_memoizer::concurrent::IntlLangMemoizer,

compiler/rustc_errors/Cargo.toml

@@ -36,8 +36,3 @@ features = [
     "Win32_Security",
     "Win32_System_Threading",
 ]
-
-[features]
-# tidy-alphabetical-start
-rustc_use_parallel_compiler = ['rustc_error_messages/rustc_use_parallel_compiler']
-# tidy-alphabetical-end

compiler/rustc_errors/src/json.rs

@@ -27,7 +27,7 @@ use termcolor::{ColorSpec, WriteColor};
 
 use crate::diagnostic::IsLint;
 use crate::emitter::{
-    ColorConfig, Destination, Emitter, HumanEmitter, HumanReadableErrorType,
+    ColorConfig, Destination, Emitter, HumanEmitter, HumanReadableErrorType, OutputTheme,
     should_show_source_code,
 };
 use crate::registry::Registry;
@@ -377,6 +377,11 @@ impl Diagnostic {
             .terminal_url(je.terminal_url)
             .ui_testing(je.ui_testing)
             .ignored_directories_in_source_blocks(je.ignored_directories_in_source_blocks.clone())
+            .theme(if let HumanReadableErrorType::Unicode = je.json_rendered {
+                OutputTheme::Unicode
+            } else {
+                OutputTheme::Ascii
+            })
             .emit_diagnostic(diag);
         let buf = Arc::try_unwrap(buf.0).unwrap().into_inner().unwrap();
         let buf = String::from_utf8(buf).unwrap();

compiler/rustc_errors/src/lib.rs

@@ -623,12 +623,25 @@ impl Drop for DiagCtxtInner {
             self.flush_delayed()
         }
 
+        // Sanity check: did we use some of the expensive `trimmed_def_paths` functions
+        // unexpectedly, that is, without producing diagnostics? If so, for debugging purposes, we
+        // suggest where this happened and how to avoid it.
         if !self.has_printed && !self.suppressed_expected_diag && !std::thread::panicking() {
             if let Some(backtrace) = &self.must_produce_diag {
+                let suggestion = match backtrace.status() {
+                    BacktraceStatus::Disabled => String::from(
+                        "Backtraces are currently disabled: set `RUST_BACKTRACE=1` and re-run \
+                        to see where it happened.",
+                    ),
+                    BacktraceStatus::Captured => format!(
+                        "This happened in the following `must_produce_diag` call's backtrace:\n\
+                        {backtrace}",
+                    ),
+                    _ => String::from("(impossible to capture backtrace where this happened)"),
+                };
                 panic!(
-                    "must_produce_diag: `trimmed_def_paths` called but no diagnostics emitted; \
-                     `with_no_trimmed_paths` for debugging. \
-                     called at: {backtrace}"
+                    "`trimmed_def_paths` called, diagnostics were expected but none were emitted. \
+                    Use `with_no_trimmed_paths` for debugging. {suggestion}"
                 );
             }
         }

compiler/rustc_errors/src/tests.rs

@@ -26,16 +26,11 @@ fn make_dummy(ftl: &'static str) -> Dummy {
 
     let langid_en = langid!("en-US");
 
-    #[cfg(parallel_compiler)]
     let mut bundle: FluentBundle =
         IntoDynSyncSend(crate::fluent_bundle::bundle::FluentBundle::new_concurrent(vec![
             langid_en,
         ]));
 
-    #[cfg(not(parallel_compiler))]
-    let mut bundle: FluentBundle =
-        IntoDynSyncSend(crate::fluent_bundle::bundle::FluentBundle::new(vec![langid_en]));
-
     bundle.add_resource(resource).expect("Failed to add FTL resources to the bundle.");
 
     Dummy { bundle }

compiler/rustc_expand/src/base.rs

@@ -866,9 +866,7 @@ impl SyntaxExtension {
             })
             .unwrap_or_else(|| (None, helper_attrs));
         let stability = attr::find_stability(sess, attrs, span);
-        // We set `is_const_fn` false to avoid getting any implicit const stability.
-        let const_stability =
-            attr::find_const_stability(sess, attrs, span, /* is_const_fn */ false);
+        let const_stability = attr::find_const_stability(sess, attrs, span);
         let body_stability = attr::find_body_stability(sess, attrs);
         if let Some((_, sp)) = const_stability {
             sess.dcx().emit_err(errors::MacroConstStability {

compiler/rustc_hir/src/lang_items.rs

@@ -241,6 +241,8 @@ language_item_table! {
     DerefMut,                sym::deref_mut,           deref_mut_trait,            Target::Trait,          GenericRequirement::Exact(0);
     DerefPure,               sym::deref_pure,          deref_pure_trait,           Target::Trait,          GenericRequirement::Exact(0);
     DerefTarget,             sym::deref_target,        deref_target,               Target::AssocTy,        GenericRequirement::None;
+    Receiver,                sym::receiver,            receiver_trait,             Target::Trait,          GenericRequirement::None;
+    ReceiverTarget,          sym::receiver_target,     receiver_target,            Target::AssocTy,        GenericRequirement::None;
     LegacyReceiver,          sym::legacy_receiver,     legacy_receiver_trait,      Target::Trait,          GenericRequirement::None;
 
     Fn,                      kw::Fn,                   fn_trait,                   Target::Trait,          GenericRequirement::Exact(1);

compiler/rustc_hir_analysis/src/check/wfcheck.rs

@@ -1427,16 +1427,6 @@ fn check_where_clauses<'tcx>(wfcx: &WfCheckingCtxt<'_, 'tcx>, span: Span, def_id
     let predicates = tcx.predicates_of(def_id.to_def_id());
     let generics = tcx.generics_of(def_id);
 
-    let is_our_default = |def: &ty::GenericParamDef| match def.kind {
-        GenericParamDefKind::Type { has_default, .. }
-        | GenericParamDefKind::Const { has_default, .. } => {
-            has_default && def.index >= generics.parent_count as u32
-        }
-        GenericParamDefKind::Lifetime => {
-            span_bug!(tcx.def_span(def.def_id), "lifetime params can have no default")
-        }
-    };
-
     // Check that concrete defaults are well-formed. See test `type-check-defaults.rs`.
     // For example, this forbids the declaration:
     //
@@ -1444,40 +1434,21 @@ fn check_where_clauses<'tcx>(wfcx: &WfCheckingCtxt<'_, 'tcx>, span: Span, def_id
     //
     // Here, the default `Vec<[u32]>` is not WF because `[u32]: Sized` does not hold.
     for param in &generics.own_params {
-        match param.kind {
-            GenericParamDefKind::Type { .. } => {
-                if is_our_default(param) {
-                    let ty = tcx.type_of(param.def_id).instantiate_identity();
-                    // Ignore dependent defaults -- that is, where the default of one type
-                    // parameter includes another (e.g., `<T, U = T>`). In those cases, we can't
-                    // be sure if it will error or not as user might always specify the other.
-                    if !ty.has_param() {
-                        wfcx.register_wf_obligation(
-                            tcx.def_span(param.def_id),
-                            Some(WellFormedLoc::Ty(param.def_id.expect_local())),
-                            ty.into(),
-                        );
-                    }
-                }
+        if let Some(default) = param.default_value(tcx).map(ty::EarlyBinder::instantiate_identity) {
+            // Ignore dependent defaults -- that is, where the default of one type
+            // parameter includes another (e.g., `<T, U = T>`). In those cases, we can't
+            // be sure if it will error or not as user might always specify the other.
+            // FIXME(generic_const_exprs): This is incorrect when dealing with unused const params.
+            // E.g: `struct Foo<const N: usize, const M: usize = { 1 - 2 }>;`. Here, we should
+            // eagerly error but we don't as we have `ConstKind::Unevaluated(.., [N, M])`.
+            if !default.has_param() {
+                wfcx.register_wf_obligation(
+                    tcx.def_span(param.def_id),
+                    matches!(param.kind, GenericParamDefKind::Type { .. })
+                        .then(|| WellFormedLoc::Ty(param.def_id.expect_local())),
+                    default,
+                );
             }
-            GenericParamDefKind::Const { .. } => {
-                if is_our_default(param) {
-                    // FIXME(const_generics_defaults): This
-                    // is incorrect when dealing with unused args, for example
-                    // for `struct Foo<const N: usize, const M: usize = { 1 - 2 }>`
-                    // we should eagerly error.
-                    let default_ct = tcx.const_param_default(param.def_id).instantiate_identity();
-                    if !default_ct.has_param() {
-                        wfcx.register_wf_obligation(
-                            tcx.def_span(param.def_id),
-                            None,
-                            default_ct.into(),
-                        );
-                    }
-                }
-            }
-            // Doesn't have defaults.
-            GenericParamDefKind::Lifetime => {}
         }
     }
 
@@ -1490,39 +1461,16 @@ fn check_where_clauses<'tcx>(wfcx: &WfCheckingCtxt<'_, 'tcx>, span: Span, def_id
     //
     // First we build the defaulted generic parameters.
     let args = GenericArgs::for_item(tcx, def_id.to_def_id(), |param, _| {
-        match param.kind {
-            GenericParamDefKind::Lifetime => {
-                // All regions are identity.
-                tcx.mk_param_from_def(param)
-            }
-
-            GenericParamDefKind::Type { .. } => {
-                // If the param has a default, ...
-                if is_our_default(param) {
-                    let default_ty = tcx.type_of(param.def_id).instantiate_identity();
-                    // ... and it's not a dependent default, ...
-                    if !default_ty.has_param() {
-                        // ... then instantiate it with the default.
-                        return default_ty.into();
-                    }
-                }
-
-                tcx.mk_param_from_def(param)
-            }
-            GenericParamDefKind::Const { .. } => {
-                // If the param has a default, ...
-                if is_our_default(param) {
-                    let default_ct = tcx.const_param_default(param.def_id).instantiate_identity();
-                    // ... and it's not a dependent default, ...
-                    if !default_ct.has_param() {
-                        // ... then instantiate it with the default.
-                        return default_ct.into();
-                    }
-                }
-
-                tcx.mk_param_from_def(param)
-            }
+        if param.index >= generics.parent_count as u32
+            // If the param has a default, ...
+            && let Some(default) = param.default_value(tcx).map(ty::EarlyBinder::instantiate_identity)
+            // ... and it's not a dependent default, ...
+            && !default.has_param()
+        {
+            // ... then instantiate it with the default.
+            return default;
         }
+        tcx.mk_param_from_def(param)
     });
 
     // Now we build the instantiated predicates.

compiler/rustc_hir_typeck/src/_match.rs

@@ -15,7 +15,7 @@ use crate::{Diverges, Expectation, FnCtxt, Needs};
 
 impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     #[instrument(skip(self), level = "debug", ret)]
-    pub(crate) fn check_match(
+    pub(crate) fn check_expr_match(
         &self,
         expr: &'tcx hir::Expr<'tcx>,
         scrut: &'tcx hir::Expr<'tcx>,

compiler/rustc_hir_typeck/src/callee.rs

@@ -62,7 +62,7 @@ enum CallStep<'tcx> {
 }
 
 impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
-    pub(crate) fn check_call(
+    pub(crate) fn check_expr_call(
         &self,
         call_expr: &'tcx hir::Expr<'tcx>,
         callee_expr: &'tcx hir::Expr<'tcx>,
@@ -74,8 +74,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 .check_expr_with_expectation_and_args(
                     callee_expr,
                     Expectation::NoExpectation,
-                    arg_exprs,
-                    Some(call_expr),
+                    Some((call_expr, arg_exprs)),
                 ),
             _ => self.check_expr(callee_expr),
         };

compiler/rustc_hir_typeck/src/check.rs

@@ -137,7 +137,7 @@ pub(super) fn check_fn<'a, 'tcx>(
     }
 
     fcx.is_whole_body.set(true);
-    fcx.check_return_expr(body.value, false);
+    fcx.check_return_or_body_tail(body.value, false);
 
     // Finalize the return check by taking the LUB of the return types
     // we saw and assigning it to the expected return type. This isn't

compiler/rustc_hir_typeck/src/expr.rs

@@ -55,6 +55,9 @@ use crate::{
 };
 
 impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
+    /// Check an expr with an expectation type, and also demand that the expr's
+    /// evaluated type is a subtype of the expectation at the end. This is a
+    /// *hard* requirement.
     pub(crate) fn check_expr_has_type_or_error(
         &self,
         expr: &'tcx hir::Expr<'tcx>,
@@ -97,6 +100,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         ty
     }
 
+    /// Check an expr with an expectation type, and also demand that the expr's
+    /// evaluated type is a coercible to the expectation at the end. This is a
+    /// *hard* requirement.
     pub(super) fn check_expr_coercible_to_type(
         &self,
         expr: &'tcx hir::Expr<'tcx>,
@@ -128,6 +134,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
     }
 
+    /// Check an expr with an expectation type. Don't actually enforce that expectation
+    /// is related to the expr's evaluated type via subtyping or coercion. This is
+    /// usually called because we want to do that subtype/coerce call manually for better
+    /// diagnostics.
     pub(super) fn check_expr_with_hint(
         &self,
         expr: &'tcx hir::Expr<'tcx>,
@@ -136,6 +146,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         self.check_expr_with_expectation(expr, ExpectHasType(expected))
     }
 
+    /// Check an expr with an expectation type, and also [`Needs`] which will
+    /// prompt typeck to convert any implicit immutable derefs to mutable derefs.
     fn check_expr_with_expectation_and_needs(
         &self,
         expr: &'tcx hir::Expr<'tcx>,
@@ -153,10 +165,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         ty
     }
 
+    /// Check an expr with no expectations.
     pub(super) fn check_expr(&self, expr: &'tcx hir::Expr<'tcx>) -> Ty<'tcx> {
         self.check_expr_with_expectation(expr, NoExpectation)
     }
 
+    /// Check an expr with no expectations, but with [`Needs`] which will
+    /// prompt typeck to convert any implicit immutable derefs to mutable derefs.
     pub(super) fn check_expr_with_needs(
         &self,
         expr: &'tcx hir::Expr<'tcx>,
@@ -165,33 +180,26 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         self.check_expr_with_expectation_and_needs(expr, NoExpectation, needs)
     }
 
-    /// Invariant:
-    /// If an expression has any sub-expressions that result in a type error,
-    /// inspecting that expression's type with `ty.references_error()` will return
-    /// true. Likewise, if an expression is known to diverge, inspecting its
-    /// type with `ty::type_is_bot` will return true (n.b.: since Rust is
-    /// strict, _|_ can appear in the type of an expression that does not,
-    /// itself, diverge: for example, fn() -> _|_.)
-    /// Note that inspecting a type's structure *directly* may expose the fact
-    /// that there are actually multiple representations for `Error`, so avoid
-    /// that when err needs to be handled differently.
+    /// Check an expr with an expectation type which may be used to eagerly
+    /// guide inference when evaluating that expr.
     #[instrument(skip(self, expr), level = "debug")]
     pub(super) fn check_expr_with_expectation(
         &self,
         expr: &'tcx hir::Expr<'tcx>,
         expected: Expectation<'tcx>,
     ) -> Ty<'tcx> {
-        self.check_expr_with_expectation_and_args(expr, expected, &[], None)
+        self.check_expr_with_expectation_and_args(expr, expected, None)
     }
 
-    /// Same as `check_expr_with_expectation`, but allows us to pass in the arguments of a
-    /// `ExprKind::Call` when evaluating its callee when it is an `ExprKind::Path`.
+    /// Same as [`Self::check_expr_with_expectation`], but allows us to pass in
+    /// the arguments of a [`ExprKind::Call`] when evaluating its callee that
+    /// is an [`ExprKind::Path`]. We use this to refine the spans for certain
+    /// well-formedness guarantees for the path expr.
     pub(super) fn check_expr_with_expectation_and_args(
         &self,
         expr: &'tcx hir::Expr<'tcx>,
         expected: Expectation<'tcx>,
-        args: &'tcx [hir::Expr<'tcx>],
-        call: Option<&'tcx hir::Expr<'tcx>>,
+        call_expr_and_args: Option<(&'tcx hir::Expr<'tcx>, &'tcx [hir::Expr<'tcx>])>,
     ) -> Ty<'tcx> {
         if self.tcx().sess.verbose_internals() {
             // make this code only run with -Zverbose-internals because it is probably slow
@@ -236,9 +244,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         };
 
         let ty = ensure_sufficient_stack(|| match &expr.kind {
+            // Intercept the callee path expr and give it better spans.
             hir::ExprKind::Path(
                 qpath @ (hir::QPath::Resolved(..) | hir::QPath::TypeRelative(..)),
-            ) => self.check_expr_path(qpath, expr, Some(args), call),
+            ) => self.check_expr_path(qpath, expr, call_expr_and_args),
             _ => self.check_expr_kind(expr, expected),
         });
         let ty = self.resolve_vars_if_possible(ty);
@@ -472,28 +481,30 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let tcx = self.tcx;
         match expr.kind {
-            ExprKind::Lit(ref lit) => self.check_lit(lit, expected),
-            ExprKind::Binary(op, lhs, rhs) => self.check_binop(expr, op, lhs, rhs, expected),
+            ExprKind::Lit(ref lit) => self.check_expr_lit(lit, expected),
+            ExprKind::Binary(op, lhs, rhs) => self.check_expr_binop(expr, op, lhs, rhs, expected),
             ExprKind::Assign(lhs, rhs, span) => {
                 self.check_expr_assign(expr, expected, lhs, rhs, span)
             }
             ExprKind::AssignOp(op, lhs, rhs) => {
-                self.check_binop_assign(expr, op, lhs, rhs, expected)
+                self.check_expr_binop_assign(expr, op, lhs, rhs, expected)
             }
-            ExprKind::Unary(unop, oprnd) => self.check_expr_unary(unop, oprnd, expected, expr),
+            ExprKind::Unary(unop, oprnd) => self.check_expr_unop(unop, oprnd, expected, expr),
             ExprKind::AddrOf(kind, mutbl, oprnd) => {
                 self.check_expr_addr_of(kind, mutbl, oprnd, expected, expr)
             }
             ExprKind::Path(QPath::LangItem(lang_item, _)) => {
                 self.check_lang_item_path(lang_item, expr)
             }
-            ExprKind::Path(ref qpath) => self.check_expr_path(qpath, expr, None, None),
+            ExprKind::Path(ref qpath) => self.check_expr_path(qpath, expr, None),
             ExprKind::InlineAsm(asm) => {
                 // We defer some asm checks as we may not have resolved the input and output types yet (they may still be infer vars).
                 self.deferred_asm_checks.borrow_mut().push((asm, expr.hir_id));
                 self.check_expr_asm(asm)
             }
-            ExprKind::OffsetOf(container, fields) => self.check_offset_of(container, fields, expr),
+            ExprKind::OffsetOf(container, fields) => {
+                self.check_expr_offset_of(container, fields, expr)
+            }
             ExprKind::Break(destination, ref expr_opt) => {
                 self.check_expr_break(destination, expr_opt.as_deref(), expr)
             }
@@ -512,13 +523,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 self.check_expr_loop(body, source, expected, expr)
             }
             ExprKind::Match(discrim, arms, match_src) => {
-                self.check_match(expr, discrim, arms, expected, match_src)
+                self.check_expr_match(expr, discrim, arms, expected, match_src)
             }
             ExprKind::Closure(closure) => self.check_expr_closure(closure, expr.span, expected),
-            ExprKind::Block(body, _) => self.check_block_with_expected(body, expected),
-            ExprKind::Call(callee, args) => self.check_call(expr, callee, args, expected),
+            ExprKind::Block(body, _) => self.check_expr_block(body, expected),
+            ExprKind::Call(callee, args) => self.check_expr_call(expr, callee, args, expected),
             ExprKind::MethodCall(segment, receiver, args, _) => {
-                self.check_method_call(expr, segment, receiver, args, expected)
+                self.check_expr_method_call(expr, segment, receiver, args, expected)
             }
             ExprKind::Cast(e, t) => self.check_expr_cast(e, t, expr),
             ExprKind::Type(e, t) => {
@@ -528,7 +539,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 ascribed_ty
             }
             ExprKind::If(cond, then_expr, opt_else_expr) => {
-                self.check_then_else(cond, then_expr, opt_else_expr, expr.span, expected)
+                self.check_expr_if(cond, then_expr, opt_else_expr, expr.span, expected)
             }
             ExprKind::DropTemps(e) => self.check_expr_with_expectation(e, expected),
             ExprKind::Array(args) => self.check_expr_array(args, expected, expr),
@@ -540,7 +551,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             ExprKind::Struct(qpath, fields, ref base_expr) => {
                 self.check_expr_struct(expr, expected, qpath, fields, base_expr)
             }
-            ExprKind::Field(base, field) => self.check_field(expr, base, field, expected),
+            ExprKind::Field(base, field) => self.check_expr_field(expr, base, field, expected),
             ExprKind::Index(base, idx, brackets_span) => {
                 self.check_expr_index(base, idx, expr, brackets_span)
             }
@@ -549,7 +560,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
     }
 
-    fn check_expr_unary(
+    fn check_expr_unop(
         &self,
         unop: hir::UnOp,
         oprnd: &'tcx hir::Expr<'tcx>,
@@ -699,8 +710,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         &self,
         qpath: &'tcx hir::QPath<'tcx>,
         expr: &'tcx hir::Expr<'tcx>,
-        args: Option<&'tcx [hir::Expr<'tcx>]>,
-        call: Option<&'tcx hir::Expr<'tcx>>,
+        call_expr_and_args: Option<(&'tcx hir::Expr<'tcx>, &'tcx [hir::Expr<'tcx>])>,
     ) -> Ty<'tcx> {
         let tcx = self.tcx;
         let (res, opt_ty, segs) =
@@ -730,7 +740,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     segs,
                     opt_ty,
                     res,
-                    call.map_or(expr.span, |e| e.span),
+                    call_expr_and_args.map_or(expr.span, |(e, _)| e.span),
                     expr.span,
                     expr.hir_id,
                 )
@@ -769,7 +779,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     // We just want to check sizedness, so instead of introducing
                     // placeholder lifetimes with probing, we just replace higher lifetimes
                     // with fresh vars.
-                    let span = args.and_then(|args| args.get(i)).map_or(expr.span, |arg| arg.span);
+                    let span = call_expr_and_args
+                        .and_then(|(_, args)| args.get(i))
+                        .map_or(expr.span, |arg| arg.span);
                     let input = self.instantiate_binder_with_fresh_vars(
                         span,
                         infer::BoundRegionConversionTime::FnCall,
@@ -795,7 +807,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             );
             self.require_type_is_sized_deferred(
                 output,
-                call.map_or(expr.span, |e| e.span),
+                call_expr_and_args.map_or(expr.span, |(e, _)| e.span),
                 ObligationCauseCode::SizedCallReturnType,
             );
         }
@@ -972,7 +984,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             if self.ret_coercion_span.get().is_none() {
                 self.ret_coercion_span.set(Some(e.span));
             }
-            self.check_return_expr(e, true);
+            self.check_return_or_body_tail(e, true);
         } else {
             let mut coercion = self.ret_coercion.as_ref().unwrap().borrow_mut();
             if self.ret_coercion_span.get().is_none() {
@@ -1035,7 +1047,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     ///
     /// `explicit_return` is `true` if we're checking an explicit `return expr`,
     /// and `false` if we're checking a trailing expression.
-    pub(super) fn check_return_expr(
+    pub(super) fn check_return_or_body_tail(
         &self,
         return_expr: &'tcx hir::Expr<'tcx>,
         explicit_return: bool,
@@ -1259,7 +1271,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
     // A generic function for checking the 'then' and 'else' clauses in an 'if'
     // or 'if-else' expression.
-    fn check_then_else(
+    fn check_expr_if(
         &self,
         cond_expr: &'tcx hir::Expr<'tcx>,
         then_expr: &'tcx hir::Expr<'tcx>,
@@ -1542,7 +1554,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     }
 
     /// Checks a method call.
-    fn check_method_call(
+    fn check_expr_method_call(
         &self,
         expr: &'tcx hir::Expr<'tcx>,
         segment: &'tcx hir::PathSegment<'tcx>,
@@ -2594,7 +2606,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     }
 
     // Check field access expressions
-    fn check_field(
+    fn check_expr_field(
         &self,
         expr: &'tcx hir::Expr<'tcx>,
         base: &'tcx hir::Expr<'tcx>,
@@ -3535,8 +3547,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     let previous_diverges = self.diverges.get();
 
                     // The label blocks should have unit return value or diverge.
-                    let ty =
-                        self.check_block_with_expected(block, ExpectHasType(self.tcx.types.unit));
+                    let ty = self.check_expr_block(block, ExpectHasType(self.tcx.types.unit));
                     if !ty.is_never() {
                         self.demand_suptype(block.span, self.tcx.types.unit, ty);
                         diverge = false;
@@ -3551,7 +3562,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         if diverge { self.tcx.types.never } else { self.tcx.types.unit }
     }
 
-    fn check_offset_of(
+    fn check_expr_offset_of(
         &self,
         container: &'tcx hir::Ty<'tcx>,
         fields: &[Ident],

compiler/rustc_hir_typeck/src/fallback.rs

@@ -621,7 +621,11 @@ impl<'tcx> AnnotateUnitFallbackVisitor<'_, 'tcx> {
                 .iter()
                 .filter(|arg| matches!(arg.unpack(), ty::GenericArgKind::Type(_)))
                 .count();
-            for (idx, arg) in args.iter().enumerate() {
+            for (idx, arg) in args
+                .iter()
+                .filter(|arg| matches!(arg.unpack(), ty::GenericArgKind::Type(_)))
+                .enumerate()
+            {
                 if let Some(ty) = arg.as_type()
                     && let Some(vid) = self.fcx.root_vid(ty)
                     && self.reachable_vids.contains(&vid)

compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs

@@ -1306,30 +1306,16 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             rustc_hir_analysis::hir_ty_lowering::RegionInferReason::Param(param),
                         )
                         .into(),
-                    GenericParamDefKind::Type { has_default, .. } => {
-                        if !infer_args && has_default {
-                            // If we have a default, then it doesn't matter that we're not
-                            // inferring the type arguments: we provide the default where any
-                            // is missing.
-                            tcx.type_of(param.def_id).instantiate(tcx, preceding_args).into()
-                        } else {
-                            // If no type arguments were provided, we have to infer them.
-                            // This case also occurs as a result of some malformed input, e.g.
-                            // a lifetime argument being given instead of a type parameter.
-                            // Using inference instead of `Error` gives better error messages.
-                            self.fcx.var_for_def(self.span, param)
+                    GenericParamDefKind::Type { .. } | GenericParamDefKind::Const { .. } => {
+                        if !infer_args && let Some(default) = param.default_value(tcx) {
+                            // If we have a default, then it doesn't matter that we're not inferring
+                            // the type/const arguments: We provide the default where any is missing.
+                            return default.instantiate(tcx, preceding_args);
                         }
-                    }
-                    GenericParamDefKind::Const { has_default, .. } => {
-                        if has_default {
-                            if !infer_args {
-                                return tcx
-                                    .const_param_default(param.def_id)
-                                    .instantiate(tcx, preceding_args)
-                                    .into();
-                            }
-                        }
-
+                        // If no type/const arguments were provided, we have to infer them.
+                        // This case also occurs as a result of some malformed input, e.g.,
+                        // a lifetime argument being given instead of a type/const parameter.
+                        // Using inference instead of `Error` gives better error messages.
                         self.fcx.var_for_def(self.span, param)
                     }
                 }
@@ -1491,7 +1477,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 }
             }
         } else if self.tcx.features().generic_const_exprs() {
-            ct.normalize_internal(self.tcx, self.param_env)
+            rustc_trait_selection::traits::evaluate_const(&self.infcx, ct, self.param_env)
         } else {
             ct
         }

compiler/rustc_hir_typeck/src/fn_ctxt/adjust_fulfillment_errors.rs

@@ -316,12 +316,24 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             .tcx
             .generics_of(def_id)
             .own_args(ty::GenericArgs::identity_for_item(self.tcx, def_id));
-        let Some((index, _)) =
-            own_args.iter().enumerate().find(|(_, arg)| **arg == param_to_point_at)
-        else {
+        let Some(mut index) = own_args.iter().position(|arg| *arg == param_to_point_at) else {
             return false;
         };
-        let Some(arg) = segment.args().args.get(index) else {
+        // SUBTLE: We may or may not turbofish lifetime arguments, which will
+        // otherwise be elided. if our "own args" starts with a lifetime, but
+        // the args list does not, then we should chop off all of the lifetimes,
+        // since they're all elided.
+        let segment_args = segment.args().args;
+        if matches!(own_args[0].unpack(), ty::GenericArgKind::Lifetime(_))
+            && segment_args.first().is_some_and(|arg| arg.is_ty_or_const())
+            && let Some(offset) = own_args.iter().position(|arg| {
+                matches!(arg.unpack(), ty::GenericArgKind::Type(_) | ty::GenericArgKind::Const(_))
+            })
+            && let Some(new_index) = index.checked_sub(offset)
+        {
+            index = new_index;
+        }
+        let Some(arg) = segment_args.get(index) else {
             return false;
         };
         error.obligation.cause.span = arg

compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs

@@ -1565,7 +1565,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     }
 
     // AST fragment checking
-    pub(in super::super) fn check_lit(
+    pub(in super::super) fn check_expr_lit(
         &self,
         lit: &hir::Lit,
         expected: Expectation<'tcx>,
@@ -1747,7 +1747,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         if let Some(blk) = decl.origin.try_get_else() {
             let previous_diverges = self.diverges.get();
-            let else_ty = self.check_block_with_expected(blk, NoExpectation);
+            let else_ty = self.check_expr_block(blk, NoExpectation);
             let cause = self.cause(blk.span, ObligationCauseCode::LetElse);
             if let Err(err) = self.demand_eqtype_with_origin(&cause, self.tcx.types.never, else_ty)
             {
@@ -1805,7 +1805,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
     pub(crate) fn check_block_no_value(&self, blk: &'tcx hir::Block<'tcx>) {
         let unit = self.tcx.types.unit;
-        let ty = self.check_block_with_expected(blk, ExpectHasType(unit));
+        let ty = self.check_expr_block(blk, ExpectHasType(unit));
 
         // if the block produces a `!` value, that can always be
         // (effectively) coerced to unit.
@@ -1814,7 +1814,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
     }
 
-    pub(in super::super) fn check_block_with_expected(
+    pub(in super::super) fn check_expr_block(
         &self,
         blk: &'tcx hir::Block<'tcx>,
         expected: Expectation<'tcx>,

compiler/rustc_hir_typeck/src/lib.rs

@@ -4,6 +4,7 @@
 #![feature(array_windows)]
 #![feature(box_patterns)]
 #![feature(if_let_guard)]
+#![feature(iter_intersperse)]
 #![feature(let_chains)]
 #![feature(never_type)]
 #![feature(try_blocks)]

compiler/rustc_hir_typeck/src/method/suggest.rs

@@ -3874,22 +3874,52 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                                 param.name.ident(),
                             ));
                             let bounds_span = hir_generics.bounds_span_for_suggestions(def_id);
+                            let mut applicability = Applicability::MaybeIncorrect;
+                            // Format the path of each suggested candidate, providing placeholders
+                            // for any generic arguments without defaults.
+                            let candidate_strs: Vec<_> = candidates
+                                .iter()
+                                .map(|cand| {
+                                    let cand_path = self.tcx.def_path_str(cand.def_id);
+                                    let cand_params = &self.tcx.generics_of(cand.def_id).own_params;
+                                    let cand_args: String = cand_params
+                                        .iter()
+                                        .skip(1)
+                                        .filter_map(|param| match param.kind {
+                                            ty::GenericParamDefKind::Type {
+                                                has_default: true,
+                                                ..
+                                            }
+                                            | ty::GenericParamDefKind::Const {
+                                                has_default: true,
+                                                ..
+                                            } => None,
+                                            _ => Some(param.name.as_str()),
+                                        })
+                                        .intersperse(", ")
+                                        .collect();
+                                    if cand_args.is_empty() {
+                                        cand_path
+                                    } else {
+                                        applicability = Applicability::HasPlaceholders;
+                                        format!("{cand_path}</* {cand_args} */>")
+                                    }
+                                })
+                                .collect();
+
                             if rcvr_ty.is_ref()
                                 && param.is_impl_trait()
                                 && let Some((bounds_span, _)) = bounds_span
                             {
                                 err.multipart_suggestions(
                                     msg,
-                                    candidates.iter().map(|t| {
+                                    candidate_strs.iter().map(|cand| {
                                         vec![
                                             (param.span.shrink_to_lo(), "(".to_string()),
-                                            (
-                                                bounds_span,
-                                                format!(" + {})", self.tcx.def_path_str(t.def_id)),
-                                            ),
+                                            (bounds_span, format!(" + {cand})")),
                                         ]
                                     }),
-                                    Applicability::MaybeIncorrect,
+                                    applicability,
                                 );
                                 return;
                             }
@@ -3905,16 +3935,12 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                                     (param.span.shrink_to_hi(), Introducer::Colon, None)
                                 };
 
-                            let all_suggs = candidates.iter().map(|cand| {
-                                let suggestion = format!(
-                                    "{} {}",
-                                    match introducer {
-                                        Introducer::Plus => " +",
-                                        Introducer::Colon => ":",
-                                        Introducer::Nothing => "",
-                                    },
-                                    self.tcx.def_path_str(cand.def_id)
-                                );
+                            let all_suggs = candidate_strs.iter().map(|cand| {
+                                let suggestion = format!("{} {cand}", match introducer {
+                                    Introducer::Plus => " +",
+                                    Introducer::Colon => ":",
+                                    Introducer::Nothing => "",
+                                },);
 
                                 let mut suggs = vec![];
 
@@ -3928,11 +3954,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                                 suggs
                             });
 
-                            err.multipart_suggestions(
-                                msg,
-                                all_suggs,
-                                Applicability::MaybeIncorrect,
-                            );
+                            err.multipart_suggestions(msg, all_suggs, applicability);
 
                             return;
                         }

compiler/rustc_hir_typeck/src/op.rs

@@ -26,7 +26,7 @@ use crate::Expectation;
 
 impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     /// Checks a `a <op>= b`
-    pub(crate) fn check_binop_assign(
+    pub(crate) fn check_expr_binop_assign(
         &self,
         expr: &'tcx hir::Expr<'tcx>,
         op: hir::BinOp,
@@ -85,7 +85,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     }
 
     /// Checks a potentially overloaded binary operator.
-    pub(crate) fn check_binop(
+    pub(crate) fn check_expr_binop(
         &self,
         expr: &'tcx hir::Expr<'tcx>,
         op: hir::BinOp,

compiler/rustc_infer/src/infer/mod.rs

@@ -25,10 +25,10 @@ use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_macros::extension;
 pub use rustc_macros::{TypeFoldable, TypeVisitable};
+use rustc_middle::bug;
 use rustc_middle::infer::canonical::{CanonicalQueryInput, CanonicalVarValues};
 use rustc_middle::infer::unify_key::{ConstVariableOrigin, ConstVariableValue, ConstVidKey};
 use rustc_middle::mir::ConstraintCategory;
-use rustc_middle::mir::interpret::{ErrorHandled, EvalToValTreeResult};
 use rustc_middle::traits::select;
 pub use rustc_middle::ty::IntVarValue;
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
@@ -40,7 +40,6 @@ use rustc_middle::ty::{
     self, ConstVid, FloatVid, GenericArg, GenericArgKind, GenericArgs, GenericArgsRef,
     GenericParamDefKind, InferConst, IntVid, Ty, TyCtxt, TyVid, TypingMode,
 };
-use rustc_middle::{bug, span_bug};
 use rustc_span::Span;
 use rustc_span::symbol::Symbol;
 use rustc_type_ir::solve::Reveal;
@@ -1279,84 +1278,6 @@ impl<'tcx> InferCtxt<'tcx> {
         u
     }
 
-    pub fn try_const_eval_resolve(
-        &self,
-        param_env: ty::ParamEnv<'tcx>,
-        unevaluated: ty::UnevaluatedConst<'tcx>,
-        span: Span,
-    ) -> Result<ty::Const<'tcx>, ErrorHandled> {
-        match self.const_eval_resolve(param_env, unevaluated, span) {
-            Ok(Ok(val)) => Ok(ty::Const::new_value(
-                self.tcx,
-                val,
-                self.tcx.type_of(unevaluated.def).instantiate(self.tcx, unevaluated.args),
-            )),
-            Ok(Err(bad_ty)) => {
-                let tcx = self.tcx;
-                let def_id = unevaluated.def;
-                span_bug!(
-                    tcx.def_span(def_id),
-                    "unable to construct a valtree for the unevaluated constant {:?}: type {bad_ty} is not valtree-compatible",
-                    unevaluated
-                );
-            }
-            Err(err) => Err(err),
-        }
-    }
-
-    /// Resolves and evaluates a constant.
-    ///
-    /// The constant can be located on a trait like `<A as B>::C`, in which case the given
-    /// generic parameters and environment are used to resolve the constant. Alternatively if the
-    /// constant has generic parameters in scope the instantiations are used to evaluate the value
-    /// of the constant. For example in `fn foo<T>() { let _ = [0; bar::<T>()]; }` the repeat count
-    /// constant `bar::<T>()` requires a instantiation for `T`, if the instantiation for `T` is
-    /// still too generic for the constant to be evaluated then `Err(ErrorHandled::TooGeneric)` is
-    /// returned.
-    ///
-    /// This handles inferences variables within both `param_env` and `args` by
-    /// performing the operation on their respective canonical forms.
-    #[instrument(skip(self), level = "debug")]
-    pub fn const_eval_resolve(
-        &self,
-        mut param_env: ty::ParamEnv<'tcx>,
-        unevaluated: ty::UnevaluatedConst<'tcx>,
-        span: Span,
-    ) -> EvalToValTreeResult<'tcx> {
-        let mut args = self.resolve_vars_if_possible(unevaluated.args);
-        debug!(?args);
-
-        // Postpone the evaluation of constants whose args depend on inference
-        // variables
-        let tcx = self.tcx;
-        if args.has_non_region_infer() {
-            if let Some(ct) = tcx.thir_abstract_const(unevaluated.def)? {
-                let ct = tcx.expand_abstract_consts(ct.instantiate(tcx, args));
-                if let Err(e) = ct.error_reported() {
-                    return Err(ErrorHandled::Reported(e.into(), span));
-                } else if ct.has_non_region_infer() || ct.has_non_region_param() {
-                    return Err(ErrorHandled::TooGeneric(span));
-                } else {
-                    args = replace_param_and_infer_args_with_placeholder(tcx, args);
-                }
-            } else {
-                args = GenericArgs::identity_for_item(tcx, unevaluated.def);
-                param_env = tcx.param_env(unevaluated.def);
-            }
-        }
-
-        let param_env_erased = tcx.erase_regions(param_env);
-        let args_erased = tcx.erase_regions(args);
-        debug!(?param_env_erased);
-        debug!(?args_erased);
-
-        let unevaluated = ty::UnevaluatedConst { def: unevaluated.def, args: args_erased };
-
-        // The return value is the evaluated value which doesn't contain any reference to inference
-        // variables, thus we don't need to instantiate back the original values.
-        tcx.const_eval_resolve_for_typeck(param_env_erased, unevaluated, span)
-    }
-
     /// The returned function is used in a fast path. If it returns `true` the variable is
     /// unchanged, `false` indicates that the status is unknown.
     #[inline]
@@ -1622,61 +1543,6 @@ impl RegionVariableOrigin {
     }
 }
 
-/// Replaces args that reference param or infer variables with suitable
-/// placeholders. This function is meant to remove these param and infer
-/// args when they're not actually needed to evaluate a constant.
-fn replace_param_and_infer_args_with_placeholder<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    args: GenericArgsRef<'tcx>,
-) -> GenericArgsRef<'tcx> {
-    struct ReplaceParamAndInferWithPlaceholder<'tcx> {
-        tcx: TyCtxt<'tcx>,
-        idx: u32,
-    }
-
-    impl<'tcx> TypeFolder<TyCtxt<'tcx>> for ReplaceParamAndInferWithPlaceholder<'tcx> {
-        fn cx(&self) -> TyCtxt<'tcx> {
-            self.tcx
-        }
-
-        fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
-            if let ty::Infer(_) = t.kind() {
-                let idx = {
-                    let idx = self.idx;
-                    self.idx += 1;
-                    idx
-                };
-                Ty::new_placeholder(self.tcx, ty::PlaceholderType {
-                    universe: ty::UniverseIndex::ROOT,
-                    bound: ty::BoundTy {
-                        var: ty::BoundVar::from_u32(idx),
-                        kind: ty::BoundTyKind::Anon,
-                    },
-                })
-            } else {
-                t.super_fold_with(self)
-            }
-        }
-
-        fn fold_const(&mut self, c: ty::Const<'tcx>) -> ty::Const<'tcx> {
-            if let ty::ConstKind::Infer(_) = c.kind() {
-                ty::Const::new_placeholder(self.tcx, ty::PlaceholderConst {
-                    universe: ty::UniverseIndex::ROOT,
-                    bound: ty::BoundVar::from_u32({
-                        let idx = self.idx;
-                        self.idx += 1;
-                        idx
-                    }),
-                })
-            } else {
-                c.super_fold_with(self)
-            }
-        }
-    }
-
-    args.fold_with(&mut ReplaceParamAndInferWithPlaceholder { tcx, idx: 0 })
-}
-
 impl<'tcx> InferCtxt<'tcx> {
     /// Given a [`hir::Block`], get the span of its last expression or
     /// statement, peeling off any inner blocks.

compiler/rustc_interface/Cargo.toml

@@ -5,8 +5,8 @@ edition = "2021"
 
 [dependencies]
 # tidy-alphabetical-start
-rustc-rayon = { version = "0.5.0", optional = true }
-rustc-rayon-core = { version = "0.5.0", optional = true }
+rustc-rayon = { version = "0.5.0" }
+rustc-rayon-core = { version = "0.5.0" }
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_lowering = { path = "../rustc_ast_lowering" }
 rustc_ast_passes = { path = "../rustc_ast_passes" }
@@ -54,10 +54,4 @@ tracing = "0.1"
 [features]
 # tidy-alphabetical-start
 llvm = ['dep:rustc_codegen_llvm']
-rustc_use_parallel_compiler = [
-    'dep:rustc-rayon',
-    'dep:rustc-rayon-core',
-    'rustc_query_impl/rustc_use_parallel_compiler',
-    'rustc_errors/rustc_use_parallel_compiler'
-]
 # tidy-alphabetical-end

compiler/rustc_interface/src/passes.rs

@@ -175,7 +175,7 @@ fn configure_and_expand(
         if cfg!(windows) {
             old_path = env::var_os("PATH").unwrap_or(old_path);
             let mut new_path = Vec::from_iter(
-                sess.host_filesearch(PathKind::All).search_paths().map(|p| p.dir.clone()),
+                sess.host_filesearch().search_paths(PathKind::All).map(|p| p.dir.clone()),
             );
             for path in env::split_paths(&old_path) {
                 if !new_path.contains(&path) {

compiler/rustc_interface/src/util.rs

@@ -6,7 +6,6 @@ use std::{env, iter, thread};
 
 use rustc_ast as ast;
 use rustc_codegen_ssa::traits::CodegenBackend;
-#[cfg(parallel_compiler)]
 use rustc_data_structures::sync;
 use rustc_metadata::{DylibError, load_symbol_from_dylib};
 use rustc_middle::ty::CurrentGcx;
@@ -117,19 +116,6 @@ fn run_in_thread_with_globals<F: FnOnce(CurrentGcx) -> R + Send, R: Send>(
     })
 }
 
-#[cfg(not(parallel_compiler))]
-pub(crate) fn run_in_thread_pool_with_globals<F: FnOnce(CurrentGcx) -> R + Send, R: Send>(
-    thread_builder_diag: &EarlyDiagCtxt,
-    edition: Edition,
-    _threads: usize,
-    sm_inputs: SourceMapInputs,
-    f: F,
-) -> R {
-    let thread_stack_size = init_stack_size(thread_builder_diag);
-    run_in_thread_with_globals(thread_stack_size, edition, sm_inputs, f)
-}
-
-#[cfg(parallel_compiler)]
 pub(crate) fn run_in_thread_pool_with_globals<F: FnOnce(CurrentGcx) -> R + Send, R: Send>(
     thread_builder_diag: &EarlyDiagCtxt,
     edition: Edition,

compiler/rustc_lint/messages.ftl

@@ -346,7 +346,6 @@ lint_impl_trait_overcaptures = `{$self_ty}` will capture more lifetimes than pos
         *[other] these lifetimes are
      } in scope but not mentioned in the type's bounds
     .note2 = all lifetimes in scope will be captured by `impl Trait`s in edition 2024
-    .suggestion = use the precise capturing `use<...>` syntax to make the captures explicit
 
 lint_impl_trait_redundant_captures = all possible in-scope parameters are already captured, so `use<...>` syntax is redundant
     .suggestion = remove the `use<...>` syntax

compiler/rustc_lint/src/builtin.rs

@@ -1298,12 +1298,30 @@ impl UnreachablePub {
         let mut applicability = Applicability::MachineApplicable;
         if cx.tcx.visibility(def_id).is_public() && !cx.effective_visibilities.is_reachable(def_id)
         {
+            // prefer suggesting `pub(super)` instead of `pub(crate)` when possible,
+            // except when `pub(super) == pub(crate)`
+            let new_vis = if let Some(ty::Visibility::Restricted(restricted_did)) =
+                cx.effective_visibilities.effective_vis(def_id).map(|effective_vis| {
+                    effective_vis.at_level(rustc_middle::middle::privacy::Level::Reachable)
+                })
+                && let parent_parent = cx.tcx.parent_module_from_def_id(
+                    cx.tcx.parent_module_from_def_id(def_id.into()).into(),
+                )
+                && *restricted_did == parent_parent.to_local_def_id()
+                && !restricted_did.to_def_id().is_crate_root()
+            {
+                "pub(super)"
+            } else {
+                "pub(crate)"
+            };
+
             if vis_span.from_expansion() {
                 applicability = Applicability::MaybeIncorrect;
             }
             let def_span = cx.tcx.def_span(def_id);
             cx.emit_span_lint(UNREACHABLE_PUB, def_span, BuiltinUnreachablePub {
                 what,
+                new_vis,
                 suggestion: (vis_span, applicability),
                 help: exportable,
             });

compiler/rustc_lint/src/impl_trait_overcaptures.rs

@@ -3,7 +3,7 @@ use std::cell::LazyCell;
 
 use rustc_data_structures::fx::{FxHashMap, FxIndexMap, FxIndexSet};
 use rustc_data_structures::unord::UnordSet;
-use rustc_errors::{Applicability, LintDiagnostic};
+use rustc_errors::{LintDiagnostic, Subdiagnostic};
 use rustc_hir as hir;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::{DefId, LocalDefId};
@@ -22,6 +22,9 @@ use rustc_session::lint::FutureIncompatibilityReason;
 use rustc_session::{declare_lint, declare_lint_pass};
 use rustc_span::edition::Edition;
 use rustc_span::{Span, Symbol};
+use rustc_trait_selection::errors::{
+    AddPreciseCapturingForOvercapture, impl_trait_overcapture_suggestion,
+};
 use rustc_trait_selection::traits::ObligationCtxt;
 use rustc_trait_selection::traits::outlives_bounds::InferCtxtExt;
 
@@ -259,7 +262,11 @@ where
             // If it's owned by this function
             && let opaque =
                 self.tcx.hir_node_by_def_id(opaque_def_id).expect_opaque_ty()
-            && let hir::OpaqueTyOrigin::FnReturn { parent, .. } = opaque.origin
+            // We want to recurse into RPITs and async fns, even though the latter
+            // doesn't overcapture on its own, it may mention additional RPITs
+            // in its bounds.
+            && let hir::OpaqueTyOrigin::FnReturn { parent, .. }
+                | hir::OpaqueTyOrigin::AsyncFn { parent, .. } = opaque.origin
             && parent == self.parent_def_id
         {
             let opaque_span = self.tcx.def_span(opaque_def_id);
@@ -334,32 +341,12 @@ where
                 // If we have uncaptured args, and if the opaque doesn't already have
                 // `use<>` syntax on it, and we're < edition 2024, then warn the user.
                 if !uncaptured_args.is_empty() {
-                    let suggestion = if let Ok(snippet) =
-                        self.tcx.sess.source_map().span_to_snippet(opaque_span)
-                        && snippet.starts_with("impl ")
-                    {
-                        let (lifetimes, others): (Vec<_>, Vec<_>) =
-                            captured.into_iter().partition(|def_id| {
-                                self.tcx.def_kind(*def_id) == DefKind::LifetimeParam
-                            });
-                        // Take all lifetime params first, then all others (ty/ct).
-                        let generics: Vec<_> = lifetimes
-                            .into_iter()
-                            .chain(others)
-                            .map(|def_id| self.tcx.item_name(def_id).to_string())
-                            .collect();
-                        // Make sure that we're not trying to name any APITs
-                        if generics.iter().all(|name| !name.starts_with("impl ")) {
-                            Some((
-                                format!(" + use<{}>", generics.join(", ")),
-                                opaque_span.shrink_to_hi(),
-                            ))
-                        } else {
-                            None
-                        }
-                    } else {
-                        None
-                    };
+                    let suggestion = impl_trait_overcapture_suggestion(
+                        self.tcx,
+                        opaque_def_id,
+                        self.parent_def_id,
+                        captured,
+                    );
 
                     let uncaptured_spans: Vec<_> = uncaptured_args
                         .into_iter()
@@ -451,7 +438,7 @@ struct ImplTraitOvercapturesLint<'tcx> {
     uncaptured_spans: Vec<Span>,
     self_ty: Ty<'tcx>,
     num_captured: usize,
-    suggestion: Option<(String, Span)>,
+    suggestion: Option<AddPreciseCapturingForOvercapture>,
 }
 
 impl<'a> LintDiagnostic<'a, ()> for ImplTraitOvercapturesLint<'_> {
@@ -461,13 +448,8 @@ impl<'a> LintDiagnostic<'a, ()> for ImplTraitOvercapturesLint<'_> {
             .arg("num_captured", self.num_captured)
             .span_note(self.uncaptured_spans, fluent::lint_note)
             .note(fluent::lint_note2);
-        if let Some((suggestion, span)) = self.suggestion {
-            diag.span_suggestion(
-                span,
-                fluent::lint_suggestion,
-                suggestion,
-                Applicability::MachineApplicable,
-            );
+        if let Some(suggestion) = self.suggestion {
+            suggestion.add_to_diag(diag);
         }
     }
 }

compiler/rustc_lint/src/lints.rs

@@ -254,7 +254,8 @@ impl<'a> LintDiagnostic<'a, ()> for BuiltinUngatedAsyncFnTrackCaller<'_> {
 #[diag(lint_builtin_unreachable_pub)]
 pub(crate) struct BuiltinUnreachablePub<'a> {
     pub what: &'a str,
-    #[suggestion(code = "pub(crate)")]
+    pub new_vis: &'a str,
+    #[suggestion(code = "{new_vis}")]
     pub suggestion: (Span, Applicability),
     #[help]
     pub help: bool,

compiler/rustc_llvm/llvm-wrapper/PassWrapper.cpp

@@ -882,10 +882,12 @@ extern "C" LLVMRustResult LLVMRustOptimize(
         SanitizerOptions->SanitizeKernelAddress) {
       OptimizerLastEPCallbacks.push_back(
 #if LLVM_VERSION_GE(20, 0)
-          [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level,
-                             ThinOrFullLTOPhase phase) {
+          [SanitizerOptions, TM](ModulePassManager &MPM,
+                                 OptimizationLevel Level,
+                                 ThinOrFullLTOPhase phase) {
 #else
-          [SanitizerOptions](ModulePassManager &MPM, OptimizationLevel Level) {
+          [SanitizerOptions, TM](ModulePassManager &MPM,
+                                 OptimizationLevel Level) {
 #endif
             auto CompileKernel = SanitizerOptions->SanitizeKernelAddress;
             AddressSanitizerOptions opts = AddressSanitizerOptions{
@@ -895,7 +897,12 @@ extern "C" LLVMRustResult LLVMRustOptimize(
                 /*UseAfterScope=*/true,
                 AsanDetectStackUseAfterReturnMode::Runtime,
             };
-            MPM.addPass(AddressSanitizerPass(opts));
+            MPM.addPass(AddressSanitizerPass(
+                opts,
+                /*UseGlobalGC*/ true,
+                // UseOdrIndicator should be false on windows machines
+                // https://reviews.llvm.org/D137227
+                !TM->getTargetTriple().isOSWindows()));
           });
     }
     if (SanitizerOptions->SanitizeHWAddress) {

compiler/rustc_metadata/src/creader.rs

@@ -507,7 +507,8 @@ impl<'a, 'tcx> CrateLoader<'a, 'tcx> {
         locator.is_proc_macro = true;
         locator.target = &self.sess.host;
         locator.tuple = TargetTuple::from_tuple(config::host_tuple());
-        locator.filesearch = self.sess.host_filesearch(path_kind);
+        locator.filesearch = self.sess.host_filesearch();
+        locator.path_kind = path_kind;
 
         let Some(host_result) = self.load(locator)? else {
             return Ok(None);

compiler/rustc_metadata/src/locator.rs

@@ -253,9 +253,10 @@ pub(crate) struct CrateLocator<'a> {
     extra_filename: Option<&'a str>,
     pub target: &'a Target,
     pub tuple: TargetTuple,
-    pub filesearch: FileSearch<'a>,
+    pub filesearch: &'a FileSearch,
     pub is_proc_macro: bool,
 
+    pub path_kind: PathKind,
     // Mutable in-progress state or output.
     crate_rejections: CrateRejections,
 }
@@ -339,7 +340,8 @@ impl<'a> CrateLocator<'a> {
             extra_filename,
             target: &sess.target,
             tuple: sess.opts.target_triple.clone(),
-            filesearch: sess.target_filesearch(path_kind),
+            filesearch: sess.target_filesearch(),
+            path_kind,
             is_proc_macro: false,
             crate_rejections: CrateRejections::default(),
         }
@@ -407,47 +409,49 @@ impl<'a> CrateLocator<'a> {
         // given that `extra_filename` comes from the `-C extra-filename`
         // option and thus can be anything, and the incorrect match will be
         // handled safely in `extract_one`.
-        for search_path in self.filesearch.search_paths() {
+        for search_path in self.filesearch.search_paths(self.path_kind) {
             debug!("searching {}", search_path.dir.display());
-            for spf in search_path.files.iter() {
-                debug!("testing {}", spf.path.display());
+            let spf = &search_path.files;
 
-                let f = &spf.file_name_str;
-                let (hash, kind) = if let Some(f) = f.strip_prefix(rlib_prefix)
-                    && let Some(f) = f.strip_suffix(rlib_suffix)
-                {
-                    (f, CrateFlavor::Rlib)
-                } else if let Some(f) = f.strip_prefix(rmeta_prefix)
-                    && let Some(f) = f.strip_suffix(rmeta_suffix)
-                {
-                    (f, CrateFlavor::Rmeta)
-                } else if let Some(f) = f.strip_prefix(dylib_prefix)
-                    && let Some(f) = f.strip_suffix(dylib_suffix.as_ref())
-                {
-                    (f, CrateFlavor::Dylib)
-                } else {
-                    if f.starts_with(staticlib_prefix) && f.ends_with(staticlib_suffix.as_ref()) {
-                        self.crate_rejections.via_kind.push(CrateMismatch {
-                            path: spf.path.clone(),
-                            got: "static".to_string(),
-                        });
+            let mut should_check_staticlibs = true;
+            for (prefix, suffix, kind) in [
+                (rlib_prefix.as_str(), rlib_suffix, CrateFlavor::Rlib),
+                (rmeta_prefix.as_str(), rmeta_suffix, CrateFlavor::Rmeta),
+                (dylib_prefix, dylib_suffix, CrateFlavor::Dylib),
+            ] {
+                if prefix == staticlib_prefix && suffix == staticlib_suffix {
+                    should_check_staticlibs = false;
+                }
+                if let Some(matches) = spf.query(prefix, suffix) {
+                    for (hash, spf) in matches {
+                        info!("lib candidate: {}", spf.path.display());
+
+                        let (rlibs, rmetas, dylibs) =
+                            candidates.entry(hash.to_string()).or_default();
+                        let path =
+                            try_canonicalize(&spf.path).unwrap_or_else(|_| spf.path.to_path_buf());
+                        if seen_paths.contains(&path) {
+                            continue;
+                        };
+                        seen_paths.insert(path.clone());
+                        match kind {
+                            CrateFlavor::Rlib => rlibs.insert(path, search_path.kind),
+                            CrateFlavor::Rmeta => rmetas.insert(path, search_path.kind),
+                            CrateFlavor::Dylib => dylibs.insert(path, search_path.kind),
+                        };
                     }
-                    continue;
-                };
-
-                info!("lib candidate: {}", spf.path.display());
-
-                let (rlibs, rmetas, dylibs) = candidates.entry(hash.to_string()).or_default();
-                let path = try_canonicalize(&spf.path).unwrap_or_else(|_| spf.path.clone());
-                if seen_paths.contains(&path) {
-                    continue;
-                };
-                seen_paths.insert(path.clone());
-                match kind {
-                    CrateFlavor::Rlib => rlibs.insert(path, search_path.kind),
-                    CrateFlavor::Rmeta => rmetas.insert(path, search_path.kind),
-                    CrateFlavor::Dylib => dylibs.insert(path, search_path.kind),
-                };
+                }
+            }
+            if let Some(static_matches) = should_check_staticlibs
+                .then(|| spf.query(staticlib_prefix, staticlib_suffix))
+                .flatten()
+            {
+                for (_, spf) in static_matches {
+                    self.crate_rejections.via_kind.push(CrateMismatch {
+                        path: spf.path.to_path_buf(),
+                        got: "static".to_string(),
+                    });
+                }
             }
         }
 

compiler/rustc_metadata/src/native_libs.rs

@@ -28,10 +28,10 @@ pub fn walk_native_lib_search_dirs<R>(
     mut f: impl FnMut(&Path, bool /*is_framework*/) -> ControlFlow<R>,
 ) -> ControlFlow<R> {
     // Library search paths explicitly supplied by user (`-L` on the command line).
-    for search_path in sess.target_filesearch(PathKind::Native).cli_search_paths() {
+    for search_path in sess.target_filesearch().cli_search_paths(PathKind::Native) {
         f(&search_path.dir, false)?;
     }
-    for search_path in sess.target_filesearch(PathKind::Framework).cli_search_paths() {
+    for search_path in sess.target_filesearch().cli_search_paths(PathKind::Framework) {
         // Frameworks are looked up strictly in framework-specific paths.
         if search_path.kind != PathKind::All {
             f(&search_path.dir, true)?;

compiler/rustc_middle/Cargo.toml

@@ -11,7 +11,7 @@ either = "1.5.0"
 field-offset = "0.3.5"
 gsgdt = "0.1.2"
 polonius-engine = "0.13.0"
-rustc-rayon-core = { version = "0.5.0", optional = true }
+rustc-rayon-core = { version = "0.5.0" }
 rustc_abi = { path = "../rustc_abi" }
 rustc_apfloat = "0.2.0"
 rustc_arena = { path = "../rustc_arena" }
@@ -43,5 +43,4 @@ tracing = "0.1"
 [features]
 # tidy-alphabetical-start
 rustc_randomized_layouts = []
-rustc_use_parallel_compiler = ["dep:rustc-rayon-core"]
 # tidy-alphabetical-end

compiler/rustc_middle/src/mir/consts.rs

@@ -1,17 +1,17 @@
 use std::fmt::{self, Debug, Display, Formatter};
 
-use either::Either;
 use rustc_abi::{HasDataLayout, Size};
 use rustc_hir::def_id::DefId;
 use rustc_macros::{HashStable, Lift, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable};
 use rustc_session::RemapFileNameExt;
 use rustc_session::config::RemapPathScopeComponents;
 use rustc_span::{DUMMY_SP, Span};
+use rustc_type_ir::visit::TypeVisitableExt;
 
 use crate::mir::interpret::{AllocId, ConstAllocation, ErrorHandled, Scalar, alloc_range};
 use crate::mir::{Promoted, pretty_print_const_value};
 use crate::ty::print::{pretty_print_const, with_no_trimmed_paths};
-use crate::ty::{self, GenericArgsRef, ScalarInt, Ty, TyCtxt};
+use crate::ty::{self, ConstKind, GenericArgsRef, ScalarInt, Ty, TyCtxt};
 
 ///////////////////////////////////////////////////////////////////////////
 /// Evaluated Constants
@@ -319,15 +319,16 @@ impl<'tcx> Const<'tcx> {
     ) -> Result<ConstValue<'tcx>, ErrorHandled> {
         match self {
             Const::Ty(_, c) => {
-                // We want to consistently have a "clean" value for type system constants (i.e., no
-                // data hidden in the padding), so we always go through a valtree here.
-                match c.eval_valtree(tcx, param_env, span) {
-                    Ok((ty, val)) => Ok(tcx.valtree_to_const_val((ty, val))),
-                    Err(Either::Left(_bad_ty)) => Err(tcx
-                        .dcx()
-                        .delayed_bug("`mir::Const::eval` called on a non-valtree-compatible type")
-                        .into()),
-                    Err(Either::Right(e)) => Err(e),
+                if c.has_non_region_param() {
+                    return Err(ErrorHandled::TooGeneric(span));
+                }
+
+                match c.kind() {
+                    ConstKind::Value(ty, val) => Ok(tcx.valtree_to_const_val((ty, val))),
+                    ConstKind::Expr(_) => {
+                        bug!("Normalization of `ty::ConstKind::Expr` is unimplemented")
+                    }
+                    _ => Err(tcx.dcx().delayed_bug("Unevaluated `ty::Const` in MIR body").into()),
                 }
             }
             Const::Unevaluated(uneval, _) => {

compiler/rustc_middle/src/mir/coverage.rs

@@ -4,7 +4,7 @@ use std::fmt::{self, Debug, Formatter};
 
 use rustc_index::IndexVec;
 use rustc_macros::{HashStable, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable};
-use rustc_span::{Span, Symbol};
+use rustc_span::Span;
 
 rustc_index::newtype_index! {
     /// Used by [`CoverageKind::BlockMarker`] to mark blocks during THIR-to-MIR
@@ -158,7 +158,6 @@ impl Debug for CoverageKind {
 #[derive(Clone, TyEncodable, TyDecodable, Hash, HashStable, PartialEq, Eq, PartialOrd, Ord)]
 #[derive(TypeFoldable, TypeVisitable)]
 pub struct SourceRegion {
-    pub file_name: Symbol,
     pub start_line: u32,
     pub start_col: u32,
     pub end_line: u32,
@@ -167,11 +166,8 @@ pub struct SourceRegion {
 
 impl Debug for SourceRegion {
     fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result {
-        write!(
-            fmt,
-            "{}:{}:{} - {}:{}",
-            self.file_name, self.start_line, self.start_col, self.end_line, self.end_col
-        )
+        let &Self { start_line, start_col, end_line, end_col } = self;
+        write!(fmt, "{start_line}:{start_col} - {end_line}:{end_col}")
     }
 }
 
@@ -246,6 +242,7 @@ pub struct Mapping {
 #[derive(TyEncodable, TyDecodable, Hash, HashStable, TypeFoldable, TypeVisitable)]
 pub struct FunctionCoverageInfo {
     pub function_source_hash: u64,
+    pub body_span: Span,
     pub num_counters: usize,
     pub mcdc_bitmap_bits: usize,
     pub expressions: IndexVec<ExpressionId, Expression>,

compiler/rustc_middle/src/mir/pretty.rs

@@ -596,8 +596,10 @@ fn write_function_coverage_info(
     function_coverage_info: &coverage::FunctionCoverageInfo,
     w: &mut dyn io::Write,
 ) -> io::Result<()> {
-    let coverage::FunctionCoverageInfo { expressions, mappings, .. } = function_coverage_info;
+    let coverage::FunctionCoverageInfo { body_span, expressions, mappings, .. } =
+        function_coverage_info;
 
+    writeln!(w, "{INDENT}coverage body span: {body_span:?}")?;
     for (id, expression) in expressions.iter_enumerated() {
         writeln!(w, "{INDENT}coverage {id:?} => {expression:?};")?;
     }

compiler/rustc_middle/src/query/mod.rs

@@ -463,7 +463,7 @@ rustc_queries! {
         separate_provide_extern
     }
 
-    /// Fetch the THIR for a given body. If typeck for that body failed, returns an empty `Thir`.
+    /// Fetch the THIR for a given body.
     query thir_body(key: LocalDefId) -> Result<(&'tcx Steal<thir::Thir<'tcx>>, thir::ExprId), ErrorGuaranteed> {
         // Perf tests revealed that hashing THIR is inefficient (see #85729).
         no_hash
@@ -2326,13 +2326,19 @@ rustc_queries! {
         separate_provide_extern
     }
 
-    /// Check the signature of this function as well as all the call expressions inside of it
-    /// to ensure that any target features required by the ABI are enabled.
-    /// Should be called on a fully monomorphized instance.
-    query check_feature_dependent_abi(key: ty::Instance<'tcx>) {
-        desc { "check for feature-dependent ABI" }
+    /// Perform monomorphization-time checking on this item.
+    /// This is used for lints/errors that can only be checked once the instance is fully
+    /// monomorphized.
+    query check_mono_item(key: ty::Instance<'tcx>) {
+        desc { "monomorphization-time checking" }
         cache_on_disk_if { true }
     }
+
+    /// Builds the set of functions that should be skipped for the move-size check.
+    query skip_move_check_fns(_: ()) -> &'tcx FxIndexSet<DefId> {
+        arena_cache
+        desc { "functions to skip for move-size check" }
+    }
 }
 
 rustc_query_append! { define_callbacks! }

compiler/rustc_middle/src/query/plumbing.rs

@@ -319,7 +319,7 @@ macro_rules! define_callbacks {
 
                 pub type Storage<'tcx> = <$($K)* as keys::Key>::Cache<Erase<$V>>;
 
-                // Ensure that keys grow no larger than 72 bytes by accident.
+                // Ensure that keys grow no larger than 80 bytes by accident.
                 // Increase this limit if necessary, but do try to keep the size low if possible
                 #[cfg(target_pointer_width = "64")]
                 const _: () = {

compiler/rustc_middle/src/thir.rs

@@ -645,7 +645,7 @@ impl<'tcx> Pat<'tcx> {
             | Binding { subpattern: Some(subpattern), .. }
             | Deref { subpattern }
             | DerefPattern { subpattern, .. }
-            | InlineConstant { subpattern, .. } => subpattern.walk_(it),
+            | ExpandedConstant { subpattern, .. } => subpattern.walk_(it),
             Leaf { subpatterns } | Variant { subpatterns, .. } => {
                 subpatterns.iter().for_each(|field| field.pattern.walk_(it))
             }
@@ -788,12 +788,17 @@ pub enum PatKind<'tcx> {
         value: mir::Const<'tcx>,
     },
 
-    /// Inline constant found while lowering a pattern.
-    InlineConstant {
-        /// [LocalDefId] of the constant, we need this so that we have a
+    /// Pattern obtained by converting a constant (inline or named) to its pattern
+    /// representation using `const_to_pat`.
+    ExpandedConstant {
+        /// [DefId] of the constant, we need this so that we have a
         /// reference that can be used by unsafety checking to visit nested
-        /// unevaluated constants.
-        def: LocalDefId,
+        /// unevaluated constants and for diagnostics. If the `DefId` doesn't
+        /// correspond to a local crate, it points at the `const` item.
+        def_id: DefId,
+        /// If `false`, then `def_id` points at a `const` item, otherwise it
+        /// corresponds to a local inline const.
+        is_inline: bool,
         /// If the inline constant is used in a range pattern, this subpattern
         /// represents the range (if both ends are inline constants, there will
         /// be multiple InlineConstant wrappers).

compiler/rustc_middle/src/thir/visit.rs

@@ -247,7 +247,7 @@ pub fn walk_pat<'thir, 'tcx: 'thir, V: Visitor<'thir, 'tcx>>(
             }
         }
         Constant { value: _ } => {}
-        InlineConstant { def: _, subpattern } => visitor.visit_pat(subpattern),
+        ExpandedConstant { def_id: _, is_inline: _, subpattern } => visitor.visit_pat(subpattern),
         Range(_) => {}
         Slice { prefix, slice, suffix } | Array { prefix, slice, suffix } => {
             for subpattern in prefix.iter() {

compiler/rustc_middle/src/ty/consts.rs

@@ -1,4 +1,3 @@
-use either::Either;
 use rustc_data_structures::intern::Interned;
 use rustc_error_messages::MultiSpan;
 use rustc_hir::def::{DefKind, Res};
@@ -9,7 +8,7 @@ use rustc_type_ir::{self as ir, TypeFlags, WithCachedTypeInfo};
 use tracing::{debug, instrument};
 
 use crate::middle::resolve_bound_vars as rbv;
-use crate::mir::interpret::{ErrorHandled, LitToConstInput, Scalar};
+use crate::mir::interpret::{LitToConstInput, Scalar};
 use crate::ty::{self, GenericArgs, ParamEnv, ParamEnvAnd, Ty, TyCtxt, TypeVisitableExt};
 
 mod int;
@@ -18,7 +17,7 @@ mod valtree;
 
 pub use int::*;
 pub use kind::*;
-use rustc_span::{DUMMY_SP, ErrorGuaranteed, Span};
+use rustc_span::{DUMMY_SP, ErrorGuaranteed};
 pub use valtree::*;
 
 pub type ConstKind<'tcx> = ir::ConstKind<TyCtxt<'tcx>>;
@@ -363,60 +362,6 @@ impl<'tcx> Const<'tcx> {
         Self::from_bits(tcx, n as u128, ParamEnv::empty().and(tcx.types.usize))
     }
 
-    /// Returns the evaluated constant as a valtree;
-    /// if that fails due to a valtree-incompatible type, indicate which type that is
-    /// by returning `Err(Left(bad_type))`.
-    #[inline]
-    pub fn eval_valtree(
-        self,
-        tcx: TyCtxt<'tcx>,
-        param_env: ParamEnv<'tcx>,
-        span: Span,
-    ) -> Result<(Ty<'tcx>, ValTree<'tcx>), Either<Ty<'tcx>, ErrorHandled>> {
-        assert!(!self.has_escaping_bound_vars(), "escaping vars in {self:?}");
-        match self.kind() {
-            ConstKind::Unevaluated(unevaluated) => {
-                // FIXME(eddyb) maybe the `const_eval_*` methods should take
-                // `ty::ParamEnvAnd` instead of having them separate.
-                let (param_env, unevaluated) = unevaluated.prepare_for_eval(tcx, param_env);
-                // try to resolve e.g. associated constants to their definition on an impl, and then
-                // evaluate the const.
-                match tcx.const_eval_resolve_for_typeck(param_env, unevaluated, span) {
-                    Ok(Ok(c)) => {
-                        Ok((tcx.type_of(unevaluated.def).instantiate(tcx, unevaluated.args), c))
-                    }
-                    Ok(Err(bad_ty)) => Err(Either::Left(bad_ty)),
-                    Err(err) => Err(Either::Right(err)),
-                }
-            }
-            ConstKind::Value(ty, val) => Ok((ty, val)),
-            ConstKind::Error(g) => Err(Either::Right(g.into())),
-            ConstKind::Param(_)
-            | ConstKind::Infer(_)
-            | ConstKind::Bound(_, _)
-            | ConstKind::Placeholder(_)
-            | ConstKind::Expr(_) => Err(Either::Right(ErrorHandled::TooGeneric(span))),
-        }
-    }
-
-    /// Normalizes the constant to a value or an error if possible.
-    #[inline]
-    pub fn normalize_internal(self, tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>) -> Self {
-        match self.eval_valtree(tcx, param_env, DUMMY_SP) {
-            Ok((ty, val)) => Self::new_value(tcx, val, ty),
-            Err(Either::Left(_bad_ty)) => {
-                // This can happen when we run on ill-typed code.
-                Self::new_error(
-                    tcx,
-                    tcx.dcx()
-                        .delayed_bug("`ty::Const::eval` called on a non-valtree-compatible type"),
-                )
-            }
-            Err(Either::Right(ErrorHandled::Reported(r, _span))) => Self::new_error(tcx, r.into()),
-            Err(Either::Right(ErrorHandled::TooGeneric(_span))) => self,
-        }
-    }
-
     /// Panics if self.kind != ty::ConstKind::Value
     pub fn to_valtree(self) -> (ty::ValTree<'tcx>, Ty<'tcx>) {
         match self.kind() {

compiler/rustc_middle/src/ty/consts/kind.rs

@@ -1,46 +1,12 @@
 use std::assert_matches::assert_matches;
 
-use rustc_macros::{HashStable, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable, extension};
+use rustc_macros::{HashStable, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable};
 
 use super::Const;
 use crate::mir;
 use crate::ty::abstract_const::CastKind;
-use crate::ty::visit::TypeVisitableExt as _;
 use crate::ty::{self, Ty, TyCtxt};
 
-#[extension(pub(crate) trait UnevaluatedConstEvalExt<'tcx>)]
-impl<'tcx> ty::UnevaluatedConst<'tcx> {
-    /// FIXME(RalfJung): I cannot explain what this does or why it makes sense, but not doing this
-    /// hurts performance.
-    #[inline]
-    fn prepare_for_eval(
-        self,
-        tcx: TyCtxt<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
-    ) -> (ty::ParamEnv<'tcx>, Self) {
-        // HACK(eddyb) this erases lifetimes even though `const_eval_resolve`
-        // also does later, but we want to do it before checking for
-        // inference variables.
-        // Note that we erase regions *before* calling `with_reveal_all_normalized`,
-        // so that we don't try to invoke this query with
-        // any region variables.
-
-        // HACK(eddyb) when the query key would contain inference variables,
-        // attempt using identity args and `ParamEnv` instead, that will succeed
-        // when the expression doesn't depend on any parameters.
-        // FIXME(eddyb, skinny121) pass `InferCtxt` into here when it's available, so that
-        // we can call `infcx.const_eval_resolve` which handles inference variables.
-        if (param_env, self).has_non_region_infer() {
-            (tcx.param_env(self.def), ty::UnevaluatedConst {
-                def: self.def,
-                args: ty::GenericArgs::identity_for_item(tcx, self.def),
-            })
-        } else {
-            (tcx.erase_regions(param_env).with_reveal_all_normalized(tcx), tcx.erase_regions(self))
-        }
-    }
-}
-
 #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)]
 #[derive(HashStable, TyEncodable, TyDecodable, TypeVisitable, TypeFoldable)]
 pub enum ExprKind {

compiler/rustc_middle/src/ty/context.rs

@@ -22,9 +22,9 @@ use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::sharded::{IntoPointer, ShardedHashMap};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::steal::Steal;
-use rustc_data_structures::sync::{self, FreezeReadGuard, Lock, Lrc, RwLock, WorkerLocal};
-#[cfg(parallel_compiler)]
-use rustc_data_structures::sync::{DynSend, DynSync};
+use rustc_data_structures::sync::{
+    self, DynSend, DynSync, FreezeReadGuard, Lock, Lrc, RwLock, WorkerLocal,
+};
 use rustc_data_structures::unord::UnordSet;
 use rustc_errors::{
     Applicability, Diag, DiagCtxtHandle, ErrorGuaranteed, LintDiagnostic, MultiSpan,
@@ -1259,9 +1259,7 @@ pub struct TyCtxt<'tcx> {
 }
 
 // Explicitly implement `DynSync` and `DynSend` for `TyCtxt` to short circuit trait resolution.
-#[cfg(parallel_compiler)]
 unsafe impl DynSend for TyCtxt<'_> {}
-#[cfg(parallel_compiler)]
 unsafe impl DynSync for TyCtxt<'_> {}
 fn _assert_tcx_fields() {
     sync::assert_dyn_sync::<&'_ GlobalCtxt<'_>>();
@@ -1383,9 +1381,7 @@ pub struct CurrentGcx {
     value: Lrc<RwLock<Option<*const ()>>>,
 }
 
-#[cfg(parallel_compiler)]
 unsafe impl DynSend for CurrentGcx {}
-#[cfg(parallel_compiler)]
 unsafe impl DynSync for CurrentGcx {}
 
 impl CurrentGcx {

compiler/rustc_middle/src/ty/context/tls.rs

@@ -1,5 +1,3 @@
-#[cfg(not(parallel_compiler))]
-use std::cell::Cell;
 use std::{mem, ptr};
 
 use rustc_data_structures::sync::{self, Lock};
@@ -50,16 +48,8 @@ impl<'a, 'tcx> ImplicitCtxt<'a, 'tcx> {
 }
 
 // Import the thread-local variable from Rayon, which is preserved for Rayon jobs.
-#[cfg(parallel_compiler)]
 use rayon_core::tlv::TLV;
 
-// Otherwise define our own
-#[cfg(not(parallel_compiler))]
-thread_local! {
-    /// A thread local variable that stores a pointer to the current `ImplicitCtxt`.
-    static TLV: Cell<*const ()> = const { Cell::new(ptr::null()) };
-}
-
 #[inline]
 fn erase(context: &ImplicitCtxt<'_, '_>) -> *const () {
     context as *const _ as *const ()

compiler/rustc_middle/src/ty/generic_args.rs

@@ -143,12 +143,10 @@ impl<'tcx> rustc_type_ir::inherent::IntoKind for GenericArg<'tcx> {
     }
 }
 
-#[cfg(parallel_compiler)]
 unsafe impl<'tcx> rustc_data_structures::sync::DynSend for GenericArg<'tcx> where
     &'tcx (Ty<'tcx>, ty::Region<'tcx>, ty::Const<'tcx>): rustc_data_structures::sync::DynSend
 {
 }
-#[cfg(parallel_compiler)]
 unsafe impl<'tcx> rustc_data_structures::sync::DynSync for GenericArg<'tcx> where
     &'tcx (Ty<'tcx>, ty::Region<'tcx>, ty::Const<'tcx>): rustc_data_structures::sync::DynSync
 {

compiler/rustc_middle/src/ty/generics.rs

@@ -86,10 +86,10 @@ impl GenericParamDef {
         tcx: TyCtxt<'tcx>,
     ) -> Option<EarlyBinder<'tcx, ty::GenericArg<'tcx>>> {
         match self.kind {
-            GenericParamDefKind::Type { has_default, .. } if has_default => {
+            GenericParamDefKind::Type { has_default: true, .. } => {
                 Some(tcx.type_of(self.def_id).map_bound(|t| t.into()))
             }
-            GenericParamDefKind::Const { has_default, .. } if has_default => {
+            GenericParamDefKind::Const { has_default: true, .. } => {
                 Some(tcx.const_param_default(self.def_id).map_bound(|c| c.into()))
             }
             _ => None,

compiler/rustc_middle/src/ty/list.rs

@@ -5,7 +5,6 @@ use std::ops::Deref;
 use std::{fmt, iter, mem, ptr, slice};
 
 use rustc_data_structures::aligned::{Aligned, align_of};
-#[cfg(parallel_compiler)]
 use rustc_data_structures::sync::DynSync;
 use rustc_serialize::{Encodable, Encoder};
 
@@ -259,7 +258,6 @@ impl<'a, H, T: Copy> IntoIterator for &'a RawList<H, T> {
 unsafe impl<H: Sync, T: Sync> Sync for RawList<H, T> {}
 
 // We need this since `List` uses extern type `OpaqueListContents`.
-#[cfg(parallel_compiler)]
 unsafe impl<H: DynSync, T: DynSync> DynSync for RawList<H, T> {}
 
 // Safety:

compiler/rustc_middle/src/ty/mod.rs

@@ -487,12 +487,10 @@ impl<'tcx> rustc_type_ir::inherent::IntoKind for Term<'tcx> {
     }
 }
 
-#[cfg(parallel_compiler)]
 unsafe impl<'tcx> rustc_data_structures::sync::DynSend for Term<'tcx> where
     &'tcx (Ty<'tcx>, Const<'tcx>): rustc_data_structures::sync::DynSend
 {
 }
-#[cfg(parallel_compiler)]
 unsafe impl<'tcx> rustc_data_structures::sync::DynSync for Term<'tcx> where
     &'tcx (Ty<'tcx>, Const<'tcx>): rustc_data_structures::sync::DynSync
 {

compiler/rustc_mir_build/src/build/custom/parse/instruction.rs

@@ -144,12 +144,20 @@ impl<'a, 'tcx> ParseCtxt<'a, 'tcx> {
         let mut targets = Vec::new();
         for arm in rest {
             let arm = &self.thir[*arm];
-            let PatKind::Constant { value } = arm.pattern.kind else {
-                return Err(ParseError {
-                    span: arm.pattern.span,
-                    item_description: format!("{:?}", arm.pattern.kind),
-                    expected: "constant pattern".to_string(),
-                });
+            let value = match arm.pattern.kind {
+                PatKind::Constant { value } => value,
+                PatKind::ExpandedConstant { ref subpattern, def_id: _, is_inline: false }
+                    if let PatKind::Constant { value } = subpattern.kind =>
+                {
+                    value
+                }
+                _ => {
+                    return Err(ParseError {
+                        span: arm.pattern.span,
+                        item_description: format!("{:?}", arm.pattern.kind),
+                        expected: "constant pattern".to_string(),
+                    });
+                }
             };
             values.push(value.eval_bits(self.tcx, self.param_env));
             targets.push(self.parse_block(arm.body)?);

compiler/rustc_mir_build/src/build/matches/match_pair.rs

@@ -162,7 +162,11 @@ impl<'pat, 'tcx> MatchPairTree<'pat, 'tcx> {
                 TestCase::Irrefutable { ascription: None, binding }
             }
 
-            PatKind::InlineConstant { subpattern: ref pattern, def, .. } => {
+            PatKind::ExpandedConstant { subpattern: ref pattern, def_id: _, is_inline: false } => {
+                subpairs.push(MatchPairTree::for_pattern(place_builder, pattern, cx));
+                default_irrefutable()
+            }
+            PatKind::ExpandedConstant { subpattern: ref pattern, def_id, is_inline: true } => {
                 // Apply a type ascription for the inline constant to the value at `match_pair.place`
                 let ascription = place.map(|source| {
                     let span = pattern.span;
@@ -173,7 +177,7 @@ impl<'pat, 'tcx> MatchPairTree<'pat, 'tcx> {
                     })
                     .args;
                     let user_ty = cx.infcx.canonicalize_user_type_annotation(ty::UserType::TypeOf(
-                        def.to_def_id(),
+                        def_id,
                         ty::UserArgs { args, user_self_ty: None },
                     ));
                     let annotation = ty::CanonicalUserTypeAnnotation {

compiler/rustc_mir_build/src/build/matches/mod.rs

@@ -917,7 +917,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 self.visit_primary_bindings(subpattern, subpattern_user_ty, f)
             }
 
-            PatKind::InlineConstant { ref subpattern, .. } => {
+            PatKind::ExpandedConstant { ref subpattern, .. } => {
                 self.visit_primary_bindings(subpattern, pattern_user_ty, f)
             }