Merge pull request #2236 from rust-lang/rustc-pull

Cargo.lock

@@ -1859,7 +1859,6 @@ checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f"
 dependencies = [
  "equivalent",
  "hashbrown 0.15.2",
- "rustc-rayon",
  "serde",
 ]
 
@@ -3240,11 +3239,12 @@ dependencies = [
 
 [[package]]
 name = "rustc-rayon"
-version = "0.5.0"
+version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eb81aadc8837ca6ecebe0fe1353f15df83b3b3cc2cf7a8afd571bc22aa121710"
+checksum = "2cd9fb077db982d7ceb42a90471e5a69a990b58f71e06f0d8340bb2cf35eb751"
 dependencies = [
  "either",
+ "indexmap",
  "rustc-rayon-core",
 ]
 
@@ -4234,6 +4234,7 @@ name = "rustc_monomorphize"
 version = "0.0.0"
 dependencies = [
  "rustc_abi",
+ "rustc_ast",
  "rustc_attr_parsing",
  "rustc_data_structures",
  "rustc_errors",
@@ -4243,6 +4244,7 @@ dependencies = [
  "rustc_middle",
  "rustc_session",
  "rustc_span",
+ "rustc_symbol_mangling",
  "rustc_target",
  "serde",
  "serde_json",

RELEASES.md

@@ -1,8 +1,22 @@
+Version 1.84.1 (2025-01-30)
+==========================
+
+<a id="1.84.1"></a>
+
+- [Fix ICE 132920 in duplicate-crate diagnostics.](https://github.com/rust-lang/rust/pull/133304/)
+- [Fix errors for overlapping impls in incremental rebuilds.](https://github.com/rust-lang/rust/pull/133828/)
+- [Fix slow compilation related to the next-generation trait solver.](https://github.com/rust-lang/rust/pull/135618/)
+- [Fix debuginfo when LLVM's location discriminator value limit is exceeded.](https://github.com/rust-lang/rust/pull/135643/)
+- Fixes for building Rust from source:
+  - [Only try to distribute `llvm-objcopy` if llvm tools are enabled.](https://github.com/rust-lang/rust/pull/134240/)
+  - [Add Profile Override for Non-Git Sources.](https://github.com/rust-lang/rust/pull/135433/)
+  - [Resolve symlinks of LLVM tool binaries before copying them.](https://github.com/rust-lang/rust/pull/135585/)
+  - [Make it possible to use ci-rustc on tarball sources.](https://github.com/rust-lang/rust/pull/135722/)
+
 Version 1.84.0 (2025-01-09)
 ==========================
 
-<a id="
-Language"></a>
+<a id="1.84.0-Language"></a>
 
 Language
 --------

REUSE.toml

@@ -92,7 +92,7 @@ SPDX-FileCopyrightText = "2015 Anders Kaseorg <andersk@mit.edu>"
 SPDX-License-Identifier = "MIT"
 
 [[annotations]]
-path = "src/librustdoc/html/static/fonts/FiraSans**"
+path = "src/librustdoc/html/static/fonts/Fira**"
 precedence = "override"
 SPDX-FileCopyrightText = ["2014, Mozilla Foundation", "2014, Telefonica S.A."]
 SPDX-License-Identifier = "OFL-1.1"

compiler/rustc_ast/src/expand/autodiff_attrs.rs

@@ -79,6 +79,7 @@ pub struct AutoDiffItem {
     pub target: String,
     pub attrs: AutoDiffAttrs,
 }
+
 #[derive(Clone, Eq, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
 pub struct AutoDiffAttrs {
     /// Conceptually either forward or reverse mode AD, as described in various autodiff papers and
@@ -231,7 +232,7 @@ impl AutoDiffAttrs {
         self.ret_activity == DiffActivity::ActiveOnly
     }
 
-    pub fn error() -> Self {
+    pub const fn error() -> Self {
         AutoDiffAttrs {
             mode: DiffMode::Error,
             ret_activity: DiffActivity::None,

compiler/rustc_ast/src/mut_visit.rs

@@ -954,8 +954,14 @@ fn walk_coroutine_kind<T: MutVisitor>(vis: &mut T, coroutine_kind: &mut Coroutin
 
 fn walk_fn<T: MutVisitor>(vis: &mut T, kind: FnKind<'_>) {
     match kind {
-        FnKind::Fn(_ctxt, _ident, FnSig { header, decl, span }, _visibility, generics, body) => {
+        FnKind::Fn(
+            _ctxt,
+            _ident,
+            _vis,
+            Fn { defaultness, generics, body, sig: FnSig { header, decl, span } },
+        ) => {
             // Identifier and visibility are visited as a part of the item.
+            visit_defaultness(vis, defaultness);
             vis.visit_fn_header(header);
             vis.visit_generics(generics);
             vis.visit_fn_decl(decl);
@@ -1205,13 +1211,8 @@ impl WalkItemKind for ItemKind {
             ItemKind::Const(item) => {
                 visit_const_item(item, vis);
             }
-            ItemKind::Fn(box Fn { defaultness, generics, sig, body }) => {
-                visit_defaultness(vis, defaultness);
-                vis.visit_fn(
-                    FnKind::Fn(FnCtxt::Free, ident, sig, visibility, generics, body),
-                    span,
-                    id,
-                );
+            ItemKind::Fn(func) => {
+                vis.visit_fn(FnKind::Fn(FnCtxt::Free, ident, visibility, &mut *func), span, id);
             }
             ItemKind::Mod(safety, mod_kind) => {
                 visit_safety(vis, safety);
@@ -1329,10 +1330,9 @@ impl WalkItemKind for AssocItemKind {
             AssocItemKind::Const(item) => {
                 visit_const_item(item, visitor);
             }
-            AssocItemKind::Fn(box Fn { defaultness, generics, sig, body }) => {
-                visit_defaultness(visitor, defaultness);
+            AssocItemKind::Fn(func) => {
                 visitor.visit_fn(
-                    FnKind::Fn(FnCtxt::Assoc(ctxt), ident, sig, visibility, generics, body),
+                    FnKind::Fn(FnCtxt::Assoc(ctxt), ident, visibility, &mut *func),
                     span,
                     id,
                 );
@@ -1476,10 +1476,9 @@ impl WalkItemKind for ForeignItemKind {
                 visitor.visit_ty(ty);
                 visit_opt(expr, |expr| visitor.visit_expr(expr));
             }
-            ForeignItemKind::Fn(box Fn { defaultness, generics, sig, body }) => {
-                visit_defaultness(visitor, defaultness);
+            ForeignItemKind::Fn(func) => {
                 visitor.visit_fn(
-                    FnKind::Fn(FnCtxt::Foreign, ident, sig, visibility, generics, body),
+                    FnKind::Fn(FnCtxt::Foreign, ident, visibility, &mut *func),
                     span,
                     id,
                 );
@@ -1965,14 +1964,7 @@ impl<N: DummyAstNode, T: DummyAstNode> DummyAstNode for crate::ast_traits::AstNo
 #[derive(Debug)]
 pub enum FnKind<'a> {
     /// E.g., `fn foo()`, `fn foo(&self)`, or `extern "Abi" fn foo()`.
-    Fn(
-        FnCtxt,
-        &'a mut Ident,
-        &'a mut FnSig,
-        &'a mut Visibility,
-        &'a mut Generics,
-        &'a mut Option<P<Block>>,
-    ),
+    Fn(FnCtxt, &'a mut Ident, &'a mut Visibility, &'a mut Fn),
 
     /// E.g., `|x, y| body`.
     Closure(

compiler/rustc_ast/src/token.rs

@@ -527,13 +527,13 @@ impl TokenKind {
 
     /// Returns tokens that are likely to be typed accidentally instead of the current token.
     /// Enables better error recovery when the wrong token is found.
-    pub fn similar_tokens(&self) -> Option<Vec<TokenKind>> {
-        match *self {
-            Comma => Some(vec![Dot, Lt, Semi]),
-            Semi => Some(vec![Colon, Comma]),
-            Colon => Some(vec![Semi]),
-            FatArrow => Some(vec![Eq, RArrow, Ge, Gt]),
-            _ => None,
+    pub fn similar_tokens(&self) -> &[TokenKind] {
+        match self {
+            Comma => &[Dot, Lt, Semi],
+            Semi => &[Colon, Comma],
+            Colon => &[Semi],
+            FatArrow => &[Eq, RArrow, Ge, Gt],
+            _ => &[],
         }
     }
 

compiler/rustc_ast/src/visit.rs

@@ -65,7 +65,7 @@ impl BoundKind {
 #[derive(Copy, Clone, Debug)]
 pub enum FnKind<'a> {
     /// E.g., `fn foo()`, `fn foo(&self)`, or `extern "Abi" fn foo()`.
-    Fn(FnCtxt, &'a Ident, &'a FnSig, &'a Visibility, &'a Generics, &'a Option<P<Block>>),
+    Fn(FnCtxt, &'a Ident, &'a Visibility, &'a Fn),
 
     /// E.g., `|x, y| body`.
     Closure(&'a ClosureBinder, &'a Option<CoroutineKind>, &'a FnDecl, &'a Expr),
@@ -74,7 +74,7 @@ pub enum FnKind<'a> {
 impl<'a> FnKind<'a> {
     pub fn header(&self) -> Option<&'a FnHeader> {
         match *self {
-            FnKind::Fn(_, _, sig, _, _, _) => Some(&sig.header),
+            FnKind::Fn(_, _, _, Fn { sig, .. }) => Some(&sig.header),
             FnKind::Closure(..) => None,
         }
     }
@@ -88,7 +88,7 @@ impl<'a> FnKind<'a> {
 
     pub fn decl(&self) -> &'a FnDecl {
         match self {
-            FnKind::Fn(_, _, sig, _, _, _) => &sig.decl,
+            FnKind::Fn(_, _, _, Fn { sig, .. }) => &sig.decl,
             FnKind::Closure(_, _, decl, _) => decl,
         }
     }
@@ -374,8 +374,8 @@ impl WalkItemKind for ItemKind {
                 try_visit!(visitor.visit_ty(ty));
                 visit_opt!(visitor, visit_expr, expr);
             }
-            ItemKind::Fn(box Fn { defaultness: _, generics, sig, body }) => {
-                let kind = FnKind::Fn(FnCtxt::Free, ident, sig, vis, generics, body);
+            ItemKind::Fn(func) => {
+                let kind = FnKind::Fn(FnCtxt::Free, ident, vis, &*func);
                 try_visit!(visitor.visit_fn(kind, span, id));
             }
             ItemKind::Mod(_unsafety, mod_kind) => match mod_kind {
@@ -715,8 +715,8 @@ impl WalkItemKind for ForeignItemKind {
                 try_visit!(visitor.visit_ty(ty));
                 visit_opt!(visitor, visit_expr, expr);
             }
-            ForeignItemKind::Fn(box Fn { defaultness: _, generics, sig, body }) => {
-                let kind = FnKind::Fn(FnCtxt::Foreign, ident, sig, vis, generics, body);
+            ForeignItemKind::Fn(func) => {
+                let kind = FnKind::Fn(FnCtxt::Foreign, ident, vis, &*func);
                 try_visit!(visitor.visit_fn(kind, span, id));
             }
             ForeignItemKind::TyAlias(box TyAlias {
@@ -858,7 +858,12 @@ pub fn walk_fn_decl<'a, V: Visitor<'a>>(
 
 pub fn walk_fn<'a, V: Visitor<'a>>(visitor: &mut V, kind: FnKind<'a>) -> V::Result {
     match kind {
-        FnKind::Fn(_ctxt, _ident, FnSig { header, decl, span: _ }, _vis, generics, body) => {
+        FnKind::Fn(
+            _ctxt,
+            _ident,
+            _vis,
+            Fn { defaultness: _, sig: FnSig { header, decl, span: _ }, generics, body },
+        ) => {
             // Identifier and visibility are visited as a part of the item.
             try_visit!(visitor.visit_fn_header(header));
             try_visit!(visitor.visit_generics(generics));
@@ -892,8 +897,8 @@ impl WalkItemKind for AssocItemKind {
                 try_visit!(visitor.visit_ty(ty));
                 visit_opt!(visitor, visit_expr, expr);
             }
-            AssocItemKind::Fn(box Fn { defaultness: _, generics, sig, body }) => {
-                let kind = FnKind::Fn(FnCtxt::Assoc(ctxt), ident, sig, vis, generics, body);
+            AssocItemKind::Fn(func) => {
+                let kind = FnKind::Fn(FnCtxt::Assoc(ctxt), ident, vis, &*func);
                 try_visit!(visitor.visit_fn(kind, span, id));
             }
             AssocItemKind::Type(box TyAlias {

compiler/rustc_ast_lowering/src/expr.rs

@@ -1391,7 +1391,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
                         None,
                     );
                     // Destructure like a unit struct.
-                    let unit_struct_pat = hir::PatKind::Path(qpath);
+                    let unit_struct_pat = hir::PatKind::Expr(self.arena.alloc(hir::PatExpr {
+                        kind: hir::PatExprKind::Path(qpath),
+                        hir_id: self.next_id(),
+                        span: self.lower_span(lhs.span),
+                    }));
                     return self.pat_without_dbm(lhs.span, unit_struct_pat);
                 }
             }
@@ -2125,7 +2129,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         self.arena.alloc(self.expr_call_mut(span, e, args))
     }
 
-    fn expr_call_lang_item_fn_mut(
+    pub(super) fn expr_call_lang_item_fn_mut(
         &mut self,
         span: Span,
         lang_item: hir::LangItem,
@@ -2135,7 +2139,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         self.expr_call_mut(span, path, args)
     }
 
-    fn expr_call_lang_item_fn(
+    pub(super) fn expr_call_lang_item_fn(
         &mut self,
         span: Span,
         lang_item: hir::LangItem,

compiler/rustc_ast_lowering/src/pat.rs

@@ -69,7 +69,16 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                             ImplTraitContext::Disallowed(ImplTraitPosition::Path),
                             None,
                         );
-                        break hir::PatKind::Path(qpath);
+                        let kind = hir::PatExprKind::Path(qpath);
+                        let span = self.lower_span(pattern.span);
+                        let expr = hir::PatExpr { hir_id: pat_hir_id, span, kind };
+                        let expr = self.arena.alloc(expr);
+                        return hir::Pat {
+                            hir_id: self.next_id(),
+                            kind: hir::PatKind::Expr(expr),
+                            span,
+                            default_binding_modes: true,
+                        };
                     }
                     PatKind::Struct(qself, path, fields, etc) => {
                         let qpath = self.lower_qpath(
@@ -304,16 +313,20 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 )
             }
             Some(res) => {
-                let hir_id = self.next_id();
                 let res = self.lower_res(res);
-                hir::PatKind::Path(hir::QPath::Resolved(
-                    None,
-                    self.arena.alloc(hir::Path {
-                        span: self.lower_span(ident.span),
-                        res,
-                        segments: arena_vec![self; hir::PathSegment::new(self.lower_ident(ident), hir_id, res)],
-                    }),
-            ))
+                let span = self.lower_span(ident.span);
+                hir::PatKind::Expr(self.arena.alloc(hir::PatExpr {
+                    kind: hir::PatExprKind::Path(hir::QPath::Resolved(
+                        None,
+                        self.arena.alloc(hir::Path {
+                            span,
+                            res,
+                            segments: arena_vec![self; hir::PathSegment::new(self.lower_ident(ident), self.next_id(), res)],
+                        }),
+                    )),
+                    hir_id: self.next_id(),
+                    span,
+                }))
             }
         }
     }

compiler/rustc_ast_passes/src/ast_validation.rs

@@ -917,7 +917,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 walk_list!(self, visit_attribute, &item.attrs);
                 return; // Avoid visiting again.
             }
-            ItemKind::Fn(box Fn { defaultness, sig, generics, body }) => {
+            ItemKind::Fn(func @ box Fn { defaultness, generics: _, sig, body }) => {
                 self.check_defaultness(item.span, *defaultness);
 
                 let is_intrinsic =
@@ -947,7 +947,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
 
                 self.visit_vis(&item.vis);
                 self.visit_ident(&item.ident);
-                let kind = FnKind::Fn(FnCtxt::Free, &item.ident, sig, &item.vis, generics, body);
+                let kind = FnKind::Fn(FnCtxt::Free, &item.ident, &item.vis, &*func);
                 self.visit_fn(kind, item.span, item.id);
                 walk_list!(self, visit_attribute, &item.attrs);
                 return; // Avoid visiting again.
@@ -1348,19 +1348,20 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
         }
 
         if let FnKind::Fn(
-            _,
-            _,
-            FnSig { header: FnHeader { ext: Extern::Implicit(extern_span), .. }, .. },
             _,
             _,
             _,
+            Fn {
+                sig: FnSig { header: FnHeader { ext: Extern::Implicit(extern_span), .. }, .. },
+                ..
+            },
         ) = fk
         {
             self.maybe_lint_missing_abi(*extern_span, id);
         }
 
         // Functions without bodies cannot have patterns.
-        if let FnKind::Fn(ctxt, _, sig, _, _, None) = fk {
+        if let FnKind::Fn(ctxt, _, _, Fn { body: None, sig, .. }) = fk {
             Self::check_decl_no_pat(&sig.decl, |span, ident, mut_ident| {
                 if mut_ident && matches!(ctxt, FnCtxt::Assoc(_)) {
                     if let Some(ident) = ident {
@@ -1394,7 +1395,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                         .is_some();
 
         let disallowed = (!tilde_const_allowed).then(|| match fk {
-            FnKind::Fn(_, ident, _, _, _, _) => TildeConstReason::Function { ident: ident.span },
+            FnKind::Fn(_, ident, _, _) => TildeConstReason::Function { ident: ident.span },
             FnKind::Closure(..) => TildeConstReason::Closure,
         });
         self.with_tilde_const(disallowed, |this| visit::walk_fn(this, fk));
@@ -1470,15 +1471,14 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
             self.outer_trait_or_trait_impl.as_ref().and_then(TraitOrTraitImpl::constness).is_some();
 
         match &item.kind {
-            AssocItemKind::Fn(box Fn { sig, generics, body, .. })
+            AssocItemKind::Fn(func)
                 if parent_is_const
                     || ctxt == AssocCtxt::Trait
-                    || matches!(sig.header.constness, Const::Yes(_)) =>
+                    || matches!(func.sig.header.constness, Const::Yes(_)) =>
             {
                 self.visit_vis(&item.vis);
                 self.visit_ident(&item.ident);
-                let kind =
-                    FnKind::Fn(FnCtxt::Assoc(ctxt), &item.ident, sig, &item.vis, generics, body);
+                let kind = FnKind::Fn(FnCtxt::Assoc(ctxt), &item.ident, &item.vis, &*func);
                 walk_list!(self, visit_attribute, &item.attrs);
                 self.visit_fn(kind, item.span, item.id);
             }

compiler/rustc_ast_pretty/src/pprust/state/item.rs

@@ -34,8 +34,8 @@ impl<'a> State<'a> {
         self.maybe_print_comment(span.lo());
         self.print_outer_attributes(attrs);
         match kind {
-            ast::ForeignItemKind::Fn(box ast::Fn { defaultness, sig, generics, body }) => {
-                self.print_fn_full(sig, ident, generics, vis, *defaultness, body.as_deref(), attrs);
+            ast::ForeignItemKind::Fn(func) => {
+                self.print_fn_full(ident, vis, attrs, &*func);
             }
             ast::ForeignItemKind::Static(box ast::StaticItem { ty, mutability, expr, safety }) => {
                 self.print_item_const(
@@ -199,16 +199,8 @@ impl<'a> State<'a> {
                     *defaultness,
                 );
             }
-            ast::ItemKind::Fn(box ast::Fn { defaultness, sig, generics, body }) => {
-                self.print_fn_full(
-                    sig,
-                    item.ident,
-                    generics,
-                    &item.vis,
-                    *defaultness,
-                    body.as_deref(),
-                    &item.attrs,
-                );
+            ast::ItemKind::Fn(func) => {
+                self.print_fn_full(item.ident, &item.vis, &item.attrs, &*func);
             }
             ast::ItemKind::Mod(safety, mod_kind) => {
                 self.head(Self::to_string(|s| {
@@ -542,8 +534,8 @@ impl<'a> State<'a> {
         self.maybe_print_comment(span.lo());
         self.print_outer_attributes(attrs);
         match kind {
-            ast::AssocItemKind::Fn(box ast::Fn { defaultness, sig, generics, body }) => {
-                self.print_fn_full(sig, ident, generics, vis, *defaultness, body.as_deref(), attrs);
+            ast::AssocItemKind::Fn(func) => {
+                self.print_fn_full(ident, vis, attrs, &*func);
             }
             ast::AssocItemKind::Const(box ast::ConstItem { defaultness, generics, ty, expr }) => {
                 self.print_item_const(
@@ -653,19 +645,17 @@ impl<'a> State<'a> {
 
     fn print_fn_full(
         &mut self,
-        sig: &ast::FnSig,
         name: Ident,
-        generics: &ast::Generics,
         vis: &ast::Visibility,
-        defaultness: ast::Defaultness,
-        body: Option<&ast::Block>,
         attrs: &[ast::Attribute],
+        func: &ast::Fn,
     ) {
+        let ast::Fn { defaultness, generics, sig, body } = func;
         if body.is_some() {
             self.head("");
         }
         self.print_visibility(vis);
-        self.print_defaultness(defaultness);
+        self.print_defaultness(*defaultness);
         self.print_fn(&sig.decl, sig.header, Some(name), generics);
         if let Some(body) = body {
             self.nbsp();

compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs

@@ -310,7 +310,7 @@ impl<'tcx> TypeOpInfo<'tcx> for AscribeUserTypeQuery<'tcx> {
         let (infcx, key, _) =
             mbcx.infcx.tcx.infer_ctxt().build_with_canonical(cause.span, &self.canonical_query);
         let ocx = ObligationCtxt::new(&infcx);
-        type_op_ascribe_user_type_with_span(&ocx, key, Some(cause.span)).ok()?;
+        type_op_ascribe_user_type_with_span(&ocx, key, cause.span).ok()?;
         let diag = try_extract_error_from_fulfill_cx(
             &ocx,
             mbcx.mir_def_id(),

compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs

@@ -3777,7 +3777,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
         let tcx = self.infcx.tcx;
         if let Some(Terminator { kind: TerminatorKind::Call { call_source, fn_span, .. }, .. }) =
             &self.body[loan.reserve_location.block].terminator
-            && let Some((method_did, method_args)) = rustc_middle::util::find_self_call(
+            && let Some((method_did, method_args)) = mir::find_self_call(
                 tcx,
                 self.body,
                 loan.assigned_place.local,

compiler/rustc_borrowck/src/diagnostics/mod.rs

@@ -17,7 +17,7 @@ use rustc_middle::mir::tcx::PlaceTy;
 use rustc_middle::mir::{
     AggregateKind, CallSource, ConstOperand, ConstraintCategory, FakeReadCause, Local, LocalInfo,
     LocalKind, Location, Operand, Place, PlaceRef, ProjectionElem, Rvalue, Statement,
-    StatementKind, Terminator, TerminatorKind,
+    StatementKind, Terminator, TerminatorKind, find_self_call,
 };
 use rustc_middle::ty::print::Print;
 use rustc_middle::ty::{self, Ty, TyCtxt};
@@ -1016,12 +1016,9 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
             kind: TerminatorKind::Call { fn_span, call_source, .. }, ..
         }) = &self.body[location.block].terminator
         {
-            let Some((method_did, method_args)) = rustc_middle::util::find_self_call(
-                self.infcx.tcx,
-                self.body,
-                target_temp,
-                location.block,
-            ) else {
+            let Some((method_did, method_args)) =
+                find_self_call(self.infcx.tcx, self.body, target_temp, location.block)
+            else {
                 return normal_ret;
             };
 

compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs

@@ -1140,10 +1140,9 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
 
         let amp_mut_sugg = match *local_decl.local_info() {
             LocalInfo::User(mir::BindingForm::ImplicitSelf(_)) => {
-                let suggestion = suggest_ampmut_self(self.infcx.tcx, decl_span);
-                let additional =
-                    local_trait.map(|span| (span, suggest_ampmut_self(self.infcx.tcx, span)));
-                Some(AmpMutSugg { has_sugg: true, span: decl_span, suggestion, additional })
+                let (span, suggestion) = suggest_ampmut_self(self.infcx.tcx, decl_span);
+                let additional = local_trait.map(|span| suggest_ampmut_self(self.infcx.tcx, span));
+                Some(AmpMutSugg { has_sugg: true, span, suggestion, additional })
             }
 
             LocalInfo::User(mir::BindingForm::Var(mir::VarBindingForm {
@@ -1202,10 +1201,11 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                                     opt_ty_info: None,
                                     ..
                                 })) => {
-                                    let sugg = suggest_ampmut_self(self.infcx.tcx, decl_span);
+                                    let (span, sugg) =
+                                        suggest_ampmut_self(self.infcx.tcx, decl_span);
                                     Some(AmpMutSugg {
                                         has_sugg: true,
-                                        span: decl_span,
+                                        span,
                                         suggestion: sugg,
                                         additional: None,
                                     })
@@ -1461,17 +1461,12 @@ fn mut_borrow_of_mutable_ref(local_decl: &LocalDecl<'_>, local_name: Option<Symb
     }
 }
 
-fn suggest_ampmut_self<'tcx>(tcx: TyCtxt<'tcx>, span: Span) -> String {
+fn suggest_ampmut_self(tcx: TyCtxt<'_>, span: Span) -> (Span, String) {
     match tcx.sess.source_map().span_to_snippet(span) {
-        Ok(snippet) => {
-            let lt_pos = snippet.find('\'');
-            if let Some(lt_pos) = lt_pos {
-                format!("&{}mut self", &snippet[lt_pos..snippet.len() - 4])
-            } else {
-                "&mut self".to_string()
-            }
+        Ok(snippet) if snippet.ends_with("self") => {
+            (span.with_hi(span.hi() - BytePos(4)).shrink_to_hi(), "mut ".to_string())
         }
-        _ => "&mut self".to_string(),
+        _ => (span, "&mut self".to_string()),
     }
 }
 

compiler/rustc_borrowck/src/lib.rs

@@ -1284,15 +1284,18 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
                 );
             }
 
-            &Rvalue::RawPtr(mutability, place) => {
-                let access_kind = match mutability {
-                    Mutability::Mut => (
+            &Rvalue::RawPtr(kind, place) => {
+                let access_kind = match kind {
+                    RawPtrKind::Mut => (
                         Deep,
                         Write(WriteKind::MutableBorrow(BorrowKind::Mut {
                             kind: MutBorrowKind::Default,
                         })),
                     ),
-                    Mutability::Not => (Deep, Read(ReadKind::Borrow(BorrowKind::Shared))),
+                    RawPtrKind::Const => (Deep, Read(ReadKind::Borrow(BorrowKind::Shared))),
+                    RawPtrKind::FakeForPtrMetadata => {
+                        (Shallow(Some(ArtificialField::ArrayLength)), Read(ReadKind::Copy))
+                    }
                 };
 
                 self.access_place(

compiler/rustc_borrowck/src/polonius/dump.rs

@@ -1,14 +1,20 @@
 use std::io;
 
+use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet};
+use rustc_index::IndexVec;
 use rustc_middle::mir::pretty::{
     PassWhere, PrettyPrintMirOptions, create_dump_file, dump_enabled, dump_mir_to_writer,
 };
-use rustc_middle::mir::{Body, ClosureRegionRequirements};
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::mir::{Body, ClosureRegionRequirements, Location};
+use rustc_middle::ty::{RegionVid, TyCtxt};
+use rustc_mir_dataflow::points::PointIndex;
 use rustc_session::config::MirIncludeSpans;
 
 use crate::borrow_set::BorrowSet;
+use crate::constraints::OutlivesConstraint;
 use crate::polonius::{LocalizedOutlivesConstraint, LocalizedOutlivesConstraintSet};
+use crate::region_infer::values::LivenessValues;
+use crate::type_check::Locations;
 use crate::{BorrowckInferCtxt, RegionInferenceContext};
 
 /// `-Zdump-mir=polonius` dumps MIR annotated with NLL and polonius specific information.
@@ -50,6 +56,8 @@ pub(crate) fn dump_polonius_mir<'tcx>(
 /// - the NLL MIR
 /// - the list of polonius localized constraints
 /// - a mermaid graph of the CFG
+/// - a mermaid graph of the NLL regions and the constraints between them
+/// - a mermaid graph of the NLL SCCs and the constraints between them
 fn emit_polonius_dump<'tcx>(
     tcx: TyCtxt<'tcx>,
     body: &Body<'tcx>,
@@ -68,25 +76,54 @@ fn emit_polonius_dump<'tcx>(
     // Section 1: the NLL + Polonius MIR.
     writeln!(out, "<div>")?;
     writeln!(out, "Raw MIR dump")?;
-    writeln!(out, "<code><pre>")?;
+    writeln!(out, "<pre><code>")?;
     emit_html_mir(
         tcx,
         body,
         regioncx,
         borrow_set,
-        localized_outlives_constraints,
+        &localized_outlives_constraints,
         closure_region_requirements,
         out,
     )?;
-    writeln!(out, "</pre></code>")?;
+    writeln!(out, "</code></pre>")?;
     writeln!(out, "</div>")?;
 
-    // Section 2: mermaid visualization of the CFG.
+    // Section 2: mermaid visualization of the polonius constraint graph.
+    writeln!(out, "<div>")?;
+    writeln!(out, "Polonius constraint graph")?;
+    writeln!(out, "<pre class='mermaid'>")?;
+    let edge_count = emit_mermaid_constraint_graph(
+        borrow_set,
+        regioncx.liveness_constraints(),
+        &localized_outlives_constraints,
+        out,
+    )?;
+    writeln!(out, "</pre>")?;
+    writeln!(out, "</div>")?;
+
+    // Section 3: mermaid visualization of the CFG.
     writeln!(out, "<div>")?;
     writeln!(out, "Control-flow graph")?;
-    writeln!(out, "<code><pre class='mermaid'>")?;
+    writeln!(out, "<pre class='mermaid'>")?;
     emit_mermaid_cfg(body, out)?;
-    writeln!(out, "</pre></code>")?;
+    writeln!(out, "</pre>")?;
+    writeln!(out, "</div>")?;
+
+    // Section 4: mermaid visualization of the NLL region graph.
+    writeln!(out, "<div>")?;
+    writeln!(out, "NLL regions")?;
+    writeln!(out, "<pre class='mermaid'>")?;
+    emit_mermaid_nll_regions(regioncx, out)?;
+    writeln!(out, "</pre>")?;
+    writeln!(out, "</div>")?;
+
+    // Section 5: mermaid visualization of the NLL SCC graph.
+    writeln!(out, "<div>")?;
+    writeln!(out, "NLL SCCs")?;
+    writeln!(out, "<pre class='mermaid'>")?;
+    emit_mermaid_nll_sccs(regioncx, out)?;
+    writeln!(out, "</pre>")?;
     writeln!(out, "</div>")?;
 
     // Finalize the dump with the HTML epilogue.
@@ -95,7 +132,11 @@ fn emit_polonius_dump<'tcx>(
         "<script src='https://cdn.jsdelivr.net/npm/mermaid/dist/mermaid.min.js'></script>"
     )?;
     writeln!(out, "<script>")?;
-    writeln!(out, "mermaid.initialize({{ startOnLoad: false, maxEdges: 100 }});")?;
+    writeln!(
+        out,
+        "mermaid.initialize({{ startOnLoad: false, maxEdges: {} }});",
+        edge_count.max(100),
+    )?;
     writeln!(out, "mermaid.run({{ querySelector: '.mermaid' }})")?;
     writeln!(out, "</script>")?;
     writeln!(out, "</body>")?;
@@ -110,7 +151,7 @@ fn emit_html_mir<'tcx>(
     body: &Body<'tcx>,
     regioncx: &RegionInferenceContext<'tcx>,
     borrow_set: &BorrowSet<'tcx>,
-    localized_outlives_constraints: LocalizedOutlivesConstraintSet,
+    localized_outlives_constraints: &LocalizedOutlivesConstraintSet,
     closure_region_requirements: &Option<ClosureRegionRequirements<'tcx>>,
     out: &mut dyn io::Write,
 ) -> io::Result<()> {
@@ -138,7 +179,7 @@ fn emit_html_mir<'tcx>(
                 regioncx,
                 closure_region_requirements,
                 borrow_set,
-                &localized_outlives_constraints,
+                localized_outlives_constraints,
                 pass_where,
                 out,
             )
@@ -261,3 +302,185 @@ fn emit_mermaid_cfg(body: &Body<'_>, out: &mut dyn io::Write) -> io::Result<()>
 
     Ok(())
 }
+
+/// Emits a region's label: index, universe, external name.
+fn render_region(
+    region: RegionVid,
+    regioncx: &RegionInferenceContext<'_>,
+    out: &mut dyn io::Write,
+) -> io::Result<()> {
+    let def = regioncx.region_definition(region);
+    let universe = def.universe;
+
+    write!(out, "'{}", region.as_usize())?;
+    if !universe.is_root() {
+        write!(out, "/{universe:?}")?;
+    }
+    if let Some(name) = def.external_name.and_then(|e| e.get_name()) {
+        write!(out, " ({name})")?;
+    }
+    Ok(())
+}
+
+/// Emits a mermaid flowchart of the NLL regions and the outlives constraints between them, similar
+/// to the graphviz version.
+fn emit_mermaid_nll_regions<'tcx>(
+    regioncx: &RegionInferenceContext<'tcx>,
+    out: &mut dyn io::Write,
+) -> io::Result<()> {
+    // The mermaid chart type: a top-down flowchart.
+    writeln!(out, "flowchart TD")?;
+
+    // Emit the region nodes.
+    for region in regioncx.var_infos.indices() {
+        write!(out, "{}[\"", region.as_usize())?;
+        render_region(region, regioncx, out)?;
+        writeln!(out, "\"]")?;
+    }
+
+    // Get a set of edges to check for the reverse edge being present.
+    let edges: FxHashSet<_> = regioncx.outlives_constraints().map(|c| (c.sup, c.sub)).collect();
+
+    // Order (and deduplicate) edges for traversal, to display them in a generally increasing order.
+    let constraint_key = |c: &OutlivesConstraint<'_>| {
+        let min = c.sup.min(c.sub);
+        let max = c.sup.max(c.sub);
+        (min, max)
+    };
+    let mut ordered_edges: Vec<_> = regioncx.outlives_constraints().collect();
+    ordered_edges.sort_by_key(|c| constraint_key(c));
+    ordered_edges.dedup_by_key(|c| constraint_key(c));
+
+    for outlives in ordered_edges {
+        // Source node.
+        write!(out, "{} ", outlives.sup.as_usize())?;
+
+        // The kind of arrow: bidirectional if the opposite edge exists in the set.
+        if edges.contains(&(outlives.sub, outlives.sup)) {
+            write!(out, "&lt;")?;
+        }
+        write!(out, "-- ")?;
+
+        // Edge label from its `Locations`.
+        match outlives.locations {
+            Locations::All(_) => write!(out, "All")?,
+            Locations::Single(location) => write!(out, "{:?}", location)?,
+        }
+
+        // Target node.
+        writeln!(out, " --> {}", outlives.sub.as_usize())?;
+    }
+    Ok(())
+}
+
+/// Emits a mermaid flowchart of the NLL SCCs and the outlives constraints between them, similar
+/// to the graphviz version.
+fn emit_mermaid_nll_sccs<'tcx>(
+    regioncx: &RegionInferenceContext<'tcx>,
+    out: &mut dyn io::Write,
+) -> io::Result<()> {
+    // The mermaid chart type: a top-down flowchart.
+    writeln!(out, "flowchart TD")?;
+
+    // Gather and emit the SCC nodes.
+    let mut nodes_per_scc: IndexVec<_, _> =
+        regioncx.constraint_sccs().all_sccs().map(|_| Vec::new()).collect();
+    for region in regioncx.var_infos.indices() {
+        let scc = regioncx.constraint_sccs().scc(region);
+        nodes_per_scc[scc].push(region);
+    }
+    for (scc, regions) in nodes_per_scc.iter_enumerated() {
+        // The node label: the regions contained in the SCC.
+        write!(out, "{scc}[\"SCC({scc}) = {{", scc = scc.as_usize())?;
+        for (idx, &region) in regions.iter().enumerate() {
+            render_region(region, regioncx, out)?;
+            if idx < regions.len() - 1 {
+                write!(out, ",")?;
+            }
+        }
+        writeln!(out, "}}\"]")?;
+    }
+
+    // Emit the edges between SCCs.
+    let edges = regioncx.constraint_sccs().all_sccs().flat_map(|source| {
+        regioncx.constraint_sccs().successors(source).iter().map(move |&target| (source, target))
+    });
+    for (source, target) in edges {
+        writeln!(out, "{} --> {}", source.as_usize(), target.as_usize())?;
+    }
+
+    Ok(())
+}
+
+/// Emits a mermaid flowchart of the polonius localized outlives constraints, with subgraphs per
+/// region, and loan introductions.
+fn emit_mermaid_constraint_graph<'tcx>(
+    borrow_set: &BorrowSet<'tcx>,
+    liveness: &LivenessValues,
+    localized_outlives_constraints: &LocalizedOutlivesConstraintSet,
+    out: &mut dyn io::Write,
+) -> io::Result<usize> {
+    let location_name = |location: Location| {
+        // A MIR location looks like `bb5[2]`. As that is not a syntactically valid mermaid node id,
+        // transform it into `BB5_2`.
+        format!("BB{}_{}", location.block.index(), location.statement_index)
+    };
+    let region_name = |region: RegionVid| format!("'{}", region.index());
+    let node_name = |region: RegionVid, point: PointIndex| {
+        let location = liveness.location_from_point(point);
+        format!("{}_{}", region_name(region), location_name(location))
+    };
+
+    // The mermaid chart type: a top-down flowchart, which supports subgraphs.
+    writeln!(out, "flowchart TD")?;
+
+    // The loans subgraph: a node per loan.
+    writeln!(out, "    subgraph \"Loans\"")?;
+    for loan_idx in 0..borrow_set.len() {
+        writeln!(out, "        L{loan_idx}")?;
+    }
+    writeln!(out, "    end\n")?;
+
+    // And an edge from that loan node to where it enters the constraint graph.
+    for (loan_idx, loan) in borrow_set.iter_enumerated() {
+        writeln!(
+            out,
+            "    L{} --> {}_{}",
+            loan_idx.index(),
+            region_name(loan.region),
+            location_name(loan.reserve_location),
+        )?;
+    }
+    writeln!(out, "")?;
+
+    // The regions subgraphs containing the region/point nodes.
+    let mut points_per_region: FxIndexMap<RegionVid, FxIndexSet<PointIndex>> =
+        FxIndexMap::default();
+    for constraint in &localized_outlives_constraints.outlives {
+        points_per_region.entry(constraint.source).or_default().insert(constraint.from);
+        points_per_region.entry(constraint.target).or_default().insert(constraint.to);
+    }
+    for (region, points) in points_per_region {
+        writeln!(out, "    subgraph \"{}\"", region_name(region))?;
+        for point in points {
+            writeln!(out, "        {}", node_name(region, point))?;
+        }
+        writeln!(out, "    end\n")?;
+    }
+
+    // The constraint graph edges.
+    for constraint in &localized_outlives_constraints.outlives {
+        // FIXME: add killed loans and constraint kind as edge labels.
+        writeln!(
+            out,
+            "    {} --> {}",
+            node_name(constraint.source, constraint.from),
+            node_name(constraint.target, constraint.to),
+        )?;
+    }
+
+    // Return the number of edges: this is the biggest graph in the dump and its edge count will be
+    // mermaid's max edge count to support.
+    let edge_count = borrow_set.len() + localized_outlives_constraints.outlives.len();
+    Ok(edge_count)
+}

compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs

@@ -3,11 +3,7 @@ use std::ops::ControlFlow;
 use rustc_data_structures::graph::dominators::Dominators;
 use rustc_middle::bug;
 use rustc_middle::mir::visit::Visitor;
-use rustc_middle::mir::{
-    self, BasicBlock, Body, BorrowKind, FakeBorrowKind, InlineAsmOperand, Location, Mutability,
-    NonDivergingIntrinsic, Operand, Place, Rvalue, Statement, StatementKind, Terminator,
-    TerminatorKind,
-};
+use rustc_middle::mir::*;
 use rustc_middle::ty::TyCtxt;
 use tracing::debug;
 
@@ -60,7 +56,7 @@ impl<'a, 'tcx> Visitor<'tcx> for LoanInvalidationsGenerator<'a, 'tcx> {
             StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(op)) => {
                 self.consume_operand(location, op);
             }
-            StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
+            StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(CopyNonOverlapping {
                 src,
                 dst,
                 count,
@@ -273,15 +269,18 @@ impl<'a, 'tcx> LoanInvalidationsGenerator<'a, 'tcx> {
                 self.access_place(location, place, access_kind, LocalMutationIsAllowed::No);
             }
 
-            &Rvalue::RawPtr(mutability, place) => {
-                let access_kind = match mutability {
-                    Mutability::Mut => (
+            &Rvalue::RawPtr(kind, place) => {
+                let access_kind = match kind {
+                    RawPtrKind::Mut => (
                         Deep,
                         Write(WriteKind::MutableBorrow(BorrowKind::Mut {
-                            kind: mir::MutBorrowKind::Default,
+                            kind: MutBorrowKind::Default,
                         })),
                     ),
-                    Mutability::Not => (Deep, Read(ReadKind::Borrow(BorrowKind::Shared))),
+                    RawPtrKind::Const => (Deep, Read(ReadKind::Borrow(BorrowKind::Shared))),
+                    RawPtrKind::FakeForPtrMetadata => {
+                        (Shallow(Some(ArtificialField::ArrayLength)), Read(ReadKind::Copy))
+                    }
                 };
 
                 self.access_place(location, place, access_kind, LocalMutationIsAllowed::No);

compiler/rustc_borrowck/src/region_infer/opaque_types.rs

@@ -1,6 +1,8 @@
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_errors::ErrorGuaranteed;
+use rustc_hir::OpaqueTyOrigin;
 use rustc_hir::def_id::LocalDefId;
+use rustc_infer::infer::outlives::env::OutlivesEnvironment;
 use rustc_infer::infer::{InferCtxt, NllRegionVariableOrigin, TyCtxtInferExt as _};
 use rustc_macros::extension;
 use rustc_middle::ty::fold::fold_regions;
@@ -10,6 +12,7 @@ use rustc_middle::ty::{
     TypingMode,
 };
 use rustc_span::Span;
+use rustc_trait_selection::regions::OutlivesEnvironmentBuildExt;
 use rustc_trait_selection::traits::ObligationCtxt;
 use tracing::{debug, instrument};
 
@@ -406,10 +409,6 @@ impl<'tcx> LazyOpaqueTyEnv<'tcx> {
     }
 
     fn get_canonical_args(&self) -> ty::GenericArgsRef<'tcx> {
-        use rustc_hir as hir;
-        use rustc_infer::infer::outlives::env::OutlivesEnvironment;
-        use rustc_trait_selection::traits::outlives_bounds::InferCtxtExt as _;
-
         if let Some(&canonical_args) = self.canonical_args.get() {
             return canonical_args;
         }
@@ -417,9 +416,9 @@ impl<'tcx> LazyOpaqueTyEnv<'tcx> {
         let &Self { tcx, def_id, .. } = self;
         let origin = tcx.local_opaque_ty_origin(def_id);
         let parent = match origin {
-            hir::OpaqueTyOrigin::FnReturn { parent, .. }
-            | hir::OpaqueTyOrigin::AsyncFn { parent, .. }
-            | hir::OpaqueTyOrigin::TyAlias { parent, .. } => parent,
+            OpaqueTyOrigin::FnReturn { parent, .. }
+            | OpaqueTyOrigin::AsyncFn { parent, .. }
+            | OpaqueTyOrigin::TyAlias { parent, .. } => parent,
         };
         let param_env = tcx.param_env(parent);
         let args = GenericArgs::identity_for_item(tcx, parent).extend_to(
@@ -439,8 +438,7 @@ impl<'tcx> LazyOpaqueTyEnv<'tcx> {
             tcx.dcx().span_delayed_bug(tcx.def_span(def_id), "error getting implied bounds");
             Default::default()
         });
-        let implied_bounds = infcx.implied_bounds_tys(param_env, parent, &wf_tys);
-        let outlives_env = OutlivesEnvironment::with_bounds(param_env, implied_bounds);
+        let outlives_env = OutlivesEnvironment::new(&infcx, parent, param_env, wf_tys);
 
         let mut seen = vec![tcx.lifetimes.re_static];
         let canonical_args = fold_regions(tcx, args, |r1, _| {

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -349,8 +349,8 @@ impl<'a, 'b, 'tcx> Visitor<'tcx> for TypeVerifier<'a, 'b, 'tcx> {
             let tcx = self.tcx();
             let maybe_uneval = match constant.const_ {
                 Const::Ty(_, ct) => match ct.kind() {
-                    ty::ConstKind::Unevaluated(_) => {
-                        bug!("should not encounter unevaluated Const::Ty here, got {:?}", ct)
+                    ty::ConstKind::Unevaluated(uv) => {
+                        Some(UnevaluatedConst { def: uv.def, args: uv.args, promoted: None })
                     }
                     _ => None,
                 },

compiler/rustc_builtin_macros/src/asm.rs

@@ -651,7 +651,7 @@ fn expand_preparsed_asm(
             .map(|span| template_span.from_inner(InnerSpan::new(span.start, span.end)));
         for piece in unverified_pieces {
             match piece {
-                parse::Piece::String(s) => {
+                parse::Piece::Lit(s) => {
                     template.push(ast::InlineAsmTemplatePiece::String(s.to_string().into()))
                 }
                 parse::Piece::NextArgument(arg) => {

compiler/rustc_builtin_macros/src/format.rs

@@ -101,15 +101,14 @@ fn parse_args<'a>(ecx: &ExtCtxt<'a>, sp: Span, tts: TokenStream) -> PResult<'a,
 
             match p.expect(exp!(Comma)) {
                 Err(err) => {
-                    match token::TokenKind::Comma.similar_tokens() {
-                        Some(tks) if tks.contains(&p.token.kind) => {
-                            // If a similar token is found, then it may be a typo. We
-                            // consider it as a comma, and continue parsing.
-                            err.emit();
-                            p.bump();
-                        }
+                    if token::TokenKind::Comma.similar_tokens().contains(&p.token.kind) {
+                        // If a similar token is found, then it may be a typo. We
+                        // consider it as a comma, and continue parsing.
+                        err.emit();
+                        p.bump();
+                    } else {
                         // Otherwise stop the parsing and return the error.
-                        _ => return Err(err),
+                        return Err(err);
                     }
                 }
                 Ok(Recovered::Yes(_)) => (),
@@ -406,7 +405,7 @@ fn make_format_args(
 
     for piece in &pieces {
         match *piece {
-            parse::Piece::String(s) => {
+            parse::Piece::Lit(s) => {
                 unfinished_literal.push_str(s);
             }
             parse::Piece::NextArgument(box parse::Argument { position, position_span, format }) => {

compiler/rustc_codegen_cranelift/src/base.rs

@@ -417,6 +417,16 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
                             Some(source_info.span),
                         );
                     }
+                    AssertKind::NullPointerDereference => {
+                        let location = fx.get_caller_location(source_info).load_scalar(fx);
+
+                        codegen_panic_inner(
+                            fx,
+                            rustc_hir::LangItem::PanicNullPointerDereference,
+                            &[location],
+                            Some(source_info.span),
+                        )
+                    }
                     _ => {
                         let location = fx.get_caller_location(source_info).load_scalar(fx);
 

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -6,7 +6,7 @@ use cranelift_module::*;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
 use rustc_middle::mir::interpret::{AllocId, GlobalAlloc, Scalar, read_target_uint};
-use rustc_middle::ty::{Binder, ExistentialTraitRef, ScalarInt};
+use rustc_middle::ty::{ExistentialTraitRef, ScalarInt};
 
 use crate::prelude::*;
 
@@ -167,7 +167,9 @@ pub(crate) fn codegen_const_value<'tcx>(
                             &mut fx.constants_cx,
                             fx.module,
                             ty,
-                            dyn_ty.principal(),
+                            dyn_ty.principal().map(|principal| {
+                                fx.tcx.instantiate_bound_regions_with_erased(principal)
+                            }),
                         );
                         let local_data_id =
                             fx.module.declare_data_in_func(data_id, &mut fx.bcx.func);
@@ -243,7 +245,7 @@ pub(crate) fn data_id_for_vtable<'tcx>(
     cx: &mut ConstantCx,
     module: &mut dyn Module,
     ty: Ty<'tcx>,
-    trait_ref: Option<Binder<'tcx, ExistentialTraitRef<'tcx>>>,
+    trait_ref: Option<ExistentialTraitRef<'tcx>>,
 ) -> DataId {
     let alloc_id = tcx.vtable_allocation((ty, trait_ref));
     data_id_for_alloc_id(cx, module, alloc_id, Mutability::Not)
@@ -460,9 +462,15 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
                 GlobalAlloc::Memory(target_alloc) => {
                     data_id_for_alloc_id(cx, module, alloc_id, target_alloc.inner().mutability)
                 }
-                GlobalAlloc::VTable(ty, dyn_ty) => {
-                    data_id_for_vtable(tcx, cx, module, ty, dyn_ty.principal())
-                }
+                GlobalAlloc::VTable(ty, dyn_ty) => data_id_for_vtable(
+                    tcx,
+                    cx,
+                    module,
+                    ty,
+                    dyn_ty
+                        .principal()
+                        .map(|principal| tcx.instantiate_bound_regions_with_erased(principal)),
+                ),
                 GlobalAlloc::Static(def_id) => {
                     if tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::THREAD_LOCAL)
                     {

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -129,12 +129,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 return;
             }
 
-            let idx = generic_args[2]
-                .expect_const()
-                .try_to_valtree()
-                .expect("expected monomorphic const in codegen")
-                .0
-                .unwrap_branch();
+            let idx = generic_args[2].expect_const().to_value().valtree.unwrap_branch();
 
             assert_eq!(x.layout(), y.layout());
             let layout = x.layout();

compiler/rustc_codegen_cranelift/src/unsize.rs

@@ -61,7 +61,12 @@ pub(crate) fn unsized_info<'tcx>(
                 old_info
             }
         }
-        (_, ty::Dynamic(data, ..)) => crate::vtable::get_vtable(fx, source, data.principal()),
+        (_, ty::Dynamic(data, ..)) => crate::vtable::get_vtable(
+            fx,
+            source,
+            data.principal()
+                .map(|principal| fx.tcx.instantiate_bound_regions_with_erased(principal)),
+        ),
         _ => bug!("unsized_info: invalid unsizing {:?} -> {:?}", source, target),
     }
 }

compiler/rustc_codegen_cranelift/src/vtable.rs

@@ -90,7 +90,7 @@ pub(crate) fn get_ptr_and_method_ref<'tcx>(
 pub(crate) fn get_vtable<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     ty: Ty<'tcx>,
-    trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
+    trait_ref: Option<ty::ExistentialTraitRef<'tcx>>,
 ) -> Value {
     let data_id = data_id_for_vtable(fx.tcx, &mut fx.constants_cx, fx.module, ty, trait_ref);
     let local_data_id = fx.module.declare_data_in_func(data_id, fx.bcx.func);

compiler/rustc_codegen_gcc/src/common.rs

@@ -234,7 +234,12 @@ impl<'gcc, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
                     GlobalAlloc::VTable(ty, dyn_ty) => {
                         let alloc = self
                             .tcx
-                            .global_alloc(self.tcx.vtable_allocation((ty, dyn_ty.principal())))
+                            .global_alloc(self.tcx.vtable_allocation((
+                                ty,
+                                dyn_ty.principal().map(|principal| {
+                                    self.tcx.instantiate_bound_regions_with_erased(principal)
+                                }),
+                            )))
                             .unwrap_memory();
                         let init = const_alloc_to_gcc(self, alloc);
                         self.static_addr_of(init, alloc.inner().align, None)

compiler/rustc_codegen_gcc/src/context.rs

@@ -14,7 +14,7 @@ use rustc_middle::ty::layout::{
     FnAbiError, FnAbiOf, FnAbiOfHelpers, FnAbiRequest, HasTyCtxt, HasTypingEnv, LayoutError,
     LayoutOfHelpers,
 };
-use rustc_middle::ty::{self, Instance, PolyExistentialTraitRef, Ty, TyCtxt};
+use rustc_middle::ty::{self, ExistentialTraitRef, Instance, Ty, TyCtxt};
 use rustc_session::Session;
 use rustc_span::source_map::respan;
 use rustc_span::{DUMMY_SP, Span};
@@ -90,7 +90,7 @@ pub struct CodegenCx<'gcc, 'tcx> {
     pub function_instances: RefCell<FxHashMap<Instance<'tcx>, Function<'gcc>>>,
     /// Cache generated vtables
     pub vtables:
-        RefCell<FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), RValue<'gcc>>>,
+        RefCell<FxHashMap<(Ty<'tcx>, Option<ty::ExistentialTraitRef<'tcx>>), RValue<'gcc>>>,
 
     // TODO(antoyo): improve the SSA API to not require those.
     /// Mapping from function pointer type to indexes of on stack parameters.
@@ -401,7 +401,7 @@ impl<'gcc, 'tcx> BackendTypes for CodegenCx<'gcc, 'tcx> {
 impl<'gcc, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
     fn vtables(
         &self,
-    ) -> &RefCell<FxHashMap<(Ty<'tcx>, Option<PolyExistentialTraitRef<'tcx>>), RValue<'gcc>>> {
+    ) -> &RefCell<FxHashMap<(Ty<'tcx>, Option<ExistentialTraitRef<'tcx>>), RValue<'gcc>>> {
         &self.vtables
     }
 

compiler/rustc_codegen_gcc/src/debuginfo.rs

@@ -7,7 +7,7 @@ use rustc_data_structures::sync::Lrc;
 use rustc_index::bit_set::DenseBitSet;
 use rustc_index::{Idx, IndexVec};
 use rustc_middle::mir::{self, Body, SourceScope};
-use rustc_middle::ty::{Instance, PolyExistentialTraitRef, Ty};
+use rustc_middle::ty::{ExistentialTraitRef, Instance, Ty};
 use rustc_session::config::DebugInfo;
 use rustc_span::{BytePos, Pos, SourceFile, SourceFileAndLine, Span, Symbol};
 use rustc_target::abi::Size;
@@ -214,7 +214,7 @@ impl<'gcc, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
     fn create_vtable_debuginfo(
         &self,
         _ty: Ty<'tcx>,
-        _trait_ref: Option<PolyExistentialTraitRef<'tcx>>,
+        _trait_ref: Option<ExistentialTraitRef<'tcx>>,
         _vtable: Self::Value,
     ) {
         // TODO(antoyo)

compiler/rustc_codegen_gcc/src/gcc_util.rs

@@ -1,10 +1,8 @@
-use std::iter::FromIterator;
-
 #[cfg(feature = "master")]
 use gccjit::Context;
 use rustc_codegen_ssa::codegen_attrs::check_tied_features;
 use rustc_codegen_ssa::errors::TargetFeatureDisableOrEnable;
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::unord::UnordSet;
 use rustc_session::Session;
 use rustc_target::target_features::RUSTC_SPECIFIC_FEATURES;
@@ -45,12 +43,6 @@ pub(crate) fn global_gcc_features(sess: &Session, diagnostics: bool) -> Vec<Stri
     let known_features = sess.target.rust_target_features();
     let mut featsmap = FxHashMap::default();
 
-    // Ensure that all ABI-required features are enabled, and the ABI-forbidden ones
-    // are disabled.
-    let abi_feature_constraints = sess.target.abi_required_features();
-    let abi_incompatible_set =
-        FxHashSet::from_iter(abi_feature_constraints.incompatible.iter().copied());
-
     // Compute implied features
     let mut all_rust_features = vec![];
     for feature in sess.opts.cg.target_feature.split(',') {
@@ -117,51 +109,11 @@ pub(crate) fn global_gcc_features(sess: &Session, diagnostics: bool) -> Vec<Stri
                 }
             }
 
-            // Ensure that the features we enable/disable are compatible with the ABI.
-            if enable {
-                if abi_incompatible_set.contains(feature) {
-                    sess.dcx().emit_warn(ForbiddenCTargetFeature {
-                        feature,
-                        enabled: "enabled",
-                        reason: "this feature is incompatible with the target ABI",
-                    });
-                }
-            } else {
-                // FIXME: we have to request implied features here since
-                // negative features do not handle implied features above.
-                for &required in abi_feature_constraints.required.iter() {
-                    let implied = sess.target.implied_target_features(std::iter::once(required));
-                    if implied.contains(feature) {
-                        sess.dcx().emit_warn(ForbiddenCTargetFeature {
-                            feature,
-                            enabled: "disabled",
-                            reason: "this feature is required by the target ABI",
-                        });
-                    }
-                }
-            }
-
             // FIXME(nagisa): figure out how to not allocate a full hashset here.
             featsmap.insert(feature, enable);
         }
     }
 
-    // To be sure the ABI-relevant features are all in the right state, we explicitly
-    // (un)set them here. This means if the target spec sets those features wrong,
-    // we will silently correct them rather than silently producing wrong code.
-    // (The target sanity check tries to catch this, but we can't know which features are
-    // enabled in GCC by default so we can't be fully sure about that check.)
-    // We add these at the beginning of the list so that `-Ctarget-features` can
-    // still override it... that's unsound, but more compatible with past behavior.
-    all_rust_features.splice(
-        0..0,
-        abi_feature_constraints
-            .required
-            .iter()
-            .map(|&f| (true, f))
-            .chain(abi_feature_constraints.incompatible.iter().map(|&f| (false, f))),
-    );
-
     // Translate this into GCC features.
     let feats =
         all_rust_features.iter().flat_map(|&(enable, feature)| {

compiler/rustc_codegen_gcc/src/lib.rs

@@ -493,9 +493,10 @@ fn target_features_cfg(
     sess.target
         .rust_target_features()
         .iter()
-        .filter(|&&(_, gate, _)| gate.in_cfg())
         .filter_map(|&(feature, gate, _)| {
-            if sess.is_nightly_build() || allow_unstable || gate.requires_nightly().is_none() {
+            if allow_unstable
+                || (gate.in_cfg() && (sess.is_nightly_build() || gate.requires_nightly().is_none()))
+            {
                 Some(feature)
             } else {
                 None

compiler/rustc_codegen_gcc/tests/run/int.rs

@@ -3,8 +3,6 @@
 // Run-time:
 //   status: 0
 
-#![feature(const_black_box)]
-
 /*
  * Code
  */

compiler/rustc_codegen_llvm/src/builder.rs

@@ -1325,7 +1325,9 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
     fn get_static(&mut self, def_id: DefId) -> &'ll Value {
         // Forward to the `get_static` method of `CodegenCx`
-        self.cx().get_static(def_id)
+        let s = self.cx().get_static(def_id);
+        // Cast to default address space if globals are in a different addrspace
+        self.cx().const_pointercast(s, self.type_ptr())
     }
 }
 

compiler/rustc_codegen_llvm/src/builder/autodiff.rs

@@ -62,8 +62,8 @@ fn generate_enzyme_call<'ll>(
     // add outer_fn name to ad_name to make it unique, in case users apply autodiff to multiple
     // functions. Unwrap will only panic, if LLVM gave us an invalid string.
     let name = llvm::get_value_name(outer_fn);
-    let outer_fn_name = std::ffi::CStr::from_bytes_with_nul(name).unwrap().to_str().unwrap();
-    ad_name.push_str(outer_fn_name.to_string().as_str());
+    let outer_fn_name = std::str::from_utf8(name).unwrap();
+    ad_name.push_str(outer_fn_name);
 
     // Let us assume the user wrote the following function square:
     //
@@ -255,14 +255,14 @@ fn generate_enzyme_call<'ll>(
             // have no debug info to copy, which would then be ok.
             trace!("no dbg info");
         }
-        // Now that we copied the metadata, get rid of dummy code.
-        llvm::LLVMRustEraseInstBefore(entry, last_inst);
-        llvm::LLVMRustEraseInstFromParent(last_inst);
 
-        if cx.val_ty(outer_fn) != cx.type_void() {
-            builder.ret(call);
-        } else {
+        // Now that we copied the metadata, get rid of dummy code.
+        llvm::LLVMRustEraseInstUntilInclusive(entry, last_inst);
+
+        if cx.val_ty(call) == cx.type_void() {
             builder.ret_void();
+        } else {
+            builder.ret(call);
         }
 
         // Let's crash in case that we messed something up above and generated invalid IR.

compiler/rustc_codegen_llvm/src/common.rs

@@ -225,6 +225,8 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                     llvm::LLVMSetUnnamedAddress(g, llvm::UnnamedAddr::Global);
                 }
                 llvm::set_linkage(g, llvm::Linkage::InternalLinkage);
+                // Cast to default address space if globals are in a different addrspace
+                let g = self.const_pointercast(g, self.type_ptr());
                 (s.to_owned(), g)
             })
             .1;
@@ -289,7 +291,7 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                             let alloc = alloc.inner();
                             let value = match alloc.mutability {
                                 Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None),
-                                _ => self.static_addr_of(init, alloc.align, None),
+                                _ => self.static_addr_of_impl(init, alloc.align, None),
                             };
                             if !self.sess().fewer_names() && llvm::get_value_name(value).is_empty()
                             {
@@ -312,10 +314,15 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                     GlobalAlloc::VTable(ty, dyn_ty) => {
                         let alloc = self
                             .tcx
-                            .global_alloc(self.tcx.vtable_allocation((ty, dyn_ty.principal())))
+                            .global_alloc(self.tcx.vtable_allocation((
+                                ty,
+                                dyn_ty.principal().map(|principal| {
+                                    self.tcx.instantiate_bound_regions_with_erased(principal)
+                                }),
+                            )))
                             .unwrap_memory();
                         let init = const_alloc_to_llvm(self, alloc, /*static*/ false);
-                        let value = self.static_addr_of(init, alloc.inner().align, None);
+                        let value = self.static_addr_of_impl(init, alloc.inner().align, None);
                         (value, AddressSpace::DATA)
                     }
                     GlobalAlloc::Static(def_id) => {
@@ -327,7 +334,8 @@ impl<'ll, 'tcx> ConstCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                 let llval = unsafe {
                     llvm::LLVMConstInBoundsGEP2(
                         self.type_i8(),
-                        self.const_bitcast(base_addr, self.type_ptr_ext(base_addr_space)),
+                        // Cast to the required address space if necessary
+                        self.const_pointercast(base_addr, self.type_ptr_ext(base_addr_space)),
                         &self.const_usize(offset.bytes()),
                         1,
                     )

compiler/rustc_codegen_llvm/src/consts.rs

@@ -210,6 +210,14 @@ impl<'ll> CodegenCx<'ll, '_> {
         unsafe { llvm::LLVMConstBitCast(val, ty) }
     }
 
+    pub(crate) fn const_pointercast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value {
+        unsafe { llvm::LLVMConstPointerCast(val, ty) }
+    }
+
+    /// Create a global variable.
+    ///
+    /// The returned global variable is a pointer in the default address space for globals.
+    /// Fails if a symbol with the given name already exists.
     pub(crate) fn static_addr_of_mut(
         &self,
         cv: &'ll Value,
@@ -233,6 +241,34 @@ impl<'ll> CodegenCx<'ll, '_> {
         gv
     }
 
+    /// Create a global constant.
+    ///
+    /// The returned global variable is a pointer in the default address space for globals.
+    pub(crate) fn static_addr_of_impl(
+        &self,
+        cv: &'ll Value,
+        align: Align,
+        kind: Option<&str>,
+    ) -> &'ll Value {
+        if let Some(&gv) = self.const_globals.borrow().get(&cv) {
+            unsafe {
+                // Upgrade the alignment in cases where the same constant is used with different
+                // alignment requirements
+                let llalign = align.bytes() as u32;
+                if llalign > llvm::LLVMGetAlignment(gv) {
+                    llvm::LLVMSetAlignment(gv, llalign);
+                }
+            }
+            return gv;
+        }
+        let gv = self.static_addr_of_mut(cv, align, kind);
+        unsafe {
+            llvm::LLVMSetGlobalConstant(gv, True);
+        }
+        self.const_globals.borrow_mut().insert(cv, gv);
+        gv
+    }
+
     #[instrument(level = "debug", skip(self))]
     pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value {
         let instance = Instance::mono(self.tcx, def_id);
@@ -505,24 +541,15 @@ impl<'ll> CodegenCx<'ll, '_> {
 }
 
 impl<'ll> StaticCodegenMethods for CodegenCx<'ll, '_> {
+    /// Get a pointer to a global variable.
+    ///
+    /// The pointer will always be in the default address space. If global variables default to a
+    /// different address space, an addrspacecast is inserted.
     fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value {
-        if let Some(&gv) = self.const_globals.borrow().get(&cv) {
-            unsafe {
-                // Upgrade the alignment in cases where the same constant is used with different
-                // alignment requirements
-                let llalign = align.bytes() as u32;
-                if llalign > llvm::LLVMGetAlignment(gv) {
-                    llvm::LLVMSetAlignment(gv, llalign);
-                }
-            }
-            return gv;
-        }
-        let gv = self.static_addr_of_mut(cv, align, kind);
-        unsafe {
-            llvm::LLVMSetGlobalConstant(gv, True);
-        }
-        self.const_globals.borrow_mut().insert(cv, gv);
-        gv
+        let gv = self.static_addr_of_impl(cv, align, kind);
+        // static_addr_of_impl returns the bare global variable, which might not be in the default
+        // address space. Cast to the default address space if necessary.
+        self.const_pointercast(gv, self.type_ptr())
     }
 
     fn codegen_static(&self, def_id: DefId) {

compiler/rustc_codegen_llvm/src/context.rs

@@ -77,8 +77,7 @@ pub(crate) struct CodegenCx<'ll, 'tcx> {
     /// Cache instances of monomorphic and polymorphic items
     pub instances: RefCell<FxHashMap<Instance<'tcx>, &'ll Value>>,
     /// Cache generated vtables
-    pub vtables:
-        RefCell<FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), &'ll Value>>,
+    pub vtables: RefCell<FxHashMap<(Ty<'tcx>, Option<ty::ExistentialTraitRef<'tcx>>), &'ll Value>>,
     /// Cache of constant strings,
     pub const_str_cache: RefCell<FxHashMap<String, &'ll Value>>,
 
@@ -663,15 +662,14 @@ impl<'ll> SimpleCx<'ll> {
 impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn vtables(
         &self,
-    ) -> &RefCell<FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), &'ll Value>>
-    {
+    ) -> &RefCell<FxHashMap<(Ty<'tcx>, Option<ty::ExistentialTraitRef<'tcx>>), &'ll Value>> {
         &self.vtables
     }
 
     fn apply_vcall_visibility_metadata(
         &self,
         ty: Ty<'tcx>,
-        poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
+        poly_trait_ref: Option<ty::ExistentialTraitRef<'tcx>>,
         vtable: &'ll Value,
     ) {
         apply_vcall_visibility_metadata(self, ty, poly_trait_ref, vtable);

compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs

@@ -298,7 +298,7 @@ struct UsageSets<'tcx> {
 /// Prepare sets of definitions that are relevant to deciding whether something
 /// is an "unused function" for coverage purposes.
 fn prepare_usage_sets<'tcx>(tcx: TyCtxt<'tcx>) -> UsageSets<'tcx> {
-    let MonoItemPartitions { all_mono_items, codegen_units } =
+    let MonoItemPartitions { all_mono_items, codegen_units, .. } =
         tcx.collect_and_partition_mono_items(());
 
     // Obtain a MIR body for each function participating in codegen, via an

compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs

@@ -13,7 +13,7 @@ use rustc_hir::def_id::{DefId, LOCAL_CRATE};
 use rustc_middle::bug;
 use rustc_middle::ty::layout::{HasTypingEnv, LayoutOf, TyAndLayout};
 use rustc_middle::ty::{
-    self, AdtKind, CoroutineArgsExt, Instance, PolyExistentialTraitRef, Ty, TyCtxt, Visibility,
+    self, AdtKind, CoroutineArgsExt, ExistentialTraitRef, Instance, Ty, TyCtxt, Visibility,
 };
 use rustc_session::config::{self, DebugInfo, Lto};
 use rustc_span::{DUMMY_SP, FileName, FileNameDisplayPreference, SourceFile, Symbol, hygiene};
@@ -919,8 +919,7 @@ pub(crate) fn build_compile_unit_di_node<'ll, 'tcx>(
         .unwrap_or_default();
     let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo);
 
-    let dwarf_version =
-        tcx.sess.opts.unstable_opts.dwarf_version.unwrap_or(tcx.sess.target.default_dwarf_version);
+    let dwarf_version = tcx.sess.dwarf_version();
     let is_dwarf_kind =
         matches!(tcx.sess.target.debuginfo_kind, DebuginfoKind::Dwarf | DebuginfoKind::DwarfDsym);
     // Don't emit `.debug_pubnames` and `.debug_pubtypes` on DWARFv4 or lower.
@@ -1400,7 +1399,7 @@ pub(crate) fn build_global_var_di_node<'ll>(
 fn build_vtable_type_di_node<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     ty: Ty<'tcx>,
-    poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
+    poly_trait_ref: Option<ty::ExistentialTraitRef<'tcx>>,
 ) -> &'ll DIType {
     let tcx = cx.tcx;
 
@@ -1488,10 +1487,30 @@ fn build_vtable_type_di_node<'ll, 'tcx>(
     .di_node
 }
 
+/// Get the global variable for the vtable.
+///
+/// When using global variables, we may have created an addrspacecast to get a pointer to the
+/// default address space if global variables are created in a different address space.
+/// For modifying the vtable, we need the real global variable. This function accepts either a
+/// global variable (which is simply returned), or an addrspacecast constant expression.
+/// If the given value is an addrspacecast, the cast is removed and the global variable behind
+/// the cast is returned.
+fn find_vtable_behind_cast<'ll>(vtable: &'ll Value) -> &'ll Value {
+    // The vtable is a global variable, which may be behind an addrspacecast.
+    unsafe {
+        if let Some(c) = llvm::LLVMIsAConstantExpr(vtable) {
+            if llvm::LLVMGetConstOpcode(c) == llvm::Opcode::AddrSpaceCast {
+                return llvm::LLVMGetOperand(c, 0).unwrap();
+            }
+        }
+    }
+    vtable
+}
+
 pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     ty: Ty<'tcx>,
-    trait_ref: Option<PolyExistentialTraitRef<'tcx>>,
+    trait_ref: Option<ExistentialTraitRef<'tcx>>,
     vtable: &'ll Value,
 ) {
     // FIXME(flip1995): The virtual function elimination optimization only works with full LTO in
@@ -1508,9 +1527,11 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>(
 
     let Some(trait_ref) = trait_ref else { return };
 
+    // Unwrap potential addrspacecast
+    let vtable = find_vtable_behind_cast(vtable);
     let trait_ref_self = trait_ref.with_self_ty(cx.tcx, ty);
     let trait_ref_self = cx.tcx.erase_regions(trait_ref_self);
-    let trait_def_id = trait_ref_self.def_id();
+    let trait_def_id = trait_ref_self.def_id;
     let trait_vis = cx.tcx.visibility(trait_def_id);
 
     let cgus = cx.sess().codegen_units().as_usize();
@@ -1569,7 +1590,7 @@ pub(crate) fn apply_vcall_visibility_metadata<'ll, 'tcx>(
 pub(crate) fn create_vtable_di_node<'ll, 'tcx>(
     cx: &CodegenCx<'ll, 'tcx>,
     ty: Ty<'tcx>,
-    poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
+    poly_trait_ref: Option<ty::ExistentialTraitRef<'tcx>>,
     vtable: &'ll Value,
 ) {
     if cx.dbg_cx.is_none() {
@@ -1581,6 +1602,9 @@ pub(crate) fn create_vtable_di_node<'ll, 'tcx>(
         return;
     }
 
+    // Unwrap potential addrspacecast
+    let vtable = find_vtable_behind_cast(vtable);
+
     // When full debuginfo is enabled, we want to try and prevent vtables from being
     // merged. Otherwise debuggers will have a hard time mapping from dyn pointer
     // to concrete type.

compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs

@@ -6,7 +6,7 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_macros::HashStable;
 use rustc_middle::bug;
-use rustc_middle::ty::{self, PolyExistentialTraitRef, Ty, TyCtxt};
+use rustc_middle::ty::{self, ExistentialTraitRef, Ty, TyCtxt};
 
 use super::{DefinitionLocation, SmallVec, UNKNOWN_LINE_NUMBER, unknown_file_metadata};
 use crate::common::{AsCCharPtr, CodegenCx};
@@ -44,7 +44,7 @@ pub(super) enum UniqueTypeId<'tcx> {
     /// The ID for the additional wrapper struct type describing an enum variant in CPP-like mode.
     VariantStructTypeCppLikeWrapper(Ty<'tcx>, VariantIdx, private::HiddenZst),
     /// The ID of the artificial type we create for VTables.
-    VTableTy(Ty<'tcx>, Option<PolyExistentialTraitRef<'tcx>>, private::HiddenZst),
+    VTableTy(Ty<'tcx>, Option<ExistentialTraitRef<'tcx>>, private::HiddenZst),
 }
 
 impl<'tcx> UniqueTypeId<'tcx> {
@@ -88,7 +88,7 @@ impl<'tcx> UniqueTypeId<'tcx> {
     pub(crate) fn for_vtable_ty(
         tcx: TyCtxt<'tcx>,
         self_type: Ty<'tcx>,
-        implemented_trait: Option<PolyExistentialTraitRef<'tcx>>,
+        implemented_trait: Option<ExistentialTraitRef<'tcx>>,
     ) -> Self {
         assert_eq!(
             self_type,

compiler/rustc_codegen_llvm/src/debuginfo/mod.rs

@@ -22,6 +22,7 @@ use rustc_session::config::{self, DebugInfo};
 use rustc_span::{
     BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span, StableSourceFileId, Symbol,
 };
+use rustc_target::spec::DebuginfoKind;
 use smallvec::SmallVec;
 use tracing::debug;
 
@@ -93,29 +94,31 @@ impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> {
 
     pub(crate) fn finalize(&self, sess: &Session) {
         unsafe { llvm::LLVMRustDIBuilderFinalize(self.builder) };
-        if !sess.target.is_like_msvc {
-            // Debuginfo generation in LLVM by default uses a higher
-            // version of dwarf than macOS currently understands. We can
-            // instruct LLVM to emit an older version of dwarf, however,
-            // for macOS to understand. For more info see #11352
-            // This can be overridden using --llvm-opts -dwarf-version,N.
-            // Android has the same issue (#22398)
-            let dwarf_version =
-                sess.opts.unstable_opts.dwarf_version.unwrap_or(sess.target.default_dwarf_version);
-            llvm::add_module_flag_u32(
-                self.llmod,
-                llvm::ModuleFlagMergeBehavior::Warning,
-                "Dwarf Version",
-                dwarf_version,
-            );
-        } else {
-            // Indicate that we want CodeView debug information on MSVC
-            llvm::add_module_flag_u32(
-                self.llmod,
-                llvm::ModuleFlagMergeBehavior::Warning,
-                "CodeView",
-                1,
-            );
+
+        match sess.target.debuginfo_kind {
+            DebuginfoKind::Dwarf | DebuginfoKind::DwarfDsym => {
+                // Debuginfo generation in LLVM by default uses a higher
+                // version of dwarf than macOS currently understands. We can
+                // instruct LLVM to emit an older version of dwarf, however,
+                // for macOS to understand. For more info see #11352
+                // This can be overridden using --llvm-opts -dwarf-version,N.
+                // Android has the same issue (#22398)
+                llvm::add_module_flag_u32(
+                    self.llmod,
+                    llvm::ModuleFlagMergeBehavior::Warning,
+                    "Dwarf Version",
+                    sess.dwarf_version(),
+                );
+            }
+            DebuginfoKind::Pdb => {
+                // Indicate that we want CodeView debug information
+                llvm::add_module_flag_u32(
+                    self.llmod,
+                    llvm::ModuleFlagMergeBehavior::Warning,
+                    "CodeView",
+                    1,
+                );
+            }
         }
 
         // Prevent bitcode readers from deleting the debug info.
@@ -585,7 +588,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
     fn create_vtable_debuginfo(
         &self,
         ty: Ty<'tcx>,
-        trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
+        trait_ref: Option<ty::ExistentialTraitRef<'tcx>>,
         vtable: Self::Value,
     ) {
         metadata::create_vtable_di_node(self, ty, trait_ref, vtable)

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -1329,7 +1329,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     }
 
     if name == sym::simd_shuffle_generic {
-        let idx = fn_args[2].expect_const().try_to_valtree().unwrap().0.unwrap_branch();
+        let idx = fn_args[2].expect_const().to_value().valtree.unwrap_branch();
         let n = idx.len() as u64;
 
         let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);

compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs

@@ -7,11 +7,13 @@ use crate::llvm::Bool;
 extern "C" {
     // Enzyme
     pub fn LLVMRustHasMetadata(I: &Value, KindID: c_uint) -> bool;
-    pub fn LLVMRustEraseInstBefore(BB: &BasicBlock, I: &Value);
+    pub fn LLVMRustEraseInstUntilInclusive(BB: &BasicBlock, I: &Value);
     pub fn LLVMRustGetLastInstruction<'a>(BB: &BasicBlock) -> Option<&'a Value>;
     pub fn LLVMRustDIGetInstMetadata(I: &Value) -> Option<&Metadata>;
     pub fn LLVMRustEraseInstFromParent(V: &Value);
     pub fn LLVMRustGetTerminator<'a>(B: &BasicBlock) -> &'a Value;
+    pub fn LLVMDumpModule(M: &Module);
+    pub fn LLVMDumpValue(V: &Value);
     pub fn LLVMRustVerifyFunction(V: &Value, action: LLVMRustVerifierFailureAction) -> Bool;
 
     pub fn LLVMGetFunctionCallConv(F: &Value) -> c_uint;

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -661,6 +661,79 @@ pub enum MemoryEffects {
     InaccessibleMemOnly,
 }
 
+/// LLVMOpcode
+#[derive(Copy, Clone, PartialEq, Eq)]
+#[repr(C)]
+pub enum Opcode {
+    Ret = 1,
+    Br = 2,
+    Switch = 3,
+    IndirectBr = 4,
+    Invoke = 5,
+    Unreachable = 7,
+    CallBr = 67,
+    FNeg = 66,
+    Add = 8,
+    FAdd = 9,
+    Sub = 10,
+    FSub = 11,
+    Mul = 12,
+    FMul = 13,
+    UDiv = 14,
+    SDiv = 15,
+    FDiv = 16,
+    URem = 17,
+    SRem = 18,
+    FRem = 19,
+    Shl = 20,
+    LShr = 21,
+    AShr = 22,
+    And = 23,
+    Or = 24,
+    Xor = 25,
+    Alloca = 26,
+    Load = 27,
+    Store = 28,
+    GetElementPtr = 29,
+    Trunc = 30,
+    ZExt = 31,
+    SExt = 32,
+    FPToUI = 33,
+    FPToSI = 34,
+    UIToFP = 35,
+    SIToFP = 36,
+    FPTrunc = 37,
+    FPExt = 38,
+    PtrToInt = 39,
+    IntToPtr = 40,
+    BitCast = 41,
+    AddrSpaceCast = 60,
+    ICmp = 42,
+    FCmp = 43,
+    PHI = 44,
+    Call = 45,
+    Select = 46,
+    UserOp1 = 47,
+    UserOp2 = 48,
+    VAArg = 49,
+    ExtractElement = 50,
+    InsertElement = 51,
+    ShuffleVector = 52,
+    ExtractValue = 53,
+    InsertValue = 54,
+    Freeze = 68,
+    Fence = 55,
+    AtomicCmpXchg = 56,
+    AtomicRMW = 57,
+    Resume = 58,
+    LandingPad = 59,
+    CleanupRet = 61,
+    CatchRet = 62,
+    CatchPad = 63,
+    CleanupPad = 64,
+    CatchSwitch = 65,
+}
+
 unsafe extern "C" {
     type Opaque;
 }
@@ -991,7 +1064,10 @@ unsafe extern "C" {
     pub fn LLVMConstPtrToInt<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value;
     pub fn LLVMConstIntToPtr<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value;
     pub fn LLVMConstBitCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value;
+    pub fn LLVMConstPointerCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value;
     pub fn LLVMGetAggregateElement(ConstantVal: &Value, Idx: c_uint) -> Option<&Value>;
+    pub fn LLVMGetConstOpcode(ConstantVal: &Value) -> Opcode;
+    pub fn LLVMIsAConstantExpr(Val: &Value) -> Option<&Value>;
 
     // Operations on global variables, functions, and aliases (globals)
     pub fn LLVMIsDeclaration(Global: &Value) -> Bool;
@@ -1048,6 +1124,7 @@ unsafe extern "C" {
     // Operations on instructions
     pub fn LLVMIsAInstruction(Val: &Value) -> Option<&Value>;
     pub fn LLVMGetFirstBasicBlock(Fn: &Value) -> &BasicBlock;
+    pub fn LLVMGetOperand(Val: &Value, Index: c_uint) -> Option<&Value>;
 
     // Operations on call sites
     pub fn LLVMSetInstructionCallConv(Instr: &Value, CC: c_uint);

compiler/rustc_codegen_llvm/src/llvm_util.rs

@@ -319,7 +319,6 @@ pub fn target_features_cfg(sess: &Session, allow_unstable: bool) -> Vec<Symbol>
         sess.target
             .rust_target_features()
             .iter()
-            .filter(|(_, gate, _)| gate.in_cfg())
             .filter(|(feature, _, _)| {
                 // skip checking special features, as LLVM may not understand them
                 if RUSTC_SPECIAL_FEATURES.contains(feature) {
@@ -388,9 +387,13 @@ pub fn target_features_cfg(sess: &Session, allow_unstable: bool) -> Vec<Symbol>
     sess.target
         .rust_target_features()
         .iter()
-        .filter(|(_, gate, _)| gate.in_cfg())
         .filter_map(|(feature, gate, _)| {
-            if sess.is_nightly_build() || allow_unstable || gate.requires_nightly().is_none() {
+            // The `allow_unstable` set is used by rustc internally to determined which target
+            // features are truly available, so we want to return even perma-unstable "forbidden"
+            // features.
+            if allow_unstable
+                || (gate.in_cfg() && (sess.is_nightly_build() || gate.requires_nightly().is_none()))
+            {
                 Some(*feature)
             } else {
                 None
@@ -670,12 +673,6 @@ pub(crate) fn global_llvm_features(
         // Will only be filled when `diagnostics` is set!
         let mut featsmap = FxHashMap::default();
 
-        // Ensure that all ABI-required features are enabled, and the ABI-forbidden ones
-        // are disabled.
-        let abi_feature_constraints = sess.target.abi_required_features();
-        let abi_incompatible_set =
-            FxHashSet::from_iter(abi_feature_constraints.incompatible.iter().copied());
-
         // Compute implied features
         let mut all_rust_features = vec![];
         for feature in sess.opts.cg.target_feature.split(',') {
@@ -746,52 +743,11 @@ pub(crate) fn global_llvm_features(
                     }
                 }
 
-                // Ensure that the features we enable/disable are compatible with the ABI.
-                if enable {
-                    if abi_incompatible_set.contains(feature) {
-                        sess.dcx().emit_warn(ForbiddenCTargetFeature {
-                            feature,
-                            enabled: "enabled",
-                            reason: "this feature is incompatible with the target ABI",
-                        });
-                    }
-                } else {
-                    // FIXME: we have to request implied features here since
-                    // negative features do not handle implied features above.
-                    for &required in abi_feature_constraints.required.iter() {
-                        let implied =
-                            sess.target.implied_target_features(std::iter::once(required));
-                        if implied.contains(feature) {
-                            sess.dcx().emit_warn(ForbiddenCTargetFeature {
-                                feature,
-                                enabled: "disabled",
-                                reason: "this feature is required by the target ABI",
-                            });
-                        }
-                    }
-                }
-
                 // FIXME(nagisa): figure out how to not allocate a full hashset here.
                 featsmap.insert(feature, enable);
             }
         }
 
-        // To be sure the ABI-relevant features are all in the right state, we explicitly
-        // (un)set them here. This means if the target spec sets those features wrong,
-        // we will silently correct them rather than silently producing wrong code.
-        // (The target sanity check tries to catch this, but we can't know which features are
-        // enabled in LLVM by default so we can't be fully sure about that check.)
-        // We add these at the beginning of the list so that `-Ctarget-features` can
-        // still override it... that's unsound, but more compatible with past behavior.
-        all_rust_features.splice(
-            0..0,
-            abi_feature_constraints
-                .required
-                .iter()
-                .map(|&f| (true, f))
-                .chain(abi_feature_constraints.incompatible.iter().map(|&f| (false, f))),
-        );
-
         // Translate this into LLVM features.
         let feats = all_rust_features
             .iter()

compiler/rustc_codegen_ssa/messages.ftl

@@ -16,6 +16,8 @@ codegen_ssa_archive_build_failure = failed to build archive at `{$path}`: {$erro
 
 codegen_ssa_atomic_compare_exchange = Atomic compare-exchange intrinsic missing failure memory ordering
 
+codegen_ssa_autodiff_without_lto = using the autodiff feature requires using fat-lto
+
 codegen_ssa_binary_output_to_tty = option `-o` or `--emit` is used to write binary output type `{$shorthand}` to stdout, but stdout is a tty
 
 codegen_ssa_cgu_not_recorded =
@@ -30,6 +32,8 @@ codegen_ssa_copy_path = could not copy {$from} to {$to}: {$error}
 
 codegen_ssa_copy_path_buf = unable to copy {$source_file} to {$output_path}: {$error}
 
+codegen_ssa_cpu_required = target requires explicitly specifying a cpu with `-C target-cpu`
+
 codegen_ssa_create_temp_dir = couldn't create a temp dir: {$error}
 
 codegen_ssa_dlltool_fail_import_library =

compiler/rustc_codegen_ssa/src/back/write.rs

@@ -7,6 +7,7 @@ use std::sync::mpsc::{Receiver, Sender, channel};
 use std::{fs, io, mem, str, thread};
 
 use rustc_ast::attr;
+use rustc_ast::expand::autodiff_attrs::AutoDiffItem;
 use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
 use rustc_data_structures::jobserver::{self, Acquired};
 use rustc_data_structures::memmap::Mmap;
@@ -40,7 +41,7 @@ use tracing::debug;
 use super::link::{self, ensure_removed};
 use super::lto::{self, SerializedModule};
 use super::symbol_export::symbol_name_for_instance_in_crate;
-use crate::errors::ErrorCreatingRemarkDir;
+use crate::errors::{AutodiffWithoutLto, ErrorCreatingRemarkDir};
 use crate::traits::*;
 use crate::{
     CachedModuleCodegen, CodegenResults, CompiledModule, CrateInfo, ModuleCodegen, ModuleKind,
@@ -118,6 +119,7 @@ pub struct ModuleConfig {
     pub merge_functions: bool,
     pub emit_lifetime_markers: bool,
     pub llvm_plugins: Vec<String>,
+    pub autodiff: Vec<config::AutoDiff>,
 }
 
 impl ModuleConfig {
@@ -266,6 +268,7 @@ impl ModuleConfig {
 
             emit_lifetime_markers: sess.emit_lifetime_markers(),
             llvm_plugins: if_regular!(sess.opts.unstable_opts.llvm_plugins.clone(), vec![]),
+            autodiff: if_regular!(sess.opts.unstable_opts.autodiff.clone(), vec![]),
         }
     }
 
@@ -389,6 +392,7 @@ impl<B: WriteBackendMethods> CodegenContext<B> {
 
 fn generate_lto_work<B: ExtraBackendMethods>(
     cgcx: &CodegenContext<B>,
+    autodiff: Vec<AutoDiffItem>,
     needs_fat_lto: Vec<FatLtoInput<B>>,
     needs_thin_lto: Vec<(String, B::ThinBuffer)>,
     import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>,
@@ -397,11 +401,19 @@ fn generate_lto_work<B: ExtraBackendMethods>(
 
     if !needs_fat_lto.is_empty() {
         assert!(needs_thin_lto.is_empty());
-        let module =
+        let mut module =
             B::run_fat_lto(cgcx, needs_fat_lto, import_only_modules).unwrap_or_else(|e| e.raise());
+        if cgcx.lto == Lto::Fat {
+            let config = cgcx.config(ModuleKind::Regular);
+            module = unsafe { module.autodiff(cgcx, autodiff, config).unwrap() };
+        }
         // We are adding a single work item, so the cost doesn't matter.
         vec![(WorkItem::LTO(module), 0)]
     } else {
+        if !autodiff.is_empty() {
+            let dcx = cgcx.create_dcx();
+            dcx.handle().emit_fatal(AutodiffWithoutLto {});
+        }
         assert!(needs_fat_lto.is_empty());
         let (lto_modules, copy_jobs) = B::run_thin_lto(cgcx, needs_thin_lto, import_only_modules)
             .unwrap_or_else(|e| e.raise());
@@ -1021,6 +1033,9 @@ pub(crate) enum Message<B: WriteBackendMethods> {
     /// Sent from a backend worker thread.
     WorkItem { result: Result<WorkItemResult<B>, Option<WorkerFatalError>>, worker_id: usize },
 
+    /// A vector containing all the AutoDiff tasks that we have to pass to Enzyme.
+    AddAutoDiffItems(Vec<AutoDiffItem>),
+
     /// The frontend has finished generating something (backend IR or a
     /// post-LTO artifact) for a codegen unit, and it should be passed to the
     /// backend. Sent from the main thread.
@@ -1348,6 +1363,7 @@ fn start_executing_work<B: ExtraBackendMethods>(
 
         // This is where we collect codegen units that have gone all the way
         // through codegen and LLVM.
+        let mut autodiff_items = Vec::new();
         let mut compiled_modules = vec![];
         let mut compiled_allocator_module = None;
         let mut needs_link = Vec::new();
@@ -1459,9 +1475,13 @@ fn start_executing_work<B: ExtraBackendMethods>(
                     let needs_thin_lto = mem::take(&mut needs_thin_lto);
                     let import_only_modules = mem::take(&mut lto_import_only_modules);
 
-                    for (work, cost) in
-                        generate_lto_work(&cgcx, needs_fat_lto, needs_thin_lto, import_only_modules)
-                    {
+                    for (work, cost) in generate_lto_work(
+                        &cgcx,
+                        autodiff_items.clone(),
+                        needs_fat_lto,
+                        needs_thin_lto,
+                        import_only_modules,
+                    ) {
                         let insertion_index = work_items
                             .binary_search_by_key(&cost, |&(_, cost)| cost)
                             .unwrap_or_else(|e| e);
@@ -1596,6 +1616,10 @@ fn start_executing_work<B: ExtraBackendMethods>(
                     main_thread_state = MainThreadState::Idle;
                 }
 
+                Message::AddAutoDiffItems(mut items) => {
+                    autodiff_items.append(&mut items);
+                }
+
                 Message::CodegenComplete => {
                     if codegen_state != Aborted {
                         codegen_state = Completed;
@@ -2070,6 +2094,10 @@ impl<B: ExtraBackendMethods> OngoingCodegen<B> {
         drop(self.coordinator.sender.send(Box::new(Message::CodegenComplete::<B>)));
     }
 
+    pub(crate) fn submit_autodiff_items(&self, items: Vec<AutoDiffItem>) {
+        drop(self.coordinator.sender.send(Box::new(Message::<B>::AddAutoDiffItems(items))));
+    }
+
     pub(crate) fn check_for_errors(&self, sess: &Session) {
         self.shared_emitter_main.check(sess, false);
     }

compiler/rustc_codegen_ssa/src/base.rs

@@ -18,14 +18,13 @@ use rustc_middle::middle::debugger_visualizer::{DebuggerVisualizerFile, Debugger
 use rustc_middle::middle::exported_symbols::SymbolExportKind;
 use rustc_middle::middle::{exported_symbols, lang_items};
 use rustc_middle::mir::BinOp;
-use rustc_middle::mir::mono::{CodegenUnit, CodegenUnitNameBuilder, MonoItem};
+use rustc_middle::mir::mono::{CodegenUnit, CodegenUnitNameBuilder, MonoItem, MonoItemPartitions};
 use rustc_middle::query::Providers;
 use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
 use rustc_session::Session;
 use rustc_session::config::{self, CrateType, EntryFnType, OptLevel, OutputType};
 use rustc_span::{DUMMY_SP, Symbol, sym};
-use rustc_trait_selection::infer::at::ToTrace;
 use rustc_trait_selection::infer::{BoundRegionConversionTime, TyCtxtInferExt};
 use rustc_trait_selection::traits::{ObligationCause, ObligationCtxt};
 use tracing::{debug, info};
@@ -129,14 +128,9 @@ pub fn validate_trivial_unsize<'tcx>(
                     BoundRegionConversionTime::HigherRankedType,
                     hr_source_principal,
                 );
-                let Ok(()) = ocx.eq_trace(
+                let Ok(()) = ocx.eq(
                     &ObligationCause::dummy(),
                     param_env,
-                    ToTrace::to_trace(
-                        &ObligationCause::dummy(),
-                        hr_target_principal,
-                        hr_source_principal,
-                    ),
                     target_principal,
                     source_principal,
                 ) else {
@@ -211,7 +205,12 @@ fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
                 old_info
             }
         }
-        (_, ty::Dynamic(data, _, _)) => meth::get_vtable(cx, source, data.principal()),
+        (_, ty::Dynamic(data, _, _)) => meth::get_vtable(
+            cx,
+            source,
+            data.principal()
+                .map(|principal| bx.tcx().instantiate_bound_regions_with_erased(principal)),
+        ),
         _ => bug!("unsized_info: invalid unsizing {:?} -> {:?}", source, target),
     }
 }
@@ -615,11 +614,18 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
         return ongoing_codegen;
     }
 
+    if tcx.sess.target.need_explicit_cpu && tcx.sess.opts.cg.target_cpu.is_none() {
+        // The target has no default cpu, but none is set explicitly
+        tcx.dcx().emit_fatal(errors::CpuRequired);
+    }
+
     let cgu_name_builder = &mut CodegenUnitNameBuilder::new(tcx);
 
     // Run the monomorphization collector and partition the collected items into
     // codegen units.
-    let codegen_units = tcx.collect_and_partition_mono_items(()).codegen_units;
+    let MonoItemPartitions { codegen_units, autodiff_items, .. } =
+        tcx.collect_and_partition_mono_items(());
+    let autodiff_fncs = autodiff_items.to_vec();
 
     // Force all codegen_unit queries so they are already either red or green
     // when compile_codegen_unit accesses them. We are not able to re-execute
@@ -690,6 +696,10 @@ pub fn codegen_crate<B: ExtraBackendMethods>(
         );
     }
 
+    if !autodiff_fncs.is_empty() {
+        ongoing_codegen.submit_autodiff_items(autodiff_fncs);
+    }
+
     // For better throughput during parallel processing by LLVM, we used to sort
     // CGUs largest to smallest. This would lead to better thread utilization
     // by, for example, preventing a large CGU from being processed last and

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

@@ -1,5 +1,10 @@
+use std::str::FromStr;
+
 use rustc_ast::attr::list_contains_name;
-use rustc_ast::{MetaItemInner, attr};
+use rustc_ast::expand::autodiff_attrs::{
+    AutoDiffAttrs, DiffActivity, DiffMode, valid_input_activity, valid_ret_activity,
+};
+use rustc_ast::{MetaItem, MetaItemInner, attr};
 use rustc_attr_parsing::{InlineAttr, InstructionSetAttr, OptimizeAttr};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::codes::*;
@@ -13,6 +18,7 @@ use rustc_middle::middle::codegen_fn_attrs::{
 };
 use rustc_middle::mir::mono::Linkage;
 use rustc_middle::query::Providers;
+use rustc_middle::span_bug;
 use rustc_middle::ty::{self as ty, TyCtxt};
 use rustc_session::parse::feature_err;
 use rustc_session::{Session, lint};
@@ -65,6 +71,13 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
         codegen_fn_attrs.flags |= CodegenFnAttrFlags::TRACK_CALLER;
     }
 
+    // If our rustc version supports autodiff/enzyme, then we call our handler
+    // to check for any `#[rustc_autodiff(...)]` attributes.
+    if cfg!(llvm_enzyme) {
+        let ad = autodiff_attrs(tcx, did.into());
+        codegen_fn_attrs.autodiff_item = ad;
+    }
+
     // When `no_builtins` is applied at the crate level, we should add the
     // `no-builtins` attribute to each function to ensure it takes effect in LTO.
     let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID);
@@ -856,6 +869,109 @@ impl<'a> MixedExportNameAndNoMangleState<'a> {
     }
 }
 
+/// We now check the #\[rustc_autodiff\] attributes which we generated from the #[autodiff(...)]
+/// macros. There are two forms. The pure one without args to mark primal functions (the functions
+/// being differentiated). The other form is #[rustc_autodiff(Mode, ActivityList)] on top of the
+/// placeholder functions. We wrote the rustc_autodiff attributes ourself, so this should never
+/// panic, unless we introduced a bug when parsing the autodiff macro.
+fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> {
+    let attrs = tcx.get_attrs(id, sym::rustc_autodiff);
+
+    let attrs =
+        attrs.filter(|attr| attr.name_or_empty() == sym::rustc_autodiff).collect::<Vec<_>>();
+
+    // check for exactly one autodiff attribute on placeholder functions.
+    // There should only be one, since we generate a new placeholder per ad macro.
+    // FIXME(ZuseZ4): re-enable this check. Currently we add multiple, which doesn't cause harm but
+    // looks strange e.g. under cargo-expand.
+    let attr = match &attrs[..] {
+        [] => return None,
+        [attr] => attr,
+        // These two attributes are the same and unfortunately duplicated due to a previous bug.
+        [attr, _attr2] => attr,
+        _ => {
+            //FIXME(ZuseZ4): Once we fixed our parser, we should also prohibit the two-attribute
+            //branch above.
+            span_bug!(attrs[1].span, "cg_ssa: rustc_autodiff should only exist once per source");
+        }
+    };
+
+    let list = attr.meta_item_list().unwrap_or_default();
+
+    // empty autodiff attribute macros (i.e. `#[autodiff]`) are used to mark source functions
+    if list.is_empty() {
+        return Some(AutoDiffAttrs::source());
+    }
+
+    let [mode, input_activities @ .., ret_activity] = &list[..] else {
+        span_bug!(attr.span, "rustc_autodiff attribute must contain mode and activities");
+    };
+    let mode = if let MetaItemInner::MetaItem(MetaItem { path: ref p1, .. }) = mode {
+        p1.segments.first().unwrap().ident
+    } else {
+        span_bug!(attr.span, "rustc_autodiff attribute must contain mode");
+    };
+
+    // parse mode
+    let mode = match mode.as_str() {
+        "Forward" => DiffMode::Forward,
+        "Reverse" => DiffMode::Reverse,
+        "ForwardFirst" => DiffMode::ForwardFirst,
+        "ReverseFirst" => DiffMode::ReverseFirst,
+        _ => {
+            span_bug!(mode.span, "rustc_autodiff attribute contains invalid mode");
+        }
+    };
+
+    // First read the ret symbol from the attribute
+    let ret_symbol = if let MetaItemInner::MetaItem(MetaItem { path: ref p1, .. }) = ret_activity {
+        p1.segments.first().unwrap().ident
+    } else {
+        span_bug!(attr.span, "rustc_autodiff attribute must contain the return activity");
+    };
+
+    // Then parse it into an actual DiffActivity
+    let Ok(ret_activity) = DiffActivity::from_str(ret_symbol.as_str()) else {
+        span_bug!(ret_symbol.span, "invalid return activity");
+    };
+
+    // Now parse all the intermediate (input) activities
+    let mut arg_activities: Vec<DiffActivity> = vec![];
+    for arg in input_activities {
+        let arg_symbol = if let MetaItemInner::MetaItem(MetaItem { path: ref p2, .. }) = arg {
+            match p2.segments.first() {
+                Some(x) => x.ident,
+                None => {
+                    span_bug!(
+                        arg.span(),
+                        "rustc_autodiff attribute must contain the input activity"
+                    );
+                }
+            }
+        } else {
+            span_bug!(arg.span(), "rustc_autodiff attribute must contain the input activity");
+        };
+
+        match DiffActivity::from_str(arg_symbol.as_str()) {
+            Ok(arg_activity) => arg_activities.push(arg_activity),
+            Err(_) => {
+                span_bug!(arg_symbol.span, "invalid input activity");
+            }
+        }
+    }
+
+    for &input in &arg_activities {
+        if !valid_input_activity(mode, input) {
+            span_bug!(attr.span, "Invalid input activity {} for {} mode", input, mode);
+        }
+    }
+    if !valid_ret_activity(mode, ret_activity) {
+        span_bug!(attr.span, "Invalid return activity {} for {} mode", ret_activity, mode);
+    }
+
+    Some(AutoDiffAttrs { mode, ret_activity, input_activity: arg_activities })
+}
+
 pub(crate) fn provide(providers: &mut Providers) {
     *providers = Providers { codegen_fn_attrs, should_inherit_track_caller, ..*providers };
 }

compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs

@@ -507,7 +507,7 @@ pub enum VTableNameKind {
 pub fn compute_debuginfo_vtable_name<'tcx>(
     tcx: TyCtxt<'tcx>,
     t: Ty<'tcx>,
-    trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
+    trait_ref: Option<ty::ExistentialTraitRef<'tcx>>,
     kind: VTableNameKind,
 ) -> String {
     let cpp_like_debuginfo = cpp_like_debuginfo(tcx);
@@ -530,8 +530,8 @@ pub fn compute_debuginfo_vtable_name<'tcx>(
     }
 
     if let Some(trait_ref) = trait_ref {
-        let trait_ref = tcx
-            .normalize_erasing_late_bound_regions(ty::TypingEnv::fully_monomorphized(), trait_ref);
+        let trait_ref =
+            tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), trait_ref);
         push_item_name(tcx, trait_ref.def_id, true, &mut vtable_name);
         visited.clear();
         push_generic_params_internal(tcx, trait_ref.args, &mut vtable_name, &mut visited);
@@ -673,25 +673,23 @@ fn push_const_param<'tcx>(tcx: TyCtxt<'tcx>, ct: ty::Const<'tcx>, output: &mut S
         ty::ConstKind::Param(param) => {
             write!(output, "{}", param.name)
         }
-        ty::ConstKind::Value(ty, valtree) => {
-            match ty.kind() {
+        ty::ConstKind::Value(cv) => {
+            match cv.ty.kind() {
                 ty::Int(ity) => {
-                    // FIXME: directly extract the bits from a valtree instead of evaluating an
-                    // already evaluated `Const` in order to get the bits.
-                    let bits = ct
+                    let bits = cv
                         .try_to_bits(tcx, ty::TypingEnv::fully_monomorphized())
                         .expect("expected monomorphic const in codegen");
                     let val = Integer::from_int_ty(&tcx, *ity).size().sign_extend(bits) as i128;
                     write!(output, "{val}")
                 }
                 ty::Uint(_) => {
-                    let val = ct
+                    let val = cv
                         .try_to_bits(tcx, ty::TypingEnv::fully_monomorphized())
                         .expect("expected monomorphic const in codegen");
                     write!(output, "{val}")
                 }
                 ty::Bool => {
-                    let val = ct.try_to_bool().expect("expected monomorphic const in codegen");
+                    let val = cv.try_to_bool().expect("expected monomorphic const in codegen");
                     write!(output, "{val}")
                 }
                 _ => {
@@ -703,9 +701,7 @@ fn push_const_param<'tcx>(tcx: TyCtxt<'tcx>, ct: ty::Const<'tcx>, output: &mut S
                     // avoiding collisions and will make the emitted type names shorter.
                     let hash_short = tcx.with_stable_hashing_context(|mut hcx| {
                         let mut hasher = StableHasher::new();
-                        hcx.while_hashing_spans(false, |hcx| {
-                            (ty, valtree).hash_stable(hcx, &mut hasher)
-                        });
+                        hcx.while_hashing_spans(false, |hcx| cv.hash_stable(hcx, &mut hasher));
                         hasher.finish::<Hash64>()
                     });
 

compiler/rustc_codegen_ssa/src/errors.rs

@@ -39,6 +39,10 @@ pub(crate) struct CguNotRecorded<'a> {
     pub cgu_name: &'a str,
 }
 
+#[derive(Diagnostic)]
+#[diag(codegen_ssa_autodiff_without_lto)]
+pub struct AutodiffWithoutLto;
+
 #[derive(Diagnostic)]
 #[diag(codegen_ssa_unknown_reuse_kind)]
 pub(crate) struct UnknownReuseKind {
@@ -523,6 +527,10 @@ pub(crate) struct CheckInstalledVisualStudio;
 #[diag(codegen_ssa_insufficient_vs_code_product)]
 pub(crate) struct InsufficientVSCodeProduct;
 
+#[derive(Diagnostic)]
+#[diag(codegen_ssa_cpu_required)]
+pub(crate) struct CpuRequired;
+
 #[derive(Diagnostic)]
 #[diag(codegen_ssa_processing_dymutil_failed)]
 #[note]

compiler/rustc_codegen_ssa/src/meth.rs

@@ -1,5 +1,5 @@
 use rustc_middle::bug;
-use rustc_middle::ty::{self, GenericArgKind, Ty};
+use rustc_middle::ty::{self, GenericArgKind, Ty, TyCtxt};
 use rustc_session::config::Lto;
 use rustc_symbol_mangling::typeid_for_trait_ref;
 use rustc_target::callconv::FnAbi;
@@ -72,12 +72,19 @@ impl<'a, 'tcx> VirtualIndex {
 
 /// This takes a valid `self` receiver type and extracts the principal trait
 /// ref of the type. Return `None` if there is no principal trait.
-fn dyn_trait_in_self(ty: Ty<'_>) -> Option<ty::PolyExistentialTraitRef<'_>> {
+fn dyn_trait_in_self<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    ty: Ty<'tcx>,
+) -> Option<ty::ExistentialTraitRef<'tcx>> {
     for arg in ty.peel_refs().walk() {
         if let GenericArgKind::Type(ty) = arg.unpack()
             && let ty::Dynamic(data, _, _) = ty.kind()
         {
-            return data.principal();
+            // FIXME(arbitrary_self_types): This is likely broken for receivers which
+            // have a "non-self" trait objects as a generic argument.
+            return data
+                .principal()
+                .map(|principal| tcx.instantiate_bound_regions_with_erased(principal));
         }
     }
 
@@ -96,7 +103,7 @@ fn dyn_trait_in_self(ty: Ty<'_>) -> Option<ty::PolyExistentialTraitRef<'_>> {
 pub(crate) fn get_vtable<'tcx, Cx: CodegenMethods<'tcx>>(
     cx: &Cx,
     ty: Ty<'tcx>,
-    trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
+    trait_ref: Option<ty::ExistentialTraitRef<'tcx>>,
 ) -> Cx::Value {
     let tcx = cx.tcx();
 
@@ -131,7 +138,7 @@ pub(crate) fn load_vtable<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     if bx.cx().sess().opts.unstable_opts.virtual_function_elimination
         && bx.cx().sess().lto() == Lto::Fat
     {
-        if let Some(trait_ref) = dyn_trait_in_self(ty) {
+        if let Some(trait_ref) = dyn_trait_in_self(bx.tcx(), ty) {
             let typeid = bx.typeid_metadata(typeid_for_trait_ref(bx.tcx(), trait_ref)).unwrap();
             let func = bx.type_checked_load(llvtable, vtable_byte_offset, typeid);
             return func;

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -713,6 +713,11 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 // and `#[track_caller]` adds an implicit third argument.
                 (LangItem::PanicMisalignedPointerDereference, vec![required, found, location])
             }
+            AssertKind::NullPointerDereference => {
+                // It's `fn panic_null_pointer_dereference()`,
+                // `#[track_caller]` adds an implicit argument.
+                (LangItem::PanicNullPointerDereference, vec![location])
+            }
             _ => {
                 // It's `pub fn panic_...()` and `#[track_caller]` adds an implicit argument.
                 (msg.panic_function(), vec![location])

compiler/rustc_codegen_ssa/src/mir/constant.rs

@@ -43,7 +43,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             mir::Const::Ty(_, c) => match c.kind() {
                 // A constant that came from a const generic but was then used as an argument to
                 // old-style simd_shuffle (passing as argument instead of as a generic param).
-                rustc_type_ir::ConstKind::Value(_, valtree) => return Ok(Ok(valtree)),
+                rustc_type_ir::ConstKind::Value(cv) => return Ok(Ok(cv.valtree)),
                 other => span_bug!(constant.span, "{other:#?}"),
             },
             // We should never encounter `Const::Val` unless MIR opts (like const prop) evaluate

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -612,9 +612,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             mir::Rvalue::CopyForDeref(place) => {
                 self.codegen_operand(bx, &mir::Operand::Copy(place))
             }
-            mir::Rvalue::RawPtr(mutability, place) => {
-                let mk_ptr =
-                    move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| Ty::new_ptr(tcx, ty, mutability);
+            mir::Rvalue::RawPtr(kind, place) => {
+                let mk_ptr = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| {
+                    Ty::new_ptr(tcx, ty, kind.to_mutbl_lossy())
+                };
                 self.codegen_place_to_pointer(bx, place, mk_ptr)
             }
 

compiler/rustc_codegen_ssa/src/traits/debuginfo.rs

@@ -2,7 +2,7 @@ use std::ops::Range;
 
 use rustc_abi::Size;
 use rustc_middle::mir;
-use rustc_middle::ty::{Instance, PolyExistentialTraitRef, Ty};
+use rustc_middle::ty::{ExistentialTraitRef, Instance, Ty};
 use rustc_span::{SourceFile, Span, Symbol};
 use rustc_target::callconv::FnAbi;
 
@@ -13,7 +13,7 @@ pub trait DebugInfoCodegenMethods<'tcx>: BackendTypes {
     fn create_vtable_debuginfo(
         &self,
         ty: Ty<'tcx>,
-        trait_ref: Option<PolyExistentialTraitRef<'tcx>>,
+        trait_ref: Option<ExistentialTraitRef<'tcx>>,
         vtable: Self::Value,
     );
 

compiler/rustc_codegen_ssa/src/traits/misc.rs

@@ -10,11 +10,11 @@ use super::BackendTypes;
 pub trait MiscCodegenMethods<'tcx>: BackendTypes {
     fn vtables(
         &self,
-    ) -> &RefCell<FxHashMap<(Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>), Self::Value>>;
+    ) -> &RefCell<FxHashMap<(Ty<'tcx>, Option<ty::ExistentialTraitRef<'tcx>>), Self::Value>>;
     fn apply_vcall_visibility_metadata(
         &self,
         _ty: Ty<'tcx>,
-        _poly_trait_ref: Option<ty::PolyExistentialTraitRef<'tcx>>,
+        _poly_trait_ref: Option<ty::ExistentialTraitRef<'tcx>>,
         _vtable: Self::Value,
     ) {
     }

compiler/rustc_const_eval/messages.ftl

@@ -403,7 +403,7 @@ const_eval_uninhabited_enum_variant_read =
 const_eval_uninhabited_enum_variant_written =
     writing discriminant of an uninhabited enum variant
 
-const_eval_unmarked_const_fn_exposed = `{$def_path}` cannot be (indirectly) exposed to stable
+const_eval_unmarked_const_item_exposed = `{$def_path}` cannot be (indirectly) exposed to stable
     .help = either mark the callee as `#[rustc_const_stable_indirect]`, or the caller as `#[rustc_const_unstable]`
 const_eval_unmarked_intrinsic_exposed = intrinsic `{$def_path}` cannot be (indirectly) exposed to stable
     .help = mark the caller as `#[rustc_const_unstable]`, or mark the intrinsic `#[rustc_intrinsic_const_stable_indirect]` (but this requires team approval)
@@ -414,6 +414,7 @@ const_eval_unreachable_unwind =
 
 const_eval_unsized_local = unsized locals are not supported
 const_eval_unstable_const_fn = `{$def_path}` is not yet stable as a const fn
+const_eval_unstable_const_trait = `{$def_path}` is not yet stable as a const trait
 const_eval_unstable_in_stable_exposed =
     const function that might be (indirectly) exposed to stable cannot use `#[feature({$gate})]`
     .is_function_call = mark the callee as `#[rustc_const_stable_indirect]` if it does not itself require any unstable features

compiler/rustc_const_eval/src/check_consts/check.rs

@@ -8,6 +8,7 @@ use std::ops::Deref;
 
 use rustc_attr_parsing::{ConstStability, StabilityLevel};
 use rustc_errors::{Diag, ErrorGuaranteed};
+use rustc_hir::def::DefKind;
 use rustc_hir::def_id::DefId;
 use rustc_hir::{self as hir, LangItem};
 use rustc_index::bit_set::DenseBitSet;
@@ -29,7 +30,7 @@ use super::ops::{self, NonConstOp, Status};
 use super::qualifs::{self, HasMutInterior, NeedsDrop, NeedsNonConstDrop};
 use super::resolver::FlowSensitiveAnalysis;
 use super::{ConstCx, Qualif};
-use crate::check_consts::is_safe_to_expose_on_stable_const_fn;
+use crate::check_consts::is_fn_or_trait_safe_to_expose_on_stable;
 use crate::errors;
 
 type QualifResults<'mir, 'tcx, Q> =
@@ -470,6 +471,88 @@ impl<'mir, 'tcx> Checker<'mir, 'tcx> {
             self.tcx.crate_level_attribute_injection_span(self.tcx.local_def_id_to_hir_id(id))
         })
     }
+
+    /// Check the const stability of the given item (fn or trait).
+    fn check_callee_stability(&mut self, def_id: DefId) {
+        match self.tcx.lookup_const_stability(def_id) {
+            Some(ConstStability { level: StabilityLevel::Stable { .. }, .. }) => {
+                // All good.
+            }
+            None => {
+                // This doesn't need a separate const-stability check -- const-stability equals
+                // regular stability, and regular stability is checked separately.
+                // However, we *do* have to worry about *recursive* const stability.
+                if self.enforce_recursive_const_stability()
+                    && !is_fn_or_trait_safe_to_expose_on_stable(self.tcx, def_id)
+                {
+                    self.dcx().emit_err(errors::UnmarkedConstItemExposed {
+                        span: self.span,
+                        def_path: self.tcx.def_path_str(def_id),
+                    });
+                }
+            }
+            Some(ConstStability {
+                level: StabilityLevel::Unstable { implied_by: implied_feature, issue, .. },
+                feature,
+                ..
+            }) => {
+                // An unstable const fn/trait with a feature gate.
+                let callee_safe_to_expose_on_stable =
+                    is_fn_or_trait_safe_to_expose_on_stable(self.tcx, def_id);
+
+                // We only honor `span.allows_unstable` aka `#[allow_internal_unstable]` if
+                // the callee is safe to expose, to avoid bypassing recursive stability.
+                // This is not ideal since it means the user sees an error, not the macro
+                // author, but that's also the case if one forgets to set
+                // `#[allow_internal_unstable]` in the first place. Note that this cannot be
+                // integrated in the check below since we want to enforce
+                // `callee_safe_to_expose_on_stable` even if
+                // `!self.enforce_recursive_const_stability()`.
+                if (self.span.allows_unstable(feature)
+                    || implied_feature.is_some_and(|f| self.span.allows_unstable(f)))
+                    && callee_safe_to_expose_on_stable
+                {
+                    return;
+                }
+
+                // We can't use `check_op` to check whether the feature is enabled because
+                // the logic is a bit different than elsewhere: local functions don't need
+                // the feature gate, and there might be an "implied" gate that also suffices
+                // to allow this.
+                let feature_enabled = def_id.is_local()
+                    || self.tcx.features().enabled(feature)
+                    || implied_feature.is_some_and(|f| self.tcx.features().enabled(f))
+                    || {
+                        // When we're compiling the compiler itself we may pull in
+                        // crates from crates.io, but those crates may depend on other
+                        // crates also pulled in from crates.io. We want to ideally be
+                        // able to compile everything without requiring upstream
+                        // modifications, so in the case that this looks like a
+                        // `rustc_private` crate (e.g., a compiler crate) and we also have
+                        // the `-Z force-unstable-if-unmarked` flag present (we're
+                        // compiling a compiler crate), then let this missing feature
+                        // annotation slide.
+                        // This matches what we do in `eval_stability_allow_unstable` for
+                        // regular stability.
+                        feature == sym::rustc_private
+                            && issue == NonZero::new(27812)
+                            && self.tcx.sess.opts.unstable_opts.force_unstable_if_unmarked
+                    };
+                // Even if the feature is enabled, we still need check_op to double-check
+                // this if the callee is not safe to expose on stable.
+                if !feature_enabled || !callee_safe_to_expose_on_stable {
+                    self.check_op(ops::CallUnstable {
+                        def_id,
+                        feature,
+                        feature_enabled,
+                        safe_to_expose_on_stable: callee_safe_to_expose_on_stable,
+                        suggestion_span: self.crate_inject_span(),
+                        is_function_call: self.tcx.def_kind(def_id) != DefKind::Trait,
+                    });
+                }
+            }
+        }
+    }
 }
 
 impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
@@ -518,7 +601,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
             }
 
             Rvalue::Ref(_, BorrowKind::Mut { .. }, place)
-            | Rvalue::RawPtr(Mutability::Mut, place) => {
+            | Rvalue::RawPtr(RawPtrKind::Mut, place) => {
                 // Inside mutable statics, we allow arbitrary mutable references.
                 // We've allowed `static mut FOO = &mut [elements];` for a long time (the exact
                 // reasons why are lost to history), and there is no reason to restrict that to
@@ -536,7 +619,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
             }
 
             Rvalue::Ref(_, BorrowKind::Shared | BorrowKind::Fake(_), place)
-            | Rvalue::RawPtr(Mutability::Not, place) => {
+            | Rvalue::RawPtr(RawPtrKind::Const, place) => {
                 let borrowed_place_has_mut_interior = qualifs::in_place::<HasMutInterior, _>(
                     self.ccx,
                     &mut |local| self.qualifs.has_mut_interior(self.ccx, local, location),
@@ -548,6 +631,17 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                 }
             }
 
+            Rvalue::RawPtr(RawPtrKind::FakeForPtrMetadata, place) => {
+                // These are only inserted for slice length, so the place must already be indirect.
+                // This implies we do not have to worry about whether the borrow escapes.
+                if !place.is_indirect() {
+                    self.tcx.dcx().span_delayed_bug(
+                        self.body.source_info(location).span,
+                        "fake borrows are always indirect",
+                    );
+                }
+            }
+
             Rvalue::Cast(
                 CastKind::PointerCoercion(
                     PointerCoercion::MutToConstPointer
@@ -586,12 +680,23 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
             ) => {}
             Rvalue::ShallowInitBox(_, _) => {}
 
-            Rvalue::UnaryOp(_, operand) => {
+            Rvalue::UnaryOp(op, operand) => {
                 let ty = operand.ty(self.body, self.tcx);
-                if is_int_bool_float_or_char(ty) {
-                    // Int, bool, float, and char operations are fine.
-                } else {
-                    span_bug!(self.span, "non-primitive type in `Rvalue::UnaryOp`: {:?}", ty);
+                match op {
+                    UnOp::Not | UnOp::Neg => {
+                        if is_int_bool_float_or_char(ty) {
+                            // Int, bool, float, and char operations are fine.
+                        } else {
+                            span_bug!(
+                                self.span,
+                                "non-primitive type in `Rvalue::UnaryOp{op:?}`: {ty:?}",
+                            );
+                        }
+                    }
+                    UnOp::PtrMetadata => {
+                        // Getting the metadata from a pointer is always const.
+                        // We already validated the type is valid in the validator.
+                    }
                 }
             }
 
@@ -716,8 +821,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                             span: *fn_span,
                             call_source,
                         });
-                        // FIXME(const_trait_impl): do a more fine-grained check whether this
-                        // particular trait can be const-stably called.
+                        self.check_callee_stability(trait_did);
                     } else {
                         // Not even a const trait.
                         self.check_op(ops::FnCallNonConst {
@@ -793,7 +897,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                     // fallback body is safe to expose on stable.
                     let is_const_stable = intrinsic.const_stable
                         || (!intrinsic.must_be_overridden
-                            && is_safe_to_expose_on_stable_const_fn(tcx, callee));
+                            && is_fn_or_trait_safe_to_expose_on_stable(tcx, callee));
                     match tcx.lookup_const_stability(callee) {
                         None => {
                             // This doesn't need a separate const-stability check -- const-stability equals
@@ -842,83 +946,7 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> {
                 }
 
                 // Finally, stability for regular function calls -- this is the big one.
-                match tcx.lookup_const_stability(callee) {
-                    Some(ConstStability { level: StabilityLevel::Stable { .. }, .. }) => {
-                        // All good.
-                    }
-                    None => {
-                        // This doesn't need a separate const-stability check -- const-stability equals
-                        // regular stability, and regular stability is checked separately.
-                        // However, we *do* have to worry about *recursive* const stability.
-                        if self.enforce_recursive_const_stability()
-                            && !is_safe_to_expose_on_stable_const_fn(tcx, callee)
-                        {
-                            self.dcx().emit_err(errors::UnmarkedConstFnExposed {
-                                span: self.span,
-                                def_path: self.tcx.def_path_str(callee),
-                            });
-                        }
-                    }
-                    Some(ConstStability {
-                        level: StabilityLevel::Unstable { implied_by: implied_feature, issue, .. },
-                        feature,
-                        ..
-                    }) => {
-                        // An unstable const fn with a feature gate.
-                        let callee_safe_to_expose_on_stable =
-                            is_safe_to_expose_on_stable_const_fn(tcx, callee);
-
-                        // We only honor `span.allows_unstable` aka `#[allow_internal_unstable]` if
-                        // the callee is safe to expose, to avoid bypassing recursive stability.
-                        // This is not ideal since it means the user sees an error, not the macro
-                        // author, but that's also the case if one forgets to set
-                        // `#[allow_internal_unstable]` in the first place. Note that this cannot be
-                        // integrated in the check below since we want to enforce
-                        // `callee_safe_to_expose_on_stable` even if
-                        // `!self.enforce_recursive_const_stability()`.
-                        if (self.span.allows_unstable(feature)
-                            || implied_feature.is_some_and(|f| self.span.allows_unstable(f)))
-                            && callee_safe_to_expose_on_stable
-                        {
-                            return;
-                        }
-
-                        // We can't use `check_op` to check whether the feature is enabled because
-                        // the logic is a bit different than elsewhere: local functions don't need
-                        // the feature gate, and there might be an "implied" gate that also suffices
-                        // to allow this.
-                        let feature_enabled = callee.is_local()
-                            || tcx.features().enabled(feature)
-                            || implied_feature.is_some_and(|f| tcx.features().enabled(f))
-                            || {
-                                // When we're compiling the compiler itself we may pull in
-                                // crates from crates.io, but those crates may depend on other
-                                // crates also pulled in from crates.io. We want to ideally be
-                                // able to compile everything without requiring upstream
-                                // modifications, so in the case that this looks like a
-                                // `rustc_private` crate (e.g., a compiler crate) and we also have
-                                // the `-Z force-unstable-if-unmarked` flag present (we're
-                                // compiling a compiler crate), then let this missing feature
-                                // annotation slide.
-                                // This matches what we do in `eval_stability_allow_unstable` for
-                                // regular stability.
-                                feature == sym::rustc_private
-                                    && issue == NonZero::new(27812)
-                                    && tcx.sess.opts.unstable_opts.force_unstable_if_unmarked
-                            };
-                        // Even if the feature is enabled, we still need check_op to double-check
-                        // this if the callee is not safe to expose on stable.
-                        if !feature_enabled || !callee_safe_to_expose_on_stable {
-                            self.check_op(ops::FnCallUnstable {
-                                def_id: callee,
-                                feature,
-                                feature_enabled,
-                                safe_to_expose_on_stable: callee_safe_to_expose_on_stable,
-                                suggestion_span: self.crate_inject_span(),
-                            });
-                        }
-                    }
-                }
+                self.check_callee_stability(callee);
             }
 
             // Forbid all `Drop` terminators unless the place being dropped is a local with no

compiler/rustc_const_eval/src/check_consts/mod.rs

@@ -56,7 +56,7 @@ impl<'mir, 'tcx> ConstCx<'mir, 'tcx> {
         self.const_kind == Some(hir::ConstContext::ConstFn)
             && (self.tcx.features().staged_api()
                 || self.tcx.sess.opts.unstable_opts.force_unstable_if_unmarked)
-            && is_safe_to_expose_on_stable_const_fn(self.tcx, self.def_id().to_def_id())
+            && is_fn_or_trait_safe_to_expose_on_stable(self.tcx, self.def_id().to_def_id())
     }
 
     fn is_async(&self) -> bool {
@@ -84,28 +84,14 @@ pub fn rustc_allow_const_fn_unstable(
     attr::rustc_allow_const_fn_unstable(tcx.sess, attrs).any(|name| name == feature_gate)
 }
 
-/// Returns `true` if the given `const fn` is "safe to expose on stable".
-///
-/// Panics if the given `DefId` does not refer to a `const fn`.
+/// Returns `true` if the given `def_id` (trait or function) is "safe to expose on stable".
 ///
 /// This is relevant within a `staged_api` crate. Unlike with normal features, the use of unstable
 /// const features *recursively* taints the functions that use them. This is to avoid accidentally
 /// exposing e.g. the implementation of an unstable const intrinsic on stable. So we partition the
 /// world into two functions: those that are safe to expose on stable (and hence may not use
 /// unstable features, not even recursively), and those that are not.
-pub fn is_safe_to_expose_on_stable_const_fn(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
-    // A default body in a `#[const_trait]` is not const-stable because const trait fns currently
-    // cannot be const-stable. These functions can't be called from anything stable, so we shouldn't
-    // restrict them to only call const-stable functions.
-    if tcx.is_const_default_method(def_id) {
-        // FIXME(const_trait_impl): we have to eventually allow some of these if these things can ever be stable.
-        // They should probably behave like regular `const fn` for that...
-        return false;
-    }
-
-    // Const-stability is only relevant for `const fn`.
-    assert!(tcx.is_const_fn(def_id));
-
+pub fn is_fn_or_trait_safe_to_expose_on_stable(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
     match tcx.lookup_const_stability(def_id) {
         None => {
             // In a `staged_api` crate, we do enforce recursive const stability for all unmarked

compiler/rustc_const_eval/src/check_consts/ops.rs

@@ -377,11 +377,11 @@ fn build_error_for_const_call<'tcx>(
     err
 }
 
-/// A call to an `#[unstable]` const fn or `#[rustc_const_unstable]` function.
+/// A call to an `#[unstable]` const fn, `#[rustc_const_unstable]` function or trait.
 ///
-/// Contains the name of the feature that would allow the use of this function.
+/// Contains the name of the feature that would allow the use of this function/trait.
 #[derive(Debug)]
-pub(crate) struct FnCallUnstable {
+pub(crate) struct CallUnstable {
     pub def_id: DefId,
     pub feature: Symbol,
     /// If this is true, then the feature is enabled, but we need to still check if it is safe to
@@ -389,24 +389,33 @@ pub(crate) struct FnCallUnstable {
     pub feature_enabled: bool,
     pub safe_to_expose_on_stable: bool,
     pub suggestion_span: Option<Span>,
+    /// true if `def_id` is the function we are calling, false if `def_id` is an unstable trait.
+    pub is_function_call: bool,
 }
 
-impl<'tcx> NonConstOp<'tcx> for FnCallUnstable {
+impl<'tcx> NonConstOp<'tcx> for CallUnstable {
     fn status_in_item(&self, _ccx: &ConstCx<'_, 'tcx>) -> Status {
         Status::Unstable {
             gate: self.feature,
             gate_already_checked: self.feature_enabled,
             safe_to_expose_on_stable: self.safe_to_expose_on_stable,
-            is_function_call: true,
+            is_function_call: self.is_function_call,
         }
     }
 
     fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> Diag<'tcx> {
         assert!(!self.feature_enabled);
-        let mut err = ccx.dcx().create_err(errors::UnstableConstFn {
-            span,
-            def_path: ccx.tcx.def_path_str(self.def_id),
-        });
+        let mut err = if self.is_function_call {
+            ccx.dcx().create_err(errors::UnstableConstFn {
+                span,
+                def_path: ccx.tcx.def_path_str(self.def_id),
+            })
+        } else {
+            ccx.dcx().create_err(errors::UnstableConstTrait {
+                span,
+                def_path: ccx.tcx.def_path_str(self.def_id),
+            })
+        };
         // FIXME: make this translatable
         let msg = format!("add `#![feature({})]` to the crate attributes to enable", self.feature);
         #[allow(rustc::untranslatable_diagnostic)]

compiler/rustc_const_eval/src/check_consts/qualifs.rs

@@ -345,7 +345,7 @@ where
         Const::Ty(_, ct)
             if matches!(
                 ct.kind(),
-                ty::ConstKind::Param(_) | ty::ConstKind::Error(_) | ty::ConstKind::Value(_, _)
+                ty::ConstKind::Param(_) | ty::ConstKind::Error(_) | ty::ConstKind::Value(_)
             ) =>
         {
             None

compiler/rustc_const_eval/src/const_eval/machine.rs

@@ -21,9 +21,10 @@ use super::error::*;
 use crate::errors::{LongRunning, LongRunningWarn};
 use crate::fluent_generated as fluent;
 use crate::interpret::{
-    self, AllocId, AllocRange, ConstAllocation, CtfeProvenance, FnArg, Frame, GlobalAlloc, ImmTy,
-    InterpCx, InterpResult, MPlaceTy, OpTy, RangeSet, Scalar, compile_time_machine, interp_ok,
-    throw_exhaust, throw_inval, throw_ub, throw_ub_custom, throw_unsup, throw_unsup_format,
+    self, AllocId, AllocInit, AllocRange, ConstAllocation, CtfeProvenance, FnArg, Frame,
+    GlobalAlloc, ImmTy, InterpCx, InterpResult, MPlaceTy, OpTy, RangeSet, Scalar,
+    compile_time_machine, interp_ok, throw_exhaust, throw_inval, throw_ub, throw_ub_custom,
+    throw_unsup, throw_unsup_format,
 };
 
 /// When hitting this many interpreted terminators we emit a deny by default lint
@@ -420,6 +421,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
                     Size::from_bytes(size),
                     align,
                     interpret::MemoryKind::Machine(MemoryKind::Heap),
+                    AllocInit::Uninit,
                 )?;
                 ecx.write_pointer(ptr, dest)?;
             }
@@ -506,6 +508,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
                     found: eval_to_int(found)?,
                 }
             }
+            NullPointerDereference => NullPointerDereference,
         };
         Err(ConstEvalErrKind::AssertFailure(err)).into()
     }

compiler/rustc_const_eval/src/const_eval/valtrees.rs

@@ -272,7 +272,8 @@ pub(crate) fn eval_to_valtree<'tcx>(
 
 /// Converts a `ValTree` to a `ConstValue`, which is needed after mir
 /// construction has finished.
-// FIXME Merge `valtree_to_const_value` and `valtree_into_mplace` into one function
+// FIXME(valtrees): Merge `valtree_to_const_value` and `valtree_into_mplace` into one function
+// FIXME(valtrees): Accept `ty::Value` instead of `Ty` and `ty::ValTree` separately.
 #[instrument(skip(tcx), level = "debug", ret)]
 pub fn valtree_to_const_value<'tcx>(
     tcx: TyCtxt<'tcx>,

compiler/rustc_const_eval/src/errors.rs

@@ -121,6 +121,14 @@ pub(crate) struct UnstableConstFn {
     pub def_path: String,
 }
 
+#[derive(Diagnostic)]
+#[diag(const_eval_unstable_const_trait)]
+pub(crate) struct UnstableConstTrait {
+    #[primary_span]
+    pub span: Span,
+    pub def_path: String,
+}
+
 #[derive(Diagnostic)]
 #[diag(const_eval_unstable_intrinsic)]
 pub(crate) struct UnstableIntrinsic {
@@ -139,9 +147,9 @@ pub(crate) struct UnstableIntrinsic {
 }
 
 #[derive(Diagnostic)]
-#[diag(const_eval_unmarked_const_fn_exposed)]
+#[diag(const_eval_unmarked_const_item_exposed)]
 #[help]
-pub(crate) struct UnmarkedConstFnExposed {
+pub(crate) struct UnmarkedConstItemExposed {
     #[primary_span]
     pub span: Span,
     pub def_path: String,

compiler/rustc_const_eval/src/interpret/call.rs

@@ -5,6 +5,7 @@ use std::borrow::Cow;
 
 use either::{Left, Right};
 use rustc_abi::{self as abi, ExternAbi, FieldIdx, Integer, VariantIdx};
+use rustc_hir::def_id::DefId;
 use rustc_middle::ty::layout::{FnAbiOf, IntegerExt, LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self, AdtDef, Instance, Ty, VariantDef};
 use rustc_middle::{bug, mir, span_bug};
@@ -693,25 +694,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 trace!("Virtual call dispatches to {fn_inst:#?}");
                 // We can also do the lookup based on `def_id` and `dyn_ty`, and check that that
                 // produces the same result.
-                if cfg!(debug_assertions) {
-                    let tcx = *self.tcx;
-
-                    let trait_def_id = tcx.trait_of_item(def_id).unwrap();
-                    let virtual_trait_ref =
-                        ty::TraitRef::from_method(tcx, trait_def_id, instance.args);
-                    let existential_trait_ref =
-                        ty::ExistentialTraitRef::erase_self_ty(tcx, virtual_trait_ref);
-                    let concrete_trait_ref = existential_trait_ref.with_self_ty(tcx, dyn_ty);
-
-                    let concrete_method = Instance::expect_resolve_for_vtable(
-                        tcx,
-                        self.typing_env,
-                        def_id,
-                        instance.args.rebase_onto(tcx, trait_def_id, concrete_trait_ref.args),
-                        self.cur_span(),
-                    );
-                    assert_eq!(fn_inst, concrete_method);
-                }
+                self.assert_virtual_instance_matches_concrete(dyn_ty, def_id, instance, fn_inst);
 
                 // Adjust receiver argument. Layout can be any (thin) ptr.
                 let receiver_ty = Ty::new_mut_ptr(self.tcx.tcx, dyn_ty);
@@ -744,6 +727,30 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         }
     }
 
+    fn assert_virtual_instance_matches_concrete(
+        &self,
+        dyn_ty: Ty<'tcx>,
+        def_id: DefId,
+        virtual_instance: ty::Instance<'tcx>,
+        concrete_instance: ty::Instance<'tcx>,
+    ) {
+        let tcx = *self.tcx;
+
+        let trait_def_id = tcx.trait_of_item(def_id).unwrap();
+        let virtual_trait_ref = ty::TraitRef::from_method(tcx, trait_def_id, virtual_instance.args);
+        let existential_trait_ref = ty::ExistentialTraitRef::erase_self_ty(tcx, virtual_trait_ref);
+        let concrete_trait_ref = existential_trait_ref.with_self_ty(tcx, dyn_ty);
+
+        let concrete_method = Instance::expect_resolve_for_vtable(
+            tcx,
+            self.typing_env,
+            def_id,
+            virtual_instance.args.rebase_onto(tcx, trait_def_id, concrete_trait_ref.args),
+            self.cur_span(),
+        );
+        assert_eq!(concrete_instance, concrete_method);
+    }
+
     /// Initiate a tail call to this function -- popping the current stack frame, pushing the new
     /// stack frame and initializing the arguments.
     pub(super) fn init_fn_tail_call(

compiler/rustc_const_eval/src/interpret/cast.rs

@@ -414,36 +414,33 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
 
                 // Sanity-check that `supertrait_vtable_slot` in this type's vtable indeed produces
                 // our destination trait.
-                if cfg!(debug_assertions) {
-                    let vptr_entry_idx =
-                        self.tcx.supertrait_vtable_slot((src_pointee_ty, dest_pointee_ty));
-                    let vtable_entries = self.vtable_entries(data_a.principal(), ty);
-                    if let Some(entry_idx) = vptr_entry_idx {
-                        let Some(&ty::VtblEntry::TraitVPtr(upcast_trait_ref)) =
-                            vtable_entries.get(entry_idx)
-                        else {
-                            span_bug!(
-                                self.cur_span(),
-                                "invalid vtable entry index in {} -> {} upcast",
-                                src_pointee_ty,
-                                dest_pointee_ty
-                            );
-                        };
-                        let erased_trait_ref = upcast_trait_ref
-                            .map_bound(|r| ty::ExistentialTraitRef::erase_self_ty(*self.tcx, r));
-                        assert!(
-                            data_b
-                                .principal()
-                                .is_some_and(|b| self.eq_in_param_env(erased_trait_ref, b))
+                let vptr_entry_idx =
+                    self.tcx.supertrait_vtable_slot((src_pointee_ty, dest_pointee_ty));
+                let vtable_entries = self.vtable_entries(data_a.principal(), ty);
+                if let Some(entry_idx) = vptr_entry_idx {
+                    let Some(&ty::VtblEntry::TraitVPtr(upcast_trait_ref)) =
+                        vtable_entries.get(entry_idx)
+                    else {
+                        span_bug!(
+                            self.cur_span(),
+                            "invalid vtable entry index in {} -> {} upcast",
+                            src_pointee_ty,
+                            dest_pointee_ty
                         );
-                    } else {
-                        // In this case codegen would keep using the old vtable. We don't want to do
-                        // that as it has the wrong trait. The reason codegen can do this is that
-                        // one vtable is a prefix of the other, so we double-check that.
-                        let vtable_entries_b = self.vtable_entries(data_b.principal(), ty);
-                        assert!(&vtable_entries[..vtable_entries_b.len()] == vtable_entries_b);
                     };
-                }
+                    let erased_trait_ref =
+                        ty::ExistentialTraitRef::erase_self_ty(*self.tcx, upcast_trait_ref);
+                    assert!(data_b.principal().is_some_and(|b| self.eq_in_param_env(
+                        erased_trait_ref,
+                        self.tcx.instantiate_bound_regions_with_erased(b)
+                    )));
+                } else {
+                    // In this case codegen would keep using the old vtable. We don't want to do
+                    // that as it has the wrong trait. The reason codegen can do this is that
+                    // one vtable is a prefix of the other, so we double-check that.
+                    let vtable_entries_b = self.vtable_entries(data_b.principal(), ty);
+                    assert!(&vtable_entries[..vtable_entries_b.len()] == vtable_entries_b);
+                };
 
                 // Get the destination trait vtable and return that.
                 let new_vptr = self.get_vtable_ptr(ty, data_b)?;

compiler/rustc_const_eval/src/interpret/intrinsics.rs

@@ -747,7 +747,13 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     {
         let a: F = self.read_scalar(&args[0])?.to_float()?;
         let b: F = self.read_scalar(&args[1])?.to_float()?;
-        let res = self.adjust_nan(a.min(b), &[a, b]);
+        let res = if a == b {
+            // They are definitely not NaN (those are never equal), but they could be `+0` and `-0`.
+            // Let the machine decide which one to return.
+            M::equal_float_min_max(self, a, b)
+        } else {
+            self.adjust_nan(a.min(b), &[a, b])
+        };
         self.write_scalar(res, dest)?;
         interp_ok(())
     }
@@ -762,7 +768,13 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     {
         let a: F = self.read_scalar(&args[0])?.to_float()?;
         let b: F = self.read_scalar(&args[1])?.to_float()?;
-        let res = self.adjust_nan(a.max(b), &[a, b]);
+        let res = if a == b {
+            // They are definitely not NaN (those are never equal), but they could be `+0` and `-0`.
+            // Let the machine decide which one to return.
+            M::equal_float_min_max(self, a, b)
+        } else {
+            self.adjust_nan(a.max(b), &[a, b])
+        };
         self.write_scalar(res, dest)?;
         interp_ok(())
     }

compiler/rustc_const_eval/src/interpret/machine.rs

@@ -278,6 +278,12 @@ pub trait Machine<'tcx>: Sized {
         F2::NAN
     }
 
+    /// Determines the result of `min`/`max` on floats when the arguments are equal.
+    fn equal_float_min_max<F: Float>(_ecx: &InterpCx<'tcx, Self>, a: F, _b: F) -> F {
+        // By default, we pick the left argument.
+        a
+    }
+
     /// Called before a basic block terminator is executed.
     #[inline]
     fn before_terminator(_ecx: &mut InterpCx<'tcx, Self>) -> InterpResult<'tcx> {

compiler/rustc_const_eval/src/interpret/memory.rs

@@ -20,10 +20,10 @@ use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
 use tracing::{debug, instrument, trace};
 
 use super::{
-    AllocBytes, AllocId, AllocMap, AllocRange, Allocation, CheckAlignMsg, CheckInAllocMsg,
-    CtfeProvenance, GlobalAlloc, InterpCx, InterpResult, Machine, MayLeak, Misalignment, Pointer,
-    PointerArithmetic, Provenance, Scalar, alloc_range, err_ub, err_ub_custom, interp_ok, throw_ub,
-    throw_ub_custom, throw_unsup, throw_unsup_format,
+    AllocBytes, AllocId, AllocInit, AllocMap, AllocRange, Allocation, CheckAlignMsg,
+    CheckInAllocMsg, CtfeProvenance, GlobalAlloc, InterpCx, InterpResult, Machine, MayLeak,
+    Misalignment, Pointer, PointerArithmetic, Provenance, Scalar, alloc_range, err_ub,
+    err_ub_custom, interp_ok, throw_ub, throw_ub_custom, throw_unsup, throw_unsup_format,
 };
 use crate::fluent_generated as fluent;
 
@@ -230,11 +230,12 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         size: Size,
         align: Align,
         kind: MemoryKind<M::MemoryKind>,
+        init: AllocInit,
     ) -> InterpResult<'tcx, Pointer<M::Provenance>> {
         let alloc = if M::PANIC_ON_ALLOC_FAIL {
-            Allocation::uninit(size, align)
+            Allocation::new(size, align, init)
         } else {
-            Allocation::try_uninit(size, align)?
+            Allocation::try_new(size, align, init)?
         };
         self.insert_allocation(alloc, kind)
     }
@@ -270,6 +271,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         M::adjust_alloc_root_pointer(self, Pointer::from(id), Some(kind))
     }
 
+    /// If this grows the allocation, `init_growth` determines
+    /// whether the additional space will be initialized.
     pub fn reallocate_ptr(
         &mut self,
         ptr: Pointer<Option<M::Provenance>>,
@@ -277,6 +280,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         new_size: Size,
         new_align: Align,
         kind: MemoryKind<M::MemoryKind>,
+        init_growth: AllocInit,
     ) -> InterpResult<'tcx, Pointer<M::Provenance>> {
         let (alloc_id, offset, _prov) = self.ptr_get_alloc_id(ptr, 0)?;
         if offset.bytes() != 0 {
@@ -289,7 +293,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
 
         // For simplicities' sake, we implement reallocate as "alloc, copy, dealloc".
         // This happens so rarely, the perf advantage is outweighed by the maintenance cost.
-        let new_ptr = self.allocate_ptr(new_size, new_align, kind)?;
+        // If requested, we zero-init the entire allocation, to ensure that a growing
+        // allocation has its new bytes properly set. For the part that is copied,
+        // `mem_copy` below will de-initialize things as necessary.
+        let new_ptr = self.allocate_ptr(new_size, new_align, kind, init_growth)?;
         let old_size = match old_size_and_align {
             Some((size, _align)) => size,
             None => self.get_alloc_raw(alloc_id)?.size(),
@@ -830,9 +837,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     /// [`InterpCx::get_alloc_info`] if all you need to check is whether the kind is
     /// [`AllocKind::Dead`] because it doesn't have to look up the type and layout of statics.
     pub fn is_alloc_live(&self, id: AllocId) -> bool {
-        self.tcx.try_get_global_alloc(id).is_some()
-            || self.memory.alloc_map.contains_key_ref(&id)
+        self.memory.alloc_map.contains_key_ref(&id)
             || self.memory.extra_fn_ptr_map.contains_key(&id)
+            // We check `tcx` last as that has to acquire a lock in `many-seeds` mode.
+            // This also matches the order in `get_alloc_info`.
+            || self.tcx.try_get_global_alloc(id).is_some()
     }
 
     /// Obtain the size and alignment of an allocation, even if that allocation has

compiler/rustc_const_eval/src/interpret/place.rs

@@ -12,9 +12,9 @@ use rustc_middle::{bug, mir, span_bug};
 use tracing::{instrument, trace};
 
 use super::{
-    AllocRef, AllocRefMut, CheckAlignMsg, CtfeProvenance, ImmTy, Immediate, InterpCx, InterpResult,
-    Machine, MemoryKind, Misalignment, OffsetMode, OpTy, Operand, Pointer, Projectable, Provenance,
-    Scalar, alloc_range, interp_ok, mir_assign_valid_types,
+    AllocInit, AllocRef, AllocRefMut, CheckAlignMsg, CtfeProvenance, ImmTy, Immediate, InterpCx,
+    InterpResult, Machine, MemoryKind, Misalignment, OffsetMode, OpTy, Operand, Pointer,
+    Projectable, Provenance, Scalar, alloc_range, interp_ok, mir_assign_valid_types,
 };
 
 #[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)]
@@ -983,7 +983,7 @@ where
         let Some((size, align)) = self.size_and_align_of(&meta, &layout)? else {
             span_bug!(self.cur_span(), "cannot allocate space for `extern` type, size is not known")
         };
-        let ptr = self.allocate_ptr(size, align, kind)?;
+        let ptr = self.allocate_ptr(size, align, kind, AllocInit::Uninit)?;
         interp_ok(self.ptr_with_meta_to_mplace(ptr.into(), meta, layout, /*unaligned*/ false))
     }
 

compiler/rustc_const_eval/src/interpret/step.rs

@@ -237,7 +237,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 self.write_immediate(*val, &dest)?;
             }
 
-            RawPtr(_, place) => {
+            RawPtr(kind, place) => {
                 // Figure out whether this is an addr_of of an already raw place.
                 let place_base_raw = if place.is_indirect_first_projection() {
                     let ty = self.frame().body.local_decls[place.local].ty;
@@ -250,8 +250,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 let src = self.eval_place(place)?;
                 let place = self.force_allocation(&src)?;
                 let mut val = ImmTy::from_immediate(place.to_ref(self), dest.layout);
-                if !place_base_raw {
-                    // If this was not already raw, it needs retagging.
+                if !place_base_raw && !kind.is_fake() {
+                    // If this was not already raw, it needs retagging -- except for "fake"
+                    // raw borrows whose defining property is that they do not get retagged.
                     val = M::retag_ptr_value(self, mir::RetagKind::Raw, &val)?;
                 }
                 self.write_immediate(*val, &dest)?;

compiler/rustc_const_eval/src/interpret/traits.rs

@@ -54,7 +54,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     ) -> &'tcx [VtblEntry<'tcx>] {
         if let Some(trait_) = trait_ {
             let trait_ref = trait_.with_self_ty(*self.tcx, dyn_ty);
-            let trait_ref = self.tcx.erase_regions(trait_ref);
+            let trait_ref =
+                self.tcx.erase_regions(self.tcx.instantiate_bound_regions_with_erased(trait_ref));
             self.tcx.vtable_entries(trait_ref)
         } else {
             TyCtxt::COMMON_VTABLE_ENTRIES

compiler/rustc_const_eval/src/interpret/util.rs

@@ -2,7 +2,7 @@ use std::ops::ControlFlow;
 
 use rustc_hir::def_id::LocalDefId;
 use rustc_middle::mir;
-use rustc_middle::mir::interpret::{Allocation, InterpResult, Pointer};
+use rustc_middle::mir::interpret::{AllocInit, Allocation, InterpResult, Pointer};
 use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::ty::{
     self, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor,
@@ -76,7 +76,7 @@ pub(crate) fn create_static_alloc<'tcx>(
     static_def_id: LocalDefId,
     layout: TyAndLayout<'tcx>,
 ) -> InterpResult<'tcx, MPlaceTy<'tcx>> {
-    let alloc = Allocation::try_uninit(layout.size, layout.align.abi)?;
+    let alloc = Allocation::try_new(layout.size, layout.align.abi, AllocInit::Uninit)?;
     let alloc_id = ecx.tcx.reserve_and_set_static_alloc(static_def_id.into());
     assert_eq!(ecx.machine.static_root_ids, None);
     ecx.machine.static_root_ids = Some((alloc_id, static_def_id));

compiler/rustc_const_eval/src/lib.rs

@@ -46,8 +46,13 @@ pub fn provide(providers: &mut Providers) {
     };
     providers.hooks.try_destructure_mir_constant_for_user_output =
         const_eval::try_destructure_mir_constant_for_user_output;
-    providers.valtree_to_const_val = |tcx, (ty, valtree)| {
-        const_eval::valtree_to_const_value(tcx, ty::TypingEnv::fully_monomorphized(), ty, valtree)
+    providers.valtree_to_const_val = |tcx, cv| {
+        const_eval::valtree_to_const_value(
+            tcx,
+            ty::TypingEnv::fully_monomorphized(),
+            cv.ty,
+            cv.valtree,
+        )
     };
     providers.check_validity_requirement = |tcx, (init_kind, param_env_and_ty)| {
         util::check_validity_requirement(tcx, init_kind, param_env_and_ty)

compiler/rustc_data_structures/Cargo.toml

@@ -10,11 +10,11 @@ bitflags = "2.4.1"
 either = "1.0"
 elsa = "=1.7.1"
 ena = "0.14.3"
-indexmap = { version = "2.4.0", features = ["rustc-rayon"] }
+indexmap = "2.4.0"
 jobserver_crate = { version = "0.1.28", package = "jobserver" }
 measureme = "11"
 rustc-hash = "2.0.0"
-rustc-rayon = "0.5.0"
+rustc-rayon = { version = "0.5.1", features = ["indexmap"] }
 rustc-stable-hash = { version = "0.1.0", features = ["nightly"] }
 rustc_arena = { path = "../rustc_arena" }
 rustc_graphviz = { path = "../rustc_graphviz" }

compiler/rustc_data_structures/src/lib.rs

@@ -76,6 +76,7 @@ pub mod sync;
 pub mod tagged_ptr;
 pub mod temp_dir;
 pub mod thinvec;
+pub mod thousands;
 pub mod transitive_relation;
 pub mod unhash;
 pub mod unord;

compiler/rustc_data_structures/src/thousands/mod.rs

@@ -0,0 +1,16 @@
+//! This is an extremely bare-bones alternative to the `thousands` crate on
+//! crates.io, for printing large numbers in a readable fashion.
+
+#[cfg(test)]
+mod tests;
+
+// Converts the number to a string, with underscores as the thousands separator.
+pub fn format_with_underscores(n: usize) -> String {
+    let mut s = n.to_string();
+    let mut i = s.len();
+    while i > 3 {
+        i -= 3;
+        s.insert(i, '_');
+    }
+    s
+}

compiler/rustc_data_structures/src/thousands/tests.rs

@@ -0,0 +1,14 @@
+use super::*;
+
+#[test]
+fn test_format_with_underscores() {
+    assert_eq!("0", format_with_underscores(0));
+    assert_eq!("1", format_with_underscores(1));
+    assert_eq!("99", format_with_underscores(99));
+    assert_eq!("345", format_with_underscores(345));
+    assert_eq!("1_000", format_with_underscores(1_000));
+    assert_eq!("12_001", format_with_underscores(12_001));
+    assert_eq!("999_999", format_with_underscores(999_999));
+    assert_eq!("1_000_000", format_with_underscores(1_000_000));
+    assert_eq!("12_345_678", format_with_underscores(12_345_678));
+}

compiler/rustc_driver_impl/src/lib.rs

@@ -160,7 +160,7 @@ pub trait Callbacks {
     fn after_crate_root_parsing(
         &mut self,
         _compiler: &interface::Compiler,
-        _queries: &ast::Crate,
+        _krate: &mut ast::Crate,
     ) -> Compilation {
         Compilation::Continue
     }
@@ -311,7 +311,7 @@ pub fn run_compiler(at_args: &[String], callbacks: &mut (dyn Callbacks + Send))
 
         // Parse the crate root source code (doesn't parse submodules yet)
         // Everything else is parsed during macro expansion.
-        let krate = passes::parse(sess);
+        let mut krate = passes::parse(sess);
 
         // If pretty printing is requested: Figure out the representation, print it and exit
         if let Some(pp_mode) = sess.opts.pretty {
@@ -328,7 +328,7 @@ pub fn run_compiler(at_args: &[String], callbacks: &mut (dyn Callbacks + Send))
             return early_exit();
         }
 
-        if callbacks.after_crate_root_parsing(compiler, &krate) == Compilation::Stop {
+        if callbacks.after_crate_root_parsing(compiler, &mut krate) == Compilation::Stop {
             return early_exit();
         }
 

compiler/rustc_driver_impl/src/pretty.rs

@@ -7,6 +7,7 @@ use rustc_ast_pretty::pprust as pprust_ast;
 use rustc_middle::bug;
 use rustc_middle::mir::{write_mir_graphviz, write_mir_pretty};
 use rustc_middle::ty::{self, TyCtxt};
+use rustc_mir_build::thir::print::{thir_flat, thir_tree};
 use rustc_session::Session;
 use rustc_session::config::{OutFileName, PpHirMode, PpMode, PpSourceMode};
 use rustc_smir::rustc_internal::pretty::write_smir_pretty;
@@ -313,7 +314,7 @@ pub fn print<'tcx>(sess: &Session, ppm: PpMode, ex: PrintExtra<'tcx>) {
             tcx.dcx().abort_if_errors();
             debug!("pretty printing THIR tree");
             for did in tcx.hir().body_owners() {
-                let _ = writeln!(out, "{:?}:\n{}\n", did, tcx.thir_tree(did));
+                let _ = writeln!(out, "{:?}:\n{}\n", did, thir_tree(tcx, did));
             }
             out
         }
@@ -324,7 +325,7 @@ pub fn print<'tcx>(sess: &Session, ppm: PpMode, ex: PrintExtra<'tcx>) {
             tcx.dcx().abort_if_errors();
             debug!("pretty printing THIR flat");
             for did in tcx.hir().body_owners() {
-                let _ = writeln!(out, "{:?}:\n{}\n", did, tcx.thir_flat(did));
+                let _ = writeln!(out, "{:?}:\n{}\n", did, thir_flat(tcx, did));
             }
             out
         }

compiler/rustc_feature/src/builtin_attrs.rs

@@ -1136,7 +1136,7 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
     ),
     rustc_attr!(
         TEST, rustc_dump_vtable, Normal, template!(Word),
-        WarnFollowing, EncodeCrossCrate::Yes
+        WarnFollowing, EncodeCrossCrate::No
     ),
     rustc_attr!(
         TEST, rustc_dummy, Normal, template!(Word /* doesn't matter*/),

compiler/rustc_hir/src/hir.rs

@@ -1437,7 +1437,7 @@ impl<'hir> Pat<'hir> {
 
         use PatKind::*;
         match self.kind {
-            Wild | Never | Expr(_) | Range(..) | Binding(.., None) | Path(_) | Err(_) => true,
+            Wild | Never | Expr(_) | Range(..) | Binding(.., None) | Err(_) => true,
             Box(s) | Deref(s) | Ref(s, _) | Binding(.., Some(s)) | Guard(s, _) => s.walk_short_(it),
             Struct(_, fields, _) => fields.iter().all(|field| field.pat.walk_short_(it)),
             TupleStruct(_, s, _) | Tuple(s, _) | Or(s) => s.iter().all(|p| p.walk_short_(it)),
@@ -1464,7 +1464,7 @@ impl<'hir> Pat<'hir> {
 
         use PatKind::*;
         match self.kind {
-            Wild | Never | Expr(_) | Range(..) | Binding(.., None) | Path(_) | Err(_) => {}
+            Wild | Never | Expr(_) | Range(..) | Binding(.., None) | Err(_) => {}
             Box(s) | Deref(s) | Ref(s, _) | Binding(.., Some(s)) | Guard(s, _) => s.walk_(it),
             Struct(_, fields, _) => fields.iter().for_each(|field| field.pat.walk_(it)),
             TupleStruct(_, s, _) | Tuple(s, _) | Or(s) => s.iter().for_each(|p| p.walk_(it)),
@@ -1618,9 +1618,6 @@ pub enum PatKind<'hir> {
     /// A never pattern `!`.
     Never,
 
-    /// A path pattern for a unit struct/variant or a (maybe-associated) constant.
-    Path(QPath<'hir>),
-
     /// A tuple pattern (e.g., `(a, b)`).
     /// If the `..` pattern fragment is present, then `Option<usize>` denotes its position.
     /// `0 <= position <= subpats.len()`

compiler/rustc_hir/src/intravisit.rs

@@ -709,9 +709,6 @@ pub fn walk_pat<'v, V: Visitor<'v>>(visitor: &mut V, pattern: &'v Pat<'v>) -> V:
             try_visit!(visitor.visit_qpath(qpath, pattern.hir_id, pattern.span));
             walk_list!(visitor, visit_pat, children);
         }
-        PatKind::Path(ref qpath) => {
-            try_visit!(visitor.visit_qpath(qpath, pattern.hir_id, pattern.span));
-        }
         PatKind::Struct(ref qpath, fields, _) => {
             try_visit!(visitor.visit_qpath(qpath, pattern.hir_id, pattern.span));
             walk_list!(visitor, visit_pat_field, fields);

compiler/rustc_hir/src/lang_items.rs

@@ -316,6 +316,7 @@ language_item_table! {
     PanicAsyncFnResumedPanic, sym::panic_const_async_fn_resumed_panic, panic_const_async_fn_resumed_panic, Target::Fn, GenericRequirement::None;
     PanicAsyncGenFnResumedPanic, sym::panic_const_async_gen_fn_resumed_panic, panic_const_async_gen_fn_resumed_panic, Target::Fn, GenericRequirement::None;
     PanicGenFnNonePanic, sym::panic_const_gen_fn_none_panic, panic_const_gen_fn_none_panic, Target::Fn, GenericRequirement::None;
+    PanicNullPointerDereference, sym::panic_null_pointer_dereference, panic_null_pointer_dereference, Target::Fn, GenericRequirement::None;
     /// libstd panic entry point. Necessary for const eval to be able to catch it
     BeginPanic,              sym::begin_panic,         begin_panic_fn,             Target::Fn,             GenericRequirement::None;
 

compiler/rustc_hir/src/pat_util.rs

@@ -105,7 +105,10 @@ impl hir::Pat<'_> {
         let mut variants = vec![];
         self.walk(|p| match &p.kind {
             PatKind::Or(_) => false,
-            PatKind::Path(hir::QPath::Resolved(_, path))
+            PatKind::Expr(hir::PatExpr {
+                kind: hir::PatExprKind::Path(hir::QPath::Resolved(_, path)),
+                ..
+            })
             | PatKind::TupleStruct(hir::QPath::Resolved(_, path), ..)
             | PatKind::Struct(hir::QPath::Resolved(_, path), ..) => {
                 if let Res::Def(DefKind::Variant | DefKind::Ctor(CtorOf::Variant, ..), id) =

compiler/rustc_hir_analysis/src/check/check.rs

@@ -27,7 +27,6 @@ use rustc_session::lint::builtin::UNINHABITED_STATIC;
 use rustc_trait_selection::error_reporting::InferCtxtErrorExt;
 use rustc_trait_selection::error_reporting::traits::on_unimplemented::OnUnimplementedDirective;
 use rustc_trait_selection::traits;
-use rustc_trait_selection::traits::outlives_bounds::InferCtxtExt as _;
 use rustc_type_ir::fold::TypeFoldable;
 use tracing::{debug, instrument};
 use ty::TypingMode;
@@ -417,9 +416,7 @@ fn check_opaque_meets_bounds<'tcx>(
     }
 
     let wf_tys = ocx.assumed_wf_types_and_report_errors(param_env, defining_use_anchor)?;
-    let implied_bounds = infcx.implied_bounds_tys(param_env, def_id, &wf_tys);
-    let outlives_env = OutlivesEnvironment::with_bounds(param_env, implied_bounds);
-    ocx.resolve_regions_and_report_errors(defining_use_anchor, &outlives_env)?;
+    ocx.resolve_regions_and_report_errors(defining_use_anchor, param_env, wf_tys)?;
 
     if infcx.next_trait_solver() {
         Ok(())

compiler/rustc_hir_analysis/src/check/compare_impl_item.rs

@@ -9,7 +9,6 @@ use rustc_errors::{Applicability, ErrorGuaranteed, pluralize, struct_span_code_e
 use rustc_hir::def::{DefKind, Res};
 use rustc_hir::intravisit::VisitorExt;
 use rustc_hir::{self as hir, AmbigArg, GenericParamKind, ImplItemKind, intravisit};
-use rustc_infer::infer::outlives::env::OutlivesEnvironment;
 use rustc_infer::infer::{self, InferCtxt, TyCtxtInferExt};
 use rustc_infer::traits::util;
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
@@ -24,7 +23,6 @@ use rustc_span::Span;
 use rustc_trait_selection::error_reporting::InferCtxtErrorExt;
 use rustc_trait_selection::infer::InferCtxtExt;
 use rustc_trait_selection::regions::InferCtxtRegionExt;
-use rustc_trait_selection::traits::outlives_bounds::InferCtxtExt as _;
 use rustc_trait_selection::traits::{
     self, FulfillmentError, ObligationCause, ObligationCauseCode, ObligationCtxt,
 };
@@ -416,11 +414,7 @@ fn compare_method_predicate_entailment<'tcx>(
 
     // Finally, resolve all regions. This catches wily misuses of
     // lifetime parameters.
-    let outlives_env = OutlivesEnvironment::with_bounds(
-        param_env,
-        infcx.implied_bounds_tys(param_env, impl_m_def_id, &wf_tys),
-    );
-    let errors = infcx.resolve_regions(&outlives_env);
+    let errors = infcx.resolve_regions(impl_m_def_id, param_env, wf_tys);
     if !errors.is_empty() {
         return Err(infcx
             .tainted_by_errors()
@@ -430,12 +424,12 @@ fn compare_method_predicate_entailment<'tcx>(
     Ok(())
 }
 
-struct RemapLateParam<'a, 'tcx> {
+struct RemapLateParam<'tcx> {
     tcx: TyCtxt<'tcx>,
-    mapping: &'a FxIndexMap<ty::LateParamRegionKind, ty::LateParamRegionKind>,
+    mapping: FxIndexMap<ty::LateParamRegionKind, ty::LateParamRegionKind>,
 }
 
-impl<'tcx> TypeFolder<TyCtxt<'tcx>> for RemapLateParam<'_, 'tcx> {
+impl<'tcx> TypeFolder<TyCtxt<'tcx>> for RemapLateParam<'tcx> {
     fn cx(&self) -> TyCtxt<'tcx> {
         self.tcx
     }
@@ -659,6 +653,7 @@ pub(super) fn collect_return_position_impl_trait_in_trait_tys<'tcx>(
                 }))),
                 terr,
                 false,
+                None,
             );
             return Err(diag.emit());
         }
@@ -725,11 +720,7 @@ pub(super) fn collect_return_position_impl_trait_in_trait_tys<'tcx>(
 
     // Finally, resolve all regions. This catches wily misuses of
     // lifetime parameters.
-    let outlives_env = OutlivesEnvironment::with_bounds(
-        param_env,
-        infcx.implied_bounds_tys(param_env, impl_m_def_id, &wf_tys),
-    );
-    ocx.resolve_regions_and_report_errors(impl_m_def_id, &outlives_env)?;
+    ocx.resolve_regions_and_report_errors(impl_m_def_id, param_env, wf_tys)?;
 
     let mut remapped_types = DefIdMap::default();
     for (def_id, (ty, args)) in collected_types {
@@ -1080,6 +1071,7 @@ fn report_trait_method_mismatch<'tcx>(
         }))),
         terr,
         false,
+        None,
     );
 
     diag.emit()
@@ -1872,6 +1864,7 @@ fn compare_const_predicate_entailment<'tcx>(
             }))),
             terr,
             false,
+            None,
         );
         return Err(diag.emit());
     };
@@ -1883,8 +1876,7 @@ fn compare_const_predicate_entailment<'tcx>(
         return Err(infcx.err_ctxt().report_fulfillment_errors(errors));
     }
 
-    let outlives_env = OutlivesEnvironment::new(param_env);
-    ocx.resolve_regions_and_report_errors(impl_ct_def_id, &outlives_env)
+    ocx.resolve_regions_and_report_errors(impl_ct_def_id, param_env, [])
 }
 
 #[instrument(level = "debug", skip(tcx))]
@@ -2017,8 +2009,7 @@ fn compare_type_predicate_entailment<'tcx>(
 
     // Finally, resolve all regions. This catches wily misuses of
     // lifetime parameters.
-    let outlives_env = OutlivesEnvironment::new(param_env);
-    ocx.resolve_regions_and_report_errors(impl_ty_def_id, &outlives_env)
+    ocx.resolve_regions_and_report_errors(impl_ty_def_id, param_env, [])
 }
 
 /// Validate that `ProjectionCandidate`s created for this associated type will
@@ -2147,9 +2138,7 @@ pub(super) fn check_type_bounds<'tcx>(
 
     // Finally, resolve all regions. This catches wily misuses of
     // lifetime parameters.
-    let implied_bounds = infcx.implied_bounds_tys(param_env, impl_ty_def_id, &assumed_wf_types);
-    let outlives_env = OutlivesEnvironment::with_bounds(param_env, implied_bounds);
-    ocx.resolve_regions_and_report_errors(impl_ty_def_id, &outlives_env)
+    ocx.resolve_regions_and_report_errors(impl_ty_def_id, param_env, assumed_wf_types)
 }
 
 struct ReplaceTy<'tcx> {

compiler/rustc_hir_analysis/src/check/compare_impl_item/refine.rs

@@ -3,7 +3,6 @@ use rustc_data_structures::fx::FxIndexSet;
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_infer::infer::TyCtxtInferExt;
-use rustc_infer::infer::outlives::env::OutlivesEnvironment;
 use rustc_lint_defs::builtin::{REFINING_IMPL_TRAIT_INTERNAL, REFINING_IMPL_TRAIT_REACHABLE};
 use rustc_middle::span_bug;
 use rustc_middle::traits::ObligationCause;
@@ -13,7 +12,6 @@ use rustc_middle::ty::{
 };
 use rustc_span::Span;
 use rustc_trait_selection::regions::InferCtxtRegionExt;
-use rustc_trait_selection::traits::outlives_bounds::InferCtxtExt;
 use rustc_trait_selection::traits::{ObligationCtxt, elaborate, normalize_param_env_or_error};
 
 /// Check that an implementation does not refine an RPITIT from a trait method signature.
@@ -170,11 +168,7 @@ pub(crate) fn check_refining_return_position_impl_trait_in_trait<'tcx>(
         tcx.dcx().delayed_bug("encountered errors when checking RPITIT refinement (selection)");
         return;
     }
-    let outlives_env = OutlivesEnvironment::with_bounds(
-        param_env,
-        infcx.implied_bounds_tys(param_env, impl_m.def_id.expect_local(), &implied_wf_types),
-    );
-    let errors = infcx.resolve_regions(&outlives_env);
+    let errors = infcx.resolve_regions(impl_m.def_id.expect_local(), param_env, implied_wf_types);
     if !errors.is_empty() {
         tcx.dcx().delayed_bug("encountered errors when checking RPITIT refinement (regions)");
         return;
@@ -305,8 +299,7 @@ fn report_mismatched_rpitit_signature<'tcx>(
     })
     .collect();
 
-    let mut return_ty =
-        trait_m_sig.output().fold_with(&mut super::RemapLateParam { tcx, mapping: &mapping });
+    let mut return_ty = trait_m_sig.output().fold_with(&mut super::RemapLateParam { tcx, mapping });
 
     if tcx.asyncness(impl_m_def_id).is_async() && tcx.asyncness(trait_m_def_id).is_async() {
         let ty::Alias(ty::Projection, future_ty) = return_ty.kind() else {

compiler/rustc_hir_analysis/src/check/dropck.rs

@@ -1,11 +1,6 @@
-// FIXME(@lcnr): Move this module out of `rustc_hir_analysis`.
-//
-// We don't do any drop checking during hir typeck.
-
 use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::codes::*;
 use rustc_errors::{ErrorGuaranteed, struct_span_code_err};
-use rustc_infer::infer::outlives::env::OutlivesEnvironment;
 use rustc_infer::infer::{RegionResolutionError, TyCtxtInferExt};
 use rustc_infer::traits::{ObligationCause, ObligationCauseCode};
 use rustc_middle::ty::util::CheckRegions;
@@ -33,7 +28,10 @@ use crate::hir::def_id::{DefId, LocalDefId};
 ///    struct/enum definition for the nominal type itself (i.e.
 ///    cannot do `struct S<T>; impl<T:Clone> Drop for S<T> { ... }`).
 ///
-pub fn check_drop_impl(tcx: TyCtxt<'_>, drop_impl_did: DefId) -> Result<(), ErrorGuaranteed> {
+pub(crate) fn check_drop_impl(
+    tcx: TyCtxt<'_>,
+    drop_impl_did: DefId,
+) -> Result<(), ErrorGuaranteed> {
     match tcx.impl_polarity(drop_impl_did) {
         ty::ImplPolarity::Positive => {}
         ty::ImplPolarity::Negative => {
@@ -192,7 +190,7 @@ fn ensure_drop_predicates_are_implied_by_item_defn<'tcx>(
         return Err(guar.unwrap());
     }
 
-    let errors = ocx.infcx.resolve_regions(&OutlivesEnvironment::new(adt_env));
+    let errors = ocx.infcx.resolve_regions(adt_def_id, adt_env, []);
     if !errors.is_empty() {
         let mut guar = None;
         for error in errors {

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -199,7 +199,8 @@ pub fn check_intrinsic_type(
         let split: Vec<&str> = name_str.split('_').collect();
         assert!(split.len() >= 2, "Atomic intrinsic in an incorrect format");
 
-        //We only care about the operation here
+        // Each atomic op has variants with different suffixes (`_seq_cst`, `_acquire`, etc.). Use
+        // string ops to strip the suffixes, because the variants all get the same treatment here.
         let (n_tps, inputs, output) = match split[1] {
             "cxchg" | "cxchgweak" => (
                 1,

compiler/rustc_hir_analysis/src/check/mod.rs

@@ -80,7 +80,6 @@ use rustc_errors::{Diag, ErrorGuaranteed, pluralize, struct_span_code_err};
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::intravisit::Visitor;
 use rustc_index::bit_set::DenseBitSet;
-use rustc_infer::infer::outlives::env::OutlivesEnvironment;
 use rustc_infer::infer::{self, TyCtxtInferExt as _};
 use rustc_infer::traits::ObligationCause;
 use rustc_middle::query::Providers;
@@ -456,18 +455,14 @@ fn fn_sig_suggestion<'tcx>(
     let mut output = sig.output();
 
     let asyncness = if tcx.asyncness(assoc.def_id).is_async() {
-        output = if let ty::Alias(_, alias_ty) = *output.kind() {
-            tcx.explicit_item_super_predicates(alias_ty.def_id)
+        output = if let ty::Alias(_, alias_ty) = *output.kind()
+            && let Some(output) = tcx
+                .explicit_item_self_bounds(alias_ty.def_id)
                 .iter_instantiated_copied(tcx, alias_ty.args)
                 .find_map(|(bound, _)| {
                     bound.as_projection_clause()?.no_bound_vars()?.term.as_type()
-                })
-                .unwrap_or_else(|| {
-                    span_bug!(
-                        ident.span,
-                        "expected async fn to have `impl Future` output, but it returns {output}"
-                    )
-                })
+                }) {
+            output
         } else {
             span_bug!(
                 ident.span,
@@ -650,13 +645,13 @@ pub fn check_function_signature<'tcx>(
                 }))),
                 err,
                 false,
+                None,
             );
             return Err(diag.emit());
         }
     }
 
-    let outlives_env = OutlivesEnvironment::new(param_env);
-    if let Err(e) = ocx.resolve_regions_and_report_errors(local_id, &outlives_env) {
+    if let Err(e) = ocx.resolve_regions_and_report_errors(local_id, param_env, []) {
         return Err(e);
     }
 

compiler/rustc_hir_analysis/src/check/region.rs

@@ -703,7 +703,6 @@ fn resolve_local<'tcx>(
             | PatKind::Binding(hir::BindingMode(hir::ByRef::No, _), ..)
             | PatKind::Wild
             | PatKind::Never
-            | PatKind::Path(_)
             | PatKind::Expr(_)
             | PatKind::Range(_, _, _)
             | PatKind::Err(_) => false,

compiler/rustc_hir_analysis/src/check/wfcheck.rs

@@ -25,11 +25,10 @@ use rustc_middle::{bug, span_bug};
 use rustc_session::parse::feature_err;
 use rustc_span::{DUMMY_SP, Ident, Span, sym};
 use rustc_trait_selection::error_reporting::InferCtxtErrorExt;
-use rustc_trait_selection::regions::InferCtxtRegionExt;
+use rustc_trait_selection::regions::{InferCtxtRegionExt, OutlivesEnvironmentBuildExt};
 use rustc_trait_selection::traits::misc::{
     ConstParamTyImplementationError, type_allowed_to_implement_const_param_ty,
 };
-use rustc_trait_selection::traits::outlives_bounds::InferCtxtExt as _;
 use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt as _;
 use rustc_trait_selection::traits::{
     self, FulfillmentError, Obligation, ObligationCause, ObligationCauseCode, ObligationCtxt,
@@ -128,13 +127,17 @@ where
     let infcx_compat = infcx.fork();
 
     // We specifically want to call the non-compat version of `implied_bounds_tys`; we do this always.
-    let implied_bounds =
-        infcx.implied_bounds_tys_compat(param_env, body_def_id, &assumed_wf_types, false);
-    let outlives_env = OutlivesEnvironment::with_bounds(param_env, implied_bounds);
+    let outlives_env = OutlivesEnvironment::new_with_implied_bounds_compat(
+        &infcx,
+        body_def_id,
+        param_env,
+        assumed_wf_types.iter().copied(),
+        false,
+    );
 
     lint_redundant_lifetimes(tcx, body_def_id, &outlives_env);
 
-    let errors = infcx.resolve_regions(&outlives_env);
+    let errors = infcx.resolve_regions_with_outlives_env(&outlives_env);
     if errors.is_empty() {
         return Ok(());
     }
@@ -172,10 +175,14 @@ where
     // but that does result in slightly more work when this option is set and
     // just obscures what we mean here anyways. Let's just be explicit.
     if is_bevy && !infcx.tcx.sess.opts.unstable_opts.no_implied_bounds_compat {
-        let implied_bounds =
-            infcx_compat.implied_bounds_tys_compat(param_env, body_def_id, &assumed_wf_types, true);
-        let outlives_env = OutlivesEnvironment::with_bounds(param_env, implied_bounds);
-        let errors_compat = infcx_compat.resolve_regions(&outlives_env);
+        let outlives_env = OutlivesEnvironment::new_with_implied_bounds_compat(
+            &infcx,
+            body_def_id,
+            param_env,
+            assumed_wf_types,
+            true,
+        );
+        let errors_compat = infcx_compat.resolve_regions_with_outlives_env(&outlives_env);
         if errors_compat.is_empty() {
             Ok(())
         } else {
@@ -769,12 +776,7 @@ fn test_region_obligations<'tcx>(
 
     add_constraints(&infcx);
 
-    let outlives_environment = OutlivesEnvironment::with_bounds(
-        param_env,
-        infcx.implied_bounds_tys(param_env, id, wf_tys),
-    );
-
-    let errors = infcx.resolve_regions(&outlives_environment);
+    let errors = infcx.resolve_regions(id, param_env, wf_tys.iter().copied());
     debug!(?errors, "errors");
 
     // If we were able to prove that the type outlives the region without
@@ -2265,14 +2267,12 @@ impl<'tcx> WfCheckingCtxt<'_, 'tcx> {
 
 fn check_mod_type_wf(tcx: TyCtxt<'_>, module: LocalModDefId) -> Result<(), ErrorGuaranteed> {
     let items = tcx.hir_module_items(module);
-    let mut res = items.par_items(|item| tcx.ensure().check_well_formed(item.owner_id.def_id));
-    res =
-        res.and(items.par_impl_items(|item| tcx.ensure().check_well_formed(item.owner_id.def_id)));
-    res =
-        res.and(items.par_trait_items(|item| tcx.ensure().check_well_formed(item.owner_id.def_id)));
-    res = res
-        .and(items.par_foreign_items(|item| tcx.ensure().check_well_formed(item.owner_id.def_id)));
-    res = res.and(items.par_opaques(|item| tcx.ensure().check_well_formed(item)));
+    let res = items
+        .par_items(|item| tcx.ensure().check_well_formed(item.owner_id.def_id))
+        .and(items.par_impl_items(|item| tcx.ensure().check_well_formed(item.owner_id.def_id)))
+        .and(items.par_trait_items(|item| tcx.ensure().check_well_formed(item.owner_id.def_id)))
+        .and(items.par_foreign_items(|item| tcx.ensure().check_well_formed(item.owner_id.def_id)))
+        .and(items.par_opaques(|item| tcx.ensure().check_well_formed(item)));
     if module == LocalModDefId::CRATE_DEF_ID {
         super::entry::check_for_entry_fn(tcx);
     }

compiler/rustc_hir_analysis/src/coherence/builtin.rs

@@ -10,7 +10,6 @@ use rustc_hir as hir;
 use rustc_hir::ItemKind;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::lang_items::LangItem;
-use rustc_infer::infer::outlives::env::OutlivesEnvironment;
 use rustc_infer::infer::{self, RegionResolutionError, TyCtxtInferExt};
 use rustc_infer::traits::Obligation;
 use rustc_middle::ty::adjustment::CoerceUnsizedInfo;
@@ -346,8 +345,7 @@ fn visit_implementation_of_dispatch_from_dyn(checker: &Checker<'_>) -> Result<()
                 }
 
                 // Finally, resolve all regions.
-                let outlives_env = OutlivesEnvironment::new(param_env);
-                res = res.and(ocx.resolve_regions_and_report_errors(impl_did, &outlives_env));
+                res = res.and(ocx.resolve_regions_and_report_errors(impl_did, param_env, []));
             }
             res
         }
@@ -406,17 +404,12 @@ pub(crate) fn coerce_unsized_info<'tcx>(
             check_mutbl(mt_a, mt_b, &|ty| Ty::new_imm_ref(tcx, r_b, ty))
         }
 
-        (&ty::Ref(_, ty_a, mutbl_a), &ty::RawPtr(ty_b, mutbl_b)) => check_mutbl(
-            ty::TypeAndMut { ty: ty_a, mutbl: mutbl_a },
-            ty::TypeAndMut { ty: ty_b, mutbl: mutbl_b },
-            &|ty| Ty::new_imm_ptr(tcx, ty),
-        ),
-
-        (&ty::RawPtr(ty_a, mutbl_a), &ty::RawPtr(ty_b, mutbl_b)) => check_mutbl(
-            ty::TypeAndMut { ty: ty_a, mutbl: mutbl_a },
-            ty::TypeAndMut { ty: ty_b, mutbl: mutbl_b },
-            &|ty| Ty::new_imm_ptr(tcx, ty),
-        ),
+        (&ty::Ref(_, ty_a, mutbl_a), &ty::RawPtr(ty_b, mutbl_b))
+        | (&ty::RawPtr(ty_a, mutbl_a), &ty::RawPtr(ty_b, mutbl_b)) => {
+            let mt_a = ty::TypeAndMut { ty: ty_a, mutbl: mutbl_a };
+            let mt_b = ty::TypeAndMut { ty: ty_b, mutbl: mutbl_b };
+            check_mutbl(mt_a, mt_b, &|ty| Ty::new_imm_ptr(tcx, ty))
+        }
 
         (&ty::Adt(def_a, args_a), &ty::Adt(def_b, args_b))
             if def_a.is_struct() && def_b.is_struct() =>
@@ -564,8 +557,7 @@ pub(crate) fn coerce_unsized_info<'tcx>(
     }
 
     // Finally, resolve all regions.
-    let outlives_env = OutlivesEnvironment::new(param_env);
-    let _ = ocx.resolve_regions_and_report_errors(impl_did, &outlives_env);
+    let _ = ocx.resolve_regions_and_report_errors(impl_did, param_env, []);
 
     Ok(CoerceUnsizedInfo { custom_kind: kind })
 }

compiler/rustc_hir_analysis/src/coherence/mod.rs

@@ -158,12 +158,12 @@ fn coherent_trait(tcx: TyCtxt<'_>, def_id: DefId) -> Result<(), ErrorGuaranteed>
         let trait_ref = trait_header.trait_ref.instantiate_identity();
         let trait_def = tcx.trait_def(trait_ref.def_id);
 
-        res = res.and(check_impl(tcx, impl_def_id, trait_ref, trait_def));
-        res = res.and(check_object_overlap(tcx, impl_def_id, trait_ref));
-
-        res = res.and(unsafety::check_item(tcx, impl_def_id, trait_header, trait_def));
-        res = res.and(tcx.ensure().orphan_check_impl(impl_def_id));
-        res = res.and(builtin::check_trait(tcx, def_id, impl_def_id, trait_header));
+        res = res
+            .and(check_impl(tcx, impl_def_id, trait_ref, trait_def))
+            .and(check_object_overlap(tcx, impl_def_id, trait_ref))
+            .and(unsafety::check_item(tcx, impl_def_id, trait_header, trait_def))
+            .and(tcx.ensure().orphan_check_impl(impl_def_id))
+            .and(builtin::check_trait(tcx, def_id, impl_def_id, trait_header));
     }
 
     res
@@ -199,10 +199,9 @@ fn check_object_overlap<'tcx>(
 
         for component_def_id in component_def_ids {
             if !tcx.is_dyn_compatible(component_def_id) {
-                // FIXME(dyn_compat_renaming): Rename test and update comment.
                 // Without the 'dyn_compatible_for_dispatch' feature this is an error
                 // which will be reported by wfcheck. Ignore it here.
-                // This is tested by `coherence-impl-trait-for-trait-object-safe.rs`.
+                // This is tested by `coherence-impl-trait-for-trait-dyn-compatible.rs`.
                 // With the feature enabled, the trait is not implemented automatically,
                 // so this is valid.
             } else {

compiler/rustc_hir_analysis/src/collect.rs

@@ -57,7 +57,7 @@ mod type_of;
 
 ///////////////////////////////////////////////////////////////////////////
 
-pub fn provide(providers: &mut Providers) {
+pub(crate) fn provide(providers: &mut Providers) {
     resolve_bound_vars::provide(providers);
     *providers = Providers {
         type_of: type_of::type_of,
@@ -65,9 +65,9 @@ pub fn provide(providers: &mut Providers) {
         type_alias_is_lazy: type_of::type_alias_is_lazy,
         item_bounds: item_bounds::item_bounds,
         explicit_item_bounds: item_bounds::explicit_item_bounds,
-        item_super_predicates: item_bounds::item_super_predicates,
-        explicit_item_super_predicates: item_bounds::explicit_item_super_predicates,
-        item_non_self_assumptions: item_bounds::item_non_self_assumptions,
+        item_self_bounds: item_bounds::item_self_bounds,
+        explicit_item_self_bounds: item_bounds::explicit_item_self_bounds,
+        item_non_self_bounds: item_bounds::item_non_self_bounds,
         impl_super_outlives: item_bounds::impl_super_outlives,
         generics_of: generics_of::generics_of,
         predicates_of: predicates_of::predicates_of,
@@ -122,7 +122,7 @@ pub fn provide(providers: &mut Providers) {
 /// `ItemCtxt` is parameterized by a `DefId` that it uses to satisfy
 /// `probe_ty_param_bounds` requests, drawing the information from
 /// the HIR (`hir::Generics`), recursively.
-pub struct ItemCtxt<'tcx> {
+pub(crate) struct ItemCtxt<'tcx> {
     tcx: TyCtxt<'tcx>,
     item_def_id: LocalDefId,
     tainted_by_errors: Cell<Option<ErrorGuaranteed>>,
@@ -148,7 +148,7 @@ impl<'v> Visitor<'v> for HirPlaceholderCollector {
     }
 }
 
-pub struct CollectItemTypesVisitor<'tcx> {
+pub(crate) struct CollectItemTypesVisitor<'tcx> {
     pub tcx: TyCtxt<'tcx>,
 }
 
@@ -328,9 +328,9 @@ impl<'tcx> Visitor<'tcx> for CollectItemTypesVisitor<'tcx> {
         self.tcx.ensure().generics_of(def_id);
         self.tcx.ensure().predicates_of(def_id);
         self.tcx.ensure().explicit_item_bounds(def_id);
-        self.tcx.ensure().explicit_item_super_predicates(def_id);
+        self.tcx.ensure().explicit_item_self_bounds(def_id);
         self.tcx.ensure().item_bounds(def_id);
-        self.tcx.ensure().item_super_predicates(def_id);
+        self.tcx.ensure().item_self_bounds(def_id);
         if self.tcx.is_conditionally_const(def_id) {
             self.tcx.ensure().explicit_implied_const_bounds(def_id);
             self.tcx.ensure().const_conditions(def_id);
@@ -364,19 +364,19 @@ fn bad_placeholder<'cx, 'tcx>(
 }
 
 impl<'tcx> ItemCtxt<'tcx> {
-    pub fn new(tcx: TyCtxt<'tcx>, item_def_id: LocalDefId) -> ItemCtxt<'tcx> {
+    pub(crate) fn new(tcx: TyCtxt<'tcx>, item_def_id: LocalDefId) -> ItemCtxt<'tcx> {
         ItemCtxt { tcx, item_def_id, tainted_by_errors: Cell::new(None) }
     }
 
-    pub fn lower_ty(&self, hir_ty: &hir::Ty<'tcx>) -> Ty<'tcx> {
+    pub(crate) fn lower_ty(&self, hir_ty: &hir::Ty<'tcx>) -> Ty<'tcx> {
         self.lowerer().lower_ty(hir_ty)
     }
 
-    pub fn hir_id(&self) -> hir::HirId {
+    pub(crate) fn hir_id(&self) -> hir::HirId {
         self.tcx.local_def_id_to_hir_id(self.item_def_id)
     }
 
-    pub fn node(&self) -> hir::Node<'tcx> {
+    pub(crate) fn node(&self) -> hir::Node<'tcx> {
         self.tcx.hir_node(self.hir_id())
     }
 
@@ -822,7 +822,7 @@ fn lower_trait_item(tcx: TyCtxt<'_>, trait_item_id: hir::TraitItemId) {
 
         hir::TraitItemKind::Type(_, Some(_)) => {
             tcx.ensure().item_bounds(def_id);
-            tcx.ensure().item_super_predicates(def_id);
+            tcx.ensure().item_self_bounds(def_id);
             tcx.ensure().type_of(def_id);
             // Account for `type T = _;`.
             let mut visitor = HirPlaceholderCollector::default();
@@ -839,7 +839,7 @@ fn lower_trait_item(tcx: TyCtxt<'_>, trait_item_id: hir::TraitItemId) {
 
         hir::TraitItemKind::Type(_, None) => {
             tcx.ensure().item_bounds(def_id);
-            tcx.ensure().item_super_predicates(def_id);
+            tcx.ensure().item_self_bounds(def_id);
             // #74612: Visit and try to find bad placeholders
             // even if there is no concrete type.
             let mut visitor = HirPlaceholderCollector::default();

compiler/rustc_hir_analysis/src/collect/dump.rs

@@ -1,7 +1,8 @@
+use rustc_hir as hir;
 use rustc_hir::def_id::{CRATE_DEF_ID, LocalDefId};
 use rustc_hir::intravisit;
 use rustc_middle::hir::nested_filter;
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::ty::{self, TyCtxt, TypeVisitableExt};
 use rustc_span::sym;
 
 pub(crate) fn opaque_hidden_types(tcx: TyCtxt<'_>) {
@@ -87,3 +88,82 @@ pub(crate) fn def_parents(tcx: TyCtxt<'_>) {
         }
     }
 }
+
+pub(crate) fn vtables<'tcx>(tcx: TyCtxt<'tcx>) {
+    for id in tcx.hir().items() {
+        let def_id = id.owner_id.def_id;
+
+        let Some(attr) = tcx.get_attr(def_id, sym::rustc_dump_vtable) else {
+            continue;
+        };
+
+        let vtable_entries = match tcx.hir().item(id).kind {
+            hir::ItemKind::Impl(hir::Impl { of_trait: Some(_), .. }) => {
+                let trait_ref = tcx.impl_trait_ref(def_id).unwrap().instantiate_identity();
+                if trait_ref.has_non_region_param() {
+                    tcx.dcx().span_err(
+                        attr.span,
+                        "`rustc_dump_vtable` must be applied to non-generic impl",
+                    );
+                    continue;
+                }
+                if !tcx.is_dyn_compatible(trait_ref.def_id) {
+                    tcx.dcx().span_err(
+                        attr.span,
+                        "`rustc_dump_vtable` must be applied to dyn-compatible trait",
+                    );
+                    continue;
+                }
+                let Ok(trait_ref) = tcx
+                    .try_normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), trait_ref)
+                else {
+                    tcx.dcx().span_err(
+                        attr.span,
+                        "`rustc_dump_vtable` applied to impl header that cannot be normalized",
+                    );
+                    continue;
+                };
+                tcx.vtable_entries(trait_ref)
+            }
+            hir::ItemKind::TyAlias(_, _) => {
+                let ty = tcx.type_of(def_id).instantiate_identity();
+                if ty.has_non_region_param() {
+                    tcx.dcx().span_err(
+                        attr.span,
+                        "`rustc_dump_vtable` must be applied to non-generic type",
+                    );
+                    continue;
+                }
+                let Ok(ty) =
+                    tcx.try_normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), ty)
+                else {
+                    tcx.dcx().span_err(
+                        attr.span,
+                        "`rustc_dump_vtable` applied to type alias that cannot be normalized",
+                    );
+                    continue;
+                };
+                let ty::Dynamic(data, _, _) = *ty.kind() else {
+                    tcx.dcx().span_err(attr.span, "`rustc_dump_vtable` to type alias of dyn type");
+                    continue;
+                };
+                if let Some(principal) = data.principal() {
+                    tcx.vtable_entries(
+                        tcx.instantiate_bound_regions_with_erased(principal).with_self_ty(tcx, ty),
+                    )
+                } else {
+                    TyCtxt::COMMON_VTABLE_ENTRIES
+                }
+            }
+            _ => {
+                tcx.dcx().span_err(
+                    attr.span,
+                    "`rustc_dump_vtable` only applies to impl, or type alias of dyn type",
+                );
+                continue;
+            }
+        };
+
+        tcx.dcx().span_err(tcx.def_span(def_id), format!("vtable entries: {vtable_entries:#?}"));
+    }
+}

compiler/rustc_hir_analysis/src/collect/item_bounds.rs

@@ -350,7 +350,7 @@ pub(super) fn explicit_item_bounds(
     explicit_item_bounds_with_filter(tcx, def_id, PredicateFilter::All)
 }
 
-pub(super) fn explicit_item_super_predicates(
+pub(super) fn explicit_item_self_bounds(
     tcx: TyCtxt<'_>,
     def_id: LocalDefId,
 ) -> ty::EarlyBinder<'_, &'_ [(ty::Clause<'_>, Span)]> {
@@ -434,11 +434,11 @@ pub(super) fn item_bounds(tcx: TyCtxt<'_>, def_id: DefId) -> ty::EarlyBinder<'_,
     })
 }
 
-pub(super) fn item_super_predicates(
+pub(super) fn item_self_bounds(
     tcx: TyCtxt<'_>,
     def_id: DefId,
 ) -> ty::EarlyBinder<'_, ty::Clauses<'_>> {
-    tcx.explicit_item_super_predicates(def_id).map_bound(|bounds| {
+    tcx.explicit_item_self_bounds(def_id).map_bound(|bounds| {
         tcx.mk_clauses_from_iter(
             util::elaborate(tcx, bounds.iter().map(|&(bound, _span)| bound)).filter_only_self(),
         )
@@ -447,13 +447,12 @@ pub(super) fn item_super_predicates(
 
 /// This exists as an optimization to compute only the item bounds of the item
 /// that are not `Self` bounds.
-pub(super) fn item_non_self_assumptions(
+pub(super) fn item_non_self_bounds(
     tcx: TyCtxt<'_>,
     def_id: DefId,
 ) -> ty::EarlyBinder<'_, ty::Clauses<'_>> {
     let all_bounds: FxIndexSet<_> = tcx.item_bounds(def_id).skip_binder().iter().collect();
-    let own_bounds: FxIndexSet<_> =
-        tcx.item_super_predicates(def_id).skip_binder().iter().collect();
+    let own_bounds: FxIndexSet<_> = tcx.item_self_bounds(def_id).skip_binder().iter().collect();
     if all_bounds.len() == own_bounds.len() {
         ty::EarlyBinder::bind(ty::ListWithCachedTypeInfo::empty())
     } else {