Auto merge of #3693 - rust-lang:rustup-2024-06-21, r=oli-obk

Automatic Rustup

.gitignore

@@ -52,6 +52,7 @@ build/
 /src/tools/x/target
 # Created by default with `src/ci/docker/run.sh`
 /obj/
+/rustc-ice*
 
 ## Temporary files
 *~

Cargo.lock

@@ -709,6 +709,7 @@ dependencies = [
  "clippy_config",
  "itertools 0.12.1",
  "rustc-semver",
+ "rustc_apfloat",
 ]
 
 [[package]]
@@ -3479,14 +3480,6 @@ dependencies = [
  "wasmparser",
 ]
 
-[[package]]
-name = "rust-demangler"
-version = "0.0.1"
-dependencies = [
- "regex",
- "rustc-demangle",
-]
-
 [[package]]
 name = "rustbook"
 version = "0.1.0"
@@ -4913,6 +4906,7 @@ version = "0.0.0"
 dependencies = [
  "bitflags 2.5.0",
  "derivative",
+ "indexmap",
  "rustc_ast_ir",
  "rustc_data_structures",
  "rustc_index",

Cargo.toml

@@ -23,7 +23,6 @@ members = [
   "src/tools/remote-test-client",
   "src/tools/remote-test-server",
   "src/tools/rust-installer",
-  "src/tools/rust-demangler",
   "src/tools/rustdoc",
   "src/tools/rls",
   "src/tools/rustfmt",

compiler/rustc_ast/src/ast.rs

@@ -975,7 +975,9 @@ impl UnOp {
     }
 }
 
-/// A statement
+/// A statement. No `attrs` or `tokens` fields because each `StmtKind` variant
+/// contains an AST node with those fields. (Except for `StmtKind::Empty`,
+/// which never has attrs or tokens)
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub struct Stmt {
     pub id: NodeId,
@@ -3161,13 +3163,16 @@ pub struct Delegation {
     pub path: Path,
     pub rename: Option<Ident>,
     pub body: Option<P<Block>>,
+    /// The item was expanded from a glob delegation item.
+    pub from_glob: bool,
 }
 
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub struct DelegationMac {
     pub qself: Option<P<QSelf>>,
     pub prefix: Path,
-    pub suffixes: ThinVec<(Ident, Option<Ident>)>,
+    // Some for list delegation, and None for glob delegation.
+    pub suffixes: Option<ThinVec<(Ident, Option<Ident>)>>,
     pub body: Option<P<Block>>,
 }
 
@@ -3179,38 +3184,6 @@ pub struct StaticItem {
     pub expr: Option<P<Expr>>,
 }
 
-/// A static item in `extern` block.
-// This struct is identical to StaticItem for now but it's going to have a safety attribute.
-#[derive(Clone, Encodable, Decodable, Debug)]
-pub struct StaticForeignItem {
-    pub ty: P<Ty>,
-    pub safety: Safety,
-    pub mutability: Mutability,
-    pub expr: Option<P<Expr>>,
-}
-
-impl From<StaticItem> for StaticForeignItem {
-    fn from(static_item: StaticItem) -> StaticForeignItem {
-        StaticForeignItem {
-            ty: static_item.ty,
-            safety: static_item.safety,
-            mutability: static_item.mutability,
-            expr: static_item.expr,
-        }
-    }
-}
-
-impl From<StaticForeignItem> for StaticItem {
-    fn from(static_item: StaticForeignItem) -> StaticItem {
-        StaticItem {
-            ty: static_item.ty,
-            safety: static_item.safety,
-            mutability: static_item.mutability,
-            expr: static_item.expr,
-        }
-    }
-}
-
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub struct ConstItem {
     pub defaultness: Defaultness,
@@ -3294,7 +3267,7 @@ pub enum ItemKind {
     ///
     /// E.g. `reuse <Type as Trait>::name { target_expr_template }`.
     Delegation(Box<Delegation>),
-    /// A list delegation item (`reuse prefix::{a, b, c}`).
+    /// A list or glob delegation item (`reuse prefix::{a, b, c}`, `reuse prefix::*`).
     /// Treated similarly to a macro call and expanded early.
     DelegationMac(Box<DelegationMac>),
 }
@@ -3375,7 +3348,7 @@ pub enum AssocItemKind {
     MacCall(P<MacCall>),
     /// An associated delegation item.
     Delegation(Box<Delegation>),
-    /// An associated delegation item list.
+    /// An associated list or glob delegation item.
     DelegationMac(Box<DelegationMac>),
 }
 
@@ -3425,7 +3398,7 @@ impl TryFrom<ItemKind> for AssocItemKind {
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub enum ForeignItemKind {
     /// A foreign static item (`static FOO: u8`).
-    Static(Box<StaticForeignItem>),
+    Static(Box<StaticItem>),
     /// An foreign function.
     Fn(Box<Fn>),
     /// An foreign type.

compiler/rustc_ast/src/mut_visit.rs

@@ -1162,7 +1162,14 @@ impl NoopVisitItemKind for ItemKind {
             }
             ItemKind::MacCall(m) => vis.visit_mac_call(m),
             ItemKind::MacroDef(def) => vis.visit_macro_def(def),
-            ItemKind::Delegation(box Delegation { id, qself, path, rename, body }) => {
+            ItemKind::Delegation(box Delegation {
+                id,
+                qself,
+                path,
+                rename,
+                body,
+                from_glob: _,
+            }) => {
                 vis.visit_id(id);
                 vis.visit_qself(qself);
                 vis.visit_path(path);
@@ -1176,10 +1183,12 @@ impl NoopVisitItemKind for ItemKind {
             ItemKind::DelegationMac(box DelegationMac { qself, prefix, suffixes, body }) => {
                 vis.visit_qself(qself);
                 vis.visit_path(prefix);
-                for (ident, rename) in suffixes {
-                    vis.visit_ident(ident);
-                    if let Some(rename) = rename {
-                        vis.visit_ident(rename);
+                if let Some(suffixes) = suffixes {
+                    for (ident, rename) in suffixes {
+                        vis.visit_ident(ident);
+                        if let Some(rename) = rename {
+                            vis.visit_ident(rename);
+                        }
                     }
                 }
                 if let Some(body) = body {
@@ -1218,7 +1227,14 @@ impl NoopVisitItemKind for AssocItemKind {
                 visit_opt(ty, |ty| visitor.visit_ty(ty));
             }
             AssocItemKind::MacCall(mac) => visitor.visit_mac_call(mac),
-            AssocItemKind::Delegation(box Delegation { id, qself, path, rename, body }) => {
+            AssocItemKind::Delegation(box Delegation {
+                id,
+                qself,
+                path,
+                rename,
+                body,
+                from_glob: _,
+            }) => {
                 visitor.visit_id(id);
                 visitor.visit_qself(qself);
                 visitor.visit_path(path);
@@ -1232,10 +1248,12 @@ impl NoopVisitItemKind for AssocItemKind {
             AssocItemKind::DelegationMac(box DelegationMac { qself, prefix, suffixes, body }) => {
                 visitor.visit_qself(qself);
                 visitor.visit_path(prefix);
-                for (ident, rename) in suffixes {
-                    visitor.visit_ident(ident);
-                    if let Some(rename) = rename {
-                        visitor.visit_ident(rename);
+                if let Some(suffixes) = suffixes {
+                    for (ident, rename) in suffixes {
+                        visitor.visit_ident(ident);
+                        if let Some(rename) = rename {
+                            visitor.visit_ident(rename);
+                        }
                     }
                 }
                 if let Some(body) = body {
@@ -1292,12 +1310,7 @@ pub fn noop_flat_map_item<K: NoopVisitItemKind>(
 impl NoopVisitItemKind for ForeignItemKind {
     fn noop_visit(&mut self, visitor: &mut impl MutVisitor) {
         match self {
-            ForeignItemKind::Static(box StaticForeignItem {
-                ty,
-                mutability: _,
-                expr,
-                safety: _,
-            }) => {
+            ForeignItemKind::Static(box StaticItem { ty, mutability: _, expr, safety: _ }) => {
                 visitor.visit_ty(ty);
                 visit_opt(expr, |expr| visitor.visit_expr(expr));
             }

compiler/rustc_ast/src/token.rs

@@ -558,9 +558,10 @@ impl Token {
     /// Returns `true` if the token can appear at the start of a const param.
     pub fn can_begin_const_arg(&self) -> bool {
         match self.kind {
-            OpenDelim(Delimiter::Brace) => true,
+            OpenDelim(Delimiter::Brace) | Literal(..) | BinOp(Minus) => true,
+            Ident(name, IdentIsRaw::No) if name.is_bool_lit() => true,
             Interpolated(ref nt) => matches!(&**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
-            _ => self.can_begin_literal_maybe_minus(),
+            _ => false,
         }
     }
 
@@ -620,6 +621,21 @@ impl Token {
         }
     }
 
+    pub fn can_begin_string_literal(&self) -> bool {
+        match self.uninterpolate().kind {
+            Literal(..) => true,
+            Interpolated(ref nt) => match &**nt {
+                NtLiteral(_) => true,
+                NtExpr(e) => match &e.kind {
+                    ast::ExprKind::Lit(_) => true,
+                    _ => false,
+                },
+                _ => false,
+            },
+            _ => false,
+        }
+    }
+
     /// A convenience function for matching on identifiers during parsing.
     /// Turns interpolated identifier (`$i: ident`) or lifetime (`$l: lifetime`) token
     /// into the regular identifier or lifetime token it refers to,
@@ -884,7 +900,11 @@ pub enum NonterminalKind {
     PatWithOr,
     Expr,
     /// Matches an expression using the rules from edition 2021 and earlier.
-    Expr2021,
+    Expr2021 {
+        /// Keep track of whether the user used `:expr` or `:expr_2021` and we inferred it from the
+        /// edition of the span. This is used for diagnostics AND feature gating.
+        inferred: bool,
+    },
     Ty,
     Ident,
     Lifetime,
@@ -913,8 +933,13 @@ impl NonterminalKind {
                 Edition::Edition2021 | Edition::Edition2024 => NonterminalKind::PatWithOr,
             },
             sym::pat_param => NonterminalKind::PatParam { inferred: false },
-            sym::expr => NonterminalKind::Expr,
-            sym::expr_2021 if edition().at_least_rust_2021() => NonterminalKind::Expr2021,
+            sym::expr => match edition() {
+                Edition::Edition2015 | Edition::Edition2018 | Edition::Edition2021 => {
+                    NonterminalKind::Expr2021 { inferred: true }
+                }
+                Edition::Edition2024 => NonterminalKind::Expr,
+            },
+            sym::expr_2021 => NonterminalKind::Expr2021 { inferred: false },
             sym::ty => NonterminalKind::Ty,
             sym::ident => NonterminalKind::Ident,
             sym::lifetime => NonterminalKind::Lifetime,
@@ -933,8 +958,8 @@ impl NonterminalKind {
             NonterminalKind::Stmt => sym::stmt,
             NonterminalKind::PatParam { inferred: false } => sym::pat_param,
             NonterminalKind::PatParam { inferred: true } | NonterminalKind::PatWithOr => sym::pat,
-            NonterminalKind::Expr => sym::expr,
-            NonterminalKind::Expr2021 => sym::expr_2021,
+            NonterminalKind::Expr | NonterminalKind::Expr2021 { inferred: true } => sym::expr,
+            NonterminalKind::Expr2021 { inferred: false } => sym::expr_2021,
             NonterminalKind::Ty => sym::ty,
             NonterminalKind::Ident => sym::ident,
             NonterminalKind::Lifetime => sym::lifetime,

compiler/rustc_ast/src/visit.rs

@@ -408,7 +408,14 @@ impl WalkItemKind for ItemKind {
             }
             ItemKind::MacCall(mac) => try_visit!(visitor.visit_mac_call(mac)),
             ItemKind::MacroDef(ts) => try_visit!(visitor.visit_mac_def(ts, item.id)),
-            ItemKind::Delegation(box Delegation { id, qself, path, rename, body }) => {
+            ItemKind::Delegation(box Delegation {
+                id,
+                qself,
+                path,
+                rename,
+                body,
+                from_glob: _,
+            }) => {
                 if let Some(qself) = qself {
                     try_visit!(visitor.visit_ty(&qself.ty));
                 }
@@ -421,10 +428,12 @@ impl WalkItemKind for ItemKind {
                     try_visit!(visitor.visit_ty(&qself.ty));
                 }
                 try_visit!(visitor.visit_path(prefix, item.id));
-                for (ident, rename) in suffixes {
-                    visitor.visit_ident(*ident);
-                    if let Some(rename) = rename {
-                        visitor.visit_ident(*rename);
+                if let Some(suffixes) = suffixes {
+                    for (ident, rename) in suffixes {
+                        visitor.visit_ident(*ident);
+                        if let Some(rename) = rename {
+                            visitor.visit_ident(*rename);
+                        }
                     }
                 }
                 visit_opt!(visitor, visit_block, body);
@@ -663,12 +672,7 @@ impl WalkItemKind for ForeignItemKind {
     ) -> V::Result {
         let &Item { id, span, ident, ref vis, .. } = item;
         match self {
-            ForeignItemKind::Static(box StaticForeignItem {
-                ty,
-                mutability: _,
-                expr,
-                safety: _,
-            }) => {
+            ForeignItemKind::Static(box StaticItem { ty, mutability: _, expr, safety: _ }) => {
                 try_visit!(visitor.visit_ty(ty));
                 visit_opt!(visitor, visit_expr, expr);
             }
@@ -837,7 +841,14 @@ impl WalkItemKind for AssocItemKind {
             AssocItemKind::MacCall(mac) => {
                 try_visit!(visitor.visit_mac_call(mac));
             }
-            AssocItemKind::Delegation(box Delegation { id, qself, path, rename, body }) => {
+            AssocItemKind::Delegation(box Delegation {
+                id,
+                qself,
+                path,
+                rename,
+                body,
+                from_glob: _,
+            }) => {
                 if let Some(qself) = qself {
                     try_visit!(visitor.visit_ty(&qself.ty));
                 }
@@ -850,10 +861,12 @@ impl WalkItemKind for AssocItemKind {
                     try_visit!(visitor.visit_ty(&qself.ty));
                 }
                 try_visit!(visitor.visit_path(prefix, item.id));
-                for (ident, rename) in suffixes {
-                    visitor.visit_ident(*ident);
-                    if let Some(rename) = rename {
-                        visitor.visit_ident(*rename);
+                if let Some(suffixes) = suffixes {
+                    for (ident, rename) in suffixes {
+                        visitor.visit_ident(*ident);
+                        if let Some(rename) = rename {
+                            visitor.visit_ident(*rename);
+                        }
                     }
                 }
                 visit_opt!(visitor, visit_block, body);

compiler/rustc_ast_lowering/src/expr.rs

@@ -1716,24 +1716,28 @@ impl<'hir> LoweringContext<'_, 'hir> {
         // `mut iter => { ... }`
         let iter_arm = self.arm(iter_pat, loop_expr);
 
-        let into_iter_expr = match loop_kind {
+        let match_expr = match loop_kind {
             ForLoopKind::For => {
                 // `::std::iter::IntoIterator::into_iter(<head>)`
-                self.expr_call_lang_item_fn(
+                let into_iter_expr = self.expr_call_lang_item_fn(
                     head_span,
                     hir::LangItem::IntoIterIntoIter,
                     arena_vec![self; head],
-                )
-            }
-            // ` unsafe { Pin::new_unchecked(&mut into_async_iter(<head>)) }`
-            ForLoopKind::ForAwait => {
-                // `::core::async_iter::IntoAsyncIterator::into_async_iter(<head>)`
-                let iter = self.expr_call_lang_item_fn(
-                    head_span,
-                    hir::LangItem::IntoAsyncIterIntoIter,
-                    arena_vec![self; head],
                 );
-                let iter = self.expr_mut_addr_of(head_span, iter);
+
+                self.arena.alloc(self.expr_match(
+                    for_span,
+                    into_iter_expr,
+                    arena_vec![self; iter_arm],
+                    hir::MatchSource::ForLoopDesugar,
+                ))
+            }
+            // `match into_async_iter(<head>) { ref mut iter => match unsafe { Pin::new_unchecked(iter) } { ... } }`
+            ForLoopKind::ForAwait => {
+                let iter_ident = iter;
+                let (async_iter_pat, async_iter_pat_id) =
+                    self.pat_ident_binding_mode(head_span, iter_ident, hir::BindingMode::REF_MUT);
+                let iter = self.expr_ident_mut(head_span, iter_ident, async_iter_pat_id);
                 // `Pin::new_unchecked(...)`
                 let iter = self.arena.alloc(self.expr_call_lang_item_fn_mut(
                     head_span,
@@ -1742,17 +1746,29 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 ));
                 // `unsafe { ... }`
                 let iter = self.arena.alloc(self.expr_unsafe(iter));
-                iter
+                let inner_match_expr = self.arena.alloc(self.expr_match(
+                    for_span,
+                    iter,
+                    arena_vec![self; iter_arm],
+                    hir::MatchSource::ForLoopDesugar,
+                ));
+
+                // `::core::async_iter::IntoAsyncIterator::into_async_iter(<head>)`
+                let iter = self.expr_call_lang_item_fn(
+                    head_span,
+                    hir::LangItem::IntoAsyncIterIntoIter,
+                    arena_vec![self; head],
+                );
+                let iter_arm = self.arm(async_iter_pat, inner_match_expr);
+                self.arena.alloc(self.expr_match(
+                    for_span,
+                    iter,
+                    arena_vec![self; iter_arm],
+                    hir::MatchSource::ForLoopDesugar,
+                ))
             }
         };
 
-        let match_expr = self.arena.alloc(self.expr_match(
-            for_span,
-            into_iter_expr,
-            arena_vec![self; iter_arm],
-            hir::MatchSource::ForLoopDesugar,
-        ));
-
         // This is effectively `{ let _result = ...; _result }`.
         // The construct was introduced in #21984 and is necessary to make sure that
         // temporaries in the `head` expression are dropped and do not leak to the

compiler/rustc_ast_lowering/src/item.rs

@@ -664,12 +664,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
                     hir::ForeignItemKind::Fn(fn_dec, fn_args, generics, safety)
                 }
-                ForeignItemKind::Static(box StaticForeignItem {
-                    ty,
-                    mutability,
-                    expr: _,
-                    safety,
-                }) => {
+                ForeignItemKind::Static(box StaticItem { ty, mutability, expr: _, safety }) => {
                     let ty = self
                         .lower_ty(ty, ImplTraitContext::Disallowed(ImplTraitPosition::StaticTy));
                     let safety = self.lower_safety(*safety, hir::Safety::Unsafe);
@@ -1319,6 +1314,14 @@ impl<'hir> LoweringContext<'_, 'hir> {
             CoroutineKind::AsyncGen { .. } => hir::CoroutineDesugaring::AsyncGen,
         };
         let closure_id = coroutine_kind.closure_id();
+
+        let span = if let FnRetTy::Default(span) = decl.output
+            && matches!(coroutine_source, rustc_hir::CoroutineSource::Closure)
+        {
+            body_span.with_lo(span.lo())
+        } else {
+            body_span
+        };
         let coroutine_expr = self.make_desugared_coroutine_expr(
             // The default capture mode here is by-ref. Later on during upvar analysis,
             // we will force the captured arguments to by-move, but for async closures,
@@ -1327,7 +1330,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             CaptureBy::Ref,
             closure_id,
             None,
-            body_span,
+            span,
             desugaring_kind,
             coroutine_source,
             mkbody,

compiler/rustc_ast_passes/src/ast_validation.rs

@@ -1232,7 +1232,7 @@ impl<'a> Visitor<'a> for AstValidator<'a> {
                 self.check_foreign_ty_genericless(generics, where_clauses);
                 self.check_foreign_item_ascii_only(fi.ident);
             }
-            ForeignItemKind::Static(box StaticForeignItem { expr, safety, .. }) => {
+            ForeignItemKind::Static(box StaticItem { expr, safety, .. }) => {
                 self.check_foreign_item_safety(fi.span, *safety);
                 self.check_foreign_kind_bodyless(fi.ident, "static", expr.as_ref().map(|b| b.span));
                 self.check_foreign_item_ascii_only(fi.ident);

compiler/rustc_ast_passes/src/feature_gate.rs

@@ -562,6 +562,10 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session, features: &Features) {
     gate_all!(precise_capturing, "precise captures on `impl Trait` are experimental");
     gate_all!(global_registration, "global registration is experimental");
     gate_all!(unsafe_attributes, "`#[unsafe()]` markers for attributes are experimental");
+    gate_all!(
+        unsafe_extern_blocks,
+        "`unsafe extern {}` blocks and `safe` keyword are experimental"
+    );
 
     if !visitor.features.never_patterns {
         if let Some(spans) = spans.get(&sym::never_patterns) {

compiler/rustc_ast_pretty/src/pprust/state/item.rs

@@ -9,6 +9,12 @@ use rustc_ast::ptr::P;
 use rustc_ast::ModKind;
 use rustc_span::symbol::Ident;
 
+enum DelegationKind<'a> {
+    Single,
+    List(&'a [(Ident, Option<Ident>)]),
+    Glob,
+}
+
 fn visibility_qualified(vis: &ast::Visibility, s: &str) -> String {
     format!("{}{}", State::to_string(|s| s.print_visibility(vis)), s)
 }
@@ -31,12 +37,7 @@ impl<'a> State<'a> {
             ast::ForeignItemKind::Fn(box ast::Fn { defaultness, sig, generics, body }) => {
                 self.print_fn_full(sig, ident, generics, vis, *defaultness, body.as_deref(), attrs);
             }
-            ast::ForeignItemKind::Static(box ast::StaticForeignItem {
-                ty,
-                mutability,
-                expr,
-                safety,
-            }) => {
+            ast::ForeignItemKind::Static(box ast::StaticItem { ty, mutability, expr, safety }) => {
                 self.print_safety(*safety);
                 self.print_item_const(
                     ident,
@@ -387,7 +388,7 @@ impl<'a> State<'a> {
                 &item.vis,
                 &deleg.qself,
                 &deleg.path,
-                None,
+                DelegationKind::Single,
                 &deleg.body,
             ),
             ast::ItemKind::DelegationMac(deleg) => self.print_delegation(
@@ -395,7 +396,7 @@ impl<'a> State<'a> {
                 &item.vis,
                 &deleg.qself,
                 &deleg.prefix,
-                Some(&deleg.suffixes),
+                deleg.suffixes.as_ref().map_or(DelegationKind::Glob, |s| DelegationKind::List(s)),
                 &deleg.body,
             ),
         }
@@ -579,7 +580,7 @@ impl<'a> State<'a> {
                 vis,
                 &deleg.qself,
                 &deleg.path,
-                None,
+                DelegationKind::Single,
                 &deleg.body,
             ),
             ast::AssocItemKind::DelegationMac(deleg) => self.print_delegation(
@@ -587,20 +588,20 @@ impl<'a> State<'a> {
                 vis,
                 &deleg.qself,
                 &deleg.prefix,
-                Some(&deleg.suffixes),
+                deleg.suffixes.as_ref().map_or(DelegationKind::Glob, |s| DelegationKind::List(s)),
                 &deleg.body,
             ),
         }
         self.ann.post(self, AnnNode::SubItem(id))
     }
 
-    pub(crate) fn print_delegation(
+    fn print_delegation(
         &mut self,
         attrs: &[ast::Attribute],
         vis: &ast::Visibility,
         qself: &Option<P<ast::QSelf>>,
         path: &ast::Path,
-        suffixes: Option<&[(Ident, Option<Ident>)]>,
+        kind: DelegationKind<'_>,
         body: &Option<P<ast::Block>>,
     ) {
         if body.is_some() {
@@ -614,21 +615,28 @@ impl<'a> State<'a> {
         } else {
             self.print_path(path, false, 0);
         }
-        if let Some(suffixes) = suffixes {
-            self.word("::");
-            self.word("{");
-            for (i, (ident, rename)) in suffixes.iter().enumerate() {
-                self.print_ident(*ident);
-                if let Some(rename) = rename {
-                    self.nbsp();
-                    self.word_nbsp("as");
-                    self.print_ident(*rename);
-                }
-                if i != suffixes.len() - 1 {
-                    self.word_space(",");
+        match kind {
+            DelegationKind::Single => {}
+            DelegationKind::List(suffixes) => {
+                self.word("::");
+                self.word("{");
+                for (i, (ident, rename)) in suffixes.iter().enumerate() {
+                    self.print_ident(*ident);
+                    if let Some(rename) = rename {
+                        self.nbsp();
+                        self.word_nbsp("as");
+                        self.print_ident(*rename);
+                    }
+                    if i != suffixes.len() - 1 {
+                        self.word_space(",");
+                    }
                 }
+                self.word("}");
+            }
+            DelegationKind::Glob => {
+                self.word("::");
+                self.word("*");
             }
-            self.word("}");
         }
         if let Some(body) = body {
             self.nbsp();

compiler/rustc_builtin_macros/src/test.rs

@@ -1,6 +1,7 @@
+//! The expansion from a test function to the appropriate test struct for libtest
+//! Ideally, this code would be in libtest but for efficiency and error messages it lives here.
+
 use crate::errors;
-/// The expansion from a test function to the appropriate test struct for libtest
-/// Ideally, this code would be in libtest but for efficiency and error messages it lives here.
 use crate::util::{check_builtin_macro_attribute, warn_on_duplicate_attribute};
 use rustc_ast::ptr::P;
 use rustc_ast::{self as ast, attr, GenericParamKind};

compiler/rustc_codegen_cranelift/src/base.rs

@@ -677,21 +677,22 @@ fn codegen_stmt<'tcx>(
                     CastKind::PointerCoercion(PointerCoercion::UnsafeFnPointer),
                     ref operand,
                     to_ty,
-                )
-                | Rvalue::Cast(
-                    CastKind::PointerCoercion(PointerCoercion::MutToConstPointer),
-                    ref operand,
-                    to_ty,
-                )
-                | Rvalue::Cast(
-                    CastKind::PointerCoercion(PointerCoercion::ArrayToPointer),
-                    ref operand,
-                    to_ty,
                 ) => {
                     let to_layout = fx.layout_of(fx.monomorphize(to_ty));
                     let operand = codegen_operand(fx, operand);
                     lval.write_cvalue(fx, operand.cast_pointer_to(to_layout));
                 }
+                Rvalue::Cast(
+                    CastKind::PointerCoercion(
+                        PointerCoercion::MutToConstPointer | PointerCoercion::ArrayToPointer,
+                    ),
+                    ..,
+                ) => {
+                    bug!(
+                        "{:?} is for borrowck, and should never appear in codegen",
+                        to_place_and_rval.1
+                    );
+                }
                 Rvalue::Cast(
                     CastKind::IntToInt
                     | CastKind::FloatToFloat

compiler/rustc_codegen_gcc/tests/lang_tests_common.rs

@@ -1,4 +1,5 @@
 //! The common code for `tests/lang_tests_*.rs`
+
 use std::{
     env::{self, current_dir},
     path::{Path, PathBuf},

compiler/rustc_codegen_ssa/src/back/symbol_export.rs

@@ -371,7 +371,7 @@ fn exported_symbols_provider_local(
                 }) => {
                     // A little sanity-check
                     debug_assert_eq!(
-                        args.non_erasable_generics(tcx, def_id).skip(1).next(),
+                        args.non_erasable_generics(tcx, def_id).next(),
                         Some(GenericArgKind::Type(ty))
                     );
                     symbols.push((
@@ -422,10 +422,7 @@ fn upstream_monomorphizations_provider(
                 }
                 ExportedSymbol::AsyncDropGlueCtorShim(ty) => {
                     if let Some(async_drop_in_place_fn_def_id) = async_drop_in_place_fn_def_id {
-                        (
-                            async_drop_in_place_fn_def_id,
-                            tcx.mk_args(&[tcx.lifetimes.re_erased.into(), ty.into()]),
-                        )
+                        (async_drop_in_place_fn_def_id, tcx.mk_args(&[ty.into()]))
                     } else {
                         // `drop_in_place` in place does not exist, don't try
                         // to use it.
@@ -473,11 +470,16 @@ fn upstream_drop_glue_for_provider<'tcx>(
     tcx: TyCtxt<'tcx>,
     args: GenericArgsRef<'tcx>,
 ) -> Option<CrateNum> {
-    if let Some(def_id) = tcx.lang_items().drop_in_place_fn() {
-        tcx.upstream_monomorphizations_for(def_id).and_then(|monos| monos.get(&args).cloned())
-    } else {
-        None
-    }
+    let def_id = tcx.lang_items().drop_in_place_fn()?;
+    tcx.upstream_monomorphizations_for(def_id)?.get(&args).cloned()
+}
+
+fn upstream_async_drop_glue_for_provider<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    args: GenericArgsRef<'tcx>,
+) -> Option<CrateNum> {
+    let def_id = tcx.lang_items().async_drop_in_place_fn()?;
+    tcx.upstream_monomorphizations_for(def_id)?.get(&args).cloned()
 }
 
 fn is_unreachable_local_definition_provider(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
@@ -491,6 +493,7 @@ pub fn provide(providers: &mut Providers) {
     providers.upstream_monomorphizations = upstream_monomorphizations_provider;
     providers.is_unreachable_local_definition = is_unreachable_local_definition_provider;
     providers.upstream_drop_glue_for = upstream_drop_glue_for_provider;
+    providers.upstream_async_drop_glue_for = upstream_async_drop_glue_for_provider;
     providers.wasm_import_module_map = wasm_import_module_map;
     providers.extern_queries.is_reachable_non_generic = is_reachable_non_generic_provider_extern;
     providers.extern_queries.upstream_monomorphizations_for =

compiler/rustc_codegen_ssa/src/mir/locals.rs

@@ -1,6 +1,7 @@
 //! Locals are in a private module as updating `LocalRef::Operand` has to
 //! be careful wrt to subtyping. To deal with this we only allow updates by using
 //! `FunctionCx::overwrite_local` which handles it automatically.
+
 use crate::mir::{FunctionCx, LocalRef};
 use crate::traits::BuilderMethods;
 use rustc_index::IndexVec;

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -456,8 +456,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             base::unsize_ptr(bx, lldata, operand.layout.ty, cast.ty, llextra);
                         OperandValue::Pair(lldata, llextra)
                     }
-                    mir::CastKind::PointerCoercion(PointerCoercion::MutToConstPointer)
-                    | mir::CastKind::PtrToPtr
+                    mir::CastKind::PointerCoercion(
+                        PointerCoercion::MutToConstPointer | PointerCoercion::ArrayToPointer,
+                    ) => {
+                        bug!("{kind:?} is for borrowck, and should never appear in codegen");
+                    }
+                    mir::CastKind::PtrToPtr
                         if bx.cx().is_backend_scalar_pair(operand.layout) =>
                     {
                         if let OperandValue::Pair(data_ptr, meta) = operand.val {
@@ -477,9 +481,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             base::cast_to_dyn_star(bx, lldata, operand.layout, cast.ty, llextra);
                         OperandValue::Pair(lldata, llextra)
                     }
-                    mir::CastKind::PointerCoercion(
-                        PointerCoercion::MutToConstPointer | PointerCoercion::ArrayToPointer,
-                    )
                     | mir::CastKind::IntToInt
                     | mir::CastKind::FloatToInt
                     | mir::CastKind::FloatToFloat
@@ -638,7 +639,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         (OperandValue::Immediate(llval), operand.layout)
                     }
                     mir::UnOp::PtrMetadata => {
-                        debug_assert!(operand.layout.ty.is_unsafe_ptr());
+                        debug_assert!(
+                            operand.layout.ty.is_unsafe_ptr() || operand.layout.ty.is_ref(),
+                        );
                         let (_, meta) = operand.val.pointer_parts();
                         assert_eq!(operand.layout.fields.count() > 1, meta.is_some());
                         if let Some(meta) = meta {

compiler/rustc_const_eval/messages.ftl

@@ -73,8 +73,6 @@ const_eval_division_by_zero =
     dividing by zero
 const_eval_division_overflow =
     overflow in signed division (dividing MIN by -1)
-const_eval_double_storage_live =
-    StorageLive on a local that was already live
 
 const_eval_dyn_call_not_a_method =
     `dyn` call trying to call something that is not a method

compiler/rustc_const_eval/src/interpret/cast.rs

@@ -70,9 +70,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             CastKind::PointerCoercion(
                 PointerCoercion::MutToConstPointer | PointerCoercion::ArrayToPointer,
             ) => {
-                // These are NOPs, but can be wide pointers.
-                let v = self.read_immediate(src)?;
-                self.write_immediate(*v, dest)?;
+                bug!("{cast_kind:?} casts are for borrowck only, not runtime MIR");
             }
 
             CastKind::PointerCoercion(PointerCoercion::ReifyFnPointer) => {

compiler/rustc_const_eval/src/interpret/discriminant.rs

@@ -241,7 +241,16 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         variant_index: VariantIdx,
     ) -> InterpResult<'tcx, Option<(ScalarInt, usize)>> {
         match self.layout_of(ty)?.variants {
-            abi::Variants::Single { .. } => Ok(None),
+            abi::Variants::Single { .. } => {
+                // The tag of a `Single` enum is like the tag of the niched
+                // variant: there's no tag as the discriminant is encoded
+                // entirely implicitly. If `write_discriminant` ever hits this
+                // case, we do a "validation read" to ensure the the right
+                // discriminant is encoded implicitly, so any attempt to write
+                // the wrong discriminant for a `Single` enum will reliably
+                // result in UB.
+                Ok(None)
+            }
 
             abi::Variants::Multiple {
                 tag_encoding: TagEncoding::Direct,

compiler/rustc_const_eval/src/interpret/eval_context.rs

@@ -1100,11 +1100,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             Operand::Immediate(Immediate::Uninit)
         });
 
-        // StorageLive expects the local to be dead, and marks it live.
+        // If the local is already live, deallocate its old memory.
         let old = mem::replace(&mut self.frame_mut().locals[local].value, local_val);
-        if !matches!(old, LocalValue::Dead) {
-            throw_ub_custom!(fluent::const_eval_double_storage_live);
-        }
+        self.deallocate_local(old)?;
         Ok(())
     }
 
@@ -1118,7 +1116,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         assert!(local != mir::RETURN_PLACE, "Cannot make return place dead");
         trace!("{:?} is now dead", local);
 
-        // It is entirely okay for this local to be already dead (at least that's how we currently generate MIR)
+        // If the local is already dead, this is a NOP.
         let old = mem::replace(&mut self.frame_mut().locals[local].value, LocalValue::Dead);
         self.deallocate_local(old)?;
         Ok(())

compiler/rustc_const_eval/src/interpret/operator.rs

@@ -460,7 +460,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 let res = ScalarInt::truncate_from_uint(res, layout.size).0;
                 Ok(ImmTy::from_scalar(res.into(), layout))
             }
-            ty::RawPtr(..) => {
+            ty::RawPtr(..) | ty::Ref(..) => {
                 assert_eq!(un_op, PtrMetadata);
                 let (_, meta) = val.to_scalar_and_meta();
                 Ok(match meta {

compiler/rustc_data_structures/src/base_n.rs

@@ -1,5 +1,6 @@
-/// Converts unsigned integers into a string representation with some base.
-/// Bases up to and including 36 can be used for case-insensitive things.
+//! Converts unsigned integers into a string representation with some base.
+//! Bases up to and including 36 can be used for case-insensitive things.
+
 use std::ascii;
 use std::fmt;
 

compiler/rustc_errors/src/lib.rs

@@ -1437,11 +1437,29 @@ impl DiagCtxtInner {
 
     // Return value is only `Some` if the level is `Error` or `DelayedBug`.
     fn emit_diagnostic(&mut self, mut diagnostic: DiagInner) -> Option<ErrorGuaranteed> {
+        match diagnostic.level {
+            Expect(expect_id) | ForceWarning(Some(expect_id)) => {
+                // The `LintExpectationId` can be stable or unstable depending on when it was
+                // created. Diagnostics created before the definition of `HirId`s are unstable and
+                // can not yet be stored. Instead, they are buffered until the `LintExpectationId`
+                // is replaced by a stable one by the `LintLevelsBuilder`.
+                if let LintExpectationId::Unstable { .. } = expect_id {
+                    // We don't call TRACK_DIAGNOSTIC because we wait for the
+                    // unstable ID to be updated, whereupon the diagnostic will be
+                    // passed into this method again.
+                    self.unstable_expect_diagnostics.push(diagnostic);
+                    return None;
+                }
+                // Continue through to the `Expect`/`ForceWarning` case below.
+            }
+            _ => {}
+        }
+
         if diagnostic.has_future_breakage() {
-            // Future breakages aren't emitted if they're `Level::Allow`,
-            // but they still need to be constructed and stashed below,
-            // so they'll trigger the must_produce_diag check.
-            assert!(matches!(diagnostic.level, Error | Warning | Allow));
+            // Future breakages aren't emitted if they're `Level::Allow` or
+            // `Level::Expect`, but they still need to be constructed and
+            // stashed below, so they'll trigger the must_produce_diag check.
+            assert!(matches!(diagnostic.level, Error | Warning | Allow | Expect(_)));
             self.future_breakage_diagnostics.push(diagnostic.clone());
         }
 
@@ -1512,16 +1530,8 @@ impl DiagCtxtInner {
                 return None;
             }
             Expect(expect_id) | ForceWarning(Some(expect_id)) => {
-                // Diagnostics created before the definition of `HirId`s are
-                // unstable and can not yet be stored. Instead, they are
-                // buffered until the `LintExpectationId` is replaced by a
-                // stable one by the `LintLevelsBuilder`.
                 if let LintExpectationId::Unstable { .. } = expect_id {
-                    // We don't call TRACK_DIAGNOSTIC because we wait for the
-                    // unstable ID to be updated, whereupon the diagnostic will
-                    // be passed into this method again.
-                    self.unstable_expect_diagnostics.push(diagnostic);
-                    return None;
+                    unreachable!(); // this case was handled at the top of this function
                 }
                 self.fulfilled_expectations.insert(expect_id.normalize());
                 if let Expect(_) = diagnostic.level {

compiler/rustc_errors/src/markdown/mod.rs

@@ -1,6 +1,7 @@
 //! A simple markdown parser that can write formatted text to the terminal
 //!
 //! Entrypoint is `MdStream::parse_str(...)`
+
 use std::io;
 
 use termcolor::{Buffer, BufferWriter, ColorChoice};

compiler/rustc_expand/messages.ftl

@@ -35,8 +35,8 @@ expand_duplicate_matcher_binding = duplicate matcher binding
     .label = duplicate binding
     .label2 = previous binding
 
-expand_empty_delegation_list =
-    empty list delegation is not supported
+expand_empty_delegation_mac =
+    empty {$kind} delegation is not supported
 
 expand_expected_paren_or_brace =
     expected `(` or `{"{"}`, found `{$token}`
@@ -58,6 +58,9 @@ expand_feature_removed =
     .label = feature has been removed
     .reason = {$reason}
 
+expand_glob_delegation_outside_impls =
+    glob delegation is only supported in impls
+
 expand_helper_attribute_name_invalid =
     `{$name}` cannot be a name of derive helper attribute
 

compiler/rustc_expand/src/base.rs

@@ -357,6 +357,10 @@ where
     }
 }
 
+pub trait GlobDelegationExpander {
+    fn expand(&self, ecx: &mut ExtCtxt<'_>) -> ExpandResult<Vec<(Ident, Option<Ident>)>, ()>;
+}
+
 // Use a macro because forwarding to a simple function has type system issues
 macro_rules! make_stmts_default {
     ($me:expr) => {
@@ -714,6 +718,9 @@ pub enum SyntaxExtensionKind {
         /// The produced AST fragment is appended to the input AST fragment.
         Box<dyn MultiItemModifier + sync::DynSync + sync::DynSend>,
     ),
+
+    /// A glob delegation.
+    GlobDelegation(Box<dyn GlobDelegationExpander + sync::DynSync + sync::DynSend>),
 }
 
 /// A struct representing a macro definition in "lowered" form ready for expansion.
@@ -748,7 +755,9 @@ impl SyntaxExtension {
     /// Returns which kind of macro calls this syntax extension.
     pub fn macro_kind(&self) -> MacroKind {
         match self.kind {
-            SyntaxExtensionKind::Bang(..) | SyntaxExtensionKind::LegacyBang(..) => MacroKind::Bang,
+            SyntaxExtensionKind::Bang(..)
+            | SyntaxExtensionKind::LegacyBang(..)
+            | SyntaxExtensionKind::GlobDelegation(..) => MacroKind::Bang,
             SyntaxExtensionKind::Attr(..)
             | SyntaxExtensionKind::LegacyAttr(..)
             | SyntaxExtensionKind::NonMacroAttr => MacroKind::Attr,
@@ -922,6 +931,32 @@ impl SyntaxExtension {
         SyntaxExtension::default(SyntaxExtensionKind::NonMacroAttr, edition)
     }
 
+    pub fn glob_delegation(
+        trait_def_id: DefId,
+        impl_def_id: LocalDefId,
+        edition: Edition,
+    ) -> SyntaxExtension {
+        struct GlobDelegationExpanderImpl {
+            trait_def_id: DefId,
+            impl_def_id: LocalDefId,
+        }
+        impl GlobDelegationExpander for GlobDelegationExpanderImpl {
+            fn expand(
+                &self,
+                ecx: &mut ExtCtxt<'_>,
+            ) -> ExpandResult<Vec<(Ident, Option<Ident>)>, ()> {
+                match ecx.resolver.glob_delegation_suffixes(self.trait_def_id, self.impl_def_id) {
+                    Ok(suffixes) => ExpandResult::Ready(suffixes),
+                    Err(Indeterminate) if ecx.force_mode => ExpandResult::Ready(Vec::new()),
+                    Err(Indeterminate) => ExpandResult::Retry(()),
+                }
+            }
+        }
+
+        let expander = GlobDelegationExpanderImpl { trait_def_id, impl_def_id };
+        SyntaxExtension::default(SyntaxExtensionKind::GlobDelegation(Box::new(expander)), edition)
+    }
+
     pub fn expn_data(
         &self,
         parent: LocalExpnId,
@@ -1030,6 +1065,16 @@ pub trait ResolverExpand {
 
     /// Tools registered with `#![register_tool]` and used by tool attributes and lints.
     fn registered_tools(&self) -> &RegisteredTools;
+
+    /// Mark this invocation id as a glob delegation.
+    fn register_glob_delegation(&mut self, invoc_id: LocalExpnId);
+
+    /// Names of specific methods to which glob delegation expands.
+    fn glob_delegation_suffixes(
+        &mut self,
+        trait_def_id: DefId,
+        impl_def_id: LocalDefId,
+    ) -> Result<Vec<(Ident, Option<Ident>)>, Indeterminate>;
 }
 
 pub trait LintStoreExpand {

compiler/rustc_expand/src/errors.rs

@@ -435,8 +435,16 @@ pub struct ExpectedParenOrBrace<'a> {
 }
 
 #[derive(Diagnostic)]
-#[diag(expand_empty_delegation_list)]
-pub(crate) struct EmptyDelegationList {
+#[diag(expand_empty_delegation_mac)]
+pub(crate) struct EmptyDelegationMac {
+    #[primary_span]
+    pub span: Span,
+    pub kind: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(expand_glob_delegation_outside_impls)]
+pub(crate) struct GlobDelegationOutsideImpls {
     #[primary_span]
     pub span: Span,
 }

compiler/rustc_expand/src/expand.rs

@@ -1,8 +1,8 @@
 use crate::base::*;
 use crate::config::StripUnconfigured;
 use crate::errors::{
-    EmptyDelegationList, IncompleteParse, RecursionLimitReached, RemoveExprNotSupported,
-    RemoveNodeNotSupported, UnsupportedKeyValue, WrongFragmentKind,
+    EmptyDelegationMac, GlobDelegationOutsideImpls, IncompleteParse, RecursionLimitReached,
+    RemoveExprNotSupported, RemoveNodeNotSupported, UnsupportedKeyValue, WrongFragmentKind,
 };
 use crate::mbe::diagnostics::annotate_err_with_kind;
 use crate::module::{mod_dir_path, parse_external_mod, DirOwnership, ParsedExternalMod};
@@ -343,6 +343,9 @@ pub enum InvocationKind {
         is_const: bool,
         item: Annotatable,
     },
+    GlobDelegation {
+        item: P<ast::AssocItem>,
+    },
 }
 
 impl InvocationKind {
@@ -370,6 +373,7 @@ impl Invocation {
             InvocationKind::Bang { span, .. } => *span,
             InvocationKind::Attr { attr, .. } => attr.span,
             InvocationKind::Derive { path, .. } => path.span,
+            InvocationKind::GlobDelegation { item } => item.span,
         }
     }
 
@@ -378,6 +382,7 @@ impl Invocation {
             InvocationKind::Bang { span, .. } => span,
             InvocationKind::Attr { attr, .. } => &mut attr.span,
             InvocationKind::Derive { path, .. } => &mut path.span,
+            InvocationKind::GlobDelegation { item } => &mut item.span,
         }
     }
 }
@@ -800,6 +805,36 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                 }
                 _ => unreachable!(),
             },
+            InvocationKind::GlobDelegation { item } => {
+                let AssocItemKind::DelegationMac(deleg) = &item.kind else { unreachable!() };
+                let suffixes = match ext {
+                    SyntaxExtensionKind::GlobDelegation(expander) => match expander.expand(self.cx)
+                    {
+                        ExpandResult::Ready(suffixes) => suffixes,
+                        ExpandResult::Retry(()) => {
+                            // Reassemble the original invocation for retrying.
+                            return ExpandResult::Retry(Invocation {
+                                kind: InvocationKind::GlobDelegation { item },
+                                ..invoc
+                            });
+                        }
+                    },
+                    SyntaxExtensionKind::LegacyBang(..) => {
+                        let msg = "expanded a dummy glob delegation";
+                        let guar = self.cx.dcx().span_delayed_bug(span, msg);
+                        return ExpandResult::Ready(fragment_kind.dummy(span, guar));
+                    }
+                    _ => unreachable!(),
+                };
+
+                type Node = AstNodeWrapper<P<ast::AssocItem>, ImplItemTag>;
+                let single_delegations = build_single_delegations::<Node>(
+                    self.cx, deleg, &item, &suffixes, item.span, true,
+                );
+                fragment_kind.expect_from_annotatables(
+                    single_delegations.map(|item| Annotatable::ImplItem(P(item))),
+                )
+            }
         })
     }
 
@@ -1067,7 +1102,7 @@ trait InvocationCollectorNode: HasAttrs + HasNodeId + Sized {
     fn take_mac_call(self) -> (P<ast::MacCall>, Self::AttrsTy, AddSemicolon) {
         unreachable!()
     }
-    fn delegation_list(&self) -> Option<(&ast::DelegationMac, &ast::Item<Self::ItemKind>)> {
+    fn delegation(&self) -> Option<(&ast::DelegationMac, &ast::Item<Self::ItemKind>)> {
         None
     }
     fn delegation_item_kind(_deleg: Box<ast::Delegation>) -> Self::ItemKind {
@@ -1126,7 +1161,7 @@ impl InvocationCollectorNode for P<ast::Item> {
             _ => unreachable!(),
         }
     }
-    fn delegation_list(&self) -> Option<(&ast::DelegationMac, &ast::Item<Self::ItemKind>)> {
+    fn delegation(&self) -> Option<(&ast::DelegationMac, &ast::Item<Self::ItemKind>)> {
         match &self.kind {
             ItemKind::DelegationMac(deleg) => Some((deleg, self)),
             _ => None,
@@ -1270,7 +1305,7 @@ impl InvocationCollectorNode for AstNodeWrapper<P<ast::AssocItem>, TraitItemTag>
             _ => unreachable!(),
         }
     }
-    fn delegation_list(&self) -> Option<(&ast::DelegationMac, &ast::Item<Self::ItemKind>)> {
+    fn delegation(&self) -> Option<(&ast::DelegationMac, &ast::Item<Self::ItemKind>)> {
         match &self.wrapped.kind {
             AssocItemKind::DelegationMac(deleg) => Some((deleg, &self.wrapped)),
             _ => None,
@@ -1311,7 +1346,7 @@ impl InvocationCollectorNode for AstNodeWrapper<P<ast::AssocItem>, ImplItemTag>
             _ => unreachable!(),
         }
     }
-    fn delegation_list(&self) -> Option<(&ast::DelegationMac, &ast::Item<Self::ItemKind>)> {
+    fn delegation(&self) -> Option<(&ast::DelegationMac, &ast::Item<Self::ItemKind>)> {
         match &self.wrapped.kind {
             AssocItemKind::DelegationMac(deleg) => Some((deleg, &self.wrapped)),
             _ => None,
@@ -1487,7 +1522,7 @@ impl InvocationCollectorNode for ast::Stmt {
         };
         (mac, attrs, if add_semicolon { AddSemicolon::Yes } else { AddSemicolon::No })
     }
-    fn delegation_list(&self) -> Option<(&ast::DelegationMac, &ast::Item<Self::ItemKind>)> {
+    fn delegation(&self) -> Option<(&ast::DelegationMac, &ast::Item<Self::ItemKind>)> {
         match &self.kind {
             StmtKind::Item(item) => match &item.kind {
                 ItemKind::DelegationMac(deleg) => Some((deleg, item)),
@@ -1684,6 +1719,44 @@ impl InvocationCollectorNode for AstNodeWrapper<P<ast::Expr>, MethodReceiverTag>
     }
 }
 
+fn build_single_delegations<'a, Node: InvocationCollectorNode>(
+    ecx: &ExtCtxt<'_>,
+    deleg: &'a ast::DelegationMac,
+    item: &'a ast::Item<Node::ItemKind>,
+    suffixes: &'a [(Ident, Option<Ident>)],
+    item_span: Span,
+    from_glob: bool,
+) -> impl Iterator<Item = ast::Item<Node::ItemKind>> + 'a {
+    if suffixes.is_empty() {
+        // Report an error for now, to avoid keeping stem for resolution and
+        // stability checks.
+        let kind = String::from(if from_glob { "glob" } else { "list" });
+        ecx.dcx().emit_err(EmptyDelegationMac { span: item.span, kind });
+    }
+
+    suffixes.iter().map(move |&(ident, rename)| {
+        let mut path = deleg.prefix.clone();
+        path.segments.push(ast::PathSegment { ident, id: ast::DUMMY_NODE_ID, args: None });
+
+        ast::Item {
+            attrs: item.attrs.clone(),
+            id: ast::DUMMY_NODE_ID,
+            span: if from_glob { item_span } else { ident.span },
+            vis: item.vis.clone(),
+            ident: rename.unwrap_or(ident),
+            kind: Node::delegation_item_kind(Box::new(ast::Delegation {
+                id: ast::DUMMY_NODE_ID,
+                qself: deleg.qself.clone(),
+                path,
+                rename,
+                body: deleg.body.clone(),
+                from_glob,
+            })),
+            tokens: None,
+        }
+    })
+}
+
 struct InvocationCollector<'a, 'b> {
     cx: &'a mut ExtCtxt<'b>,
     invocations: Vec<(Invocation, Option<Lrc<SyntaxExtension>>)>,
@@ -1702,6 +1775,11 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
 
     fn collect(&mut self, fragment_kind: AstFragmentKind, kind: InvocationKind) -> AstFragment {
         let expn_id = LocalExpnId::fresh_empty();
+        if matches!(kind, InvocationKind::GlobDelegation { .. }) {
+            // In resolver we need to know which invocation ids are delegations early,
+            // before their `ExpnData` is filled.
+            self.cx.resolver.register_glob_delegation(expn_id);
+        }
         let vis = kind.placeholder_visibility();
         self.invocations.push((
             Invocation {
@@ -1734,6 +1812,14 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
         self.collect(kind, InvocationKind::Attr { attr, pos, item, derives })
     }
 
+    fn collect_glob_delegation(
+        &mut self,
+        item: P<ast::AssocItem>,
+        kind: AstFragmentKind,
+    ) -> AstFragment {
+        self.collect(kind, InvocationKind::GlobDelegation { item })
+    }
+
     /// If `item` is an attribute invocation, remove the attribute and return it together with
     /// its position and derives following it. We have to collect the derives in order to resolve
     /// legacy derive helpers (helpers written before derives that introduce them).
@@ -1901,37 +1987,27 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
                     Node::post_flat_map_node_collect_bang(&mut res, add_semicolon);
                     res
                 }
-                None if let Some((deleg, item)) = node.delegation_list() => {
-                    if deleg.suffixes.is_empty() {
-                        // Report an error for now, to avoid keeping stem for resolution and
-                        // stability checks.
-                        self.cx.dcx().emit_err(EmptyDelegationList { span: item.span });
-                    }
-
-                    Node::flatten_outputs(deleg.suffixes.iter().map(|&(ident, rename)| {
-                        let mut path = deleg.prefix.clone();
-                        path.segments.push(ast::PathSegment {
-                            ident,
-                            id: ast::DUMMY_NODE_ID,
-                            args: None,
-                        });
-
-                        let mut item = Node::from_item(ast::Item {
-                            attrs: item.attrs.clone(),
-                            id: ast::DUMMY_NODE_ID,
-                            span: ident.span,
-                            vis: item.vis.clone(),
-                            ident: rename.unwrap_or(ident),
-                            kind: Node::delegation_item_kind(Box::new(ast::Delegation {
-                                id: ast::DUMMY_NODE_ID,
-                                qself: deleg.qself.clone(),
-                                path,
-                                rename,
-                                body: deleg.body.clone(),
-                            })),
-                            tokens: None,
-                        });
+                None if let Some((deleg, item)) = node.delegation() => {
+                    let Some(suffixes) = &deleg.suffixes else {
+                        let item = match node.to_annotatable() {
+                            Annotatable::ImplItem(item) => item,
+                            ann @ (Annotatable::Item(_)
+                            | Annotatable::TraitItem(_)
+                            | Annotatable::Stmt(_)) => {
+                                let span = ann.span();
+                                self.cx.dcx().emit_err(GlobDelegationOutsideImpls { span });
+                                return Default::default();
+                            }
+                            _ => unreachable!(),
+                        };
+                        return self.collect_glob_delegation(item, Node::KIND).make_ast::<Node>();
+                    };
 
+                    let single_delegations = build_single_delegations::<Node>(
+                        self.cx, deleg, item, suffixes, item.span, false,
+                    );
+                    Node::flatten_outputs(single_delegations.map(|item| {
+                        let mut item = Node::from_item(item);
                         assign_id!(self, item.node_id_mut(), || item.noop_flat_map(self))
                     }))
                 }
@@ -1983,7 +2059,7 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
                         self.collect_bang(mac, Node::KIND).make_ast::<Node>()
                     })
                 }
-                None if node.delegation_list().is_some() => unreachable!(),
+                None if node.delegation().is_some() => unreachable!(),
                 None => {
                     assign_id!(self, node.node_id_mut(), || node.noop_visit(self))
                 }

compiler/rustc_expand/src/mbe/macro_check.rs

@@ -104,6 +104,7 @@
 //! Kleene operators under which a meta-variable is repeating is the concatenation of the stacks
 //! stored when entering a macro definition starting from the state in which the meta-variable is
 //! bound.
+
 use crate::errors;
 use crate::mbe::{KleeneToken, TokenTree};
 

compiler/rustc_expand/src/mbe/macro_rules.rs

@@ -1292,7 +1292,9 @@ fn is_in_follow(tok: &mbe::TokenTree, kind: NonterminalKind) -> IsInFollow {
                 // maintain
                 IsInFollow::Yes
             }
-            NonterminalKind::Stmt | NonterminalKind::Expr | NonterminalKind::Expr2021 => {
+            NonterminalKind::Stmt
+            | NonterminalKind::Expr
+            | NonterminalKind::Expr2021 { inferred: _ } => {
                 const TOKENS: &[&str] = &["`=>`", "`,`", "`;`"];
                 match tok {
                     TokenTree::Token(token) => match token.kind {

compiler/rustc_expand/src/mbe/quoted.rs

@@ -113,7 +113,8 @@ pub(super) fn parse(
                                                 );
                                                 token::NonterminalKind::Ident
                                             });
-                                    if kind == token::NonterminalKind::Expr2021
+                                    if kind
+                                        == (token::NonterminalKind::Expr2021 { inferred: false })
                                         && !features.expr_fragment_specifier_2024
                                     {
                                         rustc_session::parse::feature_err(

compiler/rustc_feature/src/accepted.rs

@@ -80,6 +80,8 @@ declare_features! (
     (accepted, braced_empty_structs, "1.8.0", Some(29720)),
     /// Allows `c"foo"` literals.
     (accepted, c_str_literals, "1.77.0", Some(105723)),
+    /// Allows `extern "C-unwind" fn` to enable unwinding across ABI boundaries and treat `extern "C" fn` as nounwind.
+    (accepted, c_unwind, "CURRENT_RUSTC_VERSION", Some(74990)),
     /// Allows `#[cfg_attr(predicate, multiple, attributes, here)]`.
     (accepted, cfg_attr_multi, "1.33.0", Some(54881)),
     /// Allows the use of `#[cfg(doctest)]`, set when rustdoc is collecting doctests.

compiler/rustc_feature/src/builtin_attrs.rs

@@ -1088,14 +1088,6 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         ErrorFollowing, EncodeCrossCrate::No,
         "the `#[custom_mir]` attribute is just used for the Rust test suite",
     ),
-    rustc_attr!(
-        TEST, rustc_dump_program_clauses, Normal, template!(Word),
-        WarnFollowing, EncodeCrossCrate::No
-    ),
-    rustc_attr!(
-        TEST, rustc_dump_env_program_clauses, Normal, template!(Word),
-        WarnFollowing, EncodeCrossCrate::No
-    ),
     rustc_attr!(
         TEST, rustc_object_lifetime_default, Normal, template!(Word),
         WarnFollowing, EncodeCrossCrate::No

compiler/rustc_feature/src/unstable.rs

@@ -363,8 +363,6 @@ declare_features! (
     (unstable, async_for_loop, "1.77.0", Some(118898)),
     /// Allows builtin # foo() syntax
     (unstable, builtin_syntax, "1.71.0", Some(110680)),
-    /// Treat `extern "C"` function as nounwind.
-    (unstable, c_unwind, "1.52.0", Some(74990)),
     /// Allows using C-variadics.
     (unstable, c_variadic, "1.34.0", Some(44930)),
     /// Allows the use of `#[cfg(overflow_checks)` to check if integer overflow behaviour.
@@ -588,6 +586,8 @@ declare_features! (
     (incomplete, return_type_notation, "1.70.0", Some(109417)),
     /// Allows `extern "rust-cold"`.
     (unstable, rust_cold_cc, "1.63.0", Some(97544)),
+    /// Shortern the tail expression lifetime
+    (unstable, shorter_tail_lifetimes, "1.79.0", Some(123739)),
     /// Allows the use of SIMD types in functions declared in `extern` blocks.
     (unstable, simd_ffi, "1.0.0", Some(27731)),
     /// Allows specialization of implementations (RFC 1210).

compiler/rustc_fs_util/src/lib.rs

@@ -1,9 +1,7 @@
-// tidy-alphabetical-start
 use std::ffi::CString;
 use std::fs;
 use std::io;
 use std::path::{absolute, Path, PathBuf};
-// tidy-alphabetical-end
 
 // Unfortunately, on windows, it looks like msvcrt.dll is silently translating
 // verbatim paths under the hood to non-verbatim paths! This manifests itself as

compiler/rustc_hir/src/definitions.rs

@@ -377,22 +377,17 @@ impl Definitions {
     }
 
     #[inline(always)]
-    pub fn local_def_path_hash_to_def_id(
-        &self,
-        hash: DefPathHash,
-        err_msg: &dyn std::fmt::Debug,
-    ) -> LocalDefId {
+    /// Returns `None` if the `DefPathHash` does not correspond to a `LocalDefId`
+    /// in the current compilation session. This can legitimately happen if the
+    /// `DefPathHash` is from a `DefId` in an upstream crate or, during incr. comp.,
+    /// if the `DefPathHash` is from a previous compilation session and
+    /// the def-path does not exist anymore.
+    pub fn local_def_path_hash_to_def_id(&self, hash: DefPathHash) -> Option<LocalDefId> {
         debug_assert!(hash.stable_crate_id() == self.table.stable_crate_id);
-        #[cold]
-        #[inline(never)]
-        fn err(err_msg: &dyn std::fmt::Debug) -> ! {
-            panic!("{err_msg:?}")
-        }
         self.table
             .def_path_hash_to_index
             .get(&hash.local_hash())
             .map(|local_def_index| LocalDefId { local_def_index })
-            .unwrap_or_else(|| err(err_msg))
     }
 
     pub fn def_path_hash_to_def_index_map(&self) -> &DefPathHashMap {

compiler/rustc_hir_analysis/src/check/dropck.rs

@@ -1,6 +1,7 @@
 // FIXME(@lcnr): Move this module out of `rustc_hir_analysis`.
 //
 // We don't do any drop checking during hir typeck.
+
 use rustc_data_structures::fx::FxHashSet;
 use rustc_errors::{codes::*, struct_span_code_err, ErrorGuaranteed};
 use rustc_infer::infer::outlives::env::OutlivesEnvironment;

compiler/rustc_hir_analysis/src/check/region.rs

@@ -6,7 +6,6 @@
 //!
 //! [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/borrow_check.html
 
-use rustc_ast::visit::visit_opt;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
@@ -168,7 +167,14 @@ fn resolve_block<'tcx>(visitor: &mut RegionResolutionVisitor<'tcx>, blk: &'tcx h
                 hir::StmtKind::Expr(..) | hir::StmtKind::Semi(..) => visitor.visit_stmt(statement),
             }
         }
-        visit_opt!(visitor, visit_expr, &blk.expr);
+        if let Some(tail_expr) = blk.expr {
+            if visitor.tcx.features().shorter_tail_lifetimes
+                && blk.span.edition().at_least_rust_2024()
+            {
+                visitor.terminating_scopes.insert(tail_expr.hir_id.local_id);
+            }
+            visitor.visit_expr(tail_expr);
+        }
     }
 
     visitor.cx = prev_cx;

compiler/rustc_hir_analysis/src/check/wfcheck.rs

@@ -119,16 +119,7 @@ where
 
     let errors = wfcx.select_all_or_error();
     if !errors.is_empty() {
-        let err = infcx.err_ctxt().report_fulfillment_errors(errors);
-        if tcx.dcx().has_errors().is_some() {
-            return Err(err);
-        } else {
-            // HACK(oli-obk): tests/ui/specialization/min_specialization/specialize_on_type_error.rs
-            // causes an delayed bug during normalization, without reporting an error, so we need
-            // to act as if no error happened, in order to let our callers continue and report an
-            // error later in check_impl_items_against_trait.
-            return Ok(());
-        }
+        return Err(infcx.err_ctxt().report_fulfillment_errors(errors));
     }
 
     debug!(?assumed_wf_types);

compiler/rustc_hir_analysis/src/collect/item_bounds.rs

@@ -59,7 +59,7 @@ fn associated_type_bounds<'tcx>(
 /// impl trait it isn't possible to write a suitable predicate on the
 /// containing function and for type-alias impl trait we don't have a backwards
 /// compatibility issue.
-#[instrument(level = "trace", skip(tcx), ret)]
+#[instrument(level = "trace", skip(tcx, item_ty))]
 fn opaque_type_bounds<'tcx>(
     tcx: TyCtxt<'tcx>,
     opaque_def_id: LocalDefId,

compiler/rustc_hir_typeck/src/autoderef.rs

@@ -1,4 +1,5 @@
-//! Some helper functions for `AutoDeref`
+//! Some helper functions for `AutoDeref`.
+
 use super::method::MethodCallee;
 use super::{FnCtxt, PlaceOp};
 

compiler/rustc_hir_typeck/src/errors.rs

@@ -1,4 +1,5 @@
 //! Errors emitted by `rustc_hir_typeck`.
+
 use std::borrow::Cow;
 
 use crate::fluent_generated as fluent;

compiler/rustc_hir_typeck/src/fn_ctxt/inspect_obligations.rs

@@ -1,4 +1,5 @@
 //! A utility module to inspect currently ambiguous obligations in the current context.
+
 use crate::FnCtxt;
 use rustc_infer::traits::{self, ObligationCause};
 use rustc_middle::traits::solve::Goal;

compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs

@@ -859,10 +859,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 } else {
                     // Only point to return type if the expected type is the return type, as if they
                     // are not, the expectation must have been caused by something else.
-                    debug!("return type {:?}", hir_ty);
+                    debug!(?hir_ty, "return type");
                     let ty = self.lowerer().lower_ty(hir_ty);
-                    debug!("return type {:?}", ty);
-                    debug!("expected type {:?}", expected);
+                    debug!(?ty, "return type (lowered)");
+                    debug!(?expected, "expected type");
                     let bound_vars = self.tcx.late_bound_vars(hir_ty.hir_id.owner.into());
                     let ty = Binder::bind_with_vars(ty, bound_vars);
                     let ty = self.normalize(hir_ty.span, ty);
@@ -873,7 +873,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             expected,
                         });
                         self.try_suggest_return_impl_trait(err, expected, found, fn_id);
-                        self.note_caller_chooses_ty_for_ty_param(err, expected, found);
+                        self.try_note_caller_chooses_ty_for_ty_param(err, expected, found);
                         return true;
                     }
                 }
@@ -883,18 +883,30 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         false
     }
 
-    fn note_caller_chooses_ty_for_ty_param(
+    fn try_note_caller_chooses_ty_for_ty_param(
         &self,
         diag: &mut Diag<'_>,
         expected: Ty<'tcx>,
         found: Ty<'tcx>,
     ) {
-        if let ty::Param(expected_ty_as_param) = expected.kind() {
-            diag.subdiagnostic(errors::NoteCallerChoosesTyForTyParam {
-                ty_param_name: expected_ty_as_param.name,
-                found_ty: found,
-            });
+        // Only show the note if:
+        // 1. `expected` ty is a type parameter;
+        // 2. The `expected` type parameter does *not* occur in the return expression type. This can
+        //    happen for e.g. `fn foo<T>(t: &T) -> T { t }`, where `expected` is `T` but `found` is
+        //    `&T`. Saying "the caller chooses a type for `T` which can be different from `&T`" is
+        //    "well duh" and is only confusing and not helpful.
+        let ty::Param(expected_ty_as_param) = expected.kind() else {
+            return;
+        };
+
+        if found.contains(expected) {
+            return;
         }
+
+        diag.subdiagnostic(errors::NoteCallerChoosesTyForTyParam {
+            ty_param_name: expected_ty_as_param.name,
+            found_ty: found,
+        });
     }
 
     /// check whether the return type is a generic type with a trait bound

compiler/rustc_hir_typeck/src/method/probe.rs

@@ -1357,6 +1357,8 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
         traits::SelectionContext::new(self).select(&obligation)
     }
 
+    /// Used for ambiguous method call error reporting. Uses probing that throws away the result internally,
+    /// so do not use to make a decision that may lead to a successful compilation.
     fn candidate_source(&self, candidate: &Candidate<'tcx>, self_ty: Ty<'tcx>) -> CandidateSource {
         match candidate.kind {
             InherentImplCandidate(_) => {
@@ -1370,8 +1372,10 @@ impl<'a, 'tcx> ProbeContext<'a, 'tcx> {
                     self.instantiate_binder_with_fresh_vars(self.span, infer::FnCall, trait_ref);
                 let (xform_self_ty, _) =
                     self.xform_self_ty(candidate.item, trait_ref.self_ty(), trait_ref.args);
+                // Guide the trait selection to show impls that have methods whose type matches
+                // up with the `self` parameter of the method.
                 let _ = self.at(&ObligationCause::dummy(), self.param_env).sup(
-                    DefineOpaqueTypes::No,
+                    DefineOpaqueTypes::Yes,
                     xform_self_ty,
                     self_ty,
                 );

compiler/rustc_hir_typeck/src/method/suggest.rs

@@ -303,14 +303,57 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 }
                 if !candidates.is_empty() {
                     let help = format!(
-                        "{an}other candidate{s} {were} found in the following trait{s}, perhaps \
-                         add a `use` for {one_of_them}:",
+                        "{an}other candidate{s} {were} found in the following trait{s}",
                         an = if candidates.len() == 1 { "an" } else { "" },
                         s = pluralize!(candidates.len()),
                         were = pluralize!("was", candidates.len()),
-                        one_of_them = if candidates.len() == 1 { "it" } else { "one_of_them" },
                     );
-                    self.suggest_use_candidates(&mut err, help, candidates);
+                    self.suggest_use_candidates(
+                        candidates,
+                        |accessible_sugg, inaccessible_sugg, span| {
+                            let suggest_for_access =
+                                |err: &mut Diag<'_>, mut msg: String, sugg: Vec<_>| {
+                                    msg += &format!(
+                                        ", perhaps add a `use` for {one_of_them}:",
+                                        one_of_them =
+                                            if sugg.len() == 1 { "it" } else { "one_of_them" },
+                                    );
+                                    err.span_suggestions(
+                                        span,
+                                        msg,
+                                        sugg,
+                                        Applicability::MaybeIncorrect,
+                                    );
+                                };
+                            let suggest_for_privacy =
+                                |err: &mut Diag<'_>, mut msg: String, sugg: Vec<String>| {
+                                    if sugg.len() == 1 {
+                                        let msg = format!("\
+                                            trait `{}` provides `{item_name}` is implemented but not reachable",
+                                            sugg[0].trim()
+                                        );
+                                        err.help(msg);
+                                    } else {
+                                        msg += &format!(" but {} not reachable", pluralize!("is", sugg.len()));
+                                        err.span_suggestions(
+                                            span,
+                                            msg,
+                                            sugg,
+                                            Applicability::MaybeIncorrect,
+                                        );
+                                    }
+                                };
+                            if accessible_sugg.is_empty() {
+                                // `inaccessible_sugg` must not be empty
+                                suggest_for_privacy(&mut err, help, inaccessible_sugg);
+                            } else if inaccessible_sugg.is_empty() {
+                                suggest_for_access(&mut err, help, accessible_sugg);
+                            } else {
+                                suggest_for_access(&mut err, help.clone(), accessible_sugg);
+                                suggest_for_privacy(&mut err, help, inaccessible_sugg);
+                            }
+                        },
+                    );
                 }
                 if let ty::Ref(region, t_type, mutability) = rcvr_ty.kind() {
                     if needs_mut {
@@ -3139,49 +3182,69 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
     }
 
-    fn suggest_use_candidates(&self, err: &mut Diag<'_>, msg: String, candidates: Vec<DefId>) {
+    fn suggest_use_candidates<F>(&self, candidates: Vec<DefId>, handle_candidates: F)
+    where
+        F: FnOnce(Vec<String>, Vec<String>, Span),
+    {
         let parent_map = self.tcx.visible_parent_map(());
 
-        // Separate out candidates that must be imported with a glob, because they are named `_`
-        // and cannot be referred with their identifier.
-        let (candidates, globs): (Vec<_>, Vec<_>) = candidates.into_iter().partition(|trait_did| {
-            if let Some(parent_did) = parent_map.get(trait_did) {
-                // If the item is re-exported as `_`, we should suggest a glob-import instead.
-                if *parent_did != self.tcx.parent(*trait_did)
-                    && self
-                        .tcx
-                        .module_children(*parent_did)
-                        .iter()
-                        .filter(|child| child.res.opt_def_id() == Some(*trait_did))
-                        .all(|child| child.ident.name == kw::Underscore)
-                {
-                    return false;
-                }
-            }
+        let scope = self.tcx.parent_module_from_def_id(self.body_id);
+        let (accessible_candidates, inaccessible_candidates): (Vec<_>, Vec<_>) =
+            candidates.into_iter().partition(|id| {
+                let vis = self.tcx.visibility(*id);
+                vis.is_accessible_from(scope, self.tcx)
+            });
 
-            true
-        });
+        let sugg = |candidates: Vec<_>, visible| {
+            // Separate out candidates that must be imported with a glob, because they are named `_`
+            // and cannot be referred with their identifier.
+            let (candidates, globs): (Vec<_>, Vec<_>) =
+                candidates.into_iter().partition(|trait_did| {
+                    if let Some(parent_did) = parent_map.get(trait_did) {
+                        // If the item is re-exported as `_`, we should suggest a glob-import instead.
+                        if *parent_did != self.tcx.parent(*trait_did)
+                            && self
+                                .tcx
+                                .module_children(*parent_did)
+                                .iter()
+                                .filter(|child| child.res.opt_def_id() == Some(*trait_did))
+                                .all(|child| child.ident.name == kw::Underscore)
+                        {
+                            return false;
+                        }
+                    }
 
-        let module_did = self.tcx.parent_module_from_def_id(self.body_id);
-        let (module, _, _) = self.tcx.hir().get_module(module_did);
+                    true
+                });
+
+            let prefix = if visible { "use " } else { "" };
+            let postfix = if visible { ";" } else { "" };
+            let path_strings = candidates.iter().map(|trait_did| {
+                format!(
+                    "{prefix}{}{postfix}\n",
+                    with_crate_prefix!(self.tcx.def_path_str(*trait_did)),
+                )
+            });
+
+            let glob_path_strings = globs.iter().map(|trait_did| {
+                let parent_did = parent_map.get(trait_did).unwrap();
+                format!(
+                    "{prefix}{}::*{postfix} // trait {}\n",
+                    with_crate_prefix!(self.tcx.def_path_str(*parent_did)),
+                    self.tcx.item_name(*trait_did),
+                )
+            });
+            let mut sugg: Vec<_> = path_strings.chain(glob_path_strings).collect();
+            sugg.sort();
+            sugg
+        };
+
+        let accessible_sugg = sugg(accessible_candidates, true);
+        let inaccessible_sugg = sugg(inaccessible_candidates, false);
+
+        let (module, _, _) = self.tcx.hir().get_module(scope);
         let span = module.spans.inject_use_span;
-
-        let path_strings = candidates.iter().map(|trait_did| {
-            format!("use {};\n", with_crate_prefix!(self.tcx.def_path_str(*trait_did)),)
-        });
-
-        let glob_path_strings = globs.iter().map(|trait_did| {
-            let parent_did = parent_map.get(trait_did).unwrap();
-            format!(
-                "use {}::*; // trait {}\n",
-                with_crate_prefix!(self.tcx.def_path_str(*parent_did)),
-                self.tcx.item_name(*trait_did),
-            )
-        });
-        let mut sugg: Vec<_> = path_strings.chain(glob_path_strings).collect();
-        sugg.sort();
-
-        err.span_suggestions(span, msg, sugg, Applicability::MaybeIncorrect);
+        handle_candidates(accessible_sugg, inaccessible_sugg, span);
     }
 
     fn suggest_valid_traits(
@@ -3205,9 +3268,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
             if explain {
                 err.help("items from traits can only be used if the trait is in scope");
             }
+
             let msg = format!(
-                "{this_trait_is} implemented but not in scope; perhaps you want to import \
-                 {one_of_them}",
+                "{this_trait_is} implemented but not in scope",
                 this_trait_is = if candidates.len() == 1 {
                     format!(
                         "trait `{}` which provides `{item_name}` is",
@@ -3215,11 +3278,43 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     )
                 } else {
                     format!("the following traits which provide `{item_name}` are")
-                },
-                one_of_them = if candidates.len() == 1 { "it" } else { "one of them" },
+                }
             );
 
-            self.suggest_use_candidates(err, msg, candidates);
+            self.suggest_use_candidates(candidates, |accessible_sugg, inaccessible_sugg, span| {
+                let suggest_for_access = |err: &mut Diag<'_>, mut msg: String, sugg: Vec<_>| {
+                    msg += &format!(
+                        "; perhaps you want to import {one_of}",
+                        one_of = if sugg.len() == 1 { "it" } else { "one of them" },
+                    );
+                    err.span_suggestions(span, msg, sugg, Applicability::MaybeIncorrect);
+                };
+                let suggest_for_privacy = |err: &mut Diag<'_>, sugg: Vec<String>| {
+                    let msg = format!(
+                        "{this_trait_is} implemented but not reachable",
+                        this_trait_is = if sugg.len() == 1 {
+                            format!("trait `{}` which provides `{item_name}` is", sugg[0].trim())
+                        } else {
+                            format!("the following traits which provide `{item_name}` are")
+                        }
+                    );
+                    if sugg.len() == 1 {
+                        err.help(msg);
+                    } else {
+                        err.span_suggestions(span, msg, sugg, Applicability::MaybeIncorrect);
+                    }
+                };
+                if accessible_sugg.is_empty() {
+                    // `inaccessible_sugg` must not be empty
+                    suggest_for_privacy(err, inaccessible_sugg);
+                } else if inaccessible_sugg.is_empty() {
+                    suggest_for_access(err, msg, accessible_sugg);
+                } else {
+                    suggest_for_access(err, msg, accessible_sugg);
+                    suggest_for_privacy(err, inaccessible_sugg);
+                }
+            });
+
             if let Some(did) = edition_fix {
                 err.note(format!(
                     "'{}' is included in the prelude starting in Edition 2021",

compiler/rustc_hir_typeck/src/typeck_root_ctxt.rs

@@ -22,7 +22,7 @@ use std::ops::Deref;
 /// e.g. closures defined within the function. For example:
 /// ```ignore (illustrative)
 /// fn foo() {
-///     bar(move|| { ... })
+///     bar(move || { ... })
 /// }
 /// ```
 /// Here, the function `foo()` and the closure passed to

compiler/rustc_hir_typeck/src/writeback.rs

@@ -793,7 +793,7 @@ impl<'cx, 'tcx> Resolver<'cx, 'tcx> {
     }
 
     fn report_error(&self, p: impl Into<ty::GenericArg<'tcx>>) -> ErrorGuaranteed {
-        if let Some(guar) = self.fcx.dcx().has_errors() {
+        if let Some(guar) = self.fcx.tainted_by_errors() {
             guar
         } else {
             self.fcx

compiler/rustc_infer/src/infer/error_reporting/nice_region_error/named_anon_conflict.rs

@@ -1,5 +1,6 @@
 //! Error Reporting for Anonymous Region Lifetime Errors
 //! where one region is named and the other is anonymous.
+
 use crate::infer::error_reporting::nice_region_error::NiceRegionError;
 use crate::{
     errors::ExplicitLifetimeRequired,

compiler/rustc_infer/src/infer/freshen.rs

@@ -30,6 +30,7 @@
 //! solving a set of constraints. In contrast, the type inferencer assigns a value to each type
 //! variable only once, and it does so as soon as it can, so it is reasonable to ask what the type
 //! inferencer knows "so far".
+
 use super::InferCtxt;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::bug;

compiler/rustc_infer/src/infer/mod.rs

@@ -878,9 +878,9 @@ impl<'tcx> InferCtxt<'tcx> {
 
         self.enter_forall(predicate, |ty::SubtypePredicate { a_is_expected, a, b }| {
             if a_is_expected {
-                Ok(self.at(cause, param_env).sub(DefineOpaqueTypes::No, a, b))
+                Ok(self.at(cause, param_env).sub(DefineOpaqueTypes::Yes, a, b))
             } else {
-                Ok(self.at(cause, param_env).sup(DefineOpaqueTypes::No, b, a))
+                Ok(self.at(cause, param_env).sup(DefineOpaqueTypes::Yes, b, a))
             }
         })
     }

compiler/rustc_infer/src/infer/outlives/mod.rs

@@ -1,4 +1,5 @@
 //! Various code related to computing outlives relations.
+
 use self::env::OutlivesEnvironment;
 use super::region_constraints::RegionConstraintData;
 use super::{InferCtxt, RegionResolutionError, SubregionOrigin};

compiler/rustc_lint/messages.ftl

@@ -549,7 +549,8 @@ lint_non_local_definitions_impl = non-local `impl` definition, `impl` blocks sho
     .without_trait = methods and associated constants are still usable outside the current expression, only `impl Local` and `impl dyn Local` can ever be private, and only if the type is nested in the same item as the `impl`
     .with_trait = an `impl` is never scoped, even when it is nested inside an item, as it may impact type checking outside of that item, which can be the case if neither the trait or the self type are at the same nesting level as the `impl`
     .bounds = `impl` may be usable in bounds, etc. from outside the expression, which might e.g. make something constructible that previously wasn't, because it's still on a publicly-visible type
-    .exception = items in an anonymous const item (`const _: () = {"{"} ... {"}"}`) are treated as in the same scope as the anonymous const's declaration
+    .doctest = make this doc-test a standalone test with its own `fn main() {"{"} ... {"}"}`
+    .exception = items in an anonymous const item (`const _: () = {"{"} ... {"}"}`) are treated as in the same scope as the anonymous const's declaration for the purpose of this lint
     .const_anon = use a const-anon item to suppress this lint
     .macro_to_change = the {$macro_kind} `{$macro_to_change}` defines the non-local `impl`, and may need to be changed
 

compiler/rustc_lint/src/lints.rs

@@ -1358,6 +1358,7 @@ pub enum NonLocalDefinitionsDiag {
         cargo_update: Option<NonLocalDefinitionsCargoUpdateNote>,
         const_anon: Option<Option<Span>>,
         move_to: Option<(Span, Vec<Span>)>,
+        doctest: bool,
         may_remove: Option<(Span, String)>,
         has_trait: bool,
         self_ty_str: String,
@@ -1368,8 +1369,7 @@ pub enum NonLocalDefinitionsDiag {
         depth: u32,
         body_kind_descr: &'static str,
         body_name: String,
-        help: Option<()>,
-        doctest_help: Option<()>,
+        doctest: bool,
         cargo_update: Option<NonLocalDefinitionsCargoUpdateNote>,
     },
 }
@@ -1384,6 +1384,7 @@ impl<'a> LintDiagnostic<'a, ()> for NonLocalDefinitionsDiag {
                 cargo_update,
                 const_anon,
                 move_to,
+                doctest,
                 may_remove,
                 has_trait,
                 self_ty_str,
@@ -1422,6 +1423,9 @@ impl<'a> LintDiagnostic<'a, ()> for NonLocalDefinitionsDiag {
                     }
                     diag.span_help(ms, fluent::lint_non_local_definitions_impl_move_help);
                 }
+                if doctest {
+                    diag.help(fluent::lint_doctest);
+                }
 
                 if let Some((span, part)) = may_remove {
                     diag.arg("may_remove_part", part);
@@ -1451,8 +1455,7 @@ impl<'a> LintDiagnostic<'a, ()> for NonLocalDefinitionsDiag {
                 depth,
                 body_kind_descr,
                 body_name,
-                help,
-                doctest_help,
+                doctest,
                 cargo_update,
             } => {
                 diag.primary_message(fluent::lint_non_local_definitions_macro_rules);
@@ -1460,11 +1463,10 @@ impl<'a> LintDiagnostic<'a, ()> for NonLocalDefinitionsDiag {
                 diag.arg("body_kind_descr", body_kind_descr);
                 diag.arg("body_name", body_name);
 
-                if let Some(()) = help {
-                    diag.help(fluent::lint_help);
-                }
-                if let Some(()) = doctest_help {
+                if doctest {
                     diag.help(fluent::lint_help_doctest);
+                } else {
+                    diag.help(fluent::lint_help);
                 }
 
                 diag.note(fluent::lint_non_local);

compiler/rustc_lint/src/non_local_def.rs

@@ -111,6 +111,12 @@ impl<'tcx> LateLintPass<'tcx> for NonLocalDefinitions {
             }
         };
 
+        // determining if we are in a doctest context can't currently be determined
+        // by the code itself (there are no specific attributes), but fortunately rustdoc
+        // sets a perma-unstable env var for libtest so we just reuse that for now
+        let is_at_toplevel_doctest =
+            || self.body_depth == 2 && std::env::var("UNSTABLE_RUSTDOC_TEST_PATH").is_ok();
+
         match item.kind {
             ItemKind::Impl(impl_) => {
                 // The RFC states:
@@ -191,29 +197,6 @@ impl<'tcx> LateLintPass<'tcx> for NonLocalDefinitions {
                     None
                 };
 
-                let mut collector = PathCollector { paths: Vec::new() };
-                collector.visit_ty(&impl_.self_ty);
-                if let Some(of_trait) = &impl_.of_trait {
-                    collector.visit_trait_ref(of_trait);
-                }
-                collector.visit_generics(&impl_.generics);
-
-                let mut may_move: Vec<Span> = collector
-                    .paths
-                    .into_iter()
-                    .filter_map(|path| {
-                        if let Some(did) = path.res.opt_def_id()
-                            && did_has_local_parent(did, cx.tcx, parent, parent_parent)
-                        {
-                            Some(cx.tcx.def_span(did))
-                        } else {
-                            None
-                        }
-                    })
-                    .collect();
-                may_move.sort();
-                may_move.dedup();
-
                 let const_anon = matches!(parent_def_kind, DefKind::Const | DefKind::Static { .. })
                     .then_some(span_for_const_anon_suggestion);
 
@@ -248,14 +231,44 @@ impl<'tcx> LateLintPass<'tcx> for NonLocalDefinitions {
                 } else {
                     None
                 };
-                let move_to = if may_move.is_empty() {
-                    ms.push_span_label(
-                        cx.tcx.def_span(parent),
-                        fluent::lint_non_local_definitions_impl_move_help,
-                    );
-                    None
+
+                let (doctest, move_to) = if is_at_toplevel_doctest() {
+                    (true, None)
                 } else {
-                    Some((cx.tcx.def_span(parent), may_move))
+                    let mut collector = PathCollector { paths: Vec::new() };
+                    collector.visit_ty(&impl_.self_ty);
+                    if let Some(of_trait) = &impl_.of_trait {
+                        collector.visit_trait_ref(of_trait);
+                    }
+                    collector.visit_generics(&impl_.generics);
+
+                    let mut may_move: Vec<Span> = collector
+                        .paths
+                        .into_iter()
+                        .filter_map(|path| {
+                            if let Some(did) = path.res.opt_def_id()
+                                && did_has_local_parent(did, cx.tcx, parent, parent_parent)
+                            {
+                                Some(cx.tcx.def_span(did))
+                            } else {
+                                None
+                            }
+                        })
+                        .collect();
+                    may_move.sort();
+                    may_move.dedup();
+
+                    let move_to = if may_move.is_empty() {
+                        ms.push_span_label(
+                            cx.tcx.def_span(parent),
+                            fluent::lint_non_local_definitions_impl_move_help,
+                        );
+                        None
+                    } else {
+                        Some((cx.tcx.def_span(parent), may_move))
+                    };
+
+                    (false, move_to)
                 };
 
                 let macro_to_change =
@@ -279,6 +292,7 @@ impl<'tcx> LateLintPass<'tcx> for NonLocalDefinitions {
                         self_ty_str,
                         of_trait_str,
                         move_to,
+                        doctest,
                         may_remove,
                         has_trait: impl_.of_trait.is_some(),
                         macro_to_change,
@@ -288,12 +302,6 @@ impl<'tcx> LateLintPass<'tcx> for NonLocalDefinitions {
             ItemKind::Macro(_macro, MacroKind::Bang)
                 if cx.tcx.has_attr(item.owner_id.def_id, sym::macro_export) =>
             {
-                // determining we if are in a doctest context can't currently be determined
-                // by the code it-self (no specific attrs), but fortunatly rustdoc sets a
-                // perma-unstable env for libtest so we just re-use that env for now
-                let is_at_toplevel_doctest =
-                    self.body_depth == 2 && std::env::var("UNSTABLE_RUSTDOC_TEST_PATH").is_ok();
-
                 cx.emit_span_lint(
                     NON_LOCAL_DEFINITIONS,
                     item.span,
@@ -304,8 +312,7 @@ impl<'tcx> LateLintPass<'tcx> for NonLocalDefinitions {
                             .map(|s| s.to_ident_string())
                             .unwrap_or_else(|| "<unnameable>".to_string()),
                         cargo_update: cargo_update(),
-                        help: (!is_at_toplevel_doctest).then_some(()),
-                        doctest_help: is_at_toplevel_doctest.then_some(()),
+                        doctest: is_at_toplevel_doctest(),
                     },
                 )
             }

compiler/rustc_lint_defs/src/lib.rs

@@ -1,4 +1,3 @@
-// tidy-alphabetical-start
 pub use self::Level::*;
 use rustc_ast::node_id::NodeId;
 use rustc_ast::{AttrId, Attribute};
@@ -15,7 +14,6 @@ use rustc_span::edition::Edition;
 use rustc_span::symbol::MacroRulesNormalizedIdent;
 use rustc_span::{sym, symbol::Ident, Span, Symbol};
 use rustc_target::spec::abi::Abi;
-// tidy-alphabetical-end
 
 use serde::{Deserialize, Serialize};
 

compiler/rustc_middle/src/dep_graph/dep_node.rs

@@ -194,10 +194,7 @@ impl DepNodeExt for DepNode {
     /// has been removed.
     fn extract_def_id(&self, tcx: TyCtxt<'_>) -> Option<DefId> {
         if tcx.fingerprint_style(self.kind) == FingerprintStyle::DefPathHash {
-            Some(tcx.def_path_hash_to_def_id(
-                DefPathHash(self.hash.into()),
-                &("Failed to extract DefId", self.kind, self.hash),
-            ))
+            tcx.def_path_hash_to_def_id(DefPathHash(self.hash.into()))
         } else {
             None
         }
@@ -390,12 +387,7 @@ impl<'tcx> DepNodeParams<TyCtxt<'tcx>> for HirId {
         if tcx.fingerprint_style(dep_node.kind) == FingerprintStyle::HirId {
             let (local_hash, local_id) = Fingerprint::from(dep_node.hash).split();
             let def_path_hash = DefPathHash::new(tcx.stable_crate_id(LOCAL_CRATE), local_hash);
-            let def_id = tcx
-                .def_path_hash_to_def_id(
-                    def_path_hash,
-                    &("Failed to extract HirId", dep_node.kind, dep_node.hash),
-                )
-                .expect_local();
+            let def_id = tcx.def_path_hash_to_def_id(def_path_hash)?.expect_local();
             let local_id = local_id
                 .as_u64()
                 .try_into()

compiler/rustc_middle/src/middle/privacy.rs

@@ -1,6 +1,7 @@
 //! A pass that checks to make sure private fields and methods aren't used
 //! outside their scopes. This pass will also generate a set of exported items
 //! which are available for use externally when compiled as a library.
+
 use crate::ty::{TyCtxt, Visibility};
 use rustc_data_structures::fx::{FxIndexMap, IndexEntry};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};

compiler/rustc_middle/src/mir/statement.rs

@@ -1,4 +1,5 @@
-/// Functionality for statements, operands, places, and things that appear in them.
+//! Functionality for statements, operands, places, and things that appear in them.
+
 use super::{interpret::GlobalAlloc, *};
 
 ///////////////////////////////////////////////////////////////////////////

compiler/rustc_middle/src/mir/syntax.rs

@@ -127,6 +127,9 @@ pub enum AnalysisPhase {
     /// * [`StatementKind::AscribeUserType`]
     /// * [`StatementKind::Coverage`] with [`CoverageKind::BlockMarker`] or [`CoverageKind::SpanMarker`]
     /// * [`Rvalue::Ref`] with `BorrowKind::Fake`
+    /// * [`CastKind::PointerCoercion`] with any of the following:
+    ///   * [`PointerCoercion::ArrayToPointer`]
+    ///   * [`PointerCoercion::MutToConstPointer`]
     ///
     /// Furthermore, `Deref` projections must be the first projection within any place (if they
     /// appear at all)
@@ -361,16 +364,19 @@ pub enum StatementKind<'tcx> {
     /// At any point during the execution of a function, each local is either allocated or
     /// unallocated. Except as noted below, all locals except function parameters are initially
     /// unallocated. `StorageLive` statements cause memory to be allocated for the local while
-    /// `StorageDead` statements cause the memory to be freed. Using a local in any way (not only
-    /// reading/writing from it) while it is unallocated is UB.
+    /// `StorageDead` statements cause the memory to be freed. In other words,
+    /// `StorageLive`/`StorageDead` act like the heap operations `allocate`/`deallocate`, but for
+    /// stack-allocated local variables. Using a local in any way (not only reading/writing from it)
+    /// while it is unallocated is UB.
     ///
     /// Some locals have no `StorageLive` or `StorageDead` statements within the entire MIR body.
     /// These locals are implicitly allocated for the full duration of the function. There is a
     /// convenience method at `rustc_mir_dataflow::storage::always_storage_live_locals` for
     /// computing these locals.
     ///
-    /// If the local is already allocated, calling `StorageLive` again is UB. However, for an
-    /// unallocated local an additional `StorageDead` all is simply a nop.
+    /// If the local is already allocated, calling `StorageLive` again will implicitly free the
+    /// local and then allocate fresh uninitilized memory. If a local is already deallocated,
+    /// calling `StorageDead` again is a NOP.
     StorageLive(Local),
 
     /// See `StorageLive` above.
@@ -1281,8 +1287,7 @@ pub enum Rvalue<'tcx> {
     ///
     /// This allows for casts from/to a variety of types.
     ///
-    /// **FIXME**: Document exactly which `CastKind`s allow which types of casts. Figure out why
-    /// `ArrayToPointer` and `MutToConstPointer` are special.
+    /// **FIXME**: Document exactly which `CastKind`s allow which types of casts.
     Cast(CastKind, Operand<'tcx>, Ty<'tcx>),
 
     /// * `Offset` has the same semantics as [`offset`](pointer::offset), except that the second
@@ -1362,6 +1367,13 @@ pub enum CastKind {
     PointerWithExposedProvenance,
     /// Pointer related casts that are done by coercions. Note that reference-to-raw-ptr casts are
     /// translated into `&raw mut/const *r`, i.e., they are not actually casts.
+    ///
+    /// The following are allowed in [`AnalysisPhase::Initial`] as they're needed for borrowck,
+    /// but after that are forbidden (including in all phases of runtime MIR):
+    /// * [`PointerCoercion::ArrayToPointer`]
+    /// * [`PointerCoercion::MutToConstPointer`]
+    ///
+    /// Both are runtime nops, so should be [`CastKind::PtrToPtr`] instead in runtime MIR.
     PointerCoercion(PointerCoercion),
     /// Cast into a dyn* object.
     DynStar,
@@ -1434,10 +1446,12 @@ pub enum UnOp {
     Not,
     /// The `-` operator for negation
     Neg,
-    /// Get the metadata `M` from a `*const/mut impl Pointee<Metadata = M>`.
+    /// Gets the metadata `M` from a `*const`/`*mut`/`&`/`&mut` to
+    /// `impl Pointee<Metadata = M>`.
     ///
     /// For example, this will give a `()` from `*const i32`, a `usize` from
-    /// `*mut [u8]`, or a pointer to a vtable from a `*const dyn Foo`.
+    /// `&mut [u8]`, or a `ptr::DynMetadata<dyn Foo>` (internally a pointer)
+    /// from a `*mut dyn Foo`.
     ///
     /// Allowed only in [`MirPhase::Runtime`]; earlier it's an intrinsic.
     PtrMetadata,

compiler/rustc_middle/src/mir/terminator.rs

@@ -1,4 +1,5 @@
-/// Functionality for terminators and helper types that appear in terminators.
+//! Functionality for terminators and helper types that appear in terminators.
+
 use rustc_hir::LangItem;
 use smallvec::{smallvec, SmallVec};
 

compiler/rustc_middle/src/query/mod.rs

@@ -1554,12 +1554,13 @@ rustc_queries! {
         }
     }
 
-    /// The entire set of monomorphizations the local crate can safely link
-    /// to because they are exported from upstream crates. Do not depend on
-    /// this directly, as its value changes anytime a monomorphization gets
-    /// added or removed in any upstream crate. Instead use the narrower
-    /// `upstream_monomorphizations_for`, `upstream_drop_glue_for`, or, even
-    /// better, `Instance::upstream_monomorphization()`.
+    /// The entire set of monomorphizations the local crate can safely
+    /// link to because they are exported from upstream crates. Do
+    /// not depend on this directly, as its value changes anytime
+    /// a monomorphization gets added or removed in any upstream
+    /// crate. Instead use the narrower `upstream_monomorphizations_for`,
+    /// `upstream_drop_glue_for`, `upstream_async_drop_glue_for`, or,
+    /// even better, `Instance::upstream_monomorphization()`.
     query upstream_monomorphizations(_: ()) -> &'tcx DefIdMap<UnordMap<GenericArgsRef<'tcx>, CrateNum>> {
         arena_cache
         desc { "collecting available upstream monomorphizations" }
@@ -1601,6 +1602,26 @@ rustc_queries! {
         desc { "available upstream drop-glue for `{:?}`", args }
     }
 
+    /// Returns the upstream crate that exports async-drop-glue for
+    /// the given type (`args` is expected to be a single-item list
+    /// containing the type one wants async-drop-glue for).
+    ///
+    /// This is a subset of `upstream_monomorphizations_for` in order
+    /// to increase dep-tracking granularity. Otherwise adding or
+    /// removing any type with async-drop-glue in any upstream crate
+    /// would invalidate all functions calling async-drop-glue of an
+    /// upstream type.
+    ///
+    /// You likely want to call `Instance::upstream_monomorphization()`
+    /// instead of invoking this query directly.
+    ///
+    /// NOTE: This query could easily be extended to also support other
+    ///       common functions that have are large set of monomorphizations
+    ///       (like `Clone::clone` for example).
+    query upstream_async_drop_glue_for(args: GenericArgsRef<'tcx>) -> Option<CrateNum> {
+        desc { "available upstream async-drop-glue for `{:?}`", args }
+    }
+
     /// Returns a list of all `extern` blocks of a crate.
     query foreign_modules(_: CrateNum) -> &'tcx FxIndexMap<DefId, ForeignModule> {
         arena_cache

compiler/rustc_middle/src/query/on_disk_cache.rs

@@ -733,10 +733,10 @@ impl<'a, 'tcx> SpanDecoder for CacheDecoder<'a, 'tcx> {
         // If we get to this point, then all of the query inputs were green,
         // which means that the definition with this hash is guaranteed to
         // still exist in the current compilation session.
-        self.tcx.def_path_hash_to_def_id(
-            def_path_hash,
-            &("Failed to convert DefPathHash", def_path_hash),
-        )
+        match self.tcx.def_path_hash_to_def_id(def_path_hash) {
+            Some(r) => r,
+            None => panic!("Failed to convert DefPathHash {def_path_hash:?}"),
+        }
     }
 
     fn decode_attr_id(&mut self) -> rustc_span::AttrId {

compiler/rustc_middle/src/ty/abstract_const.rs

@@ -1,4 +1,5 @@
 //! A subset of a mir body used for const evaluability checking.
+
 use crate::ty::{
     self, Const, EarlyBinder, Ty, TyCtxt, TypeFoldable, TypeFolder, TypeSuperFoldable,
     TypeVisitableExt,

compiler/rustc_middle/src/ty/context.rs

@@ -1677,11 +1677,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Converts a `DefPathHash` to its corresponding `DefId` in the current compilation
     /// session, if it still exists. This is used during incremental compilation to
     /// turn a deserialized `DefPathHash` into its current `DefId`.
-    pub fn def_path_hash_to_def_id(
-        self,
-        hash: DefPathHash,
-        err_msg: &dyn std::fmt::Debug,
-    ) -> DefId {
+    pub fn def_path_hash_to_def_id(self, hash: DefPathHash) -> Option<DefId> {
         debug!("def_path_hash_to_def_id({:?})", hash);
 
         let stable_crate_id = hash.stable_crate_id();
@@ -1689,13 +1685,9 @@ impl<'tcx> TyCtxt<'tcx> {
         // If this is a DefPathHash from the local crate, we can look up the
         // DefId in the tcx's `Definitions`.
         if stable_crate_id == self.stable_crate_id(LOCAL_CRATE) {
-            self.untracked
-                .definitions
-                .read()
-                .local_def_path_hash_to_def_id(hash, err_msg)
-                .to_def_id()
+            Some(self.untracked.definitions.read().local_def_path_hash_to_def_id(hash)?.to_def_id())
         } else {
-            self.def_path_hash_to_def_id_extern(hash, stable_crate_id)
+            Some(self.def_path_hash_to_def_id_extern(hash, stable_crate_id))
         }
     }
 

compiler/rustc_middle/src/ty/instance.rs

@@ -219,8 +219,9 @@ impl<'tcx> Instance<'tcx> {
             InstanceKind::Item(def) => tcx
                 .upstream_monomorphizations_for(def)
                 .and_then(|monos| monos.get(&self.args).cloned()),
-            InstanceKind::DropGlue(_, Some(_)) | InstanceKind::AsyncDropGlueCtorShim(_, _) => {
-                tcx.upstream_drop_glue_for(self.args)
+            InstanceKind::DropGlue(_, Some(_)) => tcx.upstream_drop_glue_for(self.args),
+            InstanceKind::AsyncDropGlueCtorShim(_, Some(_)) => {
+                tcx.upstream_async_drop_glue_for(self.args)
             }
             _ => None,
         }
@@ -256,7 +257,7 @@ impl<'tcx> InstanceKind<'tcx> {
         match self {
             ty::InstanceKind::Item(def) => Some(def),
             ty::InstanceKind::DropGlue(def_id, Some(_))
-            | InstanceKind::AsyncDropGlueCtorShim(def_id, _)
+            | InstanceKind::AsyncDropGlueCtorShim(def_id, Some(_))
             | InstanceKind::ThreadLocalShim(def_id) => Some(def_id),
             InstanceKind::VTableShim(..)
             | InstanceKind::ReifyShim(..)
@@ -267,6 +268,7 @@ impl<'tcx> InstanceKind<'tcx> {
             | ty::InstanceKind::ConstructCoroutineInClosureShim { .. }
             | ty::InstanceKind::CoroutineKindShim { .. }
             | InstanceKind::DropGlue(..)
+            | InstanceKind::AsyncDropGlueCtorShim(..)
             | InstanceKind::CloneShim(..)
             | InstanceKind::FnPtrAddrShim(..) => None,
         }
@@ -312,7 +314,9 @@ impl<'tcx> InstanceKind<'tcx> {
         if self.requires_inline(tcx) {
             return true;
         }
-        if let ty::InstanceKind::DropGlue(.., Some(ty)) = *self {
+        if let ty::InstanceKind::DropGlue(.., Some(ty))
+        | ty::InstanceKind::AsyncDropGlueCtorShim(.., Some(ty)) = *self
+        {
             // Drop glue generally wants to be instantiated at every codegen
             // unit, but without an #[inline] hint. We should make this
             // available to normal end-users.
@@ -327,9 +331,14 @@ impl<'tcx> InstanceKind<'tcx> {
             // drops of `Option::None` before LTO. We also respect the intent of
             // `#[inline]` on `Drop::drop` implementations.
             return ty.ty_adt_def().map_or(true, |adt_def| {
-                adt_def
-                    .destructor(tcx)
-                    .map_or_else(|| adt_def.is_enum(), |dtor| tcx.cross_crate_inlinable(dtor.did))
+                match *self {
+                    ty::InstanceKind::DropGlue(..) => adt_def.destructor(tcx).map(|dtor| dtor.did),
+                    ty::InstanceKind::AsyncDropGlueCtorShim(..) => {
+                        adt_def.async_destructor(tcx).map(|dtor| dtor.ctor)
+                    }
+                    _ => unreachable!(),
+                }
+                .map_or_else(|| adt_def.is_enum(), |did| tcx.cross_crate_inlinable(did))
             });
         }
         if let ty::InstanceKind::ThreadLocalShim(..) = *self {

compiler/rustc_middle/src/ty/layout.rs

@@ -1182,37 +1182,6 @@ pub fn fn_can_unwind(tcx: TyCtxt<'_>, fn_def_id: Option<DefId>, abi: SpecAbi) ->
     // ABIs have such an option. Otherwise the only other thing here is Rust
     // itself, and those ABIs are determined by the panic strategy configured
     // for this compilation.
-    //
-    // Unfortunately at this time there's also another caveat. Rust [RFC
-    // 2945][rfc] has been accepted and is in the process of being implemented
-    // and stabilized. In this interim state we need to deal with historical
-    // rustc behavior as well as plan for future rustc behavior.
-    //
-    // Historically functions declared with `extern "C"` were marked at the
-    // codegen layer as `nounwind`. This happened regardless of `panic=unwind`
-    // or not. This is UB for functions in `panic=unwind` mode that then
-    // actually panic and unwind. Note that this behavior is true for both
-    // externally declared functions as well as Rust-defined function.
-    //
-    // To fix this UB rustc would like to change in the future to catch unwinds
-    // from function calls that may unwind within a Rust-defined `extern "C"`
-    // function and forcibly abort the process, thereby respecting the
-    // `nounwind` attribute emitted for `extern "C"`. This behavior change isn't
-    // ready to roll out, so determining whether or not the `C` family of ABIs
-    // unwinds is conditional not only on their definition but also whether the
-    // `#![feature(c_unwind)]` feature gate is active.
-    //
-    // Note that this means that unlike historical compilers rustc now, by
-    // default, unconditionally thinks that the `C` ABI may unwind. This will
-    // prevent some optimization opportunities, however, so we try to scope this
-    // change and only assume that `C` unwinds with `panic=unwind` (as opposed
-    // to `panic=abort`).
-    //
-    // Eventually the check against `c_unwind` here will ideally get removed and
-    // this'll be a little cleaner as it'll be a straightforward check of the
-    // ABI.
-    //
-    // [rfc]: https://github.com/rust-lang/rfcs/blob/master/text/2945-c-unwind-abi.md
     use SpecAbi::*;
     match abi {
         C { unwind }
@@ -1224,10 +1193,7 @@ pub fn fn_can_unwind(tcx: TyCtxt<'_>, fn_def_id: Option<DefId>, abi: SpecAbi) ->
         | Thiscall { unwind }
         | Aapcs { unwind }
         | Win64 { unwind }
-        | SysV64 { unwind } => {
-            unwind
-                || (!tcx.features().c_unwind && tcx.sess.panic_strategy() == PanicStrategy::Unwind)
-        }
+        | SysV64 { unwind } => unwind,
         PtxKernel
         | Msp430Interrupt
         | X86Interrupt

compiler/rustc_middle/src/ty/print/pretty.rs

@@ -7,7 +7,7 @@ use crate::ty::{
     ConstInt, Expr, ParamConst, ScalarInt, Term, TermKind, TypeFoldable, TypeSuperFoldable,
     TypeSuperVisitable, TypeVisitable, TypeVisitableExt,
 };
-use rustc_apfloat::ieee::{Double, Single};
+use rustc_apfloat::ieee::{Double, Half, Quad, Single};
 use rustc_apfloat::Float;
 use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
 use rustc_data_structures::unord::UnordMap;
@@ -1710,6 +1710,10 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
             ty::Bool if int == ScalarInt::FALSE => p!("false"),
             ty::Bool if int == ScalarInt::TRUE => p!("true"),
             // Float
+            ty::Float(ty::FloatTy::F16) => {
+                let val = Half::try_from(int).unwrap();
+                p!(write("{}{}f16", val, if val.is_finite() { "" } else { "_" }))
+            }
             ty::Float(ty::FloatTy::F32) => {
                 let val = Single::try_from(int).unwrap();
                 p!(write("{}{}f32", val, if val.is_finite() { "" } else { "_" }))
@@ -1718,6 +1722,10 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
                 let val = Double::try_from(int).unwrap();
                 p!(write("{}{}f64", val, if val.is_finite() { "" } else { "_" }))
             }
+            ty::Float(ty::FloatTy::F128) => {
+                let val = Quad::try_from(int).unwrap();
+                p!(write("{}{}f128", val, if val.is_finite() { "" } else { "_" }))
+            }
             // Int
             ty::Uint(_) | ty::Int(_) => {
                 let int =

compiler/rustc_mir_build/src/build/coverageinfo.rs

@@ -118,17 +118,35 @@ impl BranchInfoBuilder {
         }
     }
 
-    fn add_two_way_branch<'tcx>(
+    fn register_two_way_branch<'tcx>(
         &mut self,
+        tcx: TyCtxt<'tcx>,
         cfg: &mut CFG<'tcx>,
         source_info: SourceInfo,
         true_block: BasicBlock,
         false_block: BasicBlock,
     ) {
-        let true_marker = self.markers.inject_block_marker(cfg, source_info, true_block);
-        let false_marker = self.markers.inject_block_marker(cfg, source_info, false_block);
+        // Separate path for handling branches when MC/DC is enabled.
+        if let Some(mcdc_info) = self.mcdc_info.as_mut() {
+            let inject_block_marker =
+                |source_info, block| self.markers.inject_block_marker(cfg, source_info, block);
+            mcdc_info.visit_evaluated_condition(
+                tcx,
+                source_info,
+                true_block,
+                false_block,
+                inject_block_marker,
+            );
+        } else {
+            let true_marker = self.markers.inject_block_marker(cfg, source_info, true_block);
+            let false_marker = self.markers.inject_block_marker(cfg, source_info, false_block);
 
-        self.branch_spans.push(BranchSpan { span: source_info.span, true_marker, false_marker });
+            self.branch_spans.push(BranchSpan {
+                span: source_info.span,
+                true_marker,
+                false_marker,
+            });
+        }
     }
 
     pub(crate) fn into_done(self) -> Option<Box<mir::coverage::BranchInfo>> {
@@ -205,7 +223,14 @@ impl<'tcx> Builder<'_, 'tcx> {
             mir::TerminatorKind::if_(mir::Operand::Copy(place), true_block, false_block),
         );
 
-        branch_info.add_two_way_branch(&mut self.cfg, source_info, true_block, false_block);
+        // Separate path for handling branches when MC/DC is enabled.
+        branch_info.register_two_way_branch(
+            self.tcx,
+            &mut self.cfg,
+            source_info,
+            true_block,
+            false_block,
+        );
 
         let join_block = self.cfg.start_new_block();
         self.cfg.goto(true_block, source_info, join_block);
@@ -236,22 +261,13 @@ impl<'tcx> Builder<'_, 'tcx> {
 
         let source_info = SourceInfo { span: self.thir[expr_id].span, scope: self.source_scope };
 
-        // Separate path for handling branches when MC/DC is enabled.
-        if let Some(mcdc_info) = branch_info.mcdc_info.as_mut() {
-            let inject_block_marker = |source_info, block| {
-                branch_info.markers.inject_block_marker(&mut self.cfg, source_info, block)
-            };
-            mcdc_info.visit_evaluated_condition(
-                self.tcx,
-                source_info,
-                then_block,
-                else_block,
-                inject_block_marker,
-            );
-            return;
-        }
-
-        branch_info.add_two_way_branch(&mut self.cfg, source_info, then_block, else_block);
+        branch_info.register_two_way_branch(
+            self.tcx,
+            &mut self.cfg,
+            source_info,
+            then_block,
+            else_block,
+        );
     }
 
     /// If branch coverage is enabled, inject marker statements into `true_block`
@@ -270,6 +286,12 @@ impl<'tcx> Builder<'_, 'tcx> {
         // FIXME(#124144) This may need special handling when MC/DC is enabled.
 
         let source_info = SourceInfo { span: pattern.span, scope: self.source_scope };
-        branch_info.add_two_way_branch(&mut self.cfg, source_info, true_block, false_block);
+        branch_info.register_two_way_branch(
+            self.tcx,
+            &mut self.cfg,
+            source_info,
+            true_block,
+            false_block,
+        );
     }
 }

compiler/rustc_mir_build/src/build/custom/parse/instruction.rs

@@ -193,6 +193,10 @@ impl<'tcx, 'body> ParseCtxt<'tcx, 'body> {
                 let source = self.parse_operand(args[0])?;
                 Ok(Rvalue::Cast(CastKind::Transmute, source, expr.ty))
             },
+            @call(mir_cast_ptr_to_ptr, args) => {
+                let source = self.parse_operand(args[0])?;
+                Ok(Rvalue::Cast(CastKind::PtrToPtr, source, expr.ty))
+            },
             @call(mir_checked, args) => {
                 parse_by_kind!(self, args[0], _, "binary op",
                     ExprKind::Binary { op, lhs, rhs } => {

compiler/rustc_mir_build/src/build/expr/into.rs

@@ -150,6 +150,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             ExprKind::LogicalOp { op, lhs, rhs } => {
                 let condition_scope = this.local_scope();
                 let source_info = this.source_info(expr.span);
+
+                this.visit_coverage_branch_operation(op, expr.span);
+
                 // We first evaluate the left-hand side of the predicate ...
                 let (then_block, else_block) =
                     this.in_if_then_scope(condition_scope, expr.span, |this| {

compiler/rustc_mir_build/src/build/matches/mod.rs

@@ -1074,12 +1074,9 @@ struct Candidate<'pat, 'tcx> {
     // because that would break binding consistency.
     subcandidates: Vec<Candidate<'pat, 'tcx>>,
 
-    /// ...and the guard must be evaluated if there is one.
+    /// ...and if there is a guard it must be evaluated; if it's `false` then branch to `otherwise_block`.
     has_guard: bool,
 
-    /// If the guard is `false` then branch to `otherwise_block`.
-    otherwise_block: Option<BasicBlock>,
-
     /// If the candidate matches, bindings and ascriptions must be established.
     extra_data: PatternExtraData<'tcx>,
 
@@ -1090,6 +1087,9 @@ struct Candidate<'pat, 'tcx> {
     /// The block before the `bindings` have been established.
     pre_binding_block: Option<BasicBlock>,
 
+    /// The block to branch to if the guard or a nested candidate fails to match.
+    otherwise_block: Option<BasicBlock>,
+
     /// The earliest block that has only candidates >= this one as descendents. Used for false
     /// edges, see the doc for [`Builder::match_expr`].
     false_edge_start_block: Option<BasicBlock>,
@@ -1364,56 +1364,105 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         otherwise_block: BasicBlock,
         candidates: &mut [&mut Candidate<'pat, 'tcx>],
     ) {
-        let mut split_or_candidate = false;
-        for candidate in &mut *candidates {
-            if let [MatchPair { test_case: TestCase::Or { .. }, .. }] = &*candidate.match_pairs {
-                // Split a candidate in which the only match-pair is an or-pattern into multiple
-                // candidates. This is so that
-                //
-                // match x {
-                //     0 | 1 => { ... },
-                //     2 | 3 => { ... },
-                // }
-                //
-                // only generates a single switch.
-                let match_pair = candidate.match_pairs.pop().unwrap();
-                self.create_or_subcandidates(candidate, match_pair);
-                split_or_candidate = true;
+        // We process or-patterns here. If any candidate starts with an or-pattern, we have to
+        // expand the or-pattern before we can proceed further.
+        //
+        // We can't expand them freely however. The rule is: if the candidate has an or-pattern as
+        // its only remaining match pair, we can expand it freely. If it has other match pairs, we
+        // can expand it but we can't process more candidates after it.
+        //
+        // If we didn't stop, the `otherwise` cases could get mixed up. E.g. in the following,
+        // or-pattern simplification (in `merge_trivial_subcandidates`) makes it so the `1` and `2`
+        // cases branch to a same block (which then tests `false`). If we took `(2, _)` in the same
+        // set of candidates, when we reach the block that tests `false` we don't know whether we
+        // came from `1` or `2`, hence we can't know where to branch on failure.
+        // ```ignore(illustrative)
+        // match (1, true) {
+        //     (1 | 2, false) => {},
+        //     (2, _) => {},
+        //     _ => {}
+        // }
+        // ```
+        //
+        // We therefore split the `candidates` slice in two, expand or-patterns in the first half,
+        // and process both halves separately.
+        let mut expand_until = 0;
+        for (i, candidate) in candidates.iter().enumerate() {
+            if matches!(
+                &*candidate.match_pairs,
+                [MatchPair { test_case: TestCase::Or { .. }, .. }, ..]
+            ) {
+                expand_until = i + 1;
+                if candidate.match_pairs.len() > 1 {
+                    break;
+                }
             }
         }
+        let (candidates_to_expand, remaining_candidates) = candidates.split_at_mut(expand_until);
 
         ensure_sufficient_stack(|| {
-            if split_or_candidate {
-                // At least one of the candidates has been split into subcandidates.
-                // We need to change the candidate list to include those.
-                let mut new_candidates = Vec::new();
-                for candidate in candidates.iter_mut() {
-                    candidate.visit_leaves(|leaf_candidate| new_candidates.push(leaf_candidate));
+            if candidates_to_expand.is_empty() {
+                // No candidates start with an or-pattern, we can continue.
+                self.match_expanded_candidates(
+                    span,
+                    scrutinee_span,
+                    start_block,
+                    otherwise_block,
+                    remaining_candidates,
+                );
+            } else {
+                // Expand one level of or-patterns for each candidate in `candidates_to_expand`.
+                let mut expanded_candidates = Vec::new();
+                for candidate in candidates_to_expand.iter_mut() {
+                    if let [MatchPair { test_case: TestCase::Or { .. }, .. }, ..] =
+                        &*candidate.match_pairs
+                    {
+                        let or_match_pair = candidate.match_pairs.remove(0);
+                        // Expand the or-pattern into subcandidates.
+                        self.create_or_subcandidates(candidate, or_match_pair);
+                        // Collect the newly created subcandidates.
+                        for subcandidate in candidate.subcandidates.iter_mut() {
+                            expanded_candidates.push(subcandidate);
+                        }
+                    } else {
+                        expanded_candidates.push(candidate);
+                    }
                 }
+
+                // Process the expanded candidates.
+                let remainder_start = self.cfg.start_new_block();
+                // There might be new or-patterns obtained from expanding the old ones, so we call
+                // `match_candidates` again.
                 self.match_candidates(
                     span,
                     scrutinee_span,
                     start_block,
-                    otherwise_block,
-                    &mut *new_candidates,
+                    remainder_start,
+                    expanded_candidates.as_mut_slice(),
                 );
 
-                for candidate in candidates {
-                    self.merge_trivial_subcandidates(candidate);
+                // Simplify subcandidates and process any leftover match pairs.
+                for candidate in candidates_to_expand {
+                    if !candidate.subcandidates.is_empty() {
+                        self.finalize_or_candidate(span, scrutinee_span, candidate);
+                    }
                 }
-            } else {
-                self.match_simplified_candidates(
+
+                // Process the remaining candidates.
+                self.match_candidates(
                     span,
                     scrutinee_span,
-                    start_block,
+                    remainder_start,
                     otherwise_block,
-                    candidates,
+                    remaining_candidates,
                 );
             }
         });
     }
 
-    fn match_simplified_candidates(
+    /// Construct the decision tree for `candidates`. Caller must ensure that no candidate in
+    /// `candidates` starts with an or-pattern.
+    fn match_expanded_candidates(
         &mut self,
         span: Span,
         scrutinee_span: Span,
@@ -1438,7 +1487,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 // The first candidate has satisfied all its match pairs; we link it up and continue
                 // with the remaining candidates.
                 start_block = self.select_matched_candidate(first, start_block);
-                self.match_simplified_candidates(
+                self.match_expanded_candidates(
                     span,
                     scrutinee_span,
                     start_block,
@@ -1448,7 +1497,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
             candidates => {
                 // The first candidate has some unsatisfied match pairs; we proceed to do more tests.
-                self.test_candidates_with_or(
+                self.test_candidates(
                     span,
                     scrutinee_span,
                     candidates,
@@ -1495,16 +1544,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
 
         candidate.pre_binding_block = Some(start_block);
         let otherwise_block = self.cfg.start_new_block();
-        if candidate.has_guard {
-            // Create the otherwise block for this candidate, which is the
-            // pre-binding block for the next candidate.
-            candidate.otherwise_block = Some(otherwise_block);
-        }
+        // Create the otherwise block for this candidate, which is the
+        // pre-binding block for the next candidate.
+        candidate.otherwise_block = Some(otherwise_block);
         otherwise_block
     }
 
-    /// Tests a candidate where there are only or-patterns left to test, or
-    /// forwards to [Builder::test_candidates].
+    /// Simplify subcandidates and process any leftover match pairs. The candidate should have been
+    /// expanded with `create_or_subcandidates`.
     ///
     /// Given a pattern `(P | Q, R | S)` we (in principle) generate a CFG like
     /// so:
@@ -1556,84 +1603,56 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     ///      |
     ///     ...
     /// ```
-    fn test_candidates_with_or(
+    fn finalize_or_candidate(
         &mut self,
         span: Span,
         scrutinee_span: Span,
-        candidates: &mut [&mut Candidate<'_, 'tcx>],
-        start_block: BasicBlock,
-        otherwise_block: BasicBlock,
+        candidate: &mut Candidate<'_, 'tcx>,
     ) {
-        let (first_candidate, remaining_candidates) = candidates.split_first_mut().unwrap();
-        assert!(first_candidate.subcandidates.is_empty());
-        if !matches!(first_candidate.match_pairs[0].test_case, TestCase::Or { .. }) {
-            self.test_candidates(span, scrutinee_span, candidates, start_block, otherwise_block);
+        if candidate.subcandidates.is_empty() {
             return;
         }
 
-        let first_match_pair = first_candidate.match_pairs.remove(0);
-        let remaining_match_pairs = mem::take(&mut first_candidate.match_pairs);
-        let remainder_start = self.cfg.start_new_block();
-        // Test the alternatives of this or-pattern.
-        self.test_or_pattern(first_candidate, start_block, remainder_start, first_match_pair);
+        self.merge_trivial_subcandidates(candidate);
 
-        if !remaining_match_pairs.is_empty() {
+        if !candidate.match_pairs.is_empty() {
             // If more match pairs remain, test them after each subcandidate.
             // We could add them to the or-candidates before the call to `test_or_pattern` but this
             // would make it impossible to detect simplifiable or-patterns. That would guarantee
             // exponentially large CFGs for cases like `(1 | 2, 3 | 4, ...)`.
-            first_candidate.visit_leaves(|leaf_candidate| {
+            let mut last_otherwise = None;
+            candidate.visit_leaves(|leaf_candidate| {
+                last_otherwise = leaf_candidate.otherwise_block;
+            });
+            let remaining_match_pairs = mem::take(&mut candidate.match_pairs);
+            candidate.visit_leaves(|leaf_candidate| {
                 assert!(leaf_candidate.match_pairs.is_empty());
                 leaf_candidate.match_pairs.extend(remaining_match_pairs.iter().cloned());
                 let or_start = leaf_candidate.pre_binding_block.unwrap();
-                // In a case like `(a | b, c | d)`, if `a` succeeds and `c | d` fails, we know `(b,
-                // c | d)` will fail too. If there is no guard, we skip testing of `b` by branching
-                // directly to `remainder_start`. If there is a guard, we have to try `(b, c | d)`.
-                let or_otherwise = leaf_candidate.otherwise_block.unwrap_or(remainder_start);
-                self.test_candidates_with_or(
+                // In a case like `(P | Q, R | S)`, if `P` succeeds and `R | S` fails, we know `(Q,
+                // R | S)` will fail too. If there is no guard, we skip testing of `Q` by branching
+                // directly to `last_otherwise`. If there is a guard,
+                // `leaf_candidate.otherwise_block` can be reached by guard failure as well, so we
+                // can't skip `Q`.
+                let or_otherwise = if leaf_candidate.has_guard {
+                    leaf_candidate.otherwise_block.unwrap()
+                } else {
+                    last_otherwise.unwrap()
+                };
+                self.match_candidates(
                     span,
                     scrutinee_span,
-                    &mut [leaf_candidate],
                     or_start,
                     or_otherwise,
+                    &mut [leaf_candidate],
                 );
             });
         }
-
-        // Test the remaining candidates.
-        self.match_candidates(
-            span,
-            scrutinee_span,
-            remainder_start,
-            otherwise_block,
-            remaining_candidates,
-        );
-    }
-
-    #[instrument(skip(self, start_block, otherwise_block, candidate, match_pair), level = "debug")]
-    fn test_or_pattern<'pat>(
-        &mut self,
-        candidate: &mut Candidate<'pat, 'tcx>,
-        start_block: BasicBlock,
-        otherwise_block: BasicBlock,
-        match_pair: MatchPair<'pat, 'tcx>,
-    ) {
-        let or_span = match_pair.pattern.span;
-        self.create_or_subcandidates(candidate, match_pair);
-        let mut or_candidate_refs: Vec<_> = candidate.subcandidates.iter_mut().collect();
-        self.match_candidates(
-            or_span,
-            or_span,
-            start_block,
-            otherwise_block,
-            &mut or_candidate_refs,
-        );
-        self.merge_trivial_subcandidates(candidate);
     }
 
     /// Given a match-pair that corresponds to an or-pattern, expand each subpattern into a new
     /// subcandidate. Any candidate that has been expanded that way should be passed to
-    /// `merge_trivial_subcandidates` after its subcandidates have been processed.
+    /// `finalize_or_candidate` after its subcandidates have been processed.
     fn create_or_subcandidates<'pat>(
         &mut self,
         candidate: &mut Candidate<'pat, 'tcx>,
@@ -1651,8 +1670,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     }
 
     /// Try to merge all of the subcandidates of the given candidate into one. This avoids
-    /// exponentially large CFGs in cases like `(1 | 2, 3 | 4, ...)`. The or-pattern should have
-    /// been expanded with `create_or_subcandidates`.
+    /// exponentially large CFGs in cases like `(1 | 2, 3 | 4, ...)`. The candidate should have been
+    /// expanded with `create_or_subcandidates`.
     fn merge_trivial_subcandidates(&mut self, candidate: &mut Candidate<'_, 'tcx>) {
         if candidate.subcandidates.is_empty() || candidate.has_guard {
             // FIXME(or_patterns; matthewjasper) Don't give up if we have a guard.
@@ -1664,6 +1683,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             subcandidate.subcandidates.is_empty() && subcandidate.extra_data.is_empty()
         });
         if can_merge {
+            let mut last_otherwise = None;
             let any_matches = self.cfg.start_new_block();
             let or_span = candidate.or_span.take().unwrap();
             let source_info = self.source_info(or_span);
@@ -1674,8 +1694,11 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             for subcandidate in mem::take(&mut candidate.subcandidates) {
                 let or_block = subcandidate.pre_binding_block.unwrap();
                 self.cfg.goto(or_block, source_info, any_matches);
+                last_otherwise = subcandidate.otherwise_block;
             }
             candidate.pre_binding_block = Some(any_matches);
+            assert!(last_otherwise.is_some());
+            candidate.otherwise_block = last_otherwise;
         } else {
             // Never subcandidates may have a set of bindings inconsistent with their siblings,
             // which would break later code. So we filter them out. Note that we can't filter out

compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs

@@ -18,7 +18,8 @@
 
 use crate::MirPass;
 use rustc_middle::mir::coverage::CoverageKind;
-use rustc_middle::mir::{Body, BorrowKind, Rvalue, StatementKind, TerminatorKind};
+use rustc_middle::mir::{Body, BorrowKind, CastKind, Rvalue, StatementKind, TerminatorKind};
+use rustc_middle::ty::adjustment::PointerCoercion;
 use rustc_middle::ty::TyCtxt;
 
 pub struct CleanupPostBorrowck;
@@ -36,6 +37,22 @@ impl<'tcx> MirPass<'tcx> for CleanupPostBorrowck {
                         CoverageKind::BlockMarker { .. } | CoverageKind::SpanMarker { .. },
                     )
                     | StatementKind::FakeRead(..) => statement.make_nop(),
+                    StatementKind::Assign(box (
+                        _,
+                        Rvalue::Cast(
+                            ref mut cast_kind @ CastKind::PointerCoercion(
+                                PointerCoercion::ArrayToPointer
+                                | PointerCoercion::MutToConstPointer,
+                            ),
+                            ..,
+                        ),
+                    )) => {
+                        // BorrowCk needed to track whether these cases were coercions or casts,
+                        // to know whether to check lifetimes in their pointees,
+                        // but from now on that distinction doesn't matter,
+                        // so just make them ordinary pointer casts instead.
+                        *cast_kind = CastKind::PtrToPtr;
+                    }
                     _ => (),
                 }
             }

compiler/rustc_mir_transform/src/cost_checker.rs

@@ -1,3 +1,4 @@
+use rustc_middle::bug;
 use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
 use rustc_middle::ty::{self, ParamEnv, Ty, TyCtxt};
@@ -6,13 +7,16 @@ const INSTR_COST: usize = 5;
 const CALL_PENALTY: usize = 25;
 const LANDINGPAD_PENALTY: usize = 50;
 const RESUME_PENALTY: usize = 45;
+const LARGE_SWITCH_PENALTY: usize = 20;
+const CONST_SWITCH_BONUS: usize = 10;
 
 /// Verify that the callee body is compatible with the caller.
 #[derive(Clone)]
 pub(crate) struct CostChecker<'b, 'tcx> {
     tcx: TyCtxt<'tcx>,
     param_env: ParamEnv<'tcx>,
-    cost: usize,
+    penalty: usize,
+    bonus: usize,
     callee_body: &'b Body<'tcx>,
     instance: Option<ty::Instance<'tcx>>,
 }
@@ -24,11 +28,11 @@ impl<'b, 'tcx> CostChecker<'b, 'tcx> {
         instance: Option<ty::Instance<'tcx>>,
         callee_body: &'b Body<'tcx>,
     ) -> CostChecker<'b, 'tcx> {
-        CostChecker { tcx, param_env, callee_body, instance, cost: 0 }
+        CostChecker { tcx, param_env, callee_body, instance, penalty: 0, bonus: 0 }
     }
 
     pub fn cost(&self) -> usize {
-        self.cost
+        usize::saturating_sub(self.penalty, self.bonus)
     }
 
     fn instantiate_ty(&self, v: Ty<'tcx>) -> Ty<'tcx> {
@@ -41,36 +45,49 @@ impl<'b, 'tcx> CostChecker<'b, 'tcx> {
 }
 
 impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
-    fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) {
-        // Don't count StorageLive/StorageDead in the inlining cost.
+    fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) {
+        // Most costs are in rvalues and terminators, not in statements.
         match statement.kind {
-            StatementKind::StorageLive(_)
-            | StatementKind::StorageDead(_)
-            | StatementKind::Deinit(_)
-            | StatementKind::Nop => {}
-            _ => self.cost += INSTR_COST,
+            StatementKind::Intrinsic(ref ndi) => {
+                self.penalty += match **ndi {
+                    NonDivergingIntrinsic::Assume(..) => INSTR_COST,
+                    NonDivergingIntrinsic::CopyNonOverlapping(..) => CALL_PENALTY,
+                };
+            }
+            _ => self.super_statement(statement, location),
+        }
+    }
+
+    fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, _location: Location) {
+        match rvalue {
+            Rvalue::NullaryOp(NullOp::UbChecks, ..) if !self.tcx.sess.ub_checks() => {
+                // If this is in optimized MIR it's because it's used later,
+                // so if we don't need UB checks this session, give a bonus
+                // here to offset the cost of the call later.
+                self.bonus += CALL_PENALTY;
+            }
+            // These are essentially constants that didn't end up in an Operand,
+            // so treat them as also being free.
+            Rvalue::NullaryOp(..) => {}
+            _ => self.penalty += INSTR_COST,
         }
     }
 
     fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, _: Location) {
-        let tcx = self.tcx;
-        match terminator.kind {
-            TerminatorKind::Drop { ref place, unwind, .. } => {
+        match &terminator.kind {
+            TerminatorKind::Drop { place, unwind, .. } => {
                 // If the place doesn't actually need dropping, treat it like a regular goto.
-                let ty = self.instantiate_ty(place.ty(self.callee_body, tcx).ty);
-                if ty.needs_drop(tcx, self.param_env) {
-                    self.cost += CALL_PENALTY;
+                let ty = self.instantiate_ty(place.ty(self.callee_body, self.tcx).ty);
+                if ty.needs_drop(self.tcx, self.param_env) {
+                    self.penalty += CALL_PENALTY;
                     if let UnwindAction::Cleanup(_) = unwind {
-                        self.cost += LANDINGPAD_PENALTY;
+                        self.penalty += LANDINGPAD_PENALTY;
                     }
-                } else {
-                    self.cost += INSTR_COST;
                 }
             }
-            TerminatorKind::Call { func: Operand::Constant(ref f), unwind, .. } => {
-                let fn_ty = self.instantiate_ty(f.const_.ty());
-                self.cost += if let ty::FnDef(def_id, _) = *fn_ty.kind()
-                    && tcx.intrinsic(def_id).is_some()
+            TerminatorKind::Call { func, unwind, .. } => {
+                self.penalty += if let Some((def_id, ..)) = func.const_fn_def()
+                    && self.tcx.intrinsic(def_id).is_some()
                 {
                     // Don't give intrinsics the extra penalty for calls
                     INSTR_COST
@@ -78,23 +95,50 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
                     CALL_PENALTY
                 };
                 if let UnwindAction::Cleanup(_) = unwind {
-                    self.cost += LANDINGPAD_PENALTY;
+                    self.penalty += LANDINGPAD_PENALTY;
                 }
             }
-            TerminatorKind::Assert { unwind, .. } => {
-                self.cost += CALL_PENALTY;
+            TerminatorKind::SwitchInt { discr, targets } => {
+                if discr.constant().is_some() {
+                    // Not only will this become a `Goto`, but likely other
+                    // things will be removable as unreachable.
+                    self.bonus += CONST_SWITCH_BONUS;
+                } else if targets.all_targets().len() > 3 {
+                    // More than false/true/unreachable gets extra cost.
+                    self.penalty += LARGE_SWITCH_PENALTY;
+                } else {
+                    self.penalty += INSTR_COST;
+                }
+            }
+            TerminatorKind::Assert { unwind, msg, .. } => {
+                self.penalty +=
+                    if msg.is_optional_overflow_check() && !self.tcx.sess.overflow_checks() {
+                        INSTR_COST
+                    } else {
+                        CALL_PENALTY
+                    };
                 if let UnwindAction::Cleanup(_) = unwind {
-                    self.cost += LANDINGPAD_PENALTY;
+                    self.penalty += LANDINGPAD_PENALTY;
                 }
             }
-            TerminatorKind::UnwindResume => self.cost += RESUME_PENALTY,
+            TerminatorKind::UnwindResume => self.penalty += RESUME_PENALTY,
             TerminatorKind::InlineAsm { unwind, .. } => {
-                self.cost += INSTR_COST;
+                self.penalty += INSTR_COST;
                 if let UnwindAction::Cleanup(_) = unwind {
-                    self.cost += LANDINGPAD_PENALTY;
+                    self.penalty += LANDINGPAD_PENALTY;
                 }
             }
-            _ => self.cost += INSTR_COST,
+            TerminatorKind::Unreachable => {
+                self.bonus += INSTR_COST;
+            }
+            TerminatorKind::Goto { .. } | TerminatorKind::Return => {}
+            TerminatorKind::UnwindTerminate(..) => {}
+            kind @ (TerminatorKind::FalseUnwind { .. }
+            | TerminatorKind::FalseEdge { .. }
+            | TerminatorKind::Yield { .. }
+            | TerminatorKind::CoroutineDrop) => {
+                bug!("{kind:?} should not be in runtime MIR");
+            }
         }
     }
 }

compiler/rustc_mir_transform/src/ctfe_limit.rs

@@ -1,5 +1,6 @@
 //! A pass that inserts the `ConstEvalCounter` instruction into any blocks that have a back edge
 //! (thus indicating there is a loop in the CFG), or whose terminator is a function call.
+
 use crate::MirPass;
 
 use rustc_data_structures::graph::dominators::Dominators;

compiler/rustc_mir_transform/src/ffi_unwind_calls.rs

@@ -11,38 +11,6 @@ use rustc_target::spec::PanicStrategy;
 
 use crate::errors;
 
-/// Some of the functions declared as "may unwind" by `fn_can_unwind` can't actually unwind. In
-/// particular, `extern "C"` is still considered as can-unwind on stable, but we need to consider
-/// it cannot-unwind here. So below we check `fn_can_unwind() && abi_can_unwind()` before concluding
-/// that a function call can unwind.
-fn abi_can_unwind(abi: Abi) -> bool {
-    use Abi::*;
-    match abi {
-        C { unwind }
-        | System { unwind }
-        | Cdecl { unwind }
-        | Stdcall { unwind }
-        | Fastcall { unwind }
-        | Vectorcall { unwind }
-        | Thiscall { unwind }
-        | Aapcs { unwind }
-        | Win64 { unwind }
-        | SysV64 { unwind } => unwind,
-        PtxKernel
-        | Msp430Interrupt
-        | X86Interrupt
-        | EfiApi
-        | AvrInterrupt
-        | AvrNonBlockingInterrupt
-        | RiscvInterruptM
-        | RiscvInterruptS
-        | CCmseNonSecureCall
-        | Wasm
-        | Unadjusted => false,
-        RustIntrinsic | Rust | RustCall | RustCold => unreachable!(), // these ABIs are already skipped earlier
-    }
-}
-
 // Check if the body of this def_id can possibly leak a foreign unwind into Rust code.
 fn has_ffi_unwind_calls(tcx: TyCtxt<'_>, local_def_id: LocalDefId) -> bool {
     debug!("has_ffi_unwind_calls({local_def_id:?})");
@@ -103,7 +71,7 @@ fn has_ffi_unwind_calls(tcx: TyCtxt<'_>, local_def_id: LocalDefId) -> bool {
             _ => bug!("invalid callee of type {:?}", ty),
         };
 
-        if layout::fn_can_unwind(tcx, fn_def_id, sig.abi()) && abi_can_unwind(sig.abi()) {
+        if layout::fn_can_unwind(tcx, fn_def_id, sig.abi()) {
             // We have detected a call that can possibly leak foreign unwind.
             //
             // Because the function body itself can unwind, we are not aborting this function call

compiler/rustc_mir_transform/src/gvn.rs

@@ -571,11 +571,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                     let ret = self.ecx.ptr_to_ptr(&src, to).ok()?;
                     ret.into()
                 }
-                CastKind::PointerCoercion(
-                    ty::adjustment::PointerCoercion::MutToConstPointer
-                    | ty::adjustment::PointerCoercion::ArrayToPointer
-                    | ty::adjustment::PointerCoercion::UnsafeFnPointer,
-                ) => {
+                CastKind::PointerCoercion(ty::adjustment::PointerCoercion::UnsafeFnPointer) => {
                     let src = self.evaluated[value].as_ref()?;
                     let src = self.ecx.read_immediate(src).ok()?;
                     let to = self.ecx.layout_of(to).ok()?;
@@ -840,12 +836,8 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                 }
                 Value::BinaryOp(op, lhs, rhs)
             }
-            Rvalue::UnaryOp(op, ref mut arg) => {
-                let arg = self.simplify_operand(arg, location)?;
-                if let Some(value) = self.simplify_unary(op, arg) {
-                    return Some(value);
-                }
-                Value::UnaryOp(op, arg)
+            Rvalue::UnaryOp(op, ref mut arg_op) => {
+                return self.simplify_unary(op, arg_op, location);
             }
             Rvalue::Discriminant(ref mut place) => {
                 let place = self.simplify_place_value(place, location)?;
@@ -953,13 +945,8 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                 was_updated = true;
             }
 
-            if was_updated {
-                if let Some(const_) = self.try_as_constant(fields[0]) {
-                    field_ops[FieldIdx::ZERO] = Operand::Constant(Box::new(const_));
-                } else if let Some(local) = self.try_as_local(fields[0], location) {
-                    field_ops[FieldIdx::ZERO] = Operand::Copy(Place::from(local));
-                    self.reused_locals.insert(local);
-                }
+            if was_updated && let Some(op) = self.try_as_operand(fields[0], location) {
+                field_ops[FieldIdx::ZERO] = op;
             }
         }
 
@@ -969,11 +956,8 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             let first = fields[0];
             if fields.iter().all(|&v| v == first) {
                 let len = ty::Const::from_target_usize(self.tcx, fields.len().try_into().unwrap());
-                if let Some(const_) = self.try_as_constant(first) {
-                    *rvalue = Rvalue::Repeat(Operand::Constant(Box::new(const_)), len);
-                } else if let Some(local) = self.try_as_local(first, location) {
-                    *rvalue = Rvalue::Repeat(Operand::Copy(local.into()), len);
-                    self.reused_locals.insert(local);
+                if let Some(op) = self.try_as_operand(first, location) {
+                    *rvalue = Rvalue::Repeat(op, len);
                 }
                 return Some(self.insert(Value::Repeat(first, len)));
             }
@@ -983,8 +967,71 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
     }
 
     #[instrument(level = "trace", skip(self), ret)]
-    fn simplify_unary(&mut self, op: UnOp, value: VnIndex) -> Option<VnIndex> {
-        let value = match (op, self.get(value)) {
+    fn simplify_unary(
+        &mut self,
+        op: UnOp,
+        arg_op: &mut Operand<'tcx>,
+        location: Location,
+    ) -> Option<VnIndex> {
+        let mut arg_index = self.simplify_operand(arg_op, location)?;
+
+        // PtrMetadata doesn't care about *const vs *mut vs & vs &mut,
+        // so start by removing those distinctions so we can update the `Operand`
+        if op == UnOp::PtrMetadata {
+            let mut was_updated = false;
+            loop {
+                match self.get(arg_index) {
+                    // Pointer casts that preserve metadata, such as
+                    // `*const [i32]` <-> `*mut [i32]` <-> `*mut [f32]`.
+                    // It's critical that this not eliminate cases like
+                    // `*const [T]` -> `*const T` which remove metadata.
+                    // We run on potentially-generic MIR, though, so unlike codegen
+                    // we can't always know exactly what the metadata are.
+                    // Thankfully, equality on `ptr_metadata_ty_or_tail` gives us
+                    // what we need: `Ok(meta_ty)` if the metadata is known, or
+                    // `Err(tail_ty)` if not. Matching metadata is ok, but if
+                    // that's not known, then matching tail types is also ok,
+                    // allowing things like `*mut (?A, ?T)` <-> `*mut (?B, ?T)`.
+                    // FIXME: Would it be worth trying to normalize, rather than
+                    // passing the identity closure?  Or are the types in the
+                    // Cast realistically about as normalized as we can get anyway?
+                    Value::Cast { kind: CastKind::PtrToPtr, value: inner, from, to }
+                        if from
+                            .builtin_deref(true)
+                            .unwrap()
+                            .ptr_metadata_ty_or_tail(self.tcx, |t| t)
+                            == to
+                                .builtin_deref(true)
+                                .unwrap()
+                                .ptr_metadata_ty_or_tail(self.tcx, |t| t) =>
+                    {
+                        arg_index = *inner;
+                        was_updated = true;
+                        continue;
+                    }
+
+                    // `&mut *p`, `&raw *p`, etc don't change metadata.
+                    Value::Address { place, kind: _, provenance: _ }
+                        if let PlaceRef { local, projection: [PlaceElem::Deref] } =
+                            place.as_ref()
+                            && let Some(local_index) = self.locals[local] =>
+                    {
+                        arg_index = local_index;
+                        was_updated = true;
+                        continue;
+                    }
+
+                    _ => {
+                        if was_updated && let Some(op) = self.try_as_operand(arg_index, location) {
+                            *arg_op = op;
+                        }
+                        break;
+                    }
+                }
+            }
+        }
+
+        let value = match (op, self.get(arg_index)) {
             (UnOp::Not, Value::UnaryOp(UnOp::Not, inner)) => return Some(*inner),
             (UnOp::Neg, Value::UnaryOp(UnOp::Neg, inner)) => return Some(*inner),
             (UnOp::Not, Value::BinaryOp(BinOp::Eq, lhs, rhs)) => {
@@ -996,9 +1043,26 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             (UnOp::PtrMetadata, Value::Aggregate(AggregateTy::RawPtr { .. }, _, fields)) => {
                 return Some(fields[1]);
             }
-            _ => return None,
+            // We have an unsizing cast, which assigns the length to fat pointer metadata.
+            (
+                UnOp::PtrMetadata,
+                Value::Cast {
+                    kind: CastKind::PointerCoercion(ty::adjustment::PointerCoercion::Unsize),
+                    from,
+                    to,
+                    ..
+                },
+            ) if let ty::Slice(..) = to.builtin_deref(true).unwrap().kind()
+                && let ty::Array(_, len) = from.builtin_deref(true).unwrap().kind() =>
+            {
+                return self.insert_constant(Const::from_ty_const(
+                    *len,
+                    self.tcx.types.usize,
+                    self.tcx,
+                ));
+            }
+            _ => Value::UnaryOp(op, arg_index),
         };
-
         Some(self.insert(value))
     }
 
@@ -1164,10 +1228,10 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             }
         }
 
-        if let PtrToPtr | PointerCoercion(MutToConstPointer) = kind
+        if let PtrToPtr = kind
             && let Value::Cast { kind: inner_kind, value: inner_value, from: inner_from, to: _ } =
                 *self.get(value)
-            && let PtrToPtr | PointerCoercion(MutToConstPointer) = inner_kind
+            && let PtrToPtr = inner_kind
         {
             from = inner_from;
             value = inner_value;
@@ -1178,13 +1242,8 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             }
         }
 
-        if was_updated {
-            if let Some(const_) = self.try_as_constant(value) {
-                *operand = Operand::Constant(Box::new(const_));
-            } else if let Some(local) = self.try_as_local(value, location) {
-                *operand = Operand::Copy(local.into());
-                self.reused_locals.insert(local);
-            }
+        if was_updated && let Some(op) = self.try_as_operand(value, location) {
+            *operand = op;
         }
 
         Some(self.insert(Value::Cast { kind: *kind, value, from, to }))
@@ -1300,6 +1359,19 @@ fn op_to_prop_const<'tcx>(
 }
 
 impl<'tcx> VnState<'_, 'tcx> {
+    /// If either [`Self::try_as_constant`] as [`Self::try_as_local`] succeeds,
+    /// returns that result as an [`Operand`].
+    fn try_as_operand(&mut self, index: VnIndex, location: Location) -> Option<Operand<'tcx>> {
+        if let Some(const_) = self.try_as_constant(index) {
+            Some(Operand::Constant(Box::new(const_)))
+        } else if let Some(local) = self.try_as_local(index, location) {
+            self.reused_locals.insert(local);
+            Some(Operand::Copy(local.into()))
+        } else {
+            None
+        }
+    }
+
     /// If `index` is a `Value::Constant`, return the `Constant` to be put in the MIR.
     fn try_as_constant(&mut self, index: VnIndex) -> Option<ConstOperand<'tcx>> {
         // This was already constant in MIR, do not change it.

compiler/rustc_mir_transform/src/inline.rs

@@ -1,4 +1,5 @@
-//! Inlining pass for MIR functions
+//! Inlining pass for MIR functions.
+
 use crate::deref_separator::deref_finder;
 use rustc_attr::InlineAttr;
 use rustc_hir::def::DefKind;

compiler/rustc_mir_transform/src/lib.rs

@@ -51,11 +51,11 @@ mod abort_unwinding_calls;
 mod add_call_guards;
 mod add_moves_for_packed_drops;
 mod add_retag;
+mod add_subtyping_projections;
+mod check_alignment;
 mod check_const_item_mutation;
 mod check_packed_ref;
-mod remove_place_mention;
 // This pass is public to allow external drivers to perform MIR cleanup
-mod add_subtyping_projections;
 pub mod cleanup_post_borrowck;
 mod copy_prop;
 mod coroutine;
@@ -88,12 +88,12 @@ mod lower_slice_len;
 mod match_branches;
 mod mentioned_items;
 mod multiple_return_terminators;
-mod normalize_array_len;
 mod nrvo;
 mod prettify;
 mod promote_consts;
 mod ref_prop;
 mod remove_noop_landing_pads;
+mod remove_place_mention;
 mod remove_storage_markers;
 mod remove_uninit_drops;
 mod remove_unneeded_drops;
@@ -103,7 +103,6 @@ mod reveal_all;
 mod shim;
 mod ssa;
 // This pass is public to allow external drivers to perform MIR cleanup
-mod check_alignment;
 pub mod simplify;
 mod simplify_branches;
 mod simplify_comparison_integral;
@@ -581,9 +580,6 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
             &o1(simplify::SimplifyCfg::AfterUnreachableEnumBranching),
             // Inlining may have introduced a lot of redundant code and a large move pattern.
             // Now, we need to shrink the generated MIR.
-
-            // Has to run after `slice::len` lowering
-            &normalize_array_len::NormalizeArrayLen,
             &ref_prop::ReferencePropagation,
             &sroa::ScalarReplacementOfAggregates,
             &match_branches::MatchBranchSimplification,

compiler/rustc_mir_transform/src/lint.rs

@@ -1,6 +1,7 @@
 //! This pass statically detects code which has undefined behaviour or is likely to be erroneous.
 //! It can be used to locate problems in MIR building or optimizations. It assumes that all code
 //! can be executed, so it has false positives.
+
 use rustc_data_structures::fx::FxHashSet;
 use rustc_index::bit_set::BitSet;
 use rustc_middle::mir::visit::{PlaceContext, Visitor};

compiler/rustc_mir_transform/src/lower_slice_len.rs

@@ -1,10 +1,9 @@
-//! This pass lowers calls to core::slice::len to just Len op.
+//! This pass lowers calls to core::slice::len to just PtrMetadata op.
 //! It should run before inlining!
 
 use rustc_hir::def_id::DefId;
-use rustc_index::IndexSlice;
 use rustc_middle::mir::*;
-use rustc_middle::ty::{self, TyCtxt};
+use rustc_middle::ty::TyCtxt;
 
 pub struct LowerSliceLenCalls;
 
@@ -29,16 +28,11 @@ pub fn lower_slice_len_calls<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
     let basic_blocks = body.basic_blocks.as_mut_preserves_cfg();
     for block in basic_blocks {
         // lower `<[_]>::len` calls
-        lower_slice_len_call(tcx, block, &body.local_decls, slice_len_fn_item_def_id);
+        lower_slice_len_call(block, slice_len_fn_item_def_id);
     }
 }
 
-fn lower_slice_len_call<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    block: &mut BasicBlockData<'tcx>,
-    local_decls: &IndexSlice<Local, LocalDecl<'tcx>>,
-    slice_len_fn_item_def_id: DefId,
-) {
+fn lower_slice_len_call<'tcx>(block: &mut BasicBlockData<'tcx>, slice_len_fn_item_def_id: DefId) {
     let terminator = block.terminator();
     if let TerminatorKind::Call {
         func,
@@ -50,19 +44,17 @@ fn lower_slice_len_call<'tcx>(
     } = &terminator.kind
         // some heuristics for fast rejection
         && let [arg] = &args[..]
-        && let Some(arg) = arg.node.place()
-        && let ty::FnDef(fn_def_id, _) = func.ty(local_decls, tcx).kind()
-        && *fn_def_id == slice_len_fn_item_def_id
+        && let Some((fn_def_id, _)) = func.const_fn_def()
+        && fn_def_id == slice_len_fn_item_def_id
     {
         // perform modifications from something like:
         //     _5 = core::slice::<impl [u8]>::len(move _6) -> bb1
         // into:
-        //     _5 = Len(*_6)
+        //     _5 = PtrMetadata(move _6)
         //     goto bb1
 
         // make new RValue for Len
-        let deref_arg = tcx.mk_place_deref(arg);
-        let r_value = Rvalue::Len(deref_arg);
+        let r_value = Rvalue::UnaryOp(UnOp::PtrMetadata, arg.node.clone());
         let len_statement_kind = StatementKind::Assign(Box::new((*destination, r_value)));
         let add_statement =
             Statement { kind: len_statement_kind, source_info: terminator.source_info };

compiler/rustc_mir_transform/src/normalize_array_len.rs

@@ -1,103 +0,0 @@
-//! This pass eliminates casting of arrays into slices when their length
-//! is taken using `.len()` method. Handy to preserve information in MIR for const prop
-
-use crate::ssa::SsaLocals;
-use rustc_index::IndexVec;
-use rustc_middle::mir::visit::*;
-use rustc_middle::mir::*;
-use rustc_middle::ty::{self, TyCtxt};
-
-pub struct NormalizeArrayLen;
-
-impl<'tcx> MirPass<'tcx> for NormalizeArrayLen {
-    fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
-        sess.mir_opt_level() >= 3
-    }
-
-    #[instrument(level = "trace", skip(self, tcx, body))]
-    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        debug!(def_id = ?body.source.def_id());
-        normalize_array_len_calls(tcx, body)
-    }
-}
-
-fn normalize_array_len_calls<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-    let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id());
-    let ssa = SsaLocals::new(tcx, body, param_env);
-
-    let slice_lengths = compute_slice_length(tcx, &ssa, body);
-    debug!(?slice_lengths);
-
-    Replacer { tcx, slice_lengths }.visit_body_preserves_cfg(body);
-}
-
-fn compute_slice_length<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    ssa: &SsaLocals,
-    body: &Body<'tcx>,
-) -> IndexVec<Local, Option<ty::Const<'tcx>>> {
-    let mut slice_lengths = IndexVec::from_elem(None, &body.local_decls);
-
-    for (local, rvalue, _) in ssa.assignments(body) {
-        match rvalue {
-            Rvalue::Cast(
-                CastKind::PointerCoercion(ty::adjustment::PointerCoercion::Unsize),
-                operand,
-                cast_ty,
-            ) => {
-                let operand_ty = operand.ty(body, tcx);
-                debug!(?operand_ty);
-                if let Some(operand_ty) = operand_ty.builtin_deref(true)
-                    && let ty::Array(_, len) = operand_ty.kind()
-                    && let Some(cast_ty) = cast_ty.builtin_deref(true)
-                    && let ty::Slice(..) = cast_ty.kind()
-                {
-                    slice_lengths[local] = Some(*len);
-                }
-            }
-            // The length information is stored in the fat pointer, so we treat `operand` as a value.
-            Rvalue::Use(operand) => {
-                if let Some(rhs) = operand.place()
-                    && let Some(rhs) = rhs.as_local()
-                {
-                    slice_lengths[local] = slice_lengths[rhs];
-                }
-            }
-            // The length information is stored in the fat pointer.
-            // Reborrowing copies length information from one pointer to the other.
-            Rvalue::Ref(_, _, rhs) | Rvalue::AddressOf(_, rhs) => {
-                if let [PlaceElem::Deref] = rhs.projection[..] {
-                    slice_lengths[local] = slice_lengths[rhs.local];
-                }
-            }
-            _ => {}
-        }
-    }
-
-    slice_lengths
-}
-
-struct Replacer<'tcx> {
-    tcx: TyCtxt<'tcx>,
-    slice_lengths: IndexVec<Local, Option<ty::Const<'tcx>>>,
-}
-
-impl<'tcx> MutVisitor<'tcx> for Replacer<'tcx> {
-    fn tcx(&self) -> TyCtxt<'tcx> {
-        self.tcx
-    }
-
-    fn visit_rvalue(&mut self, rvalue: &mut Rvalue<'tcx>, loc: Location) {
-        if let Rvalue::Len(place) = rvalue
-            && let [PlaceElem::Deref] = &place.projection[..]
-            && let Some(len) = self.slice_lengths[place.local]
-        {
-            *rvalue = Rvalue::Use(Operand::Constant(Box::new(ConstOperand {
-                span: rustc_span::DUMMY_SP,
-                user_ty: None,
-                const_: Const::from_ty_const(len, self.tcx.types.usize, self.tcx),
-            })));
-        }
-        self.super_rvalue(rvalue, loc);
-    }
-}

compiler/rustc_mir_transform/src/validate.rs

@@ -1116,12 +1116,17 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                     UnOp::PtrMetadata => {
                         if !matches!(self.mir_phase, MirPhase::Runtime(_)) {
                             // It would probably be fine to support this in earlier phases,
-                            // but at the time of writing it's only ever introduced from intrinsic lowering,
+                            // but at the time of writing it's only ever introduced from intrinsic lowering
+                            // or other runtime-phase optimization passes,
                             // so earlier things can just `bug!` on it.
                             self.fail(location, "PtrMetadata should be in runtime MIR only");
                         }
 
-                        check_kinds!(a, "Cannot PtrMetadata non-pointer type {:?}", ty::RawPtr(..));
+                        check_kinds!(
+                            a,
+                            "Cannot PtrMetadata non-pointer non-reference type {:?}",
+                            ty::RawPtr(..) | ty::Ref(..)
+                        );
                     }
                 }
             }
@@ -1188,6 +1193,9 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                             "CastKind::{kind:?} output must be a raw const pointer, not {:?}",
                             ty::RawPtr(_, Mutability::Not)
                         );
+                        if self.mir_phase >= MirPhase::Analysis(AnalysisPhase::PostCleanup) {
+                            self.fail(location, format!("After borrowck, MIR disallows {kind:?}"));
+                        }
                     }
                     CastKind::PointerCoercion(PointerCoercion::ArrayToPointer) => {
                         // FIXME: Check pointee types
@@ -1201,6 +1209,9 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                             "CastKind::{kind:?} output must be a raw pointer, not {:?}",
                             ty::RawPtr(..)
                         );
+                        if self.mir_phase >= MirPhase::Analysis(AnalysisPhase::PostCleanup) {
+                            self.fail(location, format!("After borrowck, MIR disallows {kind:?}"));
+                        }
                     }
                     CastKind::PointerCoercion(PointerCoercion::Unsize) => {
                         // This is used for all `CoerceUnsized` types,
@@ -1212,7 +1223,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                         if !input_valid || !target_valid {
                             self.fail(
                                 location,
-                                format!("Wrong cast kind {kind:?} for the type {op_ty}",),
+                                format!("Wrong cast kind {kind:?} for the type {op_ty}"),
                             );
                         }
                     }

compiler/rustc_next_trait_solver/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 # tidy-alphabetical-start
 bitflags = "2.4.1"
 derivative = "2.2.0"
-rustc_ast_ir = { path = "../rustc_ast_ir" }
+rustc_ast_ir = { path = "../rustc_ast_ir", default-features = false }
 rustc_data_structures = { path = "../rustc_data_structures", optional = true }
 rustc_index = { path = "../rustc_index", default-features = false }
 rustc_macros = { path = "../rustc_macros", optional = true }

compiler/rustc_next_trait_solver/src/lib.rs

@@ -4,8 +4,6 @@
 //! but were uplifted in the process of making the new trait solver generic.
 //! So if you got to this crate from the old solver, it's totally normal.
 
-#![feature(let_chains)]
-
 pub mod canonicalizer;
 pub mod infcx;
 pub mod resolve;

compiler/rustc_next_trait_solver/src/solve/assembly/structural_traits.rs

@@ -2,7 +2,7 @@
 //! traits, `Copy`/`Clone`.
 
 use rustc_ast_ir::{Movability, Mutability};
-use rustc_data_structures::fx::FxHashMap;
+use rustc_type_ir::data_structures::HashMap;
 use rustc_type_ir::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable};
 use rustc_type_ir::inherent::*;
 use rustc_type_ir::lang_items::TraitSolverLangItem;
@@ -304,9 +304,10 @@ pub(in crate::solve) fn extract_tupled_inputs_and_output_from_callable<I: Intern
             let kind_ty = args.kind_ty();
             let sig = args.coroutine_closure_sig().skip_binder();
 
-            let coroutine_ty = if let Some(closure_kind) = kind_ty.to_opt_closure_kind()
-                && !args.tupled_upvars_ty().is_ty_var()
-            {
+            // FIXME: let_chains
+            let kind = kind_ty.to_opt_closure_kind();
+            let coroutine_ty = if kind.is_some() && !args.tupled_upvars_ty().is_ty_var() {
+                let closure_kind = kind.unwrap();
                 if !closure_kind.extends(goal_kind) {
                     return Err(NoSolution);
                 }
@@ -411,10 +412,11 @@ pub(in crate::solve) fn extract_tupled_inputs_and_output_from_async_callable<I:
             let kind_ty = args.kind_ty();
             let sig = args.coroutine_closure_sig().skip_binder();
             let mut nested = vec![];
-            let coroutine_ty = if let Some(closure_kind) = kind_ty.to_opt_closure_kind()
-                && !args.tupled_upvars_ty().is_ty_var()
-            {
-                if !closure_kind.extends(goal_kind) {
+
+            // FIXME: let_chains
+            let kind = kind_ty.to_opt_closure_kind();
+            let coroutine_ty = if kind.is_some() && !args.tupled_upvars_ty().is_ty_var() {
+                if !kind.unwrap().extends(goal_kind) {
                     return Err(NoSolution);
                 }
 
@@ -683,7 +685,7 @@ where
         );
     }
 
-    let mut replace_projection_with = FxHashMap::default();
+    let mut replace_projection_with = HashMap::default();
     for bound in object_bounds {
         if let ty::ExistentialPredicate::Projection(proj) = bound.skip_binder() {
             let proj = proj.with_self_ty(tcx, trait_ref.self_ty());
@@ -713,7 +715,7 @@ where
 struct ReplaceProjectionWith<'a, Infcx: SolverDelegate<Interner = I>, I: Interner> {
     ecx: &'a EvalCtxt<'a, Infcx>,
     param_env: I::ParamEnv,
-    mapping: FxHashMap<I::DefId, ty::Binder<I, ty::ProjectionPredicate<I>>>,
+    mapping: HashMap<I::DefId, ty::Binder<I, ty::ProjectionPredicate<I>>>,
     nested: Vec<Goal<I, I::Predicate>>,
 }
 
@@ -725,24 +727,28 @@ impl<Infcx: SolverDelegate<Interner = I>, I: Interner> TypeFolder<I>
     }
 
     fn fold_ty(&mut self, ty: I::Ty) -> I::Ty {
-        if let ty::Alias(ty::Projection, alias_ty) = ty.kind()
-            && let Some(replacement) = self.mapping.get(&alias_ty.def_id)
-        {
-            // We may have a case where our object type's projection bound is higher-ranked,
-            // but the where clauses we instantiated are not. We can solve this by instantiating
-            // the binder at the usage site.
-            let proj = self.ecx.instantiate_binder_with_infer(*replacement);
-            // FIXME: Technically this equate could be fallible...
-            self.nested.extend(
-                self.ecx
-                    .eq_and_get_goals(
-                        self.param_env,
-                        alias_ty,
-                        proj.projection_term.expect_ty(self.ecx.interner()),
-                    )
-                    .expect("expected to be able to unify goal projection with dyn's projection"),
-            );
-            proj.term.expect_ty()
+        if let ty::Alias(ty::Projection, alias_ty) = ty.kind() {
+            if let Some(replacement) = self.mapping.get(&alias_ty.def_id) {
+                // We may have a case where our object type's projection bound is higher-ranked,
+                // but the where clauses we instantiated are not. We can solve this by instantiating
+                // the binder at the usage site.
+                let proj = self.ecx.instantiate_binder_with_infer(*replacement);
+                // FIXME: Technically this equate could be fallible...
+                self.nested.extend(
+                    self.ecx
+                        .eq_and_get_goals(
+                            self.param_env,
+                            alias_ty,
+                            proj.projection_term.expect_ty(self.ecx.interner()),
+                        )
+                        .expect(
+                            "expected to be able to unify goal projection with dyn's projection",
+                        ),
+                );
+                proj.term.expect_ty()
+            } else {
+                ty.super_fold_with(self)
+            }
         } else {
             ty.super_fold_with(self)
         }

compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs

@@ -1,7 +1,8 @@
 use std::ops::ControlFlow;
 
-use rustc_data_structures::stack::ensure_sufficient_stack;
+#[cfg(feature = "nightly")]
 use rustc_macros::{HashStable_NoContext, TyDecodable, TyEncodable};
+use rustc_type_ir::data_structures::ensure_sufficient_stack;
 use rustc_type_ir::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable};
 use rustc_type_ir::inherent::*;
 use rustc_type_ir::relate::Relate;
@@ -88,7 +89,7 @@ where
 #[derive(derivative::Derivative)]
 #[derivative(Clone(bound = ""), Debug(bound = ""), Default(bound = ""))]
 #[derive(TypeVisitable_Generic, TypeFoldable_Generic, Lift_Generic)]
-#[derive(TyDecodable, TyEncodable, HashStable_NoContext)]
+#[cfg_attr(feature = "nightly", derive(TyDecodable, TyEncodable, HashStable_NoContext))]
 // FIXME: This can be made crate-private once `EvalCtxt` also lives in this crate.
 pub struct NestedGoals<I: Interner> {
     /// These normalizes-to goals are treated specially during the evaluation
@@ -116,7 +117,8 @@ impl<I: Interner> NestedGoals<I> {
     }
 }
 
-#[derive(PartialEq, Eq, Debug, Hash, HashStable_NoContext, Clone, Copy)]
+#[derive(PartialEq, Eq, Debug, Hash, Clone, Copy)]
+#[cfg_attr(feature = "nightly", derive(HashStable_NoContext))]
 pub enum GenerateProofTree {
     Yes,
     No,
@@ -689,14 +691,15 @@ where
             fn visit_ty(&mut self, t: I::Ty) -> Self::Result {
                 match t.kind() {
                     ty::Infer(ty::TyVar(vid)) => {
-                        if let ty::TermKind::Ty(term) = self.term.kind()
-                            && let ty::Infer(ty::TyVar(term_vid)) = term.kind()
-                            && self.infcx.root_ty_var(vid) == self.infcx.root_ty_var(term_vid)
-                        {
-                            ControlFlow::Break(())
-                        } else {
-                            self.check_nameable(self.infcx.universe_of_ty(vid).unwrap())
+                        if let ty::TermKind::Ty(term) = self.term.kind() {
+                            if let ty::Infer(ty::TyVar(term_vid)) = term.kind() {
+                                if self.infcx.root_ty_var(vid) == self.infcx.root_ty_var(term_vid) {
+                                    return ControlFlow::Break(());
+                                }
+                            }
                         }
+
+                        self.check_nameable(self.infcx.universe_of_ty(vid).unwrap())
                     }
                     ty::Placeholder(p) => self.check_nameable(p.universe()),
                     _ => {
@@ -712,14 +715,18 @@ where
             fn visit_const(&mut self, c: I::Const) -> Self::Result {
                 match c.kind() {
                     ty::ConstKind::Infer(ty::InferConst::Var(vid)) => {
-                        if let ty::TermKind::Const(term) = self.term.kind()
-                            && let ty::ConstKind::Infer(ty::InferConst::Var(term_vid)) = term.kind()
-                            && self.infcx.root_const_var(vid) == self.infcx.root_const_var(term_vid)
-                        {
-                            ControlFlow::Break(())
-                        } else {
-                            self.check_nameable(self.infcx.universe_of_ct(vid).unwrap())
+                        if let ty::TermKind::Const(term) = self.term.kind() {
+                            if let ty::ConstKind::Infer(ty::InferConst::Var(term_vid)) = term.kind()
+                            {
+                                if self.infcx.root_const_var(vid)
+                                    == self.infcx.root_const_var(term_vid)
+                                {
+                                    return ControlFlow::Break(());
+                                }
+                            }
                         }
+
+                        self.check_nameable(self.infcx.universe_of_ct(vid).unwrap())
                     }
                     ty::ConstKind::Placeholder(p) => self.check_nameable(p.universe()),
                     _ => {

compiler/rustc_next_trait_solver/src/solve/search_graph.rs

@@ -1,7 +1,7 @@
 use std::mem;
 
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_index::{Idx, IndexVec};
+use rustc_type_ir::data_structures::{HashMap, HashSet};
 use rustc_type_ir::inherent::*;
 use rustc_type_ir::Interner;
 use tracing::debug;
@@ -17,6 +17,7 @@ pub struct SolverLimit(usize);
 
 rustc_index::newtype_index! {
     #[orderable]
+    #[gate_rustc_only]
     pub struct StackDepth {}
 }
 
@@ -70,7 +71,7 @@ struct StackEntry<I: Interner> {
     /// C :- D
     /// D :- C
     /// ```
-    cycle_participants: FxHashSet<CanonicalInput<I>>,
+    cycle_participants: HashSet<CanonicalInput<I>>,
     /// Starts out as `None` and gets set when rerunning this
     /// goal in case we encounter a cycle.
     provisional_result: Option<QueryResult<I>>,
@@ -126,7 +127,7 @@ pub(super) struct SearchGraph<I: Interner> {
     ///
     /// An element is *deeper* in the stack if its index is *lower*.
     stack: IndexVec<StackDepth, StackEntry<I>>,
-    provisional_cache: FxHashMap<CanonicalInput<I>, ProvisionalCacheEntry<I>>,
+    provisional_cache: HashMap<CanonicalInput<I>, ProvisionalCacheEntry<I>>,
 }
 
 impl<I: Interner> SearchGraph<I> {
@@ -227,13 +228,17 @@ impl<I: Interner> SearchGraph<I> {
     }
 
     fn clear_dependent_provisional_results(
-        provisional_cache: &mut FxHashMap<CanonicalInput<I>, ProvisionalCacheEntry<I>>,
+        provisional_cache: &mut HashMap<CanonicalInput<I>, ProvisionalCacheEntry<I>>,
         head: StackDepth,
     ) {
         #[allow(rustc::potential_query_instability)]
         provisional_cache.retain(|_, entry| {
-            entry.with_coinductive_stack.take_if(|p| p.head == head);
-            entry.with_inductive_stack.take_if(|p| p.head == head);
+            if entry.with_coinductive_stack.as_ref().is_some_and(|p| p.head == head) {
+                entry.with_coinductive_stack.take();
+            }
+            if entry.with_inductive_stack.as_ref().is_some_and(|p| p.head == head) {
+                entry.with_inductive_stack.take();
+            }
             !entry.is_empty()
         });
     }

compiler/rustc_next_trait_solver/src/solve/trait_goals.rs

@@ -1,7 +1,7 @@
 //! Dealing with trait goals, i.e. `T: Trait<'a, U>`.
 
 use rustc_ast_ir::Movability;
-use rustc_data_structures::fx::FxIndexSet;
+use rustc_type_ir::data_structures::IndexSet;
 use rustc_type_ir::inherent::*;
 use rustc_type_ir::lang_items::TraitSolverLangItem;
 use rustc_type_ir::visit::TypeVisitableExt as _;
@@ -821,7 +821,7 @@ where
         // We may upcast to auto traits that are either explicitly listed in
         // the object type's bounds, or implied by the principal trait ref's
         // supertraits.
-        let a_auto_traits: FxIndexSet<I::DefId> = a_data
+        let a_auto_traits: IndexSet<I::DefId> = a_data
             .auto_traits()
             .into_iter()
             .chain(a_data.principal_def_id().into_iter().flat_map(|principal_def_id| {

compiler/rustc_parse/messages.ftl

@@ -133,6 +133,8 @@ parse_dot_dot_dot_for_remaining_fields = expected field pattern, found `{$token_
 parse_dot_dot_dot_range_to_pattern_not_allowed = range-to patterns with `...` are not allowed
     .suggestion = use `..=` instead
 
+parse_dot_dot_range_attribute = attributes are not allowed on range expressions starting with `..`
+
 parse_dotdotdot = unexpected token: `...`
     .suggest_exclusive_range = use `..` for an exclusive range
     .suggest_inclusive_range = or `..=` for an inclusive range

compiler/rustc_parse/src/errors.rs

@@ -2699,12 +2699,13 @@ pub(crate) struct SingleColonImportPath {
 
 #[derive(Diagnostic)]
 #[diag(parse_bad_item_kind)]
-#[help]
 pub(crate) struct BadItemKind {
     #[primary_span]
     pub span: Span,
     pub descr: &'static str,
     pub ctx: &'static str,
+    #[help]
+    pub help: Option<()>,
 }
 
 #[derive(Diagnostic)]
@@ -2989,3 +2990,10 @@ pub(crate) struct ExprRArrowCall {
     #[suggestion(style = "short", applicability = "machine-applicable", code = ".")]
     pub span: Span,
 }
+
+#[derive(Diagnostic)]
+#[diag(parse_dot_dot_range_attribute)]
+pub(crate) struct DotDotRangeAttribute {
+    #[primary_span]
+    pub span: Span,
+}

compiler/rustc_parse/src/parser/attr_wrapper.rs

@@ -15,7 +15,7 @@ use std::ops::Range;
 /// for the attribute target. This allows us to perform cfg-expansion on
 /// a token stream before we invoke a derive proc-macro.
 ///
-/// This wrapper prevents direct access to the underlying `ast::AttrVec>`.
+/// This wrapper prevents direct access to the underlying `ast::AttrVec`.
 /// Parsing code can only get access to the underlying attributes
 /// by passing an `AttrWrapper` to `collect_tokens_trailing_tokens`.
 /// This makes it difficult to accidentally construct an AST node
@@ -177,6 +177,10 @@ impl<'a> Parser<'a> {
     /// into a `LazyAttrTokenStream`, and returned along with the result
     /// of the callback.
     ///
+    /// The `attrs` passed in are in `AttrWrapper` form, which is opaque. The
+    /// `AttrVec` within is passed to `f`. See the comment on `AttrWrapper` for
+    /// details.
+    ///
     /// Note: If your callback consumes an opening delimiter
     /// (including the case where you call `collect_tokens`
     /// when the current token is an opening delimiter),

compiler/rustc_parse/src/parser/diagnostics.rs

@@ -2499,7 +2499,8 @@ impl<'a> Parser<'a> {
     /// wrapped in braces.
     pub(super) fn handle_unambiguous_unbraced_const_arg(&mut self) -> PResult<'a, P<Expr>> {
         let start = self.token.span;
-        let expr = self.parse_expr_res(Restrictions::CONST_EXPR, None).map_err(|mut err| {
+        let attrs = self.parse_outer_attributes()?;
+        let expr = self.parse_expr_res(Restrictions::CONST_EXPR, attrs).map_err(|mut err| {
             err.span_label(
                 start.shrink_to_lo(),
                 "while parsing a const generic argument starting here",
@@ -2621,7 +2622,10 @@ impl<'a> Parser<'a> {
         if is_op_or_dot {
             self.bump();
         }
-        match self.parse_expr_res(Restrictions::CONST_EXPR, None) {
+        match (|| {
+            let attrs = self.parse_outer_attributes()?;
+            self.parse_expr_res(Restrictions::CONST_EXPR, attrs)
+        })() {
             Ok(expr) => {
                 // Find a mistake like `MyTrait<Assoc == S::Assoc>`.
                 if token::EqEq == snapshot.token.kind {
@@ -2675,7 +2679,10 @@ impl<'a> Parser<'a> {
         &mut self,
         mut snapshot: SnapshotParser<'a>,
     ) -> Option<P<ast::Expr>> {
-        match snapshot.parse_expr_res(Restrictions::CONST_EXPR, None) {
+        match (|| {
+            let attrs = self.parse_outer_attributes()?;
+            snapshot.parse_expr_res(Restrictions::CONST_EXPR, attrs)
+        })() {
             // Since we don't know the exact reason why we failed to parse the type or the
             // expression, employ a simple heuristic to weed out some pathological cases.
             Ok(expr) if let token::Comma | token::Gt = snapshot.token.kind => {
@@ -2958,9 +2965,10 @@ impl<'a> Parser<'a> {
 
     /// This checks if this is a conflict marker, depending of the parameter passed.
     ///
-    /// * `>>>>>`
-    /// * `=====`
-    /// * `<<<<<`
+    /// * `<<<<<<<`
+    /// * `|||||||`
+    /// * `=======`
+    /// * `>>>>>>>`
     ///
     pub(super) fn is_vcs_conflict_marker(
         &mut self,
@@ -2990,14 +2998,18 @@ impl<'a> Parser<'a> {
     }
 
     pub(crate) fn err_vcs_conflict_marker(&mut self) -> PResult<'a, ()> {
+        // <<<<<<<
         let Some(start) = self.conflict_marker(&TokenKind::BinOp(token::Shl), &TokenKind::Lt)
         else {
             return Ok(());
         };
         let mut spans = Vec::with_capacity(3);
         spans.push(start);
+        // |||||||
         let mut middlediff3 = None;
+        // =======
         let mut middle = None;
+        // >>>>>>>
         let mut end = None;
         loop {
             if self.token.kind == TokenKind::Eof {
@@ -3018,29 +3030,50 @@ impl<'a> Parser<'a> {
             }
             self.bump();
         }
+
         let mut err = self.dcx().struct_span_err(spans, "encountered diff marker");
-        err.span_label(start, "after this is the code before the merge");
-        if let Some(middle) = middlediff3 {
-            err.span_label(middle, "");
-        }
+        match middlediff3 {
+            // We're using diff3
+            Some(middlediff3) => {
+                err.span_label(
+                    start,
+                    "between this marker and `|||||||` is the code that we're merging into",
+                );
+                err.span_label(middlediff3, "between this marker and `=======` is the base code (what the two refs diverged from)");
+            }
+            None => {
+                err.span_label(
+                    start,
+                    "between this marker and `=======` is the code that we're merging into",
+                );
+            }
+        };
+
         if let Some(middle) = middle {
-            err.span_label(middle, "");
+            err.span_label(middle, "between this marker and `>>>>>>>` is the incoming code");
         }
         if let Some(end) = end {
-            err.span_label(end, "above this are the incoming code changes");
+            err.span_label(end, "this marker concludes the conflict region");
         }
-        err.help(
-            "if you're having merge conflicts after pulling new code, the top section is the code \
-             you already had and the bottom section is the remote code",
-        );
-        err.help(
-            "if you're in the middle of a rebase, the top section is the code being rebased onto \
-             and the bottom section is the code coming from the current commit being rebased",
-        );
         err.note(
-            "for an explanation on these markers from the `git` documentation, visit \
-             <https://git-scm.com/book/en/v2/Git-Tools-Advanced-Merging#_checking_out_conflicts>",
+            "conflict markers indicate that a merge was started but could not be completed due \
+             to merge conflicts\n\
+             to resolve a conflict, keep only the code you want and then delete the lines \
+             containing conflict markers",
         );
+        err.help(
+            "if you're having merge conflicts after pulling new code:\n\
+             the top section is the code you already had and the bottom section is the remote code\n\
+             if you're in the middle of a rebase:\n\
+             the top section is the code being rebased onto and the bottom section is the code \
+             coming from the current commit being rebased",
+        );
+
+        err.note(
+            "for an explanation on these markers from the `git` documentation:\n\
+             visit <https://git-scm.com/book/en/v2/Git-Tools-Advanced-Merging#_checking_out_conflicts>",
+        );
+
         Err(err)
     }
 

compiler/rustc_parse/src/parser/expr.rs

@@ -1,4 +1,5 @@
 // ignore-tidy-filelength
+
 use super::diagnostics::SnapshotParser;
 use super::pat::{CommaRecoveryMode, Expected, RecoverColon, RecoverComma};
 use super::ty::{AllowPlus, RecoverQPath, RecoverReturnSign};
@@ -70,28 +71,10 @@ macro_rules! maybe_whole_expr {
 
 #[derive(Debug)]
 pub(super) enum LhsExpr {
-    NotYetParsed,
-    AttributesParsed(AttrWrapper),
-    AlreadyParsed { expr: P<Expr>, starts_statement: bool },
-}
-
-impl From<Option<AttrWrapper>> for LhsExpr {
-    /// Converts `Some(attrs)` into `LhsExpr::AttributesParsed(attrs)`
-    /// and `None` into `LhsExpr::NotYetParsed`.
-    ///
-    /// This conversion does not allocate.
-    fn from(o: Option<AttrWrapper>) -> Self {
-        if let Some(attrs) = o { LhsExpr::AttributesParsed(attrs) } else { LhsExpr::NotYetParsed }
-    }
-}
-
-impl From<P<Expr>> for LhsExpr {
-    /// Converts the `expr: P<Expr>` into `LhsExpr::AlreadyParsed { expr, starts_statement: false }`.
-    ///
-    /// This conversion does not allocate.
-    fn from(expr: P<Expr>) -> Self {
-        LhsExpr::AlreadyParsed { expr, starts_statement: false }
-    }
+    // Already parsed just the outer attributes.
+    Unparsed { attrs: AttrWrapper },
+    // Already parsed the expression.
+    Parsed { expr: P<Expr>, starts_statement: bool },
 }
 
 #[derive(Debug)]
@@ -112,12 +95,16 @@ impl<'a> Parser<'a> {
     pub fn parse_expr(&mut self) -> PResult<'a, P<Expr>> {
         self.current_closure.take();
 
-        self.parse_expr_res(Restrictions::empty(), None)
+        let attrs = self.parse_outer_attributes()?;
+        self.parse_expr_res(Restrictions::empty(), attrs)
     }
 
-    /// Parses an expression, forcing tokens to be collected
+    /// Parses an expression, forcing tokens to be collected.
     pub fn parse_expr_force_collect(&mut self) -> PResult<'a, P<Expr>> {
-        self.collect_tokens_no_attrs(|this| this.parse_expr())
+        self.current_closure.take();
+
+        let attrs = self.parse_outer_attributes()?;
+        self.collect_tokens_no_attrs(|this| this.parse_expr_res(Restrictions::empty(), attrs))
     }
 
     pub fn parse_expr_anon_const(&mut self) -> PResult<'a, AnonConst> {
@@ -125,7 +112,8 @@ impl<'a> Parser<'a> {
     }
 
     fn parse_expr_catch_underscore(&mut self, restrictions: Restrictions) -> PResult<'a, P<Expr>> {
-        match self.parse_expr_res(restrictions, None) {
+        let attrs = self.parse_outer_attributes()?;
+        match self.parse_expr_res(restrictions, attrs) {
             Ok(expr) => Ok(expr),
             Err(err) => match self.token.ident() {
                 Some((Ident { name: kw::Underscore, .. }, IdentIsRaw::No))
@@ -152,21 +140,9 @@ impl<'a> Parser<'a> {
     pub(super) fn parse_expr_res(
         &mut self,
         r: Restrictions,
-        already_parsed_attrs: Option<AttrWrapper>,
+        attrs: AttrWrapper,
     ) -> PResult<'a, P<Expr>> {
-        self.with_res(r, |this| this.parse_expr_assoc(already_parsed_attrs))
-    }
-
-    /// Parses an associative expression.
-    ///
-    /// This parses an expression accounting for associativity and precedence of the operators in
-    /// the expression.
-    #[inline]
-    fn parse_expr_assoc(
-        &mut self,
-        already_parsed_attrs: Option<AttrWrapper>,
-    ) -> PResult<'a, P<Expr>> {
-        self.parse_expr_assoc_with(0, already_parsed_attrs.into())
+        self.with_res(r, |this| this.parse_expr_assoc_with(0, LhsExpr::Unparsed { attrs }))
     }
 
     /// Parses an associative expression with operators of at least `min_prec` precedence.
@@ -176,18 +152,17 @@ impl<'a> Parser<'a> {
         lhs: LhsExpr,
     ) -> PResult<'a, P<Expr>> {
         let mut starts_stmt = false;
-        let mut lhs = if let LhsExpr::AlreadyParsed { expr, starts_statement } = lhs {
-            starts_stmt = starts_statement;
-            expr
-        } else {
-            let attrs = match lhs {
-                LhsExpr::AttributesParsed(attrs) => Some(attrs),
-                _ => None,
-            };
-            if self.token.is_range_separator() {
-                return self.parse_expr_prefix_range(attrs);
-            } else {
-                self.parse_expr_prefix(attrs)?
+        let mut lhs = match lhs {
+            LhsExpr::Parsed { expr, starts_statement } => {
+                starts_stmt = starts_statement;
+                expr
+            }
+            LhsExpr::Unparsed { attrs } => {
+                if self.token.is_range_separator() {
+                    return self.parse_expr_prefix_range(attrs);
+                } else {
+                    self.parse_expr_prefix(attrs)?
+                }
             }
         };
 
@@ -325,7 +300,8 @@ impl<'a> Parser<'a> {
                 Fixity::None => 1,
             };
             let rhs = self.with_res(restrictions - Restrictions::STMT_EXPR, |this| {
-                this.parse_expr_assoc_with(prec + prec_adjustment, LhsExpr::NotYetParsed)
+                let attrs = this.parse_outer_attributes()?;
+                this.parse_expr_assoc_with(prec + prec_adjustment, LhsExpr::Unparsed { attrs })
             })?;
 
             let span = self.mk_expr_sp(&lhs, lhs_span, rhs.span);
@@ -498,8 +474,9 @@ impl<'a> Parser<'a> {
     ) -> PResult<'a, P<Expr>> {
         let rhs = if self.is_at_start_of_range_notation_rhs() {
             let maybe_lt = self.token.clone();
+            let attrs = self.parse_outer_attributes()?;
             Some(
-                self.parse_expr_assoc_with(prec + 1, LhsExpr::NotYetParsed)
+                self.parse_expr_assoc_with(prec + 1, LhsExpr::Unparsed { attrs })
                     .map_err(|err| self.maybe_err_dotdotlt_syntax(maybe_lt, err))?,
             )
         } else {
@@ -526,7 +503,12 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses prefix-forms of range notation: `..expr`, `..`, `..=expr`.
-    fn parse_expr_prefix_range(&mut self, attrs: Option<AttrWrapper>) -> PResult<'a, P<Expr>> {
+    fn parse_expr_prefix_range(&mut self, attrs: AttrWrapper) -> PResult<'a, P<Expr>> {
+        if !attrs.is_empty() {
+            let err = errors::DotDotRangeAttribute { span: self.token.span };
+            self.dcx().emit_err(err);
+        }
+
         // Check for deprecated `...` syntax.
         if self.token == token::DotDotDot {
             self.err_dotdotdot_syntax(self.token.span);
@@ -543,20 +525,20 @@ impl<'a> Parser<'a> {
             _ => RangeLimits::Closed,
         };
         let op = AssocOp::from_token(&self.token);
-        // FIXME: `parse_prefix_range_expr` is called when the current
-        // token is `DotDot`, `DotDotDot`, or `DotDotEq`. If we haven't already
-        // parsed attributes, then trying to parse them here will always fail.
-        // We should figure out how we want attributes on range expressions to work.
-        let attrs = self.parse_or_use_outer_attributes(attrs)?;
+        let attrs = self.parse_outer_attributes()?;
         self.collect_tokens_for_expr(attrs, |this, attrs| {
             let lo = this.token.span;
             let maybe_lt = this.look_ahead(1, |t| t.clone());
             this.bump();
             let (span, opt_end) = if this.is_at_start_of_range_notation_rhs() {
                 // RHS must be parsed with more associativity than the dots.
-                this.parse_expr_assoc_with(op.unwrap().precedence() + 1, LhsExpr::NotYetParsed)
-                    .map(|x| (lo.to(x.span), Some(x)))
-                    .map_err(|err| this.maybe_err_dotdotlt_syntax(maybe_lt, err))?
+                let attrs = this.parse_outer_attributes()?;
+                this.parse_expr_assoc_with(
+                    op.unwrap().precedence() + 1,
+                    LhsExpr::Unparsed { attrs },
+                )
+                .map(|x| (lo.to(x.span), Some(x)))
+                .map_err(|err| this.maybe_err_dotdotlt_syntax(maybe_lt, err))?
             } else {
                 (lo, None)
             };
@@ -566,8 +548,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses a prefix-unary-operator expr.
-    fn parse_expr_prefix(&mut self, attrs: Option<AttrWrapper>) -> PResult<'a, P<Expr>> {
-        let attrs = self.parse_or_use_outer_attributes(attrs)?;
+    fn parse_expr_prefix(&mut self, attrs: AttrWrapper) -> PResult<'a, P<Expr>> {
         let lo = self.token.span;
 
         macro_rules! make_it {
@@ -616,7 +597,8 @@ impl<'a> Parser<'a> {
                 this.dcx().emit_err(err);
 
                 this.bump();
-                this.parse_expr_prefix(None)
+                let attrs = this.parse_outer_attributes()?;
+                this.parse_expr_prefix(attrs)
             }
             // Recover from `++x`:
             token::BinOp(token::Plus)
@@ -629,7 +611,7 @@ impl<'a> Parser<'a> {
                 this.bump();
                 this.bump();
 
-                let operand_expr = this.parse_expr_dot_or_call(Default::default())?;
+                let operand_expr = this.parse_expr_dot_or_call(attrs)?;
                 this.recover_from_prefix_increment(operand_expr, pre_span, starts_stmt)
             }
             token::Ident(..) if this.token.is_keyword(kw::Box) => {
@@ -638,13 +620,14 @@ impl<'a> Parser<'a> {
             token::Ident(..) if this.may_recover() && this.is_mistaken_not_ident_negation() => {
                 make_it!(this, attrs, |this, _| this.recover_not_expr(lo))
             }
-            _ => return this.parse_expr_dot_or_call(Some(attrs)),
+            _ => return this.parse_expr_dot_or_call(attrs),
         }
     }
 
     fn parse_expr_prefix_common(&mut self, lo: Span) -> PResult<'a, (Span, P<Expr>)> {
         self.bump();
-        let expr = self.parse_expr_prefix(None)?;
+        let attrs = self.parse_outer_attributes()?;
+        let expr = self.parse_expr_prefix(attrs)?;
         let span = self.interpolated_or_expr_span(&expr);
         Ok((lo.to(span), expr))
     }
@@ -894,10 +877,11 @@ impl<'a> Parser<'a> {
         let has_lifetime = self.token.is_lifetime() && self.look_ahead(1, |t| t != &token::Colon);
         let lifetime = has_lifetime.then(|| self.expect_lifetime()); // For recovery, see below.
         let (borrow_kind, mutbl) = self.parse_borrow_modifiers(lo);
+        let attrs = self.parse_outer_attributes()?;
         let expr = if self.token.is_range_separator() {
-            self.parse_expr_prefix_range(None)
+            self.parse_expr_prefix_range(attrs)
         } else {
-            self.parse_expr_prefix(None)
+            self.parse_expr_prefix(attrs)
         }?;
         let hi = self.interpolated_or_expr_span(&expr);
         let span = lo.to(hi);
@@ -927,8 +911,7 @@ impl<'a> Parser<'a> {
     }
 
     /// Parses `a.b` or `a(13)` or `a[4]` or just `a`.
-    fn parse_expr_dot_or_call(&mut self, attrs: Option<AttrWrapper>) -> PResult<'a, P<Expr>> {
-        let attrs = self.parse_or_use_outer_attributes(attrs)?;
+    fn parse_expr_dot_or_call(&mut self, attrs: AttrWrapper) -> PResult<'a, P<Expr>> {
         self.collect_tokens_for_expr(attrs, |this, attrs| {
             let base = this.parse_expr_bottom()?;
             let span = this.interpolated_or_expr_span(&base);
@@ -2365,7 +2348,8 @@ impl<'a> Parser<'a> {
                     self.restrictions - Restrictions::STMT_EXPR - Restrictions::ALLOW_LET;
                 let prev = self.prev_token.clone();
                 let token = self.token.clone();
-                match self.parse_expr_res(restrictions, None) {
+                let attrs = self.parse_outer_attributes()?;
+                match self.parse_expr_res(restrictions, attrs) {
                     Ok(expr) => expr,
                     Err(err) => self.recover_closure_body(err, before, prev, token, lo, decl_hi)?,
                 }
@@ -2613,8 +2597,9 @@ impl<'a> Parser<'a> {
 
     /// Parses the condition of a `if` or `while` expression.
     fn parse_expr_cond(&mut self) -> PResult<'a, P<Expr>> {
+        let attrs = self.parse_outer_attributes()?;
         let mut cond =
-            self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL | Restrictions::ALLOW_LET, None)?;
+            self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL | Restrictions::ALLOW_LET, attrs)?;
 
         CondChecker::new(self).visit_expr(&mut cond);
 
@@ -2661,7 +2646,11 @@ impl<'a> Parser<'a> {
         } else {
             self.expect(&token::Eq)?;
         }
-        let expr = self.parse_expr_assoc_with(1 + prec_let_scrutinee_needs_par(), None.into())?;
+        let attrs = self.parse_outer_attributes()?;
+        let expr = self.parse_expr_assoc_with(
+            1 + prec_let_scrutinee_needs_par(),
+            LhsExpr::Unparsed { attrs },
+        )?;
         let span = lo.to(expr.span);
         Ok(self.mk_expr(span, ExprKind::Let(pat, expr, span, recovered)))
     }
@@ -2794,7 +2783,8 @@ impl<'a> Parser<'a> {
             (Err(err), Some((start_span, left))) if self.eat_keyword(kw::In) => {
                 // We know for sure we have seen `for ($SOMETHING in`. In the happy path this would
                 // happen right before the return of this method.
-                let expr = match self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, None) {
+                let attrs = self.parse_outer_attributes()?;
+                let expr = match self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, attrs) {
                     Ok(expr) => expr,
                     Err(expr_err) => {
                         // We don't know what followed the `in`, so cancel and bubble up the
@@ -2828,7 +2818,8 @@ impl<'a> Parser<'a> {
             self.error_missing_in_for_loop();
         }
         self.check_for_for_in_in_typo(self.prev_token.span);
-        let expr = self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, None)?;
+        let attrs = self.parse_outer_attributes()?;
+        let expr = self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, attrs)?;
         Ok((pat, expr))
     }
 
@@ -2940,7 +2931,8 @@ impl<'a> Parser<'a> {
     /// Parses a `match ... { ... }` expression (`match` token already eaten).
     fn parse_expr_match(&mut self) -> PResult<'a, P<Expr>> {
         let match_span = self.prev_token.span;
-        let scrutinee = self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, None)?;
+        let attrs = self.parse_outer_attributes()?;
+        let scrutinee = self.parse_expr_res(Restrictions::NO_STRUCT_LITERAL, attrs)?;
 
         self.parse_match_block(match_span, match_span, scrutinee, MatchKind::Prefix)
     }
@@ -3144,8 +3136,9 @@ impl<'a> Parser<'a> {
                 let arrow_span = this.prev_token.span;
                 let arm_start_span = this.token.span;
 
+                let attrs = this.parse_outer_attributes()?;
                 let expr =
-                    this.parse_expr_res(Restrictions::STMT_EXPR, None).map_err(|mut err| {
+                    this.parse_expr_res(Restrictions::STMT_EXPR, attrs).map_err(|mut err| {
                         err.span_label(arrow_span, "while parsing the `match` arm starting here");
                         err
                     })?;
@@ -3350,7 +3343,8 @@ impl<'a> Parser<'a> {
     }
 
     fn parse_match_guard_condition(&mut self) -> PResult<'a, P<Expr>> {
-        self.parse_expr_res(Restrictions::ALLOW_LET | Restrictions::IN_IF_GUARD, None).map_err(
+        let attrs = self.parse_outer_attributes()?;
+        self.parse_expr_res(Restrictions::ALLOW_LET | Restrictions::IN_IF_GUARD, attrs).map_err(
             |mut err| {
                 if self.prev_token == token::OpenDelim(Delimiter::Brace) {
                     let sugg_sp = self.prev_token.span.shrink_to_lo();

compiler/rustc_parse/src/parser/item.rs

@@ -707,15 +707,25 @@ impl<'a> Parser<'a> {
         };
 
         let (ident, item_kind) = if self.eat(&token::PathSep) {
-            let (suffixes, _) = self.parse_delim_comma_seq(Delimiter::Brace, |p| {
-                Ok((p.parse_path_segment_ident()?, rename(p)?))
-            })?;
+            let suffixes = if self.eat(&token::BinOp(token::Star)) {
+                None
+            } else {
+                let parse_suffix = |p: &mut Self| Ok((p.parse_path_segment_ident()?, rename(p)?));
+                Some(self.parse_delim_comma_seq(Delimiter::Brace, parse_suffix)?.0)
+            };
             let deleg = DelegationMac { qself, prefix: path, suffixes, body: body(self)? };
             (Ident::empty(), ItemKind::DelegationMac(Box::new(deleg)))
         } else {
             let rename = rename(self)?;
             let ident = rename.unwrap_or_else(|| path.segments.last().unwrap().ident);
-            let deleg = Delegation { id: DUMMY_NODE_ID, qself, path, rename, body: body(self)? };
+            let deleg = Delegation {
+                id: DUMMY_NODE_ID,
+                qself,
+                path,
+                rename,
+                body: body(self)?,
+                from_glob: false,
+            };
             (ident, ItemKind::Delegation(Box::new(deleg)))
         };
 
@@ -1218,7 +1228,7 @@ impl<'a> Parser<'a> {
                                 ident_span: ident.span,
                                 const_span,
                             });
-                            ForeignItemKind::Static(Box::new(StaticForeignItem {
+                            ForeignItemKind::Static(Box::new(StaticItem {
                                 ty,
                                 mutability: Mutability::Not,
                                 expr,
@@ -1237,7 +1247,11 @@ impl<'a> Parser<'a> {
         // FIXME(#100717): needs variant for each `ItemKind` (instead of using `ItemKind::descr()`)
         let span = self.psess.source_map().guess_head_span(span);
         let descr = kind.descr();
-        self.dcx().emit_err(errors::BadItemKind { span, descr, ctx });
+        let help = match kind {
+            ItemKind::DelegationMac(deleg) if deleg.suffixes.is_none() => None,
+            _ => Some(()),
+        };
+        self.dcx().emit_err(errors::BadItemKind { span, descr, ctx, help });
         None
     }
 
@@ -1245,7 +1259,7 @@ impl<'a> Parser<'a> {
         self.token.is_keyword(kw::Unsafe)
             && self.is_keyword_ahead(1, &[kw::Extern])
             && self.look_ahead(
-                2 + self.look_ahead(2, |t| t.can_begin_literal_maybe_minus() as usize),
+                2 + self.look_ahead(2, |t| t.can_begin_string_literal() as usize),
                 |t| t.kind == token::OpenDelim(Delimiter::Brace),
             )
     }
@@ -2434,7 +2448,7 @@ impl<'a> Parser<'a> {
                 })
             // `extern ABI fn`
             || self.check_keyword_case(kw::Extern, case)
-                && self.look_ahead(1, |t| t.can_begin_literal_maybe_minus())
+                && self.look_ahead(1, |t| t.can_begin_string_literal())
                 && (self.look_ahead(2, |t| t.is_keyword_case(kw::Fn, case)) ||
                     // this branch is only for better diagnostic in later, `pub` is not allowed here
                     (self.may_recover()

compiler/rustc_parse/src/parser/mod.rs

@@ -1214,6 +1214,9 @@ impl<'a> Parser<'a> {
         if self.eat_keyword_case(kw::Unsafe, case) {
             Safety::Unsafe(self.prev_token.uninterpolated_span())
         } else if self.eat_keyword_case(kw::Safe, case) {
+            self.psess
+                .gated_spans
+                .gate(sym::unsafe_extern_blocks, self.prev_token.uninterpolated_span());
             Safety::Safe(self.prev_token.uninterpolated_span())
         } else {
             Safety::Default
@@ -1330,17 +1333,6 @@ impl<'a> Parser<'a> {
         })
     }
 
-    fn parse_or_use_outer_attributes(
-        &mut self,
-        already_parsed_attrs: Option<AttrWrapper>,
-    ) -> PResult<'a, AttrWrapper> {
-        if let Some(attrs) = already_parsed_attrs {
-            Ok(attrs)
-        } else {
-            self.parse_outer_attributes()
-        }
-    }
-
     /// Parses a single token tree from the input.
     pub fn parse_token_tree(&mut self) -> TokenTree {
         match self.token.kind {

compiler/rustc_parse/src/parser/nonterminal.rs

@@ -36,7 +36,7 @@ impl<'a> Parser<'a> {
         }
 
         match kind {
-            NonterminalKind::Expr2021 => {
+            NonterminalKind::Expr2021 { inferred: _ } => {
                 token.can_begin_expr()
                 // This exception is here for backwards compatibility.
                 && !token.is_keyword(kw::Let)
@@ -47,7 +47,6 @@ impl<'a> Parser<'a> {
                 token.can_begin_expr()
                 // This exception is here for backwards compatibility.
                 && !token.is_keyword(kw::Let)
-                && (!token.is_keyword(kw::Const) || token.span.edition().at_least_rust_2024())
             }
             NonterminalKind::Ty => token.can_begin_type(),
             NonterminalKind::Ident => get_macro_ident(token).is_some(),
@@ -149,7 +148,7 @@ impl<'a> Parser<'a> {
                 })?)
             }
 
-            NonterminalKind::Expr | NonterminalKind::Expr2021 => {
+            NonterminalKind::Expr | NonterminalKind::Expr2021 { inferred: _ } => {
                 NtExpr(self.parse_expr_force_collect()?)
             }
             NonterminalKind::Literal => {

compiler/rustc_parse/src/parser/pat.rs

@@ -10,7 +10,7 @@ use crate::errors::{
     UnexpectedParenInRangePatSugg, UnexpectedVertVertBeforeFunctionParam,
     UnexpectedVertVertInPattern,
 };
-use crate::parser::expr::could_be_unclosed_char_literal;
+use crate::parser::expr::{could_be_unclosed_char_literal, LhsExpr};
 use crate::{maybe_recover_from_interpolated_ty_qpath, maybe_whole};
 use rustc_ast::mut_visit::{noop_visit_pat, MutVisitor};
 use rustc_ast::ptr::P;
@@ -398,9 +398,8 @@ impl<'a> Parser<'a> {
 
             // Parse an associative expression such as `+ expr`, `% expr`, ...
             // Assignements, ranges and `|` are disabled by [`Restrictions::IS_PAT`].
-            if let Ok(expr) =
-                snapshot.parse_expr_assoc_with(0, expr.into()).map_err(|err| err.cancel())
-            {
+            let lhs = LhsExpr::Parsed { expr, starts_statement: false };
+            if let Ok(expr) = snapshot.parse_expr_assoc_with(0, lhs).map_err(|err| err.cancel()) {
                 // We got a valid expression.
                 self.restore_snapshot(snapshot);
                 self.restrictions.remove(Restrictions::IS_PAT);
@@ -940,7 +939,8 @@ impl<'a> Parser<'a> {
             || self.look_ahead(dist, |t| {
                 t.is_path_start() // e.g. `MY_CONST`;
                 || t.kind == token::Dot // e.g. `.5` for recovery;
-                || t.can_begin_literal_maybe_minus() // e.g. `42`.
+                || matches!(t.kind, token::Literal(..) | token::BinOp(token::Minus))
+                || t.is_bool_lit()
                 || t.is_whole_expr()
                 || t.is_lifetime() // recover `'a` instead of `'a'`
                 || (self.may_recover() // recover leading `(`