Merge pull request #2519 from rust-lang/rustc-pull

Rustc pull update

Cargo.lock

@@ -4255,6 +4255,7 @@ dependencies = [
  "rustc-literal-escaper",
  "rustc_ast",
  "rustc_ast_pretty",
+ "rustc_attr_parsing",
  "rustc_data_structures",
  "rustc_errors",
  "rustc_feature",
@@ -4563,7 +4564,10 @@ dependencies = [
  "rustc_macros",
  "rustc_serialize",
  "rustc_span",
+ "serde",
+ "serde_derive",
  "serde_json",
+ "serde_path_to_error",
  "tracing",
 ]
 
@@ -4955,6 +4959,16 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_path_to_error"
+version = "0.1.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a"
+dependencies = [
+ "itoa",
+ "serde",
+]
+
 [[package]]
 name = "serde_spanned"
 version = "0.6.9"

compiler/rustc_abi/src/layout.rs

@@ -313,7 +313,6 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
         scalar_valid_range: (Bound<u128>, Bound<u128>),
         discr_range_of_repr: impl Fn(i128, i128) -> (Integer, bool),
         discriminants: impl Iterator<Item = (VariantIdx, i128)>,
-        dont_niche_optimize_enum: bool,
         always_sized: bool,
     ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {
         let (present_first, present_second) = {
@@ -352,13 +351,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
             // structs. (We have also handled univariant enums
             // that allow representation optimization.)
             assert!(is_enum);
-            self.layout_of_enum(
-                repr,
-                variants,
-                discr_range_of_repr,
-                discriminants,
-                dont_niche_optimize_enum,
-            )
+            self.layout_of_enum(repr, variants, discr_range_of_repr, discriminants)
         }
     }
 
@@ -599,7 +592,6 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
         variants: &IndexSlice<VariantIdx, IndexVec<FieldIdx, F>>,
         discr_range_of_repr: impl Fn(i128, i128) -> (Integer, bool),
         discriminants: impl Iterator<Item = (VariantIdx, i128)>,
-        dont_niche_optimize_enum: bool,
     ) -> LayoutCalculatorResult<FieldIdx, VariantIdx, F> {
         // Until we've decided whether to use the tagged or
         // niche filling LayoutData, we don't want to intern the
@@ -618,7 +610,7 @@ impl<Cx: HasDataLayout> LayoutCalculator<Cx> {
         }
 
         let calculate_niche_filling_layout = || -> Option<TmpLayout<FieldIdx, VariantIdx>> {
-            if dont_niche_optimize_enum {
+            if repr.inhibit_enum_layout_opt() {
                 return None;
             }
 

compiler/rustc_abi/src/lib.rs

@@ -1376,6 +1376,28 @@ impl WrappingRange {
         }
     }
 
+    /// Returns `true` if all the values in `other` are contained in this range,
+    /// when the values are considered as having width `size`.
+    #[inline(always)]
+    pub fn contains_range(&self, other: Self, size: Size) -> bool {
+        if self.is_full_for(size) {
+            true
+        } else {
+            let trunc = |x| size.truncate(x);
+
+            let delta = self.start;
+            let max = trunc(self.end.wrapping_sub(delta));
+
+            let other_start = trunc(other.start.wrapping_sub(delta));
+            let other_end = trunc(other.end.wrapping_sub(delta));
+
+            // Having shifted both input ranges by `delta`, now we only need to check
+            // whether `0..=max` contains `other_start..=other_end`, which can only
+            // happen if the other doesn't wrap since `self` isn't everything.
+            (other_start <= other_end) && (other_end <= max)
+        }
+    }
+
     /// Returns `self` with replaced `start`
     #[inline(always)]
     fn with_start(mut self, start: u128) -> Self {

compiler/rustc_abi/src/tests.rs

@@ -5,3 +5,66 @@ fn align_constants() {
     assert_eq!(Align::ONE, Align::from_bytes(1).unwrap());
     assert_eq!(Align::EIGHT, Align::from_bytes(8).unwrap());
 }
+
+#[test]
+fn wrapping_range_contains_range() {
+    let size16 = Size::from_bytes(16);
+
+    let a = WrappingRange { start: 10, end: 20 };
+    assert!(a.contains_range(a, size16));
+    assert!(a.contains_range(WrappingRange { start: 11, end: 19 }, size16));
+    assert!(a.contains_range(WrappingRange { start: 10, end: 10 }, size16));
+    assert!(a.contains_range(WrappingRange { start: 20, end: 20 }, size16));
+    assert!(!a.contains_range(WrappingRange { start: 10, end: 21 }, size16));
+    assert!(!a.contains_range(WrappingRange { start: 9, end: 20 }, size16));
+    assert!(!a.contains_range(WrappingRange { start: 4, end: 6 }, size16));
+    assert!(!a.contains_range(WrappingRange { start: 24, end: 26 }, size16));
+
+    assert!(!a.contains_range(WrappingRange { start: 16, end: 14 }, size16));
+
+    let b = WrappingRange { start: 20, end: 10 };
+    assert!(b.contains_range(b, size16));
+    assert!(b.contains_range(WrappingRange { start: 20, end: 20 }, size16));
+    assert!(b.contains_range(WrappingRange { start: 10, end: 10 }, size16));
+    assert!(b.contains_range(WrappingRange { start: 0, end: 10 }, size16));
+    assert!(b.contains_range(WrappingRange { start: 20, end: 30 }, size16));
+    assert!(b.contains_range(WrappingRange { start: 20, end: 9 }, size16));
+    assert!(b.contains_range(WrappingRange { start: 21, end: 10 }, size16));
+    assert!(b.contains_range(WrappingRange { start: 999, end: 9999 }, size16));
+    assert!(b.contains_range(WrappingRange { start: 999, end: 9 }, size16));
+    assert!(!b.contains_range(WrappingRange { start: 19, end: 19 }, size16));
+    assert!(!b.contains_range(WrappingRange { start: 11, end: 11 }, size16));
+    assert!(!b.contains_range(WrappingRange { start: 19, end: 11 }, size16));
+    assert!(!b.contains_range(WrappingRange { start: 11, end: 19 }, size16));
+
+    let f = WrappingRange { start: 0, end: u128::MAX };
+    assert!(f.contains_range(WrappingRange { start: 10, end: 20 }, size16));
+    assert!(f.contains_range(WrappingRange { start: 20, end: 10 }, size16));
+
+    let g = WrappingRange { start: 2, end: 1 };
+    assert!(g.contains_range(WrappingRange { start: 10, end: 20 }, size16));
+    assert!(g.contains_range(WrappingRange { start: 20, end: 10 }, size16));
+
+    let size1 = Size::from_bytes(1);
+    let u8r = WrappingRange { start: 0, end: 255 };
+    let i8r = WrappingRange { start: 128, end: 127 };
+    assert!(u8r.contains_range(i8r, size1));
+    assert!(i8r.contains_range(u8r, size1));
+    assert!(!u8r.contains_range(i8r, size16));
+    assert!(i8r.contains_range(u8r, size16));
+
+    let boolr = WrappingRange { start: 0, end: 1 };
+    assert!(u8r.contains_range(boolr, size1));
+    assert!(i8r.contains_range(boolr, size1));
+    assert!(!boolr.contains_range(u8r, size1));
+    assert!(!boolr.contains_range(i8r, size1));
+
+    let cmpr = WrappingRange { start: 255, end: 1 };
+    assert!(u8r.contains_range(cmpr, size1));
+    assert!(i8r.contains_range(cmpr, size1));
+    assert!(!cmpr.contains_range(u8r, size1));
+    assert!(!cmpr.contains_range(i8r, size1));
+
+    assert!(!boolr.contains_range(cmpr, size1));
+    assert!(cmpr.contains_range(boolr, size1));
+}

compiler/rustc_ast/src/ast.rs

@@ -2849,7 +2849,7 @@ impl InlineAsmOperand {
     }
 }
 
-#[derive(Clone, Copy, Encodable, Decodable, Debug, HashStable_Generic, Walkable)]
+#[derive(Clone, Copy, Encodable, Decodable, Debug, HashStable_Generic, Walkable, PartialEq, Eq)]
 pub enum AsmMacro {
     /// The `asm!` macro
     Asm,

compiler/rustc_attr_parsing/src/attributes/stability.rs

@@ -74,8 +74,15 @@ impl<S: Stage> AttributeParser<S> for StabilityParser {
             template!(NameValueStr: "deprecation message"),
             |this, cx, args| {
                 reject_outside_std!(cx);
-                this.allowed_through_unstable_modules =
-                    args.name_value().and_then(|i| i.value_as_str())
+                let Some(nv) = args.name_value() else {
+                    cx.expected_name_value(cx.attr_span, None);
+                    return;
+                };
+                let Some(value_str) = nv.value_as_str() else {
+                    cx.expected_string_literal(nv.value_span, Some(nv.value_as_lit()));
+                    return;
+                };
+                this.allowed_through_unstable_modules = Some(value_str);
             },
         ),
     ];
@@ -247,7 +254,12 @@ pub(crate) fn parse_stability<S: Stage>(
     let mut feature = None;
     let mut since = None;
 
-    for param in args.list()?.mixed() {
+    let ArgParser::List(list) = args else {
+        cx.expected_list(cx.attr_span);
+        return None;
+    };
+
+    for param in list.mixed() {
         let param_span = param.span();
         let Some(param) = param.meta_item() else {
             cx.emit_err(session_diagnostics::UnsupportedLiteral {
@@ -322,7 +334,13 @@ pub(crate) fn parse_unstability<S: Stage>(
     let mut is_soft = false;
     let mut implied_by = None;
     let mut old_name = None;
-    for param in args.list()?.mixed() {
+
+    let ArgParser::List(list) = args else {
+        cx.expected_list(cx.attr_span);
+        return None;
+    };
+
+    for param in list.mixed() {
         let Some(param) = param.meta_item() else {
             cx.emit_err(session_diagnostics::UnsupportedLiteral {
                 span: param.span(),

compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs

@@ -1290,6 +1290,58 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 span,
                 format!("if `{ty}` implemented `Clone`, you could clone the value"),
             );
+        } else if let ty::Adt(_, _) = ty.kind()
+            && let Some(clone_trait) = self.infcx.tcx.lang_items().clone_trait()
+        {
+            // For cases like `Option<NonClone>`, where `Option<T>: Clone` if `T: Clone`, we point
+            // at the types that should be `Clone`.
+            let ocx = ObligationCtxt::new_with_diagnostics(self.infcx);
+            let cause = ObligationCause::misc(expr.span, self.mir_def_id());
+            ocx.register_bound(cause, self.infcx.param_env, ty, clone_trait);
+            let errors = ocx.select_all_or_error();
+            if errors.iter().all(|error| {
+                match error.obligation.predicate.as_clause().and_then(|c| c.as_trait_clause()) {
+                    Some(clause) => match clause.self_ty().skip_binder().kind() {
+                        ty::Adt(def, _) => def.did().is_local() && clause.def_id() == clone_trait,
+                        _ => false,
+                    },
+                    None => false,
+                }
+            }) {
+                let mut type_spans = vec![];
+                let mut types = FxIndexSet::default();
+                for clause in errors
+                    .iter()
+                    .filter_map(|e| e.obligation.predicate.as_clause())
+                    .filter_map(|c| c.as_trait_clause())
+                {
+                    let ty::Adt(def, _) = clause.self_ty().skip_binder().kind() else { continue };
+                    type_spans.push(self.infcx.tcx.def_span(def.did()));
+                    types.insert(
+                        self.infcx
+                            .tcx
+                            .short_string(clause.self_ty().skip_binder(), &mut err.long_ty_path()),
+                    );
+                }
+                let mut span: MultiSpan = type_spans.clone().into();
+                for sp in type_spans {
+                    span.push_span_label(sp, "consider implementing `Clone` for this type");
+                }
+                span.push_span_label(expr.span, "you could clone this value");
+                let types: Vec<_> = types.into_iter().collect();
+                let msg = match &types[..] {
+                    [only] => format!("`{only}`"),
+                    [head @ .., last] => format!(
+                        "{} and `{last}`",
+                        head.iter().map(|t| format!("`{t}`")).collect::<Vec<_>>().join(", ")
+                    ),
+                    [] => unreachable!(),
+                };
+                err.span_note(
+                    span,
+                    format!("if {msg} implemented `Clone`, you could clone the value"),
+                );
+            }
         }
     }
 

compiler/rustc_borrowck/src/diagnostics/move_errors.rs

@@ -115,10 +115,8 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
     fn append_to_grouped_errors(
         &self,
         grouped_errors: &mut Vec<GroupedMoveError<'tcx>>,
-        error: MoveError<'tcx>,
+        MoveError { place: original_path, location, kind }: MoveError<'tcx>,
     ) {
-        let MoveError { place: original_path, location, kind } = error;
-
         // Note: that the only time we assign a place isn't a temporary
         // to a user variable is when initializing it.
         // If that ever stops being the case, then the ever initialized
@@ -251,54 +249,47 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
     }
 
     fn report(&mut self, error: GroupedMoveError<'tcx>) {
-        let (mut err, err_span) = {
-            let (span, use_spans, original_path, kind) = match error {
-                GroupedMoveError::MovesFromPlace { span, original_path, ref kind, .. }
-                | GroupedMoveError::MovesFromValue { span, original_path, ref kind, .. } => {
-                    (span, None, original_path, kind)
-                }
-                GroupedMoveError::OtherIllegalMove { use_spans, original_path, ref kind } => {
-                    (use_spans.args_or_use(), Some(use_spans), original_path, kind)
-                }
-            };
-            debug!(
-                "report: original_path={:?} span={:?}, kind={:?} \
-                   original_path.is_upvar_field_projection={:?}",
-                original_path,
-                span,
-                kind,
-                self.is_upvar_field_projection(original_path.as_ref())
-            );
-            if self.has_ambiguous_copy(original_path.ty(self.body, self.infcx.tcx).ty) {
-                // If the type may implement Copy, skip the error.
-                // It's an error with the Copy implementation (e.g. duplicate Copy) rather than borrow check
-                self.dcx().span_delayed_bug(
-                    span,
-                    "Type may implement copy, but there is no other error.",
-                );
-                return;
+        let (span, use_spans, original_path, kind) = match error {
+            GroupedMoveError::MovesFromPlace { span, original_path, ref kind, .. }
+            | GroupedMoveError::MovesFromValue { span, original_path, ref kind, .. } => {
+                (span, None, original_path, kind)
+            }
+            GroupedMoveError::OtherIllegalMove { use_spans, original_path, ref kind } => {
+                (use_spans.args_or_use(), Some(use_spans), original_path, kind)
+            }
+        };
+        debug!(
+            "report: original_path={:?} span={:?}, kind={:?} \
+             original_path.is_upvar_field_projection={:?}",
+            original_path,
+            span,
+            kind,
+            self.is_upvar_field_projection(original_path.as_ref())
+        );
+        if self.has_ambiguous_copy(original_path.ty(self.body, self.infcx.tcx).ty) {
+            // If the type may implement Copy, skip the error.
+            // It's an error with the Copy implementation (e.g. duplicate Copy) rather than borrow check
+            self.dcx()
+                .span_delayed_bug(span, "Type may implement copy, but there is no other error.");
+            return;
+        }
+        let mut err = match kind {
+            &IllegalMoveOriginKind::BorrowedContent { target_place } => self
+                .report_cannot_move_from_borrowed_content(
+                    original_path,
+                    target_place,
+                    span,
+                    use_spans,
+                ),
+            &IllegalMoveOriginKind::InteriorOfTypeWithDestructor { container_ty: ty } => {
+                self.cannot_move_out_of_interior_of_drop(span, ty)
+            }
+            &IllegalMoveOriginKind::InteriorOfSliceOrArray { ty, is_index } => {
+                self.cannot_move_out_of_interior_noncopy(span, ty, Some(is_index))
             }
-            (
-                match kind {
-                    &IllegalMoveOriginKind::BorrowedContent { target_place } => self
-                        .report_cannot_move_from_borrowed_content(
-                            original_path,
-                            target_place,
-                            span,
-                            use_spans,
-                        ),
-                    &IllegalMoveOriginKind::InteriorOfTypeWithDestructor { container_ty: ty } => {
-                        self.cannot_move_out_of_interior_of_drop(span, ty)
-                    }
-                    &IllegalMoveOriginKind::InteriorOfSliceOrArray { ty, is_index } => {
-                        self.cannot_move_out_of_interior_noncopy(span, ty, Some(is_index))
-                    }
-                },
-                span,
-            )
         };
 
-        self.add_move_hints(error, &mut err, err_span);
+        self.add_move_hints(error, &mut err, span);
         self.buffer_error(err);
     }
 
@@ -483,7 +474,8 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 self.cannot_move_out_of_interior_noncopy(span, ty, None)
             }
             ty::Closure(def_id, closure_args)
-                if def_id.as_local() == Some(self.mir_def_id()) && upvar_field.is_some() =>
+                if def_id.as_local() == Some(self.mir_def_id())
+                    && let Some(upvar_field) = upvar_field =>
             {
                 let closure_kind_ty = closure_args.as_closure().kind_ty();
                 let closure_kind = match closure_kind_ty.to_opt_closure_kind() {
@@ -496,7 +488,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 let capture_description =
                     format!("captured variable in an `{closure_kind}` closure");
 
-                let upvar = &self.upvars[upvar_field.unwrap().index()];
+                let upvar = &self.upvars[upvar_field.index()];
                 let upvar_hir_id = upvar.get_root_variable();
                 let upvar_name = upvar.to_string(tcx);
                 let upvar_span = tcx.hir_span(upvar_hir_id);
@@ -606,7 +598,7 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
             }
             // No binding. Nothing to suggest.
             GroupedMoveError::OtherIllegalMove { ref original_path, use_spans, .. } => {
-                let use_span = use_spans.var_or_use();
+                let mut use_span = use_spans.var_or_use();
                 let place_ty = original_path.ty(self.body, self.infcx.tcx).ty;
                 let place_desc = match self.describe_place(original_path.as_ref()) {
                     Some(desc) => format!("`{desc}`"),
@@ -623,6 +615,36 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                     );
                 }
 
+                if let Some(upvar_field) = self
+                    .prefixes(original_path.as_ref(), PrefixSet::All)
+                    .find_map(|p| self.is_upvar_field_projection(p))
+                {
+                    // Look for the introduction of the original binding being moved.
+                    let upvar = &self.upvars[upvar_field.index()];
+                    let upvar_hir_id = upvar.get_root_variable();
+                    use_span = match self.infcx.tcx.parent_hir_node(upvar_hir_id) {
+                        hir::Node::Param(param) => {
+                            // Instead of pointing at the path where we access the value within a
+                            // closure, we point at the type of the outer `fn` argument.
+                            param.ty_span
+                        }
+                        hir::Node::LetStmt(stmt) => match (stmt.ty, stmt.init) {
+                            // We point at the type of the outer let-binding.
+                            (Some(ty), _) => ty.span,
+                            // We point at the initializer of the outer let-binding, but only if it
+                            // isn't something that spans multiple lines, like a closure, as the
+                            // ASCII art gets messy.
+                            (None, Some(init))
+                                if !self.infcx.tcx.sess.source_map().is_multiline(init.span) =>
+                            {
+                                init.span
+                            }
+                            _ => use_span,
+                        },
+                        _ => use_span,
+                    };
+                }
+
                 err.subdiagnostic(crate::session_diagnostics::TypeNoCopy::Label {
                     is_partial_move: false,
                     ty: place_ty,
@@ -630,12 +652,22 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                     span: use_span,
                 });
 
+                let mut pointed_at_span = false;
                 use_spans.args_subdiag(err, |args_span| {
+                    if args_span == span || args_span == use_span {
+                        pointed_at_span = true;
+                    }
                     crate::session_diagnostics::CaptureArgLabel::MoveOutPlace {
-                        place: place_desc,
+                        place: place_desc.clone(),
                         args_span,
                     }
                 });
+                if !pointed_at_span && use_span != span {
+                    err.subdiagnostic(crate::session_diagnostics::CaptureArgLabel::MoveOutPlace {
+                        place: place_desc,
+                        args_span: span,
+                    });
+                }
 
                 self.add_note_for_packed_struct_derive(err, original_path.local);
             }

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -74,7 +74,7 @@ pub(crate) fn codegen_tls_ref<'tcx>(
 pub(crate) fn eval_mir_constant<'tcx>(
     fx: &FunctionCx<'_, '_, 'tcx>,
     constant: &ConstOperand<'tcx>,
-) -> (ConstValue<'tcx>, Ty<'tcx>) {
+) -> (ConstValue, Ty<'tcx>) {
     let cv = fx.monomorphize(constant.const_);
     // This cannot fail because we checked all required_consts in advance.
     let val = cv
@@ -93,7 +93,7 @@ pub(crate) fn codegen_constant_operand<'tcx>(
 
 pub(crate) fn codegen_const_value<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
-    const_val: ConstValue<'tcx>,
+    const_val: ConstValue,
     ty: Ty<'tcx>,
 ) -> CValue<'tcx> {
     let layout = fx.layout_of(ty);
@@ -210,8 +210,7 @@ pub(crate) fn codegen_const_value<'tcx>(
                 .offset_i64(fx, i64::try_from(offset.bytes()).unwrap()),
             layout,
         ),
-        ConstValue::Slice { data, meta } => {
-            let alloc_id = fx.tcx.reserve_and_set_memory_alloc(data);
+        ConstValue::Slice { alloc_id, meta } => {
             let ptr = pointer_for_allocation(fx, alloc_id).get_addr(fx);
             let len = fx.bcx.ins().iconst(fx.pointer_type, meta as i64);
             CValue::by_val_pair(ptr, len, layout)

compiler/rustc_codegen_gcc/messages.ftl

@@ -3,12 +3,4 @@ codegen_gcc_unwinding_inline_asm =
 
 codegen_gcc_copy_bitcode = failed to copy bitcode to object file: {$err}
 
-codegen_gcc_dynamic_linking_with_lto =
-    cannot prefer dynamic linking when performing LTO
-    .note = only 'staticlib', 'bin', and 'cdylib' outputs are supported with LTO
-
-codegen_gcc_lto_disallowed = lto can only be run for executables, cdylibs and static library outputs
-
-codegen_gcc_lto_dylib = lto cannot be used for `dylib` crate type without `-Zdylib-lto`
-
 codegen_gcc_lto_bitcode_from_rlib = failed to get bitcode from object file for LTO ({$gcc_err})

compiler/rustc_codegen_gcc/src/back/lto.rs

@@ -25,35 +25,21 @@ use std::sync::Arc;
 use gccjit::{Context, OutputKind};
 use object::read::archive::ArchiveFile;
 use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule, ThinShared};
-use rustc_codegen_ssa::back::symbol_export;
 use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput};
 use rustc_codegen_ssa::traits::*;
 use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, looks_like_rust_object_file};
 use rustc_data_structures::memmap::Mmap;
 use rustc_errors::{DiagCtxtHandle, FatalError};
-use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::bug;
 use rustc_middle::dep_graph::WorkProduct;
-use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel};
-use rustc_session::config::{CrateType, Lto};
+use rustc_session::config::Lto;
 use rustc_target::spec::RelocModel;
 use tempfile::{TempDir, tempdir};
 
 use crate::back::write::save_temp_bitcode;
-use crate::errors::{DynamicLinkingWithLTO, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib};
+use crate::errors::LtoBitcodeFromRlib;
 use crate::{GccCodegenBackend, GccContext, SyncContext, to_gcc_opt_level};
 
-pub fn crate_type_allows_lto(crate_type: CrateType) -> bool {
-    match crate_type {
-        CrateType::Executable
-        | CrateType::Dylib
-        | CrateType::Staticlib
-        | CrateType::Cdylib
-        | CrateType::Sdylib => true,
-        CrateType::Rlib | CrateType::ProcMacro => false,
-    }
-}
-
 struct LtoData {
     // TODO(antoyo): use symbols_below_threshold.
     //symbols_below_threshold: Vec<String>,
@@ -63,18 +49,9 @@ struct LtoData {
 
 fn prepare_lto(
     cgcx: &CodegenContext<GccCodegenBackend>,
+    each_linked_rlib_for_lto: &[PathBuf],
     dcx: DiagCtxtHandle<'_>,
 ) -> Result<LtoData, FatalError> {
-    let export_threshold = match cgcx.lto {
-        // We're just doing LTO for our one crate
-        Lto::ThinLocal => SymbolExportLevel::Rust,
-
-        // We're doing LTO for the entire crate graph
-        Lto::Fat | Lto::Thin => symbol_export::crates_export_threshold(&cgcx.crate_types),
-
-        Lto::No => panic!("didn't request LTO but we're doing LTO"),
-    };
-
     let tmp_path = match tempdir() {
         Ok(tmp_path) => tmp_path,
         Err(error) => {
@@ -83,20 +60,6 @@ fn prepare_lto(
         }
     };
 
-    let symbol_filter = &|&(ref name, info): &(String, SymbolExportInfo)| {
-        if info.level.is_below_threshold(export_threshold) || info.used {
-            Some(name.clone())
-        } else {
-            None
-        }
-    };
-    let exported_symbols = cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
-    let mut symbols_below_threshold = {
-        let _timer = cgcx.prof.generic_activity("GCC_lto_generate_symbols_below_threshold");
-        exported_symbols[&LOCAL_CRATE].iter().filter_map(symbol_filter).collect::<Vec<String>>()
-    };
-    info!("{} symbols to preserve in this crate", symbols_below_threshold.len());
-
     // If we're performing LTO for the entire crate graph, then for each of our
     // upstream dependencies, find the corresponding rlib and load the bitcode
     // from the archive.
@@ -105,32 +68,7 @@ fn prepare_lto(
     // with either fat or thin LTO
     let mut upstream_modules = Vec::new();
     if cgcx.lto != Lto::ThinLocal {
-        // Make sure we actually can run LTO
-        for crate_type in cgcx.crate_types.iter() {
-            if !crate_type_allows_lto(*crate_type) {
-                dcx.emit_err(LtoDisallowed);
-                return Err(FatalError);
-            }
-            if *crate_type == CrateType::Dylib && !cgcx.opts.unstable_opts.dylib_lto {
-                dcx.emit_err(LtoDylib);
-                return Err(FatalError);
-            }
-        }
-
-        if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto {
-            dcx.emit_err(DynamicLinkingWithLTO);
-            return Err(FatalError);
-        }
-
-        for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() {
-            let exported_symbols =
-                cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
-            {
-                let _timer = cgcx.prof.generic_activity("GCC_lto_generate_symbols_below_threshold");
-                symbols_below_threshold
-                    .extend(exported_symbols[&cnum].iter().filter_map(symbol_filter));
-            }
-
+        for path in each_linked_rlib_for_lto {
             let archive_data = unsafe {
                 Mmap::map(File::open(path).expect("couldn't open rlib")).expect("couldn't map rlib")
             };
@@ -174,19 +112,18 @@ fn save_as_file(obj: &[u8], path: &Path) -> Result<(), LtoBitcodeFromRlib> {
 /// for further optimization.
 pub(crate) fn run_fat(
     cgcx: &CodegenContext<GccCodegenBackend>,
+    each_linked_rlib_for_lto: &[PathBuf],
     modules: Vec<FatLtoInput<GccCodegenBackend>>,
-    cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
 ) -> Result<ModuleCodegen<GccContext>, FatalError> {
     let dcx = cgcx.create_dcx();
     let dcx = dcx.handle();
-    let lto_data = prepare_lto(cgcx, dcx)?;
+    let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx)?;
     /*let symbols_below_threshold =
     lto_data.symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();*/
     fat_lto(
         cgcx,
         dcx,
         modules,
-        cached_modules,
         lto_data.upstream_modules,
         lto_data.tmp_path,
         //&lto_data.symbols_below_threshold,
@@ -197,7 +134,6 @@ fn fat_lto(
     cgcx: &CodegenContext<GccCodegenBackend>,
     _dcx: DiagCtxtHandle<'_>,
     modules: Vec<FatLtoInput<GccCodegenBackend>>,
-    cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
     mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
     tmp_path: TempDir,
     //symbols_below_threshold: &[String],
@@ -211,21 +147,12 @@ fn fat_lto(
     //   modules that are serialized in-memory.
     // * `in_memory` contains modules which are already parsed and in-memory,
     //   such as from multi-CGU builds.
-    //
-    // All of `cached_modules` (cached from previous incremental builds) can
-    // immediately go onto the `serialized_modules` modules list and then we can
-    // split the `modules` array into these two lists.
     let mut in_memory = Vec::new();
-    serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| {
-        info!("pushing cached module {:?}", wp.cgu_name);
-        (buffer, CString::new(wp.cgu_name).unwrap())
-    }));
     for module in modules {
         match module {
             FatLtoInput::InMemory(m) => in_memory.push(m),
             FatLtoInput::Serialized { name, buffer } => {
                 info!("pushing serialized module {:?}", name);
-                let buffer = SerializedModule::Local(buffer);
                 serialized_modules.push((buffer, CString::new(name).unwrap()));
             }
         }
@@ -356,12 +283,13 @@ impl ModuleBufferMethods for ModuleBuffer {
 /// can simply be copied over from the incr. comp. cache.
 pub(crate) fn run_thin(
     cgcx: &CodegenContext<GccCodegenBackend>,
+    each_linked_rlib_for_lto: &[PathBuf],
     modules: Vec<(String, ThinBuffer)>,
     cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
 ) -> Result<(Vec<ThinModule<GccCodegenBackend>>, Vec<WorkProduct>), FatalError> {
     let dcx = cgcx.create_dcx();
     let dcx = dcx.handle();
-    let lto_data = prepare_lto(cgcx, dcx)?;
+    let lto_data = prepare_lto(cgcx, each_linked_rlib_for_lto, dcx)?;
     if cgcx.opts.cg.linker_plugin_lto.enabled() {
         unreachable!(
             "We should never reach this case if the LTO step \

compiler/rustc_codegen_gcc/src/errors.rs

@@ -14,19 +14,6 @@ pub(crate) struct CopyBitcode {
     pub err: std::io::Error,
 }
 
-#[derive(Diagnostic)]
-#[diag(codegen_gcc_dynamic_linking_with_lto)]
-#[note]
-pub(crate) struct DynamicLinkingWithLTO;
-
-#[derive(Diagnostic)]
-#[diag(codegen_gcc_lto_disallowed)]
-pub(crate) struct LtoDisallowed;
-
-#[derive(Diagnostic)]
-#[diag(codegen_gcc_lto_dylib)]
-pub(crate) struct LtoDylib;
-
 #[derive(Diagnostic)]
 #[diag(codegen_gcc_lto_bitcode_from_rlib)]
 pub(crate) struct LtoBitcodeFromRlib {

compiler/rustc_codegen_gcc/src/lib.rs

@@ -81,6 +81,7 @@ mod type_of;
 use std::any::Any;
 use std::fmt::Debug;
 use std::ops::Deref;
+use std::path::PathBuf;
 #[cfg(not(feature = "master"))]
 use std::sync::atomic::AtomicBool;
 #[cfg(not(feature = "master"))]
@@ -358,23 +359,28 @@ impl WriteBackendMethods for GccCodegenBackend {
 
     fn run_and_optimize_fat_lto(
         cgcx: &CodegenContext<Self>,
+        // FIXME(bjorn3): Limit LTO exports to these symbols
+        _exported_symbols_for_lto: &[String],
+        each_linked_rlib_for_lto: &[PathBuf],
         modules: Vec<FatLtoInput<Self>>,
-        cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
         diff_fncs: Vec<AutoDiffItem>,
     ) -> Result<ModuleCodegen<Self::Module>, FatalError> {
         if !diff_fncs.is_empty() {
             unimplemented!();
         }
 
-        back::lto::run_fat(cgcx, modules, cached_modules)
+        back::lto::run_fat(cgcx, each_linked_rlib_for_lto, modules)
     }
 
     fn run_thin_lto(
         cgcx: &CodegenContext<Self>,
+        // FIXME(bjorn3): Limit LTO exports to these symbols
+        _exported_symbols_for_lto: &[String],
+        each_linked_rlib_for_lto: &[PathBuf],
         modules: Vec<(String, Self::ThinBuffer)>,
         cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
     ) -> Result<(Vec<ThinModule<Self>>, Vec<WorkProduct>), FatalError> {
-        back::lto::run_thin(cgcx, modules, cached_modules)
+        back::lto::run_thin(cgcx, each_linked_rlib_for_lto, modules, cached_modules)
     }
 
     fn print_pass_timings(&self) {

compiler/rustc_codegen_llvm/messages.ftl

@@ -2,10 +2,6 @@ codegen_llvm_autodiff_without_enable = using the autodiff feature requires -Z au
 
 codegen_llvm_copy_bitcode = failed to copy bitcode to object file: {$err}
 
-codegen_llvm_dynamic_linking_with_lto =
-    cannot prefer dynamic linking when performing LTO
-    .note = only 'staticlib', 'bin', and 'cdylib' outputs are supported with LTO
-
 
 codegen_llvm_fixed_x18_invalid_arch = the `-Zfixed-x18` flag is not supported on the `{$arch}` architecture
 
@@ -18,12 +14,6 @@ codegen_llvm_load_bitcode_with_llvm_err = failed to load bitcode of module "{$na
 
 codegen_llvm_lto_bitcode_from_rlib = failed to get bitcode from object file for LTO ({$llvm_err})
 
-codegen_llvm_lto_disallowed = lto can only be run for executables, cdylibs and static library outputs
-
-codegen_llvm_lto_dylib = lto cannot be used for `dylib` crate type without `-Zdylib-lto`
-
-codegen_llvm_lto_proc_macro = lto cannot be used for `proc-macro` crate type without `-Zdylib-lto`
-
 codegen_llvm_mismatch_data_layout =
     data-layout for target `{$rustc_target}`, `{$rustc_layout}`, differs from LLVM target's `{$llvm_target}` default layout, `{$llvm_layout}`
 

compiler/rustc_codegen_llvm/src/back/lto.rs

@@ -1,33 +1,28 @@
 use std::collections::BTreeMap;
 use std::ffi::{CStr, CString};
 use std::fs::File;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::ptr::NonNull;
 use std::sync::Arc;
 use std::{io, iter, slice};
 
 use object::read::archive::ArchiveFile;
 use rustc_codegen_ssa::back::lto::{SerializedModule, ThinModule, ThinShared};
-use rustc_codegen_ssa::back::symbol_export;
 use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput};
 use rustc_codegen_ssa::traits::*;
 use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, looks_like_rust_object_file};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::memmap::Mmap;
 use rustc_errors::{DiagCtxtHandle, FatalError};
-use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::bug;
 use rustc_middle::dep_graph::WorkProduct;
-use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel};
-use rustc_session::config::{self, CrateType, Lto};
+use rustc_session::config::{self, Lto};
 use tracing::{debug, info};
 
 use crate::back::write::{
     self, CodegenDiagnosticsStage, DiagnosticHandlers, bitcode_section_name, save_temp_bitcode,
 };
-use crate::errors::{
-    DynamicLinkingWithLTO, LlvmError, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib, LtoProcMacro,
-};
+use crate::errors::{LlvmError, LtoBitcodeFromRlib};
 use crate::llvm::AttributePlace::Function;
 use crate::llvm::{self, build_string};
 use crate::{LlvmCodegenBackend, ModuleLlvm, SimpleCx, attributes};
@@ -36,45 +31,21 @@ use crate::{LlvmCodegenBackend, ModuleLlvm, SimpleCx, attributes};
 /// session to determine which CGUs we can reuse.
 const THIN_LTO_KEYS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-keys.bin";
 
-fn crate_type_allows_lto(crate_type: CrateType) -> bool {
-    match crate_type {
-        CrateType::Executable
-        | CrateType::Dylib
-        | CrateType::Staticlib
-        | CrateType::Cdylib
-        | CrateType::ProcMacro
-        | CrateType::Sdylib => true,
-        CrateType::Rlib => false,
-    }
-}
-
 fn prepare_lto(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
+    exported_symbols_for_lto: &[String],
+    each_linked_rlib_for_lto: &[PathBuf],
     dcx: DiagCtxtHandle<'_>,
 ) -> Result<(Vec<CString>, Vec<(SerializedModule<ModuleBuffer>, CString)>), FatalError> {
-    let export_threshold = match cgcx.lto {
-        // We're just doing LTO for our one crate
-        Lto::ThinLocal => SymbolExportLevel::Rust,
+    let mut symbols_below_threshold = exported_symbols_for_lto
+        .iter()
+        .map(|symbol| CString::new(symbol.to_owned()).unwrap())
+        .collect::<Vec<CString>>();
 
-        // We're doing LTO for the entire crate graph
-        Lto::Fat | Lto::Thin => symbol_export::crates_export_threshold(&cgcx.crate_types),
-
-        Lto::No => panic!("didn't request LTO but we're doing LTO"),
-    };
-
-    let symbol_filter = &|&(ref name, info): &(String, SymbolExportInfo)| {
-        if info.level.is_below_threshold(export_threshold) || info.used {
-            Some(CString::new(name.as_str()).unwrap())
-        } else {
-            None
-        }
-    };
-    let exported_symbols = cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
-    let mut symbols_below_threshold = {
-        let _timer = cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold");
-        exported_symbols[&LOCAL_CRATE].iter().filter_map(symbol_filter).collect::<Vec<CString>>()
-    };
-    info!("{} symbols to preserve in this crate", symbols_below_threshold.len());
+    // __llvm_profile_counter_bias is pulled in at link time by an undefined reference to
+    // __llvm_profile_runtime, therefore we won't know until link time if this symbol
+    // should have default visibility.
+    symbols_below_threshold.push(c"__llvm_profile_counter_bias".to_owned());
 
     // If we're performing LTO for the entire crate graph, then for each of our
     // upstream dependencies, find the corresponding rlib and load the bitcode
@@ -84,37 +55,7 @@ fn prepare_lto(
     // with either fat or thin LTO
     let mut upstream_modules = Vec::new();
     if cgcx.lto != Lto::ThinLocal {
-        // Make sure we actually can run LTO
-        for crate_type in cgcx.crate_types.iter() {
-            if !crate_type_allows_lto(*crate_type) {
-                dcx.emit_err(LtoDisallowed);
-                return Err(FatalError);
-            } else if *crate_type == CrateType::Dylib {
-                if !cgcx.opts.unstable_opts.dylib_lto {
-                    dcx.emit_err(LtoDylib);
-                    return Err(FatalError);
-                }
-            } else if *crate_type == CrateType::ProcMacro && !cgcx.opts.unstable_opts.dylib_lto {
-                dcx.emit_err(LtoProcMacro);
-                return Err(FatalError);
-            }
-        }
-
-        if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto {
-            dcx.emit_err(DynamicLinkingWithLTO);
-            return Err(FatalError);
-        }
-
-        for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() {
-            let exported_symbols =
-                cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
-            {
-                let _timer =
-                    cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold");
-                symbols_below_threshold
-                    .extend(exported_symbols[&cnum].iter().filter_map(symbol_filter));
-            }
-
+        for path in each_linked_rlib_for_lto {
             let archive_data = unsafe {
                 Mmap::map(std::fs::File::open(&path).expect("couldn't open rlib"))
                     .expect("couldn't map rlib")
@@ -147,10 +88,6 @@ fn prepare_lto(
         }
     }
 
-    // __llvm_profile_counter_bias is pulled in at link time by an undefined reference to
-    // __llvm_profile_runtime, therefore we won't know until link time if this symbol
-    // should have default visibility.
-    symbols_below_threshold.push(c"__llvm_profile_counter_bias".to_owned());
     Ok((symbols_below_threshold, upstream_modules))
 }
 
@@ -199,15 +136,17 @@ fn get_bitcode_slice_from_object_data<'a>(
 /// for further optimization.
 pub(crate) fn run_fat(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
+    exported_symbols_for_lto: &[String],
+    each_linked_rlib_for_lto: &[PathBuf],
     modules: Vec<FatLtoInput<LlvmCodegenBackend>>,
-    cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
 ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> {
     let dcx = cgcx.create_dcx();
     let dcx = dcx.handle();
-    let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, dcx)?;
+    let (symbols_below_threshold, upstream_modules) =
+        prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx)?;
     let symbols_below_threshold =
         symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();
-    fat_lto(cgcx, dcx, modules, cached_modules, upstream_modules, &symbols_below_threshold)
+    fat_lto(cgcx, dcx, modules, upstream_modules, &symbols_below_threshold)
 }
 
 /// Performs thin LTO by performing necessary global analysis and returning two
@@ -215,12 +154,15 @@ pub(crate) fn run_fat(
 /// can simply be copied over from the incr. comp. cache.
 pub(crate) fn run_thin(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
+    exported_symbols_for_lto: &[String],
+    each_linked_rlib_for_lto: &[PathBuf],
     modules: Vec<(String, ThinBuffer)>,
     cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
 ) -> Result<(Vec<ThinModule<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> {
     let dcx = cgcx.create_dcx();
     let dcx = dcx.handle();
-    let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, dcx)?;
+    let (symbols_below_threshold, upstream_modules) =
+        prepare_lto(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, dcx)?;
     let symbols_below_threshold =
         symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();
     if cgcx.opts.cg.linker_plugin_lto.enabled() {
@@ -245,7 +187,6 @@ fn fat_lto(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
     dcx: DiagCtxtHandle<'_>,
     modules: Vec<FatLtoInput<LlvmCodegenBackend>>,
-    cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
     mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
     symbols_below_threshold: &[*const libc::c_char],
 ) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> {
@@ -258,21 +199,12 @@ fn fat_lto(
     //   modules that are serialized in-memory.
     // * `in_memory` contains modules which are already parsed and in-memory,
     //   such as from multi-CGU builds.
-    //
-    // All of `cached_modules` (cached from previous incremental builds) can
-    // immediately go onto the `serialized_modules` modules list and then we can
-    // split the `modules` array into these two lists.
     let mut in_memory = Vec::new();
-    serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| {
-        info!("pushing cached module {:?}", wp.cgu_name);
-        (buffer, CString::new(wp.cgu_name).unwrap())
-    }));
     for module in modules {
         match module {
             FatLtoInput::InMemory(m) => in_memory.push(m),
             FatLtoInput::Serialized { name, buffer } => {
                 info!("pushing serialized module {:?}", name);
-                let buffer = SerializedModule::Local(buffer);
                 serialized_modules.push((buffer, CString::new(name).unwrap()));
             }
         }

compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/spans.rs

@@ -39,10 +39,7 @@ impl Coords {
 /// or other expansions), and if it does happen then skipping a span or function is
 /// better than an ICE or `llvm-cov` failure that the user might have no way to avoid.
 pub(crate) fn make_coords(source_map: &SourceMap, file: &SourceFile, span: Span) -> Option<Coords> {
-    if span.is_empty() {
-        debug_assert!(false, "can't make coords from empty span: {span:?}");
-        return None;
-    }
+    let span = ensure_non_empty_span(source_map, span)?;
 
     let lo = span.lo();
     let hi = span.hi();
@@ -73,6 +70,29 @@ pub(crate) fn make_coords(source_map: &SourceMap, file: &SourceFile, span: Span)
     })
 }
 
+fn ensure_non_empty_span(source_map: &SourceMap, span: Span) -> Option<Span> {
+    if !span.is_empty() {
+        return Some(span);
+    }
+
+    // The span is empty, so try to enlarge it to cover an adjacent '{' or '}'.
+    source_map
+        .span_to_source(span, |src, start, end| try {
+            // Adjusting span endpoints by `BytePos(1)` is normally a bug,
+            // but in this case we have specifically checked that the character
+            // we're skipping over is one of two specific ASCII characters, so
+            // adjusting by exactly 1 byte is correct.
+            if src.as_bytes().get(end).copied() == Some(b'{') {
+                Some(span.with_hi(span.hi() + BytePos(1)))
+            } else if start > 0 && src.as_bytes()[start - 1] == b'}' {
+                Some(span.with_lo(span.lo() - BytePos(1)))
+            } else {
+                None
+            }
+        })
+        .ok()?
+}
+
 /// If `llvm-cov` sees a source region that is improperly ordered (end < start),
 /// it will immediately exit with a fatal error. To prevent that from happening,
 /// discard regions that are improperly ordered, or might be interpreted in a

compiler/rustc_codegen_llvm/src/errors.rs

@@ -20,11 +20,6 @@ pub(crate) struct SymbolAlreadyDefined<'a> {
 #[diag(codegen_llvm_sanitizer_memtag_requires_mte)]
 pub(crate) struct SanitizerMemtagRequiresMte;
 
-#[derive(Diagnostic)]
-#[diag(codegen_llvm_dynamic_linking_with_lto)]
-#[note]
-pub(crate) struct DynamicLinkingWithLTO;
-
 pub(crate) struct ParseTargetMachineConfig<'a>(pub LlvmError<'a>);
 
 impl<G: EmissionGuarantee> Diagnostic<'_, G> for ParseTargetMachineConfig<'_> {
@@ -41,18 +36,6 @@ impl<G: EmissionGuarantee> Diagnostic<'_, G> for ParseTargetMachineConfig<'_> {
 #[diag(codegen_llvm_autodiff_without_enable)]
 pub(crate) struct AutoDiffWithoutEnable;
 
-#[derive(Diagnostic)]
-#[diag(codegen_llvm_lto_disallowed)]
-pub(crate) struct LtoDisallowed;
-
-#[derive(Diagnostic)]
-#[diag(codegen_llvm_lto_dylib)]
-pub(crate) struct LtoDylib;
-
-#[derive(Diagnostic)]
-#[diag(codegen_llvm_lto_proc_macro)]
-pub(crate) struct LtoProcMacro;
-
 #[derive(Diagnostic)]
 #[diag(codegen_llvm_lto_bitcode_from_rlib)]
 pub(crate) struct LtoBitcodeFromRlib {

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -382,26 +382,16 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                 let width = size.bits();
                 let llty = self.type_ix(width);
                 match name {
-                    sym::ctlz | sym::cttz => {
-                        let y = self.const_bool(false);
-                        let ret = self.call_intrinsic(
-                            format!("llvm.{name}"),
-                            &[llty],
-                            &[args[0].immediate(), y],
-                        );
-
-                        self.intcast(ret, result.layout.llvm_type(self), false)
-                    }
-                    sym::ctlz_nonzero => {
-                        let y = self.const_bool(true);
+                    sym::ctlz | sym::ctlz_nonzero | sym::cttz | sym::cttz_nonzero => {
+                        let y =
+                            self.const_bool(name == sym::ctlz_nonzero || name == sym::cttz_nonzero);
+                        let llvm_name = if name == sym::ctlz || name == sym::ctlz_nonzero {
+                            "llvm.ctlz"
+                        } else {
+                            "llvm.cttz"
+                        };
                         let ret =
-                            self.call_intrinsic("llvm.ctlz", &[llty], &[args[0].immediate(), y]);
-                        self.intcast(ret, result.layout.llvm_type(self), false)
-                    }
-                    sym::cttz_nonzero => {
-                        let y = self.const_bool(true);
-                        let ret =
-                            self.call_intrinsic("llvm.cttz", &[llty], &[args[0].immediate(), y]);
+                            self.call_intrinsic(llvm_name, &[llty], &[args[0].immediate(), y]);
                         self.intcast(ret, result.layout.llvm_type(self), false)
                     }
                     sym::ctpop => {

compiler/rustc_codegen_llvm/src/lib.rs

@@ -22,6 +22,7 @@
 use std::any::Any;
 use std::ffi::CStr;
 use std::mem::ManuallyDrop;
+use std::path::PathBuf;
 
 use back::owned_target_machine::OwnedTargetMachine;
 use back::write::{create_informational_target_machine, create_target_machine};
@@ -176,11 +177,13 @@ impl WriteBackendMethods for LlvmCodegenBackend {
     }
     fn run_and_optimize_fat_lto(
         cgcx: &CodegenContext<Self>,
+        exported_symbols_for_lto: &[String],
+        each_linked_rlib_for_lto: &[PathBuf],
         modules: Vec<FatLtoInput<Self>>,
-        cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
         diff_fncs: Vec<AutoDiffItem>,
     ) -> Result<ModuleCodegen<Self::Module>, FatalError> {
-        let mut module = back::lto::run_fat(cgcx, modules, cached_modules)?;
+        let mut module =
+            back::lto::run_fat(cgcx, exported_symbols_for_lto, each_linked_rlib_for_lto, modules)?;
 
         if !diff_fncs.is_empty() {
             builder::autodiff::differentiate(&module, cgcx, diff_fncs)?;
@@ -194,10 +197,18 @@ impl WriteBackendMethods for LlvmCodegenBackend {
     }
     fn run_thin_lto(
         cgcx: &CodegenContext<Self>,
+        exported_symbols_for_lto: &[String],
+        each_linked_rlib_for_lto: &[PathBuf],
         modules: Vec<(String, Self::ThinBuffer)>,
         cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
     ) -> Result<(Vec<ThinModule<Self>>, Vec<WorkProduct>), FatalError> {
-        back::lto::run_thin(cgcx, modules, cached_modules)
+        back::lto::run_thin(
+            cgcx,
+            exported_symbols_for_lto,
+            each_linked_rlib_for_lto,
+            modules,
+            cached_modules,
+        )
     }
     fn optimize(
         cgcx: &CodegenContext<Self>,

compiler/rustc_codegen_llvm/src/llvm_util.rs

@@ -405,6 +405,8 @@ fn update_target_reliable_float_cfg(sess: &Session, cfg: &mut TargetConfig) {
         ("mips64" | "mips64r6", _) => false,
         // Selection bug <https://github.com/llvm/llvm-project/issues/95471>
         ("nvptx64", _) => false,
+        // Unsupported https://github.com/llvm/llvm-project/issues/121122
+        ("amdgpu", _) => false,
         // ABI bugs <https://github.com/rust-lang/rust/issues/125109> et al. (full
         // list at <https://github.com/rust-lang/rust/issues/116909>)
         ("powerpc" | "powerpc64", _) => false,
@@ -433,6 +435,9 @@ fn update_target_reliable_float_cfg(sess: &Session, cfg: &mut TargetConfig) {
         // This rules out anything that doesn't have `long double` = `binary128`; <= 32 bits
         // (ld is `f64`), anything other than Linux (Windows and MacOS use `f64`), and `x86`
         // (ld is 80-bit extended precision).
+        //
+        // musl does not implement the symbols required for f128 math at all.
+        _ if target_env == "musl" => false,
         ("x86_64", _) => false,
         (_, "linux") if target_pointer_width == 64 => true,
         _ => false,

compiler/rustc_codegen_ssa/messages.ftl

@@ -35,6 +35,10 @@ codegen_ssa_dlltool_fail_import_library =
     {$stdout}
     {$stderr}
 
+codegen_ssa_dynamic_linking_with_lto =
+    cannot prefer dynamic linking when performing LTO
+    .note = only 'staticlib', 'bin', and 'cdylib' outputs are supported with LTO
+
 codegen_ssa_error_calling_dlltool =
     Error calling dlltool '{$dlltool_path}': {$error}
 
@@ -191,6 +195,12 @@ codegen_ssa_linker_unsupported_modifier = `as-needed` modifier not supported for
 
 codegen_ssa_linking_failed = linking with `{$linker_path}` failed: {$exit_status}
 
+codegen_ssa_lto_disallowed = lto can only be run for executables, cdylibs and static library outputs
+
+codegen_ssa_lto_dylib = lto cannot be used for `dylib` crate type without `-Zdylib-lto`
+
+codegen_ssa_lto_proc_macro = lto cannot be used for `proc-macro` crate type without `-Zdylib-lto`
+
 codegen_ssa_malformed_cgu_name =
     found malformed codegen unit name `{$user_path}`. codegen units names must always start with the name of the crate (`{$crate_name}` in this case).
 

compiler/rustc_codegen_ssa/src/back/link/raw_dylib.rs

@@ -4,7 +4,7 @@ use std::path::{Path, PathBuf};
 
 use rustc_abi::Endian;
 use rustc_data_structures::base_n::{CASE_INSENSITIVE, ToBaseN};
-use rustc_data_structures::fx::FxIndexMap;
+use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
 use rustc_data_structures::stable_hasher::StableHasher;
 use rustc_hashes::Hash128;
 use rustc_session::Session;
@@ -214,7 +214,7 @@ pub(super) fn create_raw_dylib_elf_stub_shared_objects<'a>(
 /// It exports all the provided symbols, but is otherwise empty.
 fn create_elf_raw_dylib_stub(sess: &Session, soname: &str, symbols: &[DllImport]) -> Vec<u8> {
     use object::write::elf as write;
-    use object::{Architecture, elf};
+    use object::{AddressSize, Architecture, elf};
 
     let mut stub_buf = Vec::new();
 
@@ -226,47 +226,6 @@ fn create_elf_raw_dylib_stub(sess: &Session, soname: &str, symbols: &[DllImport]
     // It is important that the order of reservation matches the order of writing.
     // The object crate contains many debug asserts that fire if you get this wrong.
 
-    let endianness = match sess.target.options.endian {
-        Endian::Little => object::Endianness::Little,
-        Endian::Big => object::Endianness::Big,
-    };
-    let mut stub = write::Writer::new(endianness, true, &mut stub_buf);
-
-    // These initial reservations don't reserve any bytes in the binary yet,
-    // they just allocate in the internal data structures.
-
-    // First, we crate the dynamic symbol table. It starts with a null symbol
-    // and then all the symbols and their dynamic strings.
-    stub.reserve_null_dynamic_symbol_index();
-
-    let dynstrs = symbols
-        .iter()
-        .map(|sym| {
-            stub.reserve_dynamic_symbol_index();
-            (sym, stub.add_dynamic_string(sym.name.as_str().as_bytes()))
-        })
-        .collect::<Vec<_>>();
-
-    let soname = stub.add_dynamic_string(soname.as_bytes());
-
-    // Reserve the sections.
-    // We have the minimal sections for a dynamic SO and .text where we point our dummy symbols to.
-    stub.reserve_shstrtab_section_index();
-    let text_section_name = stub.add_section_name(".text".as_bytes());
-    let text_section = stub.reserve_section_index();
-    stub.reserve_dynstr_section_index();
-    stub.reserve_dynsym_section_index();
-    stub.reserve_dynamic_section_index();
-
-    // These reservations now determine the actual layout order of the object file.
-    stub.reserve_file_header();
-    stub.reserve_shstrtab();
-    stub.reserve_section_headers();
-    stub.reserve_dynstr();
-    stub.reserve_dynsym();
-    stub.reserve_dynamic(2); // DT_SONAME, DT_NULL
-
-    // First write the ELF header with the arch information.
     let Some((arch, sub_arch)) = sess.target.object_architecture(&sess.unstable_target_features)
     else {
         sess.dcx().fatal(format!(
@@ -274,6 +233,87 @@ fn create_elf_raw_dylib_stub(sess: &Session, soname: &str, symbols: &[DllImport]
             sess.target.arch
         ));
     };
+
+    let endianness = match sess.target.options.endian {
+        Endian::Little => object::Endianness::Little,
+        Endian::Big => object::Endianness::Big,
+    };
+
+    let is_64 = match arch.address_size() {
+        Some(AddressSize::U8 | AddressSize::U16 | AddressSize::U32) => false,
+        Some(AddressSize::U64) => true,
+        _ => sess.dcx().fatal(format!(
+            "raw-dylib is not supported for the architecture `{}`",
+            sess.target.arch
+        )),
+    };
+
+    let mut stub = write::Writer::new(endianness, is_64, &mut stub_buf);
+
+    let mut vers = Vec::new();
+    let mut vers_map = FxHashMap::default();
+    let mut syms = Vec::new();
+
+    for symbol in symbols {
+        let symbol_name = symbol.name.as_str();
+        if let Some((name, version_name)) = symbol_name.split_once('@') {
+            assert!(!version_name.contains('@'));
+            let dynstr = stub.add_dynamic_string(name.as_bytes());
+            let ver = if let Some(&ver_id) = vers_map.get(version_name) {
+                ver_id
+            } else {
+                let id = vers.len();
+                vers_map.insert(version_name, id);
+                let dynstr = stub.add_dynamic_string(version_name.as_bytes());
+                vers.push((version_name, dynstr));
+                id
+            };
+            syms.push((name, dynstr, Some(ver)));
+        } else {
+            let dynstr = stub.add_dynamic_string(symbol_name.as_bytes());
+            syms.push((symbol_name, dynstr, None));
+        }
+    }
+
+    let soname = stub.add_dynamic_string(soname.as_bytes());
+
+    // These initial reservations don't reserve any bytes in the binary yet,
+    // they just allocate in the internal data structures.
+
+    // First, we create the dynamic symbol table. It starts with a null symbol
+    // and then all the symbols and their dynamic strings.
+    stub.reserve_null_dynamic_symbol_index();
+
+    for _ in syms.iter() {
+        stub.reserve_dynamic_symbol_index();
+    }
+
+    // Reserve the sections.
+    // We have the minimal sections for a dynamic SO and .text where we point our dummy symbols to.
+    stub.reserve_shstrtab_section_index();
+    let text_section_name = stub.add_section_name(".text".as_bytes());
+    let text_section = stub.reserve_section_index();
+    stub.reserve_dynsym_section_index();
+    stub.reserve_dynstr_section_index();
+    if !vers.is_empty() {
+        stub.reserve_gnu_versym_section_index();
+        stub.reserve_gnu_verdef_section_index();
+    }
+    stub.reserve_dynamic_section_index();
+
+    // These reservations now determine the actual layout order of the object file.
+    stub.reserve_file_header();
+    stub.reserve_shstrtab();
+    stub.reserve_section_headers();
+    stub.reserve_dynsym();
+    stub.reserve_dynstr();
+    if !vers.is_empty() {
+        stub.reserve_gnu_versym();
+        stub.reserve_gnu_verdef(1 + vers.len(), 1 + vers.len());
+    }
+    stub.reserve_dynamic(2); // DT_SONAME, DT_NULL
+
+    // First write the ELF header with the arch information.
     let e_machine = match (arch, sub_arch) {
         (Architecture::Aarch64, None) => elf::EM_AARCH64,
         (Architecture::Aarch64_Ilp32, None) => elf::EM_AARCH64,
@@ -342,18 +382,19 @@ fn create_elf_raw_dylib_stub(sess: &Session, soname: &str, symbols: &[DllImport]
         sh_addralign: 1,
         sh_entsize: 0,
     });
-    stub.write_dynstr_section_header(0);
     stub.write_dynsym_section_header(0, 1);
+    stub.write_dynstr_section_header(0);
+    if !vers.is_empty() {
+        stub.write_gnu_versym_section_header(0);
+        stub.write_gnu_verdef_section_header(0);
+    }
     stub.write_dynamic_section_header(0);
 
-    // .dynstr
-    stub.write_dynstr();
-
     // .dynsym
     stub.write_null_dynamic_symbol();
-    for (_, name) in dynstrs {
+    for (_name, dynstr, _ver) in syms.iter().copied() {
         stub.write_dynamic_symbol(&write::Sym {
-            name: Some(name),
+            name: Some(dynstr),
             st_info: (elf::STB_GLOBAL << 4) | elf::STT_NOTYPE,
             st_other: elf::STV_DEFAULT,
             section: Some(text_section),
@@ -363,10 +404,47 @@ fn create_elf_raw_dylib_stub(sess: &Session, soname: &str, symbols: &[DllImport]
         });
     }
 
+    // .dynstr
+    stub.write_dynstr();
+
+    // ld.bfd is unhappy if these sections exist without any symbols, so we only generate them when necessary.
+    if !vers.is_empty() {
+        // .gnu_version
+        stub.write_null_gnu_versym();
+        for (_name, _dynstr, ver) in syms.iter().copied() {
+            stub.write_gnu_versym(if let Some(ver) = ver {
+                assert!((2 + ver as u16) < elf::VERSYM_HIDDEN);
+                elf::VERSYM_HIDDEN | (2 + ver as u16)
+            } else {
+                1
+            });
+        }
+
+        // .gnu_version_d
+        stub.write_align_gnu_verdef();
+        stub.write_gnu_verdef(&write::Verdef {
+            version: elf::VER_DEF_CURRENT,
+            flags: elf::VER_FLG_BASE,
+            index: 1,
+            aux_count: 1,
+            name: soname,
+        });
+        for (ver, (_name, dynstr)) in vers.into_iter().enumerate() {
+            stub.write_gnu_verdef(&write::Verdef {
+                version: elf::VER_DEF_CURRENT,
+                flags: 0,
+                index: 2 + ver as u16,
+                aux_count: 1,
+                name: dynstr,
+            });
+        }
+    }
+
     // .dynamic
     // the DT_SONAME will be used by the linker to populate DT_NEEDED
     // which the loader uses to find the library.
     // DT_NULL terminates the .dynamic table.
+    stub.write_align_dynamic();
     stub.write_dynamic_string(elf::DT_SONAME, soname);
     stub.write_dynamic(elf::DT_NULL, 0);
 

compiler/rustc_codegen_ssa/src/back/lto.rs

@@ -2,7 +2,15 @@ use std::ffi::CString;
 use std::sync::Arc;
 
 use rustc_data_structures::memmap::Mmap;
+use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
+use rustc_middle::middle::exported_symbols::{ExportedSymbol, SymbolExportInfo, SymbolExportLevel};
+use rustc_middle::ty::TyCtxt;
+use rustc_session::config::{CrateType, Lto};
+use tracing::info;
 
+use crate::back::symbol_export::{self, symbol_name_for_instance_in_crate};
+use crate::back::write::CodegenContext;
+use crate::errors::{DynamicLinkingWithLTO, LtoDisallowed, LtoDylib, LtoProcMacro};
 use crate::traits::*;
 
 pub struct ThinModule<B: WriteBackendMethods> {
@@ -52,3 +60,86 @@ impl<M: ModuleBufferMethods> SerializedModule<M> {
         }
     }
 }
+
+fn crate_type_allows_lto(crate_type: CrateType) -> bool {
+    match crate_type {
+        CrateType::Executable
+        | CrateType::Dylib
+        | CrateType::Staticlib
+        | CrateType::Cdylib
+        | CrateType::ProcMacro
+        | CrateType::Sdylib => true,
+        CrateType::Rlib => false,
+    }
+}
+
+pub(super) fn exported_symbols_for_lto(
+    tcx: TyCtxt<'_>,
+    each_linked_rlib_for_lto: &[CrateNum],
+) -> Vec<String> {
+    let export_threshold = match tcx.sess.lto() {
+        // We're just doing LTO for our one crate
+        Lto::ThinLocal => SymbolExportLevel::Rust,
+
+        // We're doing LTO for the entire crate graph
+        Lto::Fat | Lto::Thin => symbol_export::crates_export_threshold(&tcx.crate_types()),
+
+        Lto::No => return vec![],
+    };
+
+    let copy_symbols = |cnum| {
+        tcx.exported_non_generic_symbols(cnum)
+            .iter()
+            .chain(tcx.exported_generic_symbols(cnum))
+            .filter_map(|&(s, info): &(ExportedSymbol<'_>, SymbolExportInfo)| {
+                if info.level.is_below_threshold(export_threshold) || info.used {
+                    Some(symbol_name_for_instance_in_crate(tcx, s, cnum))
+                } else {
+                    None
+                }
+            })
+            .collect::<Vec<_>>()
+    };
+    let mut symbols_below_threshold = {
+        let _timer = tcx.prof.generic_activity("lto_generate_symbols_below_threshold");
+        copy_symbols(LOCAL_CRATE)
+    };
+    info!("{} symbols to preserve in this crate", symbols_below_threshold.len());
+
+    // If we're performing LTO for the entire crate graph, then for each of our
+    // upstream dependencies, include their exported symbols.
+    if tcx.sess.lto() != Lto::ThinLocal {
+        for &cnum in each_linked_rlib_for_lto {
+            let _timer = tcx.prof.generic_activity("lto_generate_symbols_below_threshold");
+            symbols_below_threshold.extend(copy_symbols(cnum));
+        }
+    }
+
+    symbols_below_threshold
+}
+
+pub(super) fn check_lto_allowed<B: WriteBackendMethods>(cgcx: &CodegenContext<B>) {
+    if cgcx.lto == Lto::ThinLocal {
+        // Crate local LTO is always allowed
+        return;
+    }
+
+    let dcx = cgcx.create_dcx();
+
+    // Make sure we actually can run LTO
+    for crate_type in cgcx.crate_types.iter() {
+        if !crate_type_allows_lto(*crate_type) {
+            dcx.handle().emit_fatal(LtoDisallowed);
+        } else if *crate_type == CrateType::Dylib {
+            if !cgcx.opts.unstable_opts.dylib_lto {
+                dcx.handle().emit_fatal(LtoDylib);
+            }
+        } else if *crate_type == CrateType::ProcMacro && !cgcx.opts.unstable_opts.dylib_lto {
+            dcx.handle().emit_fatal(LtoProcMacro);
+        }
+    }
+
+    if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto {
+        dcx.handle().emit_fatal(DynamicLinkingWithLTO);
+    }
+}

compiler/rustc_codegen_ssa/src/back/write.rs

@@ -9,7 +9,7 @@ use std::{fs, io, mem, str, thread};
 use rustc_abi::Size;
 use rustc_ast::attr;
 use rustc_ast::expand::autodiff_attrs::AutoDiffItem;
-use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
+use rustc_data_structures::fx::FxIndexMap;
 use rustc_data_structures::jobserver::{self, Acquired};
 use rustc_data_structures::memmap::Mmap;
 use rustc_data_structures::profiling::{SelfProfilerRef, VerboseTimingGuard};
@@ -20,14 +20,12 @@ use rustc_errors::{
     Suggestions,
 };
 use rustc_fs_util::link_or_copy;
-use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
 use rustc_incremental::{
     copy_cgu_workproduct_to_incr_comp_cache_dir, in_incr_comp_dir, in_incr_comp_dir_sess,
 };
 use rustc_metadata::fs::copy_to_stdout;
 use rustc_middle::bug;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
-use rustc_middle::middle::exported_symbols::SymbolExportInfo;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::Session;
 use rustc_session::config::{
@@ -40,7 +38,7 @@ use tracing::debug;
 
 use super::link::{self, ensure_removed};
 use super::lto::{self, SerializedModule};
-use super::symbol_export::symbol_name_for_instance_in_crate;
+use crate::back::lto::check_lto_allowed;
 use crate::errors::{AutodiffWithoutLto, ErrorCreatingRemarkDir};
 use crate::traits::*;
 use crate::{
@@ -332,8 +330,6 @@ pub type TargetMachineFactoryFn<B> = Arc<
         + Sync,
 >;
 
-type ExportedSymbols = FxHashMap<CrateNum, Arc<Vec<(String, SymbolExportInfo)>>>;
-
 /// Additional resources used by optimize_and_codegen (not module specific)
 #[derive(Clone)]
 pub struct CodegenContext<B: WriteBackendMethods> {
@@ -343,10 +339,8 @@ pub struct CodegenContext<B: WriteBackendMethods> {
     pub save_temps: bool,
     pub fewer_names: bool,
     pub time_trace: bool,
-    pub exported_symbols: Option<Arc<ExportedSymbols>>,
     pub opts: Arc<config::Options>,
     pub crate_types: Vec<CrateType>,
-    pub each_linked_rlib_for_lto: Vec<(CrateNum, PathBuf)>,
     pub output_filenames: Arc<OutputFilenames>,
     pub invocation_temp: Option<String>,
     pub regular_module_config: Arc<ModuleConfig>,
@@ -401,13 +395,21 @@ impl<B: WriteBackendMethods> CodegenContext<B> {
 
 fn generate_thin_lto_work<B: ExtraBackendMethods>(
     cgcx: &CodegenContext<B>,
+    exported_symbols_for_lto: &[String],
+    each_linked_rlib_for_lto: &[PathBuf],
     needs_thin_lto: Vec<(String, B::ThinBuffer)>,
     import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>,
 ) -> Vec<(WorkItem<B>, u64)> {
     let _prof_timer = cgcx.prof.generic_activity("codegen_thin_generate_lto_work");
 
-    let (lto_modules, copy_jobs) =
-        B::run_thin_lto(cgcx, needs_thin_lto, import_only_modules).unwrap_or_else(|e| e.raise());
+    let (lto_modules, copy_jobs) = B::run_thin_lto(
+        cgcx,
+        exported_symbols_for_lto,
+        each_linked_rlib_for_lto,
+        needs_thin_lto,
+        import_only_modules,
+    )
+    .unwrap_or_else(|e| e.raise());
     lto_modules
         .into_iter()
         .map(|module| {
@@ -723,6 +725,8 @@ pub(crate) enum WorkItem<B: WriteBackendMethods> {
     CopyPostLtoArtifacts(CachedModuleCodegen),
     /// Performs fat LTO on the given module.
     FatLto {
+        exported_symbols_for_lto: Arc<Vec<String>>,
+        each_linked_rlib_for_lto: Vec<PathBuf>,
         needs_fat_lto: Vec<FatLtoInput<B>>,
         import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>,
         autodiff: Vec<AutoDiffItem>,
@@ -810,7 +814,7 @@ pub(crate) enum WorkItemResult<B: WriteBackendMethods> {
 }
 
 pub enum FatLtoInput<B: WriteBackendMethods> {
-    Serialized { name: String, buffer: B::ModuleBuffer },
+    Serialized { name: String, buffer: SerializedModule<B::ModuleBuffer> },
     InMemory(ModuleCodegen<B::Module>),
 }
 
@@ -899,7 +903,10 @@ fn execute_optimize_work_item<B: ExtraBackendMethods>(
                 fs::write(&path, buffer.data()).unwrap_or_else(|e| {
                     panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
                 });
-                Ok(WorkItemResult::NeedsFatLto(FatLtoInput::Serialized { name, buffer }))
+                Ok(WorkItemResult::NeedsFatLto(FatLtoInput::Serialized {
+                    name,
+                    buffer: SerializedModule::Local(buffer),
+                }))
             }
             None => Ok(WorkItemResult::NeedsFatLto(FatLtoInput::InMemory(module))),
         },
@@ -992,12 +999,24 @@ fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
 
 fn execute_fat_lto_work_item<B: ExtraBackendMethods>(
     cgcx: &CodegenContext<B>,
-    needs_fat_lto: Vec<FatLtoInput<B>>,
+    exported_symbols_for_lto: &[String],
+    each_linked_rlib_for_lto: &[PathBuf],
+    mut needs_fat_lto: Vec<FatLtoInput<B>>,
     import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>,
     autodiff: Vec<AutoDiffItem>,
     module_config: &ModuleConfig,
 ) -> Result<WorkItemResult<B>, FatalError> {
-    let module = B::run_and_optimize_fat_lto(cgcx, needs_fat_lto, import_only_modules, autodiff)?;
+    for (module, wp) in import_only_modules {
+        needs_fat_lto.push(FatLtoInput::Serialized { name: wp.cgu_name, buffer: module })
+    }
+
+    let module = B::run_and_optimize_fat_lto(
+        cgcx,
+        exported_symbols_for_lto,
+        each_linked_rlib_for_lto,
+        needs_fat_lto,
+        autodiff,
+    )?;
     let module = B::codegen(cgcx, module, module_config)?;
     Ok(WorkItemResult::Finished(module))
 }
@@ -1032,7 +1051,7 @@ pub(crate) enum Message<B: WriteBackendMethods> {
 
     /// The backend has finished processing a work item for a codegen unit.
     /// Sent from a backend worker thread.
-    WorkItem { result: Result<WorkItemResult<B>, Option<WorkerFatalError>>, worker_id: usize },
+    WorkItem { result: Result<WorkItemResult<B>, Option<WorkerFatalError>> },
 
     /// The frontend has finished generating something (backend IR or a
     /// post-LTO artifact) for a codegen unit, and it should be passed to the
@@ -1113,42 +1132,18 @@ fn start_executing_work<B: ExtraBackendMethods>(
     let autodiff_items = autodiff_items.to_vec();
 
     let mut each_linked_rlib_for_lto = Vec::new();
+    let mut each_linked_rlib_file_for_lto = Vec::new();
     drop(link::each_linked_rlib(crate_info, None, &mut |cnum, path| {
         if link::ignored_for_lto(sess, crate_info, cnum) {
             return;
         }
-        each_linked_rlib_for_lto.push((cnum, path.to_path_buf()));
+        each_linked_rlib_for_lto.push(cnum);
+        each_linked_rlib_file_for_lto.push(path.to_path_buf());
     }));
 
     // Compute the set of symbols we need to retain when doing LTO (if we need to)
-    let exported_symbols = {
-        let mut exported_symbols = FxHashMap::default();
-
-        let copy_symbols = |cnum| {
-            let symbols = tcx
-                .exported_non_generic_symbols(cnum)
-                .iter()
-                .chain(tcx.exported_generic_symbols(cnum))
-                .map(|&(s, lvl)| (symbol_name_for_instance_in_crate(tcx, s, cnum), lvl))
-                .collect();
-            Arc::new(symbols)
-        };
-
-        match sess.lto() {
-            Lto::No => None,
-            Lto::ThinLocal => {
-                exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
-                Some(Arc::new(exported_symbols))
-            }
-            Lto::Fat | Lto::Thin => {
-                exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
-                for &(cnum, ref _path) in &each_linked_rlib_for_lto {
-                    exported_symbols.insert(cnum, copy_symbols(cnum));
-                }
-                Some(Arc::new(exported_symbols))
-            }
-        }
-    };
+    let exported_symbols_for_lto =
+        Arc::new(lto::exported_symbols_for_lto(tcx, &each_linked_rlib_for_lto));
 
     // First up, convert our jobserver into a helper thread so we can use normal
     // mpsc channels to manage our messages and such.
@@ -1183,14 +1178,12 @@ fn start_executing_work<B: ExtraBackendMethods>(
 
     let cgcx = CodegenContext::<B> {
         crate_types: tcx.crate_types().to_vec(),
-        each_linked_rlib_for_lto,
         lto: sess.lto(),
         fewer_names: sess.fewer_names(),
         save_temps: sess.opts.cg.save_temps,
         time_trace: sess.opts.unstable_opts.llvm_time_trace,
         opts: Arc::new(sess.opts.clone()),
         prof: sess.prof.clone(),
-        exported_symbols,
         remark: sess.opts.cg.remark.clone(),
         remark_dir,
         incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()),
@@ -1350,18 +1343,6 @@ fn start_executing_work<B: ExtraBackendMethods>(
     // necessary. There's already optimizations in place to avoid sending work
     // back to the coordinator if LTO isn't requested.
     return B::spawn_named_thread(cgcx.time_trace, "coordinator".to_string(), move || {
-        let mut worker_id_counter = 0;
-        let mut free_worker_ids = Vec::new();
-        let mut get_worker_id = |free_worker_ids: &mut Vec<usize>| {
-            if let Some(id) = free_worker_ids.pop() {
-                id
-            } else {
-                let id = worker_id_counter;
-                worker_id_counter += 1;
-                id
-            }
-        };
-
         // This is where we collect codegen units that have gone all the way
         // through codegen and LLVM.
         let mut compiled_modules = vec![];
@@ -1442,12 +1423,7 @@ fn start_executing_work<B: ExtraBackendMethods>(
                         let (item, _) =
                             work_items.pop().expect("queue empty - queue_full_enough() broken?");
                         main_thread_state = MainThreadState::Lending;
-                        spawn_work(
-                            &cgcx,
-                            &mut llvm_start_time,
-                            get_worker_id(&mut free_worker_ids),
-                            item,
-                        );
+                        spawn_work(&cgcx, &mut llvm_start_time, item);
                     }
                 }
             } else if codegen_state == Completed {
@@ -1474,12 +1450,18 @@ fn start_executing_work<B: ExtraBackendMethods>(
                     let needs_fat_lto = mem::take(&mut needs_fat_lto);
                     let needs_thin_lto = mem::take(&mut needs_thin_lto);
                     let import_only_modules = mem::take(&mut lto_import_only_modules);
+                    let each_linked_rlib_file_for_lto =
+                        mem::take(&mut each_linked_rlib_file_for_lto);
+
+                    check_lto_allowed(&cgcx);
 
                     if !needs_fat_lto.is_empty() {
                         assert!(needs_thin_lto.is_empty());
 
                         work_items.push((
                             WorkItem::FatLto {
+                                exported_symbols_for_lto: Arc::clone(&exported_symbols_for_lto),
+                                each_linked_rlib_for_lto: each_linked_rlib_file_for_lto,
                                 needs_fat_lto,
                                 import_only_modules,
                                 autodiff: autodiff_items.clone(),
@@ -1495,9 +1477,13 @@ fn start_executing_work<B: ExtraBackendMethods>(
                             dcx.handle().emit_fatal(AutodiffWithoutLto {});
                         }
 
-                        for (work, cost) in
-                            generate_thin_lto_work(&cgcx, needs_thin_lto, import_only_modules)
-                        {
+                        for (work, cost) in generate_thin_lto_work(
+                            &cgcx,
+                            &exported_symbols_for_lto,
+                            &each_linked_rlib_file_for_lto,
+                            needs_thin_lto,
+                            import_only_modules,
+                        ) {
                             let insertion_index = work_items
                                 .binary_search_by_key(&cost, |&(_, cost)| cost)
                                 .unwrap_or_else(|e| e);
@@ -1516,12 +1502,7 @@ fn start_executing_work<B: ExtraBackendMethods>(
                     MainThreadState::Idle => {
                         if let Some((item, _)) = work_items.pop() {
                             main_thread_state = MainThreadState::Lending;
-                            spawn_work(
-                                &cgcx,
-                                &mut llvm_start_time,
-                                get_worker_id(&mut free_worker_ids),
-                                item,
-                            );
+                            spawn_work(&cgcx, &mut llvm_start_time, item);
                         } else {
                             // There is no unstarted work, so let the main thread
                             // take over for a running worker. Otherwise the
@@ -1557,12 +1538,7 @@ fn start_executing_work<B: ExtraBackendMethods>(
                 while running_with_own_token < tokens.len()
                     && let Some((item, _)) = work_items.pop()
                 {
-                    spawn_work(
-                        &cgcx,
-                        &mut llvm_start_time,
-                        get_worker_id(&mut free_worker_ids),
-                        item,
-                    );
+                    spawn_work(&cgcx, &mut llvm_start_time, item);
                     running_with_own_token += 1;
                 }
             }
@@ -1570,21 +1546,6 @@ fn start_executing_work<B: ExtraBackendMethods>(
             // Relinquish accidentally acquired extra tokens.
             tokens.truncate(running_with_own_token);
 
-            // If a thread exits successfully then we drop a token associated
-            // with that worker and update our `running_with_own_token` count.
-            // We may later re-acquire a token to continue running more work.
-            // We may also not actually drop a token here if the worker was
-            // running with an "ephemeral token".
-            let mut free_worker = |worker_id| {
-                if main_thread_state == MainThreadState::Lending {
-                    main_thread_state = MainThreadState::Idle;
-                } else {
-                    running_with_own_token -= 1;
-                }
-
-                free_worker_ids.push(worker_id);
-            };
-
             let msg = coordinator_receive.recv().unwrap();
             match *msg.downcast::<Message<B>>().ok().unwrap() {
                 // Save the token locally and the next turn of the loop will use
@@ -1653,8 +1614,17 @@ fn start_executing_work<B: ExtraBackendMethods>(
                     codegen_state = Aborted;
                 }
 
-                Message::WorkItem { result, worker_id } => {
-                    free_worker(worker_id);
+                Message::WorkItem { result } => {
+                    // If a thread exits successfully then we drop a token associated
+                    // with that worker and update our `running_with_own_token` count.
+                    // We may later re-acquire a token to continue running more work.
+                    // We may also not actually drop a token here if the worker was
+                    // running with an "ephemeral token".
+                    if main_thread_state == MainThreadState::Lending {
+                        main_thread_state = MainThreadState::Idle;
+                    } else {
+                        running_with_own_token -= 1;
+                    }
 
                     match result {
                         Ok(WorkItemResult::Finished(compiled_module)) => {
@@ -1800,7 +1770,6 @@ pub(crate) struct WorkerFatalError;
 fn spawn_work<'a, B: ExtraBackendMethods>(
     cgcx: &'a CodegenContext<B>,
     llvm_start_time: &mut Option<VerboseTimingGuard<'a>>,
-    worker_id: usize,
     work: WorkItem<B>,
 ) {
     if cgcx.config(work.module_kind()).time_module && llvm_start_time.is_none() {
@@ -1815,24 +1784,21 @@ fn spawn_work<'a, B: ExtraBackendMethods>(
         struct Bomb<B: ExtraBackendMethods> {
             coordinator_send: Sender<Box<dyn Any + Send>>,
             result: Option<Result<WorkItemResult<B>, FatalError>>,
-            worker_id: usize,
         }
         impl<B: ExtraBackendMethods> Drop for Bomb<B> {
             fn drop(&mut self) {
-                let worker_id = self.worker_id;
                 let msg = match self.result.take() {
-                    Some(Ok(result)) => Message::WorkItem::<B> { result: Ok(result), worker_id },
+                    Some(Ok(result)) => Message::WorkItem::<B> { result: Ok(result) },
                     Some(Err(FatalError)) => {
-                        Message::WorkItem::<B> { result: Err(Some(WorkerFatalError)), worker_id }
+                        Message::WorkItem::<B> { result: Err(Some(WorkerFatalError)) }
                     }
-                    None => Message::WorkItem::<B> { result: Err(None), worker_id },
+                    None => Message::WorkItem::<B> { result: Err(None) },
                 };
                 drop(self.coordinator_send.send(Box::new(msg)));
             }
         }
 
-        let mut bomb =
-            Bomb::<B> { coordinator_send: cgcx.coordinator_send.clone(), result: None, worker_id };
+        let mut bomb = Bomb::<B> { coordinator_send: cgcx.coordinator_send.clone(), result: None };
 
         // Execute the work itself, and if it finishes successfully then flag
         // ourselves as a success as well.
@@ -1856,12 +1822,20 @@ fn spawn_work<'a, B: ExtraBackendMethods>(
                     );
                     Ok(execute_copy_from_cache_work_item(&cgcx, m, module_config))
                 }
-                WorkItem::FatLto { needs_fat_lto, import_only_modules, autodiff } => {
+                WorkItem::FatLto {
+                    exported_symbols_for_lto,
+                    each_linked_rlib_for_lto,
+                    needs_fat_lto,
+                    import_only_modules,
+                    autodiff,
+                } => {
                     let _timer = cgcx
                         .prof
                         .generic_activity_with_arg("codegen_module_perform_lto", "everything");
                     execute_fat_lto_work_item(
                         &cgcx,
+                        &exported_symbols_for_lto,
+                        &each_linked_rlib_for_lto,
                         needs_fat_lto,
                         import_only_modules,
                         autodiff,

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

@@ -511,15 +511,6 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
         err.emit();
     }
 
-    // Any linkage to LLVM intrinsics for now forcibly marks them all as never
-    // unwinds since LLVM sometimes can't handle codegen which `invoke`s
-    // intrinsic functions.
-    if let Some(name) = &codegen_fn_attrs.link_name
-        && name.as_str().starts_with("llvm.")
-    {
-        codegen_fn_attrs.flags |= CodegenFnAttrFlags::NEVER_UNWIND;
-    }
-
     if let Some(features) = check_tied_features(
         tcx.sess,
         &codegen_fn_attrs

compiler/rustc_codegen_ssa/src/common.rs

@@ -148,7 +148,7 @@ pub(crate) fn shift_mask_val<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 pub fn asm_const_to_str<'tcx>(
     tcx: TyCtxt<'tcx>,
     sp: Span,
-    const_value: mir::ConstValue<'tcx>,
+    const_value: mir::ConstValue,
     ty_and_layout: TyAndLayout<'tcx>,
 ) -> String {
     let mir::ConstValue::Scalar(scalar) = const_value else {

compiler/rustc_codegen_ssa/src/errors.rs

@@ -1294,3 +1294,20 @@ pub(crate) struct FeatureNotValid<'a> {
     #[help]
     pub plus_hint: bool,
 }
+
+#[derive(Diagnostic)]
+#[diag(codegen_ssa_lto_disallowed)]
+pub(crate) struct LtoDisallowed;
+
+#[derive(Diagnostic)]
+#[diag(codegen_ssa_lto_dylib)]
+pub(crate) struct LtoDylib;
+
+#[derive(Diagnostic)]
+#[diag(codegen_ssa_lto_proc_macro)]
+pub(crate) struct LtoProcMacro;
+
+#[derive(Diagnostic)]
+#[diag(codegen_ssa_dynamic_linking_with_lto)]
+#[note]
+pub(crate) struct DynamicLinkingWithLTO;

compiler/rustc_codegen_ssa/src/mir/analyze.rs

@@ -1,12 +1,13 @@
 //! An analysis to determine which locals require allocas and
 //! which do not.
 
+use rustc_abi as abi;
 use rustc_data_structures::graph::dominators::Dominators;
 use rustc_index::bit_set::DenseBitSet;
 use rustc_index::{IndexSlice, IndexVec};
 use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor};
 use rustc_middle::mir::{self, DefLocation, Location, TerminatorKind, traversal};
-use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
+use rustc_middle::ty::layout::LayoutOf;
 use rustc_middle::{bug, span_bug};
 use tracing::debug;
 
@@ -99,63 +100,75 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> LocalAnalyzer<'a, 'b, 'tcx, Bx>
         context: PlaceContext,
         location: Location,
     ) {
-        let cx = self.fx.cx;
+        if !place_ref.projection.is_empty() {
+            const COPY_CONTEXT: PlaceContext =
+                PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy);
 
-        if let Some((place_base, elem)) = place_ref.last_projection() {
-            let mut base_context = if context.is_mutating_use() {
-                PlaceContext::MutatingUse(MutatingUseContext::Projection)
-            } else {
-                PlaceContext::NonMutatingUse(NonMutatingUseContext::Projection)
-            };
-
-            // Allow uses of projections that are ZSTs or from scalar fields.
-            let is_consume = matches!(
-                context,
-                PlaceContext::NonMutatingUse(
-                    NonMutatingUseContext::Copy | NonMutatingUseContext::Move,
-                )
-            );
-            if is_consume {
-                let base_ty = place_base.ty(self.fx.mir, cx.tcx());
-                let base_ty = self.fx.monomorphize(base_ty);
-
-                // ZSTs don't require any actual memory access.
-                let elem_ty = base_ty.projection_ty(cx.tcx(), self.fx.monomorphize(elem)).ty;
-                let span = self.fx.mir.local_decls[place_ref.local].source_info.span;
-                if cx.spanned_layout_of(elem_ty, span).is_zst() {
-                    return;
+            // `PlaceElem::Index` is the only variant that can mention other `Local`s,
+            // so check for those up-front before any potential short-circuits.
+            for elem in place_ref.projection {
+                if let mir::PlaceElem::Index(index_local) = *elem {
+                    self.visit_local(index_local, COPY_CONTEXT, location);
                 }
+            }
 
-                if let mir::ProjectionElem::Field(..) = elem {
-                    let layout = cx.spanned_layout_of(base_ty.ty, span);
-                    if cx.is_backend_immediate(layout) || cx.is_backend_scalar_pair(layout) {
-                        // Recurse with the same context, instead of `Projection`,
-                        // potentially stopping at non-operand projections,
-                        // which would trigger `not_ssa` on locals.
-                        base_context = context;
+            // If our local is already memory, nothing can make it *more* memory
+            // so we don't need to bother checking the projections further.
+            if self.locals[place_ref.local] == LocalKind::Memory {
+                return;
+            }
+
+            if place_ref.is_indirect_first_projection() {
+                // If this starts with a `Deref`, we only need to record a read of the
+                // pointer being dereferenced, as all the subsequent projections are
+                // working on a place which is always supported. (And because we're
+                // looking at codegen MIR, it can only happen as the first projection.)
+                self.visit_local(place_ref.local, COPY_CONTEXT, location);
+                return;
+            }
+
+            if context.is_mutating_use() {
+                // If it's a mutating use it doesn't matter what the projections are,
+                // if there are *any* then we need a place to write. (For example,
+                // `_1 = Foo()` works in SSA but `_2.0 = Foo()` does not.)
+                let mut_projection = PlaceContext::MutatingUse(MutatingUseContext::Projection);
+                self.visit_local(place_ref.local, mut_projection, location);
+                return;
+            }
+
+            // Scan through to ensure the only projections are those which
+            // `FunctionCx::maybe_codegen_consume_direct` can handle.
+            let base_ty = self.fx.monomorphized_place_ty(mir::PlaceRef::from(place_ref.local));
+            let mut layout = self.fx.cx.layout_of(base_ty);
+            for elem in place_ref.projection {
+                layout = match *elem {
+                    mir::PlaceElem::Field(fidx, ..) => layout.field(self.fx.cx, fidx.as_usize()),
+                    mir::PlaceElem::Downcast(_, vidx)
+                        if let abi::Variants::Single { index: single_variant } =
+                            layout.variants
+                            && vidx == single_variant =>
+                    {
+                        layout.for_variant(self.fx.cx, vidx)
+                    }
+                    mir::PlaceElem::Subtype(subtype_ty) => {
+                        let subtype_ty = self.fx.monomorphize(subtype_ty);
+                        self.fx.cx.layout_of(subtype_ty)
+                    }
+                    _ => {
+                        self.locals[place_ref.local] = LocalKind::Memory;
+                        return;
                     }
                 }
             }
-
-            if let mir::ProjectionElem::Deref = elem {
-                // Deref projections typically only read the pointer.
-                base_context = PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy);
-            }
-
-            self.process_place(&place_base, base_context, location);
-            // HACK(eddyb) this emulates the old `visit_projection_elem`, this
-            // entire `visit_place`-like `process_place` method should be rewritten,
-            // now that we have moved to the "slice of projections" representation.
-            if let mir::ProjectionElem::Index(local) = elem {
-                self.visit_local(
-                    local,
-                    PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy),
-                    location,
-                );
-            }
-        } else {
-            self.visit_local(place_ref.local, context, location);
+            debug_assert!(
+                !self.fx.cx.is_backend_ref(layout),
+                "Post-projection {place_ref:?} layout should be non-Ref, but it's {layout:?}",
+            );
         }
+
+        // Even with supported projections, we still need to have `visit_local`
+        // check for things that can't be done in SSA (like `SharedBorrow`).
+        self.visit_local(place_ref.local, context, location);
     }
 }
 
@@ -170,11 +183,6 @@ impl<'a, 'b, 'tcx, Bx: BuilderMethods<'b, 'tcx>> Visitor<'tcx> for LocalAnalyzer
 
         if let Some(local) = place.as_local() {
             self.define(local, DefLocation::Assignment(location));
-            if self.locals[local] != LocalKind::Memory {
-                if !self.fx.rvalue_creates_operand(rvalue) {
-                    self.locals[local] = LocalKind::Memory;
-                }
-            }
         } else {
             self.visit_place(place, PlaceContext::MutatingUse(MutatingUseContext::Store), location);
         }

compiler/rustc_codegen_ssa/src/mir/constant.rs

@@ -20,7 +20,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         OperandRef::from_const(bx, val, ty)
     }
 
-    pub fn eval_mir_constant(&self, constant: &mir::ConstOperand<'tcx>) -> mir::ConstValue<'tcx> {
+    pub fn eval_mir_constant(&self, constant: &mir::ConstOperand<'tcx>) -> mir::ConstValue {
         // `MirUsedCollector` visited all required_consts before codegen began, so if we got here
         // there can be no more constants that fail to evaluate.
         self.monomorphize(constant.const_)

compiler/rustc_codegen_ssa/src/mir/operand.rs

@@ -140,7 +140,7 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 
     pub(crate) fn from_const<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
         bx: &mut Bx,
-        val: mir::ConstValue<'tcx>,
+        val: mir::ConstValue,
         ty: Ty<'tcx>,
     ) -> Self {
         let layout = bx.layout_of(ty);
@@ -154,14 +154,11 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
                 OperandValue::Immediate(llval)
             }
             ConstValue::ZeroSized => return OperandRef::zero_sized(layout),
-            ConstValue::Slice { data, meta } => {
+            ConstValue::Slice { alloc_id, meta } => {
                 let BackendRepr::ScalarPair(a_scalar, _) = layout.backend_repr else {
                     bug!("from_const: invalid ScalarPair layout: {:#?}", layout);
                 };
-                let a = Scalar::from_pointer(
-                    Pointer::new(bx.tcx().reserve_and_set_memory_alloc(data).into(), Size::ZERO),
-                    &bx.tcx(),
-                );
+                let a = Scalar::from_pointer(Pointer::new(alloc_id.into(), Size::ZERO), &bx.tcx());
                 let a_llval = bx.scalar_to_backend(
                     a,
                     a_scalar,
@@ -338,13 +335,6 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
 
         let val = if field.is_zst() {
             OperandValue::ZeroSized
-        } else if let BackendRepr::SimdVector { .. } = self.layout.backend_repr {
-            // codegen_transmute_operand doesn't support SIMD, but since the previous
-            // check handled ZSTs, the only possible field access into something SIMD
-            // is to the `non_1zst_field` that's the same SIMD. (Other things, even
-            // just padding, would change the wrapper's representation type.)
-            assert_eq!(field.size, self.layout.size);
-            self.val
         } else if field.size == self.layout.size {
             assert_eq!(offset.bytes(), 0);
             fx.codegen_transmute_operand(bx, *self, field)
@@ -931,9 +921,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         match self.locals[place_ref.local] {
             LocalRef::Operand(mut o) => {
-                // Moves out of scalar and scalar pair fields are trivial.
-                for elem in place_ref.projection.iter() {
-                    match elem {
+                // We only need to handle the projections that
+                // `LocalAnalyzer::process_place` let make it here.
+                for elem in place_ref.projection {
+                    match *elem {
                         mir::ProjectionElem::Field(f, _) => {
                             assert!(
                                 !o.layout.ty.is_any_ptr(),
@@ -942,17 +933,18 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                             );
                             o = o.extract_field(self, bx, f.index());
                         }
-                        mir::ProjectionElem::Index(_)
-                        | mir::ProjectionElem::ConstantIndex { .. } => {
-                            // ZSTs don't require any actual memory access.
-                            // FIXME(eddyb) deduplicate this with the identical
-                            // checks in `codegen_consume` and `extract_field`.
-                            let elem = o.layout.field(bx.cx(), 0);
-                            if elem.is_zst() {
-                                o = OperandRef::zero_sized(elem);
-                            } else {
-                                return None;
-                            }
+                        mir::PlaceElem::Downcast(_, vidx) => {
+                            debug_assert_eq!(
+                                o.layout.variants,
+                                abi::Variants::Single { index: vidx },
+                            );
+                            let layout = o.layout.for_variant(bx.cx(), vidx);
+                            o = OperandRef { val: o.val, layout }
+                        }
+                        mir::PlaceElem::Subtype(subtype_ty) => {
+                            let subtype_ty = self.monomorphize(subtype_ty);
+                            let layout = self.cx.layout_of(subtype_ty);
+                            o = OperandRef { val: o.val, layout }
                         }
                         _ => return None,
                     }

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -2,12 +2,12 @@ use rustc_abi::{self as abi, FIRST_VARIANT};
 use rustc_middle::ty::adjustment::PointerCoercion;
 use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, LayoutOf, TyAndLayout};
 use rustc_middle::ty::{self, Instance, Ty, TyCtxt};
-use rustc_middle::{bug, mir};
+use rustc_middle::{bug, mir, span_bug};
 use rustc_session::config::OptLevel;
 use tracing::{debug, instrument};
 
 use super::operand::{OperandRef, OperandRefBuilder, OperandValue};
-use super::place::{PlaceRef, codegen_tag_value};
+use super::place::{PlaceRef, PlaceValue, codegen_tag_value};
 use super::{FunctionCx, LocalRef};
 use crate::common::{IntPredicate, TypeKind};
 use crate::traits::*;
@@ -180,7 +180,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             }
 
             _ => {
-                assert!(self.rvalue_creates_operand(rvalue));
                 let temp = self.codegen_rvalue_operand(bx, rvalue);
                 temp.val.store(bx, dest);
             }
@@ -218,17 +217,26 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
     /// Transmutes an `OperandValue` to another `OperandValue`.
     ///
-    /// This is supported only for cases where [`Self::rvalue_creates_operand`]
-    /// returns `true`, and will ICE otherwise. (In particular, anything that
-    /// would need to `alloca` in order to return a `PlaceValue` will ICE,
-    /// expecting those to go via [`Self::codegen_transmute`] instead where
-    /// the destination place is already allocated.)
+    /// This is supported for all cases where the `cast` type is SSA,
+    /// but for non-ZSTs with [`abi::BackendRepr::Memory`] it ICEs.
     pub(crate) fn codegen_transmute_operand(
         &mut self,
         bx: &mut Bx,
         operand: OperandRef<'tcx, Bx::Value>,
         cast: TyAndLayout<'tcx>,
     ) -> OperandValue<Bx::Value> {
+        if let abi::BackendRepr::Memory { .. } = cast.backend_repr
+            && !cast.is_zst()
+        {
+            span_bug!(self.mir.span, "Use `codegen_transmute` to transmute to {cast:?}");
+        }
+
+        // `Layout` is interned, so we can do a cheap check for things that are
+        // exactly the same and thus don't need any handling.
+        if abi::Layout::eq(&operand.layout.layout, &cast.layout) {
+            return operand.val;
+        }
+
         // Check for transmutes that are always UB.
         if operand.layout.size != cast.size
             || operand.layout.is_uninhabited()
@@ -241,11 +249,22 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             return OperandValue::poison(bx, cast);
         }
 
+        // To or from pointers takes different methods, so we use this to restrict
+        // the SimdVector case to types which can be `bitcast` between each other.
+        #[inline]
+        fn vector_can_bitcast(x: abi::Scalar) -> bool {
+            matches!(
+                x,
+                abi::Scalar::Initialized {
+                    value: abi::Primitive::Int(..) | abi::Primitive::Float(..),
+                    ..
+                }
+            )
+        }
+
+        let cx = bx.cx();
         match (operand.val, operand.layout.backend_repr, cast.backend_repr) {
             _ if cast.is_zst() => OperandValue::ZeroSized,
-            (_, _, abi::BackendRepr::Memory { .. }) => {
-                bug!("Cannot `codegen_transmute_operand` to non-ZST memory-ABI output {cast:?}");
-            }
             (OperandValue::Ref(source_place_val), abi::BackendRepr::Memory { .. }, _) => {
                 assert_eq!(source_place_val.llextra, None);
                 // The existing alignment is part of `source_place_val`,
@@ -256,16 +275,46 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 OperandValue::Immediate(imm),
                 abi::BackendRepr::Scalar(from_scalar),
                 abi::BackendRepr::Scalar(to_scalar),
-            ) => OperandValue::Immediate(transmute_scalar(bx, imm, from_scalar, to_scalar)),
+            ) if from_scalar.size(cx) == to_scalar.size(cx) => {
+                OperandValue::Immediate(transmute_scalar(bx, imm, from_scalar, to_scalar))
+            }
+            (
+                OperandValue::Immediate(imm),
+                abi::BackendRepr::SimdVector { element: from_scalar, .. },
+                abi::BackendRepr::SimdVector { element: to_scalar, .. },
+            ) if vector_can_bitcast(from_scalar) && vector_can_bitcast(to_scalar) => {
+                let to_backend_ty = bx.cx().immediate_backend_type(cast);
+                OperandValue::Immediate(bx.bitcast(imm, to_backend_ty))
+            }
             (
                 OperandValue::Pair(imm_a, imm_b),
                 abi::BackendRepr::ScalarPair(in_a, in_b),
                 abi::BackendRepr::ScalarPair(out_a, out_b),
-            ) => OperandValue::Pair(
-                transmute_scalar(bx, imm_a, in_a, out_a),
-                transmute_scalar(bx, imm_b, in_b, out_b),
-            ),
-            _ => bug!("Cannot `codegen_transmute_operand` {operand:?} to {cast:?}"),
+            ) if in_a.size(cx) == out_a.size(cx) && in_b.size(cx) == out_b.size(cx) => {
+                OperandValue::Pair(
+                    transmute_scalar(bx, imm_a, in_a, out_a),
+                    transmute_scalar(bx, imm_b, in_b, out_b),
+                )
+            }
+            _ => {
+                // For any other potentially-tricky cases, make a temporary instead.
+                // If anything else wants the target local to be in memory this won't
+                // be hit, as `codegen_transmute` will get called directly. Thus this
+                // is only for places where everything else wants the operand form,
+                // and thus it's not worth making those places get it from memory.
+                //
+                // Notably, Scalar ⇌ ScalarPair cases go here to avoid padding
+                // and endianness issues, as do SimdVector ones to avoid worrying
+                // about things like f32x8 ⇌ ptrx4 that would need multiple steps.
+                let align = Ord::max(operand.layout.align.abi, cast.align.abi);
+                let size = Ord::max(operand.layout.size, cast.size);
+                let temp = PlaceValue::alloca(bx, size, align);
+                bx.lifetime_start(temp.llval, size);
+                operand.val.store(bx, temp.with_type(operand.layout));
+                let val = bx.load_operand(temp.with_type(cast)).val;
+                bx.lifetime_end(temp.llval, size);
+                val
+            }
         }
     }
 
@@ -288,7 +337,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         // valid ranges. For example, `char`s are passed as just `i32`, with no
         // way for LLVM to know that they're 0x10FFFF at most. Thus we assume
         // the range of the input value too, not just the output range.
-        assume_scalar_range(bx, imm, from_scalar, from_backend_ty);
+        assume_scalar_range(bx, imm, from_scalar, from_backend_ty, None);
 
         imm = match (from_scalar.primitive(), to_scalar.primitive()) {
             (Int(_, is_signed), Int(..)) => bx.intcast(imm, to_backend_ty, is_signed),
@@ -326,8 +375,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         bx: &mut Bx,
         rvalue: &mir::Rvalue<'tcx>,
     ) -> OperandRef<'tcx, Bx::Value> {
-        assert!(self.rvalue_creates_operand(rvalue), "cannot codegen {rvalue:?} to operand",);
-
         match *rvalue {
             mir::Rvalue::Cast(ref kind, ref source, mir_cast_ty) => {
                 let operand = self.codegen_operand(bx, source);
@@ -653,8 +700,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 let ty = self.monomorphize(ty);
                 let layout = self.cx.layout_of(ty);
 
-                // `rvalue_creates_operand` has arranged that we only get here if
-                // we can build the aggregate immediate from the field immediates.
                 let mut builder = OperandRefBuilder::new(layout);
                 for (field_idx, field) in fields.iter_enumerated() {
                     let op = self.codegen_operand(bx, field);
@@ -955,69 +1000,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
 
         OperandValue::Pair(val, of)
     }
-
-    /// Returns `true` if the `rvalue` can be computed into an [`OperandRef`],
-    /// rather than needing a full `PlaceRef` for the assignment destination.
-    ///
-    /// This is used by the [`super::analyze`] code to decide which MIR locals
-    /// can stay as SSA values (as opposed to generating `alloca` slots for them).
-    /// As such, some paths here return `true` even where the specific rvalue
-    /// will not actually take the operand path because the result type is such
-    /// that it always gets an `alloca`, but where it's not worth re-checking the
-    /// layout in this code when the right thing will happen anyway.
-    pub(crate) fn rvalue_creates_operand(&self, rvalue: &mir::Rvalue<'tcx>) -> bool {
-        match *rvalue {
-            mir::Rvalue::Cast(mir::CastKind::Transmute, ref operand, cast_ty) => {
-                let operand_ty = operand.ty(self.mir, self.cx.tcx());
-                let cast_layout = self.cx.layout_of(self.monomorphize(cast_ty));
-                let operand_layout = self.cx.layout_of(self.monomorphize(operand_ty));
-                match (operand_layout.backend_repr, cast_layout.backend_repr) {
-                    // When the output will be in memory anyway, just use its place
-                    // (instead of the operand path) unless it's the trivial ZST case.
-                    (_, abi::BackendRepr::Memory { .. }) => cast_layout.is_zst(),
-
-                    // Otherwise (for a non-memory output) if the input is memory
-                    // then we can just read the value from the place.
-                    (abi::BackendRepr::Memory { .. }, _) => true,
-
-                    // When we have scalar immediates, we can only convert things
-                    // where the sizes match, to avoid endianness questions.
-                    (abi::BackendRepr::Scalar(a), abi::BackendRepr::Scalar(b)) =>
-                        a.size(self.cx) == b.size(self.cx),
-                    (abi::BackendRepr::ScalarPair(a0, a1), abi::BackendRepr::ScalarPair(b0, b1)) =>
-                        a0.size(self.cx) == b0.size(self.cx) && a1.size(self.cx) == b1.size(self.cx),
-
-                    // Mixing Scalars and ScalarPairs can get quite complicated when
-                    // padding and undef get involved, so leave that to the memory path.
-                    (abi::BackendRepr::Scalar(_), abi::BackendRepr::ScalarPair(_, _)) |
-                    (abi::BackendRepr::ScalarPair(_, _), abi::BackendRepr::Scalar(_)) => false,
-
-                    // SIMD vectors aren't worth the trouble of dealing with complex
-                    // cases like from vectors of f32 to vectors of pointers or
-                    // from fat pointers to vectors of u16. (See #143194 #110021 ...)
-                    (abi::BackendRepr::SimdVector { .. }, _) | (_, abi::BackendRepr::SimdVector { .. }) => false,
-                }
-            }
-            mir::Rvalue::Ref(..) |
-            mir::Rvalue::CopyForDeref(..) |
-            mir::Rvalue::RawPtr(..) |
-            mir::Rvalue::Len(..) |
-            mir::Rvalue::Cast(..) | // (*)
-            mir::Rvalue::ShallowInitBox(..) | // (*)
-            mir::Rvalue::BinaryOp(..) |
-            mir::Rvalue::UnaryOp(..) |
-            mir::Rvalue::Discriminant(..) |
-            mir::Rvalue::NullaryOp(..) |
-            mir::Rvalue::ThreadLocalRef(_) |
-            mir::Rvalue::Use(..) |
-            mir::Rvalue::Repeat(..) | // (*)
-            mir::Rvalue::Aggregate(..) | // (*)
-            mir::Rvalue::WrapUnsafeBinder(..) => // (*)
-                true,
-        }
-
-        // (*) this is only true if the type is suitable
-    }
 }
 
 /// Transmutes a single scalar value `imm` from `from_scalar` to `to_scalar`.
@@ -1064,7 +1046,7 @@ pub(super) fn transmute_scalar<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     // That said, last time we tried removing this, it didn't actually help
     // the rustc-perf results, so might as well keep doing it
     // <https://github.com/rust-lang/rust/pull/135610#issuecomment-2599275182>
-    assume_scalar_range(bx, imm, from_scalar, from_backend_ty);
+    assume_scalar_range(bx, imm, from_scalar, from_backend_ty, Some(&to_scalar));
 
     imm = match (from_scalar.primitive(), to_scalar.primitive()) {
         (Int(..) | Float(_), Int(..) | Float(_)) => bx.bitcast(imm, to_backend_ty),
@@ -1092,22 +1074,42 @@ pub(super) fn transmute_scalar<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     // since it's never passed to something with parameter metadata (especially
     // after MIR inlining) so the only way to tell the backend about the
     // constraint that the `transmute` introduced is to `assume` it.
-    assume_scalar_range(bx, imm, to_scalar, to_backend_ty);
+    assume_scalar_range(bx, imm, to_scalar, to_backend_ty, Some(&from_scalar));
 
     imm = bx.to_immediate_scalar(imm, to_scalar);
     imm
 }
 
+/// Emits an `assume` call that `imm`'s value is within the known range of `scalar`.
+///
+/// If `known` is `Some`, only emits the assume if it's more specific than
+/// whatever is already known from the range of *that* scalar.
 fn assume_scalar_range<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     bx: &mut Bx,
     imm: Bx::Value,
     scalar: abi::Scalar,
     backend_ty: Bx::Type,
+    known: Option<&abi::Scalar>,
 ) {
-    if matches!(bx.cx().sess().opts.optimize, OptLevel::No) || scalar.is_always_valid(bx.cx()) {
+    if matches!(bx.cx().sess().opts.optimize, OptLevel::No) {
         return;
     }
 
+    match (scalar, known) {
+        (abi::Scalar::Union { .. }, _) => return,
+        (_, None) => {
+            if scalar.is_always_valid(bx.cx()) {
+                return;
+            }
+        }
+        (abi::Scalar::Initialized { valid_range, .. }, Some(known)) => {
+            let known_range = known.valid_range(bx.cx());
+            if valid_range.contains_range(known_range, scalar.size(bx.cx())) {
+                return;
+            }
+        }
+    }
+
     match scalar.primitive() {
         abi::Primitive::Int(..) => {
             let range = scalar.valid_range(bx.cx());

compiler/rustc_codegen_ssa/src/traits/write.rs

@@ -1,3 +1,5 @@
+use std::path::PathBuf;
+
 use rustc_ast::expand::autodiff_attrs::AutoDiffItem;
 use rustc_errors::{DiagCtxtHandle, FatalError};
 use rustc_middle::dep_graph::WorkProduct;
@@ -24,8 +26,9 @@ pub trait WriteBackendMethods: Clone + 'static {
     /// if necessary and running any further optimizations
     fn run_and_optimize_fat_lto(
         cgcx: &CodegenContext<Self>,
+        exported_symbols_for_lto: &[String],
+        each_linked_rlib_for_lto: &[PathBuf],
         modules: Vec<FatLtoInput<Self>>,
-        cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
         diff_fncs: Vec<AutoDiffItem>,
     ) -> Result<ModuleCodegen<Self::Module>, FatalError>;
     /// Performs thin LTO by performing necessary global analysis and returning two
@@ -33,6 +36,8 @@ pub trait WriteBackendMethods: Clone + 'static {
     /// can simply be copied over from the incr. comp. cache.
     fn run_thin_lto(
         cgcx: &CodegenContext<Self>,
+        exported_symbols_for_lto: &[String],
+        each_linked_rlib_for_lto: &[PathBuf],
         modules: Vec<(String, Self::ThinBuffer)>,
         cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>,
     ) -> Result<(Vec<ThinModule<Self>>, Vec<WorkProduct>), FatalError>;

compiler/rustc_const_eval/src/const_eval/eval_queries.rs

@@ -152,7 +152,7 @@ pub(crate) fn mk_eval_cx_to_read_const_val<'tcx>(
 pub fn mk_eval_cx_for_const_val<'tcx>(
     tcx: TyCtxtAt<'tcx>,
     typing_env: ty::TypingEnv<'tcx>,
-    val: mir::ConstValue<'tcx>,
+    val: mir::ConstValue,
     ty: Ty<'tcx>,
 ) -> Option<(CompileTimeInterpCx<'tcx>, OpTy<'tcx>)> {
     let ecx = mk_eval_cx_to_read_const_val(tcx.tcx, tcx.span, typing_env, CanAccessMutGlobal::No);
@@ -172,7 +172,7 @@ pub(super) fn op_to_const<'tcx>(
     ecx: &CompileTimeInterpCx<'tcx>,
     op: &OpTy<'tcx>,
     for_diagnostics: bool,
-) -> ConstValue<'tcx> {
+) -> ConstValue {
     // Handle ZST consistently and early.
     if op.layout.is_zst() {
         return ConstValue::ZeroSized;
@@ -241,10 +241,9 @@ pub(super) fn op_to_const<'tcx>(
                 let (prov, offset) =
                     ptr.into_pointer_or_addr().expect(msg).prov_and_relative_offset();
                 let alloc_id = prov.alloc_id();
-                let data = ecx.tcx.global_alloc(alloc_id).unwrap_memory();
                 assert!(offset == abi::Size::ZERO, "{}", msg);
                 let meta = b.to_target_usize(ecx).expect(msg);
-                ConstValue::Slice { data, meta }
+                ConstValue::Slice { alloc_id, meta }
             }
             Immediate::Uninit => bug!("`Uninit` is not a valid value for {}", op.layout.ty),
         },
@@ -256,7 +255,7 @@ pub(crate) fn turn_into_const_value<'tcx>(
     tcx: TyCtxt<'tcx>,
     constant: ConstAlloc<'tcx>,
     key: ty::PseudoCanonicalInput<'tcx, GlobalId<'tcx>>,
-) -> ConstValue<'tcx> {
+) -> ConstValue {
     let cid = key.value;
     let def_id = cid.instance.def.def_id();
     let is_static = tcx.is_static(def_id);

compiler/rustc_const_eval/src/const_eval/mod.rs

@@ -28,7 +28,7 @@ const VALTREE_MAX_NODES: usize = 100000;
 #[instrument(skip(tcx), level = "debug")]
 pub(crate) fn try_destructure_mir_constant_for_user_output<'tcx>(
     tcx: TyCtxt<'tcx>,
-    val: mir::ConstValue<'tcx>,
+    val: mir::ConstValue,
     ty: Ty<'tcx>,
 ) -> Option<mir::DestructuredConstant<'tcx>> {
     let typing_env = ty::TypingEnv::fully_monomorphized();

compiler/rustc_const_eval/src/const_eval/valtrees.rs

@@ -259,7 +259,7 @@ pub fn valtree_to_const_value<'tcx>(
     tcx: TyCtxt<'tcx>,
     typing_env: ty::TypingEnv<'tcx>,
     cv: ty::Value<'tcx>,
-) -> mir::ConstValue<'tcx> {
+) -> mir::ConstValue {
     // Basic idea: We directly construct `Scalar` values from trivial `ValTree`s
     // (those for constants with type bool, int, uint, float or char).
     // For all other types we create an `MPlace` and fill that by walking

compiler/rustc_const_eval/src/interpret/eval_context.rs

@@ -582,8 +582,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         span: Span,
         layout: Option<TyAndLayout<'tcx>>,
     ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
-        M::eval_mir_constant(self, *val, span, layout, |ecx, val, span, layout| {
-            let const_val = val.eval(*ecx.tcx, ecx.typing_env, span).map_err(|err| {
+        let const_val = val.eval(*self.tcx, self.typing_env, span).map_err(|err| {
                 if M::ALL_CONSTS_ARE_PRECHECKED {
                     match err {
                         ErrorHandled::TooGeneric(..) => {},
@@ -599,11 +598,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                         }
                     }
                 }
-                err.emit_note(*ecx.tcx);
+                err.emit_note(*self.tcx);
                 err
             })?;
-            ecx.const_val_to_op(const_val, val.ty(), layout)
-        })
+        self.const_val_to_op(const_val, val.ty(), layout)
     }
 
     #[must_use]

compiler/rustc_const_eval/src/interpret/intrinsics.rs

@@ -6,7 +6,7 @@ use std::assert_matches::assert_matches;
 
 use rustc_abi::{FieldIdx, HasDataLayout, Size};
 use rustc_apfloat::ieee::{Double, Half, Quad, Single};
-use rustc_middle::mir::interpret::{read_target_uint, write_target_uint};
+use rustc_middle::mir::interpret::{CTFE_ALLOC_SALT, read_target_uint, write_target_uint};
 use rustc_middle::mir::{self, BinOp, ConstValue, NonDivergingIntrinsic};
 use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::ty::{Ty, TyCtxt};
@@ -17,17 +17,18 @@ use tracing::trace;
 use super::memory::MemoryKind;
 use super::util::ensure_monomorphic_enough;
 use super::{
-    Allocation, CheckInAllocMsg, ConstAllocation, ImmTy, InterpCx, InterpResult, Machine, OpTy,
-    PlaceTy, Pointer, PointerArithmetic, Provenance, Scalar, err_ub_custom, err_unsup_format,
-    interp_ok, throw_inval, throw_ub_custom, throw_ub_format,
+    AllocId, CheckInAllocMsg, ImmTy, InterpCx, InterpResult, Machine, OpTy, PlaceTy, Pointer,
+    PointerArithmetic, Provenance, Scalar, err_ub_custom, err_unsup_format, interp_ok, throw_inval,
+    throw_ub_custom, throw_ub_format,
 };
 use crate::fluent_generated as fluent;
 
 /// Directly returns an `Allocation` containing an absolute path representation of the given type.
-pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ConstAllocation<'tcx> {
+pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> (AllocId, u64) {
     let path = crate::util::type_name(tcx, ty);
-    let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes(), ());
-    tcx.mk_const_alloc(alloc)
+    let bytes = path.into_bytes();
+    let len = bytes.len().try_into().unwrap();
+    (tcx.allocate_bytes_dedup(bytes, CTFE_ALLOC_SALT), len)
 }
 impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
     /// Generates a value of `TypeId` for `ty` in-place.
@@ -126,8 +127,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             sym::type_name => {
                 let tp_ty = instance.args.type_at(0);
                 ensure_monomorphic_enough(tcx, tp_ty)?;
-                let alloc = alloc_type_name(tcx, tp_ty);
-                let val = ConstValue::Slice { data: alloc, meta: alloc.inner().size().bytes() };
+                let (alloc_id, meta) = alloc_type_name(tcx, tp_ty);
+                let val = ConstValue::Slice { alloc_id, meta };
                 let val = self.const_val_to_op(val, dest.layout.ty, Some(dest.layout))?;
                 self.copy_op(&val, dest)?;
             }

compiler/rustc_const_eval/src/interpret/machine.rs

@@ -12,7 +12,6 @@ use rustc_middle::query::TyCtxtAt;
 use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::{mir, ty};
-use rustc_span::Span;
 use rustc_span::def_id::DefId;
 use rustc_target::callconv::FnAbi;
 
@@ -587,27 +586,6 @@ pub trait Machine<'tcx>: Sized {
         interp_ok(())
     }
 
-    /// Evaluate the given constant. The `eval` function will do all the required evaluation,
-    /// but this hook has the chance to do some pre/postprocessing.
-    #[inline(always)]
-    fn eval_mir_constant<F>(
-        ecx: &InterpCx<'tcx, Self>,
-        val: mir::Const<'tcx>,
-        span: Span,
-        layout: Option<TyAndLayout<'tcx>>,
-        eval: F,
-    ) -> InterpResult<'tcx, OpTy<'tcx, Self::Provenance>>
-    where
-        F: Fn(
-            &InterpCx<'tcx, Self>,
-            mir::Const<'tcx>,
-            Span,
-            Option<TyAndLayout<'tcx>>,
-        ) -> InterpResult<'tcx, OpTy<'tcx, Self::Provenance>>,
-    {
-        eval(ecx, val, span, layout)
-    }
-
     /// Returns the salt to be used for a deduplicated global alloation.
     /// If the allocation is for a function, the instance is provided as well
     /// (this lets Miri ensure unique addresses for some functions).

compiler/rustc_const_eval/src/interpret/operand.rs

@@ -836,7 +836,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
 
     pub(crate) fn const_val_to_op(
         &self,
-        val_val: mir::ConstValue<'tcx>,
+        val_val: mir::ConstValue,
         ty: Ty<'tcx>,
         layout: Option<TyAndLayout<'tcx>>,
     ) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
@@ -860,9 +860,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
             }
             mir::ConstValue::Scalar(x) => adjust_scalar(x)?.into(),
             mir::ConstValue::ZeroSized => Immediate::Uninit,
-            mir::ConstValue::Slice { data, meta } => {
+            mir::ConstValue::Slice { alloc_id, meta } => {
                 // This is const data, no mutation allowed.
-                let alloc_id = self.tcx.reserve_and_set_memory_alloc(data);
                 let ptr = Pointer::new(CtfeProvenance::from(alloc_id).as_immutable(), Size::ZERO);
                 Immediate::new_slice(self.global_root_pointer(ptr)?.into(), meta, self)
             }

compiler/rustc_const_eval/src/util/caller_location.rs

@@ -57,7 +57,7 @@ pub(crate) fn const_caller_location_provider(
     file: Symbol,
     line: u32,
     col: u32,
-) -> mir::ConstValue<'_> {
+) -> mir::ConstValue {
     trace!("const_caller_location: {}:{}:{}", file, line, col);
     let mut ecx = mk_eval_cx_to_read_const_val(
         tcx,

compiler/rustc_data_structures/src/profiling.rs

@@ -551,6 +551,11 @@ impl SelfProfilerRef {
     pub fn get_self_profiler(&self) -> Option<Arc<SelfProfiler>> {
         self.profiler.clone()
     }
+
+    /// Is expensive recording of query keys and/or function arguments enabled?
+    pub fn is_args_recording_enabled(&self) -> bool {
+        self.enabled() && self.event_filter_mask.intersects(EventFilter::ARGS)
+    }
 }
 
 /// A helper for recording costly arguments to self-profiling events. Used with

compiler/rustc_errors/src/markdown/term.rs

@@ -18,7 +18,7 @@ thread_local! {
 pub(crate) fn entrypoint(stream: &MdStream<'_>, buf: &mut Buffer) -> io::Result<()> {
     #[cfg(not(test))]
     if let Some((w, _)) = termize::dimensions() {
-        WIDTH.with(|c| c.set(std::cmp::min(w, DEFAULT_COLUMN_WIDTH)));
+        WIDTH.set(std::cmp::min(w, DEFAULT_COLUMN_WIDTH));
     }
     write_stream(stream, buf, None, 0)?;
     buf.write_all(b"\n")
@@ -84,7 +84,7 @@ fn write_tt(tt: &MdTree<'_>, buf: &mut Buffer, indent: usize) -> io::Result<()>
             reset_cursor();
         }
         MdTree::HorizontalRule => {
-            (0..WIDTH.with(Cell::get)).for_each(|_| buf.write_all(b"-").unwrap());
+            (0..WIDTH.get()).for_each(|_| buf.write_all(b"-").unwrap());
             reset_cursor();
         }
         MdTree::Heading(n, stream) => {
@@ -121,7 +121,7 @@ fn write_tt(tt: &MdTree<'_>, buf: &mut Buffer, indent: usize) -> io::Result<()>
 
 /// End of that block, just wrap the line
 fn reset_cursor() {
-    CURSOR.with(|cur| cur.set(0));
+    CURSOR.set(0);
 }
 
 /// Change to be generic on Write for testing. If we have a link URL, we don't
@@ -144,7 +144,7 @@ fn write_wrapping<B: io::Write>(
                 buf.write_all(ind_ws)?;
                 cur.set(indent);
             }
-            let ch_count = WIDTH.with(Cell::get) - cur.get();
+            let ch_count = WIDTH.get() - cur.get();
             let mut iter = to_write.char_indices();
             let Some((end_idx, _ch)) = iter.nth(ch_count) else {
                 // Write entire line

compiler/rustc_expand/src/base.rs

@@ -1141,7 +1141,7 @@ pub trait ResolverExpand {
 
     /// Names of specific methods to which glob delegation expands.
     fn glob_delegation_suffixes(
-        &mut self,
+        &self,
         trait_def_id: DefId,
         impl_def_id: LocalDefId,
     ) -> Result<Vec<(Ident, Option<Ident>)>, Indeterminate>;
@@ -1224,6 +1224,7 @@ pub struct ExtCtxt<'a> {
     pub(super) expanded_inert_attrs: MarkedAttrs,
     /// `-Zmacro-stats` data.
     pub macro_stats: FxHashMap<(Symbol, MacroKind), MacroStat>,
+    pub nb_macro_errors: usize,
 }
 
 impl<'a> ExtCtxt<'a> {
@@ -1254,6 +1255,7 @@ impl<'a> ExtCtxt<'a> {
             expanded_inert_attrs: MarkedAttrs::new(),
             buffered_early_lint: vec![],
             macro_stats: Default::default(),
+            nb_macro_errors: 0,
         }
     }
 
@@ -1315,6 +1317,12 @@ impl<'a> ExtCtxt<'a> {
         self.current_expansion.id.expansion_cause()
     }
 
+    /// This method increases the internal macro errors count and then call `trace_macros_diag`.
+    pub fn macro_error_and_trace_macros_diag(&mut self) {
+        self.nb_macro_errors += 1;
+        self.trace_macros_diag();
+    }
+
     pub fn trace_macros_diag(&mut self) {
         for (span, notes) in self.expansions.iter() {
             let mut db = self.dcx().create_note(errors::TraceMacro { span: *span });

compiler/rustc_expand/src/expand.rs

@@ -693,7 +693,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             crate_name: self.cx.ecfg.crate_name,
         });
 
-        self.cx.trace_macros_diag();
+        self.cx.macro_error_and_trace_macros_diag();
         guar
     }
 
@@ -707,7 +707,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
     ) -> ErrorGuaranteed {
         let guar =
             self.cx.dcx().emit_err(WrongFragmentKind { span, kind: kind.name(), name: &mac.path });
-        self.cx.trace_macros_diag();
+        self.cx.macro_error_and_trace_macros_diag();
         guar
     }
 
@@ -1048,7 +1048,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
                 }
                 annotate_err_with_kind(&mut err, kind, span);
                 let guar = err.emit();
-                self.cx.trace_macros_diag();
+                self.cx.macro_error_and_trace_macros_diag();
                 kind.dummy(span, guar)
             }
         }

compiler/rustc_expand/src/mbe/macro_parser.rs

@@ -299,6 +299,7 @@ enum EofMatcherPositions {
 }
 
 /// Represents the possible results of an attempted parse.
+#[derive(Debug)]
 pub(crate) enum ParseResult<T, F> {
     /// Parsed successfully.
     Success(T),

compiler/rustc_expand/src/mbe/macro_rules.rs

@@ -280,7 +280,7 @@ fn expand_macro<'cx>(
             // Retry and emit a better error.
             let (span, guar) =
                 diagnostics::failed_to_match_macro(cx.psess(), sp, def_span, name, arg, rules);
-            cx.trace_macros_diag();
+            cx.macro_error_and_trace_macros_diag();
             DummyResult::any(span, guar)
         }
     }

compiler/rustc_hir_analysis/src/check/check.rs

@@ -767,7 +767,8 @@ pub(crate) fn check_item_type(tcx: TyCtxt<'_>, def_id: LocalDefId) -> Result<(),
                 DefKind::Static { .. } => {
                     check_static_inhabited(tcx, def_id);
                     check_static_linkage(tcx, def_id);
-                    res = res.and(wfcheck::check_static_item(tcx, def_id));
+                    let ty = tcx.type_of(def_id).instantiate_identity();
+                    res = res.and(wfcheck::check_static_item(tcx, def_id, ty, true));
                 }
                 DefKind::Const => res = res.and(wfcheck::check_const_item(tcx, def_id)),
                 _ => unreachable!(),
@@ -1642,20 +1643,40 @@ fn check_enum(tcx: TyCtxt<'_>, def_id: LocalDefId) {
 
     if def.repr().int.is_none() {
         let is_unit = |var: &ty::VariantDef| matches!(var.ctor_kind(), Some(CtorKind::Const));
-        let has_disr = |var: &ty::VariantDef| matches!(var.discr, ty::VariantDiscr::Explicit(_));
+        let get_disr = |var: &ty::VariantDef| match var.discr {
+            ty::VariantDiscr::Explicit(disr) => Some(disr),
+            ty::VariantDiscr::Relative(_) => None,
+        };
 
-        let has_non_units = def.variants().iter().any(|var| !is_unit(var));
-        let disr_units = def.variants().iter().any(|var| is_unit(var) && has_disr(var));
-        let disr_non_unit = def.variants().iter().any(|var| !is_unit(var) && has_disr(var));
+        let non_unit = def.variants().iter().find(|var| !is_unit(var));
+        let disr_unit =
+            def.variants().iter().filter(|var| is_unit(var)).find_map(|var| get_disr(var));
+        let disr_non_unit =
+            def.variants().iter().filter(|var| !is_unit(var)).find_map(|var| get_disr(var));
 
-        if disr_non_unit || (disr_units && has_non_units) {
-            struct_span_code_err!(
+        if disr_non_unit.is_some() || (disr_unit.is_some() && non_unit.is_some()) {
+            let mut err = struct_span_code_err!(
                 tcx.dcx(),
                 tcx.def_span(def_id),
                 E0732,
-                "`#[repr(inttype)]` must be specified"
-            )
-            .emit();
+                "`#[repr(inttype)]` must be specified for enums with explicit discriminants and non-unit variants"
+            );
+            if let Some(disr_non_unit) = disr_non_unit {
+                err.span_label(
+                    tcx.def_span(disr_non_unit),
+                    "explicit discriminant on non-unit variant specified here",
+                );
+            } else {
+                err.span_label(
+                    tcx.def_span(disr_unit.unwrap()),
+                    "explicit discriminant specified here",
+                );
+                err.span_label(
+                    tcx.def_span(non_unit.unwrap().def_id),
+                    "non-unit discriminant declared here",
+                );
+            }
+            err.emit();
         }
     }
 

compiler/rustc_hir_analysis/src/check/wfcheck.rs

@@ -1180,12 +1180,13 @@ fn check_item_fn(
 }
 
 #[instrument(level = "debug", skip(tcx))]
-pub(super) fn check_static_item(
-    tcx: TyCtxt<'_>,
+pub(crate) fn check_static_item<'tcx>(
+    tcx: TyCtxt<'tcx>,
     item_id: LocalDefId,
+    ty: Ty<'tcx>,
+    should_check_for_sync: bool,
 ) -> Result<(), ErrorGuaranteed> {
     enter_wf_checking_ctxt(tcx, item_id, |wfcx| {
-        let ty = tcx.type_of(item_id).instantiate_identity();
         let span = tcx.ty_span(item_id);
         let item_ty = wfcx.deeply_normalize(span, Some(WellFormedLoc::Ty(item_id)), ty);
 
@@ -1212,9 +1213,9 @@ pub(super) fn check_static_item(
         }
 
         // Ensure that the end result is `Sync` in a non-thread local `static`.
-        let should_check_for_sync = tcx.static_mutability(item_id.to_def_id())
-            == Some(hir::Mutability::Not)
+        let should_check_for_sync = should_check_for_sync
             && !is_foreign_item
+            && tcx.static_mutability(item_id.to_def_id()) == Some(hir::Mutability::Not)
             && !tcx.is_thread_local_static(item_id.to_def_id());
 
         if should_check_for_sync {

compiler/rustc_hir_analysis/src/collect/type_of.rs

@@ -14,6 +14,7 @@ use rustc_middle::{bug, span_bug};
 use rustc_span::{DUMMY_SP, Ident, Span};
 
 use super::{HirPlaceholderCollector, ItemCtxt, bad_placeholder};
+use crate::check::wfcheck::check_static_item;
 use crate::errors::TypeofReservedKeywordUsed;
 use crate::hir_ty_lowering::HirTyLowerer;
 
@@ -217,7 +218,13 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<'_
                         "static variable",
                     )
                 } else {
-                    icx.lower_ty(ty)
+                    let ty = icx.lower_ty(ty);
+                    // MIR relies on references to statics being scalars.
+                    // Verify that here to avoid ill-formed MIR.
+                    match check_static_item(tcx, def_id, ty, false) {
+                        Ok(()) => ty,
+                        Err(guar) => Ty::new_error(tcx, guar),
+                    }
                 }
             }
             ItemKind::Const(ident, _, ty, body_id) => {
@@ -275,7 +282,15 @@ pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<'_
                 let args = ty::GenericArgs::identity_for_item(tcx, def_id);
                 Ty::new_fn_def(tcx, def_id.to_def_id(), args)
             }
-            ForeignItemKind::Static(t, _, _) => icx.lower_ty(t),
+            ForeignItemKind::Static(ty, _, _) => {
+                let ty = icx.lower_ty(ty);
+                // MIR relies on references to statics being scalars.
+                // Verify that here to avoid ill-formed MIR.
+                match check_static_item(tcx, def_id, ty, false) {
+                    Ok(()) => ty,
+                    Err(guar) => Ty::new_error(tcx, guar),
+                }
+            }
             ForeignItemKind::Type => Ty::new_foreign(tcx, def_id.to_def_id()),
         },
 

compiler/rustc_hir_analysis/src/hir_ty_lowering/lint.rs

@@ -447,17 +447,30 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ {
     fn maybe_suggest_assoc_ty_bound(&self, self_ty: &hir::Ty<'_>, diag: &mut Diag<'_>) {
         let mut parents = self.tcx().hir_parent_iter(self_ty.hir_id);
 
-        if let Some((_, hir::Node::AssocItemConstraint(constraint))) = parents.next()
+        if let Some((c_hir_id, hir::Node::AssocItemConstraint(constraint))) = parents.next()
             && let Some(obj_ty) = constraint.ty()
+            && let Some((_, hir::Node::TraitRef(trait_ref))) = parents.next()
         {
-            if let Some((_, hir::Node::TraitRef(..))) = parents.next()
-                && let Some((_, hir::Node::Ty(ty))) = parents.next()
+            if let Some((_, hir::Node::Ty(ty))) = parents.next()
                 && let hir::TyKind::TraitObject(..) = ty.kind
             {
                 // Assoc ty bounds aren't permitted inside trait object types.
                 return;
             }
 
+            if trait_ref
+                .path
+                .segments
+                .iter()
+                .find_map(|seg| {
+                    seg.args.filter(|args| args.constraints.iter().any(|c| c.hir_id == c_hir_id))
+                })
+                .is_none_or(|args| args.parenthesized != hir::GenericArgsParentheses::No)
+            {
+                // Only consider angle-bracketed args (where we have a `=` to replace with `:`).
+                return;
+            }
+
             let lo = if constraint.gen_args.span_ext.is_dummy() {
                 constraint.ident.span
             } else {

compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs

@@ -1302,8 +1302,10 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 None => ".clone()".to_string(),
             };
 
+            let span = expr.span.find_oldest_ancestor_in_same_ctxt().shrink_to_hi();
+
             diag.span_suggestion_verbose(
-                expr.span.shrink_to_hi(),
+                span,
                 "consider using clone here",
                 suggestion,
                 Applicability::MachineApplicable,

compiler/rustc_hir_typeck/src/method/suggest.rs

@@ -264,6 +264,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     err.span_label(within_macro_span, "due to this macro variable");
                 }
                 self.suggest_valid_traits(&mut err, item_name, out_of_scope_traits, true);
+                self.suggest_unwrapping_inner_self(&mut err, source, rcvr_ty, item_name);
                 err.emit()
             }
 

compiler/rustc_hir_typeck/src/upvar.rs

@@ -1047,7 +1047,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             }
                         }
                     }
-                    lint.note("for more information, see <https://doc.rust-lang.org/nightly/edition-guide/rust-2021/disjoint-capture-in-closures.html>");
+                    lint.note("for more information, see <https://doc.rust-lang.org/edition-guide/rust-2021/disjoint-capture-in-closures.html>");
 
                     let diagnostic_msg = format!(
                         "add a dummy let to cause {migrated_variables_concat} to be fully captured"

compiler/rustc_interface/src/passes.rs

@@ -208,6 +208,10 @@ fn configure_and_expand(
         // Expand macros now!
         let krate = sess.time("expand_crate", || ecx.monotonic_expander().expand_crate(krate));
 
+        if ecx.nb_macro_errors > 0 {
+            sess.dcx().abort_if_errors();
+        }
+
         // The rest is error reporting and stats
 
         sess.psess.buffered_lints.with_lock(|buffered_lints: &mut Vec<BufferedEarlyLint>| {

compiler/rustc_lint/messages.ftl

@@ -593,7 +593,7 @@ lint_non_camel_case_type = {$sort} `{$name}` should have an upper camel case nam
 
 lint_non_fmt_panic = panic message is not a string literal
     .note = this usage of `{$name}!()` is deprecated; it will be a hard error in Rust 2021
-    .more_info_note = for more information, see <https://doc.rust-lang.org/nightly/edition-guide/rust-2021/panic-macro-consistency.html>
+    .more_info_note = for more information, see <https://doc.rust-lang.org/edition-guide/rust-2021/panic-macro-consistency.html>
     .supports_fmt_note = the `{$name}!()` macro supports formatting, so there's no need for the `format!()` macro here
     .supports_fmt_suggestion = remove the `format!(..)` macro call
     .display_suggestion = add a "{"{"}{"}"}" format string to `Display` the message

compiler/rustc_lint/src/builtin.rs

@@ -1654,7 +1654,7 @@ declare_lint! {
     "`...` range patterns are deprecated",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2021),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2021/warnings-promoted-to-error.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2021/warnings-promoted-to-error.html>",
     };
 }
 
@@ -1835,7 +1835,7 @@ declare_lint! {
     "detects edition keywords being used as an identifier",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/gen-keyword.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/gen-keyword.html>",
     };
 }
 
@@ -2870,7 +2870,7 @@ impl<'tcx> LateLintPass<'tcx> for AsmLabels {
         if let hir::Expr {
             kind:
                 hir::ExprKind::InlineAsm(hir::InlineAsm {
-                    asm_macro: AsmMacro::Asm | AsmMacro::NakedAsm,
+                    asm_macro: asm_macro @ (AsmMacro::Asm | AsmMacro::NakedAsm),
                     template_strs,
                     options,
                     ..
@@ -2878,6 +2878,15 @@ impl<'tcx> LateLintPass<'tcx> for AsmLabels {
             ..
         } = expr
         {
+            // Non-generic naked functions are allowed to define arbitrary
+            // labels.
+            if *asm_macro == AsmMacro::NakedAsm {
+                let def_id = expr.hir_id.owner.def_id;
+                if !cx.tcx.generics_of(def_id).requires_monomorphization(cx.tcx) {
+                    return;
+                }
+            }
+
             // asm with `options(raw)` does not do replacement with `{` and `}`.
             let raw = options.contains(InlineAsmOptions::RAW);
 

compiler/rustc_lint/src/if_let_rescope.rs

@@ -87,7 +87,7 @@ declare_lint! {
     rewriting in `match` is an option to preserve the semantics up to Edition 2021",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionSemanticsChange(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/temporary-if-let-scope.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/temporary-if-let-scope.html>",
     };
 }
 

compiler/rustc_lint/src/impl_trait_overcaptures.rs

@@ -71,7 +71,7 @@ declare_lint! {
     "`impl Trait` will capture more lifetimes than possibly intended in edition 2024",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionSemanticsChange(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/rpit-lifetime-capture.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/rpit-lifetime-capture.html>",
     };
 }
 

compiler/rustc_lint/src/late.rs

@@ -356,7 +356,16 @@ pub fn late_lint_mod<'tcx, T: LateLintPass<'tcx> + 'tcx>(
     let store = unerased_lint_store(tcx.sess);
 
     if store.late_module_passes.is_empty() {
-        late_lint_mod_inner(tcx, module_def_id, context, builtin_lints);
+        // If all builtin lints can be skipped, there is no point in running `late_lint_mod_inner`
+        // at all. This happens often for dependencies built with `--cap-lints=allow`.
+        let dont_need_to_run = tcx.lints_that_dont_need_to_run(());
+        let can_skip_lints = builtin_lints
+            .get_lints()
+            .iter()
+            .all(|lint| dont_need_to_run.contains(&LintId::of(lint)));
+        if !can_skip_lints {
+            late_lint_mod_inner(tcx, module_def_id, context, builtin_lints);
+        }
     } else {
         let builtin_lints = Box::new(builtin_lints) as Box<dyn LateLintPass<'tcx>>;
         let mut binding = store

compiler/rustc_lint/src/levels.rs

@@ -933,6 +933,7 @@ impl<'s, P: LintLevelsProvider> LintLevelsBuilder<'s, P> {
     fn check_gated_lint(&self, lint_id: LintId, span: Span, lint_from_cli: bool) -> bool {
         let feature = if let Some(feature) = lint_id.lint.feature_gate
             && !self.features.enabled(feature)
+            && !span.allows_unstable(feature)
         {
             // Lint is behind a feature that is not enabled; eventually return false.
             feature

compiler/rustc_lint/src/macro_expr_fragment_specifier_2024_migration.rs

@@ -65,7 +65,7 @@ declare_lint! {
     /// to ensure the macros implement the desired behavior.
     ///
     /// [editions]: https://doc.rust-lang.org/edition-guide/
-    /// [macro matcher fragment specifiers]: https://doc.rust-lang.org/nightly/edition-guide/rust-2024/macro-fragment-specifiers.html
+    /// [macro matcher fragment specifiers]: https://doc.rust-lang.org/edition-guide/rust-2024/macro-fragment-specifiers.html
     /// [`cargo fix`]: https://doc.rust-lang.org/cargo/commands/cargo-fix.html
     pub EDITION_2024_EXPR_FRAGMENT_SPECIFIER,
     Allow,
@@ -73,7 +73,7 @@ declare_lint! {
     To keep the existing behavior, use the `expr_2021` fragment specifier.",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionSemanticsChange(Edition::Edition2024),
-        reference: "Migration Guide <https://doc.rust-lang.org/nightly/edition-guide/rust-2024/macro-fragment-specifiers.html>",
+        reference: "Migration Guide <https://doc.rust-lang.org/edition-guide/rust-2024/macro-fragment-specifiers.html>",
     };
 }
 

compiler/rustc_lint/src/passes.rs

@@ -92,7 +92,7 @@ macro_rules! expand_combined_late_lint_pass_methods {
 /// Combines multiple lints passes into a single lint pass, at compile time,
 /// for maximum speed. Each `check_foo` method in `$methods` within this pass
 /// simply calls `check_foo` once per `$pass`. Compare with
-/// `LateLintPassObjects`, which is similar, but combines lint passes at
+/// `RuntimeCombinedLateLintPass`, which is similar, but combines lint passes at
 /// runtime.
 #[macro_export]
 macro_rules! declare_combined_late_lint_pass {
@@ -123,10 +123,10 @@ macro_rules! declare_combined_late_lint_pass {
         #[allow(rustc::lint_pass_impl_without_macro)]
         impl $crate::LintPass for $name {
             fn name(&self) -> &'static str {
-                panic!()
+                stringify!($name)
             }
             fn get_lints(&self) -> LintVec {
-                panic!()
+                $name::get_lints()
             }
         }
     )

compiler/rustc_lint/src/shadowed_into_iter.rs

@@ -32,7 +32,7 @@ declare_lint! {
     "detects calling `into_iter` on arrays in Rust 2015 and 2018",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionSemanticsChange(Edition::Edition2021),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2021/IntoIterator-for-arrays.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2021/IntoIterator-for-arrays.html>",
     };
 }
 
@@ -61,7 +61,7 @@ declare_lint! {
     "detects calling `into_iter` on boxed slices in Rust 2015, 2018, and 2021",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionSemanticsChange(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/intoiterator-box-slice.html>"
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/intoiterator-box-slice.html>"
     };
 }
 

compiler/rustc_lint/src/static_mut_refs.rs

@@ -54,7 +54,7 @@ declare_lint! {
     "creating a shared reference to mutable static",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/static-mut-references.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/static-mut-references.html>",
         explain_reason: false,
     };
     @edition Edition2024 => Deny;

compiler/rustc_lint/src/unused.rs

@@ -1340,7 +1340,15 @@ impl EarlyLintPass for UnusedParens {
                 self.with_self_ty_parens = false;
             }
             ast::TyKind::Ref(_, mut_ty) | ast::TyKind::Ptr(mut_ty) => {
-                self.in_no_bounds_pos.insert(mut_ty.ty.id, NoBoundsException::OneBound);
+                // If this type itself appears in no-bounds position, we propagate its
+                // potentially tighter constraint or risk a false posive (issue 143653).
+                let own_constraint = self.in_no_bounds_pos.get(&ty.id);
+                let constraint = match own_constraint {
+                    Some(NoBoundsException::None) => NoBoundsException::None,
+                    Some(NoBoundsException::OneBound) => NoBoundsException::OneBound,
+                    None => NoBoundsException::OneBound,
+                };
+                self.in_no_bounds_pos.insert(mut_ty.ty.id, constraint);
             }
             ast::TyKind::TraitObject(bounds, _) | ast::TyKind::ImplTrait(_, bounds) => {
                 for i in 0..bounds.len() {

compiler/rustc_lint_defs/src/builtin.rs

@@ -1814,7 +1814,7 @@ declare_lint! {
     "suggest using `dyn Trait` for trait objects",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2021),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2021/warnings-promoted-to-error.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2021/warnings-promoted-to-error.html>",
     };
 }
 
@@ -2472,7 +2472,7 @@ declare_lint! {
     "unsafe operations in unsafe functions without an explicit unsafe block are deprecated",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionSemanticsChange(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/unsafe-op-in-unsafe-fn.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/unsafe-op-in-unsafe-fn.html>",
         explain_reason: false
     };
     @edition Edition2024 => Warn;
@@ -3445,7 +3445,7 @@ declare_lint! {
     "detects usage of old versions of or-patterns",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2021),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2021/or-patterns-macro-rules.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2021/or-patterns-macro-rules.html>",
     };
 }
 
@@ -3494,7 +3494,7 @@ declare_lint! {
         prelude in future editions",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2021),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2021/prelude.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2021/prelude.html>",
     };
 }
 
@@ -3534,7 +3534,7 @@ declare_lint! {
         prelude in future editions",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/prelude.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/prelude.html>",
     };
 }
 
@@ -3571,7 +3571,7 @@ declare_lint! {
     "identifiers that will be parsed as a prefix in Rust 2021",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2021),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2021/reserving-syntax.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2021/reserving-syntax.html>",
     };
     crate_level_only
 }
@@ -4100,7 +4100,7 @@ declare_lint! {
     "never type fallback affecting unsafe function calls",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionAndFutureReleaseSemanticsChange(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/never-type-fallback.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/never-type-fallback.html>",
         report_in_deps: true,
     };
     @edition Edition2024 => Deny;
@@ -4155,7 +4155,7 @@ declare_lint! {
     "never type fallback affecting unsafe function calls",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionAndFutureReleaseError(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/never-type-fallback.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/never-type-fallback.html>",
         report_in_deps: true,
     };
     report_in_external_macro
@@ -4740,7 +4740,7 @@ declare_lint! {
     "detects unsafe functions being used as safe functions",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/newly-unsafe-functions.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/newly-unsafe-functions.html>",
     };
 }
 
@@ -4776,7 +4776,7 @@ declare_lint! {
     "detects missing unsafe keyword on extern declarations",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/unsafe-extern.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/unsafe-extern.html>",
     };
 }
 
@@ -4817,7 +4817,7 @@ declare_lint! {
     "detects unsafe attributes outside of unsafe",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/unsafe-attributes.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/unsafe-attributes.html>",
     };
 }
 
@@ -5014,7 +5014,7 @@ declare_lint! {
     "Detect and warn on significant change in drop order in tail expression location",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionSemanticsChange(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/temporary-tail-expr-scope.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/temporary-tail-expr-scope.html>",
     };
 }
 
@@ -5053,7 +5053,7 @@ declare_lint! {
     "will be parsed as a guarded string in Rust 2024",
     @future_incompatible = FutureIncompatibleInfo {
         reason: FutureIncompatibilityReason::EditionError(Edition::Edition2024),
-        reference: "<https://doc.rust-lang.org/nightly/edition-guide/rust-2024/reserved-syntax.html>",
+        reference: "<https://doc.rust-lang.org/edition-guide/rust-2024/reserved-syntax.html>",
     };
     crate_level_only
 }

compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp

@@ -1610,7 +1610,7 @@ extern "C" void LLVMRustPositionBefore(LLVMBuilderRef B, LLVMValueRef Instr) {
 
 extern "C" void LLVMRustPositionAfter(LLVMBuilderRef B, LLVMValueRef Instr) {
   if (auto I = dyn_cast<Instruction>(unwrap<Value>(Instr))) {
-    auto J = I->getNextNonDebugInstruction();
+    auto J = I->getNextNode();
     unwrap(B)->SetInsertPoint(J);
   }
 }

compiler/rustc_metadata/messages.ftl

@@ -330,3 +330,6 @@ metadata_wasm_import_form =
 
 metadata_whole_archive_needs_static =
     linking modifier `whole-archive` is only compatible with `static` linking kind
+
+metadata_raw_dylib_malformed =
+    link name must be well-formed if link kind is `raw-dylib`

compiler/rustc_metadata/src/errors.rs

@@ -815,3 +815,10 @@ pub struct AsyncDropTypesInDependency {
     pub extern_crate: Symbol,
     pub local_crate: Symbol,
 }
+
+#[derive(Diagnostic)]
+#[diag(metadata_raw_dylib_malformed)]
+pub struct RawDylibMalformed {
+    #[primary_span]
+    pub span: Span,
+}

compiler/rustc_metadata/src/native_libs.rs

@@ -700,8 +700,21 @@ impl<'tcx> Collector<'tcx> {
             .link_ordinal
             .map_or(import_name_type, |ord| Some(PeImportNameType::Ordinal(ord)));
 
+        let name = codegen_fn_attrs.link_name.unwrap_or_else(|| self.tcx.item_name(item));
+
+        if self.tcx.sess.target.binary_format == BinaryFormat::Elf {
+            let name = name.as_str();
+            if name.contains('\0') {
+                self.tcx.dcx().emit_err(errors::RawDylibMalformed { span });
+            } else if let Some((left, right)) = name.split_once('@')
+                && (left.is_empty() || right.is_empty() || right.contains('@'))
+            {
+                self.tcx.dcx().emit_err(errors::RawDylibMalformed { span });
+            }
+        }
+
         DllImport {
-            name: codegen_fn_attrs.link_name.unwrap_or_else(|| self.tcx.item_name(item)),
+            name,
             import_name_type,
             calling_convention,
             span,

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -2124,11 +2124,11 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             };
             let def_id = id.owner_id.to_def_id();
 
-            self.tables.defaultness.set_some(def_id.index, tcx.defaultness(def_id));
-
             if of_trait && let Some(header) = tcx.impl_trait_header(def_id) {
                 record!(self.tables.impl_trait_header[def_id] <- header);
 
+                self.tables.defaultness.set_some(def_id.index, tcx.defaultness(def_id));
+
                 let trait_ref = header.trait_ref.instantiate_identity();
                 let simplified_self_ty = fast_reject::simplify_type(
                     self.tcx,

compiler/rustc_middle/src/hooks/mod.rs

@@ -50,10 +50,10 @@ macro_rules! declare_hooks {
 declare_hooks! {
     /// Tries to destructure an `mir::Const` ADT or array into its variant index
     /// and its field values. This should only be used for pretty printing.
-    hook try_destructure_mir_constant_for_user_output(val: mir::ConstValue<'tcx>, ty: Ty<'tcx>) -> Option<mir::DestructuredConstant<'tcx>>;
+    hook try_destructure_mir_constant_for_user_output(val: mir::ConstValue, ty: Ty<'tcx>) -> Option<mir::DestructuredConstant<'tcx>>;
 
     /// Getting a &core::panic::Location referring to a span.
-    hook const_caller_location(file: rustc_span::Symbol, line: u32, col: u32) -> mir::ConstValue<'tcx>;
+    hook const_caller_location(file: rustc_span::Symbol, line: u32, col: u32) -> mir::ConstValue;
 
     /// Returns `true` if this def is a function-like thing that is eligible for
     /// coverage instrumentation under `-Cinstrument-coverage`.

compiler/rustc_middle/src/mir/consts.rs

@@ -9,9 +9,7 @@ use rustc_span::{DUMMY_SP, Span, Symbol};
 use rustc_type_ir::TypeVisitableExt;
 
 use super::interpret::ReportedErrorInfo;
-use crate::mir::interpret::{
-    AllocId, AllocRange, ConstAllocation, ErrorHandled, GlobalAlloc, Scalar, alloc_range,
-};
+use crate::mir::interpret::{AllocId, AllocRange, ErrorHandled, GlobalAlloc, Scalar, alloc_range};
 use crate::mir::{Promoted, pretty_print_const_value};
 use crate::ty::print::{pretty_print_const, with_no_trimmed_paths};
 use crate::ty::{self, ConstKind, GenericArgsRef, ScalarInt, Ty, TyCtxt};
@@ -33,8 +31,8 @@ pub struct ConstAlloc<'tcx> {
 /// Represents a constant value in Rust. `Scalar` and `Slice` are optimizations for
 /// array length computations, enum discriminants and the pattern matching logic.
 #[derive(Copy, Clone, Debug, Eq, PartialEq, TyEncodable, TyDecodable, Hash)]
-#[derive(HashStable, Lift)]
-pub enum ConstValue<'tcx> {
+#[derive(HashStable)]
+pub enum ConstValue {
     /// Used for types with `layout::abi::Scalar` ABI.
     ///
     /// Not using the enum `Value` to encode that this must not be `Uninit`.
@@ -52,7 +50,7 @@ pub enum ConstValue<'tcx> {
     Slice {
         /// The allocation storing the slice contents.
         /// This always points to the beginning of the allocation.
-        data: ConstAllocation<'tcx>,
+        alloc_id: AllocId,
         /// The metadata field of the reference.
         /// This is a "target usize", so we use `u64` as in the interpreter.
         meta: u64,
@@ -75,9 +73,9 @@ pub enum ConstValue<'tcx> {
 }
 
 #[cfg(target_pointer_width = "64")]
-rustc_data_structures::static_assert_size!(ConstValue<'_>, 24);
+rustc_data_structures::static_assert_size!(ConstValue, 24);
 
-impl<'tcx> ConstValue<'tcx> {
+impl ConstValue {
     #[inline]
     pub fn try_to_scalar(&self) -> Option<Scalar> {
         match *self {
@@ -98,11 +96,11 @@ impl<'tcx> ConstValue<'tcx> {
         self.try_to_scalar_int()?.try_into().ok()
     }
 
-    pub fn try_to_target_usize(&self, tcx: TyCtxt<'tcx>) -> Option<u64> {
+    pub fn try_to_target_usize(&self, tcx: TyCtxt<'_>) -> Option<u64> {
         Some(self.try_to_scalar_int()?.to_target_usize(tcx))
     }
 
-    pub fn try_to_bits_for_ty(
+    pub fn try_to_bits_for_ty<'tcx>(
         &self,
         tcx: TyCtxt<'tcx>,
         typing_env: ty::TypingEnv<'tcx>,
@@ -132,12 +130,15 @@ impl<'tcx> ConstValue<'tcx> {
     }
 
     /// Must only be called on constants of type `&str` or `&[u8]`!
-    pub fn try_get_slice_bytes_for_diagnostics(&self, tcx: TyCtxt<'tcx>) -> Option<&'tcx [u8]> {
-        let (data, start, end) = match self {
+    pub fn try_get_slice_bytes_for_diagnostics<'tcx>(
+        &self,
+        tcx: TyCtxt<'tcx>,
+    ) -> Option<&'tcx [u8]> {
+        let (alloc_id, start, len) = match self {
             ConstValue::Scalar(_) | ConstValue::ZeroSized => {
                 bug!("`try_get_slice_bytes` on non-slice constant")
             }
-            &ConstValue::Slice { data, meta } => (data, 0, meta),
+            &ConstValue::Slice { alloc_id, meta } => (alloc_id, 0, meta),
             &ConstValue::Indirect { alloc_id, offset } => {
                 // The reference itself is stored behind an indirection.
                 // Load the reference, and then load the actual slice contents.
@@ -170,26 +171,29 @@ impl<'tcx> ConstValue<'tcx> {
                 // Non-empty slice, must have memory. We know this is a relative pointer.
                 let (inner_prov, offset) =
                     ptr.into_pointer_or_addr().ok()?.prov_and_relative_offset();
-                let data = tcx.global_alloc(inner_prov.alloc_id()).unwrap_memory();
-                (data, offset.bytes(), offset.bytes() + len)
+                (inner_prov.alloc_id(), offset.bytes(), len)
             }
         };
 
+        let data = tcx.global_alloc(alloc_id).unwrap_memory();
+
         // This is for diagnostics only, so we are okay to use `inspect_with_uninit_and_ptr_outside_interpreter`.
         let start = start.try_into().unwrap();
-        let end = end.try_into().unwrap();
+        let end = start + usize::try_from(len).unwrap();
         Some(data.inner().inspect_with_uninit_and_ptr_outside_interpreter(start..end))
     }
 
     /// Check if a constant may contain provenance information. This is used by MIR opts.
     /// Can return `true` even if there is no provenance.
-    pub fn may_have_provenance(&self, tcx: TyCtxt<'tcx>, size: Size) -> bool {
+    pub fn may_have_provenance(&self, tcx: TyCtxt<'_>, size: Size) -> bool {
         match *self {
             ConstValue::ZeroSized | ConstValue::Scalar(Scalar::Int(_)) => return false,
             ConstValue::Scalar(Scalar::Ptr(..)) => return true,
             // It's hard to find out the part of the allocation we point to;
             // just conservatively check everything.
-            ConstValue::Slice { data, meta: _ } => !data.inner().provenance().ptrs().is_empty(),
+            ConstValue::Slice { alloc_id, meta: _ } => {
+                !tcx.global_alloc(alloc_id).unwrap_memory().inner().provenance().ptrs().is_empty()
+            }
             ConstValue::Indirect { alloc_id, offset } => !tcx
                 .global_alloc(alloc_id)
                 .unwrap_memory()
@@ -200,7 +204,7 @@ impl<'tcx> ConstValue<'tcx> {
     }
 
     /// Check if a constant only contains uninitialized bytes.
-    pub fn all_bytes_uninit(&self, tcx: TyCtxt<'tcx>) -> bool {
+    pub fn all_bytes_uninit(&self, tcx: TyCtxt<'_>) -> bool {
         let ConstValue::Indirect { alloc_id, .. } = self else {
             return false;
         };
@@ -247,7 +251,7 @@ pub enum Const<'tcx> {
 
     /// This constant cannot go back into the type system, as it represents
     /// something the type system cannot handle (e.g. pointers).
-    Val(ConstValue<'tcx>, Ty<'tcx>),
+    Val(ConstValue, Ty<'tcx>),
 }
 
 impl<'tcx> Const<'tcx> {
@@ -343,7 +347,7 @@ impl<'tcx> Const<'tcx> {
         tcx: TyCtxt<'tcx>,
         typing_env: ty::TypingEnv<'tcx>,
         span: Span,
-    ) -> Result<ConstValue<'tcx>, ErrorHandled> {
+    ) -> Result<ConstValue, ErrorHandled> {
         match self {
             Const::Ty(_, c) => {
                 if c.has_non_region_param() {
@@ -440,7 +444,7 @@ impl<'tcx> Const<'tcx> {
     }
 
     #[inline]
-    pub fn from_value(val: ConstValue<'tcx>, ty: Ty<'tcx>) -> Self {
+    pub fn from_value(val: ConstValue, ty: Ty<'tcx>) -> Self {
         Self::Val(val, ty)
     }
 
@@ -487,9 +491,8 @@ impl<'tcx> Const<'tcx> {
     /// taking into account even pointer identity tests.
     pub fn is_deterministic(&self) -> bool {
         // Some constants may generate fresh allocations for pointers they contain,
-        // so using the same constant twice can yield two different results:
-        // - valtrees purposefully generate new allocations
-        // - ConstValue::Slice also generate new allocations
+        // so using the same constant twice can yield two different results.
+        // Notably, valtrees purposefully generate new allocations.
         match self {
             Const::Ty(_, c) => match c.kind() {
                 ty::ConstKind::Param(..) => true,
@@ -507,11 +510,11 @@ impl<'tcx> Const<'tcx> {
                 | ty::ConstKind::Placeholder(..) => bug!(),
             },
             Const::Unevaluated(..) => false,
-            // If the same slice appears twice in the MIR, we cannot guarantee that we will
-            // give the same `AllocId` to the data.
-            Const::Val(ConstValue::Slice { .. }, _) => false,
             Const::Val(
-                ConstValue::ZeroSized | ConstValue::Scalar(_) | ConstValue::Indirect { .. },
+                ConstValue::Slice { .. }
+                | ConstValue::ZeroSized
+                | ConstValue::Scalar(_)
+                | ConstValue::Indirect { .. },
                 _,
             ) => true,
         }
@@ -574,7 +577,7 @@ impl<'tcx> Display for Const<'tcx> {
 /// Const-related utilities
 
 impl<'tcx> TyCtxt<'tcx> {
-    pub fn span_as_caller_location(self, span: Span) -> ConstValue<'tcx> {
+    pub fn span_as_caller_location(self, span: Span) -> ConstValue {
         let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
         let caller = self.sess.source_map().lookup_char_pos(topmost.lo());
         self.const_caller_location(

compiler/rustc_middle/src/mir/interpret/error.rs

@@ -137,7 +137,7 @@ impl<'tcx> ValTreeCreationError<'tcx> {
 
 pub type EvalToAllocationRawResult<'tcx> = Result<ConstAlloc<'tcx>, ErrorHandled>;
 pub type EvalStaticInitializerRawResult<'tcx> = Result<ConstAllocation<'tcx>, ErrorHandled>;
-pub type EvalToConstValueResult<'tcx> = Result<ConstValue<'tcx>, ErrorHandled>;
+pub type EvalToConstValueResult<'tcx> = Result<ConstValue, ErrorHandled>;
 pub type EvalToValTreeResult<'tcx> = Result<ValTree<'tcx>, ValTreeCreationError<'tcx>>;
 
 #[cfg(target_pointer_width = "64")]

compiler/rustc_middle/src/mir/mono.rs

@@ -143,10 +143,8 @@ impl<'tcx> MonoItem<'tcx> {
         };
 
         // Similarly, the executable entrypoint must be instantiated exactly once.
-        if let Some((entry_def_id, _)) = tcx.entry_fn(()) {
-            if instance.def_id() == entry_def_id {
-                return InstantiationMode::GloballyShared { may_conflict: false };
-            }
+        if tcx.is_entrypoint(instance.def_id()) {
+            return InstantiationMode::GloballyShared { may_conflict: false };
         }
 
         // If the function is #[naked] or contains any other attribute that requires exactly-once

compiler/rustc_middle/src/mir/pretty.rs

@@ -1465,7 +1465,7 @@ impl<'tcx> Visitor<'tcx> for ExtraComments<'tcx> {
                 self.push(&format!("+ user_ty: {user_ty:?}"));
             }
 
-            let fmt_val = |val: ConstValue<'tcx>, ty: Ty<'tcx>| {
+            let fmt_val = |val: ConstValue, ty: Ty<'tcx>| {
                 let tcx = self.tcx;
                 rustc_data_structures::make_display(move |fmt| {
                     pretty_print_const_value_tcx(tcx, val, ty, fmt)
@@ -1562,16 +1562,12 @@ pub fn write_allocations<'tcx>(
         alloc.inner().provenance().ptrs().values().map(|p| p.alloc_id())
     }
 
-    fn alloc_id_from_const_val(val: ConstValue<'_>) -> Option<AllocId> {
+    fn alloc_id_from_const_val(val: ConstValue) -> Option<AllocId> {
         match val {
             ConstValue::Scalar(interpret::Scalar::Ptr(ptr, _)) => Some(ptr.provenance.alloc_id()),
             ConstValue::Scalar(interpret::Scalar::Int { .. }) => None,
             ConstValue::ZeroSized => None,
-            ConstValue::Slice { .. } => {
-                // `u8`/`str` slices, shouldn't contain pointers that we want to print.
-                None
-            }
-            ConstValue::Indirect { alloc_id, .. } => {
+            ConstValue::Slice { alloc_id, .. } | ConstValue::Indirect { alloc_id, .. } => {
                 // FIXME: we don't actually want to print all of these, since some are printed nicely directly as values inline in MIR.
                 // Really we'd want `pretty_print_const_value` to decide which allocations to print, instead of having a separate visitor.
                 Some(alloc_id)
@@ -1885,7 +1881,7 @@ fn pretty_print_byte_str(fmt: &mut Formatter<'_>, byte_str: &[u8]) -> fmt::Resul
 fn comma_sep<'tcx>(
     tcx: TyCtxt<'tcx>,
     fmt: &mut Formatter<'_>,
-    elems: Vec<(ConstValue<'tcx>, Ty<'tcx>)>,
+    elems: Vec<(ConstValue, Ty<'tcx>)>,
 ) -> fmt::Result {
     let mut first = true;
     for (ct, ty) in elems {
@@ -1900,7 +1896,7 @@ fn comma_sep<'tcx>(
 
 fn pretty_print_const_value_tcx<'tcx>(
     tcx: TyCtxt<'tcx>,
-    ct: ConstValue<'tcx>,
+    ct: ConstValue,
     ty: Ty<'tcx>,
     fmt: &mut Formatter<'_>,
 ) -> fmt::Result {
@@ -1947,7 +1943,7 @@ fn pretty_print_const_value_tcx<'tcx>(
             let ct = tcx.lift(ct).unwrap();
             let ty = tcx.lift(ty).unwrap();
             if let Some(contents) = tcx.try_destructure_mir_constant_for_user_output(ct, ty) {
-                let fields: Vec<(ConstValue<'_>, Ty<'_>)> = contents.fields.to_vec();
+                let fields: Vec<(ConstValue, Ty<'_>)> = contents.fields.to_vec();
                 match *ty.kind() {
                     ty::Array(..) => {
                         fmt.write_str("[")?;
@@ -2028,7 +2024,7 @@ fn pretty_print_const_value_tcx<'tcx>(
 }
 
 pub(crate) fn pretty_print_const_value<'tcx>(
-    ct: ConstValue<'tcx>,
+    ct: ConstValue,
     ty: Ty<'tcx>,
     fmt: &mut Formatter<'_>,
 ) -> fmt::Result {

compiler/rustc_middle/src/mir/query.rs

@@ -173,5 +173,5 @@ pub enum AnnotationSource {
 #[derive(Copy, Clone, Debug, HashStable)]
 pub struct DestructuredConstant<'tcx> {
     pub variant: Option<VariantIdx>,
-    pub fields: &'tcx [(ConstValue<'tcx>, Ty<'tcx>)],
+    pub fields: &'tcx [(ConstValue, Ty<'tcx>)],
 }

compiler/rustc_middle/src/query/erase.rs

@@ -153,8 +153,8 @@ impl EraseType for Result<mir::ConstAlloc<'_>, mir::interpret::ErrorHandled> {
     type Result = [u8; size_of::<Result<mir::ConstAlloc<'static>, mir::interpret::ErrorHandled>>()];
 }
 
-impl EraseType for Result<mir::ConstValue<'_>, mir::interpret::ErrorHandled> {
-    type Result = [u8; size_of::<Result<mir::ConstValue<'static>, mir::interpret::ErrorHandled>>()];
+impl EraseType for Result<mir::ConstValue, mir::interpret::ErrorHandled> {
+    type Result = [u8; size_of::<Result<mir::ConstValue, mir::interpret::ErrorHandled>>()];
 }
 
 impl EraseType for EvalToValTreeResult<'_> {
@@ -301,6 +301,7 @@ trivial! {
     rustc_middle::middle::resolve_bound_vars::ResolvedArg,
     rustc_middle::middle::stability::DeprecationEntry,
     rustc_middle::mir::ConstQualifs,
+    rustc_middle::mir::ConstValue,
     rustc_middle::mir::interpret::AllocId,
     rustc_middle::mir::interpret::CtfeProvenance,
     rustc_middle::mir::interpret::ErrorHandled,
@@ -362,7 +363,6 @@ tcx_lifetime! {
     rustc_middle::mir::Const,
     rustc_middle::mir::DestructuredConstant,
     rustc_middle::mir::ConstAlloc,
-    rustc_middle::mir::ConstValue,
     rustc_middle::mir::interpret::GlobalId,
     rustc_middle::mir::interpret::LitToConstInput,
     rustc_middle::mir::interpret::EvalStaticInitializerRawResult,

compiler/rustc_middle/src/query/mod.rs

@@ -1363,7 +1363,7 @@ rustc_queries! {
     }
 
     /// Converts a type-level constant value into a MIR constant value.
-    query valtree_to_const_val(key: ty::Value<'tcx>) -> mir::ConstValue<'tcx> {
+    query valtree_to_const_val(key: ty::Value<'tcx>) -> mir::ConstValue {
         desc { "converting type-level constant value to MIR constant value"}
     }
 
@@ -2152,9 +2152,6 @@ rustc_queries! {
         desc { |tcx| "collecting child items of module `{}`", tcx.def_path_str(def_id) }
         separate_provide_extern
     }
-    query extern_mod_stmt_cnum(def_id: LocalDefId) -> Option<CrateNum> {
-        desc { |tcx| "computing crate imported by `{}`", tcx.def_path_str(def_id) }
-    }
 
     /// Gets the number of definitions in a foreign crate.
     ///

compiler/rustc_middle/src/ty/context.rs

@@ -5,7 +5,7 @@
 pub mod tls;
 
 use std::assert_matches::debug_assert_matches;
-use std::borrow::Borrow;
+use std::borrow::{Borrow, Cow};
 use std::cmp::Ordering;
 use std::env::VarError;
 use std::ffi::OsStr;
@@ -1041,11 +1041,13 @@ const NUM_PREINTERNED_TY_VARS: u32 = 100;
 const NUM_PREINTERNED_FRESH_TYS: u32 = 20;
 const NUM_PREINTERNED_FRESH_INT_TYS: u32 = 3;
 const NUM_PREINTERNED_FRESH_FLOAT_TYS: u32 = 3;
+const NUM_PREINTERNED_ANON_BOUND_TYS_I: u32 = 3;
+const NUM_PREINTERNED_ANON_BOUND_TYS_V: u32 = 20;
 
 // This number may seem high, but it is reached in all but the smallest crates.
 const NUM_PREINTERNED_RE_VARS: u32 = 500;
-const NUM_PREINTERNED_RE_LATE_BOUNDS_I: u32 = 2;
-const NUM_PREINTERNED_RE_LATE_BOUNDS_V: u32 = 20;
+const NUM_PREINTERNED_ANON_RE_BOUNDS_I: u32 = 3;
+const NUM_PREINTERNED_ANON_RE_BOUNDS_V: u32 = 20;
 
 pub struct CommonTypes<'tcx> {
     pub unit: Ty<'tcx>,
@@ -1088,6 +1090,11 @@ pub struct CommonTypes<'tcx> {
 
     /// Pre-interned `Infer(ty::FreshFloatTy(n))` for small values of `n`.
     pub fresh_float_tys: Vec<Ty<'tcx>>,
+
+    /// Pre-interned values of the form:
+    /// `Bound(DebruijnIndex(i), BoundTy { var: v, kind: BoundTyKind::Anon})`
+    /// for small values of `i` and `v`.
+    pub anon_bound_tys: Vec<Vec<Ty<'tcx>>>,
 }
 
 pub struct CommonLifetimes<'tcx> {
@@ -1101,9 +1108,9 @@ pub struct CommonLifetimes<'tcx> {
     pub re_vars: Vec<Region<'tcx>>,
 
     /// Pre-interned values of the form:
-    /// `ReBound(DebruijnIndex(i), BoundRegion { var: v, kind: BrAnon })`
+    /// `ReBound(DebruijnIndex(i), BoundRegion { var: v, kind: BoundRegionKind::Anon })`
     /// for small values of `i` and `v`.
-    pub re_late_bounds: Vec<Vec<Region<'tcx>>>,
+    pub anon_re_bounds: Vec<Vec<Region<'tcx>>>,
 }
 
 pub struct CommonConsts<'tcx> {
@@ -1131,6 +1138,19 @@ impl<'tcx> CommonTypes<'tcx> {
         let fresh_float_tys: Vec<_> =
             (0..NUM_PREINTERNED_FRESH_FLOAT_TYS).map(|n| mk(Infer(ty::FreshFloatTy(n)))).collect();
 
+        let anon_bound_tys = (0..NUM_PREINTERNED_ANON_BOUND_TYS_I)
+            .map(|i| {
+                (0..NUM_PREINTERNED_ANON_BOUND_TYS_V)
+                    .map(|v| {
+                        mk(ty::Bound(
+                            ty::DebruijnIndex::from(i),
+                            ty::BoundTy { var: ty::BoundVar::from(v), kind: ty::BoundTyKind::Anon },
+                        ))
+                    })
+                    .collect()
+            })
+            .collect();
+
         CommonTypes {
             unit: mk(Tuple(List::empty())),
             bool: mk(Bool),
@@ -1161,6 +1181,7 @@ impl<'tcx> CommonTypes<'tcx> {
             fresh_tys,
             fresh_int_tys,
             fresh_float_tys,
+            anon_bound_tys,
         }
     }
 }
@@ -1176,9 +1197,9 @@ impl<'tcx> CommonLifetimes<'tcx> {
         let re_vars =
             (0..NUM_PREINTERNED_RE_VARS).map(|n| mk(ty::ReVar(ty::RegionVid::from(n)))).collect();
 
-        let re_late_bounds = (0..NUM_PREINTERNED_RE_LATE_BOUNDS_I)
+        let anon_re_bounds = (0..NUM_PREINTERNED_ANON_RE_BOUNDS_I)
             .map(|i| {
-                (0..NUM_PREINTERNED_RE_LATE_BOUNDS_V)
+                (0..NUM_PREINTERNED_ANON_RE_BOUNDS_V)
                     .map(|v| {
                         mk(ty::ReBound(
                             ty::DebruijnIndex::from(i),
@@ -1196,7 +1217,7 @@ impl<'tcx> CommonLifetimes<'tcx> {
             re_static: mk(ty::ReStatic),
             re_erased: mk(ty::ReErased),
             re_vars,
-            re_late_bounds,
+            anon_re_bounds,
         }
     }
 }
@@ -1625,7 +1646,11 @@ impl<'tcx> TyCtxt<'tcx> {
 
     /// Allocates a read-only byte or string literal for `mir::interpret` with alignment 1.
     /// Returns the same `AllocId` if called again with the same bytes.
-    pub fn allocate_bytes_dedup(self, bytes: &[u8], salt: usize) -> interpret::AllocId {
+    pub fn allocate_bytes_dedup<'a>(
+        self,
+        bytes: impl Into<Cow<'a, [u8]>>,
+        salt: usize,
+    ) -> interpret::AllocId {
         // Create an allocation that just contains these bytes.
         let alloc = interpret::Allocation::from_bytes_byte_aligned_immutable(bytes, ());
         let alloc = self.mk_const_alloc(alloc);
@@ -3373,6 +3398,11 @@ impl<'tcx> TyCtxt<'tcx> {
         self.resolutions(()).module_children.get(&def_id).map_or(&[], |v| &v[..])
     }
 
+    /// Return the crate imported by given use item.
+    pub fn extern_mod_stmt_cnum(self, def_id: LocalDefId) -> Option<CrateNum> {
+        self.resolutions(()).extern_crate_map.get(&def_id).copied()
+    }
+
     pub fn resolver_for_lowering(self) -> &'tcx Steal<(ty::ResolverAstLowering, Arc<ast::Crate>)> {
         self.resolver_for_lowering_raw(()).0
     }
@@ -3410,6 +3440,20 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn do_not_recommend_impl(self, def_id: DefId) -> bool {
         self.get_diagnostic_attr(def_id, sym::do_not_recommend).is_some()
     }
+
+    /// Whether this def is one of the special bin crate entrypoint functions that must have a
+    /// monomorphization and also not be internalized in the bin crate.
+    pub fn is_entrypoint(self, def_id: DefId) -> bool {
+        if self.is_lang_item(def_id, LangItem::Start) {
+            return true;
+        }
+        if let Some((entry_def_id, _)) = self.entry_fn(())
+            && entry_def_id == def_id
+        {
+            return true;
+        }
+        false
+    }
 }
 
 /// Parameter attributes that can only be determined by examining the body of a function instead
@@ -3428,8 +3472,6 @@ pub struct DeducedParamAttrs {
 }
 
 pub fn provide(providers: &mut Providers) {
-    providers.extern_mod_stmt_cnum =
-        |tcx, id| tcx.resolutions(()).extern_crate_map.get(&id).cloned();
     providers.is_panic_runtime =
         |tcx, LocalCrate| contains_name(tcx.hir_krate_attrs(), sym::panic_runtime);
     providers.is_compiler_builtins =

compiler/rustc_middle/src/ty/print/pretty.rs

@@ -86,7 +86,7 @@ macro_rules! define_helper {
 
             impl $helper {
                 pub fn new() -> $helper {
-                    $helper($tl.with(|c| c.replace(true)))
+                    $helper($tl.replace(true))
                 }
             }
 
@@ -100,12 +100,12 @@ macro_rules! define_helper {
 
             impl Drop for $helper {
                 fn drop(&mut self) {
-                    $tl.with(|c| c.set(self.0))
+                    $tl.set(self.0)
                 }
             }
 
             pub fn $name() -> bool {
-                $tl.with(|c| c.get())
+                $tl.get()
             }
         )+
     }

compiler/rustc_middle/src/ty/region.rs

@@ -54,7 +54,7 @@ impl<'tcx> Region<'tcx> {
     ) -> Region<'tcx> {
         // Use a pre-interned one when possible.
         if let ty::BoundRegion { var, kind: ty::BoundRegionKind::Anon } = bound_region
-            && let Some(inner) = tcx.lifetimes.re_late_bounds.get(debruijn.as_usize())
+            && let Some(inner) = tcx.lifetimes.anon_re_bounds.get(debruijn.as_usize())
             && let Some(re) = inner.get(var.as_usize()).copied()
         {
             re

compiler/rustc_middle/src/ty/structural_impls.rs

@@ -4,6 +4,7 @@
 //! to help with the tedium.
 
 use std::fmt::{self, Debug};
+use std::marker::PhantomData;
 
 use rustc_abi::TyAndLayout;
 use rustc_hir::def::Namespace;
@@ -234,6 +235,7 @@ TrivialLiftImpls! {
     rustc_abi::ExternAbi,
     rustc_abi::Size,
     rustc_hir::Safety,
+    rustc_middle::mir::ConstValue,
     rustc_type_ir::BoundConstness,
     rustc_type_ir::PredicatePolarity,
     // tidy-alphabetical-end
@@ -250,7 +252,7 @@ TrivialTypeTraversalImpls! {
     crate::mir::BlockTailInfo,
     crate::mir::BorrowKind,
     crate::mir::CastKind,
-    crate::mir::ConstValue<'tcx>,
+    crate::mir::ConstValue,
     crate::mir::CoroutineSavedLocal,
     crate::mir::FakeReadCause,
     crate::mir::Local,
@@ -311,6 +313,13 @@ TrivialTypeTraversalAndLiftImpls! {
 ///////////////////////////////////////////////////////////////////////////
 // Lift implementations
 
+impl<'tcx> Lift<TyCtxt<'tcx>> for PhantomData<&()> {
+    type Lifted = PhantomData<&'tcx ()>;
+    fn lift_to_interner(self, _: TyCtxt<'tcx>) -> Option<Self::Lifted> {
+        Some(PhantomData)
+    }
+}
+
 impl<'tcx, T: Lift<TyCtxt<'tcx>>> Lift<TyCtxt<'tcx>> for Option<T> {
     type Lifted = Option<T::Lifted>;
     fn lift_to_interner(self, tcx: TyCtxt<'tcx>) -> Option<Self::Lifted> {

compiler/rustc_middle/src/ty/sty.rs

@@ -485,7 +485,15 @@ impl<'tcx> Ty<'tcx> {
         index: ty::DebruijnIndex,
         bound_ty: ty::BoundTy,
     ) -> Ty<'tcx> {
-        Ty::new(tcx, Bound(index, bound_ty))
+        // Use a pre-interned one when possible.
+        if let ty::BoundTy { var, kind: ty::BoundTyKind::Anon } = bound_ty
+            && let Some(inner) = tcx.types.anon_bound_tys.get(index.as_usize())
+            && let Some(ty) = inner.get(var.as_usize()).copied()
+        {
+            ty
+        } else {
+            Ty::new(tcx, Bound(index, bound_ty))
+        }
     }
 
     #[inline]

compiler/rustc_mir_build/messages.ftl

@@ -86,10 +86,16 @@ mir_build_confused = missing patterns are not covered because `{$variable}` is i
 
 mir_build_const_continue_bad_const = could not determine the target branch for this `#[const_continue]`
     .label = this value is too generic
-    .note = the value must be a literal or a monomorphic const
 
 mir_build_const_continue_missing_value = a `#[const_continue]` must break to a label with a value
 
+mir_build_const_continue_not_const = could not determine the target branch for this `#[const_continue]`
+    .help = try extracting the expression into a `const` item
+
+mir_build_const_continue_not_const_const_block = `const` blocks may use generics, and are not evaluated early enough
+mir_build_const_continue_not_const_const_other = this value must be a literal or a monomorphic const
+mir_build_const_continue_not_const_constant_parameter = constant parameters may use generics, and are not evaluated early enough
+
 mir_build_const_continue_unknown_jump_target = the target of this `#[const_continue]` is not statically known
     .label = this value must be a literal or a monomorphic const
 

compiler/rustc_mir_build/src/builder/expr/as_constant.rs

@@ -3,7 +3,7 @@
 use rustc_abi::Size;
 use rustc_ast as ast;
 use rustc_hir::LangItem;
-use rustc_middle::mir::interpret::{Allocation, CTFE_ALLOC_SALT, LitToConstInput, Scalar};
+use rustc_middle::mir::interpret::{CTFE_ALLOC_SALT, LitToConstInput, Scalar};
 use rustc_middle::mir::*;
 use rustc_middle::thir::*;
 use rustc_middle::ty::{
@@ -120,17 +120,18 @@ fn lit_to_mir_constant<'tcx>(tcx: TyCtxt<'tcx>, lit_input: LitToConstInput<'tcx>
 
     let value = match (lit, lit_ty.kind()) {
         (ast::LitKind::Str(s, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_str() => {
-            let s = s.as_str();
-            let allocation = Allocation::from_bytes_byte_aligned_immutable(s.as_bytes(), ());
-            let allocation = tcx.mk_const_alloc(allocation);
-            ConstValue::Slice { data: allocation, meta: allocation.inner().size().bytes() }
+            let s = s.as_str().as_bytes();
+            let len = s.len();
+            let allocation = tcx.allocate_bytes_dedup(s, CTFE_ALLOC_SALT);
+            ConstValue::Slice { alloc_id: allocation, meta: len.try_into().unwrap() }
         }
-        (ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _))
+        (ast::LitKind::ByteStr(byte_sym, _), ty::Ref(_, inner_ty, _))
             if matches!(inner_ty.kind(), ty::Slice(_)) =>
         {
-            let allocation = Allocation::from_bytes_byte_aligned_immutable(data.as_byte_str(), ());
-            let allocation = tcx.mk_const_alloc(allocation);
-            ConstValue::Slice { data: allocation, meta: allocation.inner().size().bytes() }
+            let data = byte_sym.as_byte_str();
+            let len = data.len();
+            let allocation = tcx.allocate_bytes_dedup(data, CTFE_ALLOC_SALT);
+            ConstValue::Slice { alloc_id: allocation, meta: len.try_into().unwrap() }
         }
         (ast::LitKind::ByteStr(byte_sym, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_array() => {
             let id = tcx.allocate_bytes_dedup(byte_sym.as_byte_str(), CTFE_ALLOC_SALT);
@@ -138,10 +139,10 @@ fn lit_to_mir_constant<'tcx>(tcx: TyCtxt<'tcx>, lit_input: LitToConstInput<'tcx>
         }
         (ast::LitKind::CStr(byte_sym, _), ty::Ref(_, inner_ty, _)) if matches!(inner_ty.kind(), ty::Adt(def, _) if tcx.is_lang_item(def.did(), LangItem::CStr)) =>
         {
-            let allocation =
-                Allocation::from_bytes_byte_aligned_immutable(byte_sym.as_byte_str(), ());
-            let allocation = tcx.mk_const_alloc(allocation);
-            ConstValue::Slice { data: allocation, meta: allocation.inner().size().bytes() }
+            let data = byte_sym.as_byte_str();
+            let len = data.len();
+            let allocation = tcx.allocate_bytes_dedup(data, CTFE_ALLOC_SALT);
+            ConstValue::Slice { alloc_id: allocation, meta: len.try_into().unwrap() }
         }
         (ast::LitKind::Byte(n), ty::Uint(ty::UintTy::U8)) => {
             ConstValue::Scalar(Scalar::from_uint(n, Size::from_bytes(1)))

compiler/rustc_mir_build/src/builder/expr/as_rvalue.rs

@@ -1,6 +1,6 @@
 //! See docs in `build/expr/mod.rs`.
 
-use rustc_abi::{BackendRepr, FieldIdx, Primitive};
+use rustc_abi::FieldIdx;
 use rustc_hir::lang_items::LangItem;
 use rustc_index::{Idx, IndexVec};
 use rustc_middle::bug;
@@ -9,7 +9,6 @@ use rustc_middle::mir::interpret::Scalar;
 use rustc_middle::mir::*;
 use rustc_middle::thir::*;
 use rustc_middle::ty::cast::{CastTy, mir_cast_kind};
-use rustc_middle::ty::layout::IntegerExt;
 use rustc_middle::ty::util::IntTypeExt;
 use rustc_middle::ty::{self, Ty, UpvarArgs};
 use rustc_span::source_map::Spanned;
@@ -200,8 +199,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 {
                     let discr_ty = adt_def.repr().discr_type().to_ty(this.tcx);
                     let temp = unpack!(block = this.as_temp(block, scope, source, Mutability::Not));
-                    let layout =
-                        this.tcx.layout_of(this.typing_env().as_query_input(source_expr.ty));
                     let discr = this.temp(discr_ty, source_expr.span);
                     this.cfg.push_assign(
                         block,
@@ -209,80 +206,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                         discr,
                         Rvalue::Discriminant(temp.into()),
                     );
-                    let (op, ty) = (Operand::Move(discr), discr_ty);
-
-                    if let BackendRepr::Scalar(scalar) = layout.unwrap().backend_repr
-                        && !scalar.is_always_valid(&this.tcx)
-                        && let Primitive::Int(int_width, _signed) = scalar.primitive()
-                    {
-                        let unsigned_ty = int_width.to_ty(this.tcx, false);
-                        let unsigned_place = this.temp(unsigned_ty, expr_span);
-                        this.cfg.push_assign(
-                            block,
-                            source_info,
-                            unsigned_place,
-                            Rvalue::Cast(CastKind::IntToInt, Operand::Copy(discr), unsigned_ty),
-                        );
-
-                        let bool_ty = this.tcx.types.bool;
-                        let range = scalar.valid_range(&this.tcx);
-                        let merge_op =
-                            if range.start <= range.end { BinOp::BitAnd } else { BinOp::BitOr };
-
-                        let mut comparer = |range: u128, bin_op: BinOp| -> Place<'tcx> {
-                            // We can use `ty::TypingEnv::fully_monomorphized()` here
-                            // as we only need it to compute the layout of a primitive.
-                            let range_val = Const::from_bits(
-                                this.tcx,
-                                range,
-                                ty::TypingEnv::fully_monomorphized(),
-                                unsigned_ty,
-                            );
-                            let lit_op = this.literal_operand(expr.span, range_val);
-                            let is_bin_op = this.temp(bool_ty, expr_span);
-                            this.cfg.push_assign(
-                                block,
-                                source_info,
-                                is_bin_op,
-                                Rvalue::BinaryOp(
-                                    bin_op,
-                                    Box::new((Operand::Copy(unsigned_place), lit_op)),
-                                ),
-                            );
-                            is_bin_op
-                        };
-                        let assert_place = if range.start == 0 {
-                            comparer(range.end, BinOp::Le)
-                        } else {
-                            let start_place = comparer(range.start, BinOp::Ge);
-                            let end_place = comparer(range.end, BinOp::Le);
-                            let merge_place = this.temp(bool_ty, expr_span);
-                            this.cfg.push_assign(
-                                block,
-                                source_info,
-                                merge_place,
-                                Rvalue::BinaryOp(
-                                    merge_op,
-                                    Box::new((
-                                        Operand::Move(start_place),
-                                        Operand::Move(end_place),
-                                    )),
-                                ),
-                            );
-                            merge_place
-                        };
-                        this.cfg.push(
-                            block,
-                            Statement::new(
-                                source_info,
-                                StatementKind::Intrinsic(Box::new(NonDivergingIntrinsic::Assume(
-                                    Operand::Move(assert_place),
-                                ))),
-                            ),
-                        );
-                    }
-
-                    (op, ty)
+                    (Operand::Move(discr), discr_ty)
                 } else {
                     let ty = source_expr.ty;
                     let source = unpack!(

compiler/rustc_mir_build/src/builder/mod.rs

@@ -1045,11 +1045,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
     }
 }
 
-fn parse_float_into_constval<'tcx>(
-    num: Symbol,
-    float_ty: ty::FloatTy,
-    neg: bool,
-) -> Option<ConstValue<'tcx>> {
+fn parse_float_into_constval(num: Symbol, float_ty: ty::FloatTy, neg: bool) -> Option<ConstValue> {
     parse_float_into_scalar(num, float_ty, neg).map(|s| ConstValue::Scalar(s.into()))
 }
 

compiler/rustc_mir_build/src/builder/scope.rs

@@ -100,7 +100,9 @@ use tracing::{debug, instrument};
 
 use super::matches::BuiltMatchTree;
 use crate::builder::{BlockAnd, BlockAndExtension, BlockFrame, Builder, CFG};
-use crate::errors::{ConstContinueBadConst, ConstContinueUnknownJumpTarget};
+use crate::errors::{
+    ConstContinueBadConst, ConstContinueNotMonomorphicConst, ConstContinueUnknownJumpTarget,
+};
 
 #[derive(Debug)]
 pub(crate) struct Scopes<'tcx> {
@@ -867,7 +869,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             span_bug!(span, "break value must be a scope")
         };
 
-        let constant = match &self.thir[value].kind {
+        let expr = &self.thir[value];
+        let constant = match &expr.kind {
             ExprKind::Adt(box AdtExpr { variant_index, fields, base, .. }) => {
                 assert!(matches!(base, AdtExprBase::None));
                 assert!(fields.is_empty());
@@ -887,7 +890,27 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     ),
                 }
             }
-            _ => self.as_constant(&self.thir[value]),
+
+            ExprKind::Literal { .. }
+            | ExprKind::NonHirLiteral { .. }
+            | ExprKind::ZstLiteral { .. }
+            | ExprKind::NamedConst { .. } => self.as_constant(&self.thir[value]),
+
+            other => {
+                use crate::errors::ConstContinueNotMonomorphicConstReason as Reason;
+
+                let span = expr.span;
+                let reason = match other {
+                    ExprKind::ConstParam { .. } => Reason::ConstantParameter { span },
+                    ExprKind::ConstBlock { .. } => Reason::ConstBlock { span },
+                    _ => Reason::Other { span },
+                };
+
+                self.tcx
+                    .dcx()
+                    .emit_err(ConstContinueNotMonomorphicConst { span: expr.span, reason });
+                return block.unit();
+            }
         };
 
         let break_index = self

compiler/rustc_mir_build/src/errors.rs

@@ -1213,6 +1213,38 @@ pub(crate) struct LoopMatchArmWithGuard {
     pub span: Span,
 }
 
+#[derive(Diagnostic)]
+#[diag(mir_build_const_continue_not_const)]
+#[help]
+pub(crate) struct ConstContinueNotMonomorphicConst {
+    #[primary_span]
+    pub span: Span,
+
+    #[subdiagnostic]
+    pub reason: ConstContinueNotMonomorphicConstReason,
+}
+
+#[derive(Subdiagnostic)]
+pub(crate) enum ConstContinueNotMonomorphicConstReason {
+    #[label(mir_build_const_continue_not_const_constant_parameter)]
+    ConstantParameter {
+        #[primary_span]
+        span: Span,
+    },
+
+    #[label(mir_build_const_continue_not_const_const_block)]
+    ConstBlock {
+        #[primary_span]
+        span: Span,
+    },
+
+    #[label(mir_build_const_continue_not_const_const_other)]
+    Other {
+        #[primary_span]
+        span: Span,
+    },
+}
+
 #[derive(Diagnostic)]
 #[diag(mir_build_const_continue_bad_const)]
 pub(crate) struct ConstContinueBadConst {

compiler/rustc_mir_transform/src/coverage/spans.rs

@@ -1,8 +1,7 @@
 use rustc_data_structures::fx::FxHashSet;
 use rustc_middle::mir;
 use rustc_middle::ty::TyCtxt;
-use rustc_span::source_map::SourceMap;
-use rustc_span::{BytePos, DesugaringKind, ExpnKind, MacroKind, Span};
+use rustc_span::{DesugaringKind, ExpnKind, MacroKind, Span};
 use tracing::instrument;
 
 use crate::coverage::graph::{BasicCoverageBlock, CoverageGraph};
@@ -84,18 +83,8 @@ pub(super) fn extract_refined_covspans<'tcx>(
     // Discard any span that overlaps with a hole.
     discard_spans_overlapping_holes(&mut covspans, &holes);
 
-    // Discard spans that overlap in unwanted ways.
+    // Perform more refinement steps after holes have been dealt with.
     let mut covspans = remove_unwanted_overlapping_spans(covspans);
-
-    // For all empty spans, either enlarge them to be non-empty, or discard them.
-    let source_map = tcx.sess.source_map();
-    covspans.retain_mut(|covspan| {
-        let Some(span) = ensure_non_empty_span(source_map, covspan.span) else { return false };
-        covspan.span = span;
-        true
-    });
-
-    // Merge covspans that can be merged.
     covspans.dedup_by(|b, a| a.merge_if_eligible(b));
 
     code_mappings.extend(covspans.into_iter().map(|Covspan { span, bcb }| {
@@ -241,26 +230,3 @@ fn compare_spans(a: Span, b: Span) -> std::cmp::Ordering {
         // - Both have the same start and span A extends further right
         .then_with(|| Ord::cmp(&a.hi(), &b.hi()).reverse())
 }
-
-fn ensure_non_empty_span(source_map: &SourceMap, span: Span) -> Option<Span> {
-    if !span.is_empty() {
-        return Some(span);
-    }
-
-    // The span is empty, so try to enlarge it to cover an adjacent '{' or '}'.
-    source_map
-        .span_to_source(span, |src, start, end| try {
-            // Adjusting span endpoints by `BytePos(1)` is normally a bug,
-            // but in this case we have specifically checked that the character
-            // we're skipping over is one of two specific ASCII characters, so
-            // adjusting by exactly 1 byte is correct.
-            if src.as_bytes().get(end).copied() == Some(b'{') {
-                Some(span.with_hi(span.hi() + BytePos(1)))
-            } else if start > 0 && src.as_bytes()[start - 1] == b'}' {
-                Some(span.with_lo(span.lo() - BytePos(1)))
-            } else {
-                None
-            }
-        })
-        .ok()?
-}

compiler/rustc_mir_transform/src/gvn.rs

@@ -1542,7 +1542,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
 fn op_to_prop_const<'tcx>(
     ecx: &mut InterpCx<'tcx, DummyMachine>,
     op: &OpTy<'tcx>,
-) -> Option<ConstValue<'tcx>> {
+) -> Option<ConstValue> {
     // Do not attempt to propagate unsized locals.
     if op.layout.is_unsized() {
         return None;

compiler/rustc_monomorphize/src/collector.rs

@@ -659,10 +659,7 @@ impl<'a, 'tcx> MirUsedCollector<'a, 'tcx> {
     }
 
     /// Evaluates a *not yet monomorphized* constant.
-    fn eval_constant(
-        &mut self,
-        constant: &mir::ConstOperand<'tcx>,
-    ) -> Option<mir::ConstValue<'tcx>> {
+    fn eval_constant(&mut self, constant: &mir::ConstOperand<'tcx>) -> Option<mir::ConstValue> {
         let const_ = self.monomorphize(constant.const_);
         // Evaluate the constant. This makes const eval failure a collection-time error (rather than
         // a codegen-time error). rustc stops after collection if there was an error, so this
@@ -1355,19 +1352,15 @@ fn visit_mentioned_item<'tcx>(
 #[instrument(skip(tcx, output), level = "debug")]
 fn collect_const_value<'tcx>(
     tcx: TyCtxt<'tcx>,
-    value: mir::ConstValue<'tcx>,
+    value: mir::ConstValue,
     output: &mut MonoItems<'tcx>,
 ) {
     match value {
         mir::ConstValue::Scalar(Scalar::Ptr(ptr, _size)) => {
             collect_alloc(tcx, ptr.provenance.alloc_id(), output)
         }
-        mir::ConstValue::Indirect { alloc_id, .. } => collect_alloc(tcx, alloc_id, output),
-        mir::ConstValue::Slice { data, meta: _ } => {
-            for &prov in data.inner().provenance().ptrs().values() {
-                collect_alloc(tcx, prov.alloc_id(), output);
-            }
-        }
+        mir::ConstValue::Indirect { alloc_id, .. }
+        | mir::ConstValue::Slice { alloc_id, meta: _ } => collect_alloc(tcx, alloc_id, output),
         _ => {}
     }
 }
@@ -1582,6 +1575,15 @@ impl<'v> RootCollector<'_, 'v> {
             return;
         };
 
+        let main_instance = Instance::mono(self.tcx, main_def_id);
+        if self.tcx.should_codegen_locally(main_instance) {
+            self.output.push(create_fn_mono_item(
+                self.tcx,
+                main_instance,
+                self.tcx.def_span(main_def_id),
+            ));
+        }
+
         let Some(start_def_id) = self.tcx.lang_items().start_fn() else {
             self.tcx.dcx().emit_fatal(errors::StartNotFound);
         };

compiler/rustc_monomorphize/src/partitioning.rs

@@ -223,11 +223,7 @@ where
         // So even if its mode is LocalCopy, we need to treat it like a root.
         match mono_item.instantiation_mode(cx.tcx) {
             InstantiationMode::GloballyShared { .. } => {}
-            InstantiationMode::LocalCopy => {
-                if !cx.tcx.is_lang_item(mono_item.def_id(), LangItem::Start) {
-                    continue;
-                }
-            }
+            InstantiationMode::LocalCopy => continue,
         }
 
         let characteristic_def_id = characteristic_def_id_of_mono_item(cx.tcx, mono_item);
@@ -821,10 +817,9 @@ fn mono_item_visibility<'tcx>(
         | InstanceKind::FnPtrAddrShim(..) => return Visibility::Hidden,
     };
 
-    // The `start_fn` lang item is actually a monomorphized instance of a
-    // function in the standard library, used for the `main` function. We don't
-    // want to export it so we tag it with `Hidden` visibility but this symbol
-    // is only referenced from the actual `main` symbol which we unfortunately
+    // Both the `start_fn` lang item and `main` itself should not be exported,
+    // so we give them with `Hidden` visibility but these symbols are
+    // only referenced from the actual `main` symbol which we unfortunately
     // don't know anything about during partitioning/collection. As a result we
     // forcibly keep this symbol out of the `internalization_candidates` set.
     //
@@ -834,7 +829,7 @@ fn mono_item_visibility<'tcx>(
     //        from the `main` symbol we'll generate later.
     //
     //        This may be fixable with a new `InstanceKind` perhaps? Unsure!
-    if tcx.is_lang_item(def_id, LangItem::Start) {
+    if tcx.is_entrypoint(def_id) {
         *can_be_internalized = false;
         return Visibility::Hidden;
     }

compiler/rustc_next_trait_solver/src/solve/assembly/mod.rs

@@ -10,8 +10,8 @@ use rustc_type_ir::inherent::*;
 use rustc_type_ir::lang_items::TraitSolverLangItem;
 use rustc_type_ir::solve::SizedTraitKind;
 use rustc_type_ir::{
-    self as ty, Interner, TypeFoldable, TypeSuperVisitable, TypeVisitable, TypeVisitableExt as _,
-    TypeVisitor, TypingMode, Upcast as _, elaborate,
+    self as ty, Interner, TypeFlags, TypeFoldable, TypeSuperVisitable, TypeVisitable,
+    TypeVisitableExt as _, TypeVisitor, TypingMode, Upcast as _, elaborate,
 };
 use tracing::{debug, instrument};
 
@@ -132,6 +132,7 @@ where
         })
         .enter(|ecx| {
             Self::match_assumption(ecx, goal, assumption, |ecx| {
+                ecx.try_evaluate_added_goals()?;
                 source.set(ecx.characterize_param_env_assumption(goal.param_env, assumption)?);
                 ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
             })
@@ -1069,8 +1070,10 @@ where
             } else {
                 ControlFlow::Continue(())
             }
-        } else {
+        } else if ty.has_type_flags(TypeFlags::HAS_PLACEHOLDER | TypeFlags::HAS_RE_INFER) {
             ty.super_visit_with(self)
+        } else {
+            ControlFlow::Continue(())
         }
     }
 
@@ -1086,8 +1089,10 @@ where
             } else {
                 ControlFlow::Continue(())
             }
-        } else {
+        } else if ct.has_type_flags(TypeFlags::HAS_PLACEHOLDER | TypeFlags::HAS_RE_INFER) {
             ct.super_visit_with(self)
+        } else {
+            ControlFlow::Continue(())
         }
     }
 

compiler/rustc_parse/Cargo.toml

@@ -9,6 +9,7 @@ bitflags = "2.4.1"
 rustc-literal-escaper = "0.0.5"
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
+rustc_attr_parsing = { path = "../rustc_attr_parsing" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_feature = { path = "../rustc_feature" }

compiler/rustc_parse/src/validate_attr.rs

@@ -1,11 +1,14 @@
 //! Meta-syntax validation logic of attributes for post-expansion.
 
+use std::slice;
+
 use rustc_ast::token::Delimiter;
 use rustc_ast::tokenstream::DelimSpan;
 use rustc_ast::{
     self as ast, AttrArgs, Attribute, DelimArgs, MetaItem, MetaItemInner, MetaItemKind, NodeId,
     Path, Safety,
 };
+use rustc_attr_parsing::{AttributeParser, Late};
 use rustc_errors::{Applicability, DiagCtxtHandle, FatalError, PResult};
 use rustc_feature::{AttributeSafety, AttributeTemplate, BUILTIN_ATTRIBUTE_MAP, BuiltinAttribute};
 use rustc_session::errors::report_lit_error;
@@ -266,64 +269,7 @@ pub fn check_builtin_meta_item(
 ) {
     if !is_attr_template_compatible(&template, &meta.kind) {
         // attrs with new parsers are locally validated so excluded here
-        if matches!(
-            name,
-            sym::inline
-                | sym::export_stable
-                | sym::ffi_const
-                | sym::ffi_pure
-                | sym::rustc_std_internal_symbol
-                | sym::may_dangle
-                | sym::rustc_as_ptr
-                | sym::rustc_pub_transparent
-                | sym::rustc_const_stable_indirect
-                | sym::rustc_force_inline
-                | sym::rustc_confusables
-                | sym::rustc_skip_during_method_dispatch
-                | sym::rustc_pass_by_value
-                | sym::rustc_deny_explicit_impl
-                | sym::rustc_do_not_implement_via_object
-                | sym::rustc_coinductive
-                | sym::const_trait
-                | sym::rustc_specialization_trait
-                | sym::rustc_unsafe_specialization_marker
-                | sym::rustc_allow_incoherent_impl
-                | sym::rustc_coherence_is_core
-                | sym::marker
-                | sym::fundamental
-                | sym::rustc_paren_sugar
-                | sym::type_const
-                | sym::repr
-                // FIXME(#82232, #143834): temporarily renamed to mitigate `#[align]` nameres
-                // ambiguity
-                | sym::rustc_align
-                | sym::deprecated
-                | sym::optimize
-                | sym::pointee
-                | sym::cold
-                | sym::target_feature
-                | sym::rustc_allow_const_fn_unstable
-                | sym::macro_use
-                | sym::macro_escape
-                | sym::naked
-                | sym::no_mangle
-                | sym::non_exhaustive
-                | sym::omit_gdb_pretty_printer_section
-                | sym::path
-                | sym::ignore
-                | sym::must_use
-                | sym::track_caller
-                | sym::link_name
-                | sym::link_ordinal
-                | sym::export_name
-                | sym::rustc_macro_transparency
-                | sym::link_section
-                | sym::rustc_layout_scalar_valid_range_start
-                | sym::rustc_layout_scalar_valid_range_end
-                | sym::no_implicit_prelude
-                | sym::automatically_derived
-                | sym::coverage
-        ) {
+        if AttributeParser::<Late>::is_parsed_attribute(slice::from_ref(&name)) {
             return;
         }
         emit_malformed_attribute(psess, style, meta.span, name, template);

compiler/rustc_passes/src/check_attr.rs

@@ -1258,7 +1258,7 @@ impl<'tcx> CheckAttrVisitor<'tcx> {
             return;
         }
 
-        if self.tcx.extern_mod_stmt_cnum(hir_id.owner).is_none() {
+        if self.tcx.extern_mod_stmt_cnum(hir_id.owner.def_id).is_none() {
             self.tcx.emit_node_span_lint(
                 INVALID_DOC_ATTRIBUTES,
                 hir_id,

compiler/rustc_pattern_analysis/src/constructor.rs

@@ -950,9 +950,7 @@ impl<Cx: PatCx> Constructor<Cx> {
                 }
             }
             Never => write!(f, "!")?,
-            Wildcard | Missing | NonExhaustive | Hidden | PrivateUninhabited => {
-                write!(f, "_ : {:?}", ty)?
-            }
+            Wildcard | Missing | NonExhaustive | Hidden | PrivateUninhabited => write!(f, "_")?,
         }
         Ok(())
     }

compiler/rustc_pattern_analysis/src/lib.rs

@@ -57,6 +57,13 @@ pub trait PatCx: Sized + fmt::Debug {
 
     fn is_exhaustive_patterns_feature_on(&self) -> bool;
 
+    /// Whether to ensure the non-exhaustiveness witnesses we report for a complete set. This is
+    /// `false` by default to avoid some exponential blowup cases such as
+    /// <https://github.com/rust-lang/rust/issues/118437>.
+    fn exhaustive_witnesses(&self) -> bool {
+        false
+    }
+
     /// The number of fields for this constructor.
     fn ctor_arity(&self, ctor: &Constructor<Self>, ty: &Self::Ty) -> usize;