Merge from rustc

.git-blame-ignore-revs

@@ -4,3 +4,5 @@ a06baa56b95674fc626b3c3fd680d6a65357fe60
 95e00bfed801e264e9c4ac817004153ca0f19eb6
 # reformat with new rustfmt
 971c549ca334b7b7406e61e958efcca9c4152822
+# refactor infcx building
+283abbf0e7d20176f76006825b5c52e9a4234e4c

.mailmap

@@ -217,7 +217,7 @@ Hsiang-Cheng Yang <rick68@users.noreply.github.com>
 Ian Jackson <ijackson@chiark.greenend.org.uk> <ian.jackson@citrix.com>
 Ian Jackson <ijackson@chiark.greenend.org.uk> <ijackson+github@slimy.greenend.org.uk>
 Ian Jackson <ijackson@chiark.greenend.org.uk> <iwj@xenproject.org>
-Ibraheem Ahmed <ibrah1440@gmail.com>
+Ibraheem Ahmed <ibraheem@ibraheem.ca> <ibrah1440@gmail.com>
 Ilyong Cho <ilyoan@gmail.com>
 inquisitivecrystal <22333129+inquisitivecrystal@users.noreply.github.com>
 Irina Popa <irinagpopa@gmail.com>

.reuse/dep5

@@ -6,11 +6,6 @@ Files: *
 Copyright: The Rust Project Developers (see https://thanks.rust-lang.org)
 License: MIT or Apache-2.0
 
-Files: library/std/src/sync/mpsc/mpsc_queue.rs
-       library/std/src/sync/mpsc/spsc_queue.rs
-Copyright: 2010-2011 Dmitry Vyukov
-License: BSD-2-Clause
-
 Files: src/librustdoc/html/static/fonts/FiraSans*
 Copyright: 2014, Mozilla Foundation, 2014, Telefonica S.A.
 License: OFL-1.1

Cargo.lock

@@ -273,7 +273,7 @@ dependencies = [
 
 [[package]]
 name = "cargo"
-version = "0.67.0"
+version = "0.68.0"
 dependencies = [
  "anyhow",
  "atty",
@@ -473,9 +473,9 @@ version = "0.1.0"
 
 [[package]]
 name = "cc"
-version = "1.0.73"
+version = "1.0.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
+checksum = "76a284da2e6fe2092f2353e51713435363112dfd60030e22add80be333fb928f"
 dependencies = [
  "jobserver",
 ]
@@ -502,9 +502,9 @@ dependencies = [
 
 [[package]]
 name = "chalk-derive"
-version = "0.80.0"
+version = "0.87.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0001adf0cf12361e08b65e1898ea138f8f77d8f5177cbf29b6b3b3532252bd6"
+checksum = "d552b2fa341f5fc35c6b917b1d289d3c3a34d0b74e579390ea6192d6152a8cdb"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -514,9 +514,9 @@ dependencies = [
 
 [[package]]
 name = "chalk-engine"
-version = "0.80.0"
+version = "0.87.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c44ee96f2d67cb5193d1503f185db1abad9933a1c6e6b4169c176f90baecd393"
+checksum = "7e54ac43048cb31c470d7b3e3acd409090ef4a5abddfe02455187aebc3d6879f"
 dependencies = [
  "chalk-derive",
  "chalk-ir",
@@ -527,9 +527,9 @@ dependencies = [
 
 [[package]]
 name = "chalk-ir"
-version = "0.80.0"
+version = "0.87.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92d8a95548f23618fda86426e4304e563ec2bb7ba0216139f0748d63c107b5f1"
+checksum = "43aa55deff4e7fbdb09fa014543372f2c95a06835ac487b9ce57b5099b950838"
 dependencies = [
  "bitflags",
  "chalk-derive",
@@ -538,9 +538,9 @@ dependencies = [
 
 [[package]]
 name = "chalk-solve"
-version = "0.80.0"
+version = "0.87.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f37f492dacfafe2e21319b80827da2779932909bb392f0cc86b2bd5c07c1b4e1"
+checksum = "61213deefc36ba265ad01c4d997e18bcddf7922862a4594a47ca4575afb3dab4"
 dependencies = [
  "chalk-derive",
  "chalk-ir",

LICENSES/BSD-2-Clause.txt

@@ -1,9 +0,0 @@
-Copyright (c) <year> <owner> 
-
-Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

compiler/rustc_ast/src/ast.rs

@@ -1208,7 +1208,7 @@ impl Expr {
             ExprKind::Tup(_) => ExprPrecedence::Tup,
             ExprKind::Binary(op, ..) => ExprPrecedence::Binary(op.node),
             ExprKind::Unary(..) => ExprPrecedence::Unary,
-            ExprKind::Lit(_) => ExprPrecedence::Lit,
+            ExprKind::Lit(_) | ExprKind::IncludedBytes(..) => ExprPrecedence::Lit,
             ExprKind::Type(..) | ExprKind::Cast(..) => ExprPrecedence::Cast,
             ExprKind::Let(..) => ExprPrecedence::Let,
             ExprKind::If(..) => ExprPrecedence::If,
@@ -1446,6 +1446,12 @@ pub enum ExprKind {
     /// with an optional value to be returned.
     Yeet(Option<P<Expr>>),
 
+    /// Bytes included via `include_bytes!`
+    /// Added for optimization purposes to avoid the need to escape
+    /// large binary blobs - should always behave like [`ExprKind::Lit`]
+    /// with a `ByteStr` literal.
+    IncludedBytes(Lrc<[u8]>),
+
     /// Placeholder for an expression that wasn't syntactically well formed in some way.
     Err,
 }

compiler/rustc_ast/src/lib.rs

@@ -29,6 +29,7 @@ extern crate rustc_macros;
 extern crate tracing;
 
 pub mod util {
+    pub mod case;
     pub mod classify;
     pub mod comments;
     pub mod literal;

compiler/rustc_ast/src/mut_visit.rs

@@ -1428,7 +1428,7 @@ pub fn noop_visit_expr<T: MutVisitor>(
         }
         ExprKind::Try(expr) => vis.visit_expr(expr),
         ExprKind::TryBlock(body) => vis.visit_block(body),
-        ExprKind::Lit(_) | ExprKind::Err => {}
+        ExprKind::Lit(_) | ExprKind::IncludedBytes(..) | ExprKind::Err => {}
     }
     vis.visit_id(id);
     vis.visit_span(span);

compiler/rustc_ast/src/token.rs

@@ -5,6 +5,7 @@ pub use TokenKind::*;
 
 use crate::ast;
 use crate::ptr::P;
+use crate::util::case::Case;
 
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::sync::Lrc;
@@ -615,6 +616,15 @@ impl Token {
         self.is_non_raw_ident_where(|id| id.name == kw)
     }
 
+    /// Returns `true` if the token is a given keyword, `kw` or if `case` is `Insensitive` and this token is an identifier equal to `kw` ignoring the case.
+    pub fn is_keyword_case(&self, kw: Symbol, case: Case) -> bool {
+        self.is_keyword(kw)
+            || (case == Case::Insensitive
+                && self.is_non_raw_ident_where(|id| {
+                    id.name.as_str().to_lowercase() == kw.as_str().to_lowercase()
+                }))
+    }
+
     pub fn is_path_segment_keyword(&self) -> bool {
         self.is_non_raw_ident_where(Ident::is_path_segment_keyword)
     }

compiler/rustc_ast/src/util/case.rs

@@ -0,0 +1,6 @@
+/// Whatever to ignore case (`fn` vs `Fn` vs `FN`) or not. Used for recovering.
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
+pub enum Case {
+    Sensitive,
+    Insensitive,
+}

compiler/rustc_ast/src/util/literal.rs

@@ -2,6 +2,7 @@
 
 use crate::ast::{self, Lit, LitKind};
 use crate::token::{self, Token};
+use rustc_data_structures::sync::Lrc;
 use rustc_lexer::unescape::{byte_from_char, unescape_byte, unescape_char, unescape_literal, Mode};
 use rustc_span::symbol::{kw, sym, Symbol};
 use rustc_span::Span;
@@ -231,6 +232,13 @@ impl Lit {
         Lit { token_lit: kind.to_token_lit(), kind, span }
     }
 
+    /// Recovers an AST literal from a string of bytes produced by `include_bytes!`.
+    /// This requires ASCII-escaping the string, which can result in poor performance
+    /// for very large strings of bytes.
+    pub fn from_included_bytes(bytes: &Lrc<[u8]>, span: Span) -> Lit {
+        Self::from_lit_kind(LitKind::ByteStr(bytes.clone()), span)
+    }
+
     /// Losslessly convert an AST literal into a token.
     pub fn to_token(&self) -> Token {
         let kind = match self.token_lit.kind {

compiler/rustc_ast/src/visit.rs

@@ -251,7 +251,7 @@ pub trait Visitor<'ast>: Sized {
 macro_rules! walk_list {
     ($visitor: expr, $method: ident, $list: expr $(, $($extra_args: expr),* )?) => {
         {
-            #[cfg_attr(not(bootstrap), allow(for_loops_over_fallibles))]
+            #[allow(for_loops_over_fallibles)]
             for elem in $list {
                 $visitor.$method(elem $(, $($extra_args,)* )?)
             }
@@ -901,7 +901,7 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) {
         }
         ExprKind::Try(ref subexpression) => visitor.visit_expr(subexpression),
         ExprKind::TryBlock(ref body) => visitor.visit_block(body),
-        ExprKind::Lit(_) | ExprKind::Err => {}
+        ExprKind::Lit(_) | ExprKind::IncludedBytes(..) | ExprKind::Err => {}
     }
 
     visitor.visit_expr_post(expression)

compiler/rustc_ast_lowering/src/expr.rs

@@ -87,6 +87,10 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 ExprKind::Lit(ref l) => {
                     hir::ExprKind::Lit(respan(self.lower_span(l.span), l.kind.clone()))
                 }
+                ExprKind::IncludedBytes(ref bytes) => hir::ExprKind::Lit(respan(
+                    self.lower_span(e.span),
+                    LitKind::ByteStr(bytes.clone()),
+                )),
                 ExprKind::Cast(ref expr, ref ty) => {
                     let expr = self.lower_expr(expr);
                     let ty =

compiler/rustc_ast_lowering/src/pat.rs

@@ -323,7 +323,10 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
     // ```
     fn lower_expr_within_pat(&mut self, expr: &Expr, allow_paths: bool) -> &'hir hir::Expr<'hir> {
         match expr.kind {
-            ExprKind::Lit(..) | ExprKind::ConstBlock(..) | ExprKind::Err => {}
+            ExprKind::Lit(..)
+            | ExprKind::ConstBlock(..)
+            | ExprKind::IncludedBytes(..)
+            | ExprKind::Err => {}
             ExprKind::Path(..) if allow_paths => {}
             ExprKind::Unary(UnOp::Neg, ref inner) if matches!(inner.kind, ExprKind::Lit(_)) => {}
             _ => {

compiler/rustc_ast_pretty/src/pprust/state/expr.rs

@@ -322,6 +322,10 @@ impl<'a> State<'a> {
             ast::ExprKind::Lit(ref lit) => {
                 self.print_literal(lit);
             }
+            ast::ExprKind::IncludedBytes(ref bytes) => {
+                let lit = ast::Lit::from_included_bytes(bytes, expr.span);
+                self.print_literal(&lit)
+            }
             ast::ExprKind::Cast(ref expr, ref ty) => {
                 let prec = AssocOp::As.precedence() as i8;
                 self.print_expr_maybe_paren(expr, prec);

compiler/rustc_borrowck/src/borrow_set.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 use crate::nll::ToRegionVid;
 use crate::path_utils::allow_two_phase_borrow;
 use crate::place_ext::PlaceExt;

compiler/rustc_borrowck/src/borrowck_errors.rs

@@ -8,9 +8,18 @@ impl<'cx, 'tcx> crate::MirBorrowckCtxt<'cx, 'tcx> {
     pub(crate) fn cannot_move_when_borrowed(
         &self,
         span: Span,
-        desc: &str,
+        borrow_span: Span,
+        place: &str,
+        borrow_place: &str,
+        value_place: &str,
     ) -> DiagnosticBuilder<'cx, ErrorGuaranteed> {
-        struct_span_err!(self, span, E0505, "cannot move out of {} because it is borrowed", desc,)
+        self.infcx.tcx.sess.create_err(crate::session_diagnostics::MoveBorrow {
+            place,
+            span,
+            borrow_place,
+            value_place,
+            borrow_span,
+        })
     }
 
     pub(crate) fn cannot_use_when_mutably_borrowed(

compiler/rustc_borrowck/src/constraint_generation.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 use rustc_infer::infer::InferCtxt;
 use rustc_middle::mir::visit::TyContext;
 use rustc_middle::mir::visit::Visitor;

compiler/rustc_borrowck/src/constraints/mod.rs

@@ -1,3 +1,6 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
+
 use rustc_data_structures::graph::scc::Sccs;
 use rustc_index::vec::IndexVec;
 use rustc_middle::mir::ConstraintCategory;

compiler/rustc_borrowck/src/consumers.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 //! This file provides API for compiler consumers.
 
 use rustc_hir::def_id::LocalDefId;

compiler/rustc_borrowck/src/dataflow.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 use rustc_data_structures::fx::FxHashMap;
 use rustc_index::bit_set::BitSet;
 use rustc_middle::mir::{self, BasicBlock, Body, Location, Place};

compiler/rustc_borrowck/src/def_use.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 use rustc_middle::mir::visit::{
     MutatingUseContext, NonMutatingUseContext, NonUseContext, PlaceContext,
 };

compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs

@@ -1,3 +1,6 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
+
 use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed};
 use rustc_infer::infer::canonical::Canonical;
 use rustc_infer::infer::error_reporting::nice_region_error::NiceRegionError;

compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs

@@ -224,10 +224,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 }
             }
 
-            use_spans.var_span_label_path_only(
-                &mut err,
-                format!("{} occurs due to use{}", desired_action.as_noun(), use_spans.describe()),
-            );
+            use_spans.var_path_only_subdiag(&mut err, desired_action);
 
             if !is_loop_move {
                 err.span_label(
@@ -404,10 +401,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         let used = desired_action.as_general_verb_in_past_tense();
         let mut err =
             struct_span_err!(self, span, E0381, "{used} binding {desc}{isnt_initialized}");
-        use_spans.var_span_label_path_only(
-            &mut err,
-            format!("{} occurs due to use{}", desired_action.as_noun(), use_spans.describe()),
-        );
+        use_spans.var_path_only_subdiag(&mut err, desired_action);
 
         if let InitializationRequiringAction::PartialAssignment
         | InitializationRequiringAction::Assignment = desired_action
@@ -673,16 +667,16 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         let move_spans = self.move_spans(place.as_ref(), location);
         let span = move_spans.args_or_use();
 
-        let mut err =
-            self.cannot_move_when_borrowed(span, &self.describe_any_place(place.as_ref()));
-        err.span_label(borrow_span, format!("borrow of {} occurs here", borrow_msg));
-        err.span_label(span, format!("move out of {} occurs here", value_msg));
-
-        borrow_spans.var_span_label_path_only(
-            &mut err,
-            format!("borrow occurs due to use{}", borrow_spans.describe()),
+        let mut err = self.cannot_move_when_borrowed(
+            span,
+            borrow_span,
+            &self.describe_any_place(place.as_ref()),
+            &borrow_msg,
+            &value_msg,
         );
 
+        borrow_spans.var_path_only_subdiag(&mut err, crate::InitializationRequiringAction::Borrow);
+
         move_spans.var_span_label(
             &mut err,
             format!("move occurs due to use{}", move_spans.describe()),
@@ -724,22 +718,15 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             borrow_span,
             &self.describe_any_place(borrow.borrowed_place.as_ref()),
         );
-        borrow_spans.var_subdiag(
-            &mut err,
-            |var_span| {
-                use crate::session_diagnostics::CaptureVarCause::*;
-                let place = &borrow.borrowed_place;
-                let desc_place = self.describe_any_place(place.as_ref());
-                match borrow_spans {
-                    UseSpans::ClosureUse { generator_kind, .. } => match generator_kind {
-                        Some(_) => BorrowUsePlaceGenerator { place: desc_place, var_span },
-                        None => BorrowUsePlaceClosure { place: desc_place, var_span },
-                    },
-                    _ => BorrowUsePlace { place: desc_place, var_span },
-                }
-            },
-            "mutable",
-        );
+        borrow_spans.var_subdiag(&mut err, Some(borrow.kind), |kind, var_span| {
+            use crate::session_diagnostics::CaptureVarCause::*;
+            let place = &borrow.borrowed_place;
+            let desc_place = self.describe_any_place(place.as_ref());
+            match kind {
+                Some(_) => BorrowUsePlaceGenerator { place: desc_place, var_span },
+                None => BorrowUsePlaceClosure { place: desc_place, var_span },
+            }
+        });
 
         self.explain_why_borrow_contains_point(location, borrow, None)
             .add_explanation_to_diagnostic(

compiler/rustc_borrowck/src/diagnostics/find_all_local_uses.rs

@@ -1,3 +1,6 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
+
 use std::collections::BTreeSet;
 
 use rustc_middle::mir::visit::{PlaceContext, Visitor};

compiler/rustc_borrowck/src/diagnostics/find_use.rs

@@ -1,3 +1,6 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
+
 use std::collections::VecDeque;
 use std::rc::Rc;
 

compiler/rustc_borrowck/src/diagnostics/mod.rs

@@ -595,11 +595,34 @@ impl UseSpans<'_> {
         }
     }
 
-    // Add a span label to the use of the captured variable, if it exists.
-    // only adds label to the `path_span`
-    pub(super) fn var_span_label_path_only(self, err: &mut Diagnostic, message: impl Into<String>) {
-        if let UseSpans::ClosureUse { path_span, .. } = self {
-            err.span_label(path_span, message);
+    /// Add a span label to the use of the captured variable, if it exists.
+    /// only adds label to the `path_span`
+    pub(super) fn var_path_only_subdiag(
+        self,
+        err: &mut Diagnostic,
+        action: crate::InitializationRequiringAction,
+    ) {
+        use crate::session_diagnostics::CaptureVarPathUseCause::*;
+        use crate::InitializationRequiringAction::*;
+        if let UseSpans::ClosureUse { generator_kind, path_span, .. } = self {
+            match generator_kind {
+                Some(_) => {
+                    err.subdiagnostic(match action {
+                        Borrow => BorrowInGenerator { path_span },
+                        MatchOn | Use => UseInGenerator { path_span },
+                        Assignment => AssignInGenerator { path_span },
+                        PartialAssignment => AssignPartInGenerator { path_span },
+                    });
+                }
+                None => {
+                    err.subdiagnostic(match action {
+                        Borrow => BorrowInClosure { path_span },
+                        MatchOn | Use => UseInClosure { path_span },
+                        Assignment => AssignInClosure { path_span },
+                        PartialAssignment => AssignPartInClosure { path_span },
+                    });
+                }
+            }
         }
     }
 
@@ -627,19 +650,28 @@ impl UseSpans<'_> {
     pub(super) fn var_subdiag(
         self,
         err: &mut Diagnostic,
-        f: impl Fn(Span) -> crate::session_diagnostics::CaptureVarCause,
-        kind_desc: impl Into<String>,
+        kind: Option<rustc_middle::mir::BorrowKind>,
+        f: impl Fn(Option<GeneratorKind>, Span) -> crate::session_diagnostics::CaptureVarCause,
     ) {
-        if let UseSpans::ClosureUse { capture_kind_span, path_span, .. } = self {
-            if capture_kind_span == path_span {
-                err.subdiagnostic(f(capture_kind_span));
-            } else {
-                err.subdiagnostic(crate::session_diagnostics::CaptureVarKind {
-                    kind_desc: kind_desc.into(),
-                    kind_span: capture_kind_span,
+        use crate::session_diagnostics::CaptureVarKind::*;
+        if let UseSpans::ClosureUse { generator_kind, capture_kind_span, path_span, .. } = self {
+            if capture_kind_span != path_span {
+                err.subdiagnostic(match kind {
+                    Some(kd) => match kd {
+                        rustc_middle::mir::BorrowKind::Shared
+                        | rustc_middle::mir::BorrowKind::Shallow
+                        | rustc_middle::mir::BorrowKind::Unique => {
+                            Immute { kind_span: capture_kind_span }
+                        }
+
+                        rustc_middle::mir::BorrowKind::Mut { .. } => {
+                            Mut { kind_span: capture_kind_span }
+                        }
+                    },
+                    None => Move { kind_span: capture_kind_span },
                 });
-                err.subdiagnostic(f(path_span));
-            }
+            };
+            err.subdiagnostic(f(generator_kind, path_span));
         }
     }
 

compiler/rustc_borrowck/src/diagnostics/mutability_errors.rs

@@ -1,6 +1,4 @@
-use rustc_errors::{
-    Applicability, Diagnostic, DiagnosticBuilder, EmissionGuarantee, ErrorGuaranteed,
-};
+use rustc_errors::{Applicability, Diagnostic};
 use rustc_hir as hir;
 use rustc_hir::intravisit::Visitor;
 use rustc_hir::Node;
@@ -629,25 +627,20 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
         self.buffer_error(err);
     }
 
-    fn suggest_map_index_mut_alternatives(
-        &self,
-        ty: Ty<'_>,
-        err: &mut DiagnosticBuilder<'_, ErrorGuaranteed>,
-        span: Span,
-    ) {
+    fn suggest_map_index_mut_alternatives(&self, ty: Ty<'tcx>, err: &mut Diagnostic, span: Span) {
         let Some(adt) = ty.ty_adt_def() else { return };
         let did = adt.did();
         if self.infcx.tcx.is_diagnostic_item(sym::HashMap, did)
             || self.infcx.tcx.is_diagnostic_item(sym::BTreeMap, did)
         {
-            struct V<'a, 'b, 'tcx, G: EmissionGuarantee> {
+            struct V<'a, 'tcx> {
                 assign_span: Span,
-                err: &'a mut DiagnosticBuilder<'b, G>,
+                err: &'a mut Diagnostic,
                 ty: Ty<'tcx>,
                 suggested: bool,
             }
-            impl<'a, 'b: 'a, 'hir, 'tcx, G: EmissionGuarantee> Visitor<'hir> for V<'a, 'b, 'tcx, G> {
-                fn visit_stmt(&mut self, stmt: &'hir hir::Stmt<'hir>) {
+            impl<'a, 'tcx> Visitor<'tcx> for V<'a, 'tcx> {
+                fn visit_stmt(&mut self, stmt: &'tcx hir::Stmt<'tcx>) {
                     hir::intravisit::walk_stmt(self, stmt);
                     let expr = match stmt.kind {
                         hir::StmtKind::Semi(expr) | hir::StmtKind::Expr(expr) => expr,
@@ -705,7 +698,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, 'tcx> {
                                     ),
                                     (rv.span.shrink_to_hi(), ")".to_string()),
                                 ],
-                            ].into_iter(),
+                            ],
                             Applicability::MachineApplicable,
                         );
                         self.suggested = true;

compiler/rustc_borrowck/src/diagnostics/var_name.rs

@@ -1,3 +1,6 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
+
 use crate::Upvar;
 use crate::{nll::ToRegionVid, region_infer::RegionInferenceContext};
 use rustc_index::vec::{Idx, IndexVec};

compiler/rustc_borrowck/src/facts.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 use crate::location::{LocationIndex, LocationTable};
 use crate::BorrowIndex;
 use polonius_engine::AllFacts as PoloniusFacts;

compiler/rustc_borrowck/src/invalidation.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 use rustc_data_structures::graph::dominators::Dominators;
 use rustc_middle::mir::visit::Visitor;
 use rustc_middle::mir::{self, BasicBlock, Body, Location, NonDivergingIntrinsic, Place, Rvalue};

compiler/rustc_borrowck/src/location.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 use rustc_index::vec::{Idx, IndexVec};
 use rustc_middle::mir::{BasicBlock, Body, Location};
 

compiler/rustc_borrowck/src/member_constraints.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_index::vec::IndexVec;

compiler/rustc_borrowck/src/nll.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 //! The entry point of the NLL borrow checker.
 
 use rustc_data_structures::vec_map::VecMap;

compiler/rustc_borrowck/src/path_utils.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 use crate::borrow_set::{BorrowData, BorrowSet, TwoPhaseActivation};
 use crate::places_conflict;
 use crate::AccessDepth;

compiler/rustc_borrowck/src/place_ext.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 use crate::borrow_set::LocalsStateAtExit;
 use rustc_hir as hir;
 use rustc_middle::mir::ProjectionElem;

compiler/rustc_borrowck/src/places_conflict.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 use crate::ArtificialField;
 use crate::Overlap;
 use crate::{AccessDepth, Deep, Shallow};

compiler/rustc_borrowck/src/prefixes.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 //! From the NLL RFC: "The deep [aka 'supporting'] prefixes for an
 //! place are formed by stripping away fields and derefs, except that
 //! we stop when we reach the deref of a shared reference. [...] "

compiler/rustc_borrowck/src/region_infer/dump_mir.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 //! As part of generating the regions, if you enable `-Zdump-mir=nll`,
 //! we will generate an annotated copy of the MIR that includes the
 //! state of region inference. This code handles emitting the region

compiler/rustc_borrowck/src/region_infer/graphviz.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 //! This module provides linkage between RegionInferenceContext and
 //! `rustc_graphviz` traits, specialized to attaching borrowck analysis
 //! data to rendered labels.

compiler/rustc_borrowck/src/region_infer/reverse_sccs.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 use crate::constraints::ConstraintSccIndex;
 use crate::RegionInferenceContext;
 use itertools::Itertools;

compiler/rustc_borrowck/src/region_infer/values.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 use rustc_data_structures::fx::FxIndexSet;
 use rustc_index::bit_set::SparseBitMatrix;
 use rustc_index::interval::IntervalSet;

compiler/rustc_borrowck/src/renumber.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 use rustc_index::vec::IndexVec;
 use rustc_infer::infer::{InferCtxt, NllRegionVariableOrigin};
 use rustc_middle::mir::visit::{MutVisitor, TyContext};

compiler/rustc_borrowck/src/session_diagnostics.rs

@@ -150,21 +150,70 @@ pub(crate) enum RequireStaticErr {
 }
 
 #[derive(Subdiagnostic)]
-#[label(borrowck_capture_kind_label)]
-pub(crate) struct CaptureVarKind {
-    pub kind_desc: String,
-    #[primary_span]
-    pub kind_span: Span,
+pub(crate) enum CaptureVarPathUseCause {
+    #[label(borrowck_borrow_due_to_use_generator)]
+    BorrowInGenerator {
+        #[primary_span]
+        path_span: Span,
+    },
+    #[label(borrowck_use_due_to_use_generator)]
+    UseInGenerator {
+        #[primary_span]
+        path_span: Span,
+    },
+    #[label(borrowck_assign_due_to_use_generator)]
+    AssignInGenerator {
+        #[primary_span]
+        path_span: Span,
+    },
+    #[label(borrowck_assign_part_due_to_use_generator)]
+    AssignPartInGenerator {
+        #[primary_span]
+        path_span: Span,
+    },
+    #[label(borrowck_borrow_due_to_use_closure)]
+    BorrowInClosure {
+        #[primary_span]
+        path_span: Span,
+    },
+    #[label(borrowck_use_due_to_use_closure)]
+    UseInClosure {
+        #[primary_span]
+        path_span: Span,
+    },
+    #[label(borrowck_assign_due_to_use_closure)]
+    AssignInClosure {
+        #[primary_span]
+        path_span: Span,
+    },
+    #[label(borrowck_assign_part_due_to_use_closure)]
+    AssignPartInClosure {
+        #[primary_span]
+        path_span: Span,
+    },
+}
+
+#[derive(Subdiagnostic)]
+pub(crate) enum CaptureVarKind {
+    #[label(borrowck_capture_immute)]
+    Immute {
+        #[primary_span]
+        kind_span: Span,
+    },
+    #[label(borrowck_capture_mut)]
+    Mut {
+        #[primary_span]
+        kind_span: Span,
+    },
+    #[label(borrowck_capture_move)]
+    Move {
+        #[primary_span]
+        kind_span: Span,
+    },
 }
 
 #[derive(Subdiagnostic)]
 pub(crate) enum CaptureVarCause {
-    #[label(borrowck_var_borrow_by_use_place)]
-    BorrowUsePlace {
-        place: String,
-        #[primary_span]
-        var_span: Span,
-    },
     #[label(borrowck_var_borrow_by_use_place_in_generator)]
     BorrowUsePlaceGenerator {
         place: String,
@@ -178,3 +227,16 @@ pub(crate) enum CaptureVarCause {
         var_span: Span,
     },
 }
+
+#[derive(Diagnostic)]
+#[diag(borrowck_cannot_move_when_borrowed, code = "E0505")]
+pub(crate) struct MoveBorrow<'a> {
+    pub place: &'a str,
+    pub borrow_place: &'a str,
+    pub value_place: &'a str,
+    #[primary_span]
+    #[label(move_label)]
+    pub span: Span,
+    #[label]
+    pub borrow_span: Span,
+}

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 //! This pass type-checks the MIR to ensure it is not broken.
 
 use std::rc::Rc;

compiler/rustc_borrowck/src/used_muts.rs

@@ -1,3 +1,5 @@
+#![deny(rustc::untranslatable_diagnostic)]
+#![deny(rustc::diagnostic_outside_of_impl)]
 use rustc_data_structures::fx::FxHashSet;
 use rustc_middle::mir::visit::{PlaceContext, Visitor};
 use rustc_middle::mir::{

compiler/rustc_builtin_macros/src/assert/context.rs

@@ -303,6 +303,7 @@ impl<'cx, 'a> Context<'cx, 'a> {
             | ExprKind::Field(_, _)
             | ExprKind::ForLoop(_, _, _, _)
             | ExprKind::If(_, _, _)
+            | ExprKind::IncludedBytes(..)
             | ExprKind::InlineAsm(_)
             | ExprKind::Let(_, _, _)
             | ExprKind::Lit(_)

compiler/rustc_builtin_macros/src/cfg_accessible.rs

@@ -34,6 +34,7 @@ impl MultiItemModifier for Expander {
         span: Span,
         meta_item: &ast::MetaItem,
         item: Annotatable,
+        _is_derive_const: bool,
     ) -> ExpandResult<Vec<Annotatable>, Annotatable> {
         let template = AttributeTemplate { list: Some("path"), ..Default::default() };
         let attr = &ecx.attribute(meta_item.clone());

compiler/rustc_builtin_macros/src/concat.rs

@@ -43,6 +43,9 @@ pub fn expand_concat(
                     has_errors = true;
                 }
             },
+            ast::ExprKind::IncludedBytes(..) => {
+                cx.span_err(e.span, "cannot concatenate a byte string literal")
+            }
             ast::ExprKind::Err => {
                 has_errors = true;
             }

compiler/rustc_builtin_macros/src/concat_bytes.rs

@@ -108,6 +108,16 @@ fn handle_array_element(
                 None
             }
         },
+        ast::ExprKind::IncludedBytes(..) => {
+            if !*has_errors {
+                cx.struct_span_err(expr.span, "cannot concatenate doubly nested array")
+                    .note("byte strings are treated as arrays of bytes")
+                    .help("try flattening the array")
+                    .emit();
+            }
+            *has_errors = true;
+            None
+        }
         _ => {
             missing_literals.push(expr.span);
             None
@@ -167,6 +177,9 @@ pub fn expand_concat_bytes(
                     has_errors = true;
                 }
             },
+            ast::ExprKind::IncludedBytes(ref bytes) => {
+                accumulator.extend_from_slice(bytes);
+            }
             ast::ExprKind::Err => {
                 has_errors = true;
             }

compiler/rustc_builtin_macros/src/derive.rs

@@ -10,7 +10,7 @@ use rustc_session::Session;
 use rustc_span::symbol::{sym, Ident};
 use rustc_span::Span;
 
-pub(crate) struct Expander;
+pub(crate) struct Expander(pub bool);
 
 impl MultiItemModifier for Expander {
     fn expand(
@@ -19,6 +19,7 @@ impl MultiItemModifier for Expander {
         span: Span,
         meta_item: &ast::MetaItem,
         item: Annotatable,
+        _: bool,
     ) -> ExpandResult<Vec<Annotatable>, Annotatable> {
         let sess = ecx.sess;
         if report_bad_target(sess, &item, span) {
@@ -58,20 +59,20 @@ impl MultiItemModifier for Expander {
                         report_path_args(sess, &meta);
                         meta.path
                     })
-                    .map(|path| (path, dummy_annotatable(), None))
+                    .map(|path| (path, dummy_annotatable(), None, self.0))
                     .collect();
 
                 // Do not configure or clone items unless necessary.
                 match &mut resolutions[..] {
                     [] => {}
-                    [(_, first_item, _), others @ ..] => {
+                    [(_, first_item, ..), others @ ..] => {
                         *first_item = cfg_eval(
                             sess,
                             features,
                             item.clone(),
                             ecx.current_expansion.lint_node_id,
                         );
-                        for (_, item, _) in others {
+                        for (_, item, _, _) in others {
                             *item = first_item.clone();
                         }
                     }

compiler/rustc_builtin_macros/src/deriving/bounds.rs

@@ -12,6 +12,7 @@ pub fn expand_deriving_copy(
     mitem: &MetaItem,
     item: &Annotatable,
     push: &mut dyn FnMut(Annotatable),
+    is_const: bool,
 ) {
     let trait_def = TraitDef {
         span,
@@ -22,6 +23,7 @@ pub fn expand_deriving_copy(
         supports_unions: true,
         methods: Vec::new(),
         associated_types: Vec::new(),
+        is_const,
     };
 
     trait_def.expand(cx, mitem, item, push);

compiler/rustc_builtin_macros/src/deriving/clone.rs

@@ -14,6 +14,7 @@ pub fn expand_deriving_clone(
     mitem: &MetaItem,
     item: &Annotatable,
     push: &mut dyn FnMut(Annotatable),
+    is_const: bool,
 ) {
     // The simple form is `fn clone(&self) -> Self { *self }`, possibly with
     // some additional `AssertParamIsClone` assertions.
@@ -87,6 +88,7 @@ pub fn expand_deriving_clone(
             combine_substructure: substructure,
         }],
         associated_types: Vec::new(),
+        is_const,
     };
 
     trait_def.expand_ext(cx, mitem, item, push, is_simple)

compiler/rustc_builtin_macros/src/deriving/cmp/eq.rs

@@ -15,6 +15,7 @@ pub fn expand_deriving_eq(
     mitem: &MetaItem,
     item: &Annotatable,
     push: &mut dyn FnMut(Annotatable),
+    is_const: bool,
 ) {
     let span = cx.with_def_site_ctxt(span);
     let inline = cx.meta_word(span, sym::inline);
@@ -42,6 +43,7 @@ pub fn expand_deriving_eq(
             })),
         }],
         associated_types: Vec::new(),
+        is_const,
     };
 
     super::inject_impl_of_structural_trait(cx, span, item, path_std!(marker::StructuralEq), push);

compiler/rustc_builtin_macros/src/deriving/cmp/ord.rs

@@ -13,6 +13,7 @@ pub fn expand_deriving_ord(
     mitem: &MetaItem,
     item: &Annotatable,
     push: &mut dyn FnMut(Annotatable),
+    is_const: bool,
 ) {
     let inline = cx.meta_word(span, sym::inline);
     let attrs = thin_vec![cx.attribute(inline)];
@@ -34,6 +35,7 @@ pub fn expand_deriving_ord(
             combine_substructure: combine_substructure(Box::new(|a, b, c| cs_cmp(a, b, c))),
         }],
         associated_types: Vec::new(),
+        is_const,
     };
 
     trait_def.expand(cx, mitem, item, push)

compiler/rustc_builtin_macros/src/deriving/cmp/partial_eq.rs

@@ -14,6 +14,7 @@ pub fn expand_deriving_partial_eq(
     mitem: &MetaItem,
     item: &Annotatable,
     push: &mut dyn FnMut(Annotatable),
+    is_const: bool,
 ) {
     fn cs_eq(cx: &mut ExtCtxt<'_>, span: Span, substr: &Substructure<'_>) -> BlockOrExpr {
         let base = true;
@@ -89,6 +90,7 @@ pub fn expand_deriving_partial_eq(
         supports_unions: false,
         methods,
         associated_types: Vec::new(),
+        is_const,
     };
     trait_def.expand(cx, mitem, item, push)
 }

compiler/rustc_builtin_macros/src/deriving/cmp/partial_ord.rs

@@ -13,6 +13,7 @@ pub fn expand_deriving_partial_ord(
     mitem: &MetaItem,
     item: &Annotatable,
     push: &mut dyn FnMut(Annotatable),
+    is_const: bool,
 ) {
     let ordering_ty = Path(path_std!(cmp::Ordering));
     let ret_ty =
@@ -43,6 +44,7 @@ pub fn expand_deriving_partial_ord(
         supports_unions: false,
         methods: vec![partial_cmp_def],
         associated_types: Vec::new(),
+        is_const,
     };
     trait_def.expand(cx, mitem, item, push)
 }

compiler/rustc_builtin_macros/src/deriving/debug.rs

@@ -13,6 +13,7 @@ pub fn expand_deriving_debug(
     mitem: &MetaItem,
     item: &Annotatable,
     push: &mut dyn FnMut(Annotatable),
+    is_const: bool,
 ) {
     // &mut ::std::fmt::Formatter
     let fmtr = Ref(Box::new(Path(path_std!(fmt::Formatter))), ast::Mutability::Mut);
@@ -37,6 +38,7 @@ pub fn expand_deriving_debug(
             })),
         }],
         associated_types: Vec::new(),
+        is_const,
     };
     trait_def.expand(cx, mitem, item, push)
 }

compiler/rustc_builtin_macros/src/deriving/decodable.rs

@@ -16,6 +16,7 @@ pub fn expand_deriving_rustc_decodable(
     mitem: &MetaItem,
     item: &Annotatable,
     push: &mut dyn FnMut(Annotatable),
+    is_const: bool,
 ) {
     let krate = sym::rustc_serialize;
     let typaram = sym::__D;
@@ -55,6 +56,7 @@ pub fn expand_deriving_rustc_decodable(
             })),
         }],
         associated_types: Vec::new(),
+        is_const,
     };
 
     trait_def.expand(cx, mitem, item, push)

compiler/rustc_builtin_macros/src/deriving/default.rs

@@ -16,6 +16,7 @@ pub fn expand_deriving_default(
     mitem: &ast::MetaItem,
     item: &Annotatable,
     push: &mut dyn FnMut(Annotatable),
+    is_const: bool,
 ) {
     item.visit_with(&mut DetectNonVariantDefaultAttr { cx });
 
@@ -47,6 +48,7 @@ pub fn expand_deriving_default(
             })),
         }],
         associated_types: Vec::new(),
+        is_const,
     };
     trait_def.expand(cx, mitem, item, push)
 }

compiler/rustc_builtin_macros/src/deriving/encodable.rs

@@ -100,6 +100,7 @@ pub fn expand_deriving_rustc_encodable(
     mitem: &MetaItem,
     item: &Annotatable,
     push: &mut dyn FnMut(Annotatable),
+    is_const: bool,
 ) {
     let krate = sym::rustc_serialize;
     let typaram = sym::__S;
@@ -139,6 +140,7 @@ pub fn expand_deriving_rustc_encodable(
             })),
         }],
         associated_types: Vec::new(),
+        is_const,
     };
 
     trait_def.expand(cx, mitem, item, push)

compiler/rustc_builtin_macros/src/deriving/generic/mod.rs

@@ -171,7 +171,7 @@ use rustc_ast::{GenericArg, GenericParamKind, VariantData};
 use rustc_attr as attr;
 use rustc_expand::base::{Annotatable, ExtCtxt};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
-use rustc_span::Span;
+use rustc_span::{Span, DUMMY_SP};
 use std::cell::RefCell;
 use std::iter;
 use std::ops::Not;
@@ -204,6 +204,8 @@ pub struct TraitDef<'a> {
     pub methods: Vec<MethodDef<'a>>,
 
     pub associated_types: Vec<(Ident, Ty)>,
+
+    pub is_const: bool,
 }
 
 pub struct MethodDef<'a> {
@@ -730,7 +732,7 @@ impl<'a> TraitDef<'a> {
                 unsafety: ast::Unsafe::No,
                 polarity: ast::ImplPolarity::Positive,
                 defaultness: ast::Defaultness::Final,
-                constness: ast::Const::No,
+                constness: if self.is_const { ast::Const::Yes(DUMMY_SP) } else { ast::Const::No },
                 generics: trait_generics,
                 of_trait: opt_trait_ref,
                 self_ty: self_type,

compiler/rustc_builtin_macros/src/deriving/hash.rs

@@ -13,6 +13,7 @@ pub fn expand_deriving_hash(
     mitem: &MetaItem,
     item: &Annotatable,
     push: &mut dyn FnMut(Annotatable),
+    is_const: bool,
 ) {
     let path = Path::new_(pathvec_std!(hash::Hash), vec![], PathKind::Std);
 
@@ -39,6 +40,7 @@ pub fn expand_deriving_hash(
             })),
         }],
         associated_types: Vec::new(),
+        is_const,
     };
 
     hash_trait_def.expand(cx, mitem, item, push);

compiler/rustc_builtin_macros/src/deriving/mod.rs

@@ -38,9 +38,10 @@ pub mod partial_ord;
 
 pub mod generic;
 
-pub(crate) struct BuiltinDerive(
-    pub(crate) fn(&mut ExtCtxt<'_>, Span, &MetaItem, &Annotatable, &mut dyn FnMut(Annotatable)),
-);
+pub(crate) type BuiltinDeriveFn =
+    fn(&mut ExtCtxt<'_>, Span, &MetaItem, &Annotatable, &mut dyn FnMut(Annotatable), bool);
+
+pub(crate) struct BuiltinDerive(pub(crate) BuiltinDeriveFn);
 
 impl MultiItemModifier for BuiltinDerive {
     fn expand(
@@ -49,6 +50,7 @@ impl MultiItemModifier for BuiltinDerive {
         span: Span,
         meta_item: &MetaItem,
         item: Annotatable,
+        is_derive_const: bool,
     ) -> ExpandResult<Vec<Annotatable>, Annotatable> {
         // FIXME: Built-in derives often forget to give spans contexts,
         // so we are doing it here in a centralized way.
@@ -57,21 +59,28 @@ impl MultiItemModifier for BuiltinDerive {
         match item {
             Annotatable::Stmt(stmt) => {
                 if let ast::StmtKind::Item(item) = stmt.into_inner().kind {
-                    (self.0)(ecx, span, meta_item, &Annotatable::Item(item), &mut |a| {
-                        // Cannot use 'ecx.stmt_item' here, because we need to pass 'ecx'
-                        // to the function
-                        items.push(Annotatable::Stmt(P(ast::Stmt {
-                            id: ast::DUMMY_NODE_ID,
-                            kind: ast::StmtKind::Item(a.expect_item()),
-                            span,
-                        })));
-                    });
+                    (self.0)(
+                        ecx,
+                        span,
+                        meta_item,
+                        &Annotatable::Item(item),
+                        &mut |a| {
+                            // Cannot use 'ecx.stmt_item' here, because we need to pass 'ecx'
+                            // to the function
+                            items.push(Annotatable::Stmt(P(ast::Stmt {
+                                id: ast::DUMMY_NODE_ID,
+                                kind: ast::StmtKind::Item(a.expect_item()),
+                                span,
+                            })));
+                        },
+                        is_derive_const,
+                    );
                 } else {
                     unreachable!("should have already errored on non-item statement")
                 }
             }
             _ => {
-                (self.0)(ecx, span, meta_item, &item, &mut |a| items.push(a));
+                (self.0)(ecx, span, meta_item, &item, &mut |a| items.push(a), is_derive_const);
             }
         }
         ExpandResult::Ready(items)

compiler/rustc_builtin_macros/src/lib.rs

@@ -99,7 +99,8 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) {
         bench: test::expand_bench,
         cfg_accessible: cfg_accessible::Expander,
         cfg_eval: cfg_eval::expand,
-        derive: derive::Expander,
+        derive: derive::Expander(false),
+        derive_const: derive::Expander(true),
         global_allocator: global_allocator::expand,
         test: test::expand_test,
         test_case: test::expand_test_case,

compiler/rustc_builtin_macros/src/source_util.rs

@@ -216,7 +216,10 @@ pub fn expand_include_bytes(
         }
     };
     match cx.source_map().load_binary_file(&file) {
-        Ok(bytes) => base::MacEager::expr(cx.expr_byte_str(sp, bytes)),
+        Ok(bytes) => {
+            let expr = cx.expr(sp, ast::ExprKind::IncludedBytes(bytes.into()));
+            base::MacEager::expr(expr)
+        }
         Err(e) => {
             cx.span_err(sp, &format!("couldn't read {}: {}", file.display(), e));
             DummyResult::any(sp)

compiler/rustc_builtin_macros/src/test.rs

@@ -112,7 +112,7 @@ pub fn expand_test_or_bench(
     };
 
     // Note: non-associated fn items are already handled by `expand_test_or_bench`
-    if !matches!(item.kind, ast::ItemKind::Fn(_)) {
+    let ast::ItemKind::Fn(fn_) = &item.kind else {
         let diag = &cx.sess.parse_sess.span_diagnostic;
         let msg = "the `#[test]` attribute may only be used on a non-associated function";
         let mut err = match item.kind {
@@ -130,7 +130,7 @@ pub fn expand_test_or_bench(
             .emit();
 
         return vec![Annotatable::Item(item)];
-    }
+    };
 
     // has_*_signature will report any errors in the type so compilation
     // will fail. We shouldn't try to expand in this case because the errors
@@ -141,12 +141,14 @@ pub fn expand_test_or_bench(
         return vec![Annotatable::Item(item)];
     }
 
-    let (sp, attr_sp) = (cx.with_def_site_ctxt(item.span), cx.with_def_site_ctxt(attr_sp));
+    let sp = cx.with_def_site_ctxt(item.span);
+    let ret_ty_sp = cx.with_def_site_ctxt(fn_.sig.decl.output.span());
+    let attr_sp = cx.with_def_site_ctxt(attr_sp);
 
     let test_id = Ident::new(sym::test, attr_sp);
 
     // creates test::$name
-    let test_path = |name| cx.path(sp, vec![test_id, Ident::from_str_and_span(name, sp)]);
+    let test_path = |name| cx.path(ret_ty_sp, vec![test_id, Ident::from_str_and_span(name, sp)]);
 
     // creates test::ShouldPanic::$name
     let should_panic_path = |name| {
@@ -192,7 +194,7 @@ pub fn expand_test_or_bench(
                         vec![
                             // super::$test_fn(b)
                             cx.expr_call(
-                                sp,
+                                ret_ty_sp,
                                 cx.expr_path(cx.path(sp, vec![item.ident])),
                                 vec![cx.expr_ident(sp, b)],
                             ),
@@ -216,7 +218,11 @@ pub fn expand_test_or_bench(
                         cx.expr_path(test_path("assert_test_result")),
                         vec![
                             // $test_fn()
-                            cx.expr_call(sp, cx.expr_path(cx.path(sp, vec![item.ident])), vec![]), // )
+                            cx.expr_call(
+                                ret_ty_sp,
+                                cx.expr_path(cx.path(sp, vec![item.ident])),
+                                vec![],
+                            ), // )
                         ],
                     ), // }
                 ), // )

compiler/rustc_codegen_cranelift/src/constant.rs

@@ -128,7 +128,7 @@ pub(crate) fn codegen_const_value<'tcx>(
     ty: Ty<'tcx>,
 ) -> CValue<'tcx> {
     let layout = fx.layout_of(ty);
-    assert!(!layout.is_unsized(), "sized const value");
+    assert!(layout.is_sized(), "unsized const value");
 
     if layout.is_zst() {
         return CValue::by_ref(crate::Pointer::dangling(layout.align.pref), layout);

compiler/rustc_codegen_cranelift/src/value_and_place.rs

@@ -19,7 +19,7 @@ fn codegen_field<'tcx>(
     };
 
     if let Some(extra) = extra {
-        if !field_layout.is_unsized() {
+        if field_layout.is_sized() {
             return simple(fx);
         }
         match field_layout.ty.kind() {
@@ -364,7 +364,7 @@ impl<'tcx> CPlace<'tcx> {
         fx: &mut FunctionCx<'_, '_, 'tcx>,
         layout: TyAndLayout<'tcx>,
     ) -> CPlace<'tcx> {
-        assert!(!layout.is_unsized());
+        assert!(layout.is_sized());
         if layout.size.bytes() == 0 {
             return CPlace {
                 inner: CPlaceInner::Addr(Pointer::dangling(layout.align.pref), None),
@@ -825,7 +825,7 @@ impl<'tcx> CPlace<'tcx> {
         fx: &FunctionCx<'_, '_, 'tcx>,
         variant: VariantIdx,
     ) -> Self {
-        assert!(!self.layout().is_unsized());
+        assert!(self.layout().is_sized());
         let layout = self.layout().for_variant(fx, variant);
         CPlace { inner: self.inner, layout }
     }

compiler/rustc_codegen_gcc/src/type_.rs

@@ -277,7 +277,7 @@ pub fn struct_fields<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, layout: TyAndLayout
         offset = target_offset + field.size;
         prev_effective_align = effective_field_align;
     }
-    if !layout.is_unsized() && field_count > 0 {
+    if layout.is_sized() && field_count > 0 {
         if offset > layout.size {
             bug!("layout: {:#?} stride: {:?} offset: {:?}", layout, layout.size, offset);
         }

compiler/rustc_codegen_llvm/src/back/write.rs

@@ -765,11 +765,21 @@ pub(crate) unsafe fn codegen(
         drop(handlers);
     }
 
+    // `.dwo` files are only emitted if:
+    //
+    // - Object files are being emitted (i.e. bitcode only or metadata only compilations will not
+    //   produce dwarf objects, even if otherwise enabled)
+    // - Target supports Split DWARF
+    // - Split debuginfo is enabled
+    // - Split DWARF kind is `split` (i.e. debuginfo is split into `.dwo` files, not different
+    //   sections in the `.o` files).
+    let dwarf_object_emitted = matches!(config.emit_obj, EmitObj::ObjectCode(_))
+        && cgcx.target_can_use_split_dwarf
+        && cgcx.split_debuginfo != SplitDebuginfo::Off
+        && cgcx.split_dwarf_kind == SplitDwarfKind::Split;
     Ok(module.into_compiled_module(
         config.emit_obj != EmitObj::None,
-        cgcx.target_can_use_split_dwarf
-            && cgcx.split_debuginfo != SplitDebuginfo::Off
-            && cgcx.split_dwarf_kind == SplitDwarfKind::Split,
+        dwarf_object_emitted,
         config.emit_bc,
         &cgcx.output_filenames,
     ))

compiler/rustc_codegen_llvm/src/debuginfo/utils.rs

@@ -72,7 +72,7 @@ pub(crate) fn fat_pointer_kind<'ll, 'tcx>(
         layout.is_unsized()
     );
 
-    if !layout.is_unsized() {
+    if layout.is_sized() {
         return None;
     }
 

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -340,17 +340,26 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
 
             sym::black_box => {
                 args[0].val.store(self, result);
-
+                let result_val_span = [result.llval];
                 // We need to "use" the argument in some way LLVM can't introspect, and on
                 // targets that support it we can typically leverage inline assembly to do
                 // this. LLVM's interpretation of inline assembly is that it's, well, a black
                 // box. This isn't the greatest implementation since it probably deoptimizes
                 // more than we want, but it's so far good enough.
+                //
+                // For zero-sized types, the location pointed to by the result may be
+                // uninitialized. Do not "use" the result in this case; instead just clobber
+                // the memory.
+                let (constraint, inputs): (&str, &[_]) = if result.layout.is_zst() {
+                    ("~{memory}", &[])
+                } else {
+                    ("r,~{memory}", &result_val_span)
+                };
                 crate::asm::inline_asm_call(
                     self,
                     "",
-                    "r,~{memory}",
-                    &[result.llval],
+                    constraint,
+                    inputs,
                     self.type_void(),
                     true,
                     false,

compiler/rustc_codegen_llvm/src/type_of.rs

@@ -140,7 +140,7 @@ fn struct_llfields<'a, 'tcx>(
         prev_effective_align = effective_field_align;
     }
     let padding_used = result.len() > field_count;
-    if !layout.is_unsized() && field_count > 0 {
+    if layout.is_sized() && field_count > 0 {
         if offset > layout.size {
             bug!("layout: {:#?} stride: {:?} offset: {:?}", layout, layout.size, offset);
         }

compiler/rustc_codegen_ssa/src/back/archive.rs

@@ -4,8 +4,11 @@ use rustc_session::cstore::DllImport;
 use rustc_session::Session;
 use rustc_span::symbol::Symbol;
 
+use super::metadata::search_for_section;
+
 use object::read::archive::ArchiveFile;
 
+use std::error::Error;
 use std::fs::File;
 use std::io;
 use std::path::{Path, PathBuf};
@@ -56,6 +59,9 @@ pub trait ArchiveBuilderBuilder {
             if !bundled_lib_file_names.contains(&Symbol::intern(name)) {
                 continue; // We need to extract only native libraries.
             }
+            let data = search_for_section(rlib, data, ".bundled_lib").map_err(|e| {
+                ExtractBundledLibsError::ExtractSection { rlib, error: Box::<dyn Error>::from(e) }
+            })?;
             std::fs::write(&outdir.join(&name), data)
                 .map_err(|e| ExtractBundledLibsError::WriteFile { rlib, error: Box::new(e) })?;
         }

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -6,9 +6,9 @@ use rustc_data_structures::memmap::Mmap;
 use rustc_data_structures::temp_dir::MaybeTempDir;
 use rustc_errors::{ErrorGuaranteed, Handler};
 use rustc_fs_util::fix_windows_verbatim_for_gcc;
-use rustc_hir::def_id::CrateNum;
+use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
 use rustc_metadata::find_native_static_library;
-use rustc_metadata::fs::{emit_metadata, METADATA_FILENAME};
+use rustc_metadata::fs::{emit_wrapper_file, METADATA_FILENAME};
 use rustc_middle::middle::dependency_format::Linkage;
 use rustc_middle::middle::exported_symbols::SymbolExportKind;
 use rustc_session::config::{self, CFGuard, CrateType, DebugInfo, LdImpl, Lto, Strip};
@@ -24,12 +24,12 @@ use rustc_span::symbol::Symbol;
 use rustc_span::DebuggerVisualizerFile;
 use rustc_target::spec::crt_objects::{CrtObjects, LinkSelfContainedDefault};
 use rustc_target::spec::{Cc, LinkOutputKind, LinkerFlavor, LinkerFlavorCli, Lld, PanicStrategy};
-use rustc_target::spec::{RelocModel, RelroLevel, SanitizerSet, SplitDebuginfo, Target};
+use rustc_target::spec::{RelocModel, RelroLevel, SanitizerSet, SplitDebuginfo};
 
 use super::archive::{ArchiveBuilder, ArchiveBuilderBuilder};
 use super::command::Command;
 use super::linker::{self, Linker};
-use super::metadata::{create_rmeta_file, MetadataPosition};
+use super::metadata::{create_wrapper_file, MetadataPosition};
 use super::rpath::{self, RPathConfig};
 use crate::{
     errors, looks_like_rust_object_file, CodegenResults, CompiledModule, CrateInfo, NativeLib,
@@ -44,7 +44,7 @@ use std::borrow::Borrow;
 use std::cell::OnceCell;
 use std::collections::BTreeSet;
 use std::ffi::OsString;
-use std::fs::{File, OpenOptions};
+use std::fs::{read, File, OpenOptions};
 use std::io::{BufWriter, Write};
 use std::ops::Deref;
 use std::path::{Path, PathBuf};
@@ -292,8 +292,8 @@ fn link_rlib<'a>(
     let trailing_metadata = match flavor {
         RlibFlavor::Normal => {
             let (metadata, metadata_position) =
-                create_rmeta_file(sess, codegen_results.metadata.raw_data());
-            let metadata = emit_metadata(sess, &metadata, tmpdir);
+                create_wrapper_file(sess, b".rmeta".to_vec(), codegen_results.metadata.raw_data());
+            let metadata = emit_wrapper_file(sess, &metadata, tmpdir, METADATA_FILENAME);
             match metadata_position {
                 MetadataPosition::First => {
                     // Most of the time metadata in rlib files is wrapped in a "dummy" object
@@ -376,12 +376,18 @@ fn link_rlib<'a>(
             let location =
                 find_native_static_library(name.as_str(), lib.verbatim, &lib_search_paths, sess);
             if sess.opts.unstable_opts.packed_bundled_libs && flavor == RlibFlavor::Normal {
-                packed_bundled_libs.push(find_native_static_library(
-                    lib.filename.unwrap().as_str(),
+                let filename = lib.filename.unwrap();
+                let lib_path = find_native_static_library(
+                    filename.as_str(),
                     Some(true),
                     &lib_search_paths,
                     sess,
-                ));
+                );
+                let src = read(lib_path)
+                    .map_err(|e| sess.emit_fatal(errors::ReadFileError { message: e }))?;
+                let (data, _) = create_wrapper_file(sess, b".bundled_lib".to_vec(), &src);
+                let wrapper_file = emit_wrapper_file(sess, &data, tmpdir, filename.as_str());
+                packed_bundled_libs.push(wrapper_file);
                 continue;
             }
             ab.add_archive(&location, Box::new(|_| false)).unwrap_or_else(|error| {
@@ -2007,15 +2013,9 @@ fn linker_with_args<'a>(
     cmd.add_as_needed();
 
     // Local native libraries of all kinds.
-    //
-    // If `-Zlink-native-libraries=false` is set, then the assumption is that an
-    // external build system already has the native dependencies defined, and it
-    // will provide them to the linker itself.
-    if sess.opts.unstable_opts.link_native_libraries {
-        add_local_native_libraries(cmd, sess, codegen_results);
-    }
+    add_local_native_libraries(cmd, sess, archive_builder_builder, codegen_results, tmpdir);
 
-    // Upstream rust libraries and their (possibly bundled) static native libraries.
+    // Upstream rust crates and their non-dynamic native libraries.
     add_upstream_rust_crates(
         cmd,
         sess,
@@ -2026,13 +2026,7 @@ fn linker_with_args<'a>(
     );
 
     // Dynamic native libraries from upstream crates.
-    //
-    // FIXME: Merge this to `add_upstream_rust_crates` so that all native libraries are linked
-    // together with their respective upstream crates, and in their originally specified order.
-    // This may be slightly breaking due to our use of `--as-needed` and needs a crater run.
-    if sess.opts.unstable_opts.link_native_libraries {
-        add_upstream_native_libraries(cmd, sess, codegen_results);
-    }
+    add_upstream_native_libraries(cmd, sess, archive_builder_builder, codegen_results, tmpdir);
 
     // Link with the import library generated for any raw-dylib functions.
     for (raw_dylib_name, raw_dylib_imports) in
@@ -2276,42 +2270,46 @@ fn collect_natvis_visualizers(
     visualizer_paths
 }
 
-/// # Native library linking
-///
-/// User-supplied library search paths (-L on the command line). These are the same paths used to
-/// find Rust crates, so some of them may have been added already by the previous crate linking
-/// code. This only allows them to be found at compile time so it is still entirely up to outside
-/// forces to make sure that library can be found at runtime.
-///
-/// Also note that the native libraries linked here are only the ones located in the current crate.
-/// Upstream crates with native library dependencies may have their native library pulled in above.
-fn add_local_native_libraries(
+fn add_native_libs_from_crate(
     cmd: &mut dyn Linker,
     sess: &Session,
+    archive_builder_builder: &dyn ArchiveBuilderBuilder,
     codegen_results: &CodegenResults,
+    tmpdir: &Path,
+    search_paths: &OnceCell<Vec<PathBuf>>,
+    bundled_libs: &FxHashSet<Symbol>,
+    cnum: CrateNum,
+    link_static: bool,
+    link_dynamic: bool,
 ) {
-    let filesearch = sess.target_filesearch(PathKind::All);
-    for search_path in filesearch.search_paths() {
-        match search_path.kind {
-            PathKind::Framework => {
-                cmd.framework_path(&search_path.dir);
-            }
-            _ => {
-                cmd.include_path(&fix_windows_verbatim_for_gcc(&search_path.dir));
-            }
-        }
+    if !sess.opts.unstable_opts.link_native_libraries {
+        // If `-Zlink-native-libraries=false` is set, then the assumption is that an
+        // external build system already has the native dependencies defined, and it
+        // will provide them to the linker itself.
+        return;
     }
 
-    let relevant_libs =
-        codegen_results.crate_info.used_libraries.iter().filter(|l| relevant_lib(sess, l));
+    if link_static && cnum != LOCAL_CRATE && !bundled_libs.is_empty() {
+        // If rlib contains native libs as archives, unpack them to tmpdir.
+        let rlib = &codegen_results.crate_info.used_crate_source[&cnum].rlib.as_ref().unwrap().0;
+        archive_builder_builder
+            .extract_bundled_libs(rlib, tmpdir, &bundled_libs)
+            .unwrap_or_else(|e| sess.emit_fatal(e));
+    }
+
+    let native_libs = match cnum {
+        LOCAL_CRATE => &codegen_results.crate_info.used_libraries,
+        _ => &codegen_results.crate_info.native_libraries[&cnum],
+    };
 
-    let search_path = OnceCell::new();
     let mut last = (None, NativeLibKind::Unspecified, None);
-    for lib in relevant_libs {
+    for lib in native_libs {
         let Some(name) = lib.name else {
             continue;
         };
-        let name = name.as_str();
+        if !relevant_lib(sess, lib) {
+            continue;
+        }
 
         // Skip if this library is the same as the last.
         last = if (lib.name, lib.kind, lib.verbatim) == last {
@@ -2320,46 +2318,119 @@ fn add_local_native_libraries(
             (lib.name, lib.kind, lib.verbatim)
         };
 
+        let name = name.as_str();
         let verbatim = lib.verbatim.unwrap_or(false);
         match lib.kind {
+            NativeLibKind::Static { bundle, whole_archive } => {
+                if link_static {
+                    let bundle = bundle.unwrap_or(true);
+                    let whole_archive = whole_archive == Some(true)
+                        // Backward compatibility case: this can be a rlib (so `+whole-archive`
+                        // cannot be added explicitly if necessary, see the error in `fn link_rlib`)
+                        // compiled as an executable due to `--test`. Use whole-archive implicitly,
+                        // like before the introduction of native lib modifiers.
+                        || (whole_archive == None
+                            && bundle
+                            && cnum == LOCAL_CRATE
+                            && sess.opts.test);
+
+                    if bundle && cnum != LOCAL_CRATE {
+                        if let Some(filename) = lib.filename {
+                            // If rlib contains native libs as archives, they are unpacked to tmpdir.
+                            let path = tmpdir.join(filename.as_str());
+                            if whole_archive {
+                                cmd.link_whole_rlib(&path);
+                            } else {
+                                cmd.link_rlib(&path);
+                            }
+                        }
+                    } else {
+                        if whole_archive {
+                            cmd.link_whole_staticlib(
+                                name,
+                                verbatim,
+                                &search_paths.get_or_init(|| archive_search_paths(sess)),
+                            );
+                        } else {
+                            // HACK/FIXME: Fixup a circular dependency between libgcc and libc
+                            // with glibc. This logic should be moved to the libc crate.
+                            if cnum != LOCAL_CRATE
+                                && sess.target.os == "linux"
+                                && sess.target.env == "gnu"
+                                && name == "c"
+                            {
+                                cmd.link_staticlib("gcc", false);
+                            }
+                            cmd.link_staticlib(name, verbatim)
+                        }
+                    }
+                }
+            }
             NativeLibKind::Dylib { as_needed } => {
-                cmd.link_dylib(name, verbatim, as_needed.unwrap_or(true))
+                if link_dynamic {
+                    cmd.link_dylib(name, verbatim, as_needed.unwrap_or(true))
+                }
+            }
+            NativeLibKind::Unspecified => {
+                if link_dynamic {
+                    cmd.link_dylib(name, verbatim, true);
+                }
             }
-            NativeLibKind::Unspecified => cmd.link_dylib(name, verbatim, true),
             NativeLibKind::Framework { as_needed } => {
-                cmd.link_framework(name, as_needed.unwrap_or(true))
-            }
-            NativeLibKind::Static { whole_archive, bundle, .. } => {
-                if whole_archive == Some(true)
-                    // Backward compatibility case: this can be a rlib (so `+whole-archive` cannot
-                    // be added explicitly if necessary, see the error in `fn link_rlib`) compiled
-                    // as an executable due to `--test`. Use whole-archive implicitly, like before
-                    // the introduction of native lib modifiers.
-                    || (whole_archive == None && bundle != Some(false) && sess.opts.test)
-                {
-                    cmd.link_whole_staticlib(
-                        name,
-                        verbatim,
-                        &search_path.get_or_init(|| archive_search_paths(sess)),
-                    );
-                } else {
-                    cmd.link_staticlib(name, verbatim)
+                if link_dynamic {
+                    cmd.link_framework(name, as_needed.unwrap_or(true))
                 }
             }
             NativeLibKind::RawDylib => {
-                // Ignore RawDylib here, they are handled separately in linker_with_args().
+                // Handled separately in `linker_with_args`.
             }
             NativeLibKind::LinkArg => {
-                cmd.arg(name);
+                if link_static {
+                    cmd.arg(name);
+                }
             }
         }
     }
 }
 
-/// # Linking Rust crates and their non-bundled static libraries
-///
-/// Rust crates are not considered at all when creating an rlib output. All dependencies will be
-/// linked when producing the final output (instead of the intermediate rlib version).
+fn add_local_native_libraries(
+    cmd: &mut dyn Linker,
+    sess: &Session,
+    archive_builder_builder: &dyn ArchiveBuilderBuilder,
+    codegen_results: &CodegenResults,
+    tmpdir: &Path,
+) {
+    if sess.opts.unstable_opts.link_native_libraries {
+        // User-supplied library search paths (-L on the command line). These are the same paths
+        // used to find Rust crates, so some of them may have been added already by the previous
+        // crate linking code. This only allows them to be found at compile time so it is still
+        // entirely up to outside forces to make sure that library can be found at runtime.
+        for search_path in sess.target_filesearch(PathKind::All).search_paths() {
+            match search_path.kind {
+                PathKind::Framework => cmd.framework_path(&search_path.dir),
+                _ => cmd.include_path(&fix_windows_verbatim_for_gcc(&search_path.dir)),
+            }
+        }
+    }
+
+    let search_paths = OnceCell::new();
+    // All static and dynamic native library dependencies are linked to the local crate.
+    let link_static = true;
+    let link_dynamic = true;
+    add_native_libs_from_crate(
+        cmd,
+        sess,
+        archive_builder_builder,
+        codegen_results,
+        tmpdir,
+        &search_paths,
+        &Default::default(),
+        LOCAL_CRATE,
+        link_static,
+        link_dynamic,
+    );
+}
+
 fn add_upstream_rust_crates<'a>(
     cmd: &mut dyn Linker,
     sess: &'a Session,
@@ -2375,7 +2446,6 @@ fn add_upstream_rust_crates<'a>(
     // Linking to a rlib involves just passing it to the linker (the linker
     // will slurp up the object files inside), and linking to a dynamic library
     // involves just passing the right -l flag.
-
     let (_, data) = codegen_results
         .crate_info
         .dependency_formats
@@ -2383,348 +2453,236 @@ fn add_upstream_rust_crates<'a>(
         .find(|(ty, _)| *ty == crate_type)
         .expect("failed to find crate type in dependency format list");
 
-    // Invoke get_used_crates to ensure that we get a topological sorting of
-    // crates.
-    let deps = &codegen_results.crate_info.used_crates;
+    let search_paths = OnceCell::new();
+    for &cnum in &codegen_results.crate_info.used_crates {
+        // We may not pass all crates through to the linker. Some crates may appear statically in
+        // an existing dylib, meaning we'll pick up all the symbols from the dylib.
+        // We must always link crates `compiler_builtins` and `profiler_builtins` statically.
+        // Even if they were already included into a dylib
+        // (e.g. `libstd` when `-C prefer-dynamic` is used).
+        // FIXME: `dependency_formats` can report `profiler_builtins` as `NotLinked` for some
+        // reason, it shouldn't do that because `profiler_builtins` should indeed be linked.
+        let linkage = data[cnum.as_usize() - 1];
+        let link_static_crate = linkage == Linkage::Static
+            || (linkage == Linkage::IncludedFromDylib || linkage == Linkage::NotLinked)
+                && (codegen_results.crate_info.compiler_builtins == Some(cnum)
+                    || codegen_results.crate_info.profiler_runtime == Some(cnum));
 
-    let mut compiler_builtins = None;
-    let search_path = OnceCell::new();
-
-    for &cnum in deps.iter() {
-        // We may not pass all crates through to the linker. Some crates may
-        // appear statically in an existing dylib, meaning we'll pick up all the
-        // symbols from the dylib.
-        let src = &codegen_results.crate_info.used_crate_source[&cnum];
-        match data[cnum.as_usize() - 1] {
-            _ if codegen_results.crate_info.profiler_runtime == Some(cnum) => {
-                add_static_crate(
-                    cmd,
-                    sess,
-                    archive_builder_builder,
-                    codegen_results,
-                    tmpdir,
-                    cnum,
-                    &Default::default(),
-                );
-            }
-            // compiler-builtins are always placed last to ensure that they're
-            // linked correctly.
-            _ if codegen_results.crate_info.compiler_builtins == Some(cnum) => {
-                assert!(compiler_builtins.is_none());
-                compiler_builtins = Some(cnum);
-            }
-            Linkage::NotLinked | Linkage::IncludedFromDylib => {}
-            Linkage::Static => {
-                let bundled_libs = if sess.opts.unstable_opts.packed_bundled_libs {
-                    codegen_results.crate_info.native_libraries[&cnum]
+        let mut bundled_libs = Default::default();
+        match linkage {
+            Linkage::Static | Linkage::IncludedFromDylib | Linkage::NotLinked => {
+                if link_static_crate {
+                    bundled_libs = codegen_results.crate_info.native_libraries[&cnum]
                         .iter()
                         .filter_map(|lib| lib.filename)
-                        .collect::<FxHashSet<_>>()
-                } else {
-                    Default::default()
-                };
-                add_static_crate(
-                    cmd,
-                    sess,
-                    archive_builder_builder,
-                    codegen_results,
-                    tmpdir,
-                    cnum,
-                    &bundled_libs,
-                );
-
-                // Link static native libs with "-bundle" modifier only if the crate they originate from
-                // is being linked statically to the current crate.  If it's linked dynamically
-                // or is an rlib already included via some other dylib crate, the symbols from
-                // native libs will have already been included in that dylib.
-                //
-                // If `-Zlink-native-libraries=false` is set, then the assumption is that an
-                // external build system already has the native dependencies defined, and it
-                // will provide them to the linker itself.
-                if sess.opts.unstable_opts.link_native_libraries {
-                    if sess.opts.unstable_opts.packed_bundled_libs {
-                        // If rlib contains native libs as archives, unpack them to tmpdir.
-                        let rlib = &src.rlib.as_ref().unwrap().0;
-                        archive_builder_builder
-                            .extract_bundled_libs(rlib, tmpdir, &bundled_libs)
-                            .unwrap_or_else(|e| sess.emit_fatal(e));
-                    }
-
-                    let mut last = (None, NativeLibKind::Unspecified, None);
-                    for lib in &codegen_results.crate_info.native_libraries[&cnum] {
-                        let Some(name) = lib.name else {
-                            continue;
-                        };
-                        let name = name.as_str();
-                        if !relevant_lib(sess, lib) {
-                            continue;
-                        }
-
-                        // Skip if this library is the same as the last.
-                        last = if (lib.name, lib.kind, lib.verbatim) == last {
-                            continue;
-                        } else {
-                            (lib.name, lib.kind, lib.verbatim)
-                        };
-
-                        match lib.kind {
-                            NativeLibKind::Static {
-                                bundle: Some(false),
-                                whole_archive: Some(true),
-                            } => {
-                                cmd.link_whole_staticlib(
-                                    name,
-                                    lib.verbatim.unwrap_or(false),
-                                    search_path.get_or_init(|| archive_search_paths(sess)),
-                                );
-                            }
-                            NativeLibKind::Static {
-                                bundle: Some(false),
-                                whole_archive: Some(false) | None,
-                            } => {
-                                // HACK/FIXME: Fixup a circular dependency between libgcc and libc
-                                // with glibc. This logic should be moved to the libc crate.
-                                if sess.target.os == "linux"
-                                    && sess.target.env == "gnu"
-                                    && name == "c"
-                                {
-                                    cmd.link_staticlib("gcc", false);
-                                }
-                                cmd.link_staticlib(name, lib.verbatim.unwrap_or(false));
-                            }
-                            NativeLibKind::LinkArg => {
-                                cmd.arg(name);
-                            }
-                            NativeLibKind::Dylib { .. }
-                            | NativeLibKind::Framework { .. }
-                            | NativeLibKind::Unspecified
-                            | NativeLibKind::RawDylib => {}
-                            NativeLibKind::Static { bundle: Some(true) | None, whole_archive } => {
-                                if sess.opts.unstable_opts.packed_bundled_libs {
-                                    // If rlib contains native libs as archives, they are unpacked to tmpdir.
-                                    let path = tmpdir.join(lib.filename.unwrap().as_str());
-                                    if whole_archive == Some(true) {
-                                        cmd.link_whole_rlib(&path);
-                                    } else {
-                                        cmd.link_rlib(&path);
-                                    }
-                                }
-                            }
-                        }
-                    }
+                        .collect();
+                    add_static_crate(
+                        cmd,
+                        sess,
+                        archive_builder_builder,
+                        codegen_results,
+                        tmpdir,
+                        cnum,
+                        &bundled_libs,
+                    );
                 }
             }
-            Linkage::Dynamic => add_dynamic_crate(cmd, sess, &src.dylib.as_ref().unwrap().0),
+            Linkage::Dynamic => {
+                let src = &codegen_results.crate_info.used_crate_source[&cnum];
+                add_dynamic_crate(cmd, sess, &src.dylib.as_ref().unwrap().0);
+            }
         }
-    }
 
-    // compiler-builtins are always placed last to ensure that they're
-    // linked correctly.
-    // We must always link the `compiler_builtins` crate statically. Even if it
-    // was already "included" in a dylib (e.g., `libstd` when `-C prefer-dynamic`
-    // is used)
-    if let Some(cnum) = compiler_builtins {
-        add_static_crate(
+        // Static libraries are linked for a subset of linked upstream crates.
+        // 1. If the upstream crate is a directly linked rlib then we must link the native library
+        // because the rlib is just an archive.
+        // 2. If the upstream crate is a dylib or a rlib linked through dylib, then we do not link
+        // the native library because it is already linked into the dylib, and even if
+        // inline/const/generic functions from the dylib can refer to symbols from the native
+        // library, those symbols should be exported and available from the dylib anyway.
+        // 3. Libraries bundled into `(compiler,profiler)_builtins` are special, see above.
+        let link_static = link_static_crate;
+        // Dynamic libraries are not linked here, see the FIXME in `add_upstream_native_libraries`.
+        let link_dynamic = false;
+        add_native_libs_from_crate(
             cmd,
             sess,
             archive_builder_builder,
             codegen_results,
             tmpdir,
+            &search_paths,
+            &bundled_libs,
             cnum,
-            &Default::default(),
-        );
-    }
-
-    // Converts a library file-stem into a cc -l argument
-    fn unlib<'a>(target: &Target, stem: &'a str) -> &'a str {
-        if stem.starts_with("lib") && !target.is_like_windows { &stem[3..] } else { stem }
-    }
-
-    // Adds the static "rlib" versions of all crates to the command line.
-    // There's a bit of magic which happens here specifically related to LTO,
-    // namely that we remove upstream object files.
-    //
-    // When performing LTO, almost(*) all of the bytecode from the upstream
-    // libraries has already been included in our object file output. As a
-    // result we need to remove the object files in the upstream libraries so
-    // the linker doesn't try to include them twice (or whine about duplicate
-    // symbols). We must continue to include the rest of the rlib, however, as
-    // it may contain static native libraries which must be linked in.
-    //
-    // (*) Crates marked with `#![no_builtins]` don't participate in LTO and
-    // their bytecode wasn't included. The object files in those libraries must
-    // still be passed to the linker.
-    //
-    // Note, however, that if we're not doing LTO we can just pass the rlib
-    // blindly to the linker (fast) because it's fine if it's not actually
-    // included as we're at the end of the dependency chain.
-    fn add_static_crate<'a>(
-        cmd: &mut dyn Linker,
-        sess: &'a Session,
-        archive_builder_builder: &dyn ArchiveBuilderBuilder,
-        codegen_results: &CodegenResults,
-        tmpdir: &Path,
-        cnum: CrateNum,
-        bundled_lib_file_names: &FxHashSet<Symbol>,
-    ) {
-        let src = &codegen_results.crate_info.used_crate_source[&cnum];
-        let cratepath = &src.rlib.as_ref().unwrap().0;
-
-        let mut link_upstream = |path: &Path| {
-            cmd.link_rlib(&fix_windows_verbatim_for_gcc(path));
-        };
-
-        // See the comment above in `link_staticlib` and `link_rlib` for why if
-        // there's a static library that's not relevant we skip all object
-        // files.
-        let native_libs = &codegen_results.crate_info.native_libraries[&cnum];
-        let skip_native = native_libs.iter().any(|lib| {
-            matches!(lib.kind, NativeLibKind::Static { bundle: None | Some(true), .. })
-                && !relevant_lib(sess, lib)
-        });
-
-        if (!are_upstream_rust_objects_already_included(sess)
-            || ignored_for_lto(sess, &codegen_results.crate_info, cnum))
-            && !skip_native
-        {
-            link_upstream(cratepath);
-            return;
-        }
-
-        let dst = tmpdir.join(cratepath.file_name().unwrap());
-        let name = cratepath.file_name().unwrap().to_str().unwrap();
-        let name = &name[3..name.len() - 5]; // chop off lib/.rlib
-        let bundled_lib_file_names = bundled_lib_file_names.clone();
-
-        sess.prof.generic_activity_with_arg("link_altering_rlib", name).run(|| {
-            let canonical_name = name.replace('-', "_");
-            let upstream_rust_objects_already_included =
-                are_upstream_rust_objects_already_included(sess);
-            let is_builtins = sess.target.no_builtins
-                || !codegen_results.crate_info.is_no_builtins.contains(&cnum);
-
-            let mut archive = archive_builder_builder.new_archive_builder(sess);
-            if let Err(error) = archive.add_archive(
-                cratepath,
-                Box::new(move |f| {
-                    if f == METADATA_FILENAME {
-                        return true;
-                    }
-
-                    let canonical = f.replace('-', "_");
-
-                    let is_rust_object =
-                        canonical.starts_with(&canonical_name) && looks_like_rust_object_file(&f);
-
-                    // If we've been requested to skip all native object files
-                    // (those not generated by the rust compiler) then we can skip
-                    // this file. See above for why we may want to do this.
-                    let skip_because_cfg_say_so = skip_native && !is_rust_object;
-
-                    // If we're performing LTO and this is a rust-generated object
-                    // file, then we don't need the object file as it's part of the
-                    // LTO module. Note that `#![no_builtins]` is excluded from LTO,
-                    // though, so we let that object file slide.
-                    let skip_because_lto =
-                        upstream_rust_objects_already_included && is_rust_object && is_builtins;
-
-                    // We skip native libraries because:
-                    // 1. This native libraries won't be used from the generated rlib,
-                    //    so we can throw them away to avoid the copying work.
-                    // 2. We can't allow it to be a single remaining entry in archive
-                    //    as some linkers may complain on that.
-                    if bundled_lib_file_names.contains(&Symbol::intern(f)) {
-                        return true;
-                    }
-
-                    if skip_because_cfg_say_so || skip_because_lto {
-                        return true;
-                    }
-
-                    false
-                }),
-            ) {
-                sess.emit_fatal(errors::RlibArchiveBuildFailure { error });
-            }
-            if archive.build(&dst) {
-                link_upstream(&dst);
-            }
-        });
-    }
-
-    // Same thing as above, but for dynamic crates instead of static crates.
-    fn add_dynamic_crate(cmd: &mut dyn Linker, sess: &Session, cratepath: &Path) {
-        // Just need to tell the linker about where the library lives and
-        // what its name is
-        let parent = cratepath.parent();
-        if let Some(dir) = parent {
-            cmd.include_path(&fix_windows_verbatim_for_gcc(dir));
-        }
-        let filestem = cratepath.file_stem().unwrap().to_str().unwrap();
-        cmd.link_rust_dylib(
-            &unlib(&sess.target, filestem),
-            parent.unwrap_or_else(|| Path::new("")),
+            link_static,
+            link_dynamic,
         );
     }
 }
 
-/// Link in all of our upstream crates' native dependencies. Remember that all of these upstream
-/// native dependencies are all non-static dependencies. We've got two cases then:
-///
-/// 1. The upstream crate is an rlib. In this case we *must* link in the native dependency because
-/// the rlib is just an archive.
-///
-/// 2. The upstream crate is a dylib. In order to use the dylib, we have to have the dependency
-/// present on the system somewhere. Thus, we don't gain a whole lot from not linking in the
-/// dynamic dependency to this crate as well.
-///
-/// The use case for this is a little subtle. In theory the native dependencies of a crate are
-/// purely an implementation detail of the crate itself, but the problem arises with generic and
-/// inlined functions. If a generic function calls a native function, then the generic function
-/// must be instantiated in the target crate, meaning that the native symbol must also be resolved
-/// in the target crate.
 fn add_upstream_native_libraries(
     cmd: &mut dyn Linker,
     sess: &Session,
+    archive_builder_builder: &dyn ArchiveBuilderBuilder,
     codegen_results: &CodegenResults,
+    tmpdir: &Path,
 ) {
-    let mut last = (None, NativeLibKind::Unspecified, None);
+    let search_path = OnceCell::new();
     for &cnum in &codegen_results.crate_info.used_crates {
-        for lib in codegen_results.crate_info.native_libraries[&cnum].iter() {
-            let Some(name) = lib.name else {
-                continue;
-            };
-            let name = name.as_str();
-            if !relevant_lib(sess, &lib) {
-                continue;
-            }
-
-            // Skip if this library is the same as the last.
-            last = if (lib.name, lib.kind, lib.verbatim) == last {
-                continue;
-            } else {
-                (lib.name, lib.kind, lib.verbatim)
-            };
-
-            let verbatim = lib.verbatim.unwrap_or(false);
-            match lib.kind {
-                NativeLibKind::Dylib { as_needed } => {
-                    cmd.link_dylib(name, verbatim, as_needed.unwrap_or(true))
-                }
-                NativeLibKind::Unspecified => cmd.link_dylib(name, verbatim, true),
-                NativeLibKind::Framework { as_needed } => {
-                    cmd.link_framework(name, as_needed.unwrap_or(true))
-                }
-                // ignore static native libraries here as we've
-                // already included them in add_local_native_libraries and
-                // add_upstream_rust_crates
-                NativeLibKind::Static { .. } => {}
-                NativeLibKind::RawDylib | NativeLibKind::LinkArg => {}
-            }
-        }
+        // Static libraries are not linked here, they are linked in `add_upstream_rust_crates`.
+        // FIXME: Merge this function to `add_upstream_rust_crates` so that all native libraries
+        // are linked together with their respective upstream crates, and in their originally
+        // specified order. This is slightly breaking due to our use of `--as-needed` (see crater
+        // results in https://github.com/rust-lang/rust/pull/102832#issuecomment-1279772306).
+        let link_static = false;
+        // Dynamic libraries are linked for all linked upstream crates.
+        // 1. If the upstream crate is a directly linked rlib then we must link the native library
+        // because the rlib is just an archive.
+        // 2. If the upstream crate is a dylib or a rlib linked through dylib, then we have to link
+        // the native library too because inline/const/generic functions from the dylib can refer
+        // to symbols from the native library, so the native library providing those symbols should
+        // be available when linking our final binary.
+        let link_dynamic = true;
+        add_native_libs_from_crate(
+            cmd,
+            sess,
+            archive_builder_builder,
+            codegen_results,
+            tmpdir,
+            &search_path,
+            &Default::default(),
+            cnum,
+            link_static,
+            link_dynamic,
+        );
     }
 }
 
+// Adds the static "rlib" versions of all crates to the command line.
+// There's a bit of magic which happens here specifically related to LTO,
+// namely that we remove upstream object files.
+//
+// When performing LTO, almost(*) all of the bytecode from the upstream
+// libraries has already been included in our object file output. As a
+// result we need to remove the object files in the upstream libraries so
+// the linker doesn't try to include them twice (or whine about duplicate
+// symbols). We must continue to include the rest of the rlib, however, as
+// it may contain static native libraries which must be linked in.
+//
+// (*) Crates marked with `#![no_builtins]` don't participate in LTO and
+// their bytecode wasn't included. The object files in those libraries must
+// still be passed to the linker.
+//
+// Note, however, that if we're not doing LTO we can just pass the rlib
+// blindly to the linker (fast) because it's fine if it's not actually
+// included as we're at the end of the dependency chain.
+fn add_static_crate<'a>(
+    cmd: &mut dyn Linker,
+    sess: &'a Session,
+    archive_builder_builder: &dyn ArchiveBuilderBuilder,
+    codegen_results: &CodegenResults,
+    tmpdir: &Path,
+    cnum: CrateNum,
+    bundled_lib_file_names: &FxHashSet<Symbol>,
+) {
+    let src = &codegen_results.crate_info.used_crate_source[&cnum];
+    let cratepath = &src.rlib.as_ref().unwrap().0;
+
+    let mut link_upstream = |path: &Path| {
+        cmd.link_rlib(&fix_windows_verbatim_for_gcc(path));
+    };
+
+    // See the comment above in `link_staticlib` and `link_rlib` for why if
+    // there's a static library that's not relevant we skip all object
+    // files.
+    let native_libs = &codegen_results.crate_info.native_libraries[&cnum];
+    let skip_native = native_libs.iter().any(|lib| {
+        matches!(lib.kind, NativeLibKind::Static { bundle: None | Some(true), .. })
+            && !relevant_lib(sess, lib)
+    });
+
+    if (!are_upstream_rust_objects_already_included(sess)
+        || ignored_for_lto(sess, &codegen_results.crate_info, cnum))
+        && !skip_native
+    {
+        link_upstream(cratepath);
+        return;
+    }
+
+    let dst = tmpdir.join(cratepath.file_name().unwrap());
+    let name = cratepath.file_name().unwrap().to_str().unwrap();
+    let name = &name[3..name.len() - 5]; // chop off lib/.rlib
+    let bundled_lib_file_names = bundled_lib_file_names.clone();
+
+    sess.prof.generic_activity_with_arg("link_altering_rlib", name).run(|| {
+        let canonical_name = name.replace('-', "_");
+        let upstream_rust_objects_already_included =
+            are_upstream_rust_objects_already_included(sess);
+        let is_builtins =
+            sess.target.no_builtins || !codegen_results.crate_info.is_no_builtins.contains(&cnum);
+
+        let mut archive = archive_builder_builder.new_archive_builder(sess);
+        if let Err(e) = archive.add_archive(
+            cratepath,
+            Box::new(move |f| {
+                if f == METADATA_FILENAME {
+                    return true;
+                }
+
+                let canonical = f.replace('-', "_");
+
+                let is_rust_object =
+                    canonical.starts_with(&canonical_name) && looks_like_rust_object_file(&f);
+
+                // If we've been requested to skip all native object files
+                // (those not generated by the rust compiler) then we can skip
+                // this file. See above for why we may want to do this.
+                let skip_because_cfg_say_so = skip_native && !is_rust_object;
+
+                // If we're performing LTO and this is a rust-generated object
+                // file, then we don't need the object file as it's part of the
+                // LTO module. Note that `#![no_builtins]` is excluded from LTO,
+                // though, so we let that object file slide.
+                let skip_because_lto =
+                    upstream_rust_objects_already_included && is_rust_object && is_builtins;
+
+                // We skip native libraries because:
+                // 1. This native libraries won't be used from the generated rlib,
+                //    so we can throw them away to avoid the copying work.
+                // 2. We can't allow it to be a single remaining entry in archive
+                //    as some linkers may complain on that.
+                if bundled_lib_file_names.contains(&Symbol::intern(f)) {
+                    return true;
+                }
+
+                if skip_because_cfg_say_so || skip_because_lto {
+                    return true;
+                }
+
+                false
+            }),
+        ) {
+            sess.fatal(&format!("failed to build archive from rlib: {}", e));
+        }
+        if archive.build(&dst) {
+            link_upstream(&dst);
+        }
+    });
+}
+
+// Same thing as above, but for dynamic crates instead of static crates.
+fn add_dynamic_crate(cmd: &mut dyn Linker, sess: &Session, cratepath: &Path) {
+    // Just need to tell the linker about where the library lives and
+    // what its name is
+    let parent = cratepath.parent();
+    if let Some(dir) = parent {
+        cmd.include_path(&fix_windows_verbatim_for_gcc(dir));
+    }
+    let stem = cratepath.file_stem().unwrap().to_str().unwrap();
+    // Convert library file-stem into a cc -l argument.
+    let prefix = if stem.starts_with("lib") && !sess.target.is_like_windows { 3 } else { 0 };
+    cmd.link_rust_dylib(&stem[prefix..], parent.unwrap_or_else(|| Path::new("")));
+}
+
 fn relevant_lib(sess: &Session, lib: &NativeLib) -> bool {
     match lib.cfg {
         Some(ref cfg) => rustc_attr::cfg_matches(cfg, &sess.parse_sess, CRATE_NODE_ID, None),

compiler/rustc_codegen_ssa/src/back/linker.rs

@@ -1260,11 +1260,11 @@ impl<'a> Linker for WasmLd<'a> {
     }
 
     fn link_whole_staticlib(&mut self, lib: &str, _verbatim: bool, _search_path: &[PathBuf]) {
-        self.cmd.arg("-l").arg(lib);
+        self.cmd.arg("--whole-archive").arg("-l").arg(lib).arg("--no-whole-archive");
     }
 
     fn link_whole_rlib(&mut self, lib: &Path) {
-        self.cmd.arg(lib);
+        self.cmd.arg("--whole-archive").arg(lib).arg("--no-whole-archive");
     }
 
     fn gc_sections(&mut self, _keep_metadata: bool) {

compiler/rustc_codegen_ssa/src/back/metadata.rs

@@ -60,7 +60,7 @@ impl MetadataLoader for DefaultMetadataLoader {
                     let data = entry
                         .data(data)
                         .map_err(|e| format!("failed to parse rlib '{}': {}", path.display(), e))?;
-                    return search_for_metadata(path, data, ".rmeta");
+                    return search_for_section(path, data, ".rmeta");
                 }
             }
 
@@ -69,11 +69,11 @@ impl MetadataLoader for DefaultMetadataLoader {
     }
 
     fn get_dylib_metadata(&self, _target: &Target, path: &Path) -> Result<MetadataRef, String> {
-        load_metadata_with(path, |data| search_for_metadata(path, data, ".rustc"))
+        load_metadata_with(path, |data| search_for_section(path, data, ".rustc"))
     }
 }
 
-fn search_for_metadata<'a>(
+pub(super) fn search_for_section<'a>(
     path: &Path,
     bytes: &'a [u8],
     section: &str,
@@ -223,7 +223,11 @@ pub enum MetadataPosition {
 // * ELF - All other targets are similar to Windows in that there's a
 //   `SHF_EXCLUDE` flag we can set on sections in an object file to get
 //   automatically removed from the final output.
-pub fn create_rmeta_file(sess: &Session, metadata: &[u8]) -> (Vec<u8>, MetadataPosition) {
+pub fn create_wrapper_file(
+    sess: &Session,
+    section_name: Vec<u8>,
+    data: &[u8],
+) -> (Vec<u8>, MetadataPosition) {
     let Some(mut file) = create_object_file(sess) else {
         // This is used to handle all "other" targets. This includes targets
         // in two categories:
@@ -241,11 +245,11 @@ pub fn create_rmeta_file(sess: &Session, metadata: &[u8]) -> (Vec<u8>, MetadataP
         // WebAssembly and for targets not supported by the `object` crate
         // yet it means that work will need to be done in the `object` crate
         // to add a case above.
-        return (metadata.to_vec(), MetadataPosition::Last);
+        return (data.to_vec(), MetadataPosition::Last);
     };
     let section = file.add_section(
         file.segment_name(StandardSegment::Debug).to_vec(),
-        b".rmeta".to_vec(),
+        section_name,
         SectionKind::Debug,
     );
     match file.format() {
@@ -259,7 +263,7 @@ pub fn create_rmeta_file(sess: &Session, metadata: &[u8]) -> (Vec<u8>, MetadataP
         }
         _ => {}
     };
-    file.append_section_data(section, metadata, 1);
+    file.append_section_data(section, data, 1);
     (file.write().unwrap(), MetadataPosition::First)
 }
 

compiler/rustc_codegen_ssa/src/base.rs

@@ -833,20 +833,30 @@ impl CrateInfo {
         //
         // In order to get this left-to-right dependency ordering, we use the reverse
         // postorder of all crates putting the leaves at the right-most positions.
-        let used_crates = tcx
+        let mut compiler_builtins = None;
+        let mut used_crates: Vec<_> = tcx
             .postorder_cnums(())
             .iter()
             .rev()
             .copied()
-            .filter(|&cnum| !tcx.dep_kind(cnum).macros_only())
+            .filter(|&cnum| {
+                let link = !tcx.dep_kind(cnum).macros_only();
+                if link && tcx.is_compiler_builtins(cnum) {
+                    compiler_builtins = Some(cnum);
+                    return false;
+                }
+                link
+            })
             .collect();
+        // `compiler_builtins` are always placed last to ensure that they're linked correctly.
+        used_crates.extend(compiler_builtins);
 
         let mut info = CrateInfo {
             target_cpu,
             exported_symbols,
             linked_symbols,
             local_crate_name,
-            compiler_builtins: None,
+            compiler_builtins,
             profiler_runtime: None,
             is_no_builtins: Default::default(),
             native_libraries: Default::default(),
@@ -872,9 +882,6 @@ impl CrateInfo {
 
             let used_crate_source = tcx.used_crate_source(cnum);
             info.used_crate_source.insert(cnum, used_crate_source.clone());
-            if tcx.is_compiler_builtins(cnum) {
-                info.compiler_builtins = Some(cnum);
-            }
             if tcx.is_profiler_runtime(cnum) {
                 info.profiler_runtime = Some(cnum);
             }

compiler/rustc_codegen_ssa/src/errors.rs

@@ -507,6 +507,9 @@ pub enum ExtractBundledLibsError<'a> {
 
     #[diag(codegen_ssa_extract_bundled_libs_write_file)]
     WriteFile { rlib: &'a Path, error: Box<dyn std::error::Error> },
+
+    #[diag(codegen_ssa_extract_bundled_libs_write_file)]
+    ExtractSection { rlib: &'a Path, error: Box<dyn std::error::Error> },
 }
 
 #[derive(Diagnostic)]
@@ -521,3 +524,9 @@ pub enum AppleSdkRootError<'a> {
     #[diag(codegen_ssa_apple_sdk_error_sdk_path)]
     SdkPath { sdk_name: &'a str, error: Error },
 }
+
+#[derive(Diagnostic)]
+#[diag(codegen_ssa_read_file)]
+pub struct ReadFileError {
+    pub message: std::io::Error,
+}

compiler/rustc_codegen_ssa/src/glue.rs

@@ -15,7 +15,7 @@ pub fn size_and_align_of_dst<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
 ) -> (Bx::Value, Bx::Value) {
     let layout = bx.layout_of(t);
     debug!("size_and_align_of_dst(ty={}, info={:?}): layout: {:?}", t, info, layout);
-    if !layout.is_unsized() {
+    if layout.is_sized() {
         let size = bx.const_usize(layout.size.bytes());
         let align = bx.const_usize(layout.align.abi.bytes());
         return (size, align);

compiler/rustc_codegen_ssa/src/mir/place.rs

@@ -29,7 +29,7 @@ pub struct PlaceRef<'tcx, V> {
 
 impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
     pub fn new_sized(llval: V, layout: TyAndLayout<'tcx>) -> PlaceRef<'tcx, V> {
-        assert!(!layout.is_unsized());
+        assert!(layout.is_sized());
         PlaceRef { llval, llextra: None, layout, align: layout.align.abi }
     }
 
@@ -38,7 +38,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
         layout: TyAndLayout<'tcx>,
         align: Align,
     ) -> PlaceRef<'tcx, V> {
-        assert!(!layout.is_unsized());
+        assert!(layout.is_sized());
         PlaceRef { llval, llextra: None, layout, align }
     }
 
@@ -48,7 +48,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
         bx: &mut Bx,
         layout: TyAndLayout<'tcx>,
     ) -> Self {
-        assert!(!layout.is_unsized(), "tried to statically allocate unsized place");
+        assert!(layout.is_sized(), "tried to statically allocate unsized place");
         let tmp = bx.alloca(bx.cx().backend_type(layout), layout.align.abi);
         Self::new_sized(tmp, layout)
     }
@@ -145,7 +145,7 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
                 );
                 return simple();
             }
-            _ if !field.is_unsized() => return simple(),
+            _ if field.is_sized() => return simple(),
             ty::Slice(..) | ty::Str | ty::Foreign(..) => return simple(),
             ty::Adt(def, _) => {
                 if def.repr().packed() {
@@ -209,7 +209,9 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
         bx: &mut Bx,
         cast_to: Ty<'tcx>,
     ) -> V {
-        let cast_to = bx.cx().immediate_backend_type(bx.cx().layout_of(cast_to));
+        let cast_to_layout = bx.cx().layout_of(cast_to);
+        let cast_to_size = cast_to_layout.layout.size();
+        let cast_to = bx.cx().immediate_backend_type(cast_to_layout);
         if self.layout.abi.is_uninhabited() {
             return bx.cx().const_undef(cast_to);
         }
@@ -229,7 +231,8 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
 
         // Read the tag/niche-encoded discriminant from memory.
         let tag = self.project_field(bx, tag_field);
-        let tag = bx.load_operand(tag);
+        let tag_op = bx.load_operand(tag);
+        let tag_imm = tag_op.immediate();
 
         // Decode the discriminant (specifically if it's niche-encoded).
         match *tag_encoding {
@@ -242,68 +245,161 @@ impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
                     Int(_, signed) => !tag_scalar.is_bool() && signed,
                     _ => false,
                 };
-                bx.intcast(tag.immediate(), cast_to, signed)
+                bx.intcast(tag_imm, cast_to, signed)
             }
             TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => {
-                // Rebase from niche values to discriminants, and check
-                // whether the result is in range for the niche variants.
-                let niche_llty = bx.cx().immediate_backend_type(tag.layout);
-                let tag = tag.immediate();
-
-                // We first compute the "relative discriminant" (wrt `niche_variants`),
-                // that is, if `n = niche_variants.end() - niche_variants.start()`,
-                // we remap `niche_start..=niche_start + n` (which may wrap around)
-                // to (non-wrap-around) `0..=n`, to be able to check whether the
-                // discriminant corresponds to a niche variant with one comparison.
-                // We also can't go directly to the (variant index) discriminant
-                // and check that it is in the range `niche_variants`, because
-                // that might not fit in the same type, on top of needing an extra
-                // comparison (see also the comment on `let niche_discr`).
-                let relative_discr = if niche_start == 0 {
-                    // Avoid subtracting `0`, which wouldn't work for pointers.
-                    // FIXME(eddyb) check the actual primitive type here.
-                    tag
+                // Cast to an integer so we don't have to treat a pointer as a
+                // special case.
+                let (tag, tag_llty) = if tag_scalar.primitive().is_ptr() {
+                    let t = bx.type_isize();
+                    let tag = bx.ptrtoint(tag_imm, t);
+                    (tag, t)
                 } else {
-                    bx.sub(tag, bx.cx().const_uint_big(niche_llty, niche_start))
+                    (tag_imm, bx.cx().immediate_backend_type(tag_op.layout))
                 };
+
+                let tag_size = tag_scalar.size(bx.cx());
+                let max_unsigned = tag_size.unsigned_int_max();
+                let max_signed = tag_size.signed_int_max() as u128;
+                let min_signed = max_signed + 1;
                 let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32();
-                let is_niche = if relative_max == 0 {
-                    // Avoid calling `const_uint`, which wouldn't work for pointers.
-                    // Also use canonical == 0 instead of non-canonical u<= 0.
-                    // FIXME(eddyb) check the actual primitive type here.
-                    bx.icmp(IntPredicate::IntEQ, relative_discr, bx.cx().const_null(niche_llty))
+                let niche_end = niche_start.wrapping_add(relative_max as u128) & max_unsigned;
+                let range = tag_scalar.valid_range(bx.cx());
+
+                let sle = |lhs: u128, rhs: u128| -> bool {
+                    // Signed and unsigned comparisons give the same results,
+                    // except that in signed comparisons an integer with the
+                    // sign bit set is less than one with the sign bit clear.
+                    // Toggle the sign bit to do a signed comparison.
+                    (lhs ^ min_signed) <= (rhs ^ min_signed)
+                };
+
+                // We have a subrange `niche_start..=niche_end` inside `range`.
+                // If the value of the tag is inside this subrange, it's a
+                // "niche value", an increment of the discriminant. Otherwise it
+                // indicates the untagged variant.
+                // A general algorithm to extract the discriminant from the tag
+                // is:
+                // relative_tag = tag - niche_start
+                // is_niche = relative_tag <= (ule) relative_max
+                // discr = if is_niche {
+                //     cast(relative_tag) + niche_variants.start()
+                // } else {
+                //     untagged_variant
+                // }
+                // However, we will likely be able to emit simpler code.
+
+                // Find the least and greatest values in `range`, considered
+                // both as signed and unsigned.
+                let (low_unsigned, high_unsigned) = if range.start <= range.end {
+                    (range.start, range.end)
                 } else {
-                    let relative_max = bx.cx().const_uint(niche_llty, relative_max as u64);
-                    bx.icmp(IntPredicate::IntULE, relative_discr, relative_max)
+                    (0, max_unsigned)
+                };
+                let (low_signed, high_signed) = if sle(range.start, range.end) {
+                    (range.start, range.end)
+                } else {
+                    (min_signed, max_signed)
                 };
 
-                // NOTE(eddyb) this addition needs to be performed on the final
-                // type, in case the niche itself can't represent all variant
-                // indices (e.g. `u8` niche with more than `256` variants,
-                // but enough uninhabited variants so that the remaining variants
-                // fit in the niche).
-                // In other words, `niche_variants.end - niche_variants.start`
-                // is representable in the niche, but `niche_variants.end`
-                // might not be, in extreme cases.
-                let niche_discr = {
-                    let relative_discr = if relative_max == 0 {
-                        // HACK(eddyb) since we have only one niche, we know which
-                        // one it is, and we can avoid having a dynamic value here.
-                        bx.cx().const_uint(cast_to, 0)
+                let niches_ule = niche_start <= niche_end;
+                let niches_sle = sle(niche_start, niche_end);
+                let cast_smaller = cast_to_size <= tag_size;
+
+                // In the algorithm above, we can change
+                // cast(relative_tag) + niche_variants.start()
+                // into
+                // cast(tag) + (niche_variants.start() - niche_start)
+                // if either the casted type is no larger than the original
+                // type, or if the niche values are contiguous (in either the
+                // signed or unsigned sense).
+                let can_incr_after_cast = cast_smaller || niches_ule || niches_sle;
+
+                let data_for_boundary_niche = || -> Option<(IntPredicate, u128)> {
+                    if !can_incr_after_cast {
+                        None
+                    } else if niche_start == low_unsigned {
+                        Some((IntPredicate::IntULE, niche_end))
+                    } else if niche_end == high_unsigned {
+                        Some((IntPredicate::IntUGE, niche_start))
+                    } else if niche_start == low_signed {
+                        Some((IntPredicate::IntSLE, niche_end))
+                    } else if niche_end == high_signed {
+                        Some((IntPredicate::IntSGE, niche_start))
                     } else {
-                        bx.intcast(relative_discr, cast_to, false)
-                    };
-                    bx.add(
-                        relative_discr,
-                        bx.cx().const_uint(cast_to, niche_variants.start().as_u32() as u64),
-                    )
+                        None
+                    }
                 };
 
-                bx.select(
+                let (is_niche, tagged_discr, delta) = if relative_max == 0 {
+                    // Best case scenario: only one tagged variant. This will
+                    // likely become just a comparison and a jump.
+                    // The algorithm is:
+                    // is_niche = tag == niche_start
+                    // discr = if is_niche {
+                    //     niche_start
+                    // } else {
+                    //     untagged_variant
+                    // }
+                    let niche_start = bx.cx().const_uint_big(tag_llty, niche_start);
+                    let is_niche = bx.icmp(IntPredicate::IntEQ, tag, niche_start);
+                    let tagged_discr =
+                        bx.cx().const_uint(cast_to, niche_variants.start().as_u32() as u64);
+                    (is_niche, tagged_discr, 0)
+                } else if let Some((predicate, constant)) = data_for_boundary_niche() {
+                    // The niche values are either the lowest or the highest in
+                    // `range`. We can avoid the first subtraction in the
+                    // algorithm.
+                    // The algorithm is now this:
+                    // is_niche = tag <= niche_end
+                    // discr = if is_niche {
+                    //     cast(tag) + (niche_variants.start() - niche_start)
+                    // } else {
+                    //     untagged_variant
+                    // }
+                    // (the first line may instead be tag >= niche_start,
+                    // and may be a signed or unsigned comparison)
+                    let is_niche =
+                        bx.icmp(predicate, tag, bx.cx().const_uint_big(tag_llty, constant));
+                    let cast_tag = if cast_smaller {
+                        bx.intcast(tag, cast_to, false)
+                    } else if niches_ule {
+                        bx.zext(tag, cast_to)
+                    } else {
+                        bx.sext(tag, cast_to)
+                    };
+
+                    let delta = (niche_variants.start().as_u32() as u128).wrapping_sub(niche_start);
+                    (is_niche, cast_tag, delta)
+                } else {
+                    // The special cases don't apply, so we'll have to go with
+                    // the general algorithm.
+                    let relative_discr = bx.sub(tag, bx.cx().const_uint_big(tag_llty, niche_start));
+                    let cast_tag = bx.intcast(relative_discr, cast_to, false);
+                    let is_niche = bx.icmp(
+                        IntPredicate::IntULE,
+                        relative_discr,
+                        bx.cx().const_uint(tag_llty, relative_max as u64),
+                    );
+                    (is_niche, cast_tag, niche_variants.start().as_u32() as u128)
+                };
+
+                let tagged_discr = if delta == 0 {
+                    tagged_discr
+                } else {
+                    bx.add(tagged_discr, bx.cx().const_uint_big(cast_to, delta))
+                };
+
+                let discr = bx.select(
                     is_niche,
-                    niche_discr,
+                    tagged_discr,
                     bx.cx().const_uint(cast_to, untagged_variant.as_u32() as u64),
-                )
+                );
+
+                // In principle we could insert assumes on the possible range of `discr`, but
+                // currently in LLVM this seems to be a pessimization.
+
+                discr
             }
         }
     }

compiler/rustc_const_eval/src/const_eval/eval_queries.rs

@@ -46,7 +46,7 @@ fn eval_body_using_ecx<'mir, 'tcx>(
         ecx.tcx.def_kind(cid.instance.def_id())
     );
     let layout = ecx.layout_of(body.bound_return_ty().subst(tcx, cid.instance.substs))?;
-    assert!(!layout.is_unsized());
+    assert!(layout.is_sized());
     let ret = ecx.allocate(layout, MemoryKind::Stack)?;
 
     trace!(

compiler/rustc_const_eval/src/interpret/eval_context.rs

@@ -572,7 +572,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         metadata: &MemPlaceMeta<M::Provenance>,
         layout: &TyAndLayout<'tcx>,
     ) -> InterpResult<'tcx, Option<(Size, Align)>> {
-        if !layout.is_unsized() {
+        if layout.is_sized() {
             return Ok(Some((layout.size, layout.align.abi)));
         }
         match layout.ty.kind() {

compiler/rustc_const_eval/src/interpret/intrinsics.rs

@@ -713,7 +713,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         rhs: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::Provenance>,
     ) -> InterpResult<'tcx, Scalar<M::Provenance>> {
         let layout = self.layout_of(lhs.layout.ty.builtin_deref(true).unwrap().ty)?;
-        assert!(!layout.is_unsized());
+        assert!(layout.is_sized());
 
         let get_bytes = |this: &InterpCx<'mir, 'tcx, M>,
                          op: &OpTy<'tcx, <M as Machine<'mir, 'tcx>>::Provenance>,

compiler/rustc_const_eval/src/interpret/memory.rs

@@ -683,7 +683,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 // Use size and align of the type.
                 let ty = self.tcx.type_of(def_id);
                 let layout = self.tcx.layout_of(ParamEnv::empty().and(ty)).unwrap();
-                assert!(!layout.is_unsized());
+                assert!(layout.is_sized());
                 (layout.size, layout.align.abi, AllocKind::LiveData)
             }
             Some(GlobalAlloc::Memory(alloc)) => {

compiler/rustc_const_eval/src/interpret/operand.rs

@@ -280,7 +280,7 @@ impl<'tcx, Prov: Provenance> OpTy<'tcx, Prov> {
         layout: TyAndLayout<'tcx>,
         cx: &impl HasDataLayout,
     ) -> InterpResult<'tcx, Self> {
-        assert!(!layout.is_unsized());
+        assert!(layout.is_sized());
         self.offset_with_meta(offset, MemPlaceMeta::None, layout, cx)
     }
 }
@@ -376,7 +376,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
     /// Read an immediate from a place, asserting that that is possible with the given layout.
     ///
-    /// If this suceeds, the `ImmTy` is never `Uninit`.
+    /// If this succeeds, the `ImmTy` is never `Uninit`.
     #[inline(always)]
     pub fn read_immediate(
         &self,

compiler/rustc_const_eval/src/interpret/place.rs

@@ -201,7 +201,7 @@ impl<'tcx, Prov: Provenance> MPlaceTy<'tcx, Prov> {
         layout: TyAndLayout<'tcx>,
         cx: &impl HasDataLayout,
     ) -> InterpResult<'tcx, Self> {
-        assert!(!layout.is_unsized());
+        assert!(layout.is_sized());
         self.offset_with_meta(offset, MemPlaceMeta::None, layout, cx)
     }
 
@@ -316,8 +316,7 @@ where
         Ok(MPlaceTy { mplace, layout, align })
     }
 
-    /// Take an operand, representing a pointer, and dereference it to a place -- that
-    /// will always be a MemPlace.  Lives in `place.rs` because it creates a place.
+    /// Take an operand, representing a pointer, and dereference it to a place.
     #[instrument(skip(self), level = "debug")]
     pub fn deref_operand(
         &self,
@@ -331,7 +330,7 @@ where
         }
 
         let mplace = self.ref_to_mplace(&val)?;
-        self.check_mplace_access(mplace, CheckInAllocMsg::DerefTest)?;
+        self.check_mplace(mplace)?;
         Ok(mplace)
     }
 
@@ -340,7 +339,7 @@ where
         &self,
         place: &MPlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, Option<AllocRef<'_, 'tcx, M::Provenance, M::AllocExtra>>> {
-        assert!(!place.layout.is_unsized());
+        assert!(place.layout.is_sized());
         assert!(!place.meta.has_meta());
         let size = place.layout.size;
         self.get_ptr_alloc(place.ptr, size, place.align)
@@ -351,24 +350,25 @@ where
         &mut self,
         place: &MPlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx, Option<AllocRefMut<'_, 'tcx, M::Provenance, M::AllocExtra>>> {
-        assert!(!place.layout.is_unsized());
+        assert!(place.layout.is_sized());
         assert!(!place.meta.has_meta());
         let size = place.layout.size;
         self.get_ptr_alloc_mut(place.ptr, size, place.align)
     }
 
     /// Check if this mplace is dereferenceable and sufficiently aligned.
-    fn check_mplace_access(
-        &self,
-        mplace: MPlaceTy<'tcx, M::Provenance>,
-        msg: CheckInAllocMsg,
-    ) -> InterpResult<'tcx> {
+    pub fn check_mplace(&self, mplace: MPlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx> {
         let (size, align) = self
             .size_and_align_of_mplace(&mplace)?
             .unwrap_or((mplace.layout.size, mplace.layout.align.abi));
         assert!(mplace.align <= align, "dynamic alignment less strict than static one?");
         let align = M::enforce_alignment(self).then_some(align);
-        self.check_ptr_access_align(mplace.ptr, size, align.unwrap_or(Align::ONE), msg)?;
+        self.check_ptr_access_align(
+            mplace.ptr,
+            size,
+            align.unwrap_or(Align::ONE),
+            CheckInAllocMsg::DerefTest,
+        )?;
         Ok(())
     }
 
@@ -485,7 +485,7 @@ where
         src: Immediate<M::Provenance>,
         dest: &PlaceTy<'tcx, M::Provenance>,
     ) -> InterpResult<'tcx> {
-        assert!(!dest.layout.is_unsized(), "Cannot write unsized data");
+        assert!(dest.layout.is_sized(), "Cannot write unsized data");
         trace!("write_immediate: {:?} <- {:?}: {}", *dest, src, dest.layout.ty);
 
         // See if we can avoid an allocation. This is the counterpart to `read_immediate_raw`,
@@ -746,7 +746,7 @@ where
         layout: TyAndLayout<'tcx>,
         kind: MemoryKind<M::MemoryKind>,
     ) -> InterpResult<'tcx, MPlaceTy<'tcx, M::Provenance>> {
-        assert!(!layout.is_unsized());
+        assert!(layout.is_sized());
         let ptr = self.allocate_ptr(layout.size, layout.align.abi, kind)?;
         Ok(MPlaceTy::from_aligned_ptr(ptr.into(), layout))
     }

compiler/rustc_const_eval/src/interpret/step.rs

@@ -209,7 +209,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
             Repeat(ref operand, _) => {
                 let src = self.eval_operand(operand, None)?;
-                assert!(!src.layout.is_unsized());
+                assert!(src.layout.is_sized());
                 let dest = self.force_allocation(&dest)?;
                 let length = dest.len(self)?;
 

compiler/rustc_const_eval/src/interpret/traits.rs

@@ -53,7 +53,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
     ) -> InterpResult<'tcx, (Size, Align)> {
         let (ty, _trait_ref) = self.get_ptr_vtable(vtable)?;
         let layout = self.layout_of(ty)?;
-        assert!(!layout.is_unsized(), "there are no vtables for unsized types");
+        assert!(layout.is_sized(), "there are no vtables for unsized types");
         Ok((layout.size, layout.align.abi))
     }
 }

compiler/rustc_data_structures/src/intern.rs

@@ -110,11 +110,6 @@ where
     }
 }
 
-/// A helper trait so that `Interned` things can cache stable hashes reproducibly.
-pub trait InternedHashingContext {
-    fn with_def_path_and_no_spans(&mut self, f: impl FnOnce(&mut Self));
-}
-
 /// A helper type that you can wrap round your own type in order to automatically
 /// cache the stable hash on creation and not recompute it whenever the stable hash
 /// of the type is computed.
@@ -161,11 +156,15 @@ impl<T> Deref for WithStableHash<T> {
 impl<T: Hash> Hash for WithStableHash<T> {
     #[inline]
     fn hash<H: Hasher>(&self, s: &mut H) {
-        self.internee.hash(s)
+        if self.stable_hash != Fingerprint::ZERO {
+            self.stable_hash.hash(s)
+        } else {
+            self.internee.hash(s)
+        }
     }
 }
 
-impl<T: HashStable<CTX>, CTX: InternedHashingContext> HashStable<CTX> for WithStableHash<T> {
+impl<T: HashStable<CTX>, CTX> HashStable<CTX> for WithStableHash<T> {
     fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
         if self.stable_hash == Fingerprint::ZERO || cfg!(debug_assertions) {
             // No cached hash available. This can only mean that incremental is disabled.
@@ -176,7 +175,7 @@ impl<T: HashStable<CTX>, CTX: InternedHashingContext> HashStable<CTX> for WithSt
             // otherwise the hashes will differ between cached and non-cached mode.
             let stable_hash: Fingerprint = {
                 let mut hasher = StableHasher::new();
-                hcx.with_def_path_and_no_spans(|hcx| self.internee.hash_stable(hcx, &mut hasher));
+                self.internee.hash_stable(hcx, &mut hasher);
                 hasher.finish()
             };
             if cfg!(debug_assertions) && self.stable_hash != Fingerprint::ZERO {

compiler/rustc_error_codes/src/error_codes/E0706.md

@@ -56,4 +56,4 @@ You might be interested in visiting the [async book] for further information.
 [`async-trait` crate]: https://crates.io/crates/async-trait
 [async-is-hard]: https://smallcultfollowing.com/babysteps/blog/2019/10/26/async-fn-in-traits-are-hard/
 [Generic Associated Types]: https://github.com/rust-lang/rust/issues/44265
-[async book]: https://rust-lang.github.io/async-book/07_workarounds/06_async_in_traits.html
+[async book]: https://rust-lang.github.io/async-book/07_workarounds/05_async_in_traits.html

compiler/rustc_error_messages/locales/en-US/borrowck.ftl

@@ -70,3 +70,56 @@ borrowck_var_borrow_by_use_place_in_closure =
 
 borrowck_var_borrow_by_use_place =
     borrow occurs due to use of {$place}
+
+borrowck_borrow_due_to_use_generator =
+    borrow occurs due to use in generator
+
+borrowck_use_due_to_use_generator =
+    use occurs due to use in generator
+
+borrowck_assign_due_to_use_generator =
+    assign occurs due to use in generator
+
+borrowck_assign_part_due_to_use_generator =
+    assign to part occurs due to use in generator
+
+borrowck_borrow_due_to_use_closure =
+    borrow occurs due to use in closure
+
+borrowck_use_due_to_use_closure =
+    use occurs due to use in closure
+
+borrowck_assign_due_to_use_closure =
+    assign occurs due to use in closure
+
+borrowck_assign_part_due_to_use_closure =
+    assign to part occurs due to use in closure
+
+borrowck_capture_immute =
+    capture is immutable because of use here
+
+borrowck_capture_mut =
+    capture is mutable because of use here
+
+borrowck_capture_move =
+    capture is moved because of use here
+
+borrowck_var_move_by_use_place_in_generator =
+    move occurs due to use of {$place} in generator
+
+borrowck_var_move_by_use_place_in_closure =
+    move occurs due to use of {$place} in closure
+
+borrowck_cannot_move_when_borrowed =
+    cannot move out of {$place ->
+        [value] value
+        *[other] {$place}
+    } because it is borrowed
+    .label = borrow of {$borrow_place ->
+        [value] value
+        *[other] {$borrow_place}
+    } occurs here
+    .move_label = move out of {$value_place ->
+        [value] value
+        *[other] {$value_place}
+    } occurs here

compiler/rustc_error_messages/locales/en-US/codegen_ssa.ftl

@@ -182,3 +182,5 @@ codegen_ssa_extract_bundled_libs_write_file = failed to write file '{$rlib}': {$
 codegen_ssa_unsupported_arch = unsupported arch `{$arch}` for os `{$os}`
 
 codegen_ssa_apple_sdk_error_sdk_path = failed to get {$sdk_name} SDK path: {error}
+
+codegen_ssa_read_file = failed to read file: {message}

compiler/rustc_error_messages/locales/en-US/hir_analysis.ftl

@@ -150,3 +150,6 @@ hir_analysis_const_bound_for_non_const_trait =
 hir_analysis_self_in_impl_self =
     `Self` is not valid in the self type of an impl block
     .note = replace `Self` with a different type
+
+hir_analysis_op_trait_generic_params =
+    `{$method_name}` must not have any generic parameters

compiler/rustc_error_messages/locales/en-US/infer.ftl

@@ -126,10 +126,10 @@ infer_data_lifetime_flow = ...but data with one lifetime flows into the other he
 infer_declared_multiple = this type is declared with multiple lifetimes...
 infer_types_declared_different = these two types are declared with different lifetimes...
 infer_data_flows = ...but data{$label_var1_exists ->
-    [true] -> {" "}from `{$label_var1}`
+    [true] {" "}from `{$label_var1}`
     *[false] -> {""}
 } flows{$label_var2_exists ->
-    [true] -> {" "}into `{$label_var2}`
+    [true] {" "}into `{$label_var2}`
     *[false] -> {""}
 } here
 

compiler/rustc_error_messages/locales/en-US/parser.ftl

@@ -375,3 +375,12 @@ parser_async_move_order_incorrect = the order of `move` and `async` is incorrect
 
 parser_double_colon_in_bound = expected `:` followed by trait or lifetime
     .suggestion = use single colon
+
+parser_fn_ptr_with_generics = function pointer types may not have generic parameters
+    .suggestion = consider moving the lifetime {$arity ->
+        [one] parameter
+        *[other] parameters
+    } to {$for_param_list_exists ->
+        [true] the
+        *[false] a
+    } `for` parameter list

compiler/rustc_error_messages/src/lib.rs

@@ -30,7 +30,8 @@ use intl_memoizer::concurrent::IntlLangMemoizer;
 #[cfg(not(parallel_compiler))]
 use intl_memoizer::IntlLangMemoizer;
 
-pub use fluent_bundle::{FluentArgs, FluentError, FluentValue};
+pub use fluent_bundle::{self, FluentArgs, FluentError, FluentValue};
+
 pub use unic_langid::{langid, LanguageIdentifier};
 
 // Generates `DEFAULT_LOCALE_RESOURCES` static and `fluent_generated` module.

compiler/rustc_errors/src/diagnostic.rs

@@ -64,8 +64,7 @@ impl<'source> Into<FluentValue<'source>> for DiagnosticArgValue<'source> {
 
 /// Trait implemented by error types. This should not be implemented manually. Instead, use
 /// `#[derive(Subdiagnostic)]` -- see [rustc_macros::Subdiagnostic].
-#[cfg_attr(bootstrap, rustc_diagnostic_item = "AddSubdiagnostic")]
-#[cfg_attr(not(bootstrap), rustc_diagnostic_item = "AddToDiagnostic")]
+#[rustc_diagnostic_item = "AddToDiagnostic"]
 pub trait AddToDiagnostic
 where
     Self: Sized,
@@ -742,7 +741,7 @@ impl Diagnostic {
         &mut self,
         sp: Span,
         msg: impl Into<SubdiagnosticMessage>,
-        suggestions: impl Iterator<Item = String>,
+        suggestions: impl IntoIterator<Item = String>,
         applicability: Applicability,
     ) -> &mut Self {
         self.span_suggestions_with_style(
@@ -759,11 +758,11 @@ impl Diagnostic {
         &mut self,
         sp: Span,
         msg: impl Into<SubdiagnosticMessage>,
-        suggestions: impl Iterator<Item = String>,
+        suggestions: impl IntoIterator<Item = String>,
         applicability: Applicability,
         style: SuggestionStyle,
     ) -> &mut Self {
-        let mut suggestions: Vec<_> = suggestions.collect();
+        let mut suggestions: Vec<_> = suggestions.into_iter().collect();
         suggestions.sort();
 
         debug_assert!(
@@ -790,10 +789,10 @@ impl Diagnostic {
     pub fn multipart_suggestions(
         &mut self,
         msg: impl Into<SubdiagnosticMessage>,
-        suggestions: impl Iterator<Item = Vec<(Span, String)>>,
+        suggestions: impl IntoIterator<Item = Vec<(Span, String)>>,
         applicability: Applicability,
     ) -> &mut Self {
-        let suggestions: Vec<_> = suggestions.collect();
+        let suggestions: Vec<_> = suggestions.into_iter().collect();
         debug_assert!(
             !(suggestions
                 .iter()

compiler/rustc_errors/src/diagnostic_builder.rs

@@ -16,8 +16,7 @@ use std::thread::panicking;
 
 /// Trait implemented by error types. This should not be implemented manually. Instead, use
 /// `#[derive(Diagnostic)]` -- see [rustc_macros::Diagnostic].
-#[cfg_attr(bootstrap, rustc_diagnostic_item = "SessionDiagnostic")]
-#[cfg_attr(not(bootstrap), rustc_diagnostic_item = "IntoDiagnostic")]
+#[rustc_diagnostic_item = "IntoDiagnostic"]
 pub trait IntoDiagnostic<'a, T: EmissionGuarantee = ErrorGuaranteed> {
     /// Write out as a diagnostic out of `Handler`.
     #[must_use]
@@ -599,13 +598,13 @@ impl<'a, G: EmissionGuarantee> DiagnosticBuilder<'a, G> {
         &mut self,
         sp: Span,
         msg: impl Into<SubdiagnosticMessage>,
-        suggestions: impl Iterator<Item = String>,
+        suggestions: impl IntoIterator<Item = String>,
         applicability: Applicability,
     ) -> &mut Self);
     forward!(pub fn multipart_suggestions(
         &mut self,
         msg: impl Into<SubdiagnosticMessage>,
-        suggestions: impl Iterator<Item = Vec<(Span, String)>>,
+        suggestions: impl IntoIterator<Item = Vec<(Span, String)>>,
         applicability: Applicability,
     ) -> &mut Self);
     forward!(pub fn span_suggestion_short(