Merge from rustc

.editorconfig

@@ -9,6 +9,8 @@ end_of_line = lf
 charset = utf-8
 trim_trailing_whitespace = true
 insert_final_newline = true
+
+[!src/llvm-project]
 indent_style = space
 indent_size = 4
 

.git-blame-ignore-revs

@@ -29,3 +29,5 @@ ec2cc761bc7067712ecc7734502f703fe3b024c8
 99cb0c6bc399fb94a0ddde7e9b38e9c00d523bad
 # reformat with rustfmt edition 2024
 c682aa162b0d41e21cc6748f4fecfe01efb69d1f
+# reformat with updated edition 2024
+1fcae03369abb4c2cc180cd5a49e1f4440a81300

Cargo.lock

@@ -420,9 +420,9 @@ version = "0.1.0"
 
 [[package]]
 name = "cc"
-version = "1.2.7"
+version = "1.2.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a012a0df96dd6d06ba9a1b29d6402d1a5d77c6befd2566afdc26e10603dc93d7"
+checksum = "c7777341816418c02e033934a09f20dc0ccaf65a5201ef8a450ae0105a573fda"
 dependencies = [
  "shlex",
 ]
@@ -1209,16 +1209,6 @@ dependencies = [
  "tidy",
 ]
 
-[[package]]
-name = "field-offset"
-version = "0.3.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "38e2275cc4e4fc009b0669731a1e5ab7ebf11f469eaede2bab9309a5b4d6057f"
-dependencies = [
- "memoffset",
- "rustc_version",
-]
-
 [[package]]
 name = "filetime"
 version = "0.2.25"
@@ -2295,15 +2285,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "memoffset"
-version = "0.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
-dependencies = [
- "autocfg",
-]
-
 [[package]]
 name = "mime"
 version = "0.3.17"
@@ -2322,9 +2303,9 @@ dependencies = [
 
 [[package]]
 name = "minifier"
-version = "0.3.2"
+version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd559bbf5d350ac7f2c1cf92ed71a869b847a92bce0c1318b47932a5b5f65cdd"
+checksum = "1cf47565b1430f5fe6c81d3afcb4b835271348d7eb35294a4d592e38dd09ea22"
 
 [[package]]
 name = "minimal-lexical"
@@ -3287,13 +3268,6 @@ dependencies = [
  "tikv-jemalloc-sys",
 ]
 
-[[package]]
-name = "rustc-perf-wrapper"
-version = "0.1.0"
-dependencies = [
- "clap",
-]
-
 [[package]]
 name = "rustc-rayon"
 version = "0.5.1"
@@ -3400,6 +3374,7 @@ dependencies = [
 name = "rustc_ast_lowering"
 version = "0.0.0"
 dependencies = [
+ "rustc_abi",
  "rustc_ast",
  "rustc_ast_pretty",
  "rustc_data_structures",
@@ -3422,6 +3397,7 @@ name = "rustc_ast_passes"
 version = "0.0.0"
 dependencies = [
  "itertools",
+ "rustc_abi",
  "rustc_ast",
  "rustc_ast_pretty",
  "rustc_attr_parsing",
@@ -4015,6 +3991,7 @@ version = "0.0.0"
 dependencies = [
  "rustc-rayon",
  "rustc-rayon-core",
+ "rustc_abi",
  "rustc_ast",
  "rustc_ast_lowering",
  "rustc_ast_passes",
@@ -4177,7 +4154,6 @@ version = "0.0.0"
 dependencies = [
  "bitflags",
  "either",
- "field-offset",
  "gsgdt",
  "polonius-engine",
  "rustc-rayon-core",
@@ -4425,7 +4401,6 @@ dependencies = [
 name = "rustc_query_impl"
 version = "0.0.0"
 dependencies = [
- "field-offset",
  "measureme",
  "rustc_data_structures",
  "rustc_errors",

Cargo.toml

@@ -45,7 +45,6 @@ members = [
   "src/tools/rustdoc-gui-test",
   "src/tools/opt-dist",
   "src/tools/coverage-dump",
-  "src/tools/rustc-perf-wrapper",
   "src/tools/wasm-component-ld",
   "src/tools/features-status-dump",
 ]

compiler/rustc_abi/src/lib.rs

@@ -1184,10 +1184,13 @@ impl Scalar {
     #[inline]
     pub fn is_bool(&self) -> bool {
         use Integer::*;
-        matches!(self, Scalar::Initialized {
-            value: Primitive::Int(I8, false),
-            valid_range: WrappingRange { start: 0, end: 1 }
-        })
+        matches!(
+            self,
+            Scalar::Initialized {
+                value: Primitive::Int(I8, false),
+                valid_range: WrappingRange { start: 0, end: 1 }
+            }
+        )
     }
 
     /// Get the primitive representation of this type, ignoring the valid range and whether the

compiler/rustc_ast/src/ast.rs

@@ -3225,7 +3225,7 @@ pub enum Extern {
     ///
     /// E.g. `extern fn foo() {}`.
     ///
-    /// This is just `extern "C"` (see `rustc_target::spec::abi::Abi::FALLBACK`).
+    /// This is just `extern "C"` (see `rustc_abi::ExternAbi::FALLBACK`).
     Implicit(Span),
     /// An explicit extern keyword was used with an explicit ABI.
     ///

compiler/rustc_ast/src/expand/autodiff_attrs.rs

@@ -30,14 +30,6 @@ pub enum DiffMode {
     Forward,
     /// The target function, to be created using reverse mode AD.
     Reverse,
-    /// The target function, to be created using forward mode AD.
-    /// This target function will also be used as a source for higher order derivatives,
-    /// so compute it before all Forward/Reverse targets and optimize it through llvm.
-    ForwardFirst,
-    /// The target function, to be created using reverse mode AD.
-    /// This target function will also be used as a source for higher order derivatives,
-    /// so compute it before all Forward/Reverse targets and optimize it through llvm.
-    ReverseFirst,
 }
 
 /// Dual and Duplicated (and their Only variants) are getting lowered to the same Enzyme Activity.
@@ -92,10 +84,10 @@ pub struct AutoDiffAttrs {
 
 impl DiffMode {
     pub fn is_rev(&self) -> bool {
-        matches!(self, DiffMode::Reverse | DiffMode::ReverseFirst)
+        matches!(self, DiffMode::Reverse)
     }
     pub fn is_fwd(&self) -> bool {
-        matches!(self, DiffMode::Forward | DiffMode::ForwardFirst)
+        matches!(self, DiffMode::Forward)
     }
 }
 
@@ -106,8 +98,6 @@ impl Display for DiffMode {
             DiffMode::Source => write!(f, "Source"),
             DiffMode::Forward => write!(f, "Forward"),
             DiffMode::Reverse => write!(f, "Reverse"),
-            DiffMode::ForwardFirst => write!(f, "ForwardFirst"),
-            DiffMode::ReverseFirst => write!(f, "ReverseFirst"),
         }
     }
 }
@@ -125,12 +115,12 @@ pub fn valid_ret_activity(mode: DiffMode, activity: DiffActivity) -> bool {
     match mode {
         DiffMode::Error => false,
         DiffMode::Source => false,
-        DiffMode::Forward | DiffMode::ForwardFirst => {
+        DiffMode::Forward => {
             activity == DiffActivity::Dual
                 || activity == DiffActivity::DualOnly
                 || activity == DiffActivity::Const
         }
-        DiffMode::Reverse | DiffMode::ReverseFirst => {
+        DiffMode::Reverse => {
             activity == DiffActivity::Const
                 || activity == DiffActivity::Active
                 || activity == DiffActivity::ActiveOnly
@@ -166,10 +156,10 @@ pub fn valid_input_activity(mode: DiffMode, activity: DiffActivity) -> bool {
     return match mode {
         DiffMode::Error => false,
         DiffMode::Source => false,
-        DiffMode::Forward | DiffMode::ForwardFirst => {
+        DiffMode::Forward => {
             matches!(activity, Dual | DualOnly | Const)
         }
-        DiffMode::Reverse | DiffMode::ReverseFirst => {
+        DiffMode::Reverse => {
             matches!(activity, Active | ActiveOnly | Duplicated | DuplicatedOnly | Const)
         }
     };
@@ -200,8 +190,6 @@ impl FromStr for DiffMode {
             "Source" => Ok(DiffMode::Source),
             "Forward" => Ok(DiffMode::Forward),
             "Reverse" => Ok(DiffMode::Reverse),
-            "ForwardFirst" => Ok(DiffMode::ForwardFirst),
-            "ReverseFirst" => Ok(DiffMode::ReverseFirst),
             _ => Err(()),
         }
     }

compiler/rustc_ast_lowering/Cargo.toml

@@ -8,6 +8,7 @@ doctest = false
 
 [dependencies]
 # tidy-alphabetical-start
+rustc_abi = { path = "../rustc_abi" }
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_data_structures = { path = "../rustc_data_structures" }

compiler/rustc_ast_lowering/src/expr.rs

@@ -828,15 +828,18 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 span,
                 Some(Arc::clone(&self.allow_gen_future)),
             );
-            self.lower_attrs(inner_hir_id, &[Attribute {
-                kind: AttrKind::Normal(ptr::P(NormalAttr::from_ident(Ident::new(
-                    sym::track_caller,
-                    span,
-                )))),
-                id: self.tcx.sess.psess.attr_id_generator.mk_attr_id(),
-                style: AttrStyle::Outer,
-                span: unstable_span,
-            }]);
+            self.lower_attrs(
+                inner_hir_id,
+                &[Attribute {
+                    kind: AttrKind::Normal(ptr::P(NormalAttr::from_ident(Ident::new(
+                        sym::track_caller,
+                        span,
+                    )))),
+                    id: self.tcx.sess.psess.attr_id_generator.mk_attr_id(),
+                    style: AttrStyle::Outer,
+                    span: unstable_span,
+                }],
+            );
         }
     }
 

compiler/rustc_ast_lowering/src/format.rs

@@ -362,13 +362,16 @@ fn make_format_spec<'hir>(
         debug_hex,
     } = &placeholder.format_options;
     let fill = ctx.expr_char(sp, fill.unwrap_or(' '));
-    let align =
-        ctx.expr_lang_item_type_relative(sp, hir::LangItem::FormatAlignment, match alignment {
+    let align = ctx.expr_lang_item_type_relative(
+        sp,
+        hir::LangItem::FormatAlignment,
+        match alignment {
             Some(FormatAlignment::Left) => sym::Left,
             Some(FormatAlignment::Right) => sym::Right,
             Some(FormatAlignment::Center) => sym::Center,
             None => sym::Unknown,
-        });
+        },
+    );
     // This needs to match `Flag` in library/core/src/fmt/rt.rs.
     let flags: u32 = ((sign == Some(FormatSign::Plus)) as u32)
         | ((sign == Some(FormatSign::Minus)) as u32) << 1

compiler/rustc_ast_lowering/src/item.rs

@@ -1,3 +1,4 @@
+use rustc_abi::ExternAbi;
 use rustc_ast::ptr::P;
 use rustc_ast::visit::AssocCtxt;
 use rustc_ast::*;
@@ -11,7 +12,6 @@ use rustc_middle::span_bug;
 use rustc_middle::ty::{ResolverAstLowering, TyCtxt};
 use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::{DesugaringKind, Ident, Span, Symbol, kw, sym};
-use rustc_target::spec::abi;
 use smallvec::{SmallVec, smallvec};
 use thin_vec::ThinVec;
 use tracing::instrument;
@@ -275,7 +275,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 ModKind::Unloaded => panic!("`mod` items should have been loaded by now"),
             },
             ItemKind::ForeignMod(fm) => hir::ItemKind::ForeignMod {
-                abi: fm.abi.map_or(abi::Abi::FALLBACK, |abi| self.lower_abi(abi)),
+                abi: fm.abi.map_or(ExternAbi::FALLBACK, |abi| self.lower_abi(abi)),
                 items: self
                     .arena
                     .alloc_from_iter(fm.items.iter().map(|x| self.lower_foreign_item_ref(x))),
@@ -304,12 +304,15 @@ impl<'hir> LoweringContext<'_, 'hir> {
                             );
                             this.arena.alloc(this.ty(span, hir::TyKind::Err(guar)))
                         }
-                        Some(ty) => this.lower_ty(ty, ImplTraitContext::OpaqueTy {
-                            origin: hir::OpaqueTyOrigin::TyAlias {
-                                parent: this.local_def_id(id),
-                                in_assoc_ty: false,
+                        Some(ty) => this.lower_ty(
+                            ty,
+                            ImplTraitContext::OpaqueTy {
+                                origin: hir::OpaqueTyOrigin::TyAlias {
+                                    parent: this.local_def_id(id),
+                                    in_assoc_ty: false,
+                                },
                             },
-                        }),
+                        ),
                     },
                 );
                 hir::ItemKind::TyAlias(ty, generics)
@@ -966,12 +969,15 @@ impl<'hir> LoweringContext<'_, 'hir> {
                             hir::ImplItemKind::Type(ty)
                         }
                         Some(ty) => {
-                            let ty = this.lower_ty(ty, ImplTraitContext::OpaqueTy {
-                                origin: hir::OpaqueTyOrigin::TyAlias {
-                                    parent: this.local_def_id(i.id),
-                                    in_assoc_ty: true,
+                            let ty = this.lower_ty(
+                                ty,
+                                ImplTraitContext::OpaqueTy {
+                                    origin: hir::OpaqueTyOrigin::TyAlias {
+                                        parent: this.local_def_id(i.id),
+                                        in_assoc_ty: true,
+                                    },
                                 },
-                            });
+                            );
                             hir::ImplItemKind::Type(ty)
                         }
                     },
@@ -1152,10 +1158,13 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
     pub(super) fn lower_const_body(&mut self, span: Span, expr: Option<&Expr>) -> hir::BodyId {
         self.lower_body(|this| {
-            (&[], match expr {
-                Some(expr) => this.lower_expr_mut(expr),
-                None => this.expr_err(span, this.dcx().span_delayed_bug(span, "no block")),
-            })
+            (
+                &[],
+                match expr {
+                    Some(expr) => this.lower_expr_mut(expr),
+                    None => this.expr_err(span, this.dcx().span_delayed_bug(span, "no block")),
+                },
+            )
         })
     }
 
@@ -1470,23 +1479,23 @@ impl<'hir> LoweringContext<'_, 'hir> {
         }
     }
 
-    pub(super) fn lower_abi(&mut self, abi: StrLit) -> abi::Abi {
-        abi::lookup(abi.symbol_unescaped.as_str()).unwrap_or_else(|err| {
+    pub(super) fn lower_abi(&mut self, abi: StrLit) -> ExternAbi {
+        rustc_abi::lookup(abi.symbol_unescaped.as_str()).unwrap_or_else(|err| {
             self.error_on_invalid_abi(abi, err);
-            abi::Abi::Rust
+            ExternAbi::Rust
         })
     }
 
-    pub(super) fn lower_extern(&mut self, ext: Extern) -> abi::Abi {
+    pub(super) fn lower_extern(&mut self, ext: Extern) -> ExternAbi {
         match ext {
-            Extern::None => abi::Abi::Rust,
-            Extern::Implicit(_) => abi::Abi::FALLBACK,
+            Extern::None => ExternAbi::Rust,
+            Extern::Implicit(_) => ExternAbi::FALLBACK,
             Extern::Explicit(abi, _) => self.lower_abi(abi),
         }
     }
 
-    fn error_on_invalid_abi(&self, abi: StrLit, err: abi::AbiUnsupported) {
-        let abi_names = abi::enabled_names(self.tcx.features(), abi.span)
+    fn error_on_invalid_abi(&self, abi: StrLit, err: rustc_abi::AbiUnsupported) {
+        let abi_names = rustc_abi::enabled_names(self.tcx.features(), abi.span)
             .iter()
             .map(|s| Symbol::intern(s))
             .collect::<Vec<_>>();
@@ -1495,7 +1504,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             abi: abi.symbol_unescaped,
             span: abi.span,
             explain: match err {
-                abi::AbiUnsupported::Reason { explain } => Some(InvalidAbiReason(explain)),
+                rustc_abi::AbiUnsupported::Reason { explain } => Some(InvalidAbiReason(explain)),
                 _ => None,
             },
             suggestion: suggested_name.map(|suggested_name| InvalidAbiSuggestion {

compiler/rustc_ast_passes/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2021"
 [dependencies]
 # tidy-alphabetical-start
 itertools = "0.12"
+rustc_abi = { path = "../rustc_abi" }
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_attr_parsing = { path = "../rustc_attr_parsing" }

compiler/rustc_ast_passes/src/feature_gate.rs

@@ -6,7 +6,6 @@ use rustc_session::Session;
 use rustc_session::parse::{feature_err, feature_err_issue, feature_warn};
 use rustc_span::source_map::Spanned;
 use rustc_span::{Span, Symbol, sym};
-use rustc_target::spec::abi;
 use thin_vec::ThinVec;
 
 use crate::errors;
@@ -77,12 +76,12 @@ impl<'a> PostExpansionVisitor<'a> {
     fn check_abi(&self, abi: ast::StrLit) {
         let ast::StrLit { symbol_unescaped, span, .. } = abi;
 
-        match abi::is_enabled(self.features, span, symbol_unescaped.as_str()) {
+        match rustc_abi::is_enabled(self.features, span, symbol_unescaped.as_str()) {
             Ok(()) => (),
-            Err(abi::AbiDisabled::Unstable { feature, explain }) => {
+            Err(rustc_abi::AbiDisabled::Unstable { feature, explain }) => {
                 feature_err_issue(&self.sess, feature, span, GateIssue::Language, explain).emit();
             }
-            Err(abi::AbiDisabled::Unrecognized) => {
+            Err(rustc_abi::AbiDisabled::Unrecognized) => {
                 if self.sess.opts.pretty.is_none_or(|ppm| ppm.needs_hir()) {
                     self.sess.dcx().span_delayed_bug(
                         span,

compiler/rustc_attr_parsing/src/session_diagnostics.rs

@@ -204,21 +204,27 @@ pub(crate) struct UnsupportedLiteral {
 
 impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for UnsupportedLiteral {
     fn into_diag(self, dcx: DiagCtxtHandle<'a>, level: Level) -> Diag<'a, G> {
-        let mut diag = Diag::new(dcx, level, match self.reason {
-            UnsupportedLiteralReason::Generic => fluent::attr_parsing_unsupported_literal_generic,
-            UnsupportedLiteralReason::CfgString => {
-                fluent::attr_parsing_unsupported_literal_cfg_string
-            }
-            UnsupportedLiteralReason::CfgBoolean => {
-                fluent::attr_parsing_unsupported_literal_cfg_boolean
-            }
-            UnsupportedLiteralReason::DeprecatedString => {
-                fluent::attr_parsing_unsupported_literal_deprecated_string
-            }
-            UnsupportedLiteralReason::DeprecatedKvPair => {
-                fluent::attr_parsing_unsupported_literal_deprecated_kv_pair
-            }
-        });
+        let mut diag = Diag::new(
+            dcx,
+            level,
+            match self.reason {
+                UnsupportedLiteralReason::Generic => {
+                    fluent::attr_parsing_unsupported_literal_generic
+                }
+                UnsupportedLiteralReason::CfgString => {
+                    fluent::attr_parsing_unsupported_literal_cfg_string
+                }
+                UnsupportedLiteralReason::CfgBoolean => {
+                    fluent::attr_parsing_unsupported_literal_cfg_boolean
+                }
+                UnsupportedLiteralReason::DeprecatedString => {
+                    fluent::attr_parsing_unsupported_literal_deprecated_string
+                }
+                UnsupportedLiteralReason::DeprecatedKvPair => {
+                    fluent::attr_parsing_unsupported_literal_deprecated_kv_pair
+                }
+            },
+        );
         diag.span(self.span);
         diag.code(E0565);
         if self.is_bytestr {

compiler/rustc_borrowck/src/diagnostics/bound_region_errors.rs

@@ -156,24 +156,25 @@ pub(crate) trait TypeOpInfo<'tcx> {
             return;
         };
 
-        let placeholder_region = ty::Region::new_placeholder(tcx, ty::Placeholder {
-            universe: adjusted_universe.into(),
-            bound: placeholder.bound,
-        });
+        let placeholder_region = ty::Region::new_placeholder(
+            tcx,
+            ty::Placeholder { universe: adjusted_universe.into(), bound: placeholder.bound },
+        );
 
-        let error_region =
-            if let RegionElement::PlaceholderRegion(error_placeholder) = error_element {
-                let adjusted_universe =
-                    error_placeholder.universe.as_u32().checked_sub(base_universe.as_u32());
-                adjusted_universe.map(|adjusted| {
-                    ty::Region::new_placeholder(tcx, ty::Placeholder {
-                        universe: adjusted.into(),
-                        bound: error_placeholder.bound,
-                    })
-                })
-            } else {
-                None
-            };
+        let error_region = if let RegionElement::PlaceholderRegion(error_placeholder) =
+            error_element
+        {
+            let adjusted_universe =
+                error_placeholder.universe.as_u32().checked_sub(base_universe.as_u32());
+            adjusted_universe.map(|adjusted| {
+                ty::Region::new_placeholder(
+                    tcx,
+                    ty::Placeholder { universe: adjusted.into(), bound: error_placeholder.bound },
+                )
+            })
+        } else {
+            None
+        };
 
         debug!(?placeholder_region);
 

compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs

@@ -147,10 +147,10 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 span,
                 desired_action.as_noun(),
                 partially_str,
-                self.describe_place_with_options(moved_place, DescribePlaceOpt {
-                    including_downcast: true,
-                    including_tuple_field: true,
-                }),
+                self.describe_place_with_options(
+                    moved_place,
+                    DescribePlaceOpt { including_downcast: true, including_tuple_field: true },
+                ),
             );
 
             let reinit_spans = maybe_reinitialized_locations
@@ -280,10 +280,10 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
                 self.suggest_adding_bounds(&mut err, ty, copy_did, span);
             }
 
-            let opt_name = self.describe_place_with_options(place.as_ref(), DescribePlaceOpt {
-                including_downcast: true,
-                including_tuple_field: true,
-            });
+            let opt_name = self.describe_place_with_options(
+                place.as_ref(),
+                DescribePlaceOpt { including_downcast: true, including_tuple_field: true },
+            );
             let note_msg = match opt_name {
                 Some(name) => format!("`{name}`"),
                 None => "value".to_owned(),
@@ -765,17 +765,17 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
         }
         let spans: Vec<_> = spans_set.into_iter().collect();
 
-        let (name, desc) = match self.describe_place_with_options(moved_place, DescribePlaceOpt {
-            including_downcast: true,
-            including_tuple_field: true,
-        }) {
+        let (name, desc) = match self.describe_place_with_options(
+            moved_place,
+            DescribePlaceOpt { including_downcast: true, including_tuple_field: true },
+        ) {
             Some(name) => (format!("`{name}`"), format!("`{name}` ")),
             None => ("the variable".to_string(), String::new()),
         };
-        let path = match self.describe_place_with_options(used_place, DescribePlaceOpt {
-            including_downcast: true,
-            including_tuple_field: true,
-        }) {
+        let path = match self.describe_place_with_options(
+            used_place,
+            DescribePlaceOpt { including_downcast: true, including_tuple_field: true },
+        ) {
             Some(name) => format!("`{name}`"),
             None => "value".to_string(),
         };

compiler/rustc_borrowck/src/diagnostics/mod.rs

@@ -304,10 +304,10 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
     /// End-user visible description of `place` if one can be found.
     /// If the place is a temporary for instance, `None` will be returned.
     pub(super) fn describe_place(&self, place_ref: PlaceRef<'tcx>) -> Option<String> {
-        self.describe_place_with_options(place_ref, DescribePlaceOpt {
-            including_downcast: false,
-            including_tuple_field: true,
-        })
+        self.describe_place_with_options(
+            place_ref,
+            DescribePlaceOpt { including_downcast: false, including_tuple_field: true },
+        )
     }
 
     /// End-user visible description of `place` if one can be found. If the place is a temporary

compiler/rustc_borrowck/src/diagnostics/region_errors.rs

@@ -1100,12 +1100,15 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
         let closure_ty = Ty::new_closure(
             tcx,
             closure_def_id.to_def_id(),
-            ty::ClosureArgs::new(tcx, ty::ClosureArgsParts {
-                parent_args: args.parent_args(),
-                closure_kind_ty: args.kind_ty(),
-                tupled_upvars_ty: args.tupled_upvars_ty(),
-                closure_sig_as_fn_ptr_ty,
-            })
+            ty::ClosureArgs::new(
+                tcx,
+                ty::ClosureArgsParts {
+                    parent_args: args.parent_args(),
+                    closure_kind_ty: args.kind_ty(),
+                    tupled_upvars_ty: args.tupled_upvars_ty(),
+                    closure_sig_as_fn_ptr_ty,
+                },
+            )
             .args,
         );
 

compiler/rustc_borrowck/src/lib.rs

@@ -1668,9 +1668,13 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
             match elem {
                 ProjectionElem::Deref => match place_ty.ty.kind() {
                     ty::Ref(..) | ty::RawPtr(..) => {
-                        self.move_errors.push(MoveError::new(place, location, BorrowedContent {
-                            target_place: place_ref.project_deeper(&[elem], tcx),
-                        }));
+                        self.move_errors.push(MoveError::new(
+                            place,
+                            location,
+                            BorrowedContent {
+                                target_place: place_ref.project_deeper(&[elem], tcx),
+                            },
+                        ));
                         return;
                     }
                     ty::Adt(adt, _) => {

compiler/rustc_borrowck/src/region_infer/opaque_types.rs

@@ -166,10 +166,10 @@ impl<'tcx> RegionInferenceContext<'tcx> {
                 // FIXME(oli-obk): collect multiple spans for better diagnostics down the road.
                 prev.span = prev.span.substitute_dummy(concrete_type.span);
             } else {
-                result.insert(opaque_type_key.def_id, OpaqueHiddenType {
-                    ty,
-                    span: concrete_type.span,
-                });
+                result.insert(
+                    opaque_type_key.def_id,
+                    OpaqueHiddenType { ty, span: concrete_type.span },
+                );
             }
 
             // Check that all opaque types have the same region parameters if they have the same

compiler/rustc_borrowck/src/type_check/input_output.rs

@@ -75,17 +75,20 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             let output_ty = Ty::new_coroutine(
                 self.tcx(),
                 self.tcx().coroutine_for_closure(mir_def_id),
-                ty::CoroutineArgs::new(self.tcx(), ty::CoroutineArgsParts {
-                    parent_args: args.parent_args(),
-                    kind_ty: Ty::from_coroutine_closure_kind(self.tcx(), args.kind()),
-                    return_ty: user_provided_sig.output(),
-                    tupled_upvars_ty,
-                    // For async closures, none of these can be annotated, so just fill
-                    // them with fresh ty vars.
-                    resume_ty: next_ty_var(),
-                    yield_ty: next_ty_var(),
-                    witness: next_ty_var(),
-                })
+                ty::CoroutineArgs::new(
+                    self.tcx(),
+                    ty::CoroutineArgsParts {
+                        parent_args: args.parent_args(),
+                        kind_ty: Ty::from_coroutine_closure_kind(self.tcx(), args.kind()),
+                        return_ty: user_provided_sig.output(),
+                        tupled_upvars_ty,
+                        // For async closures, none of these can be annotated, so just fill
+                        // them with fresh ty vars.
+                        resume_ty: next_ty_var(),
+                        yield_ty: next_ty_var(),
+                        witness: next_ty_var(),
+                    },
+                )
                 .args,
             );
 

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -411,10 +411,10 @@ impl<'a, 'b, 'tcx> Visitor<'tcx> for TypeVerifier<'a, 'b, 'tcx> {
                 } else {
                     self.typeck.ascribe_user_type(
                         constant.const_.ty(),
-                        ty::UserType::new(ty::UserTypeKind::TypeOf(uv.def, UserArgs {
-                            args: uv.args,
-                            user_self_ty: None,
-                        })),
+                        ty::UserType::new(ty::UserTypeKind::TypeOf(
+                            uv.def,
+                            UserArgs { args: uv.args, user_self_ty: None },
+                        )),
                         locations.span(self.typeck.body),
                     );
                 }
@@ -1642,10 +1642,11 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             }
 
             &Rvalue::NullaryOp(NullOp::SizeOf | NullOp::AlignOf, ty) => {
-                let trait_ref =
-                    ty::TraitRef::new(tcx, tcx.require_lang_item(LangItem::Sized, Some(span)), [
-                        ty,
-                    ]);
+                let trait_ref = ty::TraitRef::new(
+                    tcx,
+                    tcx.require_lang_item(LangItem::Sized, Some(span)),
+                    [ty],
+                );
 
                 self.prove_trait_ref(
                     trait_ref,
@@ -1659,10 +1660,11 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             Rvalue::ShallowInitBox(operand, ty) => {
                 self.check_operand(operand, location);
 
-                let trait_ref =
-                    ty::TraitRef::new(tcx, tcx.require_lang_item(LangItem::Sized, Some(span)), [
-                        *ty,
-                    ]);
+                let trait_ref = ty::TraitRef::new(
+                    tcx,
+                    tcx.require_lang_item(LangItem::Sized, Some(span)),
+                    [*ty],
+                );
 
                 self.prove_trait_ref(
                     trait_ref,

compiler/rustc_borrowck/src/type_check/opaque_types.rs

@@ -284,7 +284,7 @@ where
             return;
         }
 
-        match ty.kind() {
+        match *ty.kind() {
             ty::Closure(_, args) => {
                 // Skip lifetime parameters of the enclosing item(s)
 
@@ -316,10 +316,12 @@ where
                 args.as_coroutine().resume_ty().visit_with(self);
             }
 
-            ty::Alias(ty::Opaque, ty::AliasTy { def_id, args, .. }) => {
-                // Skip lifetime parameters that are not captures.
-                let variances = self.tcx.variances_of(*def_id);
-
+            ty::Alias(kind, ty::AliasTy { def_id, args, .. })
+                if let Some(variances) = self.tcx.opt_alias_variances(kind, def_id) =>
+            {
+                // Skip lifetime parameters that are not captured, since they do
+                // not need member constraints registered for them; we'll erase
+                // them (and hopefully in the future replace them with placeholders).
                 for (v, s) in std::iter::zip(variances, args.iter()) {
                     if *v != ty::Bivariant {
                         s.visit_with(self);

compiler/rustc_borrowck/src/universal_regions.rs

@@ -620,10 +620,10 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> {
                     let ty = tcx
                         .typeck(self.mir_def)
                         .node_type(tcx.local_def_id_to_hir_id(self.mir_def));
-                    let args = InlineConstArgs::new(tcx, InlineConstArgsParts {
-                        parent_args: identity_args,
-                        ty,
-                    })
+                    let args = InlineConstArgs::new(
+                        tcx,
+                        InlineConstArgsParts { parent_args: identity_args, ty },
+                    )
                     .args;
                     let args = self.infcx.replace_free_regions_with_nll_infer_vars(FR, args);
                     DefiningTy::InlineConst(self.mir_def.to_def_id(), args)

compiler/rustc_builtin_macros/src/alloc_error_handler.rs

@@ -67,10 +67,11 @@ fn generate_handler(cx: &ExtCtxt<'_>, handler: Ident, span: Span, sig_span: Span
 
     let layout_new = cx.std_path(&[sym::alloc, sym::Layout, sym::from_size_align_unchecked]);
     let layout_new = cx.expr_path(cx.path(span, layout_new));
-    let layout = cx.expr_call(span, layout_new, thin_vec![
-        cx.expr_ident(span, size),
-        cx.expr_ident(span, align)
-    ]);
+    let layout = cx.expr_call(
+        span,
+        layout_new,
+        thin_vec![cx.expr_ident(span, size), cx.expr_ident(span, align)],
+    );
 
     let call = cx.expr_call_ident(sig_span, handler, thin_vec![layout]);
 

compiler/rustc_builtin_macros/src/autodiff.rs

@@ -406,19 +406,21 @@ mod llvm_enzyme {
         let unsf_expr = ecx.expr_block(P(unsf_block));
         let blackbox_call_expr = ecx.expr_path(ecx.path(span, blackbox_path));
         let primal_call = gen_primal_call(ecx, span, primal, idents);
-        let black_box_primal_call =
-            ecx.expr_call(new_decl_span, blackbox_call_expr.clone(), thin_vec![
-                primal_call.clone()
-            ]);
+        let black_box_primal_call = ecx.expr_call(
+            new_decl_span,
+            blackbox_call_expr.clone(),
+            thin_vec![primal_call.clone()],
+        );
         let tup_args = new_names
             .iter()
             .map(|arg| ecx.expr_path(ecx.path_ident(span, Ident::from_str(arg))))
             .collect();
 
-        let black_box_remaining_args =
-            ecx.expr_call(sig_span, blackbox_call_expr.clone(), thin_vec![
-                ecx.expr_tuple(sig_span, tup_args)
-            ]);
+        let black_box_remaining_args = ecx.expr_call(
+            sig_span,
+            blackbox_call_expr.clone(),
+            thin_vec![ecx.expr_tuple(sig_span, tup_args)],
+        );
 
         let mut body = ecx.block(span, ThinVec::new());
         body.stmts.push(ecx.stmt_semi(unsf_expr));
@@ -532,8 +534,11 @@ mod llvm_enzyme {
                 return body;
             }
             [arg] => {
-                ret = ecx
-                    .expr_call(new_decl_span, blackbox_call_expr.clone(), thin_vec![arg.clone()]);
+                ret = ecx.expr_call(
+                    new_decl_span,
+                    blackbox_call_expr.clone(),
+                    thin_vec![arg.clone()],
+                );
             }
             args => {
                 let ret_tuple: P<ast::Expr> = ecx.expr_tuple(span, args.into());

compiler/rustc_builtin_macros/src/deriving/clone.rs

@@ -114,10 +114,13 @@ fn cs_clone_simple(
                 // type parameters.
             } else {
                 // let _: AssertParamIsClone<FieldTy>;
-                super::assert_ty_bounds(cx, &mut stmts, field.ty.clone(), field.span, &[
-                    sym::clone,
-                    sym::AssertParamIsClone,
-                ]);
+                super::assert_ty_bounds(
+                    cx,
+                    &mut stmts,
+                    field.ty.clone(),
+                    field.span,
+                    &[sym::clone, sym::AssertParamIsClone],
+                );
             }
         }
     };
@@ -126,10 +129,13 @@ fn cs_clone_simple(
         // Just a single assertion for unions, that the union impls `Copy`.
         // let _: AssertParamIsCopy<Self>;
         let self_ty = cx.ty_path(cx.path_ident(trait_span, Ident::with_dummy_span(kw::SelfUpper)));
-        super::assert_ty_bounds(cx, &mut stmts, self_ty, trait_span, &[
-            sym::clone,
-            sym::AssertParamIsCopy,
-        ]);
+        super::assert_ty_bounds(
+            cx,
+            &mut stmts,
+            self_ty,
+            trait_span,
+            &[sym::clone, sym::AssertParamIsCopy],
+        );
     } else {
         match *substr.fields {
             StaticStruct(vdata, ..) => {

compiler/rustc_builtin_macros/src/deriving/cmp/eq.rs

@@ -65,10 +65,13 @@ fn cs_total_eq_assert(
                 // Already produced an assertion for this type.
             } else {
                 // let _: AssertParamIsEq<FieldTy>;
-                super::assert_ty_bounds(cx, &mut stmts, field.ty.clone(), field.span, &[
-                    sym::cmp,
-                    sym::AssertParamIsEq,
-                ]);
+                super::assert_ty_bounds(
+                    cx,
+                    &mut stmts,
+                    field.ty.clone(),
+                    field.span,
+                    &[sym::cmp, sym::AssertParamIsEq],
+                );
             }
         }
     };

compiler/rustc_builtin_macros/src/deriving/default.rs

@@ -108,30 +108,38 @@ fn default_enum_substructure(
         Ok(default_variant) => {
             // We now know there is exactly one unit variant with exactly one `#[default]` attribute.
             match &default_variant.data {
-                VariantData::Unit(_) => cx.expr_path(cx.path(default_variant.span, vec![
-                    Ident::new(kw::SelfUpper, default_variant.span),
-                    default_variant.ident,
-                ])),
+                VariantData::Unit(_) => cx.expr_path(cx.path(
+                    default_variant.span,
+                    vec![Ident::new(kw::SelfUpper, default_variant.span), default_variant.ident],
+                )),
                 VariantData::Struct { fields, .. } => {
                     // This only happens if `#![feature(default_field_values)]`. We have validated
                     // all fields have default values in the definition.
                     let default_fields = fields
                         .iter()
                         .map(|field| {
-                            cx.field_imm(field.span, field.ident.unwrap(), match &field.default {
-                                // We use `Default::default()`.
-                                None => default_call(cx, field.span),
-                                // We use the field default const expression.
-                                Some(val) => {
-                                    cx.expr(val.value.span, ast::ExprKind::ConstBlock(val.clone()))
-                                }
-                            })
+                            cx.field_imm(
+                                field.span,
+                                field.ident.unwrap(),
+                                match &field.default {
+                                    // We use `Default::default()`.
+                                    None => default_call(cx, field.span),
+                                    // We use the field default const expression.
+                                    Some(val) => cx.expr(
+                                        val.value.span,
+                                        ast::ExprKind::ConstBlock(val.clone()),
+                                    ),
+                                },
+                            )
                         })
                         .collect();
-                    let path = cx.path(default_variant.span, vec![
-                        Ident::new(kw::SelfUpper, default_variant.span),
-                        default_variant.ident,
-                    ]);
+                    let path = cx.path(
+                        default_variant.span,
+                        vec![
+                            Ident::new(kw::SelfUpper, default_variant.span),
+                            default_variant.ident,
+                        ],
+                    );
                     cx.expr_struct(default_variant.span, path, default_fields)
                 }
                 // Logic error in `extract_default_variant`.

compiler/rustc_builtin_macros/src/deriving/generic/mod.rs

@@ -1220,10 +1220,12 @@ impl<'a> MethodDef<'a> {
 
             let discr_let_stmts: ThinVec<_> = iter::zip(&discr_idents, &selflike_args)
                 .map(|(&ident, selflike_arg)| {
-                    let variant_value =
-                        deriving::call_intrinsic(cx, span, sym::discriminant_value, thin_vec![
-                            selflike_arg.clone()
-                        ]);
+                    let variant_value = deriving::call_intrinsic(
+                        cx,
+                        span,
+                        sym::discriminant_value,
+                        thin_vec![selflike_arg.clone()],
+                    );
                     cx.stmt_let(span, false, ident, variant_value)
                 })
                 .collect();

compiler/rustc_builtin_macros/src/env.rs

@@ -77,11 +77,11 @@ pub(crate) fn expand_option_env<'cx>(
             let guar = cx.dcx().emit_err(errors::EnvNotUnicode { span: sp, var: *symbol });
             return ExpandResult::Ready(DummyResult::any(sp, guar));
         }
-        Ok(value) => {
-            cx.expr_call_global(sp, cx.std_path(&[sym::option, sym::Option, sym::Some]), thin_vec![
-                cx.expr_str(sp, value)
-            ])
-        }
+        Ok(value) => cx.expr_call_global(
+            sp,
+            cx.std_path(&[sym::option, sym::Option, sym::Some]),
+            thin_vec![cx.expr_str(sp, value)],
+        ),
     };
     ExpandResult::Ready(MacEager::expr(e))
 }

compiler/rustc_builtin_macros/src/format_foreign.rs

@@ -183,10 +183,14 @@ pub(crate) mod printf {
             s.push('{');
 
             if let Some(arg) = self.parameter {
-                match write!(s, "{}", match arg.checked_sub(1) {
-                    Some(a) => a,
-                    None => return Err(None),
-                }) {
+                match write!(
+                    s,
+                    "{}",
+                    match arg.checked_sub(1) {
+                        Some(a) => a,
+                        None => return Err(None),
+                    }
+                ) {
                     Err(_) => return Err(None),
                     _ => {}
                 }

compiler/rustc_builtin_macros/src/format_foreign/printf/tests.rs

@@ -99,12 +99,10 @@ fn test_parse() {
 fn test_iter() {
     let s = "The %d'th word %% is: `%.*s` %!\n";
     let subs: Vec<_> = iter_subs(s, 0).map(|sub| sub.translate().ok()).collect();
-    assert_eq!(subs.iter().map(Option::as_deref).collect::<Vec<_>>(), vec![
-        Some("{}"),
-        None,
-        Some("{:.*}"),
-        None
-    ]);
+    assert_eq!(
+        subs.iter().map(Option::as_deref).collect::<Vec<_>>(),
+        vec![Some("{}"), None, Some("{:.*}"), None]
+    );
 }
 
 /// Checks that the translations are what we expect.

compiler/rustc_builtin_macros/src/format_foreign/shell/tests.rs

@@ -38,11 +38,10 @@ fn test_iter() {
     use super::iter_subs;
     let s = "The $0'th word $$ is: `$WORD` $!\n";
     let subs: Vec<_> = iter_subs(s, 0).map(|sub| sub.translate().ok()).collect();
-    assert_eq!(subs.iter().map(Option::as_deref).collect::<Vec<_>>(), vec![
-        Some("{0}"),
-        None,
-        Some("{WORD}")
-    ]);
+    assert_eq!(
+        subs.iter().map(Option::as_deref).collect::<Vec<_>>(),
+        vec![Some("{0}"), None, Some("{WORD}")]
+    );
 }
 
 #[test]

compiler/rustc_builtin_macros/src/proc_macro_harness.rs

@@ -301,13 +301,10 @@ fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> P<ast::Item> {
             };
             let local_path = |cx: &ExtCtxt<'_>, name| cx.expr_path(cx.path(span, vec![name]));
             let proc_macro_ty_method_path = |cx: &ExtCtxt<'_>, method| {
-                cx.expr_path(cx.path(span.with_ctxt(harness_span.ctxt()), vec![
-                    proc_macro,
-                    bridge,
-                    client,
-                    proc_macro_ty,
-                    method,
-                ]))
+                cx.expr_path(cx.path(
+                    span.with_ctxt(harness_span.ctxt()),
+                    vec![proc_macro, bridge, client, proc_macro_ty, method],
+                ))
             };
             match m {
                 ProcMacro::Derive(cd) => {
@@ -340,10 +337,14 @@ fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> P<ast::Item> {
 
                     // The call needs to use `harness_span` so that the const stability checker
                     // accepts it.
-                    cx.expr_call(harness_span, proc_macro_ty_method_path(cx, ident), thin_vec![
-                        cx.expr_str(span, ca.function_name.name),
-                        local_path(cx, ca.function_name),
-                    ])
+                    cx.expr_call(
+                        harness_span,
+                        proc_macro_ty_method_path(cx, ident),
+                        thin_vec![
+                            cx.expr_str(span, ca.function_name.name),
+                            local_path(cx, ca.function_name),
+                        ],
+                    )
                 }
             }
         })
@@ -357,12 +358,9 @@ fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> P<ast::Item> {
                 span,
                 cx.ty(
                     span,
-                    ast::TyKind::Slice(cx.ty_path(cx.path(span, vec![
-                        proc_macro,
-                        bridge,
-                        client,
-                        proc_macro_ty,
-                    ]))),
+                    ast::TyKind::Slice(
+                        cx.ty_path(cx.path(span, vec![proc_macro, bridge, client, proc_macro_ty])),
+                    ),
                 ),
                 None,
                 ast::Mutability::Not,

compiler/rustc_builtin_macros/src/test.rs

@@ -169,20 +169,26 @@ pub(crate) fn expand_test_or_bench(
 
     // creates test::ShouldPanic::$name
     let should_panic_path = |name| {
-        cx.path(sp, vec![
-            test_id,
-            Ident::from_str_and_span("ShouldPanic", sp),
-            Ident::from_str_and_span(name, sp),
-        ])
+        cx.path(
+            sp,
+            vec![
+                test_id,
+                Ident::from_str_and_span("ShouldPanic", sp),
+                Ident::from_str_and_span(name, sp),
+            ],
+        )
     };
 
     // creates test::TestType::$name
     let test_type_path = |name| {
-        cx.path(sp, vec![
-            test_id,
-            Ident::from_str_and_span("TestType", sp),
-            Ident::from_str_and_span(name, sp),
-        ])
+        cx.path(
+            sp,
+            vec![
+                test_id,
+                Ident::from_str_and_span("TestType", sp),
+                Ident::from_str_and_span(name, sp),
+            ],
+        )
     };
 
     // creates $name: $expr
@@ -202,39 +208,55 @@ pub(crate) fn expand_test_or_bench(
         // A simple ident for a lambda
         let b = Ident::from_str_and_span("b", attr_sp);
 
-        cx.expr_call(sp, cx.expr_path(test_path("StaticBenchFn")), thin_vec![
-            // #[coverage(off)]
-            // |b| self::test::assert_test_result(
-            coverage_off(cx.lambda1(
-                sp,
-                cx.expr_call(sp, cx.expr_path(test_path("assert_test_result")), thin_vec![
-                    // super::$test_fn(b)
+        cx.expr_call(
+            sp,
+            cx.expr_path(test_path("StaticBenchFn")),
+            thin_vec![
+                // #[coverage(off)]
+                // |b| self::test::assert_test_result(
+                coverage_off(cx.lambda1(
+                    sp,
                     cx.expr_call(
-                        ret_ty_sp,
-                        cx.expr_path(cx.path(sp, vec![item.ident])),
-                        thin_vec![cx.expr_ident(sp, b)],
+                        sp,
+                        cx.expr_path(test_path("assert_test_result")),
+                        thin_vec![
+                            // super::$test_fn(b)
+                            cx.expr_call(
+                                ret_ty_sp,
+                                cx.expr_path(cx.path(sp, vec![item.ident])),
+                                thin_vec![cx.expr_ident(sp, b)],
+                            ),
+                        ],
                     ),
-                ],),
-                b,
-            )), // )
-        ])
+                    b,
+                )), // )
+            ],
+        )
     } else {
-        cx.expr_call(sp, cx.expr_path(test_path("StaticTestFn")), thin_vec![
-            // #[coverage(off)]
-            // || {
-            coverage_off(cx.lambda0(
-                sp,
-                // test::assert_test_result(
-                cx.expr_call(sp, cx.expr_path(test_path("assert_test_result")), thin_vec![
-                    // $test_fn()
+        cx.expr_call(
+            sp,
+            cx.expr_path(test_path("StaticTestFn")),
+            thin_vec![
+                // #[coverage(off)]
+                // || {
+                coverage_off(cx.lambda0(
+                    sp,
+                    // test::assert_test_result(
                     cx.expr_call(
-                        ret_ty_sp,
-                        cx.expr_path(cx.path(sp, vec![item.ident])),
-                        ThinVec::new(),
-                    ), // )
-                ],), // }
-            )), // )
-        ])
+                        sp,
+                        cx.expr_path(test_path("assert_test_result")),
+                        thin_vec![
+                            // $test_fn()
+                            cx.expr_call(
+                                ret_ty_sp,
+                                cx.expr_path(cx.path(sp, vec![item.ident])),
+                                ThinVec::new(),
+                            ), // )
+                        ],
+                    ), // }
+                )), // )
+            ],
+        )
     };
 
     let test_path_symbol = Symbol::intern(&item_path(
@@ -245,26 +267,30 @@ pub(crate) fn expand_test_or_bench(
 
     let location_info = get_location_info(cx, &item);
 
-    let mut test_const = cx.item(
-        sp,
-        Ident::new(item.ident.name, sp),
-        thin_vec![
-            // #[cfg(test)]
-            cx.attr_nested_word(sym::cfg, sym::test, attr_sp),
-            // #[rustc_test_marker = "test_case_sort_key"]
-            cx.attr_name_value_str(sym::rustc_test_marker, test_path_symbol, attr_sp),
-            // #[doc(hidden)]
-            cx.attr_nested_word(sym::doc, sym::hidden, attr_sp),
-        ],
-        // const $ident: test::TestDescAndFn =
-        ast::ItemKind::Const(
-            ast::ConstItem {
-                defaultness: ast::Defaultness::Final,
-                generics: ast::Generics::default(),
-                ty: cx.ty(sp, ast::TyKind::Path(None, test_path("TestDescAndFn"))),
-                // test::TestDescAndFn {
-                expr: Some(
-                    cx.expr_struct(sp, test_path("TestDescAndFn"), thin_vec![
+    let mut test_const =
+        cx.item(
+            sp,
+            Ident::new(item.ident.name, sp),
+            thin_vec![
+                // #[cfg(test)]
+                cx.attr_nested_word(sym::cfg, sym::test, attr_sp),
+                // #[rustc_test_marker = "test_case_sort_key"]
+                cx.attr_name_value_str(sym::rustc_test_marker, test_path_symbol, attr_sp),
+                // #[doc(hidden)]
+                cx.attr_nested_word(sym::doc, sym::hidden, attr_sp),
+            ],
+            // const $ident: test::TestDescAndFn =
+            ast::ItemKind::Const(
+                ast::ConstItem {
+                    defaultness: ast::Defaultness::Final,
+                    generics: ast::Generics::default(),
+                    ty: cx.ty(sp, ast::TyKind::Path(None, test_path("TestDescAndFn"))),
+                    // test::TestDescAndFn {
+                    expr: Some(
+                        cx.expr_struct(
+                            sp,
+                            test_path("TestDescAndFn"),
+                            thin_vec![
                         // desc: test::TestDesc {
                         field(
                             "desc",
@@ -340,12 +366,13 @@ pub(crate) fn expand_test_or_bench(
                         ),
                         // testfn: test::StaticTestFn(...) | test::StaticBenchFn(...)
                         field("testfn", test_fn), // }
-                    ]), // }
-                ),
-            }
-            .into(),
-        ),
-    );
+                    ],
+                        ), // }
+                    ),
+                }
+                .into(),
+            ),
+        );
     test_const = test_const.map(|mut tc| {
         tc.vis.kind = ast::VisibilityKind::Public;
         tc

compiler/rustc_codegen_cranelift/Cargo.lock

@@ -43,24 +43,24 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.115.0"
+version = "0.116.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac89549be94911dd0e839b4a7db99e9ed29c17517e1c026f61066884c168aa3c"
+checksum = "e15d04a0ce86cb36ead88ad68cf693ffd6cda47052b9e0ac114bc47fd9cd23c4"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-bitset"
-version = "0.115.0"
+version = "0.116.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9bd49369f76c77e34e641af85d0956869237832c118964d08bf5f51f210875a"
+checksum = "7c6e3969a7ce267259ce244b7867c5d3bc9e65b0a87e81039588dfdeaede9f34"
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.115.0"
+version = "0.116.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd96ce9cf8efebd7f5ab8ced5a0ce44250280bbae9f593d74a6d7effc3582a35"
+checksum = "2c22032c4cb42558371cf516bb47f26cdad1819d3475c133e93c49f50ebf304e"
 dependencies = [
  "bumpalo",
  "cranelift-bforest",
@@ -82,42 +82,42 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.115.0"
+version = "0.116.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5a68e358827afe4bfb6239fcbf6fbd5ac56206ece8a99c8f5f9bbd518773281a"
+checksum = "c904bc71c61b27fc57827f4a1379f29de64fe95653b620a3db77d59655eee0b8"
 dependencies = [
  "cranelift-codegen-shared",
 ]
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.115.0"
+version = "0.116.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e184c9767afbe73d50c55ec29abcf4c32f9baf0d9d22b86d58c4d55e06dee181"
+checksum = "40180f5497572f644ce88c255480981ae2ec1d7bb4d8e0c0136a13b87a2f2ceb"
 
 [[package]]
 name = "cranelift-control"
-version = "0.115.0"
+version = "0.116.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cc7664f2a66f053e33f149e952bb5971d138e3af637f5097727ed6dc0ed95dd"
+checksum = "26d132c6d0bd8a489563472afc171759da0707804a65ece7ceb15a8c6d7dd5ef"
 dependencies = [
  "arbitrary",
 ]
 
 [[package]]
 name = "cranelift-entity"
-version = "0.115.0"
+version = "0.116.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "118597e3a9cf86c3556fa579a7a23b955fa18231651a52a77a2475d305a9cf84"
+checksum = "4b2d0d9618275474fbf679dd018ac6e009acbd6ae6850f6a67be33fb3b00b323"
 dependencies = [
  "cranelift-bitset",
 ]
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.115.0"
+version = "0.116.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7638ea1efb069a0aa18d8ee67401b6b0d19f6bfe5de5e9ede348bfc80bb0d8c7"
+checksum = "4fac41e16729107393174b0c9e3730fb072866100e1e64e80a1a963b2e484d57"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -127,15 +127,15 @@ dependencies = [
 
 [[package]]
 name = "cranelift-isle"
-version = "0.115.0"
+version = "0.116.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15c53e1152a0b01c4ed2b1e0535602b8e86458777dd9d18b28732b16325c7dc0"
+checksum = "1ca20d576e5070044d0a72a9effc2deacf4d6aa650403189d8ea50126483944d"
 
 [[package]]
 name = "cranelift-jit"
-version = "0.115.0"
+version = "0.116.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36972cab12ff246afe8d45b6a427669cf814bd393c661e5e8a8dedc26a81c73f"
+checksum = "5e65c42755a719b09662b00c700daaf76cc35d5ace1f5c002ad404b591ff1978"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -153,9 +153,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.115.0"
+version = "0.116.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11841b3f54ac480db1e8e8d5678ba901a13b387012d315e3f8fba3e7b7a80447"
+checksum = "4d55612bebcf16ff7306c8a6f5bdb6d45662b8aa1ee058ecce8807ad87db719b"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -164,9 +164,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.115.0"
+version = "0.116.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b7d8f895444fa52dd7bdd0bed11bf007a7fb43af65a6deac8fcc4094c6372f7"
+checksum = "b8dee82f3f1f2c4cba9177f1cc5e350fe98764379bcd29340caa7b01f85076c7"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -175,9 +175,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.115.0"
+version = "0.116.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e235ddfd19f100855ad03358c7ae0a13070c38a000701054cab46458cca6e81"
+checksum = "aad5a6d3e379493c3f8b35dc61c93d0bf5f27003bbe20614e0200b0ec372ef52"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -413,9 +413,9 @@ dependencies = [
 
 [[package]]
 name = "target-lexicon"
-version = "0.12.16"
+version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
+checksum = "dc12939a1c9b9d391e0b7135f72fd30508b73450753e28341fed159317582a77"
 
 [[package]]
 name = "unicode-ident"
@@ -425,9 +425,9 @@ checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
 
 [[package]]
 name = "wasmtime-jit-icache-coherence"
-version = "28.0.0"
+version = "29.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d40d7722b9e1fbeae135715710a8a2570b1e6cf72b74dd653962d89831c6c70d"
+checksum = "ec5e8552e01692e6c2e5293171704fed8abdec79d1a6995a0870ab190e5747d1"
 dependencies = [
  "anyhow",
  "cfg-if",

compiler/rustc_codegen_cranelift/Cargo.toml

@@ -8,13 +8,13 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { version = "0.115.0", default-features = false, features = ["std", "unwind", "all-native-arch"] }
-cranelift-frontend = { version = "0.115.0" }
-cranelift-module = { version = "0.115.0" }
-cranelift-native = { version = "0.115.0" }
-cranelift-jit = { version = "0.115.0", optional = true }
-cranelift-object = { version = "0.115.0" }
-target-lexicon = "0.12.0"
+cranelift-codegen = { version = "0.116.0", default-features = false, features = ["std", "unwind", "all-native-arch"] }
+cranelift-frontend = { version = "0.116.0" }
+cranelift-module = { version = "0.116.0" }
+cranelift-native = { version = "0.116.0" }
+cranelift-jit = { version = "0.116.0", optional = true }
+cranelift-object = { version = "0.116.0" }
+target-lexicon = "0.13"
 gimli = { version = "0.31", default-features = false, features = ["write"] }
 object = { version = "0.36", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
 
@@ -24,12 +24,12 @@ smallvec = "1.8.1"
 
 [patch.crates-io]
 # Uncomment to use an unreleased version of cranelift
-#cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-28.0.0", version = "0.115.0" }
-#cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-28.0.0", version = "0.115.0" }
-#cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-28.0.0", version = "0.115.0" }
-#cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-28.0.0", version = "0.115.0" }
-#cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-28.0.0", version = "0.115.0" }
-#cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-28.0.0", version = "0.115.0" }
+#cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-29.0.0", version = "0.116.0" }
+#cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-29.0.0", version = "0.116.0" }
+#cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-29.0.0", version = "0.116.0" }
+#cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-29.0.0", version = "0.116.0" }
+#cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-29.0.0", version = "0.116.0" }
+#cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-29.0.0", version = "0.116.0" }
 
 # Uncomment to use local checkout of cranelift
 #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" }

compiler/rustc_codegen_cranelift/build_system/prepare.rs

@@ -186,7 +186,7 @@ fn init_git_repo(repo_dir: &Path) {
     spawn_and_wait(git_add_cmd);
 
     let mut git_commit_cmd = git_command(repo_dir, "commit");
-    git_commit_cmd.arg("-m").arg("Initial commit").arg("-q");
+    git_commit_cmd.arg("-m").arg("Initial commit").arg("-q").arg("--no-verify");
     spawn_and_wait(git_commit_cmd);
 }
 

compiler/rustc_codegen_cranelift/build_system/tests.rs

@@ -330,14 +330,6 @@ impl<'a> TestRunner<'a> {
         target_compiler.rustflags.extend(rustflags_from_env("RUSTFLAGS"));
         target_compiler.rustdocflags.extend(rustflags_from_env("RUSTDOCFLAGS"));
 
-        // FIXME fix `#[linkage = "extern_weak"]` without this
-        if target_compiler.triple.contains("darwin") {
-            target_compiler.rustflags.extend([
-                "-Clink-arg=-undefined".to_owned(),
-                "-Clink-arg=dynamic_lookup".to_owned(),
-            ]);
-        }
-
         let jit_supported = use_unstable_features
             && is_native
             && target_compiler.triple.contains("x86_64")

compiler/rustc_codegen_cranelift/example/mini_core_hello_world.rs

@@ -274,7 +274,7 @@ fn main() {
 
     assert_eq!(((|()| 42u8) as fn(()) -> u8)(()), 42);
 
-    #[cfg(not(any(jit, windows)))]
+    #[cfg(not(any(jit, target_vendor = "apple", windows)))]
     {
         extern "C" {
             #[linkage = "extern_weak"]

compiler/rustc_codegen_cranelift/example/std_example.rs

@@ -241,9 +241,10 @@ unsafe fn test_simd() {
     let (zero0, zero1) = std::mem::transmute::<_, (u64, u64)>(x);
     assert_eq!((zero0, zero1), (0, 0));
     assert_eq!(std::mem::transmute::<_, [u16; 8]>(or), [7, 7, 7, 7, 7, 7, 7, 7]);
-    assert_eq!(std::mem::transmute::<_, [u16; 8]>(cmp_eq), [
-        0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff
-    ]);
+    assert_eq!(
+        std::mem::transmute::<_, [u16; 8]>(cmp_eq),
+        [0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff]
+    );
     assert_eq!(std::mem::transmute::<_, [u16; 8]>(cmp_lt), [0, 0, 0, 0, 0, 0, 0, 0]);
 
     test_mm_slli_si128();

compiler/rustc_codegen_cranelift/patches/0028-coretests-Disable-long-running-tests.patch

@@ -36,8 +36,8 @@ index 8402833..84592e0 100644
  #[cfg(not(miri))] // unused in Miri
  macro_rules! empty_max_mut {
 @@ -2485,6 +2486,7 @@ take_tests! {
-     (take_mut_oob_max_range_to_inclusive, (..=usize::MAX), None, empty_max_mut!()),
-     (take_mut_in_bounds_max_range_from, (usize::MAX..), Some(&mut [] as _), empty_max_mut!()),
+     (split_off_mut_oob_max_range_to_inclusive, (..=usize::MAX), None, empty_max_mut!()),
+     (split_off_mut_in_bounds_max_range_from, (usize::MAX..), Some(&mut [] as _), empty_max_mut!()),
  }
 +*/
  

compiler/rustc_codegen_cranelift/patches/0029-stdlib-Disable-f16-and-f128-in-compiler-builtins.patch

@@ -16,8 +16,8 @@ index 7165c3e48af..968552ad435 100644
  
  [dependencies]
  core = { path = "../core" }
--compiler_builtins = { version = "=0.1.145", features = ['rustc-dep-of-std'] }
-+compiler_builtins = { version = "=0.1.145", features = ['rustc-dep-of-std', 'no-f16-f128'] }
+-compiler_builtins = { version = "=0.1.146", features = ['rustc-dep-of-std'] }
++compiler_builtins = { version = "=0.1.146", features = ['rustc-dep-of-std', 'no-f16-f128'] }
  
  [dev-dependencies]
  rand = { version = "0.8.5", default-features = false, features = ["alloc"] }

compiler/rustc_codegen_cranelift/rust-toolchain

@@ -1,4 +1,4 @@
 [toolchain]
-channel = "nightly-2025-01-20"
+channel = "nightly-2025-02-07"
 components = ["rust-src", "rustc-dev", "llvm-tools"]
 profile = "minimal"

compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh

@@ -83,7 +83,6 @@ rm tests/ui/match/match-float.rs
 # ==================
 rm tests/ui/codegen/issue-28950.rs # depends on stack size optimizations
 rm tests/ui/codegen/init-large-type.rs # same
-rm tests/ui/issues/issue-40883.rs # same
 rm -r tests/run-make/fmt-write-bloat/ # tests an optimization
 rm tests/ui/statics/const_generics.rs # same
 
@@ -91,13 +90,11 @@ rm tests/ui/statics/const_generics.rs # same
 # ======================
 rm tests/incremental/thinlto/cgu_invalidated_when_import_{added,removed}.rs # requires LLVM
 rm -r tests/run-make/cross-lang-lto # same
-rm -r tests/run-make/sepcomp-inlining # same
-rm -r tests/run-make/sepcomp-separate # same
-rm -r tests/run-make/sepcomp-cci-copies # same
 rm -r tests/run-make/volatile-intrinsics # same
 rm -r tests/run-make/llvm-ident # same
 rm -r tests/run-make/no-builtins-attribute # same
 rm -r tests/run-make/pgo-gen-no-imp-symbols # same
+rm -r tests/run-make/llvm-location-discriminator-limit-dummy-span # same
 rm tests/ui/abi/stack-protector.rs # requires stack protector support
 rm -r tests/run-make/emit-stack-sizes # requires support for -Z emit-stack-sizes
 rm -r tests/run-make/optimization-remarks-dir # remarks are LLVM specific
@@ -130,6 +127,7 @@ rm tests/ui/abi/large-byval-align.rs # exceeds implementation limit of Cranelift
 rm -r tests/run-make/remap-path-prefix-dwarf # requires llvm-dwarfdump
 rm -r tests/run-make/strip # same
 rm -r tests/run-make/compiler-builtins # Expects lib/rustlib/src/rust to contains the standard library source
+rm -r tests/run-make/translation # same
 rm -r tests/run-make/missing-unstable-trait-bound # This disables support for unstable features, but running cg_clif needs some unstable features
 rm -r tests/run-make/const-trait-stable-toolchain # same
 rm -r tests/run-make/incr-add-rust-src-component
@@ -156,8 +154,6 @@ cp $(../dist/rustc-clif --print target-libdir)/libstd-*.so ../dist/lib/
 
 # prevent $(RUSTDOC) from picking up the sysroot built by x.py. It conflicts with the one used by
 # rustdoc-clif
-# FIXME remove the bootstrap changes once it is no longer necessary to revert rust-lang/rust#130642
-# to avoid building rustc when testing stage0 run-make.
 cat <<EOF | git apply -
 diff --git a/tests/run-make/tools.mk b/tests/run-make/tools.mk
 index ea06b620c4c..b969d0009c6 100644
@@ -196,6 +192,20 @@ index e7ae773ffa1d3..04bc2d7787da7 100644
              // Provide necessary library search paths for rustc.
              .env(dylib_env_var(), &env::join_paths(host_dylib_search_paths).unwrap());
 
+diff --git a/tests/run-make/linker-warning/rmake.rs b/tests/run-make/linker-warning/rmake.rs
+index 30387af428c..f7895b12961 100644
+--- a/tests/run-make/linker-warning/rmake.rs
++++ b/tests/run-make/linker-warning/rmake.rs
+@@ -57,7 +57,8 @@ fn main() {
+             .actual_text("(linker error)", out.stderr())
+-            .normalize(r#"/rustc[^/]*/"#, "/rustc/")
++            .normalize(r#"/tmp/rustc[^/]*/"#, "/tmp/rustc/")
++            .normalize("libpanic_abort", "libpanic_unwind")
+             .normalize(
+                 regex::escape(run_make_support::build_root().to_str().unwrap()),
+                 "/build-root",
+             )
+             .run();
 EOF
 
 echo "[TEST] rustc test suite"

compiler/rustc_codegen_cranelift/src/cast.rs

@@ -96,9 +96,12 @@ pub(crate) fn clif_int_or_float_cast(
                 },
             );
 
-            fx.lib_call(&name, vec![AbiParam::new(from_ty)], vec![AbiParam::new(types::I128)], &[
-                from,
-            ])[0]
+            fx.lib_call(
+                &name,
+                vec![AbiParam::new(from_ty)],
+                vec![AbiParam::new(types::I128)],
+                &[from],
+            )[0]
         } else if to_ty == types::I8 || to_ty == types::I16 {
             // FIXME implement fcvt_to_*int_sat.i8/i16
             let val = if to_signed {

compiler/rustc_codegen_cranelift/src/compiler_builtins.rs

@@ -44,6 +44,7 @@ builtin_functions! {
     fn __umodti3(n: u128, d: u128) -> u128;
     fn __modti3(n: i128, d: i128) -> i128;
     fn __rust_u128_mulo(a: u128, b: u128, oflow: &mut i32) -> u128;
+    fn __rust_i128_mulo(a: i128, b: i128, oflow: &mut i32) -> i128;
 
     // floats
     fn __floattisf(i: i128) -> f32;

compiler/rustc_codegen_cranelift/src/debuginfo/object.rs

@@ -73,16 +73,19 @@ impl WriteDebugInfo for ObjectProduct {
             }
         };
         self.object
-            .add_relocation(from.0, Relocation {
-                offset: u64::from(reloc.offset),
-                symbol,
-                flags: RelocationFlags::Generic {
-                    kind: reloc.kind,
-                    encoding: RelocationEncoding::Generic,
-                    size: reloc.size * 8,
+            .add_relocation(
+                from.0,
+                Relocation {
+                    offset: u64::from(reloc.offset),
+                    symbol,
+                    flags: RelocationFlags::Generic {
+                        kind: reloc.kind,
+                        encoding: RelocationEncoding::Generic,
+                        size: reloc.size * 8,
+                    },
+                    addend: i64::try_from(symbol_offset).unwrap() + reloc.addend,
                 },
-                addend: i64::try_from(symbol_offset).unwrap() + reloc.addend,
-            })
+            )
             .unwrap();
     }
 }

compiler/rustc_codegen_cranelift/src/driver/jit.rs

@@ -46,7 +46,7 @@ unsafe impl Send for UnsafeMessage {}
 
 impl UnsafeMessage {
     /// Send the message.
-    fn send(self) -> Result<(), mpsc::SendError<UnsafeMessage>> {
+    fn send(self) {
         thread_local! {
             /// The Sender owned by the local thread
             static LOCAL_MESSAGE_SENDER: mpsc::Sender<UnsafeMessage> =
@@ -55,7 +55,9 @@ impl UnsafeMessage {
                     .lock().unwrap()
                     .clone();
         }
-        LOCAL_MESSAGE_SENDER.with(|sender| sender.send(self))
+        LOCAL_MESSAGE_SENDER.with(|sender| {
+            sender.send(self).expect("rustc thread hung up before lazy JIT request was sent")
+        })
     }
 }
 
@@ -90,7 +92,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, codegen_mode: CodegenMode, jit_args: Vec<
         create_jit_module(tcx, matches!(codegen_mode, CodegenMode::JitLazy));
     let mut cached_context = Context::new();
 
-    let (_, cgus) = tcx.collect_and_partition_mono_items(());
+    let cgus = tcx.collect_and_partition_mono_items(()).codegen_units;
     let mono_items = cgus
         .iter()
         .map(|cgu| cgu.items_in_deterministic_order(tcx).into_iter())
@@ -231,9 +233,7 @@ extern "C" fn clif_jit_fn(
 ) -> *const u8 {
     // send the JIT request to the rustc thread, with a channel for the response
     let (tx, rx) = mpsc::channel();
-    UnsafeMessage::JitFn { instance_ptr, trampoline_ptr, tx }
-        .send()
-        .expect("rustc thread hung up before lazy JIT request was sent");
+    UnsafeMessage::JitFn { instance_ptr, trampoline_ptr, tx }.send();
 
     // block on JIT compilation result
     rx.recv().expect("rustc thread hung up before responding to sent lazy JIT request")
@@ -342,11 +342,15 @@ fn codegen_shim<'tcx>(
     let instance_ptr = Box::into_raw(Box::new(inst));
 
     let jit_fn = module
-        .declare_function("__clif_jit_fn", Linkage::Import, &Signature {
-            call_conv: module.target_config().default_call_conv,
-            params: vec![AbiParam::new(pointer_type), AbiParam::new(pointer_type)],
-            returns: vec![AbiParam::new(pointer_type)],
-        })
+        .declare_function(
+            "__clif_jit_fn",
+            Linkage::Import,
+            &Signature {
+                call_conv: module.target_config().default_call_conv,
+                params: vec![AbiParam::new(pointer_type), AbiParam::new(pointer_type)],
+                returns: vec![AbiParam::new(pointer_type)],
+            },
+        )
         .unwrap();
 
     let context = cached_context;

compiler/rustc_codegen_cranelift/src/inline_asm.rs

@@ -875,11 +875,15 @@ fn call_inline_asm<'tcx>(
 
     let inline_asm_func = fx
         .module
-        .declare_function(asm_name, Linkage::Import, &Signature {
-            call_conv: CallConv::SystemV,
-            params: vec![AbiParam::new(fx.pointer_type)],
-            returns: vec![],
-        })
+        .declare_function(
+            asm_name,
+            Linkage::Import,
+            &Signature {
+                call_conv: CallConv::SystemV,
+                params: vec![AbiParam::new(fx.pointer_type)],
+                returns: vec![],
+            },
+        )
         .unwrap();
     let inline_asm_func = fx.module.declare_func_in_func(inline_asm_func, fx.bcx.func);
     if fx.clif_comments.enabled() {

compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs

@@ -17,6 +17,14 @@ pub(crate) fn codegen_aarch64_llvm_intrinsic_call<'tcx>(
             fx.bcx.ins().fence();
         }
 
+        "llvm.aarch64.neon.ld1x4.v16i8.p0i8" => {
+            intrinsic_args!(fx, args => (ptr); intrinsic);
+
+            let ptr = ptr.load_scalar(fx);
+            let val = CPlace::for_ptr(Pointer::new(ptr), ret.layout()).to_cvalue(fx);
+            ret.write_cvalue(fx, val);
+        }
+
         _ if intrinsic.starts_with("llvm.aarch64.neon.abs.v") => {
             intrinsic_args!(fx, args => (a); intrinsic);
 
@@ -115,6 +123,22 @@ pub(crate) fn codegen_aarch64_llvm_intrinsic_call<'tcx>(
             );
         }
 
+        "llvm.aarch64.neon.uaddlv.i32.v16i8" => {
+            intrinsic_args!(fx, args => (v); intrinsic);
+
+            let mut res_val = fx.bcx.ins().iconst(types::I16, 0);
+            for lane_idx in 0..16 {
+                let lane = v.value_lane(fx, lane_idx).load_scalar(fx);
+                let lane = fx.bcx.ins().uextend(types::I16, lane);
+                res_val = fx.bcx.ins().iadd(res_val, lane);
+            }
+            let res = CValue::by_val(
+                fx.bcx.ins().uextend(types::I32, res_val),
+                fx.layout_of(fx.tcx.types.u32),
+            );
+            ret.write_cvalue(fx, res);
+        }
+
         _ if intrinsic.starts_with("llvm.aarch64.neon.faddv.f32.v") => {
             intrinsic_args!(fx, args => (v); intrinsic);
 

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -558,9 +558,12 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                     (sym::simd_round, types::F64) => "round",
                     _ => unreachable!("{:?}", intrinsic),
                 };
-                fx.lib_call(name, vec![AbiParam::new(lane_ty)], vec![AbiParam::new(lane_ty)], &[
-                    lane,
-                ])[0]
+                fx.lib_call(
+                    name,
+                    vec![AbiParam::new(lane_ty)],
+                    vec![AbiParam::new(lane_ty)],
+                    &[lane],
+                )[0]
             });
         }
 

compiler/rustc_codegen_cranelift/src/lib.rs

@@ -183,8 +183,8 @@ impl CodegenBackend for CraneliftCodegenBackend {
     ) -> Vec<rustc_span::Symbol> {
         // FIXME return the actually used target features. this is necessary for #[cfg(target_feature)]
         if sess.target.arch == "x86_64" && sess.target.os != "none" {
-            // x86_64 mandates SSE2 support
-            vec![sym::fsxr, sym::sse, sym::sse2]
+            // x86_64 mandates SSE2 support and rustc requires the x87 feature to be enabled
+            vec![sym::fsxr, sym::sse, sym::sse2, Symbol::intern("x87")]
         } else if sess.target.arch == "aarch64" {
             match &*sess.target.os {
                 "none" => vec![],
@@ -209,7 +209,6 @@ impl CodegenBackend for CraneliftCodegenBackend {
         metadata: EncodedMetadata,
         need_metadata_module: bool,
     ) -> Box<dyn Any> {
-        tcx.dcx().abort_if_errors();
         info!("codegen crate {}", tcx.crate_name(LOCAL_CRATE));
         let config = self.config.clone().unwrap_or_else(|| {
             BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)

compiler/rustc_codegen_cranelift/src/main_shim.rs

@@ -15,9 +15,12 @@ pub(crate) fn maybe_create_entry_wrapper(
     is_primary_cgu: bool,
 ) {
     let (main_def_id, sigpipe) = match tcx.entry_fn(()) {
-        Some((def_id, entry_ty)) => (def_id, match entry_ty {
-            EntryFnType::Main { sigpipe } => sigpipe,
-        }),
+        Some((def_id, entry_ty)) => (
+            def_id,
+            match entry_ty {
+                EntryFnType::Main { sigpipe } => sigpipe,
+            },
+        ),
         None => return,
     };
 

compiler/rustc_codegen_cranelift/src/trap.rs

@@ -5,11 +5,15 @@ use crate::prelude::*;
 fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
     let puts = fx
         .module
-        .declare_function("puts", Linkage::Import, &Signature {
-            call_conv: fx.target_config.default_call_conv,
-            params: vec![AbiParam::new(fx.pointer_type)],
-            returns: vec![AbiParam::new(types::I32)],
-        })
+        .declare_function(
+            "puts",
+            Linkage::Import,
+            &Signature {
+                call_conv: fx.target_config.default_call_conv,
+                params: vec![AbiParam::new(fx.pointer_type)],
+                returns: vec![AbiParam::new(types::I32)],
+            },
+        )
         .unwrap();
     let puts = fx.module.declare_func_in_func(puts, &mut fx.bcx.func);
     if fx.clif_comments.enabled() {

compiler/rustc_codegen_gcc/src/abi.rs

@@ -1,6 +1,7 @@
 #[cfg(feature = "master")]
 use gccjit::FnAttribute;
 use gccjit::{ToLValue, ToRValue, Type};
+use rustc_abi::{Reg, RegKind};
 use rustc_codegen_ssa::traits::{AbiBuilderMethods, BaseTypeCodegenMethods};
 use rustc_data_structures::fx::FxHashSet;
 use rustc_middle::bug;
@@ -8,7 +9,7 @@ use rustc_middle::ty::Ty;
 use rustc_middle::ty::layout::LayoutOf;
 #[cfg(feature = "master")]
 use rustc_session::config;
-use rustc_target::callconv::{ArgAttributes, CastTarget, FnAbi, PassMode, Reg, RegKind};
+use rustc_target::callconv::{ArgAttributes, CastTarget, FnAbi, PassMode};
 
 use crate::builder::Builder;
 use crate::context::CodegenCx;

compiler/rustc_codegen_gcc/src/base.rs

@@ -49,9 +49,7 @@ pub fn global_linkage_to_gcc(linkage: Linkage) -> GlobalKind {
         Linkage::LinkOnceODR => unimplemented!(),
         Linkage::WeakAny => unimplemented!(),
         Linkage::WeakODR => unimplemented!(),
-        Linkage::Appending => unimplemented!(),
         Linkage::Internal => GlobalKind::Internal,
-        Linkage::Private => GlobalKind::Internal,
         Linkage::ExternalWeak => GlobalKind::Imported, // TODO(antoyo): should be weak linkage.
         Linkage::Common => unimplemented!(),
     }
@@ -66,9 +64,7 @@ pub fn linkage_to_gcc(linkage: Linkage) -> FunctionType {
         Linkage::LinkOnceODR => unimplemented!(),
         Linkage::WeakAny => FunctionType::Exported, // FIXME(antoyo): should be similar to linkonce.
         Linkage::WeakODR => unimplemented!(),
-        Linkage::Appending => unimplemented!(),
         Linkage::Internal => FunctionType::Internal,
-        Linkage::Private => FunctionType::Internal,
         Linkage::ExternalWeak => unimplemented!(),
         Linkage::Common => unimplemented!(),
     }

compiler/rustc_codegen_gcc/src/builder.rs

@@ -155,14 +155,11 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
         // NOTE: not sure why, but we have the wrong type here.
         let int_type = compare_exchange.get_param(2).to_rvalue().get_type();
         let src = self.context.new_bitcast(self.location, src, int_type);
-        self.context.new_call(self.location, compare_exchange, &[
-            dst,
-            expected,
-            src,
-            weak,
-            order,
-            failure_order,
-        ])
+        self.context.new_call(
+            self.location,
+            compare_exchange,
+            &[dst, expected, src, weak, order, failure_order],
+        )
     }
 
     pub fn assign(&self, lvalue: LValue<'gcc>, value: RValue<'gcc>) {
@@ -1076,9 +1073,11 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         let align = dest.val.align.restrict_for_offset(dest.layout.field(self.cx(), 0).size);
         cg_elem.val.store(self, PlaceRef::new_sized_aligned(current_val, cg_elem.layout, align));
 
-        let next = self.inbounds_gep(self.backend_type(cg_elem.layout), current.to_rvalue(), &[
-            self.const_usize(1),
-        ]);
+        let next = self.inbounds_gep(
+            self.backend_type(cg_elem.layout),
+            current.to_rvalue(),
+            &[self.const_usize(1)],
+        );
         self.llbb().add_assignment(self.location, current, next);
         self.br(header_bb);
 

compiler/rustc_codegen_gcc/src/intrinsic/llvm.rs

@@ -687,11 +687,12 @@ pub fn adjust_intrinsic_return_value<'a, 'gcc, 'tcx>(
                 let field2 = builder.context.new_field(None, args[1].get_type(), "carryResult");
                 let struct_type =
                     builder.context.new_struct_type(None, "addcarryResult", &[field1, field2]);
-                return_value =
-                    builder.context.new_struct_constructor(None, struct_type.as_type(), None, &[
-                        return_value,
-                        last_arg.dereference(None).to_rvalue(),
-                    ]);
+                return_value = builder.context.new_struct_constructor(
+                    None,
+                    struct_type.as_type(),
+                    None,
+                    &[return_value, last_arg.dereference(None).to_rvalue()],
+                );
             }
         }
         "__builtin_ia32_stmxcsr" => {
@@ -716,11 +717,12 @@ pub fn adjust_intrinsic_return_value<'a, 'gcc, 'tcx>(
             let field2 = builder.context.new_field(None, return_value.get_type(), "success");
             let struct_type =
                 builder.context.new_struct_type(None, "rdrand_result", &[field1, field2]);
-            return_value =
-                builder.context.new_struct_constructor(None, struct_type.as_type(), None, &[
-                    random_number,
-                    success_variable.to_rvalue(),
-                ]);
+            return_value = builder.context.new_struct_constructor(
+                None,
+                struct_type.as_type(),
+                None,
+                &[random_number, success_variable.to_rvalue()],
+            );
         }
         "fma" => {
             let f16_type = builder.context.new_c_type(CType::Float16);

compiler/rustc_codegen_gcc/src/intrinsic/simd.rs

@@ -62,11 +62,10 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
     let arg_tys = sig.inputs();
 
     if name == sym::simd_select_bitmask {
-        require_simd!(arg_tys[1], InvalidMonomorphization::SimdArgument {
-            span,
-            name,
-            ty: arg_tys[1]
-        });
+        require_simd!(
+            arg_tys[1],
+            InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] }
+        );
         let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
 
         let expected_int_bits = (len.max(8) - 1).next_power_of_two();
@@ -140,14 +139,17 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
 
         let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx());
-        require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType {
-            span,
-            name,
-            in_len,
-            in_ty,
-            ret_ty,
-            out_len
-        });
+        require!(
+            in_len == out_len,
+            InvalidMonomorphization::ReturnLengthInputType {
+                span,
+                name,
+                in_len,
+                in_ty,
+                ret_ty,
+                out_len
+            }
+        );
         require!(
             bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
             InvalidMonomorphization::ReturnIntegerType { span, name, ret_ty, out_ty }
@@ -269,23 +271,17 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         let lo_nibble =
             bx.context.new_rvalue_from_vector(None, long_byte_vector_type, &lo_nibble_elements);
 
-        let mask = bx.context.new_rvalue_from_vector(None, long_byte_vector_type, &vec![
-            bx.context
-                .new_rvalue_from_int(
-                    bx.u8_type, 0x0f
-                );
-            byte_vector_type_size
-                as _
-        ]);
+        let mask = bx.context.new_rvalue_from_vector(
+            None,
+            long_byte_vector_type,
+            &vec![bx.context.new_rvalue_from_int(bx.u8_type, 0x0f); byte_vector_type_size as _],
+        );
 
-        let four_vec = bx.context.new_rvalue_from_vector(None, long_byte_vector_type, &vec![
-                bx.context
-                    .new_rvalue_from_int(
-                        bx.u8_type, 4
-                    );
-                byte_vector_type_size
-                    as _
-            ]);
+        let four_vec = bx.context.new_rvalue_from_vector(
+            None,
+            long_byte_vector_type,
+            &vec![bx.context.new_rvalue_from_int(bx.u8_type, 4); byte_vector_type_size as _],
+        );
 
         // Step 2: Byte-swap the input.
         let swapped = simd_bswap(bx, args[0].immediate());
@@ -388,21 +384,14 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
 
         let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx());
-        require!(out_len == n, InvalidMonomorphization::ReturnLength {
-            span,
-            name,
-            in_len: n,
-            ret_ty,
-            out_len
-        });
-        require!(in_elem == out_ty, InvalidMonomorphization::ReturnElement {
-            span,
-            name,
-            in_elem,
-            in_ty,
-            ret_ty,
-            out_ty
-        });
+        require!(
+            out_len == n,
+            InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
+        );
+        require!(
+            in_elem == out_ty,
+            InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
+        );
 
         let vector = args[2].immediate();
 
@@ -411,13 +400,16 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
 
     #[cfg(feature = "master")]
     if name == sym::simd_insert {
-        require!(in_elem == arg_tys[2], InvalidMonomorphization::InsertedType {
-            span,
-            name,
-            in_elem,
-            in_ty,
-            out_ty: arg_tys[2]
-        });
+        require!(
+            in_elem == arg_tys[2],
+            InvalidMonomorphization::InsertedType {
+                span,
+                name,
+                in_elem,
+                in_ty,
+                out_ty: arg_tys[2]
+            }
+        );
         let vector = args[0].immediate();
         let index = args[1].immediate();
         let value = args[2].immediate();
@@ -431,13 +423,10 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
 
     #[cfg(feature = "master")]
     if name == sym::simd_extract {
-        require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType {
-            span,
-            name,
-            in_elem,
-            in_ty,
-            ret_ty
-        });
+        require!(
+            ret_ty == in_elem,
+            InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
+        );
         let vector = args[0].immediate();
         return Ok(bx.context.new_vector_access(None, vector, args[1].immediate()).to_rvalue());
     }
@@ -445,18 +434,15 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
     if name == sym::simd_select {
         let m_elem_ty = in_elem;
         let m_len = in_len;
-        require_simd!(arg_tys[1], InvalidMonomorphization::SimdArgument {
-            span,
-            name,
-            ty: arg_tys[1]
-        });
+        require_simd!(
+            arg_tys[1],
+            InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] }
+        );
         let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
-        require!(m_len == v_len, InvalidMonomorphization::MismatchedLengths {
-            span,
-            name,
-            m_len,
-            v_len
-        });
+        require!(
+            m_len == v_len,
+            InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
+        );
         match *m_elem_ty.kind() {
             ty::Int(_) => {}
             _ => return_error!(InvalidMonomorphization::MaskType { span, name, ty: m_elem_ty }),
@@ -468,25 +454,27 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
         let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
 
-        require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType {
-            span,
-            name,
-            in_len,
-            in_ty,
-            ret_ty,
-            out_len
-        });
+        require!(
+            in_len == out_len,
+            InvalidMonomorphization::ReturnLengthInputType {
+                span,
+                name,
+                in_len,
+                in_ty,
+                ret_ty,
+                out_len
+            }
+        );
 
         match *in_elem.kind() {
             ty::RawPtr(p_ty, _) => {
                 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
                     bx.tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), ty)
                 });
-                require!(metadata.is_unit(), InvalidMonomorphization::CastWidePointer {
-                    span,
-                    name,
-                    ty: in_elem
-                });
+                require!(
+                    metadata.is_unit(),
+                    InvalidMonomorphization::CastWidePointer { span, name, ty: in_elem }
+                );
             }
             _ => {
                 return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
@@ -497,11 +485,10 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
                 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
                     bx.tcx.normalize_erasing_regions(ty::TypingEnv::fully_monomorphized(), ty)
                 });
-                require!(metadata.is_unit(), InvalidMonomorphization::CastWidePointer {
-                    span,
-                    name,
-                    ty: out_elem
-                });
+                require!(
+                    metadata.is_unit(),
+                    InvalidMonomorphization::CastWidePointer { span, name, ty: out_elem }
+                );
             }
             _ => {
                 return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
@@ -524,14 +511,17 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
         let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
 
-        require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType {
-            span,
-            name,
-            in_len,
-            in_ty,
-            ret_ty,
-            out_len
-        });
+        require!(
+            in_len == out_len,
+            InvalidMonomorphization::ReturnLengthInputType {
+                span,
+                name,
+                in_len,
+                in_ty,
+                ret_ty,
+                out_len
+            }
+        );
 
         match *in_elem.kind() {
             ty::RawPtr(_, _) => {}
@@ -560,14 +550,17 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
         let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
 
-        require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType {
-            span,
-            name,
-            in_len,
-            in_ty,
-            ret_ty,
-            out_len
-        });
+        require!(
+            in_len == out_len,
+            InvalidMonomorphization::ReturnLengthInputType {
+                span,
+                name,
+                in_len,
+                in_ty,
+                ret_ty,
+                out_len
+            }
+        );
 
         match *in_elem.kind() {
             ty::Uint(ty::UintTy::Usize) => {}
@@ -596,14 +589,17 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
     if name == sym::simd_cast || name == sym::simd_as {
         require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
         let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
-        require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType {
-            span,
-            name,
-            in_len,
-            in_ty,
-            ret_ty,
-            out_len
-        });
+        require!(
+            in_len == out_len,
+            InvalidMonomorphization::ReturnLengthInputType {
+                span,
+                name,
+                in_len,
+                in_ty,
+                ret_ty,
+                out_len
+            }
+        );
         // casting cares about nominal type, not just structural type
         if in_elem == out_elem {
             return Ok(args[0].immediate());
@@ -629,14 +625,17 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
 
         match (in_style, out_style) {
             (Style::Unsupported, Style::Unsupported) => {
-                require!(false, InvalidMonomorphization::UnsupportedCast {
-                    span,
-                    name,
-                    in_ty,
-                    in_elem,
-                    ret_ty,
-                    out_elem
-                });
+                require!(
+                    false,
+                    InvalidMonomorphization::UnsupportedCast {
+                        span,
+                        name,
+                        in_ty,
+                        in_elem,
+                        ret_ty,
+                        out_elem
+                    }
+                );
             }
             _ => return Ok(bx.context.convert_vector(None, args[0].immediate(), llret_ty)),
         }
@@ -914,45 +913,47 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
 
         // All types must be simd vector types
         require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty });
-        require_simd!(arg_tys[1], InvalidMonomorphization::SimdSecond {
-            span,
-            name,
-            ty: arg_tys[1]
-        });
-        require_simd!(arg_tys[2], InvalidMonomorphization::SimdThird {
-            span,
-            name,
-            ty: arg_tys[2]
-        });
+        require_simd!(
+            arg_tys[1],
+            InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] }
+        );
+        require_simd!(
+            arg_tys[2],
+            InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] }
+        );
         require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
 
         // Of the same length:
         let (out_len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
         let (out_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx());
-        require!(in_len == out_len, InvalidMonomorphization::SecondArgumentLength {
-            span,
-            name,
-            in_len,
-            in_ty,
-            arg_ty: arg_tys[1],
-            out_len
-        });
-        require!(in_len == out_len2, InvalidMonomorphization::ThirdArgumentLength {
-            span,
-            name,
-            in_len,
-            in_ty,
-            arg_ty: arg_tys[2],
-            out_len: out_len2
-        });
+        require!(
+            in_len == out_len,
+            InvalidMonomorphization::SecondArgumentLength {
+                span,
+                name,
+                in_len,
+                in_ty,
+                arg_ty: arg_tys[1],
+                out_len
+            }
+        );
+        require!(
+            in_len == out_len2,
+            InvalidMonomorphization::ThirdArgumentLength {
+                span,
+                name,
+                in_len,
+                in_ty,
+                arg_ty: arg_tys[2],
+                out_len: out_len2
+            }
+        );
 
         // The return type must match the first argument type
-        require!(ret_ty == in_ty, InvalidMonomorphization::ExpectedReturnType {
-            span,
-            name,
-            in_ty,
-            ret_ty
-        });
+        require!(
+            ret_ty == in_ty,
+            InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty }
+        );
 
         // This counts how many pointers
         fn ptr_count(t: Ty<'_>) -> usize {
@@ -979,15 +980,18 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
                 (ptr_count(element_ty1), non_ptr(element_ty1))
             }
             _ => {
-                require!(false, InvalidMonomorphization::ExpectedElementType {
-                    span,
-                    name,
-                    expected_element: element_ty1,
-                    second_arg: arg_tys[1],
-                    in_elem,
-                    in_ty,
-                    mutability: ExpectedPointerMutability::Not,
-                });
+                require!(
+                    false,
+                    InvalidMonomorphization::ExpectedElementType {
+                        span,
+                        name,
+                        expected_element: element_ty1,
+                        second_arg: arg_tys[1],
+                        in_elem,
+                        in_ty,
+                        mutability: ExpectedPointerMutability::Not,
+                    }
+                );
                 unreachable!();
             }
         };
@@ -1000,12 +1004,15 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         match *element_ty2.kind() {
             ty::Int(_) => (),
             _ => {
-                require!(false, InvalidMonomorphization::ThirdArgElementType {
-                    span,
-                    name,
-                    expected_element: element_ty2,
-                    third_arg: arg_tys[2]
-                });
+                require!(
+                    false,
+                    InvalidMonomorphization::ThirdArgElementType {
+                        span,
+                        name,
+                        expected_element: element_ty2,
+                        third_arg: arg_tys[2]
+                    }
+                );
             }
         }
 
@@ -1029,36 +1036,40 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
 
         // All types must be simd vector types
         require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty });
-        require_simd!(arg_tys[1], InvalidMonomorphization::SimdSecond {
-            span,
-            name,
-            ty: arg_tys[1]
-        });
-        require_simd!(arg_tys[2], InvalidMonomorphization::SimdThird {
-            span,
-            name,
-            ty: arg_tys[2]
-        });
+        require_simd!(
+            arg_tys[1],
+            InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] }
+        );
+        require_simd!(
+            arg_tys[2],
+            InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] }
+        );
 
         // Of the same length:
         let (element_len1, _) = arg_tys[1].simd_size_and_type(bx.tcx());
         let (element_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx());
-        require!(in_len == element_len1, InvalidMonomorphization::SecondArgumentLength {
-            span,
-            name,
-            in_len,
-            in_ty,
-            arg_ty: arg_tys[1],
-            out_len: element_len1
-        });
-        require!(in_len == element_len2, InvalidMonomorphization::ThirdArgumentLength {
-            span,
-            name,
-            in_len,
-            in_ty,
-            arg_ty: arg_tys[2],
-            out_len: element_len2
-        });
+        require!(
+            in_len == element_len1,
+            InvalidMonomorphization::SecondArgumentLength {
+                span,
+                name,
+                in_len,
+                in_ty,
+                arg_ty: arg_tys[1],
+                out_len: element_len1
+            }
+        );
+        require!(
+            in_len == element_len2,
+            InvalidMonomorphization::ThirdArgumentLength {
+                span,
+                name,
+                in_len,
+                in_ty,
+                arg_ty: arg_tys[2],
+                out_len: element_len2
+            }
+        );
 
         // This counts how many pointers
         fn ptr_count(t: Ty<'_>) -> usize {
@@ -1086,15 +1097,18 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
                 (ptr_count(element_ty1), non_ptr(element_ty1))
             }
             _ => {
-                require!(false, InvalidMonomorphization::ExpectedElementType {
-                    span,
-                    name,
-                    expected_element: element_ty1,
-                    second_arg: arg_tys[1],
-                    in_elem,
-                    in_ty,
-                    mutability: ExpectedPointerMutability::Mut,
-                });
+                require!(
+                    false,
+                    InvalidMonomorphization::ExpectedElementType {
+                        span,
+                        name,
+                        expected_element: element_ty1,
+                        second_arg: arg_tys[1],
+                        in_elem,
+                        in_ty,
+                        mutability: ExpectedPointerMutability::Mut,
+                    }
+                );
                 unreachable!();
             }
         };
@@ -1106,12 +1120,15 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         match *element_ty2.kind() {
             ty::Int(_) => (),
             _ => {
-                require!(false, InvalidMonomorphization::ThirdArgElementType {
-                    span,
-                    name,
-                    expected_element: element_ty2,
-                    third_arg: arg_tys[2]
-                });
+                require!(
+                    false,
+                    InvalidMonomorphization::ThirdArgElementType {
+                        span,
+                        name,
+                        expected_element: element_ty2,
+                        third_arg: arg_tys[2]
+                    }
+                );
             }
         }
 
@@ -1278,13 +1295,10 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         ($name:ident : $vec_op:expr, $float_reduce:ident, $ordered:expr, $op:ident,
          $identity:expr) => {
             if name == sym::$name {
-                require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType {
-                    span,
-                    name,
-                    in_elem,
-                    in_ty,
-                    ret_ty
-                });
+                require!(
+                    ret_ty == in_elem,
+                    InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
+                );
                 return match *in_elem.kind() {
                     ty::Int(_) | ty::Uint(_) => {
                         let r = bx.vector_reduce_op(args[0].immediate(), $vec_op);
@@ -1350,13 +1364,10 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
     macro_rules! minmax_red {
         ($name:ident: $int_red:ident, $float_red:ident) => {
             if name == sym::$name {
-                require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType {
-                    span,
-                    name,
-                    in_elem,
-                    in_ty,
-                    ret_ty
-                });
+                require!(
+                    ret_ty == in_elem,
+                    InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
+                );
                 return match *in_elem.kind() {
                     ty::Int(_) | ty::Uint(_) => Ok(bx.$int_red(args[0].immediate())),
                     ty::Float(_) => Ok(bx.$float_red(args[0].immediate())),
@@ -1380,13 +1391,10 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
         ($name:ident : $op:expr, $boolean:expr) => {
             if name == sym::$name {
                 let input = if !$boolean {
-                    require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType {
-                        span,
-                        name,
-                        in_elem,
-                        in_ty,
-                        ret_ty
-                    });
+                    require!(
+                        ret_ty == in_elem,
+                        InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
+                    );
                     args[0].immediate()
                 } else {
                     match *in_elem.kind() {

compiler/rustc_codegen_gcc/src/mono_item.rs

@@ -61,10 +61,7 @@ impl<'gcc, 'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
         // compiler-rt, then we want to implicitly compile everything with hidden
         // visibility as we're going to link this object all over the place but
         // don't want the symbols to get exported.
-        if linkage != Linkage::Internal
-            && linkage != Linkage::Private
-            && self.tcx.is_compiler_builtins(LOCAL_CRATE)
-        {
+        if linkage != Linkage::Internal && self.tcx.is_compiler_builtins(LOCAL_CRATE) {
             #[cfg(feature = "master")]
             decl.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Hidden));
         } else {

compiler/rustc_codegen_llvm/src/abi.rs

@@ -4,8 +4,7 @@ use std::cmp;
 use libc::c_uint;
 use rustc_abi as abi;
 pub(crate) use rustc_abi::ExternAbi;
-use rustc_abi::Primitive::Int;
-use rustc_abi::{HasDataLayout, Size};
+use rustc_abi::{HasDataLayout, Primitive, Reg, RegKind, Size};
 use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
 use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue};
@@ -440,7 +439,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         let apply_range_attr = |idx: AttributePlace, scalar: rustc_abi::Scalar| {
             if cx.sess().opts.optimize != config::OptLevel::No
                 && llvm_util::get_version() >= (19, 0, 0)
-                && matches!(scalar.primitive(), Int(..))
+                && matches!(scalar.primitive(), Primitive::Int(..))
                 // If the value is a boolean, the range is 0..2 and that ultimately
                 // become 0..0 when the type becomes i1, which would be rejected
                 // by the LLVM verifier.
@@ -448,11 +447,11 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                 // LLVM also rejects full range.
                 && !scalar.is_always_valid(cx)
             {
-                attributes::apply_to_llfn(llfn, idx, &[llvm::CreateRangeAttr(
-                    cx.llcx,
-                    scalar.size(cx),
-                    scalar.valid_range(cx),
-                )]);
+                attributes::apply_to_llfn(
+                    llfn,
+                    idx,
+                    &[llvm::CreateRangeAttr(cx.llcx, scalar.size(cx), scalar.valid_range(cx))],
+                );
             }
         };
 
@@ -472,10 +471,14 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                 );
                 attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]);
                 if cx.sess().opts.optimize != config::OptLevel::No {
-                    attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[
-                        llvm::AttributeKind::Writable.create_attr(cx.llcx),
-                        llvm::AttributeKind::DeadOnUnwind.create_attr(cx.llcx),
-                    ]);
+                    attributes::apply_to_llfn(
+                        llfn,
+                        llvm::AttributePlace::Argument(i),
+                        &[
+                            llvm::AttributeKind::Writable.create_attr(cx.llcx),
+                            llvm::AttributeKind::DeadOnUnwind.create_attr(cx.llcx),
+                        ],
+                    );
                 }
             }
             PassMode::Cast { cast, pad_i32: _ } => {
@@ -574,7 +577,7 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         if bx.cx.sess().opts.optimize != config::OptLevel::No
                 && llvm_util::get_version() < (19, 0, 0)
                 && let abi::BackendRepr::Scalar(scalar) = self.ret.layout.backend_repr
-                && matches!(scalar.primitive(), Int(..))
+                && matches!(scalar.primitive(), Primitive::Int(..))
                 // If the value is a boolean, the range is 0..2 and that ultimately
                 // become 0..0 when the type becomes i1, which would be rejected
                 // by the LLVM verifier.
@@ -593,9 +596,11 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                         bx.cx.llcx,
                         bx.cx.type_array(bx.cx.type_i8(), arg.layout.size.bytes()),
                     );
-                    attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[
-                        byval,
-                    ]);
+                    attributes::apply_to_callsite(
+                        callsite,
+                        llvm::AttributePlace::Argument(i),
+                        &[byval],
+                    );
                 }
                 PassMode::Direct(attrs)
                 | PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
@@ -627,9 +632,11 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             // This will probably get ignored on all targets but those supporting the TrustZone-M
             // extension (thumbv8m targets).
             let cmse_nonsecure_call = llvm::CreateAttrString(bx.cx.llcx, "cmse_nonsecure_call");
-            attributes::apply_to_callsite(callsite, llvm::AttributePlace::Function, &[
-                cmse_nonsecure_call,
-            ]);
+            attributes::apply_to_callsite(
+                callsite,
+                llvm::AttributePlace::Function,
+                &[cmse_nonsecure_call],
+            );
         }
 
         // Some intrinsics require that an elementtype attribute (with the pointee type of a

compiler/rustc_codegen_llvm/src/back/lto.rs

@@ -606,10 +606,31 @@ pub(crate) fn run_pass_manager(
 
     // If this rustc version was build with enzyme/autodiff enabled, and if users applied the
     // `#[autodiff]` macro at least once, then we will later call llvm_optimize a second time.
-    let first_run = true;
     debug!("running llvm pm opt pipeline");
     unsafe {
-        write::llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage, first_run)?;
+        write::llvm_optimize(
+            cgcx,
+            dcx,
+            module,
+            config,
+            opt_level,
+            opt_stage,
+            write::AutodiffStage::DuringAD,
+        )?;
+    }
+    // FIXME(ZuseZ4): Make this more granular
+    if cfg!(llvm_enzyme) && !thin {
+        unsafe {
+            write::llvm_optimize(
+                cgcx,
+                dcx,
+                module,
+                config,
+                opt_level,
+                llvm::OptStage::FatLTO,
+                write::AutodiffStage::PostAD,
+            )?;
+        }
     }
     debug!("lto done");
     Ok(())

compiler/rustc_codegen_llvm/src/back/write.rs

@@ -530,6 +530,16 @@ fn get_instr_profile_output_path(config: &ModuleConfig) -> Option<CString> {
     config.instrument_coverage.then(|| c"default_%m_%p.profraw".to_owned())
 }
 
+// PreAD will run llvm opts but disable size increasing opts (vectorization, loop unrolling)
+// DuringAD is the same as above, but also runs the enzyme opt and autodiff passes.
+// PostAD will run all opts, including size increasing opts.
+#[derive(Debug, Eq, PartialEq)]
+pub(crate) enum AutodiffStage {
+    PreAD,
+    DuringAD,
+    PostAD,
+}
+
 pub(crate) unsafe fn llvm_optimize(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
     dcx: DiagCtxtHandle<'_>,
@@ -537,7 +547,7 @@ pub(crate) unsafe fn llvm_optimize(
     config: &ModuleConfig,
     opt_level: config::OptLevel,
     opt_stage: llvm::OptStage,
-    skip_size_increasing_opts: bool,
+    autodiff_stage: AutodiffStage,
 ) -> Result<(), FatalError> {
     // Enzyme:
     // The whole point of compiler based AD is to differentiate optimized IR instead of unoptimized
@@ -550,12 +560,16 @@ pub(crate) unsafe fn llvm_optimize(
     let unroll_loops;
     let vectorize_slp;
     let vectorize_loop;
+    let run_enzyme = cfg!(llvm_enzyme) && autodiff_stage == AutodiffStage::DuringAD;
 
     // When we build rustc with enzyme/autodiff support, we want to postpone size-increasing
-    // optimizations until after differentiation. FIXME(ZuseZ4): Before shipping on nightly,
+    // optimizations until after differentiation. Our pipeline is thus: (opt + enzyme), (full opt).
+    // We therefore have two calls to llvm_optimize, if autodiff is used.
+    //
+    // FIXME(ZuseZ4): Before shipping on nightly,
     // we should make this more granular, or at least check that the user has at least one autodiff
     // call in their code, to justify altering the compilation pipeline.
-    if skip_size_increasing_opts && cfg!(llvm_enzyme) {
+    if cfg!(llvm_enzyme) && autodiff_stage != AutodiffStage::PostAD {
         unroll_loops = false;
         vectorize_slp = false;
         vectorize_loop = false;
@@ -565,7 +579,7 @@ pub(crate) unsafe fn llvm_optimize(
         vectorize_slp = config.vectorize_slp;
         vectorize_loop = config.vectorize_loop;
     }
-    trace!(?unroll_loops, ?vectorize_slp, ?vectorize_loop);
+    trace!(?unroll_loops, ?vectorize_slp, ?vectorize_loop, ?run_enzyme);
     let using_thin_buffers = opt_stage == llvm::OptStage::PreLinkThinLTO || config.bitcode_needed();
     let pgo_gen_path = get_pgo_gen_path(config);
     let pgo_use_path = get_pgo_use_path(config);
@@ -633,6 +647,7 @@ pub(crate) unsafe fn llvm_optimize(
             vectorize_loop,
             config.no_builtins,
             config.emit_lifetime_markers,
+            run_enzyme,
             sanitizer_options.as_ref(),
             pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
             pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
@@ -684,18 +699,14 @@ pub(crate) unsafe fn optimize(
             _ => llvm::OptStage::PreLinkNoLTO,
         };
 
-        // If we know that we will later run AD, then we disable vectorization and loop unrolling
-        let skip_size_increasing_opts = cfg!(llvm_enzyme);
+        // If we know that we will later run AD, then we disable vectorization and loop unrolling.
+        // Otherwise we pretend AD is already done and run the normal opt pipeline (=PostAD).
+        // FIXME(ZuseZ4): Make this more granular, only set PreAD if we actually have autodiff
+        // usages, not just if we build rustc with autodiff support.
+        let autodiff_stage =
+            if cfg!(llvm_enzyme) { AutodiffStage::PreAD } else { AutodiffStage::PostAD };
         return unsafe {
-            llvm_optimize(
-                cgcx,
-                dcx,
-                module,
-                config,
-                opt_level,
-                opt_stage,
-                skip_size_increasing_opts,
-            )
+            llvm_optimize(cgcx, dcx, module, config, opt_level, opt_stage, autodiff_stage)
         };
     }
     Ok(())

compiler/rustc_codegen_llvm/src/base.rs

@@ -157,9 +157,7 @@ pub(crate) fn linkage_to_llvm(linkage: Linkage) -> llvm::Linkage {
         Linkage::LinkOnceODR => llvm::Linkage::LinkOnceODRLinkage,
         Linkage::WeakAny => llvm::Linkage::WeakAnyLinkage,
         Linkage::WeakODR => llvm::Linkage::WeakODRLinkage,
-        Linkage::Appending => llvm::Linkage::AppendingLinkage,
         Linkage::Internal => llvm::Linkage::InternalLinkage,
-        Linkage::Private => llvm::Linkage::PrivateLinkage,
         Linkage::ExternalWeak => llvm::Linkage::ExternalWeakLinkage,
         Linkage::Common => llvm::Linkage::CommonLinkage,
     }

compiler/rustc_codegen_llvm/src/builder/autodiff.rs

@@ -4,10 +4,9 @@ use rustc_ast::expand::autodiff_attrs::{AutoDiffAttrs, AutoDiffItem, DiffActivit
 use rustc_codegen_ssa::ModuleCodegen;
 use rustc_codegen_ssa::back::write::ModuleConfig;
 use rustc_errors::FatalError;
-use rustc_session::config::Lto;
 use tracing::{debug, trace};
 
-use crate::back::write::{llvm_err, llvm_optimize};
+use crate::back::write::llvm_err;
 use crate::builder::SBuilder;
 use crate::context::SimpleCx;
 use crate::declare::declare_simple_fn;
@@ -53,8 +52,6 @@ fn generate_enzyme_call<'ll>(
     let mut ad_name: String = match attrs.mode {
         DiffMode::Forward => "__enzyme_fwddiff",
         DiffMode::Reverse => "__enzyme_autodiff",
-        DiffMode::ForwardFirst => "__enzyme_fwddiff",
-        DiffMode::ReverseFirst => "__enzyme_autodiff",
         _ => panic!("logic bug in autodiff, unrecognized mode"),
     }
     .to_string();
@@ -153,7 +150,7 @@ fn generate_enzyme_call<'ll>(
             _ => {}
         }
 
-        trace!("matching autodiff arguments");
+        debug!("matching autodiff arguments");
         // We now handle the issue that Rust level arguments not always match the llvm-ir level
         // arguments. A slice, `&[f32]`, for example, is represented as a pointer and a length on
         // llvm-ir level. The number of activities matches the number of Rust level arguments, so we
@@ -164,10 +161,10 @@ fn generate_enzyme_call<'ll>(
         let mut activity_pos = 0;
         let outer_args: Vec<&llvm::Value> = get_params(outer_fn);
         while activity_pos < inputs.len() {
-            let activity = inputs[activity_pos as usize];
+            let diff_activity = inputs[activity_pos as usize];
             // Duplicated arguments received a shadow argument, into which enzyme will write the
             // gradient.
-            let (activity, duplicated): (&Metadata, bool) = match activity {
+            let (activity, duplicated): (&Metadata, bool) = match diff_activity {
                 DiffActivity::None => panic!("not a valid input activity"),
                 DiffActivity::Const => (enzyme_const, false),
                 DiffActivity::Active => (enzyme_out, false),
@@ -222,7 +219,15 @@ fn generate_enzyme_call<'ll>(
                     // A duplicated pointer will have the following two outer_fn arguments:
                     // (..., ptr, ptr, ...). We add the following llvm-ir to our __enzyme call:
                     // (..., metadata! enzyme_dup, ptr, ptr, ...).
-                    assert!(llvm::LLVMRustGetTypeKind(next_outer_ty) == llvm::TypeKind::Pointer);
+                    if matches!(
+                        diff_activity,
+                        DiffActivity::Duplicated | DiffActivity::DuplicatedOnly
+                    ) {
+                        assert!(
+                            llvm::LLVMRustGetTypeKind(next_outer_ty) == llvm::TypeKind::Pointer
+                        );
+                    }
+                    // In the case of Dual we don't have assumptions, e.g. f32 would be valid.
                     args.push(next_outer_arg);
                     outer_pos += 2;
                     activity_pos += 1;
@@ -277,7 +282,7 @@ pub(crate) fn differentiate<'ll>(
     module: &'ll ModuleCodegen<ModuleLlvm>,
     cgcx: &CodegenContext<LlvmCodegenBackend>,
     diff_items: Vec<AutoDiffItem>,
-    config: &ModuleConfig,
+    _config: &ModuleConfig,
 ) -> Result<(), FatalError> {
     for item in &diff_items {
         trace!("{}", item);
@@ -291,20 +296,26 @@ pub(crate) fn differentiate<'ll>(
         let name = item.source.clone();
         let fn_def: Option<&llvm::Value> = cx.get_function(&name);
         let Some(fn_def) = fn_def else {
-            return Err(llvm_err(diag_handler.handle(), LlvmError::PrepareAutoDiff {
-                src: item.source.clone(),
-                target: item.target.clone(),
-                error: "could not find source function".to_owned(),
-            }));
+            return Err(llvm_err(
+                diag_handler.handle(),
+                LlvmError::PrepareAutoDiff {
+                    src: item.source.clone(),
+                    target: item.target.clone(),
+                    error: "could not find source function".to_owned(),
+                },
+            ));
         };
         debug!(?item.target);
         let fn_target: Option<&llvm::Value> = cx.get_function(&item.target);
         let Some(fn_target) = fn_target else {
-            return Err(llvm_err(diag_handler.handle(), LlvmError::PrepareAutoDiff {
-                src: item.source.clone(),
-                target: item.target.clone(),
-                error: "could not find target function".to_owned(),
-            }));
+            return Err(llvm_err(
+                diag_handler.handle(),
+                LlvmError::PrepareAutoDiff {
+                    src: item.source.clone(),
+                    target: item.target.clone(),
+                    error: "could not find target function".to_owned(),
+                },
+            ));
         };
 
         generate_enzyme_call(&cx, fn_def, fn_target, item.attrs.clone());
@@ -312,29 +323,6 @@ pub(crate) fn differentiate<'ll>(
 
     // FIXME(ZuseZ4): support SanitizeHWAddress and prevent illegal/unsupported opts
 
-    if let Some(opt_level) = config.opt_level {
-        let opt_stage = match cgcx.lto {
-            Lto::Fat => llvm::OptStage::PreLinkFatLTO,
-            Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO,
-            _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO,
-            _ => llvm::OptStage::PreLinkNoLTO,
-        };
-        // This is our second opt call, so now we run all opts,
-        // to make sure we get the best performance.
-        let skip_size_increasing_opts = false;
-        trace!("running Module Optimization after differentiation");
-        unsafe {
-            llvm_optimize(
-                cgcx,
-                diag_handler.handle(),
-                module,
-                config,
-                opt_level,
-                opt_stage,
-                skip_size_increasing_opts,
-            )?
-        };
-    }
     trace!("done with differentiate()");
 
     Ok(())

compiler/rustc_codegen_llvm/src/coverageinfo/mapgen/covfun.rs

@@ -11,7 +11,8 @@ use rustc_codegen_ssa::traits::{
     BaseTypeCodegenMethods, ConstCodegenMethods, StaticCodegenMethods,
 };
 use rustc_middle::mir::coverage::{
-    CovTerm, CoverageIdsInfo, Expression, FunctionCoverageInfo, Mapping, MappingKind, Op,
+    BasicCoverageBlock, CovTerm, CoverageIdsInfo, Expression, FunctionCoverageInfo, Mapping,
+    MappingKind, Op,
 };
 use rustc_middle::ty::{Instance, TyCtxt};
 use rustc_span::Span;
@@ -53,7 +54,7 @@ pub(crate) fn prepare_covfun_record<'tcx>(
     let fn_cov_info = tcx.instance_mir(instance.def).function_coverage_info.as_deref()?;
     let ids_info = tcx.coverage_ids_info(instance.def)?;
 
-    let expressions = prepare_expressions(fn_cov_info, ids_info, is_used);
+    let expressions = prepare_expressions(ids_info);
 
     let mut covfun = CovfunRecord {
         mangled_function_name: tcx.symbol_name(instance).name,
@@ -75,26 +76,14 @@ pub(crate) fn prepare_covfun_record<'tcx>(
 }
 
 /// Convert the function's coverage-counter expressions into a form suitable for FFI.
-fn prepare_expressions(
-    fn_cov_info: &FunctionCoverageInfo,
-    ids_info: &CoverageIdsInfo,
-    is_used: bool,
-) -> Vec<ffi::CounterExpression> {
-    // If any counters or expressions were removed by MIR opts, replace their
-    // terms with zero.
-    let counter_for_term = |term| {
-        if !is_used || ids_info.is_zero_term(term) {
-            ffi::Counter::ZERO
-        } else {
-            ffi::Counter::from_term(term)
-        }
-    };
+fn prepare_expressions(ids_info: &CoverageIdsInfo) -> Vec<ffi::CounterExpression> {
+    let counter_for_term = ffi::Counter::from_term;
 
     // We know that LLVM will optimize out any unused expressions before
     // producing the final coverage map, so there's no need to do the same
     // thing on the Rust side unless we're confident we can do much better.
     // (See `CounterExpressionsMinimizer` in `CoverageMappingWriter.cpp`.)
-    fn_cov_info
+    ids_info
         .expressions
         .iter()
         .map(move |&Expression { lhs, op, rhs }| ffi::CounterExpression {
@@ -136,11 +125,16 @@ fn fill_region_tables<'tcx>(
 
     // For each counter/region pair in this function+file, convert it to a
     // form suitable for FFI.
-    let is_zero_term = |term| !covfun.is_used || ids_info.is_zero_term(term);
     for &Mapping { ref kind, span } in &fn_cov_info.mappings {
-        // If the mapping refers to counters/expressions that were removed by
-        // MIR opts, replace those occurrences with zero.
-        let kind = kind.map_terms(|term| if is_zero_term(term) { CovTerm::Zero } else { term });
+        // If this function is unused, replace all counters with zero.
+        let counter_for_bcb = |bcb: BasicCoverageBlock| -> ffi::Counter {
+            let term = if covfun.is_used {
+                ids_info.term_for_bcb[bcb].expect("every BCB in a mapping was given a term")
+            } else {
+                CovTerm::Zero
+            };
+            ffi::Counter::from_term(term)
+        };
 
         // Convert the `Span` into coordinates that we can pass to LLVM, or
         // discard the span if conversion fails. In rare, cases _all_ of a
@@ -154,23 +148,22 @@ fn fill_region_tables<'tcx>(
             continue;
         }
 
-        match kind {
-            MappingKind::Code(term) => {
-                code_regions
-                    .push(ffi::CodeRegion { cov_span, counter: ffi::Counter::from_term(term) });
+        match *kind {
+            MappingKind::Code { bcb } => {
+                code_regions.push(ffi::CodeRegion { cov_span, counter: counter_for_bcb(bcb) });
             }
-            MappingKind::Branch { true_term, false_term } => {
+            MappingKind::Branch { true_bcb, false_bcb } => {
                 branch_regions.push(ffi::BranchRegion {
                     cov_span,
-                    true_counter: ffi::Counter::from_term(true_term),
-                    false_counter: ffi::Counter::from_term(false_term),
+                    true_counter: counter_for_bcb(true_bcb),
+                    false_counter: counter_for_bcb(false_bcb),
                 });
             }
-            MappingKind::MCDCBranch { true_term, false_term, mcdc_params } => {
+            MappingKind::MCDCBranch { true_bcb, false_bcb, mcdc_params } => {
                 mcdc_branch_regions.push(ffi::MCDCBranchRegion {
                     cov_span,
-                    true_counter: ffi::Counter::from_term(true_term),
-                    false_counter: ffi::Counter::from_term(false_term),
+                    true_counter: counter_for_bcb(true_bcb),
+                    false_counter: counter_for_bcb(false_bcb),
                     mcdc_branch_params: ffi::mcdc::BranchParameters::from(mcdc_params),
                 });
             }

compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs

@@ -160,21 +160,12 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
             CoverageKind::SpanMarker | CoverageKind::BlockMarker { .. } => unreachable!(
                 "marker statement {kind:?} should have been removed by CleanupPostBorrowck"
             ),
-            CoverageKind::CounterIncrement { id } => {
-                // The number of counters passed to `llvm.instrprof.increment` might
-                // be smaller than the number originally inserted by the instrumentor,
-                // if some high-numbered counters were removed by MIR optimizations.
-                // If so, LLVM's profiler runtime will use fewer physical counters.
-                let num_counters = ids_info.num_counters_after_mir_opts();
-                assert!(
-                    num_counters as usize <= function_coverage_info.num_counters,
-                    "num_counters disagreement: query says {num_counters} but function info only has {}",
-                    function_coverage_info.num_counters
-                );
-
+            CoverageKind::VirtualCounter { bcb }
+                if let Some(&id) = ids_info.phys_counter_for_node.get(&bcb) =>
+            {
                 let fn_name = bx.get_pgo_func_name_var(instance);
                 let hash = bx.const_u64(function_coverage_info.function_source_hash);
-                let num_counters = bx.const_u32(num_counters);
+                let num_counters = bx.const_u32(ids_info.num_counters);
                 let index = bx.const_u32(id.as_u32());
                 debug!(
                     "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})",
@@ -182,10 +173,8 @@ impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> {
                 );
                 bx.instrprof_increment(fn_name, hash, num_counters, index);
             }
-            CoverageKind::ExpressionUsed { id: _ } => {
-                // Expression-used statements are markers that are handled by
-                // `coverage_ids_info`, so there's nothing to codegen here.
-            }
+            // If a BCB doesn't have an associated physical counter, there's nothing to codegen.
+            CoverageKind::VirtualCounter { .. } => {}
             CoverageKind::CondBitmapUpdate { index, decision_depth } => {
                 let cond_bitmap = coverage_cx
                     .try_get_mcdc_condition_bitmap(&instance, decision_depth)

compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs

@@ -319,19 +319,16 @@ fn build_subroutine_type_di_node<'ll, 'tcx>(
     // This is actually a function pointer, so wrap it in pointer DI.
     let name = compute_debuginfo_type_name(cx.tcx, fn_ty, false);
     let (size, align) = match fn_ty.kind() {
-        ty::FnDef(..) => (0, 1),
-        ty::FnPtr(..) => (
-            cx.tcx.data_layout.pointer_size.bits(),
-            cx.tcx.data_layout.pointer_align.abi.bits() as u32,
-        ),
+        ty::FnDef(..) => (Size::ZERO, Align::ONE),
+        ty::FnPtr(..) => (cx.tcx.data_layout.pointer_size, cx.tcx.data_layout.pointer_align.abi),
         _ => unreachable!(),
     };
     let di_node = unsafe {
         llvm::LLVMRustDIBuilderCreatePointerType(
             DIB(cx),
             fn_di_node,
-            size,
-            align,
+            size.bits(),
+            align.bits() as u32,
             0, // Ignore DWARF address space.
             name.as_c_char_ptr(),
             name.len(),

compiler/rustc_codegen_llvm/src/debuginfo/mod.rs

@@ -548,14 +548,17 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                 }
             }
 
-            let scope = namespace::item_namespace(cx, DefId {
-                krate: instance.def_id().krate,
-                index: cx
-                    .tcx
-                    .def_key(instance.def_id())
-                    .parent
-                    .expect("get_containing_scope: missing parent?"),
-            });
+            let scope = namespace::item_namespace(
+                cx,
+                DefId {
+                    krate: instance.def_id().krate,
+                    index: cx
+                        .tcx
+                        .def_key(instance.def_id())
+                        .parent
+                        .expect("get_containing_scope: missing parent?"),
+                },
+            );
             (scope, false)
         }
     }
@@ -637,7 +640,7 @@ impl<'ll, 'tcx> DebugInfoCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                 true,
                 DIFlags::FlagZero,
                 argument_index,
-                align.bytes() as u32,
+                align.bits() as u32,
             )
         }
     }

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -333,12 +333,15 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     sym::prefetch_write_instruction => (1, 0),
                     _ => bug!(),
                 };
-                self.call_intrinsic("llvm.prefetch", &[
-                    args[0].immediate(),
-                    self.const_i32(rw),
-                    args[1].immediate(),
-                    self.const_i32(cache_type),
-                ])
+                self.call_intrinsic(
+                    "llvm.prefetch",
+                    &[
+                        args[0].immediate(),
+                        self.const_i32(rw),
+                        args[1].immediate(),
+                        self.const_i32(cache_type),
+                    ],
+                )
             }
             sym::carrying_mul_add => {
                 let (size, signed) = fn_args.type_at(0).int_size_and_signed(self.tcx);
@@ -396,10 +399,10 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                 match name {
                     sym::ctlz | sym::cttz => {
                         let y = self.const_bool(false);
-                        let ret = self.call_intrinsic(&format!("llvm.{name}.i{width}"), &[
-                            args[0].immediate(),
-                            y,
-                        ]);
+                        let ret = self.call_intrinsic(
+                            &format!("llvm.{name}.i{width}"),
+                            &[args[0].immediate(), y],
+                        );
 
                         self.intcast(ret, llret_ty, false)
                     }
@@ -416,24 +419,26 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                         self.intcast(ret, llret_ty, false)
                     }
                     sym::ctpop => {
-                        let ret = self.call_intrinsic(&format!("llvm.ctpop.i{width}"), &[
-                            args[0].immediate()
-                        ]);
+                        let ret = self.call_intrinsic(
+                            &format!("llvm.ctpop.i{width}"),
+                            &[args[0].immediate()],
+                        );
                         self.intcast(ret, llret_ty, false)
                     }
                     sym::bswap => {
                         if width == 8 {
                             args[0].immediate() // byte swap a u8/i8 is just a no-op
                         } else {
-                            self.call_intrinsic(&format!("llvm.bswap.i{width}"), &[
-                                args[0].immediate()
-                            ])
+                            self.call_intrinsic(
+                                &format!("llvm.bswap.i{width}"),
+                                &[args[0].immediate()],
+                            )
                         }
                     }
-                    sym::bitreverse => self
-                        .call_intrinsic(&format!("llvm.bitreverse.i{width}"), &[
-                            args[0].immediate()
-                        ]),
+                    sym::bitreverse => self.call_intrinsic(
+                        &format!("llvm.bitreverse.i{width}"),
+                        &[args[0].immediate()],
+                    ),
                     sym::rotate_left | sym::rotate_right => {
                         let is_left = name == sym::rotate_left;
                         let val = args[0].immediate();
@@ -500,11 +505,10 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
 
             sym::compare_bytes => {
                 // Here we assume that the `memcmp` provided by the target is a NOP for size 0.
-                let cmp = self.call_intrinsic("memcmp", &[
-                    args[0].immediate(),
-                    args[1].immediate(),
-                    args[2].immediate(),
-                ]);
+                let cmp = self.call_intrinsic(
+                    "memcmp",
+                    &[args[0].immediate(), args[1].immediate(), args[2].immediate()],
+                );
                 // Some targets have `memcmp` returning `i16`, but the intrinsic is always `i32`.
                 self.sext(cmp, self.type_ix(32))
             }
@@ -1305,14 +1309,17 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     if let Some(cmp_op) = comparison {
         let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
 
-        require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType {
-            span,
-            name,
-            in_len,
-            in_ty,
-            ret_ty,
-            out_len
-        });
+        require!(
+            in_len == out_len,
+            InvalidMonomorphization::ReturnLengthInputType {
+                span,
+                name,
+                in_len,
+                in_ty,
+                ret_ty,
+                out_len
+            }
+        );
         require!(
             bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
             InvalidMonomorphization::ReturnIntegerType { span, name, ret_ty, out_ty }
@@ -1333,21 +1340,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         let n = idx.len() as u64;
 
         let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
-        require!(out_len == n, InvalidMonomorphization::ReturnLength {
-            span,
-            name,
-            in_len: n,
-            ret_ty,
-            out_len
-        });
-        require!(in_elem == out_ty, InvalidMonomorphization::ReturnElement {
-            span,
-            name,
-            in_elem,
-            in_ty,
-            ret_ty,
-            out_ty
-        });
+        require!(
+            out_len == n,
+            InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
+        );
+        require!(
+            in_elem == out_ty,
+            InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
+        );
 
         let total_len = in_len * 2;
 
@@ -1392,21 +1392,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         };
 
         let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
-        require!(out_len == n, InvalidMonomorphization::ReturnLength {
-            span,
-            name,
-            in_len: n,
-            ret_ty,
-            out_len
-        });
-        require!(in_elem == out_ty, InvalidMonomorphization::ReturnElement {
-            span,
-            name,
-            in_elem,
-            in_ty,
-            ret_ty,
-            out_ty
-        });
+        require!(
+            out_len == n,
+            InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
+        );
+        require!(
+            in_elem == out_ty,
+            InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty }
+        );
 
         let total_len = u128::from(in_len) * 2;
 
@@ -1431,13 +1424,16 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     }
 
     if name == sym::simd_insert {
-        require!(in_elem == arg_tys[2], InvalidMonomorphization::InsertedType {
-            span,
-            name,
-            in_elem,
-            in_ty,
-            out_ty: arg_tys[2]
-        });
+        require!(
+            in_elem == arg_tys[2],
+            InvalidMonomorphization::InsertedType {
+                span,
+                name,
+                in_elem,
+                in_ty,
+                out_ty: arg_tys[2]
+            }
+        );
         let idx = bx
             .const_to_opt_u128(args[1].immediate(), false)
             .expect("typeck should have ensure that this is a const");
@@ -1456,13 +1452,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         ));
     }
     if name == sym::simd_extract {
-        require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType {
-            span,
-            name,
-            in_elem,
-            in_ty,
-            ret_ty
-        });
+        require!(
+            ret_ty == in_elem,
+            InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
+        );
         let idx = bx
             .const_to_opt_u128(args[1].immediate(), false)
             .expect("typeck should have ensure that this is a const");
@@ -1481,18 +1474,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         let m_elem_ty = in_elem;
         let m_len = in_len;
         let (v_len, _) = require_simd!(arg_tys[1], SimdArgument);
-        require!(m_len == v_len, InvalidMonomorphization::MismatchedLengths {
-            span,
-            name,
-            m_len,
-            v_len
-        });
-        let in_elem_bitwidth =
-            require_int_ty!(m_elem_ty.kind(), InvalidMonomorphization::MaskType {
-                span,
-                name,
-                ty: m_elem_ty
-            });
+        require!(
+            m_len == v_len,
+            InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
+        );
+        let in_elem_bitwidth = require_int_ty!(
+            m_elem_ty.kind(),
+            InvalidMonomorphization::MaskType { span, name, ty: m_elem_ty }
+        );
         let m_i1s = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, m_len);
         return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate()));
     }
@@ -1510,13 +1499,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         let expected_bytes = in_len.div_ceil(8);
 
         // Integer vector <i{in_bitwidth} x in_len>:
-        let in_elem_bitwidth =
-            require_int_or_uint_ty!(in_elem.kind(), InvalidMonomorphization::VectorArgument {
-                span,
-                name,
-                in_ty,
-                in_elem
-            });
+        let in_elem_bitwidth = require_int_or_uint_ty!(
+            in_elem.kind(),
+            InvalidMonomorphization::VectorArgument { span, name, in_ty, in_elem }
+        );
 
         let i1xn = vector_mask_to_bitmask(bx, args[0].immediate(), in_elem_bitwidth, in_len);
         // Bitcast <i1 x N> to iN:
@@ -1698,30 +1684,34 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         require_simd!(ret_ty, SimdReturn);
 
         // Of the same length:
-        require!(in_len == out_len, InvalidMonomorphization::SecondArgumentLength {
-            span,
-            name,
-            in_len,
-            in_ty,
-            arg_ty: arg_tys[1],
-            out_len
-        });
-        require!(in_len == out_len2, InvalidMonomorphization::ThirdArgumentLength {
-            span,
-            name,
-            in_len,
-            in_ty,
-            arg_ty: arg_tys[2],
-            out_len: out_len2
-        });
+        require!(
+            in_len == out_len,
+            InvalidMonomorphization::SecondArgumentLength {
+                span,
+                name,
+                in_len,
+                in_ty,
+                arg_ty: arg_tys[1],
+                out_len
+            }
+        );
+        require!(
+            in_len == out_len2,
+            InvalidMonomorphization::ThirdArgumentLength {
+                span,
+                name,
+                in_len,
+                in_ty,
+                arg_ty: arg_tys[2],
+                out_len: out_len2
+            }
+        );
 
         // The return type must match the first argument type
-        require!(ret_ty == in_ty, InvalidMonomorphization::ExpectedReturnType {
-            span,
-            name,
-            in_ty,
-            ret_ty
-        });
+        require!(
+            ret_ty == in_ty,
+            InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty }
+        );
 
         require!(
             matches!(
@@ -1739,13 +1729,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             }
         );
 
-        let mask_elem_bitwidth =
-            require_int_ty!(element_ty2.kind(), InvalidMonomorphization::ThirdArgElementType {
+        let mask_elem_bitwidth = require_int_ty!(
+            element_ty2.kind(),
+            InvalidMonomorphization::ThirdArgElementType {
                 span,
                 name,
                 expected_element: element_ty2,
                 third_arg: arg_tys[2]
-            });
+            }
+        );
 
         // Alignment of T, must be a constant integer value:
         let alignment_ty = bx.type_i32();
@@ -1805,22 +1797,23 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         require_simd!(ret_ty, SimdReturn);
 
         // Of the same length:
-        require!(values_len == mask_len, InvalidMonomorphization::ThirdArgumentLength {
-            span,
-            name,
-            in_len: mask_len,
-            in_ty: mask_ty,
-            arg_ty: values_ty,
-            out_len: values_len
-        });
+        require!(
+            values_len == mask_len,
+            InvalidMonomorphization::ThirdArgumentLength {
+                span,
+                name,
+                in_len: mask_len,
+                in_ty: mask_ty,
+                arg_ty: values_ty,
+                out_len: values_len
+            }
+        );
 
         // The return type must match the last argument type
-        require!(ret_ty == values_ty, InvalidMonomorphization::ExpectedReturnType {
-            span,
-            name,
-            in_ty: values_ty,
-            ret_ty
-        });
+        require!(
+            ret_ty == values_ty,
+            InvalidMonomorphization::ExpectedReturnType { span, name, in_ty: values_ty, ret_ty }
+        );
 
         require!(
             matches!(
@@ -1838,13 +1831,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             }
         );
 
-        let m_elem_bitwidth =
-            require_int_ty!(mask_elem.kind(), InvalidMonomorphization::ThirdArgElementType {
+        let m_elem_bitwidth = require_int_ty!(
+            mask_elem.kind(),
+            InvalidMonomorphization::ThirdArgElementType {
                 span,
                 name,
                 expected_element: values_elem,
                 third_arg: mask_ty,
-            });
+            }
+        );
 
         let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
         let mask_ty = bx.type_vector(bx.type_i1(), mask_len);
@@ -1896,14 +1891,17 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         let (values_len, values_elem) = require_simd!(values_ty, SimdThird);
 
         // Of the same length:
-        require!(values_len == mask_len, InvalidMonomorphization::ThirdArgumentLength {
-            span,
-            name,
-            in_len: mask_len,
-            in_ty: mask_ty,
-            arg_ty: values_ty,
-            out_len: values_len
-        });
+        require!(
+            values_len == mask_len,
+            InvalidMonomorphization::ThirdArgumentLength {
+                span,
+                name,
+                in_len: mask_len,
+                in_ty: mask_ty,
+                arg_ty: values_ty,
+                out_len: values_len
+            }
+        );
 
         // The second argument must be a mutable pointer type matching the element type
         require!(
@@ -1923,13 +1921,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             }
         );
 
-        let m_elem_bitwidth =
-            require_int_ty!(mask_elem.kind(), InvalidMonomorphization::ThirdArgElementType {
+        let m_elem_bitwidth = require_int_ty!(
+            mask_elem.kind(),
+            InvalidMonomorphization::ThirdArgElementType {
                 span,
                 name,
                 expected_element: values_elem,
                 third_arg: mask_ty,
-            });
+            }
+        );
 
         let mask = vector_mask_to_bitmask(bx, args[0].immediate(), m_elem_bitwidth, mask_len);
         let mask_ty = bx.type_vector(bx.type_i1(), mask_len);
@@ -1976,22 +1976,28 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         let (element_len2, element_ty2) = require_simd!(arg_tys[2], SimdThird);
 
         // Of the same length:
-        require!(in_len == element_len1, InvalidMonomorphization::SecondArgumentLength {
-            span,
-            name,
-            in_len,
-            in_ty,
-            arg_ty: arg_tys[1],
-            out_len: element_len1
-        });
-        require!(in_len == element_len2, InvalidMonomorphization::ThirdArgumentLength {
-            span,
-            name,
-            in_len,
-            in_ty,
-            arg_ty: arg_tys[2],
-            out_len: element_len2
-        });
+        require!(
+            in_len == element_len1,
+            InvalidMonomorphization::SecondArgumentLength {
+                span,
+                name,
+                in_len,
+                in_ty,
+                arg_ty: arg_tys[1],
+                out_len: element_len1
+            }
+        );
+        require!(
+            in_len == element_len2,
+            InvalidMonomorphization::ThirdArgumentLength {
+                span,
+                name,
+                in_len,
+                in_ty,
+                arg_ty: arg_tys[2],
+                out_len: element_len2
+            }
+        );
 
         require!(
             matches!(
@@ -2011,13 +2017,15 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         );
 
         // The element type of the third argument must be a signed integer type of any width:
-        let mask_elem_bitwidth =
-            require_int_ty!(element_ty2.kind(), InvalidMonomorphization::ThirdArgElementType {
+        let mask_elem_bitwidth = require_int_ty!(
+            element_ty2.kind(),
+            InvalidMonomorphization::ThirdArgElementType {
                 span,
                 name,
                 expected_element: element_ty2,
                 third_arg: arg_tys[2]
-            });
+            }
+        );
 
         // Alignment of T, must be a constant integer value:
         let alignment_ty = bx.type_i32();
@@ -2058,13 +2066,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         ($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident,
          $identity:expr) => {
             if name == sym::$name {
-                require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType {
-                    span,
-                    name,
-                    in_elem,
-                    in_ty,
-                    ret_ty
-                });
+                require!(
+                    ret_ty == in_elem,
+                    InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
+                );
                 return match in_elem.kind() {
                     ty::Int(_) | ty::Uint(_) => {
                         let r = bx.$integer_reduce(args[0].immediate());
@@ -2133,13 +2138,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     macro_rules! minmax_red {
         ($name:ident: $int_red:ident, $float_red:ident) => {
             if name == sym::$name {
-                require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType {
-                    span,
-                    name,
-                    in_elem,
-                    in_ty,
-                    ret_ty
-                });
+                require!(
+                    ret_ty == in_elem,
+                    InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
+                );
                 return match in_elem.kind() {
                     ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)),
                     ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)),
@@ -2164,13 +2166,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         ($name:ident : $red:ident, $boolean:expr) => {
             if name == sym::$name {
                 let input = if !$boolean {
-                    require!(ret_ty == in_elem, InvalidMonomorphization::ReturnType {
-                        span,
-                        name,
-                        in_elem,
-                        in_ty,
-                        ret_ty
-                    });
+                    require!(
+                        ret_ty == in_elem,
+                        InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty }
+                    );
                     args[0].immediate()
                 } else {
                     let bitwidth = match in_elem.kind() {
@@ -2218,25 +2217,27 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
 
     if name == sym::simd_cast_ptr {
         let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
-        require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType {
-            span,
-            name,
-            in_len,
-            in_ty,
-            ret_ty,
-            out_len
-        });
+        require!(
+            in_len == out_len,
+            InvalidMonomorphization::ReturnLengthInputType {
+                span,
+                name,
+                in_len,
+                in_ty,
+                ret_ty,
+                out_len
+            }
+        );
 
         match in_elem.kind() {
             ty::RawPtr(p_ty, _) => {
                 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
                     bx.tcx.normalize_erasing_regions(bx.typing_env(), ty)
                 });
-                require!(metadata.is_unit(), InvalidMonomorphization::CastWidePointer {
-                    span,
-                    name,
-                    ty: in_elem
-                });
+                require!(
+                    metadata.is_unit(),
+                    InvalidMonomorphization::CastWidePointer { span, name, ty: in_elem }
+                );
             }
             _ => {
                 return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem })
@@ -2247,11 +2248,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
                 let metadata = p_ty.ptr_metadata_ty(bx.tcx, |ty| {
                     bx.tcx.normalize_erasing_regions(bx.typing_env(), ty)
                 });
-                require!(metadata.is_unit(), InvalidMonomorphization::CastWidePointer {
-                    span,
-                    name,
-                    ty: out_elem
-                });
+                require!(
+                    metadata.is_unit(),
+                    InvalidMonomorphization::CastWidePointer { span, name, ty: out_elem }
+                );
             }
             _ => {
                 return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem })
@@ -2263,14 +2263,17 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
 
     if name == sym::simd_expose_provenance {
         let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
-        require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType {
-            span,
-            name,
-            in_len,
-            in_ty,
-            ret_ty,
-            out_len
-        });
+        require!(
+            in_len == out_len,
+            InvalidMonomorphization::ReturnLengthInputType {
+                span,
+                name,
+                in_len,
+                in_ty,
+                ret_ty,
+                out_len
+            }
+        );
 
         match in_elem.kind() {
             ty::RawPtr(_, _) => {}
@@ -2288,14 +2291,17 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
 
     if name == sym::simd_with_exposed_provenance {
         let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
-        require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType {
-            span,
-            name,
-            in_len,
-            in_ty,
-            ret_ty,
-            out_len
-        });
+        require!(
+            in_len == out_len,
+            InvalidMonomorphization::ReturnLengthInputType {
+                span,
+                name,
+                in_len,
+                in_ty,
+                ret_ty,
+                out_len
+            }
+        );
 
         match in_elem.kind() {
             ty::Uint(ty::UintTy::Usize) => {}
@@ -2313,14 +2319,17 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
 
     if name == sym::simd_cast || name == sym::simd_as {
         let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
-        require!(in_len == out_len, InvalidMonomorphization::ReturnLengthInputType {
-            span,
-            name,
-            in_len,
-            in_ty,
-            ret_ty,
-            out_len
-        });
+        require!(
+            in_len == out_len,
+            InvalidMonomorphization::ReturnLengthInputType {
+                span,
+                name,
+                in_len,
+                in_ty,
+                ret_ty,
+                out_len
+            }
+        );
         // casting cares about nominal type, not just structural type
         if in_elem == out_elem {
             return Ok(args[0].immediate());

compiler/rustc_codegen_llvm/src/lib.rs

@@ -13,6 +13,7 @@
 #![feature(extern_types)]
 #![feature(file_buffered)]
 #![feature(hash_raw_entry)]
+#![feature(if_let_guard)]
 #![feature(impl_trait_in_assoc_type)]
 #![feature(iter_intersperse)]
 #![feature(let_chains)]

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -2382,6 +2382,7 @@ unsafe extern "C" {
         LoopVectorize: bool,
         DisableSimplifyLibCalls: bool,
         EmitLifetimeMarkers: bool,
+        RunEnzyme: bool,
         SanitizerOptions: Option<&SanitizerOptions>,
         PGOGenPath: *const c_char,
         PGOUsePath: *const c_char,

compiler/rustc_codegen_llvm/src/mono_item.rs

@@ -71,10 +71,7 @@ impl<'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'_, 'tcx> {
         // compiler-rt, then we want to implicitly compile everything with hidden
         // visibility as we're going to link this object all over the place but
         // don't want the symbols to get exported.
-        if linkage != Linkage::Internal
-            && linkage != Linkage::Private
-            && self.tcx.is_compiler_builtins(LOCAL_CRATE)
-        {
+        if linkage != Linkage::Internal && self.tcx.is_compiler_builtins(LOCAL_CRATE) {
             llvm::set_visibility(lldecl, llvm::Visibility::Hidden);
         } else {
             llvm::set_visibility(lldecl, base::visibility_to_llvm(visibility));

compiler/rustc_codegen_llvm/src/type_.rs

@@ -1,14 +1,14 @@
 use std::{fmt, ptr};
 
 use libc::{c_char, c_uint};
-use rustc_abi::{AddressSpace, Align, Integer, Size};
+use rustc_abi::{AddressSpace, Align, Integer, Reg, Size};
 use rustc_codegen_ssa::common::TypeKind;
 use rustc_codegen_ssa::traits::*;
 use rustc_data_structures::small_c_str::SmallCStr;
 use rustc_middle::bug;
 use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::ty::{self, Ty};
-use rustc_target::callconv::{CastTarget, FnAbi, Reg};
+use rustc_target::callconv::{CastTarget, FnAbi};
 
 use crate::abi::{FnAbiLlvmExt, LlvmType};
 use crate::context::{CodegenCx, SimpleCx};

compiler/rustc_codegen_ssa/Cargo.toml

@@ -11,7 +11,7 @@ bitflags = "2.4.1"
 bstr = "1.11.3"
 # Pinned so `cargo update` bumps don't cause breakage. Please also update the
 # `cc` in `rustc_llvm` if you update the `cc` here.
-cc = "=1.2.7"
+cc = "=1.2.13"
 either = "1.5.0"
 itertools = "0.12"
 pathdiff = "0.2.0"

compiler/rustc_codegen_ssa/src/back/archive.rs

@@ -414,10 +414,10 @@ impl<'a> ArchiveBuilder for ArArchiveBuilder<'a> {
                     let member_path = archive_path.parent().unwrap().join(Path::new(&file_name));
                     self.entries.push((file_name.into_bytes(), ArchiveEntry::File(member_path)));
                 } else {
-                    self.entries.push((file_name.into_bytes(), ArchiveEntry::FromArchive {
-                        archive_index,
-                        file_range: entry.file_range(),
-                    }));
+                    self.entries.push((
+                        file_name.into_bytes(),
+                        ArchiveEntry::FromArchive { archive_index, file_range: entry.file_range() },
+                    ));
                 }
             }
         }

compiler/rustc_codegen_ssa/src/back/linker/tests.rs

@@ -10,23 +10,22 @@ fn test_rpaths_to_args() {
 #[test]
 fn test_xlinker() {
     let mut cmd = Command::new("foo");
-    convert_link_args_to_cc_args(&mut cmd, &[
-        "arg1",
-        "arg2",
-        "arg3,with,comma",
-        "arg4,with,comma",
-        "arg5",
-        "arg6,with,comma",
-    ]);
+    convert_link_args_to_cc_args(
+        &mut cmd,
+        &["arg1", "arg2", "arg3,with,comma", "arg4,with,comma", "arg5", "arg6,with,comma"],
+    );
 
-    assert_eq!(cmd.get_args(), [
-        OsStr::new("-Wl,arg1,arg2"),
-        OsStr::new("-Xlinker"),
-        OsStr::new("arg3,with,comma"),
-        OsStr::new("-Xlinker"),
-        OsStr::new("arg4,with,comma"),
-        OsStr::new("-Wl,arg5"),
-        OsStr::new("-Xlinker"),
-        OsStr::new("arg6,with,comma"),
-    ]);
+    assert_eq!(
+        cmd.get_args(),
+        [
+            OsStr::new("-Wl,arg1,arg2"),
+            OsStr::new("-Xlinker"),
+            OsStr::new("arg3,with,comma"),
+            OsStr::new("-Xlinker"),
+            OsStr::new("arg4,with,comma"),
+            OsStr::new("-Wl,arg5"),
+            OsStr::new("-Xlinker"),
+            OsStr::new("arg6,with,comma"),
+        ]
+    );
 }

compiler/rustc_codegen_ssa/src/back/metadata.rs

@@ -704,13 +704,17 @@ pub fn create_metadata_file_for_wasm(sess: &Session, data: &[u8], section_name:
     let mut imports = wasm_encoder::ImportSection::new();
 
     if sess.target.pointer_width == 64 {
-        imports.import("env", "__linear_memory", wasm_encoder::MemoryType {
-            minimum: 0,
-            maximum: None,
-            memory64: true,
-            shared: false,
-            page_size_log2: None,
-        });
+        imports.import(
+            "env",
+            "__linear_memory",
+            wasm_encoder::MemoryType {
+                minimum: 0,
+                maximum: None,
+                memory64: true,
+                shared: false,
+                page_size_log2: None,
+            },
+        );
     }
 
     if imports.len() > 0 {

compiler/rustc_codegen_ssa/src/back/symbol_export.rs

@@ -140,11 +140,14 @@ fn reachable_non_generics_provider(tcx: TyCtxt<'_>, _: LocalCrate) -> DefIdMap<S
         .into();
 
     if let Some(id) = tcx.proc_macro_decls_static(()) {
-        reachable_non_generics.insert(id.to_def_id(), SymbolExportInfo {
-            level: SymbolExportLevel::C,
-            kind: SymbolExportKind::Data,
-            used: false,
-        });
+        reachable_non_generics.insert(
+            id.to_def_id(),
+            SymbolExportInfo {
+                level: SymbolExportLevel::C,
+                kind: SymbolExportKind::Data,
+                used: false,
+            },
+        );
     }
 
     reachable_non_generics
@@ -185,11 +188,14 @@ fn exported_symbols_provider_local(
     if !tcx.sess.target.dll_tls_export {
         symbols.extend(sorted.iter().filter_map(|(&def_id, &info)| {
             tcx.needs_thread_local_shim(def_id).then(|| {
-                (ExportedSymbol::ThreadLocalShim(def_id), SymbolExportInfo {
-                    level: info.level,
-                    kind: SymbolExportKind::Text,
-                    used: info.used,
-                })
+                (
+                    ExportedSymbol::ThreadLocalShim(def_id),
+                    SymbolExportInfo {
+                        level: info.level,
+                        kind: SymbolExportKind::Text,
+                        used: info.used,
+                    },
+                )
             })
         }))
     }
@@ -198,11 +204,14 @@ fn exported_symbols_provider_local(
         let exported_symbol =
             ExportedSymbol::NoDefId(SymbolName::new(tcx, tcx.sess.target.entry_name.as_ref()));
 
-        symbols.push((exported_symbol, SymbolExportInfo {
-            level: SymbolExportLevel::C,
-            kind: SymbolExportKind::Text,
-            used: false,
-        }));
+        symbols.push((
+            exported_symbol,
+            SymbolExportInfo {
+                level: SymbolExportLevel::C,
+                kind: SymbolExportKind::Text,
+                used: false,
+            },
+        ));
     }
 
     // Mark allocator shim symbols as exported only if they were generated.
@@ -214,20 +223,26 @@ fn exported_symbols_provider_local(
         {
             let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name));
 
-            symbols.push((exported_symbol, SymbolExportInfo {
-                level: SymbolExportLevel::Rust,
-                kind: SymbolExportKind::Text,
-                used: false,
-            }));
+            symbols.push((
+                exported_symbol,
+                SymbolExportInfo {
+                    level: SymbolExportLevel::Rust,
+                    kind: SymbolExportKind::Text,
+                    used: false,
+                },
+            ));
         }
 
         let exported_symbol =
             ExportedSymbol::NoDefId(SymbolName::new(tcx, NO_ALLOC_SHIM_IS_UNSTABLE));
-        symbols.push((exported_symbol, SymbolExportInfo {
-            level: SymbolExportLevel::Rust,
-            kind: SymbolExportKind::Data,
-            used: false,
-        }))
+        symbols.push((
+            exported_symbol,
+            SymbolExportInfo {
+                level: SymbolExportLevel::Rust,
+                kind: SymbolExportKind::Data,
+                used: false,
+            },
+        ))
     }
 
     if tcx.sess.instrument_coverage() || tcx.sess.opts.cg.profile_generate.enabled() {
@@ -239,11 +254,14 @@ fn exported_symbols_provider_local(
 
         symbols.extend(PROFILER_WEAK_SYMBOLS.iter().map(|sym| {
             let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, sym));
-            (exported_symbol, SymbolExportInfo {
-                level: SymbolExportLevel::C,
-                kind: SymbolExportKind::Data,
-                used: false,
-            })
+            (
+                exported_symbol,
+                SymbolExportInfo {
+                    level: SymbolExportLevel::C,
+                    kind: SymbolExportKind::Data,
+                    used: false,
+                },
+            )
         }));
     }
 
@@ -261,11 +279,14 @@ fn exported_symbols_provider_local(
 
         symbols.extend(msan_weak_symbols.into_iter().map(|sym| {
             let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, sym));
-            (exported_symbol, SymbolExportInfo {
-                level: SymbolExportLevel::C,
-                kind: SymbolExportKind::Data,
-                used: false,
-            })
+            (
+                exported_symbol,
+                SymbolExportInfo {
+                    level: SymbolExportLevel::C,
+                    kind: SymbolExportKind::Data,
+                    used: false,
+                },
+            )
         }));
     }
 
@@ -275,11 +296,14 @@ fn exported_symbols_provider_local(
         let symbol_name = metadata_symbol_name(tcx);
         let exported_symbol = ExportedSymbol::NoDefId(SymbolName::new(tcx, &symbol_name));
 
-        symbols.push((exported_symbol, SymbolExportInfo {
-            level: SymbolExportLevel::C,
-            kind: SymbolExportKind::Data,
-            used: true,
-        }));
+        symbols.push((
+            exported_symbol,
+            SymbolExportInfo {
+                level: SymbolExportLevel::C,
+                kind: SymbolExportKind::Data,
+                used: true,
+            },
+        ));
     }
 
     if tcx.local_crate_exports_generics() {
@@ -325,21 +349,27 @@ fn exported_symbols_provider_local(
                 MonoItem::Fn(Instance { def: InstanceKind::Item(def), args }) => {
                     if args.non_erasable_generics().next().is_some() {
                         let symbol = ExportedSymbol::Generic(def, args);
-                        symbols.push((symbol, SymbolExportInfo {
-                            level: SymbolExportLevel::Rust,
-                            kind: SymbolExportKind::Text,
-                            used: false,
-                        }));
+                        symbols.push((
+                            symbol,
+                            SymbolExportInfo {
+                                level: SymbolExportLevel::Rust,
+                                kind: SymbolExportKind::Text,
+                                used: false,
+                            },
+                        ));
                     }
                 }
                 MonoItem::Fn(Instance { def: InstanceKind::DropGlue(_, Some(ty)), args }) => {
                     // A little sanity-check
                     assert_eq!(args.non_erasable_generics().next(), Some(GenericArgKind::Type(ty)));
-                    symbols.push((ExportedSymbol::DropGlue(ty), SymbolExportInfo {
-                        level: SymbolExportLevel::Rust,
-                        kind: SymbolExportKind::Text,
-                        used: false,
-                    }));
+                    symbols.push((
+                        ExportedSymbol::DropGlue(ty),
+                        SymbolExportInfo {
+                            level: SymbolExportLevel::Rust,
+                            kind: SymbolExportKind::Text,
+                            used: false,
+                        },
+                    ));
                 }
                 MonoItem::Fn(Instance {
                     def: InstanceKind::AsyncDropGlueCtorShim(_, Some(ty)),
@@ -347,11 +377,14 @@ fn exported_symbols_provider_local(
                 }) => {
                     // A little sanity-check
                     assert_eq!(args.non_erasable_generics().next(), Some(GenericArgKind::Type(ty)));
-                    symbols.push((ExportedSymbol::AsyncDropGlueCtorShim(ty), SymbolExportInfo {
-                        level: SymbolExportLevel::Rust,
-                        kind: SymbolExportKind::Text,
-                        used: false,
-                    }));
+                    symbols.push((
+                        ExportedSymbol::AsyncDropGlueCtorShim(ty),
+                        SymbolExportInfo {
+                            level: SymbolExportLevel::Rust,
+                            kind: SymbolExportKind::Text,
+                            used: false,
+                        },
+                    ));
                 }
                 _ => {
                     // Any other symbols don't qualify for sharing

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

@@ -41,7 +41,6 @@ fn linkage_by_name(tcx: TyCtxt<'_>, def_id: LocalDefId, name: &str) -> Linkage {
     // ghost, dllimport, dllexport and linkonce_odr_autohide are not supported
     // and don't have to be, LLVM treats them as no-ops.
     match name {
-        "appending" => Appending,
         "available_externally" => AvailableExternally,
         "common" => Common,
         "extern_weak" => ExternalWeak,
@@ -49,7 +48,6 @@ fn linkage_by_name(tcx: TyCtxt<'_>, def_id: LocalDefId, name: &str) -> Linkage {
         "internal" => Internal,
         "linkonce" => LinkOnceAny,
         "linkonce_odr" => LinkOnceODR,
-        "private" => Private,
         "weak" => WeakAny,
         "weak_odr" => WeakODR,
         _ => tcx.dcx().span_fatal(tcx.def_span(def_id), "invalid linkage specified"),
@@ -916,8 +914,6 @@ fn autodiff_attrs(tcx: TyCtxt<'_>, id: DefId) -> Option<AutoDiffAttrs> {
     let mode = match mode.as_str() {
         "Forward" => DiffMode::Forward,
         "Reverse" => DiffMode::Reverse,
-        "ForwardFirst" => DiffMode::ForwardFirst,
-        "ReverseFirst" => DiffMode::ReverseFirst,
         _ => {
             span_bug!(mode.span, "rustc_autodiff attribute contains invalid mode");
         }

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -1,6 +1,6 @@
 use std::cmp;
 
-use rustc_abi::{self as abi, ExternAbi, HasDataLayout, WrappingRange};
+use rustc_abi::{BackendRepr, ExternAbi, HasDataLayout, Reg, WrappingRange};
 use rustc_ast as ast;
 use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_hir::lang_items::LangItem;
@@ -14,7 +14,7 @@ use rustc_middle::{bug, span_bug};
 use rustc_session::config::OptLevel;
 use rustc_span::source_map::Spanned;
 use rustc_span::{Span, sym};
-use rustc_target::callconv::{ArgAbi, FnAbi, PassMode, Reg};
+use rustc_target::callconv::{ArgAbi, FnAbi, PassMode};
 use tracing::{debug, info};
 
 use super::operand::OperandRef;
@@ -1568,7 +1568,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 // the load would just produce `OperandValue::Ref` instead
                 // of the `OperandValue::Immediate` we need for the call.
                 llval = bx.load(bx.backend_type(arg.layout), llval, align);
-                if let abi::BackendRepr::Scalar(scalar) = arg.layout.backend_repr {
+                if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr {
                     if scalar.is_bool() {
                         bx.range_metadata(llval, WrappingRange { start: 0, end: 1 });
                     }
@@ -1627,10 +1627,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         if let Some(slot) = self.personality_slot {
             slot
         } else {
-            let layout = cx.layout_of(Ty::new_tup(cx.tcx(), &[
-                Ty::new_mut_ptr(cx.tcx(), cx.tcx().types.u8),
-                cx.tcx().types.i32,
-            ]));
+            let layout = cx.layout_of(Ty::new_tup(
+                cx.tcx(),
+                &[Ty::new_mut_ptr(cx.tcx(), cx.tcx().types.u8), cx.tcx().types.i32],
+            ));
             let slot = PlaceRef::alloca(bx, layout);
             self.personality_slot = Some(slot);
             slot
@@ -1731,15 +1731,32 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             let mut cs_bx = Bx::build(self.cx, llbb);
             let cs = cs_bx.catch_switch(None, None, &[cp_llbb]);
 
-            // The "null" here is actually a RTTI type descriptor for the
-            // C++ personality function, but `catch (...)` has no type so
-            // it's null. The 64 here is actually a bitfield which
-            // represents that this is a catch-all block.
             bx = Bx::build(self.cx, cp_llbb);
             let null =
                 bx.const_null(bx.type_ptr_ext(bx.cx().data_layout().instruction_address_space));
-            let sixty_four = bx.const_i32(64);
-            funclet = Some(bx.catch_pad(cs, &[null, sixty_four, null]));
+
+            // The `null` in first argument here is actually a RTTI type
+            // descriptor for the C++ personality function, but `catch (...)`
+            // has no type so it's null.
+            let args = if base::wants_msvc_seh(self.cx.sess()) {
+                // This bitmask is a single `HT_IsStdDotDot` flag, which
+                // represents that this is a C++-style `catch (...)` block that
+                // only captures programmatic exceptions, not all SEH
+                // exceptions. The second `null` points to a non-existent
+                // `alloca` instruction, which an LLVM pass would inline into
+                // the initial SEH frame allocation.
+                let adjectives = bx.const_i32(0x40);
+                &[null, adjectives, null] as &[_]
+            } else {
+                // Specifying more arguments than necessary usually doesn't
+                // hurt, but the `WasmEHPrepare` LLVM pass does not recognize
+                // anything other than a single `null` as a `catch (...)` block,
+                // leading to problems down the line during instruction
+                // selection.
+                &[null] as &[_]
+            };
+
+            funclet = Some(bx.catch_pad(cs, args));
         } else {
             llbb = Bx::append_block(self.cx, self.llfn, "terminate");
             bx = Bx::build(self.cx, llbb);

compiler/rustc_codegen_ssa/src/mir/naked_asm.rs

@@ -187,10 +187,9 @@ fn prefix_and_suffix<'tcx>(
                     }
                 }
             }
-            Linkage::Internal | Linkage::Private => {
+            Linkage::Internal => {
                 // write nothing
             }
-            Linkage::Appending => emit_fatal("Only global variables can have appending linkage!"),
             Linkage::Common => emit_fatal("Functions may not have common linkage"),
             Linkage::AvailableExternally => {
                 // this would make the function equal an extern definition

compiler/rustc_codegen_ssa/src/mir/place.rs

@@ -474,10 +474,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             LocalRef::Operand(..) => {
                 if place_ref.is_indirect_first_projection() {
                     base = 1;
-                    let cg_base = self.codegen_consume(bx, mir::PlaceRef {
-                        projection: &place_ref.projection[..0],
-                        ..place_ref
-                    });
+                    let cg_base = self.codegen_consume(
+                        bx,
+                        mir::PlaceRef { projection: &place_ref.projection[..0], ..place_ref },
+                    );
                     cg_base.deref(bx.cx())
                 } else {
                     bug!("using operand local {:?} as place", place_ref);

compiler/rustc_codegen_ssa/src/traits/consts.rs

@@ -1,5 +1,5 @@
+use rustc_abi as abi;
 use rustc_middle::mir::interpret::{ConstAllocation, Scalar};
-use rustc_target::abi;
 
 use super::BackendTypes;
 

compiler/rustc_codegen_ssa/src/traits/type_.rs

@@ -1,8 +1,8 @@
-use rustc_abi::{AddressSpace, Float, Integer};
+use rustc_abi::{AddressSpace, Float, Integer, Reg};
 use rustc_middle::bug;
 use rustc_middle::ty::layout::{HasTyCtxt, HasTypingEnv, TyAndLayout};
 use rustc_middle::ty::{self, Ty};
-use rustc_target::callconv::{ArgAbi, CastTarget, FnAbi, Reg};
+use rustc_target::callconv::{ArgAbi, CastTarget, FnAbi};
 
 use super::BackendTypes;
 use super::misc::MiscCodegenMethods;

compiler/rustc_const_eval/src/check_consts/qualifs.rs

@@ -188,12 +188,14 @@ impl Qualif for NeedsNonConstDrop {
             ObligationCause::misc(cx.body.span, cx.def_id()),
             param_env,
             ty::Binder::dummy(ty::TraitRef::new(cx.tcx, destruct_def_id, [ty]))
-                .to_host_effect_clause(cx.tcx, match cx.const_kind() {
-                    rustc_hir::ConstContext::ConstFn => ty::BoundConstness::Maybe,
-                    rustc_hir::ConstContext::Static(_) | rustc_hir::ConstContext::Const { .. } => {
-                        ty::BoundConstness::Const
-                    }
-                }),
+                .to_host_effect_clause(
+                    cx.tcx,
+                    match cx.const_kind() {
+                        rustc_hir::ConstContext::ConstFn => ty::BoundConstness::Maybe,
+                        rustc_hir::ConstContext::Static(_)
+                        | rustc_hir::ConstContext::Const { .. } => ty::BoundConstness::Const,
+                    },
+                ),
         ));
         !ocx.select_all_or_error().is_empty()
     }

compiler/rustc_const_eval/src/errors.rs

@@ -578,10 +578,13 @@ impl<'a> ReportErrorExt for UndefinedBehaviorInfo<'a> {
             }
             ShiftOverflow { intrinsic, shift_amount } => {
                 diag.arg("intrinsic", intrinsic);
-                diag.arg("shift_amount", match shift_amount {
-                    Either::Left(v) => v.to_string(),
-                    Either::Right(v) => v.to_string(),
-                });
+                diag.arg(
+                    "shift_amount",
+                    match shift_amount {
+                        Either::Left(v) => v.to_string(),
+                        Either::Right(v) => v.to_string(),
+                    },
+                );
             }
             BoundsCheckFailed { len, index } => {
                 diag.arg("len", len);

compiler/rustc_const_eval/src/interpret/call.rs

@@ -381,10 +381,13 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
                 "caller ABI: {:#?}, args: {:#?}",
                 caller_fn_abi,
                 args.iter()
-                    .map(|arg| (arg.layout().ty, match arg {
-                        FnArg::Copy(op) => format!("copy({op:?})"),
-                        FnArg::InPlace(mplace) => format!("in-place({mplace:?})"),
-                    }))
+                    .map(|arg| (
+                        arg.layout().ty,
+                        match arg {
+                            FnArg::Copy(op) => format!("copy({op:?})"),
+                            FnArg::InPlace(mplace) => format!("in-place({mplace:?})"),
+                        }
+                    ))
                     .collect::<Vec<_>>()
             );
             trace!(
@@ -874,10 +877,13 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
         );
 
         // Check `unwinding`.
-        assert_eq!(unwinding, match self.frame().loc {
-            Left(loc) => self.body().basic_blocks[loc.block].is_cleanup,
-            Right(_) => true,
-        });
+        assert_eq!(
+            unwinding,
+            match self.frame().loc {
+                Left(loc) => self.body().basic_blocks[loc.block].is_cleanup,
+                Right(_) => true,
+            }
+        );
         if unwinding && self.frame_idx() == 0 {
             throw_ub_custom!(fluent::const_eval_unwind_past_top);
         }

compiler/rustc_const_eval/src/interpret/intern.rs

@@ -102,11 +102,11 @@ fn intern_as_new_static<'tcx>(
     alloc_id: AllocId,
     alloc: ConstAllocation<'tcx>,
 ) {
-    let feed = tcx.create_def(static_id, sym::nested, DefKind::Static {
-        safety: hir::Safety::Safe,
-        mutability: alloc.0.mutability,
-        nested: true,
-    });
+    let feed = tcx.create_def(
+        static_id,
+        sym::nested,
+        DefKind::Static { safety: hir::Safety::Safe, mutability: alloc.0.mutability, nested: true },
+    );
     tcx.set_nested_alloc_id_static(alloc_id, feed.def_id());
 
     if tcx.is_thread_local_static(static_id.into()) {

compiler/rustc_const_eval/src/interpret/validity.rs

@@ -812,10 +812,10 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
                 if start == 1 && end == max_value {
                     // Only null is the niche. So make sure the ptr is NOT null.
                     if self.ecx.scalar_may_be_null(scalar)? {
-                        throw_validation_failure!(self.path, NullablePtrOutOfRange {
-                            range: valid_range,
-                            max_value
-                        })
+                        throw_validation_failure!(
+                            self.path,
+                            NullablePtrOutOfRange { range: valid_range, max_value }
+                        )
                     } else {
                         return interp_ok(());
                     }
@@ -825,10 +825,10 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
                 } else {
                     // Conservatively, we reject, because the pointer *could* have a bad
                     // value.
-                    throw_validation_failure!(self.path, PtrOutOfRange {
-                        range: valid_range,
-                        max_value
-                    })
+                    throw_validation_failure!(
+                        self.path,
+                        PtrOutOfRange { range: valid_range, max_value }
+                    )
                 }
             }
         };
@@ -836,11 +836,10 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
         if valid_range.contains(bits) {
             interp_ok(())
         } else {
-            throw_validation_failure!(self.path, OutOfRange {
-                value: format!("{bits}"),
-                range: valid_range,
-                max_value
-            })
+            throw_validation_failure!(
+                self.path,
+                OutOfRange { value: format!("{bits}"), range: valid_range, max_value }
+            )
         }
     }
 

compiler/rustc_data_structures/src/graph/dominators/tests.rs

@@ -15,17 +15,10 @@ fn diamond() {
 #[test]
 fn paper() {
     // example from the paper:
-    let graph = TestGraph::new(6, &[
-        (6, 5),
-        (6, 4),
-        (5, 1),
-        (4, 2),
-        (4, 3),
-        (1, 2),
-        (2, 3),
-        (3, 2),
-        (2, 1),
-    ]);
+    let graph = TestGraph::new(
+        6,
+        &[(6, 5), (6, 4), (5, 1), (4, 2), (4, 3), (1, 2), (2, 3), (3, 2), (2, 1)],
+    );
 
     let d = dominators(&graph);
     assert_eq!(d.immediate_dominator(0), None); // <-- note that 0 is not in graph
@@ -40,19 +33,10 @@ fn paper() {
 #[test]
 fn paper_slt() {
     // example from the paper:
-    let graph = TestGraph::new(1, &[
-        (1, 2),
-        (1, 3),
-        (2, 3),
-        (2, 7),
-        (3, 4),
-        (3, 6),
-        (4, 5),
-        (5, 4),
-        (6, 7),
-        (7, 8),
-        (8, 5),
-    ]);
+    let graph = TestGraph::new(
+        1,
+        &[(1, 2), (1, 3), (2, 3), (2, 7), (3, 4), (3, 6), (4, 5), (5, 4), (6, 7), (7, 8), (8, 5)],
+    );
 
     dominators(&graph);
 }
@@ -69,21 +53,24 @@ fn immediate_dominator() {
 
 #[test]
 fn transitive_dominator() {
-    let graph = TestGraph::new(0, &[
-        // First tree branch.
-        (0, 1),
-        (1, 2),
-        (2, 3),
-        (3, 4),
-        // Second tree branch.
-        (1, 5),
-        (5, 6),
-        // Third tree branch.
-        (0, 7),
-        // These links make 0 the dominator for 2 and 3.
-        (7, 2),
-        (5, 3),
-    ]);
+    let graph = TestGraph::new(
+        0,
+        &[
+            // First tree branch.
+            (0, 1),
+            (1, 2),
+            (2, 3),
+            (3, 4),
+            // Second tree branch.
+            (1, 5),
+            (5, 6),
+            // Third tree branch.
+            (0, 7),
+            // These links make 0 the dominator for 2 and 3.
+            (7, 2),
+            (5, 3),
+        ],
+    );
 
     let d = dominators(&graph);
     assert_eq!(d.immediate_dominator(2), Some(0));

compiler/rustc_data_structures/src/graph/implementation/tests.rs

@@ -110,10 +110,13 @@ fn each_adjacent_from_a() {
 #[test]
 fn each_adjacent_from_b() {
     let graph = create_graph();
-    test_adjacent_edges(&graph, NodeIndex(1), "B", &[("FB", "F"), ("AB", "A")], &[
-        ("BD", "D"),
-        ("BC", "C"),
-    ]);
+    test_adjacent_edges(
+        &graph,
+        NodeIndex(1),
+        "B",
+        &[("FB", "F"), ("AB", "A")],
+        &[("BD", "D"), ("BC", "C")],
+    );
 }
 
 #[test]

compiler/rustc_data_structures/src/graph/scc/tests.rs

@@ -326,46 +326,49 @@ fn test_bug_max_leak_minimised() {
 
 #[test]
 fn test_bug_max_leak() {
-    let graph = TestGraph::new(8, &[
-        (0, 0),
-        (0, 18),
-        (0, 19),
-        (0, 1),
-        (0, 2),
-        (0, 7),
-        (0, 8),
-        (0, 23),
-        (18, 0),
-        (18, 12),
-        (19, 0),
-        (19, 25),
-        (12, 18),
-        (12, 3),
-        (12, 5),
-        (3, 12),
-        (3, 21),
-        (3, 22),
-        (5, 13),
-        (21, 3),
-        (22, 3),
-        (13, 5),
-        (13, 4),
-        (4, 13),
-        (4, 0),
-        (2, 11),
-        (7, 6),
-        (6, 20),
-        (20, 6),
-        (8, 17),
-        (17, 9),
-        (9, 16),
-        (16, 26),
-        (26, 15),
-        (15, 10),
-        (10, 14),
-        (14, 27),
-        (23, 24),
-    ]);
+    let graph = TestGraph::new(
+        8,
+        &[
+            (0, 0),
+            (0, 18),
+            (0, 19),
+            (0, 1),
+            (0, 2),
+            (0, 7),
+            (0, 8),
+            (0, 23),
+            (18, 0),
+            (18, 12),
+            (19, 0),
+            (19, 25),
+            (12, 18),
+            (12, 3),
+            (12, 5),
+            (3, 12),
+            (3, 21),
+            (3, 22),
+            (5, 13),
+            (21, 3),
+            (22, 3),
+            (13, 5),
+            (13, 4),
+            (4, 13),
+            (4, 0),
+            (2, 11),
+            (7, 6),
+            (6, 20),
+            (20, 6),
+            (8, 17),
+            (17, 9),
+            (9, 16),
+            (16, 26),
+            (26, 15),
+            (15, 10),
+            (10, 14),
+            (14, 27),
+            (23, 24),
+        ],
+    );
     let sccs: MaxReachedSccs = Sccs::new_with_annotation(&graph, |w| match w {
         22 => MaxReached(1),
         24 => MaxReached(2),

compiler/rustc_data_structures/src/obligation_forest/tests.rs

@@ -349,10 +349,10 @@ fn diamond() {
     ));
     assert_eq!(d_count, 1);
     assert_eq!(ok.len(), 0);
-    assert_eq!(err, vec![super::Error {
-        error: "operation failed",
-        backtrace: vec!["D'", "A'.1", "A'"]
-    }]);
+    assert_eq!(
+        err,
+        vec![super::Error { error: "operation failed", backtrace: vec!["D'", "A'.1", "A'"] }]
+    );
 
     let errors = forest.to_errors(());
     assert_eq!(errors.len(), 0);

compiler/rustc_data_structures/src/vec_cache/tests.rs

@@ -58,11 +58,14 @@ fn concurrent_stress_check() {
 
 #[test]
 fn slot_entries_table() {
-    assert_eq!(ENTRIES_BY_BUCKET, [
-        4096, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576, 2097152, 4194304,
-        8388608, 16777216, 33554432, 67108864, 134217728, 268435456, 536870912, 1073741824,
-        2147483648
-    ]);
+    assert_eq!(
+        ENTRIES_BY_BUCKET,
+        [
+            4096, 4096, 8192, 16384, 32768, 65536, 131072, 262144, 524288, 1048576, 2097152,
+            4194304, 8388608, 16777216, 33554432, 67108864, 134217728, 268435456, 536870912,
+            1073741824, 2147483648
+        ]
+    );
 }
 
 #[test]

compiler/rustc_errors/src/json/tests.rs

@@ -69,96 +69,128 @@ fn test_positions(code: &str, span: (u32, u32), expected_output: SpanTestData) {
 
 #[test]
 fn empty() {
-    test_positions(" ", (0, 1), SpanTestData {
-        byte_start: 0,
-        byte_end: 1,
-        line_start: 1,
-        column_start: 1,
-        line_end: 1,
-        column_end: 2,
-    })
+    test_positions(
+        " ",
+        (0, 1),
+        SpanTestData {
+            byte_start: 0,
+            byte_end: 1,
+            line_start: 1,
+            column_start: 1,
+            line_end: 1,
+            column_end: 2,
+        },
+    )
 }
 
 #[test]
 fn bom() {
-    test_positions("\u{feff} ", (0, 1), SpanTestData {
-        byte_start: 3,
-        byte_end: 4,
-        line_start: 1,
-        column_start: 1,
-        line_end: 1,
-        column_end: 2,
-    })
+    test_positions(
+        "\u{feff} ",
+        (0, 1),
+        SpanTestData {
+            byte_start: 3,
+            byte_end: 4,
+            line_start: 1,
+            column_start: 1,
+            line_end: 1,
+            column_end: 2,
+        },
+    )
 }
 
 #[test]
 fn lf_newlines() {
-    test_positions("\nmod foo;\nmod bar;\n", (5, 12), SpanTestData {
-        byte_start: 5,
-        byte_end: 12,
-        line_start: 2,
-        column_start: 5,
-        line_end: 3,
-        column_end: 3,
-    })
+    test_positions(
+        "\nmod foo;\nmod bar;\n",
+        (5, 12),
+        SpanTestData {
+            byte_start: 5,
+            byte_end: 12,
+            line_start: 2,
+            column_start: 5,
+            line_end: 3,
+            column_end: 3,
+        },
+    )
 }
 
 #[test]
 fn crlf_newlines() {
-    test_positions("\r\nmod foo;\r\nmod bar;\r\n", (5, 12), SpanTestData {
-        byte_start: 6,
-        byte_end: 14,
-        line_start: 2,
-        column_start: 5,
-        line_end: 3,
-        column_end: 3,
-    })
+    test_positions(
+        "\r\nmod foo;\r\nmod bar;\r\n",
+        (5, 12),
+        SpanTestData {
+            byte_start: 6,
+            byte_end: 14,
+            line_start: 2,
+            column_start: 5,
+            line_end: 3,
+            column_end: 3,
+        },
+    )
 }
 
 #[test]
 fn crlf_newlines_with_bom() {
-    test_positions("\u{feff}\r\nmod foo;\r\nmod bar;\r\n", (5, 12), SpanTestData {
-        byte_start: 9,
-        byte_end: 17,
-        line_start: 2,
-        column_start: 5,
-        line_end: 3,
-        column_end: 3,
-    })
+    test_positions(
+        "\u{feff}\r\nmod foo;\r\nmod bar;\r\n",
+        (5, 12),
+        SpanTestData {
+            byte_start: 9,
+            byte_end: 17,
+            line_start: 2,
+            column_start: 5,
+            line_end: 3,
+            column_end: 3,
+        },
+    )
 }
 
 #[test]
 fn span_before_crlf() {
-    test_positions("foo\r\nbar", (2, 3), SpanTestData {
-        byte_start: 2,
-        byte_end: 3,
-        line_start: 1,
-        column_start: 3,
-        line_end: 1,
-        column_end: 4,
-    })
+    test_positions(
+        "foo\r\nbar",
+        (2, 3),
+        SpanTestData {
+            byte_start: 2,
+            byte_end: 3,
+            line_start: 1,
+            column_start: 3,
+            line_end: 1,
+            column_end: 4,
+        },
+    )
 }
 
 #[test]
 fn span_on_crlf() {
-    test_positions("foo\r\nbar", (3, 4), SpanTestData {
-        byte_start: 3,
-        byte_end: 5,
-        line_start: 1,
-        column_start: 4,
-        line_end: 2,
-        column_end: 1,
-    })
+    test_positions(
+        "foo\r\nbar",
+        (3, 4),
+        SpanTestData {
+            byte_start: 3,
+            byte_end: 5,
+            line_start: 1,
+            column_start: 4,
+            line_end: 2,
+            column_end: 1,
+        },
+    )
 }
 
 #[test]
 fn span_after_crlf() {
-    test_positions("foo\r\nbar", (4, 5), SpanTestData {
-        byte_start: 5,
-        byte_end: 6,
-        line_start: 2,
-        column_start: 1,
-        line_end: 2,
-        column_end: 2,
-    })
+    test_positions(
+        "foo\r\nbar",
+        (4, 5),
+        SpanTestData {
+            byte_start: 5,
+            byte_end: 6,
+            line_start: 2,
+            column_start: 1,
+            line_end: 2,
+            column_end: 2,
+        },
+    )
 }

compiler/rustc_expand/src/build.rs

@@ -233,11 +233,14 @@ impl<'a> ExtCtxt<'a> {
     }
 
     pub fn block_expr(&self, expr: P<ast::Expr>) -> P<ast::Block> {
-        self.block(expr.span, thin_vec![ast::Stmt {
-            id: ast::DUMMY_NODE_ID,
-            span: expr.span,
-            kind: ast::StmtKind::Expr(expr),
-        }])
+        self.block(
+            expr.span,
+            thin_vec![ast::Stmt {
+                id: ast::DUMMY_NODE_ID,
+                span: expr.span,
+                kind: ast::StmtKind::Expr(expr),
+            }],
+        )
     }
     pub fn block(&self, span: Span, stmts: ThinVec<ast::Stmt>) -> P<ast::Block> {
         P(ast::Block {

compiler/rustc_expand/src/mbe/macro_rules.rs

@@ -405,26 +405,35 @@ pub fn compile_declarative_macro(
     // ...quasiquoting this would be nice.
     // These spans won't matter, anyways
     let argument_gram = vec![
-        mbe::TokenTree::Sequence(DelimSpan::dummy(), mbe::SequenceRepetition {
-            tts: vec![
-                mbe::TokenTree::MetaVarDecl(span, lhs_nm, tt_spec),
-                mbe::TokenTree::token(token::FatArrow, span),
-                mbe::TokenTree::MetaVarDecl(span, rhs_nm, tt_spec),
-            ],
-            separator: Some(Token::new(if macro_rules { token::Semi } else { token::Comma }, span)),
-            kleene: mbe::KleeneToken::new(mbe::KleeneOp::OneOrMore, span),
-            num_captures: 2,
-        }),
+        mbe::TokenTree::Sequence(
+            DelimSpan::dummy(),
+            mbe::SequenceRepetition {
+                tts: vec![
+                    mbe::TokenTree::MetaVarDecl(span, lhs_nm, tt_spec),
+                    mbe::TokenTree::token(token::FatArrow, span),
+                    mbe::TokenTree::MetaVarDecl(span, rhs_nm, tt_spec),
+                ],
+                separator: Some(Token::new(
+                    if macro_rules { token::Semi } else { token::Comma },
+                    span,
+                )),
+                kleene: mbe::KleeneToken::new(mbe::KleeneOp::OneOrMore, span),
+                num_captures: 2,
+            },
+        ),
         // to phase into semicolon-termination instead of semicolon-separation
-        mbe::TokenTree::Sequence(DelimSpan::dummy(), mbe::SequenceRepetition {
-            tts: vec![mbe::TokenTree::token(
-                if macro_rules { token::Semi } else { token::Comma },
-                span,
-            )],
-            separator: None,
-            kleene: mbe::KleeneToken::new(mbe::KleeneOp::ZeroOrMore, span),
-            num_captures: 0,
-        }),
+        mbe::TokenTree::Sequence(
+            DelimSpan::dummy(),
+            mbe::SequenceRepetition {
+                tts: vec![mbe::TokenTree::token(
+                    if macro_rules { token::Semi } else { token::Comma },
+                    span,
+                )],
+                separator: None,
+                kleene: mbe::KleeneToken::new(mbe::KleeneOp::ZeroOrMore, span),
+                num_captures: 0,
+            },
+        ),
     ];
     // Convert it into `MatcherLoc` form.
     let argument_gram = mbe::macro_parser::compute_locs(&argument_gram);