From 29b859335bf41c3037a9f5d386fef1c951259c3c Mon Sep 17 00:00:00 2001
From: David CARLIER <devnexen@gmail.com>
Date: Tue, 29 Aug 2023 20:46:57 +0100
Subject: [PATCH 001/297] std::thread::set_name exploit the return on haiku

---
 library/std/src/sys/unix/thread.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/library/std/src/sys/unix/thread.rs b/library/std/src/sys/unix/thread.rs
index 4f2d9cf36553..b479642c2f5e 100644
--- a/library/std/src/sys/unix/thread.rs
+++ b/library/std/src/sys/unix/thread.rs
@@ -204,7 +204,9 @@ impl Thread {
     pub fn set_name(name: &CStr) {
         unsafe {
             let thread_self = libc::find_thread(ptr::null_mut());
-            libc::rename_thread(thread_self, name.as_ptr());
+            let res = libc::rename_thread(thread_self, name.as_ptr());
+            // We have no good way of propagating errors here, but in debug-builds let's check that this actually worked.
+            debug_assert_eq!(res, libc::B_OK);
         }
     }
 

From ecefd4eb3f3bbd14d715586f8b6c5caae6779a6c Mon Sep 17 00:00:00 2001
From: Taiki Endo <te316e89@gmail.com>
Date: Sat, 23 Sep 2023 20:44:03 +0900
Subject: [PATCH 002/297] Set max_atomic_width for riscv32*-esp-espidf to 32

---
 compiler/rustc_target/src/spec/riscv32imac_esp_espidf.rs | 2 +-
 compiler/rustc_target/src/spec/riscv32imc_esp_espidf.rs  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_target/src/spec/riscv32imac_esp_espidf.rs b/compiler/rustc_target/src/spec/riscv32imac_esp_espidf.rs
index 0795065409ad..25f5c3bc2a64 100644
--- a/compiler/rustc_target/src/spec/riscv32imac_esp_espidf.rs
+++ b/compiler/rustc_target/src/spec/riscv32imac_esp_espidf.rs
@@ -17,7 +17,7 @@ pub fn target() -> Target {
 
             // As RiscV32IMAC architecture does natively support atomics,
             // automatically enable the support for the Rust STD library.
-            max_atomic_width: Some(64),
+            max_atomic_width: Some(32),
             atomic_cas: true,
 
             features: "+m,+a,+c".into(),
diff --git a/compiler/rustc_target/src/spec/riscv32imc_esp_espidf.rs b/compiler/rustc_target/src/spec/riscv32imc_esp_espidf.rs
index 25638a092b5d..3aa9923ee4dd 100644
--- a/compiler/rustc_target/src/spec/riscv32imc_esp_espidf.rs
+++ b/compiler/rustc_target/src/spec/riscv32imc_esp_espidf.rs
@@ -16,11 +16,11 @@ pub fn target() -> Target {
             cpu: "generic-rv32".into(),
 
             // While the RiscV32IMC architecture does not natively support atomics, ESP-IDF does support
-            // the __atomic* and __sync* GCC builtins, so setting `max_atomic_width` to `Some(64)`
+            // the __atomic* and __sync* GCC builtins, so setting `max_atomic_width` to `Some(32)`
             // and `atomic_cas` to `true` will cause the compiler to emit libcalls to these builtins.
             //
             // Support for atomics is necessary for the Rust STD library, which is supported by the ESP-IDF framework.
-            max_atomic_width: Some(64),
+            max_atomic_width: Some(32),
             atomic_cas: true,
 
             features: "+m,+c".into(),

From fd207e67d5923f4cec2a96c4b095a1e3bbdac5a7 Mon Sep 17 00:00:00 2001
From: David Tolnay <dtolnay@gmail.com>
Date: Thu, 28 Sep 2023 10:07:40 -0700
Subject: [PATCH 003/297] Const stabilize mem::discriminant

---
 library/core/src/intrinsics.rs        | 2 +-
 library/core/src/lib.rs               | 1 -
 library/core/src/mem/mod.rs           | 2 +-
 tests/ui/consts/const-eval/ub-enum.rs | 2 +-
 tests/ui/consts/const_discriminant.rs | 1 -
 5 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/library/core/src/intrinsics.rs b/library/core/src/intrinsics.rs
index 4c76662ac097..e5362fd5aeaa 100644
--- a/library/core/src/intrinsics.rs
+++ b/library/core/src/intrinsics.rs
@@ -2277,7 +2277,7 @@ extern "rust-intrinsic" {
     /// any safety invariants.
     ///
     /// The stabilized version of this intrinsic is [`core::mem::discriminant`].
-    #[rustc_const_unstable(feature = "const_discriminant", issue = "69821")]
+    #[rustc_const_stable(feature = "const_discriminant", since = "CURRENT_RUSTC_VERSION")]
     #[rustc_safe_intrinsic]
     #[rustc_nounwind]
     pub fn discriminant_value<T>(v: &T) -> <T as DiscriminantKind>::Discriminant;
diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs
index 8b04bafcda54..1bcde4242cde 100644
--- a/library/core/src/lib.rs
+++ b/library/core/src/lib.rs
@@ -126,7 +126,6 @@
 #![feature(const_caller_location)]
 #![feature(const_cell_into_inner)]
 #![feature(const_char_from_u32_unchecked)]
-#![feature(const_discriminant)]
 #![feature(const_eval_select)]
 #![feature(const_exact_div)]
 #![feature(const_float_bits_conv)]
diff --git a/library/core/src/mem/mod.rs b/library/core/src/mem/mod.rs
index d7abc9a0e23a..ae88231ae6e4 100644
--- a/library/core/src/mem/mod.rs
+++ b/library/core/src/mem/mod.rs
@@ -1202,7 +1202,7 @@ impl<T> fmt::Debug for Discriminant<T> {
 /// // assert_eq!(0, unsafe { std::mem::transmute::<_, u8>(std::mem::discriminant(&unit_like)) });
 /// ```
 #[stable(feature = "discriminant_value", since = "1.21.0")]
-#[rustc_const_unstable(feature = "const_discriminant", issue = "69821")]
+#[rustc_const_stable(feature = "const_discriminant", since = "CURRENT_RUSTC_VERSION")]
 #[cfg_attr(not(test), rustc_diagnostic_item = "mem_discriminant")]
 #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
 pub const fn discriminant<T>(v: &T) -> Discriminant<T> {
diff --git a/tests/ui/consts/const-eval/ub-enum.rs b/tests/ui/consts/const-eval/ub-enum.rs
index 078283fbd1f7..72a0c9efed2c 100644
--- a/tests/ui/consts/const-eval/ub-enum.rs
+++ b/tests/ui/consts/const-eval/ub-enum.rs
@@ -2,7 +2,7 @@
 // Strip out raw byte dumps to make comparison platform-independent:
 // normalize-stderr-test "(the raw bytes of the constant) \(size: [0-9]*, align: [0-9]*\)" -> "$1 (size: $$SIZE, align: $$ALIGN)"
 // normalize-stderr-test "([0-9a-f][0-9a-f] |╾─*a(lloc)?[0-9]+(\+[a-z0-9]+)?─*╼ )+ *│.*" -> "HEX_DUMP"
-#![feature(never_type, const_discriminant)]
+#![feature(never_type)]
 #![allow(invalid_value)]
 
 use std::mem;
diff --git a/tests/ui/consts/const_discriminant.rs b/tests/ui/consts/const_discriminant.rs
index b1180faa6977..80deb0f784d2 100644
--- a/tests/ui/consts/const_discriminant.rs
+++ b/tests/ui/consts/const_discriminant.rs
@@ -1,5 +1,4 @@
 // run-pass
-#![feature(const_discriminant)]
 #![allow(dead_code)]
 
 use std::mem::{discriminant, Discriminant};

From 450d6c56ebc2dbc827c0db7a909f9f89d2a17961 Mon Sep 17 00:00:00 2001
From: Sebastian Imlay <sebastian.imlay@gmail.com>
Date: Tue, 12 Sep 2023 08:48:57 -0400
Subject: [PATCH 004/297] Initial target specification for
 aarch64-apple-tvos-sim

---
 .../src/spec/aarch64_apple_tvos_sim.rs        | 33 +++++++++++++++++++
 compiler/rustc_target/src/spec/mod.rs         |  1 +
 2 files changed, 34 insertions(+)
 create mode 100644 compiler/rustc_target/src/spec/aarch64_apple_tvos_sim.rs

diff --git a/compiler/rustc_target/src/spec/aarch64_apple_tvos_sim.rs b/compiler/rustc_target/src/spec/aarch64_apple_tvos_sim.rs
new file mode 100644
index 000000000000..f203ba53c2ae
--- /dev/null
+++ b/compiler/rustc_target/src/spec/aarch64_apple_tvos_sim.rs
@@ -0,0 +1,33 @@
+use super::apple_base::{opts, tvos_sim_llvm_target, Arch};
+use crate::spec::{FramePointer, SanitizerSet, Target, TargetOptions};
+
+pub fn target() -> Target {
+    let arch = Arch::Arm64_sim;
+    let mut base = opts("ios", arch);
+    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::THREAD;
+    Target {
+        llvm_target: tvos_sim_llvm_target(arch).into(),
+        pointer_width: 64,
+        data_layout: "e-m:o-i64:64-i128:128-n32:64-S128".into(),
+        arch: arch.target_arch(),
+        options: TargetOptions {
+            features: "+neon,+fp-armv8,+apple-a7".into(),
+            max_atomic_width: Some(128),
+            forces_embed_bitcode: true,
+            frame_pointer: FramePointer::NonLeaf,
+            // Taken from (and slightly modified) the aarch64-apple-ios-sim spec which says:
+            // Taken from a clang build on Xcode 11.4.1.
+            // These arguments are not actually invoked - they just have
+            // to look right to pass App Store validation.
+            bitcode_llvm_cmdline: "-triple\0\
+                arm64-apple-tvos14.0-simulator\0\
+                -emit-obj\0\
+                -disable-llvm-passes\0\
+                -target-abi\0\
+                darwinpcs\0\
+                -Os\0"
+                .into(),
+            ..base
+        },
+    }
+}
diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index 1bcb1f353159..cf8c5fff862b 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -1399,6 +1399,7 @@ supported_targets! {
     ("aarch64-apple-ios-macabi", aarch64_apple_ios_macabi),
     ("aarch64-apple-ios-sim", aarch64_apple_ios_sim),
     ("aarch64-apple-tvos", aarch64_apple_tvos),
+    ("aarch64-apple-tvos-sim", aarch64_apple_tvos_sim),
     ("x86_64-apple-tvos", x86_64_apple_tvos),
 
     ("armv7k-apple-watchos", armv7k_apple_watchos),

From 6c43244ff6e8b9725510680b5b03ac30f5d23d66 Mon Sep 17 00:00:00 2001
From: Sebastian Imlay <sebastian.imlay@gmail.com>
Date: Tue, 12 Sep 2023 09:22:24 -0400
Subject: [PATCH 005/297] Fix typos

---
 compiler/rustc_target/src/spec/aarch64_apple_tvos_sim.rs | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/compiler/rustc_target/src/spec/aarch64_apple_tvos_sim.rs b/compiler/rustc_target/src/spec/aarch64_apple_tvos_sim.rs
index f203ba53c2ae..909a52a5097e 100644
--- a/compiler/rustc_target/src/spec/aarch64_apple_tvos_sim.rs
+++ b/compiler/rustc_target/src/spec/aarch64_apple_tvos_sim.rs
@@ -1,10 +1,8 @@
 use super::apple_base::{opts, tvos_sim_llvm_target, Arch};
-use crate::spec::{FramePointer, SanitizerSet, Target, TargetOptions};
+use crate::spec::{FramePointer, Target, TargetOptions};
 
 pub fn target() -> Target {
     let arch = Arch::Arm64_sim;
-    let mut base = opts("ios", arch);
-    base.supported_sanitizers = SanitizerSet::ADDRESS | SanitizerSet::THREAD;
     Target {
         llvm_target: tvos_sim_llvm_target(arch).into(),
         pointer_width: 64,
@@ -20,14 +18,14 @@ pub fn target() -> Target {
             // These arguments are not actually invoked - they just have
             // to look right to pass App Store validation.
             bitcode_llvm_cmdline: "-triple\0\
-                arm64-apple-tvos14.0-simulator\0\
+                arm64-apple-tvos15.0-simulator\0\
                 -emit-obj\0\
                 -disable-llvm-passes\0\
                 -target-abi\0\
                 darwinpcs\0\
                 -Os\0"
                 .into(),
-            ..base
+            ..opts("tvos", arch)
         },
     }
 }

From ba5eeda64eef357f7e848f08d5f8ec8a27c2b574 Mon Sep 17 00:00:00 2001
From: Sebastian Imlay <sebastian.imlay@gmail.com>
Date: Wed, 13 Sep 2023 13:35:29 -0400
Subject: [PATCH 006/297] Fix sdkname for tvos simulator

---
 compiler/rustc_codegen_ssa/src/back/link.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs
index c4a0f6291e7f..0c88302f01f7 100644
--- a/compiler/rustc_codegen_ssa/src/back/link.rs
+++ b/compiler/rustc_codegen_ssa/src/back/link.rs
@@ -2871,6 +2871,7 @@ fn add_apple_sdk(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) {
     }
 
     let sdk_name = match (arch.as_ref(), os.as_ref()) {
+        ("aarch64", "tvos") if llvm_target.ends_with("-simulator") => "appletvsimulator",
         ("aarch64", "tvos") => "appletvos",
         ("x86_64", "tvos") => "appletvsimulator",
         ("arm", "ios") => "iphoneos",

From 4ab4d48ee59968d8d519ccda5e12c9d200cc092f Mon Sep 17 00:00:00 2001
From: Sebastian Imlay <sebastian.imlay@gmail.com>
Date: Fri, 6 Oct 2023 17:53:13 -0400
Subject: [PATCH 007/297] Update platform docs for aarch64-apple-tvos-sim

---
 src/doc/rustc/src/platform-support.md            | 1 +
 src/doc/rustc/src/platform-support/apple-tvos.md | 4 +++-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/doc/rustc/src/platform-support.md b/src/doc/rustc/src/platform-support.md
index ef4eb5a19657..a45e8e02f0b4 100644
--- a/src/doc/rustc/src/platform-support.md
+++ b/src/doc/rustc/src/platform-support.md
@@ -225,6 +225,7 @@ target | std | host | notes
 -------|:---:|:----:|-------
 `aarch64-apple-ios-macabi` | ? |  | Apple Catalyst on ARM64
 [`aarch64-apple-tvos`](platform-support/apple-tvos.md) | ? |  | ARM64 tvOS
+[`aarch64-apple-tvos-sim`](platform-support/apple-tvos.md) | ? |  | ARM64 tvOS Simulator
 [`aarch64-apple-watchos-sim`](platform-support/apple-watchos.md) | ✓ |  | ARM64 Apple WatchOS Simulator
 [`aarch64-kmc-solid_asp3`](platform-support/kmc-solid.md) | ✓ |  | ARM64 SOLID with TOPPERS/ASP3
 [`aarch64-nintendo-switch-freestanding`](platform-support/aarch64-nintendo-switch-freestanding.md) | * |  | ARM64 Nintendo Switch, Horizon
diff --git a/src/doc/rustc/src/platform-support/apple-tvos.md b/src/doc/rustc/src/platform-support/apple-tvos.md
index d87fd1959b49..e7ea109df1ba 100644
--- a/src/doc/rustc/src/platform-support/apple-tvos.md
+++ b/src/doc/rustc/src/platform-support/apple-tvos.md
@@ -52,7 +52,7 @@ The targets can be built by enabling them for a `rustc` build in `config.toml`,
 ```toml
 [build]
 build-stage = 1
-target = ["aarch64-apple-tvos", "x86_64-apple-tvos"]
+target = ["aarch64-apple-tvos", "x86_64-apple-tvos", "aarch64-apple-tvos-sim"]
 ```
 
 It's possible that cargo under `-Zbuild-std` may also be used to target them.
@@ -67,6 +67,8 @@ Rust programs can be built for these targets
 $ rustc --target aarch64-apple-tvos your-code.rs
 ...
 $ rustc --target x86_64-apple-tvos your-code.rs
+...
+$ rustc --target aarch64-apple-tvos-sim your-code.rs
 ```
 
 ## Testing

From e960d0e751f0348726505a3d000023315259488e Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sun, 8 Oct 2023 12:21:58 +0000
Subject: [PATCH 008/297] Remove -Zdep-tasks.

---
 compiler/rustc_interface/src/tests.rs         |  1 -
 .../rustc_query_system/src/dep_graph/graph.rs | 26 +++----------------
 compiler/rustc_session/src/options.rs         |  3 ---
 3 files changed, 4 insertions(+), 26 deletions(-)

diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index 2510ce714602..60b9946f8266 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -681,7 +681,6 @@ fn test_unstable_options_tracking_hash() {
     // tidy-alphabetical-start
     untracked!(assert_incr_state, Some(String::from("loaded")));
     untracked!(deduplicate_diagnostics, false);
-    untracked!(dep_tasks, true);
     untracked!(dont_buffer_diagnostics, true);
     untracked!(dump_dep_graph, true);
     untracked!(dump_mir, Some(String::from("abc")));
diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs
index c7e92d7b2b22..a44000700cbf 100644
--- a/compiler/rustc_query_system/src/dep_graph/graph.rs
+++ b/compiler/rustc_query_system/src/dep_graph/graph.rs
@@ -149,7 +149,6 @@ impl<D: Deps> DepGraph<D> {
             DepNode { kind: D::DEP_KIND_RED, hash: Fingerprint::ZERO.into() },
             EdgesVec::new(),
             None,
-            false,
         );
         assert_eq!(red_node_index, DepNodeIndex::FOREVER_RED_NODE);
         match red_node_prev_index_and_color {
@@ -373,8 +372,6 @@ impl<D: Deps> DepGraphData<D> {
         let current_fingerprint =
             hash_result.map(|f| dcx.with_stable_hashing_context(|mut hcx| f(&mut hcx, &result)));
 
-        let print_status = cfg!(debug_assertions) && dcx.sess().opts.unstable_opts.dep_tasks;
-
         // Intern the new `DepNode`.
         let (dep_node_index, prev_and_color) = self.current.intern_node(
             dcx.profiler(),
@@ -382,7 +379,6 @@ impl<D: Deps> DepGraphData<D> {
             key,
             edges,
             current_fingerprint,
-            print_status,
         );
 
         hashing_timer.finish_with_query_invocation_id(dep_node_index.into());
@@ -589,8 +585,6 @@ impl<D: Deps> DepGraph<D> {
                 cx.with_stable_hashing_context(|mut hcx| hash_result(&mut hcx, result))
             });
 
-            let print_status = cfg!(debug_assertions) && cx.sess().opts.unstable_opts.dep_tasks;
-
             // Intern the new `DepNode` with the dependencies up-to-now.
             let (dep_node_index, prev_and_color) = data.current.intern_node(
                 cx.profiler(),
@@ -598,7 +592,6 @@ impl<D: Deps> DepGraph<D> {
                 node,
                 edges,
                 current_fingerprint,
-                print_status,
             );
 
             hashing_timer.finish_with_query_invocation_id(dep_node_index.into());
@@ -1219,20 +1212,13 @@ impl<D: Deps> CurrentDepGraph<D> {
         key: DepNode,
         edges: EdgesVec,
         fingerprint: Option<Fingerprint>,
-        print_status: bool,
     ) -> (DepNodeIndex, Option<(SerializedDepNodeIndex, DepNodeColor)>) {
-        let print_status = cfg!(debug_assertions) && print_status;
-
         // Get timer for profiling `DepNode` interning
         let _node_intern_timer =
             self.node_intern_event_id.map(|eid| profiler.generic_activity_with_event_id(eid));
 
         if let Some(prev_index) = prev_graph.node_to_index_opt(&key) {
-            let get_dep_node_index = |color, fingerprint| {
-                if print_status {
-                    eprintln!("[task::{color:}] {key:?}");
-                }
-
+            let get_dep_node_index = |fingerprint| {
                 let mut prev_index_to_index = self.prev_index_to_index.lock();
 
                 let dep_node_index = match prev_index_to_index[prev_index] {
@@ -1256,12 +1242,12 @@ impl<D: Deps> CurrentDepGraph<D> {
                 if fingerprint == prev_graph.fingerprint_by_index(prev_index) {
                     // This is a green node: it existed in the previous compilation,
                     // its query was re-executed, and it has the same result as before.
-                    let dep_node_index = get_dep_node_index("green", fingerprint);
+                    let dep_node_index = get_dep_node_index(fingerprint);
                     (dep_node_index, Some((prev_index, DepNodeColor::Green(dep_node_index))))
                 } else {
                     // This is a red node: it existed in the previous compilation, its query
                     // was re-executed, but it has a different result from before.
-                    let dep_node_index = get_dep_node_index("red", fingerprint);
+                    let dep_node_index = get_dep_node_index(fingerprint);
                     (dep_node_index, Some((prev_index, DepNodeColor::Red)))
                 }
             } else {
@@ -1269,14 +1255,10 @@ impl<D: Deps> CurrentDepGraph<D> {
                 // session, its query was re-executed, but it doesn't compute a result hash
                 // (i.e. it represents a `no_hash` query), so we have no way of determining
                 // whether or not the result was the same as before.
-                let dep_node_index = get_dep_node_index("unknown", Fingerprint::ZERO);
+                let dep_node_index = get_dep_node_index(Fingerprint::ZERO);
                 (dep_node_index, Some((prev_index, DepNodeColor::Red)))
             }
         } else {
-            if print_status {
-                eprintln!("[task::new] {key:?}");
-            }
-
             let fingerprint = fingerprint.unwrap_or(Fingerprint::ZERO);
 
             // This is a new node: it didn't exist in the previous compilation session.
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index c1424db600ef..5c9438ab9bd9 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -1452,9 +1452,6 @@ options! {
     dep_info_omit_d_target: bool = (false, parse_bool, [TRACKED],
         "in dep-info output, omit targets for tracking dependencies of the dep-info files \
         themselves (default: no)"),
-    dep_tasks: bool = (false, parse_bool, [UNTRACKED],
-        "print tasks that execute and the color their dep node gets (requires debug build) \
-        (default: no)"),
     dont_buffer_diagnostics: bool = (false, parse_bool, [UNTRACKED],
         "emit diagnostics rather than buffering (breaks NLL error downgrading, sorting) \
         (default: no)"),

From 8531ebfc951a2002207bc5e604bc7584c1c1a560 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sun, 8 Oct 2023 12:24:20 +0000
Subject: [PATCH 009/297] Remove useless debugging.

---
 compiler/rustc_query_impl/src/plumbing.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
index 4516708ce17c..5bdf88aea1c5 100644
--- a/compiler/rustc_query_impl/src/plumbing.rs
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -437,8 +437,6 @@ where
     );
 
     if let Some(key) = Q::Key::recover(tcx, &dep_node) {
-        #[cfg(debug_assertions)]
-        let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered();
         force_query(query, QueryCtxt::new(tcx), key, dep_node);
         true
     } else {

From 242a482c88ea629b54a2d870e0d66738dd3900df Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Mon, 9 Oct 2023 15:53:34 -0400
Subject: [PATCH 010/297] Merge commit
 '11a0cceab966e5ff1058ddbcab5977e8a1d6d290' into
 subtree-update_cg_gcc_2023-10-09

---
 .github/workflows/ci.yml                      |  18 +-
 .github/workflows/release.yml                 |  26 +-
 .github/workflows/stdarch.yml                 |  24 +-
 .gitignore                                    |   1 +
 Cargo.lock                                    | 261 ++++++++------
 Cargo.toml                                    |   6 +
 Readme.md                                     |  94 ++++-
 build.sh                                      |  67 ----
 build_sysroot/Cargo.toml                      |   2 +
 build_sysroot/build_sysroot.sh                |   4 +-
 build_sysroot/prepare_sysroot_src.sh          |  39 --
 build_system/Cargo.lock                       |   7 +
 build_system/Cargo.toml                       |   8 +
 build_system/src/build.rs                     | 233 ++++++++++++
 build_system/src/config.rs                    | 125 +++++++
 build_system/src/main.rs                      |  62 ++++
 build_system/src/prepare.rs                   | 227 ++++++++++++
 build_system/src/rustc_info.rs                |  12 +
 build_system/src/utils.rs                     | 240 ++++++++++++
 config.sh                                     |  15 +-
 doc/add-attribute.md                          |  17 +
 doc/gimple.md                                 | 111 ++++++
 doc/sending-gcc-patch.md                      |  44 +++
 example/alloc_example.rs                      |   1 +
 example/alloc_system.rs                       |   2 +-
 ...itrary_self_types_pointers_and_wrappers.rs |   1 +
 example/mini_core.rs                          |  11 +-
 example/mini_core_hello_world.rs              |   2 +-
 example/mod_bench.rs                          |   1 +
 failing-lto-tests.txt                         |  23 ++
 failing-non-lto-tests.txt                     |  11 +
 failing-ui-tests.txt                          |  27 +-
 failing-ui-tests12.txt                        |   1 +
 messages.ftl                                  |  30 ++
 ...1-Add-stdarch-Cargo.toml-for-testing.patch |  19 +-
 patches/0001-Disable-examples.patch           |  25 --
 prepare.sh                                    |  30 --
 prepare_build.sh                              |   5 -
 rust-toolchain                                |   2 +-
 rustup.sh                                     |   2 +-
 src/abi.rs                                    |  44 ++-
 src/allocator.rs                              | 135 ++++---
 src/asm.rs                                    |   4 -
 src/attributes.rs                             | 102 ++----
 src/back/lto.rs                               | 341 ++++++++++++++++++
 src/back/mod.rs                               |   1 +
 src/back/write.rs                             | 102 +++++-
 src/base.rs                                   |  71 ++--
 src/builder.rs                                |  29 +-
 src/declare.rs                                |  40 ++
 src/errors.rs                                 |  87 ++++-
 src/gcc_util.rs                               | 223 ++++++++++++
 src/int.rs                                    |  81 ++++-
 src/intrinsic/archs.rs                        | 114 +++++-
 src/intrinsic/llvm.rs                         |  20 +
 src/intrinsic/mod.rs                          |   4 +-
 src/lib.rs                                    | 182 ++++++----
 src/type_of.rs                                |  10 +-
 test.sh                                       |  14 +-
 tests/run/abort1.rs                           |   3 +-
 tests/run/abort2.rs                           |   3 +-
 tests/run/array.rs                            |   3 +-
 tests/run/asm.rs                              |   2 +-
 tests/run/assign.rs                           |   4 +-
 tests/run/closure.rs                          |   3 +-
 tests/run/condition.rs                        |   3 +-
 tests/run/empty_main.rs                       |   1 +
 tests/run/exit.rs                             |   1 +
 tests/run/exit_code.rs                        |   1 +
 tests/run/fun_ptr.rs                          |   3 +-
 tests/run/gep.rs                              |  10 +
 tests/run/int_overflow.rs                     |   4 +-
 tests/run/mut_ref.rs                          |   4 +-
 tests/run/operations.rs                       |   4 +-
 tests/run/ptr_cast.rs                         |   3 +-
 tests/run/return-tuple.rs                     |   1 +
 tests/run/slice.rs                            |   3 +-
 tests/run/static.rs                           |   3 +-
 tests/run/structs.rs                          |   1 +
 tests/run/tuple.rs                            |   1 +
 tools/generate_intrinsics.py                  |   8 +-
 y.sh                                          |   8 +
 82 files changed, 2848 insertions(+), 669 deletions(-)
 delete mode 100755 build.sh
 delete mode 100755 build_sysroot/prepare_sysroot_src.sh
 create mode 100644 build_system/Cargo.lock
 create mode 100644 build_system/Cargo.toml
 create mode 100644 build_system/src/build.rs
 create mode 100644 build_system/src/config.rs
 create mode 100644 build_system/src/main.rs
 create mode 100644 build_system/src/prepare.rs
 create mode 100644 build_system/src/rustc_info.rs
 create mode 100644 build_system/src/utils.rs
 create mode 100644 doc/add-attribute.md
 create mode 100644 doc/gimple.md
 create mode 100644 doc/sending-gcc-patch.md
 create mode 100644 failing-lto-tests.txt
 create mode 100644 failing-non-lto-tests.txt
 delete mode 100644 patches/0001-Disable-examples.patch
 delete mode 100755 prepare.sh
 delete mode 100755 prepare_build.sh
 create mode 100644 src/back/lto.rs
 create mode 100644 src/gcc_util.rs
 create mode 100644 tests/run/gep.rs
 create mode 100755 y.sh

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d2b7724a2215..f075c744e457 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -57,8 +57,8 @@ jobs:
       uses: dawidd6/action-download-artifact@v2
       with:
           workflow: main.yml
-          name: ${{ matrix.libgccjit_version.gcc }}
-          path: gcc-build
+          name: gcc-13
+          path: gcc-13
           repo: antoyo/gcc
           branch: ${{ matrix.libgccjit_version.artifacts_branch }}
           event: push
@@ -71,9 +71,8 @@ jobs:
     - name: Setup path to libgccjit
       if: matrix.libgccjit_version.gcc != 'libgccjit12.so'
       run: |
-          echo $(readlink -f gcc-build) > gcc_path
-          # NOTE: the filename is still libgccjit.so even when the artifact name is different.
-          ln gcc-build/libgccjit.so gcc-build/libgccjit.so.0
+          sudo dpkg --force-overwrite -i gcc-13/gcc-13.deb
+          echo /usr/lib/ > gcc_path
 
     - name: Set env
       run: |
@@ -119,8 +118,8 @@ jobs:
 
     - name: Build
       run: |
-        ./prepare_build.sh
-        ${{ matrix.libgccjit_version.env_extra }} ./build.sh ${{ matrix.libgccjit_version.extra }}
+        ./y.sh prepare --only-libcore
+        ${{ matrix.libgccjit_version.env_extra }} ./y.sh build ${{ matrix.libgccjit_version.extra }}
         ${{ matrix.libgccjit_version.env_extra }} cargo test ${{ matrix.libgccjit_version.extra }}
         ./clean_all.sh
 
@@ -128,7 +127,7 @@ jobs:
       run: |
         git config --global user.email "user@example.com"
         git config --global user.name "User"
-        ./prepare.sh
+        ./y.sh prepare
 
     # Compile is a separate step, as the actions-rs/cargo action supports error annotations
     - name: Compile
@@ -141,6 +140,9 @@ jobs:
       if: ${{ matrix.libgccjit_version.gcc == 'libgccjit12.so' }}
       run: cat failing-ui-tests12.txt >> failing-ui-tests.txt
 
+    - name: Add more failing tests because the sysroot is not compiled with LTO
+      run: cat failing-non-lto-tests.txt >> failing-ui-tests.txt
+
     - name: Run tests
       run: |
         ${{ matrix.libgccjit_version.env_extra }} ./test.sh --release --clean --build-sysroot ${{ matrix.commands }} ${{ matrix.libgccjit_version.extra }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index c4e99469bc20..bd0415040e7e 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -18,8 +18,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        libgccjit_version:
-          - { gcc: "libgccjit.so", artifacts_branch: "master" }
         commands: [
           "--test-successful-rustc --nb-parts 2 --current-part 0",
           "--test-successful-rustc --nb-parts 2 --current-part 1",
@@ -40,18 +38,17 @@ jobs:
       uses: dawidd6/action-download-artifact@v2
       with:
           workflow: main.yml
-          name: ${{ matrix.libgccjit_version.gcc }}
-          path: gcc-build
+          name: gcc-13
+          path: gcc-13
           repo: antoyo/gcc
-          branch: ${{ matrix.libgccjit_version.artifacts_branch }}
+          branch: "master"
           event: push
           search_artifacts: true # Because, instead, the action only check the last job ran and that won't work since we want multiple artifacts.
 
     - name: Setup path to libgccjit
       run: |
-          echo $(readlink -f gcc-build) > gcc_path
-          # NOTE: the filename is still libgccjit.so even when the artifact name is different.
-          ln gcc-build/libgccjit.so gcc-build/libgccjit.so.0
+          sudo dpkg --force-overwrite -i gcc-13/gcc-13.deb
+          echo /usr/lib/ > gcc_path
 
     - name: Set env
       run: |
@@ -88,8 +85,8 @@ jobs:
 
     - name: Build
       run: |
-        ./prepare_build.sh
-        ./build.sh --release --release-sysroot
+        ./y.sh prepare --only-libcore
+        EMBED_LTO_BITCODE=1 ./y.sh build --release --release-sysroot
         cargo test
         ./clean_all.sh
 
@@ -97,7 +94,9 @@ jobs:
       run: |
         git config --global user.email "user@example.com"
         git config --global user.name "User"
-        ./prepare.sh
+        ./y.sh prepare
+        # FIXME(antoyo): we cannot enable LTO for stdarch tests currently because of some failing LTO tests using proc-macros.
+        echo -n 'lto = "fat"' >> build_sysroot/Cargo.toml
 
     # Compile is a separate step, as the actions-rs/cargo action supports error annotations
     - name: Compile
@@ -106,6 +105,9 @@ jobs:
         command: build
         args: --release
 
+    - name: Add more failing tests because of undefined symbol errors (FIXME)
+      run: cat failing-lto-tests.txt >> failing-ui-tests.txt
+
     - name: Run tests
       run: |
-        ./test.sh --release --clean --release-sysroot --build-sysroot ${{ matrix.commands }}
+        EMBED_LTO_BITCODE=1 ./test.sh --release --clean --release-sysroot --build-sysroot ${{ matrix.commands }}
diff --git a/.github/workflows/stdarch.yml b/.github/workflows/stdarch.yml
index 556c64448332..6c28326823cc 100644
--- a/.github/workflows/stdarch.yml
+++ b/.github/workflows/stdarch.yml
@@ -18,8 +18,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        libgccjit_version:
-          - { gcc: "libgccjit.so", artifacts_branch: "master" }
         cargo_runner: [
           "sde -future -rtm_mode full --",
           "",
@@ -54,18 +52,17 @@ jobs:
       uses: dawidd6/action-download-artifact@v2
       with:
           workflow: main.yml
-          name: ${{ matrix.libgccjit_version.gcc }}
-          path: gcc-build
+          name: gcc-13
+          path: gcc-13
           repo: antoyo/gcc
-          branch: ${{ matrix.libgccjit_version.artifacts_branch }}
+          branch: "master"
           event: push
           search_artifacts: true # Because, instead, the action only check the last job ran and that won't work since we want multiple artifacts.
 
     - name: Setup path to libgccjit
       run: |
-          echo $(readlink -f gcc-build) > gcc_path
-          # NOTE: the filename is still libgccjit.so even when the artifact name is different.
-          ln gcc-build/libgccjit.so gcc-build/libgccjit.so.0
+          sudo dpkg --force-overwrite -i gcc-13/gcc-13.deb
+          echo /usr/lib/ > gcc_path
 
     - name: Set env
       run: |
@@ -102,8 +99,8 @@ jobs:
 
     - name: Build
       run: |
-        ./prepare_build.sh
-        ./build.sh --release --release-sysroot
+        ./y.sh prepare --only-libcore
+        ./y.sh build --release --release-sysroot
         cargo test
 
     - name: Clean
@@ -115,7 +112,7 @@ jobs:
       run: |
         git config --global user.email "user@example.com"
         git config --global user.name "User"
-        ./prepare.sh
+        ./y.sh prepare
 
     # Compile is a separate step, as the actions-rs/cargo action supports error annotations
     - name: Compile
@@ -133,10 +130,11 @@ jobs:
       if: ${{ !matrix.cargo_runner }}
       run: |
         cd build_sysroot/sysroot_src/library/stdarch/
-        CHANNEL=release TARGET=x86_64-unknown-linux-gnu ../../../../cargo.sh test
+        CHANNEL=release TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ../../../../cargo.sh test
 
     - name: Run stdarch tests
       if: ${{ matrix.cargo_runner }}
       run: |
         cd build_sysroot/sysroot_src/library/stdarch/
-        STDARCH_TEST_EVERYTHING=1 CHANNEL=release CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="${{ matrix.cargo_runner }}" TARGET=x86_64-unknown-linux-gnu ../../../../cargo.sh test -- --skip rtm --skip tbm --skip sse4a
+        # FIXME: these tests fail when the sysroot is compiled with LTO because of a missing symbol in proc-macro.
+        STDARCH_TEST_EVERYTHING=1 CHANNEL=release CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="${{ matrix.cargo_runner }}" TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ../../../../cargo.sh test -- --skip rtm --skip tbm --skip sse4a
diff --git a/.gitignore b/.gitignore
index c5ed7de200c2..b44d1aa78c2e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,3 +25,4 @@ tools/llvmint
 tools/llvmint-2
 # The `llvm` folder is generated by the `tools/generate_intrinsics.py` script to update intrinsics.
 llvm
+build_system/target
diff --git a/Cargo.lock b/Cargo.lock
index 1c8754bf675e..85675fc40c34 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4,9 +4,9 @@ version = 3
 
 [[package]]
 name = "aho-corasick"
-version = "0.7.18"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f"
+checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
 dependencies = [
  "memchr",
 ]
@@ -17,12 +17,51 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "bitflags"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
+
+[[package]]
+name = "cc"
+version = "1.0.79"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f"
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "errno"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
+dependencies = [
+ "errno-dragonfly",
+ "libc",
+ "windows-sys",
+]
+
+[[package]]
+name = "errno-dragonfly"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
+dependencies = [
+ "cc",
+ "libc",
+]
+
+[[package]]
+name = "fastrand"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764"
+
 [[package]]
 name = "fm"
 version = "0.1.4"
@@ -35,7 +74,7 @@ dependencies = [
 [[package]]
 name = "gccjit"
 version = "1.0.0"
-source = "git+https://github.com/antoyo/gccjit.rs#d6e52626cfc6f487094a5d5ac66302baf3439984"
+source = "git+https://github.com/antoyo/gccjit.rs#0b158c68bf7e46732869d90550a98e886dee8858"
 dependencies = [
  "gccjit_sys",
 ]
@@ -43,7 +82,7 @@ dependencies = [
 [[package]]
 name = "gccjit_sys"
 version = "0.0.1"
-source = "git+https://github.com/antoyo/gccjit.rs#d6e52626cfc6f487094a5d5ac66302baf3439984"
+source = "git+https://github.com/antoyo/gccjit.rs#0b158c68bf7e46732869d90550a98e886dee8858"
 dependencies = [
  "libc",
 ]
@@ -57,25 +96,11 @@ dependencies = [
  "unicode-width",
 ]
 
-[[package]]
-name = "getrandom"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
-dependencies = [
- "cfg-if",
- "libc",
- "wasi",
-]
-
 [[package]]
 name = "hermit-abi"
-version = "0.1.19"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
-dependencies = [
- "libc",
-]
+checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
 
 [[package]]
 name = "lang_tester"
@@ -95,86 +120,55 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.112"
+version = "0.2.147"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125"
+checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
+
+[[package]]
+name = "linux-raw-sys"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503"
 
 [[package]]
 name = "memchr"
-version = "2.4.1"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
+checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
 
 [[package]]
 name = "num_cpus"
-version = "1.13.0"
+version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
+checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
 dependencies = [
  "hermit-abi",
  "libc",
 ]
 
 [[package]]
-name = "ppv-lite86"
-version = "0.2.15"
+name = "object"
+version = "0.30.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ed0cfbc8191465bed66e1718596ee0b0b35d5ee1f41c5df2189d0fe8bde535ba"
-
-[[package]]
-name = "rand"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
+checksum = "03b4680b86d9cfafba8fc491dc9b6df26b68cf40e9e6cd73909194759a63c385"
 dependencies = [
- "libc",
- "rand_chacha",
- "rand_core",
- "rand_hc",
-]
-
-[[package]]
-name = "rand_chacha"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
-dependencies = [
- "ppv-lite86",
- "rand_core",
-]
-
-[[package]]
-name = "rand_core"
-version = "0.6.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
-dependencies = [
- "getrandom",
-]
-
-[[package]]
-name = "rand_hc"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7"
-dependencies = [
- "rand_core",
+ "memchr",
 ]
 
 [[package]]
 name = "redox_syscall"
-version = "0.2.10"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"
+checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 ]
 
 [[package]]
 name = "regex"
-version = "1.5.4"
+version = "1.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461"
+checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -183,18 +177,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.6.25"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
-
-[[package]]
-name = "remove_dir_all"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
-dependencies = [
- "winapi",
-]
+checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
 
 [[package]]
 name = "rustc_codegen_gcc"
@@ -202,10 +187,24 @@ version = "0.1.0"
 dependencies = [
  "gccjit",
  "lang_tester",
+ "object",
  "smallvec",
  "tempfile",
 ]
 
+[[package]]
+name = "rustix"
+version = "0.38.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f"
+dependencies = [
+ "bitflags 2.4.0",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys",
+]
+
 [[package]]
 name = "same-file"
 version = "1.0.6"
@@ -223,23 +222,22 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
 
 [[package]]
 name = "tempfile"
-version = "3.2.0"
+version = "3.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
+checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651"
 dependencies = [
  "cfg-if",
- "libc",
- "rand",
+ "fastrand",
  "redox_syscall",
- "remove_dir_all",
- "winapi",
+ "rustix",
+ "windows-sys",
 ]
 
 [[package]]
 name = "termcolor"
-version = "1.1.2"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4"
+checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6"
 dependencies = [
  "winapi-util",
 ]
@@ -255,9 +253,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-width"
-version = "0.1.9"
+version = "0.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
+checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
 
 [[package]]
 name = "wait-timeout"
@@ -270,21 +268,14 @@ dependencies = [
 
 [[package]]
 name = "walkdir"
-version = "2.3.2"
+version = "2.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56"
+checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698"
 dependencies = [
  "same-file",
- "winapi",
  "winapi-util",
 ]
 
-[[package]]
-name = "wasi"
-version = "0.10.2+wasi-snapshot-preview1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
-
 [[package]]
 name = "winapi"
 version = "0.3.9"
@@ -315,3 +306,69 @@ name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
+name = "windows-sys"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+dependencies = [
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.48.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f"
+dependencies = [
+ "windows_aarch64_gnullvm",
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_gnullvm",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
diff --git a/Cargo.toml b/Cargo.toml
index 81066d9ce1f0..51fab147aa27 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -27,7 +27,13 @@ gccjit = { git = "https://github.com/antoyo/gccjit.rs" }
 # Local copy.
 #gccjit = { path = "../gccjit.rs" }
 
+object = { version = "0.30.1", default-features = false, features = [
+    "std",
+    "read",
+] }
 smallvec = { version = "1.6.1", features = ["union", "may_dangle"] }
+# TODO(antoyo): make tempfile optional.
+tempfile = "3.7.1"
 
 [dev-dependencies]
 lang_tester = "0.3.9"
diff --git a/Readme.md b/Readme.md
index a93637d9038d..f001c83b08d3 100644
--- a/Readme.md
+++ b/Readme.md
@@ -1,6 +1,7 @@
 # WIP libgccjit codegen backend for rust
 
 [![Chat on IRC](https://img.shields.io/badge/irc.libera.chat-%23rustc__codegen__gcc-blue.svg)](https://web.libera.chat/#rustc_codegen_gcc)
+[![Chat on Matrix](https://img.shields.io/badge/matrix.org-%23rustc__codegen__gcc-blue.svg)](https://matrix.to/#/#rustc_codegen_gcc:matrix.org)
 
 This is a GCC codegen for rustc, which means it can be loaded by the existing rustc frontend, but benefits from GCC: more architectures are supported and GCC's optimizations are used.
 
@@ -14,9 +15,7 @@ A secondary goal is to check if using the gcc backend will provide any run-time
 ## Building
 
 **This requires a patched libgccjit in order to work.
-The patches in [this repository](https://github.com/antoyo/libgccjit-patches) need to be applied.
-(Those patches should work when applied on master, but in case it doesn't work, they are known to work when applied on 079c23cfe079f203d5df83fea8e92a60c7d7e878.)
-You can also use my [fork of gcc](https://github.com/antoyo/gcc) which already includes these patches.**
+You need to use my [fork of gcc](https://github.com/antoyo/gcc) which already includes these patches.**
 
 To build it (most of these instructions come from [here](https://gcc.gnu.org/onlinedocs/jit/internals/index.html), so don't hesitate to take a look there if you encounter an issue):
 
@@ -66,8 +65,8 @@ $ export RUST_COMPILER_RT_ROOT="$PWD/llvm/compiler-rt"
 Then you can run commands like this:
 
 ```bash
-$ ./prepare.sh # download and patch sysroot src and install hyperfine for benchmarking
-$ LIBRARY_PATH=$(cat gcc_path) LD_LIBRARY_PATH=$(cat gcc_path) ./build.sh --release
+$ ./y.sh prepare # download and patch sysroot src and install hyperfine for benchmarking
+$ LIBRARY_PATH=$(cat gcc_path) LD_LIBRARY_PATH=$(cat gcc_path) ./y.sh build --release
 ```
 
 To run the tests:
@@ -78,22 +77,29 @@ $ ./test.sh --release
 
 ## Usage
 
-`$cg_gccjit_dir` is the directory you cloned this repo into in the following instructions.
+`$CG_GCCJIT_DIR` is the directory you cloned this repo into in the following instructions:
+
+```bash
+export CG_GCCJIT_DIR=[the full path to rustc_codegen_gcc]
+```
 
 ### Cargo
 
 ```bash
-$ CHANNEL="release" $cg_gccjit_dir/cargo.sh run
+$ CHANNEL="release" $CG_GCCJIT_DIR/cargo.sh run
 ```
 
 If you compiled cg_gccjit in debug mode (aka you didn't pass `--release` to `./test.sh`) you should use `CHANNEL="debug"` instead or omit `CHANNEL="release"` completely.
 
+To use LTO, you need to set the variable `FAT_LTO=1` and `EMBED_LTO_BITCODE=1` in addition to setting `lto = "fat"` in the `Cargo.toml`.
+Don't set `FAT_LTO` when compiling the sysroot, though: only set `EMBED_LTO_BITCODE=1`.
+
 ### Rustc
 
 > You should prefer using the Cargo method.
 
 ```bash
-$ rustc +$(cat $cg_gccjit_dir/rust-toolchain) -Cpanic=abort -Zcodegen-backend=$cg_gccjit_dir/target/release/librustc_codegen_gcc.so --sysroot $cg_gccjit_dir/build_sysroot/sysroot my_crate.rs
+$ LIBRARY_PATH=$(cat gcc_path) LD_LIBRARY_PATH=$(cat gcc_path) rustc +$(cat $CG_GCCJIT_DIR/rust-toolchain | grep 'channel' | cut -d '=' -f 2 | sed 's/"//g' | sed 's/ //g') -Cpanic=abort -Zcodegen-backend=$CG_GCCJIT_DIR/target/release/librustc_codegen_gcc.so --sysroot $CG_GCCJIT_DIR/build_sysroot/sysroot my_crate.rs
 ```
 
 ## Env vars
@@ -105,8 +111,18 @@ $ rustc +$(cat $cg_gccjit_dir/rust-toolchain) -Cpanic=abort -Zcodegen-backend=$c
     object files when their content should have been changed by a change to cg_gccjit.</dd>
     <dt>CG_GCCJIT_DISPLAY_CG_TIME</dt>
     <dd>Display the time it took to perform codegen for a crate</dd>
+    <dt>CG_RUSTFLAGS</dt>
+    <dd>Send additional flags to rustc. Can be used to build the sysroot without unwinding by setting `CG_RUSTFLAGS=-Cpanic=abort`.</dd>
+    <dt>CG_GCCJIT_DUMP_TO_FILE</dt>
+    <dd>Dump a C-like representation to /tmp/gccjit_dumps and enable debug info in order to debug this C-like representation.</dd>
 </dl>
 
+## Licensing
+
+While this crate is licensed under a dual Apache/MIT license, it links to `libgccjit` which is under the GPLv3+ and thus, the resulting toolchain (rustc + GCC codegen) will need to be released under the GPL license.
+
+However, programs compiled with `rustc_codegen_gcc` do not need to be released under a GPL license.
+
 ## Debugging
 
 Sometimes, libgccjit will crash and output an error like this:
@@ -182,6 +198,61 @@ set substitute-path /usr/src/debug/gcc /path/to/gcc-repo/gcc
 
 TODO(antoyo): but that's not what I remember I was doing.
 
+### `failed to build archive` error
+
+When you get this error:
+
+```
+error: failed to build archive: failed to open object file: No such file or directory (os error 2)
+```
+
+That can be caused by the fact that you try to compile with `lto = "fat"`, but you didn't compile the sysroot with LTO.
+(Not sure if that's the reason since I cannot reproduce anymore. Maybe it happened when forgetting setting `FAT_LTO`.)
+
+### ld: cannot find crtbegin.o
+
+When compiling an executable with libgccijt, if setting the `*LIBRARY_PATH` variables to the install directory, you will get the following errors:
+
+```
+ld: cannot find crtbegin.o: No such file or directory
+ld: cannot find -lgcc: No such file or directory
+ld: cannot find -lgcc: No such file or directory
+libgccjit.so: error: error invoking gcc driver
+```
+
+To fix this, set the variables to `gcc-build/build/gcc`.
+
+### How to debug GCC LTO
+
+Run do the command with `-v -save-temps` and then extract the `lto1` line from the output and run that under the debugger.
+
+### How to send arguments to the GCC linker
+
+```
+CG_RUSTFLAGS="-Clink-args=-save-temps -v" ../cargo.sh build
+```
+
+### How to see the personality functions in the asm dump
+
+```
+CG_RUSTFLAGS="-Clink-arg=-save-temps -v -Clink-arg=-dA" ../cargo.sh build
+```
+
+### How to see the LLVM IR for a sysroot crate
+
+```
+cargo build -v --target x86_64-unknown-linux-gnu -Zbuild-std
+# Take the command from the output and add --emit=llvm-ir
+```
+
+### To prevent the linker from unmangling symbols
+
+Run with:
+
+```
+COLLECT_NO_DEMANGLE=1
+```
+
 ### How to use a custom-build rustc
 
  * Build the stage2 compiler (`rustup toolchain link debug-current build/x86_64-unknown-linux-gnu/stage2`).
@@ -223,6 +294,11 @@ https://rust-lang.zulipchat.com/#narrow/stream/301329-t-devtools/topic/subtree.2
 
 `rustc` needs to be built without `jemalloc` so that `mem-trace` can overload `malloc` since `jemalloc` is linked statically, so a `LD_PRELOAD`-ed library won't a chance to intercept the calls to `malloc`.
 
+### How to generate GIMPLE
+
+If you need to check what gccjit is generating (GIMPLE), then take a look at how to
+generate it in [gimple.md](./doc/gimple.md).
+
 ### How to build a cross-compiling libgccjit
 
 #### Building libgccjit
@@ -239,4 +315,4 @@ https://rust-lang.zulipchat.com/#narrow/stream/301329-t-devtools/topic/subtree.2
  * Set `linker='-Clinker=m68k-linux-gcc'`.
  * Set the path to the cross-compiling libgccjit in `gcc_path`.
  * Comment the line: `context.add_command_line_option("-masm=intel");` in src/base.rs.
- * (might not be necessary) Disable the compilation of libstd.so (and possibly libcore.so?).
+ * (might not be necessary) Disable the compilation of libstd.so (and possibly libcore.so?): Remove dylib from build_sysroot/sysroot_src/library/std/Cargo.toml.
diff --git a/build.sh b/build.sh
deleted file mode 100755
index ba0d0d04948a..000000000000
--- a/build.sh
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/usr/bin/env bash
-
-#set -x
-set -e
-
-codegen_channel=debug
-sysroot_channel=debug
-
-flags=
-
-while [[ $# -gt 0 ]]; do
-    case $1 in
-        --release)
-            codegen_channel=release
-            shift
-            ;;
-        --release-sysroot)
-            sysroot_channel=release
-            shift
-            ;;
-        --no-default-features)
-            flags="$flags --no-default-features"
-            shift
-            ;;
-        --features)
-            shift
-            flags="$flags --features $1"
-            shift
-            ;;
-        *)
-            echo "Unknown option $1"
-            exit 1
-            ;;
-    esac
-done
-
-if [ -f ./gcc_path ]; then
-    export GCC_PATH=$(cat gcc_path)
-else
-    echo 'Please put the path to your custom build of libgccjit in the file `gcc_path`, see Readme.md for details'
-    exit 1
-fi
-
-export LD_LIBRARY_PATH="$GCC_PATH"
-export LIBRARY_PATH="$GCC_PATH"
-
-if [[ "$codegen_channel" == "release" ]]; then
-    export CHANNEL='release'
-    CARGO_INCREMENTAL=1 cargo rustc --release $flags
-else
-    echo $LD_LIBRARY_PATH
-    export CHANNEL='debug'
-    cargo rustc $flags
-fi
-
-source config.sh
-
-rm -r target/out || true
-mkdir -p target/out/gccjit
-
-echo "[BUILD] sysroot"
-if [[ "$sysroot_channel" == "release" ]]; then
-    time ./build_sysroot/build_sysroot.sh --release
-else
-    time ./build_sysroot/build_sysroot.sh
-fi
-
diff --git a/build_sysroot/Cargo.toml b/build_sysroot/Cargo.toml
index a84f86a82189..e5658273c978 100644
--- a/build_sysroot/Cargo.toml
+++ b/build_sysroot/Cargo.toml
@@ -2,6 +2,7 @@
 authors = ["bjorn3 <bjorn3@users.noreply.github.com>"]
 name = "sysroot"
 version = "0.0.0"
+resolver = "2"
 
 [dependencies]
 core = { path = "./sysroot_src/library/core" }
@@ -18,3 +19,4 @@ rustc-std-workspace-std = { path = "./sysroot_src/library/rustc-std-workspace-st
 
 [profile.release]
 debug = true
+#lto = "fat" # TODO(antoyo): re-enable when the failing LTO tests regarding proc-macros are fixed.
diff --git a/build_sysroot/build_sysroot.sh b/build_sysroot/build_sysroot.sh
index 9d692d599f6b..851e9895ce2b 100755
--- a/build_sysroot/build_sysroot.sh
+++ b/build_sysroot/build_sysroot.sh
@@ -5,9 +5,9 @@
 set -e
 cd $(dirname "$0")
 
-pushd ../ >/dev/null
+pushd ../
 source ./config.sh
-popd >/dev/null
+popd
 
 # Cleanup for previous run
 #     v Clean target dir except for build scripts and incremental cache
diff --git a/build_sysroot/prepare_sysroot_src.sh b/build_sysroot/prepare_sysroot_src.sh
deleted file mode 100755
index 71b3876bac2c..000000000000
--- a/build_sysroot/prepare_sysroot_src.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-set -e
-cd $(dirname "$0")
-
-SRC_DIR=$(dirname $(rustup which rustc))"/../lib/rustlib/src/rust/"
-DST_DIR="sysroot_src"
-
-if [ ! -e $SRC_DIR ]; then
-    echo "Please install rust-src component"
-    exit 1
-fi
-
-rm -rf $DST_DIR
-mkdir -p $DST_DIR/library
-cp -r $SRC_DIR/library $DST_DIR/
-
-pushd $DST_DIR
-echo "[GIT] init"
-git init
-echo "[GIT] add"
-git add .
-echo "[GIT] commit"
-
-# This is needed on systems where nothing is configured.
-# git really needs something here, or it will fail.
-# Even using --author is not enough.
-git config user.email || git config user.email "none@example.com"
-git config user.name || git config user.name "None"
-
-git commit -m "Initial commit" -q
-for file in $(ls ../../patches/ | grep -v patcha); do
-    echo "[GIT] apply" $file
-    git apply ../../patches/$file
-    git add -A
-    git commit --no-gpg-sign -m "Patch $file"
-done
-popd
-
-echo "Successfully prepared libcore for building"
diff --git a/build_system/Cargo.lock b/build_system/Cargo.lock
new file mode 100644
index 000000000000..86268e191603
--- /dev/null
+++ b/build_system/Cargo.lock
@@ -0,0 +1,7 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "y"
+version = "0.1.0"
diff --git a/build_system/Cargo.toml b/build_system/Cargo.toml
new file mode 100644
index 000000000000..f36709ea0360
--- /dev/null
+++ b/build_system/Cargo.toml
@@ -0,0 +1,8 @@
+[package]
+name = "y"
+version = "0.1.0"
+edition = "2021"
+
+[[bin]]
+name = "y"
+path = "src/main.rs"
diff --git a/build_system/src/build.rs b/build_system/src/build.rs
new file mode 100644
index 000000000000..e2819c37ad9b
--- /dev/null
+++ b/build_system/src/build.rs
@@ -0,0 +1,233 @@
+use crate::config::set_config;
+use crate::utils::{
+    get_gcc_path, run_command, run_command_with_env, run_command_with_output_and_env, walk_dir,
+};
+use std::collections::HashMap;
+use std::ffi::OsStr;
+use std::fs;
+use std::path::Path;
+
+#[derive(Default)]
+struct BuildArg {
+    codegen_release_channel: bool,
+    sysroot_release_channel: bool,
+    features: Vec<String>,
+    gcc_path: String,
+}
+
+impl BuildArg {
+    fn new() -> Result<Option<Self>, String> {
+        let gcc_path = get_gcc_path()?;
+        let mut build_arg = Self {
+            gcc_path,
+            ..Default::default()
+        };
+        // We skip binary name and the `build` command.
+        let mut args = std::env::args().skip(2);
+
+        while let Some(arg) = args.next() {
+            match arg.as_str() {
+                "--release" => build_arg.codegen_release_channel = true,
+                "--release-sysroot" => build_arg.sysroot_release_channel = true,
+                "--no-default-features" => {
+                    build_arg.features.push("--no-default-features".to_string());
+                }
+                "--features" => {
+                    if let Some(arg) = args.next() {
+                        build_arg.features.push("--features".to_string());
+                        build_arg.features.push(arg.as_str().into());
+                    } else {
+                        return Err(
+                            "Expected a value after `--features`, found nothing".to_string()
+                        );
+                    }
+                }
+                "--help" => {
+                    Self::usage();
+                    return Ok(None);
+                }
+                arg => return Err(format!("Unknown argument `{}`", arg)),
+            }
+        }
+        Ok(Some(build_arg))
+    }
+
+    fn usage() {
+        println!(
+            r#"
+`build` command help:
+
+    --release              : Build codegen in release mode
+    --release-sysroot      : Build sysroot in release mode
+    --no-default-features  : Add `--no-default-features` flag
+    --features [arg]       : Add a new feature [arg]
+    --help                 : Show this help
+"#
+        )
+    }
+}
+
+fn build_sysroot(
+    env: &mut HashMap<String, String>,
+    release_mode: bool,
+    target_triple: &str,
+) -> Result<(), String> {
+    std::env::set_current_dir("build_sysroot")
+        .map_err(|error| format!("Failed to go to `build_sysroot` directory: {:?}", error))?;
+    // Cleanup for previous run
+    // Clean target dir except for build scripts and incremental cache
+    let _ = walk_dir(
+        "target",
+        |dir: &Path| {
+            for top in &["debug", "release"] {
+                let _ = fs::remove_dir_all(dir.join(top).join("build"));
+                let _ = fs::remove_dir_all(dir.join(top).join("deps"));
+                let _ = fs::remove_dir_all(dir.join(top).join("examples"));
+                let _ = fs::remove_dir_all(dir.join(top).join("native"));
+
+                let _ = walk_dir(
+                    dir.join(top),
+                    |sub_dir: &Path| {
+                        if sub_dir
+                            .file_name()
+                            .map(|filename| filename.to_str().unwrap().starts_with("libsysroot"))
+                            .unwrap_or(false)
+                        {
+                            let _ = fs::remove_dir_all(sub_dir);
+                        }
+                        Ok(())
+                    },
+                    |file: &Path| {
+                        if file
+                            .file_name()
+                            .map(|filename| filename.to_str().unwrap().starts_with("libsysroot"))
+                            .unwrap_or(false)
+                        {
+                            let _ = fs::remove_file(file);
+                        }
+                        Ok(())
+                    },
+                );
+            }
+            Ok(())
+        },
+        |_| Ok(()),
+    );
+
+    let _ = fs::remove_file("Cargo.lock");
+    let _ = fs::remove_file("test_target/Cargo.lock");
+    let _ = fs::remove_dir_all("sysroot");
+
+    // Builds libs
+    let channel = if release_mode {
+        let rustflags = env
+            .get("RUSTFLAGS")
+            .cloned()
+            .unwrap_or_default();
+        env.insert(
+            "RUSTFLAGS".to_string(),
+            format!("{} -Zmir-opt-level=3", rustflags),
+        );
+        run_command_with_output_and_env(
+            &[
+                &"cargo",
+                &"build",
+                &"--target",
+                &target_triple,
+                &"--release",
+            ],
+            None,
+            Some(&env),
+        )?;
+        "release"
+    } else {
+        run_command_with_output_and_env(
+            &[
+                &"cargo",
+                &"build",
+                &"--target",
+                &target_triple,
+                &"--features",
+                &"compiler_builtins/c",
+            ],
+            None,
+            Some(env),
+        )?;
+        "debug"
+    };
+
+    // Copy files to sysroot
+    let sysroot_path = format!("sysroot/lib/rustlib/{}/lib/", target_triple);
+    fs::create_dir_all(&sysroot_path)
+        .map_err(|error| format!("Failed to create directory `{}`: {:?}", sysroot_path, error))?;
+    let copier = |dir_to_copy: &Path| {
+        run_command(&[&"cp", &"-r", &dir_to_copy, &sysroot_path], None).map(|_| ())
+    };
+    walk_dir(
+        &format!("target/{}/{}/deps", target_triple, channel),
+        copier,
+        copier,
+    )?;
+
+    Ok(())
+}
+
+fn build_codegen(args: &BuildArg) -> Result<(), String> {
+    let mut env = HashMap::new();
+
+    let current_dir =
+        std::env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?;
+    if let Ok(rt_root) = std::env::var("RUST_COMPILER_RT_ROOT") {
+        env.insert("RUST_COMPILER_RT_ROOT".to_string(), rt_root);
+    } else {
+        env.insert(
+            "RUST_COMPILER_RT_ROOT".to_string(),
+            format!("{}", current_dir.join("llvm/compiler-rt").display()),
+        );
+    }
+    env.insert("LD_LIBRARY_PATH".to_string(), args.gcc_path.clone());
+    env.insert("LIBRARY_PATH".to_string(), args.gcc_path.clone());
+
+    let mut command: Vec<&dyn AsRef<OsStr>> = vec![&"cargo", &"rustc"];
+    if args.codegen_release_channel {
+        command.push(&"--release");
+        env.insert("CHANNEL".to_string(), "release".to_string());
+        env.insert("CARGO_INCREMENTAL".to_string(), "1".to_string());
+    } else {
+        env.insert("CHANNEL".to_string(), "debug".to_string());
+    }
+    let ref_features = args.features.iter().map(|s| s.as_str()).collect::<Vec<_>>();
+    for feature in &ref_features {
+        command.push(feature);
+    }
+    run_command_with_env(&command, None, Some(&env))?;
+
+    let config = set_config(&mut env, &[], Some(&args.gcc_path))?;
+
+    // We voluntarily ignore the error.
+    let _ = fs::remove_dir_all("target/out");
+    let gccjit_target = "target/out/gccjit";
+    fs::create_dir_all(gccjit_target).map_err(|error| {
+        format!(
+            "Failed to create directory `{}`: {:?}",
+            gccjit_target, error
+        )
+    })?;
+
+    println!("[BUILD] sysroot");
+    build_sysroot(
+        &mut env,
+        args.sysroot_release_channel,
+        &config.target_triple,
+    )?;
+    Ok(())
+}
+
+pub fn run() -> Result<(), String> {
+    let args = match BuildArg::new()? {
+        Some(args) => args,
+        None => return Ok(()),
+    };
+    build_codegen(&args)?;
+    Ok(())
+}
diff --git a/build_system/src/config.rs b/build_system/src/config.rs
new file mode 100644
index 000000000000..4f2e33f0f998
--- /dev/null
+++ b/build_system/src/config.rs
@@ -0,0 +1,125 @@
+use crate::utils::{get_gcc_path, get_os_name, get_rustc_host_triple};
+use std::collections::HashMap;
+use std::env as std_env;
+
+pub struct ConfigInfo {
+    pub target_triple: String,
+    pub rustc_command: Vec<String>,
+    pub run_wrapper: Option<&'static str>,
+}
+
+// Returns the beginning for the command line of rustc.
+pub fn set_config(
+    env: &mut HashMap<String, String>,
+    test_flags: &[String],
+    gcc_path: Option<&str>,
+) -> Result<ConfigInfo, String> {
+    env.insert("CARGO_INCREMENTAL".to_string(), "0".to_string());
+
+    let gcc_path = match gcc_path {
+        Some(path) => path.to_string(),
+        None => get_gcc_path()?,
+    };
+    env.insert("GCC_PATH".to_string(), gcc_path.clone());
+
+    let os_name = get_os_name()?;
+    let dylib_ext = match os_name.as_str() {
+        "Linux" => "so",
+        "Darwin" => "dylib",
+        os => return Err(format!("unsupported OS `{}`", os)),
+    };
+    let host_triple = get_rustc_host_triple()?;
+    let mut linker = None;
+    let mut target_triple = host_triple.as_str();
+    let mut run_wrapper = None;
+    // FIXME: handle this with a command line flag?
+    // let mut target_triple = "m68k-unknown-linux-gnu";
+
+    if host_triple != target_triple {
+        if target_triple == "m68k-unknown-linux-gnu" {
+            target_triple = "mips-unknown-linux-gnu";
+            linker = Some("-Clinker=m68k-linux-gcc");
+        } else if target_triple == "aarch64-unknown-linux-gnu" {
+            // We are cross-compiling for aarch64. Use the correct linker and run tests in qemu.
+            linker = Some("-Clinker=aarch64-linux-gnu-gcc");
+            run_wrapper = Some("qemu-aarch64 -L /usr/aarch64-linux-gnu");
+        } else {
+            return Err(format!("unknown non-native platform `{}`", target_triple));
+        }
+    }
+    let current_dir =
+        std_env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?;
+    let channel = if let Some(channel) = env.get("CHANNEL") {
+        channel.as_str()
+    } else {
+        "debug"
+    };
+    let cg_backend_path = current_dir
+        .join("target")
+        .join(channel)
+        .join(&format!("librustc_codegen_gcc.{}", dylib_ext));
+    let sysroot_path = current_dir.join("build_sysroot/sysroot");
+    let mut rustflags = Vec::new();
+    if let Some(cg_rustflags) = env.get("CG_RUSTFLAGS") {
+        rustflags.push(cg_rustflags.clone());
+    }
+    if let Some(linker) = linker {
+        rustflags.push(linker.to_string());
+    }
+    rustflags.extend_from_slice(&[
+        "-Csymbol-mangling-version=v0".to_string(),
+        "-Cdebuginfo=2".to_string(),
+        format!("-Zcodegen-backend={}", cg_backend_path.display()),
+        "--sysroot".to_string(),
+        sysroot_path.display().to_string(),
+    ]);
+
+    // Since we don't support ThinLTO, disable LTO completely when not trying to do LTO.
+    // TODO(antoyo): remove when we can handle ThinLTO.
+    if !env.contains_key(&"FAT_LTO".to_string()) {
+        rustflags.push("-Clto=off".to_string());
+    }
+    rustflags.extend_from_slice(test_flags);
+    // FIXME(antoyo): remove once the atomic shim is gone
+    if os_name == "Darwin" {
+        rustflags.extend_from_slice(&[
+            "-Clink-arg=-undefined".to_string(),
+            "-Clink-arg=dynamic_lookup".to_string(),
+        ]);
+    }
+    env.insert("RUSTFLAGS".to_string(), rustflags.join(" "));
+    // display metadata load errors
+    env.insert("RUSTC_LOG".to_string(), "warn".to_string());
+
+    let sysroot = current_dir.join(&format!(
+        "build_sysroot/sysroot/lib/rustlib/{}/lib",
+        target_triple
+    ));
+    let ld_library_path = format!(
+        "{target}:{sysroot}:{gcc_path}",
+        target = current_dir.join("target/out").display(),
+        sysroot = sysroot.display(),
+    );
+    env.insert("LD_LIBRARY_PATH".to_string(), ld_library_path.clone());
+    env.insert("DYLD_LIBRARY_PATH".to_string(), ld_library_path);
+
+    // NOTE: To avoid the -fno-inline errors, use /opt/gcc/bin/gcc instead of cc.
+    // To do so, add a symlink for cc to /opt/gcc/bin/gcc in our PATH.
+    // Another option would be to add the following Rust flag: -Clinker=/opt/gcc/bin/gcc
+    let path = std::env::var("PATH").unwrap_or_default();
+    env.insert("PATH".to_string(), format!("/opt/gcc/bin:{}", path));
+
+    let mut rustc_command = vec!["rustc".to_string()];
+    rustc_command.extend_from_slice(&rustflags);
+    rustc_command.extend_from_slice(&[
+        "-L".to_string(),
+        "crate=target/out".to_string(),
+        "--out-dir".to_string(),
+        "target/out".to_string(),
+    ]);
+    Ok(ConfigInfo {
+        target_triple: target_triple.to_string(),
+        rustc_command,
+        run_wrapper,
+    })
+}
diff --git a/build_system/src/main.rs b/build_system/src/main.rs
new file mode 100644
index 000000000000..332a14ff0a28
--- /dev/null
+++ b/build_system/src/main.rs
@@ -0,0 +1,62 @@
+use std::env;
+use std::process;
+
+mod build;
+mod config;
+mod prepare;
+mod rustc_info;
+mod utils;
+
+macro_rules! arg_error {
+    ($($err:tt)*) => {{
+        eprintln!($($err)*);
+        eprintln!();
+        usage();
+        std::process::exit(1);
+    }};
+}
+
+fn usage() {
+    println!(
+        "\
+Available commands for build_system:
+
+    prepare  : Run prepare command
+    build    : Run build command
+    --help   : Show this message"
+    );
+}
+
+pub enum Command {
+    Prepare,
+    Build,
+}
+
+fn main() {
+    if env::var("RUST_BACKTRACE").is_err() {
+        env::set_var("RUST_BACKTRACE", "1");
+    }
+
+    let command = match env::args().nth(1).as_deref() {
+        Some("prepare") => Command::Prepare,
+        Some("build") => Command::Build,
+        Some("--help") => {
+            usage();
+            process::exit(0);
+        }
+        Some(flag) if flag.starts_with('-') => arg_error!("Expected command found flag {}", flag),
+        Some(command) => arg_error!("Unknown command {}", command),
+        None => {
+            usage();
+            process::exit(0);
+        }
+    };
+
+    if let Err(e) = match command {
+        Command::Prepare => prepare::run(),
+        Command::Build => build::run(),
+    } {
+        eprintln!("Command failed to run: {e:?}");
+        process::exit(1);
+    }
+}
diff --git a/build_system/src/prepare.rs b/build_system/src/prepare.rs
new file mode 100644
index 000000000000..b258ddf36648
--- /dev/null
+++ b/build_system/src/prepare.rs
@@ -0,0 +1,227 @@
+use crate::rustc_info::get_rustc_path;
+use crate::utils::{cargo_install, git_clone, run_command, run_command_with_output, walk_dir};
+
+use std::fs;
+use std::path::Path;
+
+fn prepare_libcore(sysroot_path: &Path) -> Result<(), String> {
+    let rustc_path = match get_rustc_path() {
+        Some(path) => path,
+        None => return Err("`rustc` path not found".to_string()),
+    };
+
+    let parent = match rustc_path.parent() {
+        Some(path) => path,
+        None => return Err(format!("No parent for `{}`", rustc_path.display())),
+    };
+
+    let rustlib_dir = parent
+        .join("../lib/rustlib/src/rust")
+        .canonicalize()
+        .map_err(|error| format!("Failed to canonicalize path: {:?}", error))?;
+    if !rustlib_dir.is_dir() {
+        return Err("Please install `rust-src` component".to_string());
+    }
+
+    let sysroot_dir = sysroot_path.join("sysroot_src");
+    if sysroot_dir.is_dir() {
+        if let Err(error) = fs::remove_dir_all(&sysroot_dir) {
+            return Err(format!(
+                "Failed to remove `{}`: {:?}",
+                sysroot_dir.display(),
+                error,
+            ));
+        }
+    }
+
+    let sysroot_library_dir = sysroot_dir.join("library");
+    fs::create_dir_all(&sysroot_library_dir).map_err(|error| {
+        format!(
+            "Failed to create folder `{}`: {:?}",
+            sysroot_library_dir.display(),
+            error,
+        )
+    })?;
+
+    run_command(
+        &[&"cp", &"-r", &rustlib_dir.join("library"), &sysroot_dir],
+        None,
+    )?;
+
+    println!("[GIT] init (cwd): `{}`", sysroot_dir.display());
+    run_command(&[&"git", &"init"], Some(&sysroot_dir))?;
+    println!("[GIT] add (cwd): `{}`", sysroot_dir.display());
+    run_command(&[&"git", &"add", &"."], Some(&sysroot_dir))?;
+    println!("[GIT] commit (cwd): `{}`", sysroot_dir.display());
+
+    // This is needed on systems where nothing is configured.
+    // git really needs something here, or it will fail.
+    // Even using --author is not enough.
+    run_command(
+        &[&"git", &"config", &"user.email", &"none@example.com"],
+        Some(&sysroot_dir),
+    )?;
+    run_command(
+        &[&"git", &"config", &"user.name", &"None"],
+        Some(&sysroot_dir),
+    )?;
+    run_command(
+        &[&"git", &"config", &"core.autocrlf", &"false"],
+        Some(&sysroot_dir),
+    )?;
+    run_command(
+        &[&"git", &"config", &"commit.gpgSign", &"false"],
+        Some(&sysroot_dir),
+    )?;
+    run_command(
+        &[&"git", &"commit", &"-m", &"Initial commit", &"-q"],
+        Some(&sysroot_dir),
+    )?;
+
+    let mut patches = Vec::new();
+    walk_dir(
+        "patches",
+        |_| Ok(()),
+        |file_path: &Path| {
+            patches.push(file_path.to_path_buf());
+            Ok(())
+        },
+    )?;
+    patches.sort();
+    for file_path in patches {
+        println!("[GIT] apply `{}`", file_path.display());
+        let path = Path::new("../..").join(file_path);
+        run_command_with_output(&[&"git", &"apply", &path], Some(&sysroot_dir))?;
+        run_command_with_output(&[&"git", &"add", &"-A"], Some(&sysroot_dir))?;
+        run_command_with_output(
+            &[
+                &"git",
+                &"commit",
+                &"--no-gpg-sign",
+                &"-m",
+                &format!("Patch {}", path.display()),
+            ],
+            Some(&sysroot_dir),
+        )?;
+    }
+    println!("Successfully prepared libcore for building");
+    Ok(())
+}
+
+// build with cg_llvm for perf comparison
+fn build_raytracer(repo_dir: &Path) -> Result<(), String> {
+    run_command(&[&"cargo", &"build"], Some(repo_dir))?;
+    let mv_target = repo_dir.join("raytracer_cg_llvm");
+    if mv_target.is_file() {
+        std::fs::remove_file(&mv_target)
+            .map_err(|e| format!("Failed to remove file `{}`: {e:?}", mv_target.display()))?;
+    }
+    run_command(
+        &[&"mv", &"target/debug/main", &"raytracer_cg_llvm"],
+        Some(repo_dir),
+    )?;
+    Ok(())
+}
+
+fn clone_and_setup<F>(repo_url: &str, checkout_commit: &str, extra: Option<F>) -> Result<(), String>
+where
+    F: Fn(&Path) -> Result<(), String>,
+{
+    let clone_result = git_clone(repo_url, None)?;
+    if !clone_result.ran_clone {
+        println!("`{}` has already been cloned", clone_result.repo_name);
+    }
+    let repo_path = Path::new(&clone_result.repo_name);
+    run_command(&[&"git", &"checkout", &"--", &"."], Some(&repo_path))?;
+    run_command(&[&"git", &"checkout", &checkout_commit], Some(&repo_path))?;
+    let filter = format!("-{}-", clone_result.repo_name);
+    walk_dir(
+        "crate_patches",
+        |_| Ok(()),
+        |file_path| {
+            let patch = file_path.as_os_str().to_str().unwrap();
+            if patch.contains(&filter) && patch.ends_with(".patch") {
+                run_command_with_output(
+                    &[&"git", &"am", &file_path.canonicalize().unwrap()],
+                    Some(&repo_path),
+                )?;
+            }
+            Ok(())
+        },
+    )?;
+    if let Some(extra) = extra {
+        extra(&repo_path)?;
+    }
+    Ok(())
+}
+
+struct PrepareArg {
+    only_libcore: bool,
+}
+
+impl PrepareArg {
+    fn new() -> Result<Option<Self>, String> {
+        let mut only_libcore = false;
+
+        for arg in std::env::args().skip(2) {
+            match arg.as_str() {
+                "--only-libcore" => only_libcore = true,
+                "--help" => {
+                    Self::usage();
+                    return Ok(None);
+                }
+                a => return Err(format!("Unknown argument `{a}`")),
+            }
+        }
+        Ok(Some(Self { only_libcore }))
+    }
+
+    fn usage() {
+        println!(
+            r#"
+`prepare` command help:
+
+    --only-libcore  : Only setup libcore and don't clone other repositories
+    --help          : Show this help
+"#
+        )
+    }
+}
+
+pub fn run() -> Result<(), String> {
+    let args = match PrepareArg::new()? {
+        Some(a) => a,
+        None => return Ok(()),
+    };
+    let sysroot_path = Path::new("build_sysroot");
+    prepare_libcore(sysroot_path)?;
+
+    if !args.only_libcore {
+        cargo_install("hyperfine")?;
+
+        let to_clone = &[
+            (
+                "https://github.com/rust-random/rand.git",
+                "0f933f9c7176e53b2a3c7952ded484e1783f0bf1",
+                None,
+            ),
+            (
+                "https://github.com/rust-lang/regex.git",
+                "341f207c1071f7290e3f228c710817c280c8dca1",
+                None,
+            ),
+            (
+                "https://github.com/ebobby/simple-raytracer",
+                "804a7a21b9e673a482797aa289a18ed480e4d813",
+                Some(build_raytracer),
+            ),
+        ];
+
+        for (repo_url, checkout_commit, cb) in to_clone {
+            clone_and_setup(repo_url, checkout_commit, *cb)?;
+        }
+    }
+
+    println!("Successfully ran `prepare`");
+    Ok(())
+}
diff --git a/build_system/src/rustc_info.rs b/build_system/src/rustc_info.rs
new file mode 100644
index 000000000000..0988b56d81eb
--- /dev/null
+++ b/build_system/src/rustc_info.rs
@@ -0,0 +1,12 @@
+use std::path::{Path, PathBuf};
+
+use crate::utils::run_command;
+
+pub fn get_rustc_path() -> Option<PathBuf> {
+    if let Ok(rustc) = std::env::var("RUSTC") {
+        return Some(PathBuf::from(rustc));
+    }
+    run_command(&[&"rustup", &"which", &"rustc"], None)
+        .ok()
+        .map(|out| Path::new(String::from_utf8(out.stdout).unwrap().trim()).to_path_buf())
+}
diff --git a/build_system/src/utils.rs b/build_system/src/utils.rs
new file mode 100644
index 000000000000..536f33a80293
--- /dev/null
+++ b/build_system/src/utils.rs
@@ -0,0 +1,240 @@
+use std::collections::HashMap;
+use std::ffi::OsStr;
+use std::fmt::Debug;
+use std::fs;
+use std::path::Path;
+use std::process::{Command, ExitStatus, Output};
+
+fn get_command_inner(
+    input: &[&dyn AsRef<OsStr>],
+    cwd: Option<&Path>,
+    env: Option<&HashMap<String, String>>,
+) -> Command {
+    let (cmd, args) = match input {
+        [] => panic!("empty command"),
+        [cmd, args @ ..] => (cmd, args),
+    };
+    let mut command = Command::new(cmd);
+    command.args(args);
+    if let Some(cwd) = cwd {
+        command.current_dir(cwd);
+    }
+    if let Some(env) = env {
+        command.envs(env.iter().map(|(k, v)| (k.as_str(), v.as_str())));
+    }
+    command
+}
+
+fn check_exit_status(
+    input: &[&dyn AsRef<OsStr>],
+    cwd: Option<&Path>,
+    exit_status: ExitStatus,
+) -> Result<(), String> {
+    if exit_status.success() {
+        Ok(())
+    } else {
+        Err(format!(
+            "Command `{}`{} exited with status {:?}",
+            input
+                .iter()
+                .map(|s| s.as_ref().to_str().unwrap())
+                .collect::<Vec<_>>()
+                .join(" "),
+            cwd.map(|cwd| format!(" (running in folder `{}`)", cwd.display()))
+                .unwrap_or_default(),
+            exit_status.code(),
+        ))
+    }
+}
+
+fn command_error<D: Debug>(input: &[&dyn AsRef<OsStr>], cwd: &Option<&Path>, error: D) -> String {
+    format!(
+        "Command `{}`{} failed to run: {error:?}",
+        input
+            .iter()
+            .map(|s| s.as_ref().to_str().unwrap())
+            .collect::<Vec<_>>()
+            .join(" "),
+        cwd.as_ref()
+            .map(|cwd| format!(" (running in folder `{}`)", cwd.display(),))
+            .unwrap_or_default(),
+    )
+}
+
+pub fn run_command(input: &[&dyn AsRef<OsStr>], cwd: Option<&Path>) -> Result<Output, String> {
+    run_command_with_env(input, cwd, None)
+}
+
+pub fn run_command_with_env(
+    input: &[&dyn AsRef<OsStr>],
+    cwd: Option<&Path>,
+    env: Option<&HashMap<String, String>>,
+) -> Result<Output, String> {
+    let output = get_command_inner(input, cwd, env)
+        .output()
+        .map_err(|e| command_error(input, &cwd, e))?;
+    check_exit_status(input, cwd, output.status)?;
+    Ok(output)
+}
+
+pub fn run_command_with_output(
+    input: &[&dyn AsRef<OsStr>],
+    cwd: Option<&Path>,
+) -> Result<(), String> {
+    let exit_status = get_command_inner(input, cwd, None)
+        .spawn()
+        .map_err(|e| command_error(input, &cwd, e))?
+        .wait()
+        .map_err(|e| command_error(input, &cwd, e))?;
+    check_exit_status(input, cwd, exit_status)?;
+    Ok(())
+}
+
+pub fn run_command_with_output_and_env(
+    input: &[&dyn AsRef<OsStr>],
+    cwd: Option<&Path>,
+    env: Option<&HashMap<String, String>>,
+) -> Result<(), String> {
+    let exit_status = get_command_inner(input, cwd, env)
+        .spawn()
+        .map_err(|e| command_error(input, &cwd, e))?
+        .wait()
+        .map_err(|e| command_error(input, &cwd, e))?;
+    check_exit_status(input, cwd, exit_status)?;
+    Ok(())
+}
+
+pub fn cargo_install(to_install: &str) -> Result<(), String> {
+    let output = run_command(&[&"cargo", &"install", &"--list"], None)?;
+
+    let to_install_needle = format!("{to_install} ");
+    // cargo install --list returns something like this:
+    //
+    // mdbook-toc v0.8.0:
+    //     mdbook-toc
+    // rust-reduce v0.1.0:
+    //     rust-reduce
+    //
+    // We are only interested into the command name so we only look for lines ending with `:`.
+    if String::from_utf8(output.stdout)
+        .unwrap()
+        .lines()
+        .any(|line| line.ends_with(':') && line.starts_with(&to_install_needle))
+    {
+        return Ok(());
+    }
+    // We voluntarily ignore this error.
+    if run_command_with_output(&[&"cargo", &"install", &to_install], None).is_err() {
+        println!("Skipping installation of `{to_install}`");
+    }
+    Ok(())
+}
+
+pub fn get_os_name() -> Result<String, String> {
+    let output = run_command(&[&"uname"], None)?;
+    let name = std::str::from_utf8(&output.stdout)
+        .unwrap_or("")
+        .trim()
+        .to_string();
+    if !name.is_empty() {
+        Ok(name)
+    } else {
+        Err("Failed to retrieve the OS name".to_string())
+    }
+}
+
+pub fn get_rustc_host_triple() -> Result<String, String> {
+    let output = run_command(&[&"rustc", &"-vV"], None)?;
+    let content = std::str::from_utf8(&output.stdout).unwrap_or("");
+
+    for line in content.split('\n').map(|line| line.trim()) {
+        if !line.starts_with("host:") {
+            continue;
+        }
+        return Ok(line.split(':').nth(1).unwrap().trim().to_string());
+    }
+    Err("Cannot find host triple".to_string())
+}
+
+pub fn get_gcc_path() -> Result<String, String> {
+    let content = match fs::read_to_string("gcc_path") {
+        Ok(content) => content,
+        Err(_) => {
+            return Err(
+                "Please put the path to your custom build of libgccjit in the file \
+                   `gcc_path`, see Readme.md for details"
+                    .into(),
+            )
+        }
+    };
+    match content
+        .split('\n')
+        .map(|line| line.trim())
+        .filter(|line| !line.is_empty())
+        .next()
+    {
+        Some(gcc_path) => {
+            let path = Path::new(gcc_path);
+            if !path.exists() {
+                Err(format!(
+                    "Path `{}` contained in the `gcc_path` file doesn't exist",
+                    gcc_path,
+                ))
+            } else {
+                Ok(gcc_path.into())
+            }
+        }
+        None => Err("No path found in `gcc_path` file".into()),
+    }
+}
+
+pub struct CloneResult {
+    pub ran_clone: bool,
+    pub repo_name: String,
+}
+
+pub fn git_clone(to_clone: &str, dest: Option<&Path>) -> Result<CloneResult, String> {
+    let repo_name = to_clone.split('/').last().unwrap();
+    let repo_name = match repo_name.strip_suffix(".git") {
+        Some(n) => n.to_string(),
+        None => repo_name.to_string(),
+    };
+
+    let dest = dest
+        .map(|dest| dest.join(&repo_name))
+        .unwrap_or_else(|| Path::new(&repo_name).into());
+    if dest.is_dir() {
+        return Ok(CloneResult {
+            ran_clone: false,
+            repo_name,
+        });
+    }
+
+    run_command_with_output(&[&"git", &"clone", &to_clone, &dest], None)?;
+    Ok(CloneResult {
+        ran_clone: true,
+        repo_name,
+    })
+}
+
+pub fn walk_dir<P, D, F>(dir: P, mut dir_cb: D, mut file_cb: F) -> Result<(), String>
+where
+    P: AsRef<Path>,
+    D: FnMut(&Path) -> Result<(), String>,
+    F: FnMut(&Path) -> Result<(), String>,
+{
+    let dir = dir.as_ref();
+    for entry in fs::read_dir(dir)
+        .map_err(|error| format!("Failed to read dir `{}`: {:?}", dir.display(), error))?
+    {
+        let entry = entry
+            .map_err(|error| format!("Failed to read entry in `{}`: {:?}", dir.display(), error))?;
+        let entry_path = entry.path();
+        if entry_path.is_dir() {
+            dir_cb(&entry_path)?;
+        } else {
+            file_cb(&entry_path)?;
+        }
+    }
+    Ok(())
+}
diff --git a/config.sh b/config.sh
index 166e83901c4f..c686df0c72a2 100644
--- a/config.sh
+++ b/config.sh
@@ -38,10 +38,17 @@ if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
    fi
 fi
 
-export RUSTFLAGS="$CG_RUSTFLAGS $linker -Csymbol-mangling-version=v0 -Cdebuginfo=2 -Clto=off -Zcodegen-backend=$(pwd)/target/${CHANNEL:-debug}/librustc_codegen_gcc.$dylib_ext --sysroot $(pwd)/build_sysroot/sysroot $TEST_FLAGS"
+# Since we don't support ThinLTO, disable LTO completely when not trying to do LTO.
+# TODO(antoyo): remove when we can handle ThinLTO.
+disable_lto_flags=''
+if [[ ! -v FAT_LTO ]]; then
+    disable_lto_flags='-Clto=off'
+fi
+
+export RUSTFLAGS="$CG_RUSTFLAGS $linker -Csymbol-mangling-version=v0 -Cdebuginfo=2 $disable_lto_flags -Zcodegen-backend=$(pwd)/target/${CHANNEL:-debug}/librustc_codegen_gcc.$dylib_ext --sysroot $(pwd)/build_sysroot/sysroot $TEST_FLAGS"
 
 # FIXME(antoyo): remove once the atomic shim is gone
-if [[ `uname` == 'Darwin' ]]; then
+if [[ unamestr == 'Darwin' ]]; then
    export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
 fi
 
@@ -50,3 +57,7 @@ export RUSTC_LOG=warn # display metadata load errors
 
 export LD_LIBRARY_PATH="$(pwd)/target/out:$(pwd)/build_sysroot/sysroot/lib/rustlib/$TARGET_TRIPLE/lib:$GCC_PATH"
 export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH
+# NOTE: To avoid the -fno-inline errors, use /opt/gcc/bin/gcc instead of cc.
+# To do so, add a symlink for cc to /opt/gcc/bin/gcc in our PATH.
+# Another option would be to add the following Rust flag: -Clinker=/opt/gcc/bin/gcc
+export PATH="/opt/gcc/bin:$PATH"
diff --git a/doc/add-attribute.md b/doc/add-attribute.md
new file mode 100644
index 000000000000..ae3bcc5e2ebe
--- /dev/null
+++ b/doc/add-attribute.md
@@ -0,0 +1,17 @@
+# Add support for a new function attribute
+
+To add support for a new function attribute in libgccjit, you need to do the following steps:
+
+ 1. Copy the corresponding function from `c-family/c-attribs.cc` into `jit/dummy-frontend.cc`. For example if you add the `target` attribute, the function name will be `handle_target_attribute`.
+ 2. Copy the corresponding entry from the `c_common_attribute_table` variable in the `c-family/c-attribs.cc` file into the `jit_attribute_table` variable in `jit/dummy-frontend.cc`.
+ 3. Add a new variant in the `gcc_jit_fn_attribute` enum in the `jit/libgccjit.h` file.
+ 4. Add a test to ensure the attribute is correctly applied in `gcc/testsuite/jit.dg/`. Take a look at `gcc/testsuite/jit.dg/test-nonnull.c` if you want an example.
+ 5. Run the example like this (in your `gcc-build` folder): `make check-jit RUNTESTFLAGS="-v -v -v jit.exp=jit.dg/test-nonnull.c"`
+
+Once done, you need to update the [gccjit.rs] crate to add the new enum variant in the corresponding enum (`FnAttribute`).
+
+Finally, you need to update this repository by calling the relevant API you added in [gccjit.rs].
+
+To test it, build `gcc`, run `cargo update -p gccjit` and then you can test the generated output for a given Rust crate.
+
+[gccjit.rs]: https://github.com/antoyo/gccjit.rs
diff --git a/doc/gimple.md b/doc/gimple.md
new file mode 100644
index 000000000000..145c4eda3c1c
--- /dev/null
+++ b/doc/gimple.md
@@ -0,0 +1,111 @@
+# GIMPLE
+
+You can see the full documentation about what GIMPLE is [here](https://gcc.gnu.org/onlinedocs/gccint/GIMPLE.html). In this document we will explain how to generate it.
+
+First, we'll copy the content from `gcc/gcc/testsuite/jit.dg/test-const-attribute.c` into a
+file named `local.c` and remove the content we're not interested into:
+
+```diff
+- /* { dg-do compile { target x86_64-*-* } } */
+...
+- /* We don't want set_options() in harness.h to set -O3 to see that the const
+-    attribute affects the optimizations. */
+- #define TEST_ESCHEWS_SET_OPTIONS
+- static void set_options (gcc_jit_context *ctxt, const char *argv0)
+- {
+-   // Set "-O3".
+-   gcc_jit_context_set_int_option(ctxt, GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL, 3);
+- }
+-
+- #define TEST_COMPILING_TO_FILE
+- #define OUTPUT_KIND      GCC_JIT_OUTPUT_KIND_ASSEMBLER
+- #define OUTPUT_FILENAME  "output-of-test-const-attribute.c.s"
+- #include "harness.h"
+...
+- /* { dg-final { jit-verify-output-file-was-created "" } } */
+- /* Check that the loop was optimized away */
+- /* { dg-final { jit-verify-assembler-output-not "jne" } } */
+```
+
+Then we'll add a `main` function which will call the `create_code` function but
+also add the calls we need to generate the GIMPLE:
+
+```C
+int main() {
+    gcc_jit_context *ctxt = gcc_jit_context_acquire();
+    // To set `-O3`, update it depending on your needs.
+    gcc_jit_context_set_int_option(ctxt, GCC_JIT_INT_OPTION_OPTIMIZATION_LEVEL, 3);
+    // Very important option to generate the gimple format.
+    gcc_jit_context_set_bool_option(ctxt, GCC_JIT_BOOL_OPTION_DUMP_INITIAL_GIMPLE, 1);
+    create_code(ctxt, NULL);
+
+    gcc_jit_context_compile(ctxt);
+    // If you want to compile to assembly (or any other format) directly, you can
+    // use the following call instead:
+    // gcc_jit_context_compile_to_file(ctxt, GCC_JIT_OUTPUT_KIND_ASSEMBLER, "out.s");
+
+    return 0;
+}
+```
+
+Then we can compile it by using:
+
+```console
+gcc local.c -I `pwd`/gcc/gcc/jit/ -L `pwd`/gcc-build/gcc -lgccjit -o out
+```
+
+And finally when you run it:
+
+```console
+LD_LIBRARY_PATH=`pwd`/gcc-build/gcc LIBRARY_PATH=`pwd`/gcc-build/gcc ./out
+```
+
+It should display:
+
+```c
+__attribute__((const))
+int xxx ()
+{
+  int D.3394;
+  int sum;
+  int x;
+
+  <D.3377>:
+  x = 45;
+  sum = 0;
+  goto loop_cond;
+  loop_cond:
+  x = x >> 1;
+  if (x != 0) goto after_loop; else goto loop_body;
+  loop_body:
+  _1 = foo (x);
+  _2 = _1 * 2;
+  x = x + _2;
+  goto loop_cond;
+  after_loop:
+  D.3394 = sum;
+  return D.3394;
+}
+```
+
+An alternative way to generate the GIMPLE is to replace:
+
+```c
+    gcc_jit_context_set_bool_option(ctxt, GCC_JIT_BOOL_OPTION_DUMP_INITIAL_GIMPLE, 1);
+```
+
+with:
+
+```c
+    gcc_jit_context_add_command_line_option(ctxt, "-fdump-tree-gimple");
+```
+
+(although you can have both at the same time too). Then you can compile it like previously. Only one difference: before executing it, I recommend to run:
+
+```console
+rm -rf /tmp/libgccjit-*
+```
+
+to make it easier for you to know which folder to look into.
+
+Once the execution is done, you should now have a file with path looking like `/tmp/libgccjit-9OFqkD/fake.c.006t.gimple` which contains the GIMPLE format.
diff --git a/doc/sending-gcc-patch.md b/doc/sending-gcc-patch.md
new file mode 100644
index 000000000000..7a47ef29f3c2
--- /dev/null
+++ b/doc/sending-gcc-patch.md
@@ -0,0 +1,44 @@
+This guide explains what to do to send a GCC patch for review.
+
+All the commands are supposed to be run in the folder where you cloned GCC.
+
+```bash
+./contrib/gcc-changelog/git_check_commit.py
+```
+
+You can provide a specific commit hash:
+
+```bash
+./contrib/gcc-changelog/git_check_commit.py abdef78989
+```
+
+a range:
+
+```bash
+./contrib/gcc-changelog/git_check_commit.py HEAD~2
+```
+
+or even a comparison with a remote branch:
+
+```bash
+./contrib/gcc-changelog/git_check_commit.py upstream/master..HEAD
+```
+
+When there is no more errors, generate the git patch:
+
+```bash
+git format-patch -1 `git rev-parse --short HEAD`
+```
+
+Then you can run the remaining checks using:
+
+```bash
+contrib/check_GNU_style.sh 0001-your-patch.patch
+```
+
+When you have no more errors, you can send the `.patch` file to GCC by sending an
+email to `gcc-patches@gcc.gnu.org` and to the relevant GCC mailing lists
+depending on what your patch changes. You can find the list of the mailing lists
+[here](https://gcc.gnu.org/lists.html).
+
+You can find more information about "contributing to GCC" [here](https://gcc.gnu.org/contribute.html).
diff --git a/example/alloc_example.rs b/example/alloc_example.rs
index 754e7931412d..f1954a30cf86 100644
--- a/example/alloc_example.rs
+++ b/example/alloc_example.rs
@@ -1,5 +1,6 @@
 #![feature(start, core_intrinsics, alloc_error_handler, lang_items)]
 #![no_std]
+#![allow(internal_features)]
 
 extern crate alloc;
 extern crate alloc_system;
diff --git a/example/alloc_system.rs b/example/alloc_system.rs
index 3deef419f42e..56ff84e4bdfb 100644
--- a/example/alloc_system.rs
+++ b/example/alloc_system.rs
@@ -12,7 +12,7 @@
               target_arch = "mips",
               target_arch = "mips32r6",
               target_arch = "powerpc",
-              target_arch = "csky"
+              target_arch = "csky",
               target_arch = "powerpc64"))]
 const MIN_ALIGN: usize = 8;
 #[cfg(any(target_arch = "x86_64",
diff --git a/example/arbitrary_self_types_pointers_and_wrappers.rs b/example/arbitrary_self_types_pointers_and_wrappers.rs
index 3af0ba09e0ba..b299aa879740 100644
--- a/example/arbitrary_self_types_pointers_and_wrappers.rs
+++ b/example/arbitrary_self_types_pointers_and_wrappers.rs
@@ -2,6 +2,7 @@
 
 #![feature(arbitrary_self_types, unsize, coerce_unsized, dispatch_from_dyn)]
 #![feature(rustc_attrs)]
+#![allow(internal_features)]
 
 use std::{
     ops::{Deref, CoerceUnsized, DispatchFromDyn},
diff --git a/example/mini_core.rs b/example/mini_core.rs
index 0cd7e6047c20..343285203432 100644
--- a/example/mini_core.rs
+++ b/example/mini_core.rs
@@ -4,7 +4,7 @@
     thread_local
 )]
 #![no_core]
-#![allow(dead_code)]
+#![allow(dead_code, internal_features)]
 
 #[no_mangle]
 unsafe extern "C" fn _Unwind_Resume() {
@@ -429,6 +429,15 @@ fn panic_cannot_unwind() -> ! {
     }
 }
 
+#[lang = "panic_in_cleanup"]
+#[rustc_nounwind]
+fn panic_in_cleanup() -> ! {
+    unsafe {
+        libc::printf("panic in a destructor during cleanup\n\0" as *const str as *const i8);
+        intrinsics::abort();
+    }
+}
+
 #[lang = "panic_bounds_check"]
 #[track_caller]
 fn panic_bounds_check(index: usize, len: usize) -> ! {
diff --git a/example/mini_core_hello_world.rs b/example/mini_core_hello_world.rs
index b93d68597063..c3aea5718154 100644
--- a/example/mini_core_hello_world.rs
+++ b/example/mini_core_hello_world.rs
@@ -5,7 +5,7 @@
     extern_types, thread_local
 )]
 #![no_core]
-#![allow(dead_code, non_camel_case_types)]
+#![allow(dead_code, internal_features, non_camel_case_types)]
 
 extern crate mini_core;
 
diff --git a/example/mod_bench.rs b/example/mod_bench.rs
index 5e2e7f25a2c0..c60bc7fb724e 100644
--- a/example/mod_bench.rs
+++ b/example/mod_bench.rs
@@ -1,5 +1,6 @@
 #![feature(start, core_intrinsics, lang_items)]
 #![no_std]
+#![allow(internal_features)]
 
 #[link(name = "c")]
 extern {}
diff --git a/failing-lto-tests.txt b/failing-lto-tests.txt
new file mode 100644
index 000000000000..2e0b6134070b
--- /dev/null
+++ b/failing-lto-tests.txt
@@ -0,0 +1,23 @@
+tests/ui/lint/unsafe_code/forge_unsafe_block.rs
+tests/ui/lint/unused-qualification-in-derive-expansion.rs
+tests/ui/macro-quote-test.rs
+tests/ui/macros/proc_macro.rs
+tests/ui/panic-runtime/lto-unwind.rs
+tests/ui/resolve/derive-macro-1.rs
+tests/ui/resolve/derive-macro-2.rs
+tests/ui/rfcs/rfc-2565-param-attrs/param-attrs-pretty.rs
+tests/ui/rfcs/rfc-2565-param-attrs/issue-64682-dropping-first-attrs-in-impl-fns.rs
+tests/ui/rfcs/rfc-3348-c-string-literals/edition-spans.rs
+tests/ui/rust-2018/suggestions-not-always-applicable.rs
+tests/ui/rust-2021/reserved-prefixes-via-macro.rs
+tests/ui/underscore-imports/duplicate.rs
+tests/ui/async-await/issues/issue-60674.rs
+tests/ui/attributes/main-removed-2/main.rs
+tests/ui/cfg/assume-incomplete-release/assume-incomplete.rs
+tests/ui/crate-loading/cross-compiled-proc-macro.rs
+tests/ui/derives/derive-marker-tricky.rs
+tests/ui/diagnostic_namespace/existing_proc_macros.rs
+tests/ui/fmt/format-args-capture-issue-106408.rs
+tests/ui/fmt/indoc-issue-106408.rs
+tests/ui/hygiene/issue-77523-def-site-async-await.rs
+tests/ui/inherent-impls-overlap-check/no-overlap.rs
diff --git a/failing-non-lto-tests.txt b/failing-non-lto-tests.txt
new file mode 100644
index 000000000000..2f338f7d3c87
--- /dev/null
+++ b/failing-non-lto-tests.txt
@@ -0,0 +1,11 @@
+tests/ui/issues/issue-44056.rs
+tests/ui/lto/fat-lto.rs
+tests/ui/lto/debuginfo-lto.rs
+tests/ui/lto/lto-many-codegen-units.rs
+tests/ui/lto/issue-100772.rs
+tests/ui/lto/lto-rustc-loads-linker-plugin.rs
+tests/ui/panic-runtime/lto-unwind.rs
+tests/ui/sanitize/issue-111184-generator-witness.rs
+tests/ui/sepcomp/sepcomp-lib-lto.rs
+tests/ui/lto/lto-opt-level-s.rs
+tests/ui/lto/lto-opt-level-z.rs
diff --git a/failing-ui-tests.txt b/failing-ui-tests.txt
index 801464daae9a..ed56a11a1709 100644
--- a/failing-ui-tests.txt
+++ b/failing-ui-tests.txt
@@ -1,11 +1,5 @@
-tests/ui/allocator/custom-in-block.rs
-tests/ui/allocator/custom-in-submodule.rs
-tests/ui/allocator/custom.rs
-tests/ui/allocator/hygiene.rs
 tests/ui/allocator/no_std-alloc-error-handler-custom.rs
 tests/ui/allocator/no_std-alloc-error-handler-default.rs
-tests/ui/allocator/xcrate-use.rs
-tests/ui/allocator/xcrate-use2.rs
 tests/ui/asm/may_unwind.rs
 tests/ui/asm/x86_64/multiple-clobber-abi.rs
 tests/ui/debuginfo/debuginfo-emit-llvm-ir-and-split-debuginfo.rs
@@ -14,15 +8,12 @@ tests/ui/linkage-attr/linkage1.rs
 tests/ui/lto/dylib-works.rs
 tests/ui/numbers-arithmetic/saturating-float-casts.rs
 tests/ui/polymorphization/promoted-function.rs
-tests/ui/process/nofile-limit.rs
 tests/ui/sepcomp/sepcomp-cci.rs
 tests/ui/sepcomp/sepcomp-extern.rs
 tests/ui/sepcomp/sepcomp-fns-backwards.rs
 tests/ui/sepcomp/sepcomp-fns.rs
 tests/ui/sepcomp/sepcomp-statics.rs
 tests/ui/simd/intrinsic/generic-arithmetic-pass.rs
-tests/ui/sse2.rs
-tests/ui/target-feature/missing-plusminus.rs
 tests/ui/asm/x86_64/may_unwind.rs
 tests/ui/backtrace.rs
 tests/ui/catch-unwind-bang.rs
@@ -54,8 +45,8 @@ tests/ui/issues/issue-40883.rs
 tests/ui/issues/issue-43853.rs
 tests/ui/issues/issue-47364.rs
 tests/ui/macros/rfc-2011-nicer-assert-messages/assert-without-captures-does-not-create-unnecessary-code.rs
-tests/ui/rfcs/rfc-2091-track-caller/std-panic-locations.rs
 tests/ui/rfcs/rfc-1857-stabilize-drop-order/drop-order.rs
+tests/ui/rfcs/rfc-2091-track-caller/std-panic-locations.rs
 tests/ui/simd/issue-17170.rs
 tests/ui/simd/issue-39720.rs
 tests/ui/simd/issue-89193.rs
@@ -65,6 +56,18 @@ tests/ui/alloc-error/default-alloc-error-hook.rs
 tests/ui/generator/panic-safe.rs
 tests/ui/issues/issue-14875.rs
 tests/ui/issues/issue-29948.rs
-tests/ui/panic-while-printing.rs
-tests/ui/enum-discriminant/get_discr.rs
 tests/ui/panics/nested_panic_caught.rs
+tests/ui/simd/intrinsic/generic-bswap-byte.rs
+tests/ui/const_prop/ice-issue-111353.rs
+tests/ui/process/println-with-broken-pipe.rs
+tests/ui/panic-runtime/lto-abort.rs
+tests/ui/lto/thin-lto-inlines2.rs
+tests/ui/lto/weak-works.rs
+tests/ui/lto/thin-lto-inlines.rs
+tests/ui/lto/thin-lto-global-allocator.rs
+tests/ui/lto/msvc-imp-present.rs
+tests/ui/lto/lto-thin-rustc-loads-linker-plugin.rs
+tests/ui/lto/all-crates.rs
+tests/ui/async-await/deep-futures-are-freeze.rs
+tests/ui/closures/capture-unsized-by-ref.rs
+tests/ui/generator/resume-after-return.rs
diff --git a/failing-ui-tests12.txt b/failing-ui-tests12.txt
index 8c27bd8b8ca8..0ac0a034af40 100644
--- a/failing-ui-tests12.txt
+++ b/failing-ui-tests12.txt
@@ -37,3 +37,4 @@ tests/ui/simd/intrinsic/generic-gather-pass.rs
 tests/ui/simd/issue-85915-simd-ptrs.rs
 tests/ui/issues/issue-68010-large-zst-consts.rs
 tests/ui/rust-2018/proc-macro-crate-in-paths.rs
+tests/ui/target-feature/missing-plusminus.rs
diff --git a/messages.ftl b/messages.ftl
index 2fd0daee3e73..5ca0a2e1b6db 100644
--- a/messages.ftl
+++ b/messages.ftl
@@ -1,3 +1,7 @@
+codegen_gcc_unknown_ctarget_feature_prefix =
+    unknown feature specified for `-Ctarget-feature`: `{$feature}`
+    .note = features must begin with a `+` to enable or `-` to disable it
+
 codegen_gcc_invalid_minimum_alignment =
     invalid minimum global alignment: {$err}
 
@@ -9,3 +13,29 @@ codegen_gcc_tied_target_features = the target features {$features} must all be e
 
 codegen_gcc_unwinding_inline_asm =
     GCC backend does not support unwinding from inline asm
+
+codegen_gcc_copy_bitcode = failed to copy bitcode to object file: {$err}
+
+codegen_gcc_dynamic_linking_with_lto =
+    cannot prefer dynamic linking when performing LTO
+    .note = only 'staticlib', 'bin', and 'cdylib' outputs are supported with LTO
+
+codegen_gcc_load_bitcode = failed to load bitcode of module "{$name}"
+
+codegen_gcc_lto_disallowed = lto can only be run for executables, cdylibs and static library outputs
+
+codegen_gcc_lto_dylib = lto cannot be used for `dylib` crate type without `-Zdylib-lto`
+
+codegen_gcc_lto_bitcode_from_rlib = failed to get bitcode from object file for LTO ({$gcc_err})
+
+codegen_gcc_unknown_ctarget_feature =
+    unknown feature specified for `-Ctarget-feature`: `{$feature}`
+    .note = it is still passed through to the codegen backend
+    .possible_feature = you might have meant: `{$rust_feature}`
+    .consider_filing_feature_request = consider filing a feature request
+
+codegen_gcc_missing_features =
+    add the missing features in a `target_feature` attribute
+
+codegen_gcc_target_feature_disable_or_enable =
+    the target features {$features} must all be either enabled or disabled together
diff --git a/patches/0001-Add-stdarch-Cargo.toml-for-testing.patch b/patches/0001-Add-stdarch-Cargo.toml-for-testing.patch
index 93c63b5dcacf..2a55f2cb796f 100644
--- a/patches/0001-Add-stdarch-Cargo.toml-for-testing.patch
+++ b/patches/0001-Add-stdarch-Cargo.toml-for-testing.patch
@@ -1,25 +1,26 @@
-From c3821e02fbd6cb5ad6e06d759fccdc9073712375 Mon Sep 17 00:00:00 2001
+From b8f3eed3053c9333b5dfbeaeb2a6a65a4b3156df Mon Sep 17 00:00:00 2001
 From: Antoni Boucher <bouanto@zoho.com>
-Date: Tue, 7 Jun 2022 21:40:13 -0400
-Subject: [PATCH] Add stdarch Cargo.toml for testing
+Date: Tue, 29 Aug 2023 13:06:34 -0400
+Subject: [PATCH] Patch 0001-Add-stdarch-Cargo.toml-for-testing.patch
 
 ---
- library/stdarch/Cargo.toml | 20 ++++++++++++++++++++
- 1 file changed, 20 insertions(+)
+ library/stdarch/Cargo.toml | 23 +++++++++++++++++++++++
+ 1 file changed, 23 insertions(+)
  create mode 100644 library/stdarch/Cargo.toml
 
 diff --git a/library/stdarch/Cargo.toml b/library/stdarch/Cargo.toml
 new file mode 100644
-index 0000000..fbe0a95
+index 0000000..4c63700
 --- /dev/null
 +++ b/library/stdarch/Cargo.toml
-@@ -0,0 +1,20 @@
+@@ -0,0 +1,21 @@
 +[workspace]
++resolver = "1"
 +members = [
 +  "crates/core_arch",
 +  "crates/std_detect",
 +  "crates/stdarch-gen",
-+  "examples/"
++  #"examples/"
 +]
 +exclude = [
 +  "crates/wasm-assert-instr-tests"
@@ -35,5 +36,5 @@ index 0000000..fbe0a95
 +opt-level = 3
 +incremental = true
 -- 
-2.26.2.7.g19db9cfb68.dirty
+2.42.0
 
diff --git a/patches/0001-Disable-examples.patch b/patches/0001-Disable-examples.patch
deleted file mode 100644
index 1b71df1ca8df..000000000000
--- a/patches/0001-Disable-examples.patch
+++ /dev/null
@@ -1,25 +0,0 @@
-From a2d53a324a02c04b76c0e9d39dc15cd443a3b8b2 Mon Sep 17 00:00:00 2001
-From: Antoni Boucher <bouanto@zoho.com>
-Date: Fri, 25 Nov 2022 11:18:11 -0500
-Subject: [PATCH] Disable examples
-
----
- library/stdarch/Cargo.toml | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/library/stdarch/Cargo.toml b/library/stdarch/Cargo.toml
-index fbe0a95..748d72d 100644
---- a/library/stdarch/Cargo.toml
-+++ b/library/stdarch/Cargo.toml
-@@ -3,7 +3,7 @@ members = [
-   "crates/core_arch",
-   "crates/std_detect",
-   "crates/stdarch-gen",
--  "examples/"
-+  #"examples/"
- ]
- exclude = [
-   "crates/wasm-assert-instr-tests"
--- 
-2.26.2.7.g19db9cfb68.dirty
-
diff --git a/prepare.sh b/prepare.sh
deleted file mode 100755
index e98f24c6e128..000000000000
--- a/prepare.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-set -e
-set -v
-
-source prepare_build.sh
-
-cargo install hyperfine || echo "Skipping hyperfine install"
-
-git clone https://github.com/rust-random/rand.git || echo "rust-random/rand has already been cloned"
-pushd rand
-git checkout -- .
-git checkout 0f933f9c7176e53b2a3c7952ded484e1783f0bf1
-git am ../crate_patches/*-rand-*.patch
-popd
-
-git clone https://github.com/rust-lang/regex.git || echo "rust-lang/regex has already been cloned"
-pushd regex
-git checkout -- .
-git checkout 341f207c1071f7290e3f228c710817c280c8dca1
-popd
-
-git clone https://github.com/ebobby/simple-raytracer || echo "ebobby/simple-raytracer has already been cloned"
-pushd simple-raytracer
-git checkout -- .
-git checkout 804a7a21b9e673a482797aa289a18ed480e4d813
-
-# build with cg_llvm for perf comparison
-cargo build
-mv target/debug/main raytracer_cg_llvm
-popd
diff --git a/prepare_build.sh b/prepare_build.sh
deleted file mode 100755
index 8194360da4ba..000000000000
--- a/prepare_build.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/usr/bin/env bash
-set -e
-set -v
-
-./build_sysroot/prepare_sysroot_src.sh
diff --git a/rust-toolchain b/rust-toolchain
index ebb04d0069cf..25a1cea98cce 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2023-06-19"
+channel = "nightly-2023-10-08"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]
diff --git a/rustup.sh b/rustup.sh
index 041079bc9c6f..a4f938e4b5b7 100755
--- a/rustup.sh
+++ b/rustup.sh
@@ -16,7 +16,7 @@ case $1 in
         done
 
         ./clean_all.sh
-        ./prepare.sh
+        ./y.sh prepare
         ;;
     "commit")
         git add rust-toolchain
diff --git a/src/abi.rs b/src/abi.rs
index a49530ebb4c2..35bb0b6e5f4e 100644
--- a/src/abi.rs
+++ b/src/abi.rs
@@ -3,7 +3,9 @@ use rustc_codegen_ssa::traits::{AbiBuilderMethods, BaseTypeMethods};
 use rustc_data_structures::fx::FxHashSet;
 use rustc_middle::bug;
 use rustc_middle::ty::Ty;
-use rustc_target::abi::call::{CastTarget, FnAbi, PassMode, Reg, RegKind};
+#[cfg(feature = "master")]
+use rustc_session::config;
+use rustc_target::abi::call::{ArgAttributes, CastTarget, FnAbi, PassMode, Reg, RegKind};
 
 use crate::builder::Builder;
 use crate::context::CodegenCx;
@@ -120,30 +122,50 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                 }
             };
 
+        #[cfg(feature = "master")]
+        let apply_attrs = |ty: Type<'gcc>, attrs: &ArgAttributes| {
+            if cx.sess().opts.optimize != config::OptLevel::No
+                && attrs.regular.contains(rustc_target::abi::call::ArgAttribute::NoAlias)
+            {
+                ty.make_restrict()
+            } else {
+                ty
+            }
+        };
+        #[cfg(not(feature = "master"))]
+        let apply_attrs = |ty: Type<'gcc>, _attrs: &ArgAttributes| {
+            ty
+        };
+
         for arg in self.args.iter() {
             let arg_ty = match arg.mode {
                 PassMode::Ignore => continue,
-                PassMode::Direct(_) => arg.layout.immediate_gcc_type(cx),
-                PassMode::Pair(..) => {
-                    argument_tys.push(arg.layout.scalar_pair_element_gcc_type(cx, 0));
-                    argument_tys.push(arg.layout.scalar_pair_element_gcc_type(cx, 1));
+                PassMode::Pair(a, b) => {
+                    argument_tys.push(apply_attrs(arg.layout.scalar_pair_element_gcc_type(cx, 0), &a));
+                    argument_tys.push(apply_attrs(arg.layout.scalar_pair_element_gcc_type(cx, 1), &b));
                     continue;
                 }
-                PassMode::Indirect { meta_attrs: Some(_), .. } => {
-                    unimplemented!();
-                }
                 PassMode::Cast { ref cast, pad_i32 } => {
                     // add padding
                     if pad_i32 {
                         argument_tys.push(Reg::i32().gcc_type(cx));
                     }
-                    cast.gcc_type(cx)
+                    let ty = cast.gcc_type(cx);
+                    apply_attrs(ty, &cast.attrs)
                 }
-                PassMode::Indirect { meta_attrs: None, on_stack: true, .. } => {
+                PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: true } => {
+                    // This is a "byval" argument, so we don't apply the `restrict` attribute on it.
                     on_stack_param_indices.insert(argument_tys.len());
                     arg.memory_ty(cx)
                 },
-                PassMode::Indirect { meta_attrs: None, on_stack: false, .. } => cx.type_ptr_to(arg.memory_ty(cx)),
+                PassMode::Direct(attrs) => apply_attrs(arg.layout.immediate_gcc_type(cx), &attrs),
+                PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
+                    apply_attrs(cx.type_ptr_to(arg.memory_ty(cx)), &attrs)
+                }
+                PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => {
+                    assert!(!on_stack);
+                    apply_attrs(apply_attrs(cx.type_ptr_to(arg.memory_ty(cx)), &attrs), &meta_attrs)
+                }
             };
             argument_tys.push(arg_ty);
         }
diff --git a/src/allocator.rs b/src/allocator.rs
index edd7ab722f61..c8c098e2973f 100644
--- a/src/allocator.rs
+++ b/src/allocator.rs
@@ -1,6 +1,6 @@
 #[cfg(feature="master")]
 use gccjit::FnAttribute;
-use gccjit::{FunctionType, GlobalKind, ToRValue};
+use gccjit::{Context, FunctionType, GlobalKind, ToRValue, Type};
 use rustc_ast::expand::allocator::{
     alloc_error_handler_name, default_fn_name, global_fn_name, AllocatorKind, AllocatorTy,
     ALLOCATOR_METHODS, NO_ALLOC_SHIM_IS_UNSTABLE,
@@ -22,7 +22,6 @@ pub(crate) unsafe fn codegen(tcx: TyCtxt<'_>, mods: &mut GccContext, _module_nam
         };
     let i8 = context.new_type::<i8>();
     let i8p = i8.make_pointer();
-    let void = context.new_type::<()>();
 
     if kind == AllocatorKind::Default {
         for method in ALLOCATOR_METHODS {
@@ -47,80 +46,22 @@ pub(crate) unsafe fn codegen(tcx: TyCtxt<'_>, mods: &mut GccContext, _module_nam
                     panic!("invalid allocator output")
                 }
             };
-            let name = global_fn_name(method.name);
+            let from_name = global_fn_name(method.name);
+            let to_name = default_fn_name(method.name);
 
-            let args: Vec<_> = types.iter().enumerate()
-                .map(|(index, typ)| context.new_parameter(None, *typ, &format!("param{}", index)))
-                .collect();
-            let func = context.new_function(None, FunctionType::Exported, output.unwrap_or(void), &args, name, false);
-
-            if tcx.sess.target.options.default_hidden_visibility {
-                #[cfg(feature="master")]
-                func.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Hidden));
-            }
-            if tcx.sess.must_emit_unwind_tables() {
-                // TODO(antoyo): emit unwind tables.
-            }
-
-            let callee = default_fn_name(method.name);
-            let args: Vec<_> = types.iter().enumerate()
-                .map(|(index, typ)| context.new_parameter(None, *typ, &format!("param{}", index)))
-                .collect();
-            let callee = context.new_function(None, FunctionType::Extern, output.unwrap_or(void), &args, callee, false);
-            #[cfg(feature="master")]
-            callee.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Hidden));
-
-            let block = func.new_block("entry");
-
-            let args = args
-                .iter()
-                .enumerate()
-                .map(|(i, _)| func.get_param(i as i32).to_rvalue())
-                .collect::<Vec<_>>();
-            let ret = context.new_call(None, callee, &args);
-            //llvm::LLVMSetTailCall(ret, True);
-            if output.is_some() {
-                block.end_with_return(None, ret);
-            }
-            else {
-                block.end_with_void_return(None);
-            }
-
-            // TODO(@Commeownist): Check if we need to emit some extra debugging info in certain circumstances
-            // as described in https://github.com/rust-lang/rust/commit/77a96ed5646f7c3ee8897693decc4626fe380643
+            create_wrapper_function(tcx, context, &from_name, &to_name, &types, output);
         }
     }
 
-    let types = [usize, usize];
-    let name = "__rust_alloc_error_handler".to_string();
-    let args: Vec<_> = types.iter().enumerate()
-        .map(|(index, typ)| context.new_parameter(None, *typ, &format!("param{}", index)))
-        .collect();
-    let func = context.new_function(None, FunctionType::Exported, void, &args, name, false);
-
-    if tcx.sess.target.default_hidden_visibility {
-        #[cfg(feature="master")]
-        func.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Hidden));
-    }
-
-    let callee = alloc_error_handler_name(alloc_error_handler_kind);
-    let args: Vec<_> = types.iter().enumerate()
-        .map(|(index, typ)| context.new_parameter(None, *typ, &format!("param{}", index)))
-        .collect();
-    let callee = context.new_function(None, FunctionType::Extern, void, &args, callee, false);
-    #[cfg(feature="master")]
-    callee.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Hidden));
-
-    let block = func.new_block("entry");
-
-    let args = args
-        .iter()
-        .enumerate()
-        .map(|(i, _)| func.get_param(i as i32).to_rvalue())
-        .collect::<Vec<_>>();
-    let _ret = context.new_call(None, callee, &args);
-    //llvm::LLVMSetTailCall(ret, True);
-    block.end_with_void_return(None);
+    // FIXME(bjorn3): Add noreturn attribute
+    create_wrapper_function(
+        tcx,
+        context,
+        "__rust_alloc_error_handler",
+        &alloc_error_handler_name(alloc_error_handler_kind),
+        &[usize, usize],
+        None,
+    );
 
     let name = OomStrategy::SYMBOL.to_string();
     let global = context.new_global(None, GlobalKind::Exported, i8, name);
@@ -133,3 +74,53 @@ pub(crate) unsafe fn codegen(tcx: TyCtxt<'_>, mods: &mut GccContext, _module_nam
     let value = context.new_rvalue_from_int(i8, 0);
     global.global_set_initializer_rvalue(value);
 }
+
+fn create_wrapper_function(
+    tcx: TyCtxt<'_>,
+    context: &Context<'_>,
+    from_name: &str,
+    to_name: &str,
+    types: &[Type<'_>],
+    output: Option<Type<'_>>,
+) {
+    let void = context.new_type::<()>();
+
+    let args: Vec<_> = types.iter().enumerate()
+        .map(|(index, typ)| context.new_parameter(None, *typ, &format!("param{}", index)))
+        .collect();
+    let func = context.new_function(None, FunctionType::Exported, output.unwrap_or(void), &args, from_name, false);
+
+    if tcx.sess.target.options.default_hidden_visibility {
+        #[cfg(feature="master")]
+        func.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Hidden));
+    }
+    if tcx.sess.must_emit_unwind_tables() {
+        // TODO(antoyo): emit unwind tables.
+    }
+
+    let args: Vec<_> = types.iter().enumerate()
+        .map(|(index, typ)| context.new_parameter(None, *typ, &format!("param{}", index)))
+        .collect();
+    let callee = context.new_function(None, FunctionType::Extern, output.unwrap_or(void), &args, to_name, false);
+    #[cfg(feature="master")]
+    callee.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Hidden));
+
+    let block = func.new_block("entry");
+
+    let args = args
+        .iter()
+        .enumerate()
+        .map(|(i, _)| func.get_param(i as i32).to_rvalue())
+        .collect::<Vec<_>>();
+    let ret = context.new_call(None, callee, &args);
+    //llvm::LLVMSetTailCall(ret, True);
+    if output.is_some() {
+        block.end_with_return(None, ret);
+    }
+    else {
+        block.end_with_void_return(None);
+    }
+
+    // TODO(@Commeownist): Check if we need to emit some extra debugging info in certain circumstances
+    // as described in https://github.com/rust-lang/rust/commit/77a96ed5646f7c3ee8897693decc4626fe380643
+}
diff --git a/src/asm.rs b/src/asm.rs
index 905fdac92e9e..f3a9ca77a67b 100644
--- a/src/asm.rs
+++ b/src/asm.rs
@@ -452,10 +452,6 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                         }
 
                         InlineAsmOperandRef::Const { ref string } => {
-                            // Const operands get injected directly into the template
-                            if att_dialect {
-                                template_str.push('$');
-                            }
                             template_str.push_str(string);
                         }
                     }
diff --git a/src/attributes.rs b/src/attributes.rs
index eb0cce19b85c..971e019a4f60 100644
--- a/src/attributes.rs
+++ b/src/attributes.rs
@@ -4,72 +4,13 @@ use gccjit::Function;
 use rustc_attr::InstructionSetAttr;
 #[cfg(feature="master")]
 use rustc_attr::InlineAttr;
-use rustc_codegen_ssa::target_features::tied_target_features;
-use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::ty;
 #[cfg(feature="master")]
 use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
-use rustc_session::Session;
 use rustc_span::symbol::sym;
-use smallvec::{smallvec, SmallVec};
 
 use crate::{context::CodegenCx, errors::TiedTargetFeatures};
-
-// Given a map from target_features to whether they are enabled or disabled,
-// ensure only valid combinations are allowed.
-pub fn check_tied_features(sess: &Session, features: &FxHashMap<&str, bool>) -> Option<&'static [&'static str]> {
-    for tied in tied_target_features(sess) {
-        // Tied features must be set to the same value, or not set at all
-        let mut tied_iter = tied.iter();
-        let enabled = features.get(tied_iter.next().unwrap());
-        if tied_iter.any(|feature| enabled != features.get(feature)) {
-            return Some(tied);
-        }
-    }
-    None
-}
-
-// TODO(antoyo): maybe move to a new module gcc_util.
-// To find a list of GCC's names, check https://gcc.gnu.org/onlinedocs/gcc/Function-Attributes.html
-fn to_gcc_features<'a>(sess: &Session, s: &'a str) -> SmallVec<[&'a str; 2]> {
-    let arch = if sess.target.arch == "x86_64" { "x86" } else { &*sess.target.arch };
-    match (arch, s) {
-        ("x86", "sse4.2") => smallvec!["sse4.2", "crc32"],
-        ("x86", "pclmulqdq") => smallvec!["pclmul"],
-        ("x86", "rdrand") => smallvec!["rdrnd"],
-        ("x86", "bmi1") => smallvec!["bmi"],
-        ("x86", "cmpxchg16b") => smallvec!["cx16"],
-        ("x86", "avx512vaes") => smallvec!["vaes"],
-        ("x86", "avx512gfni") => smallvec!["gfni"],
-        ("x86", "avx512vpclmulqdq") => smallvec!["vpclmulqdq"],
-        // NOTE: seems like GCC requires 'avx512bw' for 'avx512vbmi2'.
-        ("x86", "avx512vbmi2") => smallvec!["avx512vbmi2", "avx512bw"],
-        // NOTE: seems like GCC requires 'avx512bw' for 'avx512bitalg'.
-        ("x86", "avx512bitalg") => smallvec!["avx512bitalg", "avx512bw"],
-        ("aarch64", "rcpc2") => smallvec!["rcpc-immo"],
-        ("aarch64", "dpb") => smallvec!["ccpp"],
-        ("aarch64", "dpb2") => smallvec!["ccdp"],
-        ("aarch64", "frintts") => smallvec!["fptoint"],
-        ("aarch64", "fcma") => smallvec!["complxnum"],
-        ("aarch64", "pmuv3") => smallvec!["perfmon"],
-        ("aarch64", "paca") => smallvec!["pauth"],
-        ("aarch64", "pacg") => smallvec!["pauth"],
-        // Rust ties fp and neon together. In LLVM neon implicitly enables fp,
-        // but we manually enable neon when a feature only implicitly enables fp
-        ("aarch64", "f32mm") => smallvec!["f32mm", "neon"],
-        ("aarch64", "f64mm") => smallvec!["f64mm", "neon"],
-        ("aarch64", "fhm") => smallvec!["fp16fml", "neon"],
-        ("aarch64", "fp16") => smallvec!["fullfp16", "neon"],
-        ("aarch64", "jsconv") => smallvec!["jsconv", "neon"],
-        ("aarch64", "sve") => smallvec!["sve", "neon"],
-        ("aarch64", "sve2") => smallvec!["sve2", "neon"],
-        ("aarch64", "sve2-aes") => smallvec!["sve2-aes", "neon"],
-        ("aarch64", "sve2-sm4") => smallvec!["sve2-sm4", "neon"],
-        ("aarch64", "sve2-sha3") => smallvec!["sve2-sha3", "neon"],
-        ("aarch64", "sve2-bitperm") => smallvec!["sve2-bitperm", "neon"],
-        (_, s) => smallvec![s],
-    }
-}
+use crate::gcc_util::{check_tied_features, to_gcc_features};
 
 /// Get GCC attribute for the provided inline heuristic.
 #[cfg(feature="master")]
@@ -114,6 +55,19 @@ pub fn from_fn_attrs<'gcc, 'tcx>(
         if let Some(attr) = inline_attr(cx, inline) {
             func.add_attribute(attr);
         }
+
+        if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::COLD) {
+            func.add_attribute(FnAttribute::Cold);
+        }
+        if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_RETURNS_TWICE) {
+            func.add_attribute(FnAttribute::ReturnsTwice);
+        }
+        if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_PURE) {
+            func.add_attribute(FnAttribute::Pure);
+        }
+        if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_CONST) {
+            func.add_attribute(FnAttribute::Const);
+        }
     }
 
     let function_features =
@@ -140,11 +94,33 @@ pub fn from_fn_attrs<'gcc, 'tcx>(
         }))
         .collect::<Vec<_>>();
 
-    // TODO(antoyo): check if we really need global backend features. (Maybe they could be applied
-    // globally?)
+    // TODO(antoyo): cg_llvm adds global features to each function so that LTO keep them.
+    // Check if GCC requires the same.
     let mut global_features = cx.tcx.global_backend_features(()).iter().map(|s| s.as_str());
     function_features.extend(&mut global_features);
-    let target_features = function_features.join(",");
+    let target_features = function_features
+        .iter()
+        .filter_map(|feature| {
+            // FIXME(antoyo): for some reasons, disabling SSE results in the following error when
+            // compiling Rust for Linux:
+            // SSE register return with SSE disabled
+            // TODO(antoyo): support soft-float and retpoline-external-thunk.
+            if feature.contains("soft-float") || feature.contains("retpoline-external-thunk") || *feature == "-sse" {
+                return None;
+            }
+
+            if feature.starts_with('-') {
+                Some(format!("no{}", feature))
+            }
+            else if feature.starts_with('+') {
+                Some(feature[1..].to_string())
+            }
+            else {
+                Some(feature.to_string())
+            }
+        })
+        .collect::<Vec<_>>()
+        .join(",");
     if !target_features.is_empty() {
         #[cfg(feature="master")]
         func.add_attribute(FnAttribute::Target(&target_features));
diff --git a/src/back/lto.rs b/src/back/lto.rs
new file mode 100644
index 000000000000..529454b119e8
--- /dev/null
+++ b/src/back/lto.rs
@@ -0,0 +1,341 @@
+/// GCC requires to use the same toolchain for the whole compilation when doing LTO.
+/// So, we need the same version/commit of the linker (gcc) and lto front-end binaries (lto1,
+/// lto-wrapper, liblto_plugin.so).
+
+// FIXME(antoyo): the executables compiled with LTO are bigger than those compiled without LTO.
+// Since it is the opposite for cg_llvm, check if this is normal.
+//
+// Maybe we embed the bitcode in the final binary?
+// It doesn't look like we try to generate fat objects for the final binary.
+// Check if the way we combine the object files make it keep the LTO sections on the final link.
+// Maybe that's because the combined object files contain the IR (true) and the final link
+// does not remove it?
+//
+// TODO(antoyo): for performance, check which optimizations the C++ frontend enables.
+//
+// Fix these warnings:
+// /usr/bin/ld: warning: type of symbol `_RNvNvNvNtCs5JWOrf9uCus_5rayon11thread_pool19WORKER_THREAD_STATE7___getit5___KEY' changed from 1 to 6 in /tmp/ccKeUSiR.ltrans0.ltrans.o
+// /usr/bin/ld: warning: type of symbol `_RNvNvNvNvNtNtNtCsAj5i4SGTR7_3std4sync4mpmc5waker17current_thread_id5DUMMY7___getit5___KEY' changed from 1 to 6 in /tmp/ccKeUSiR.ltrans0.ltrans.o
+// /usr/bin/ld: warning: incremental linking of LTO and non-LTO objects; using -flinker-output=nolto-rel which will bypass whole program optimization
+
+use std::ffi::CString;
+use std::fs::{self, File};
+use std::path::{Path, PathBuf};
+
+use gccjit::OutputKind;
+use object::read::archive::ArchiveFile;
+use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule};
+use rustc_codegen_ssa::back::symbol_export;
+use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput};
+use rustc_codegen_ssa::traits::*;
+use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind};
+use rustc_data_structures::memmap::Mmap;
+use rustc_errors::{FatalError, Handler};
+use rustc_hir::def_id::LOCAL_CRATE;
+use rustc_middle::dep_graph::WorkProduct;
+use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel};
+use rustc_session::config::{CrateType, Lto};
+use tempfile::{TempDir, tempdir};
+
+use crate::back::write::save_temp_bitcode;
+use crate::errors::{
+    DynamicLinkingWithLTO, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib,
+};
+use crate::{GccCodegenBackend, GccContext, to_gcc_opt_level};
+
+/// We keep track of the computed LTO cache keys from the previous
+/// session to determine which CGUs we can reuse.
+//pub const THIN_LTO_KEYS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-keys.bin";
+
+pub fn crate_type_allows_lto(crate_type: CrateType) -> bool {
+    match crate_type {
+        CrateType::Executable | CrateType::Dylib | CrateType::Staticlib | CrateType::Cdylib => true,
+        CrateType::Rlib | CrateType::ProcMacro => false,
+    }
+}
+
+struct LtoData {
+    // TODO(antoyo): use symbols_below_threshold.
+    //symbols_below_threshold: Vec<CString>,
+    upstream_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
+    tmp_path: TempDir,
+}
+
+fn prepare_lto(cgcx: &CodegenContext<GccCodegenBackend>, diag_handler: &Handler) -> Result<LtoData, FatalError> {
+    let export_threshold = match cgcx.lto {
+        // We're just doing LTO for our one crate
+        Lto::ThinLocal => SymbolExportLevel::Rust,
+
+        // We're doing LTO for the entire crate graph
+        Lto::Fat | Lto::Thin => symbol_export::crates_export_threshold(&cgcx.crate_types),
+
+        Lto::No => panic!("didn't request LTO but we're doing LTO"),
+    };
+
+    let tmp_path =
+        match tempdir() {
+            Ok(tmp_path) => tmp_path,
+            Err(error) => {
+                eprintln!("Cannot create temporary directory: {}", error);
+                return Err(FatalError);
+            },
+        };
+
+    let symbol_filter = &|&(ref name, info): &(String, SymbolExportInfo)| {
+        if info.level.is_below_threshold(export_threshold) || info.used {
+            Some(CString::new(name.as_str()).unwrap())
+        } else {
+            None
+        }
+    };
+    let exported_symbols = cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
+    let mut symbols_below_threshold = {
+        let _timer = cgcx.prof.generic_activity("GCC_lto_generate_symbols_below_threshold");
+        exported_symbols[&LOCAL_CRATE].iter().filter_map(symbol_filter).collect::<Vec<CString>>()
+    };
+    info!("{} symbols to preserve in this crate", symbols_below_threshold.len());
+
+    // If we're performing LTO for the entire crate graph, then for each of our
+    // upstream dependencies, find the corresponding rlib and load the bitcode
+    // from the archive.
+    //
+    // We save off all the bytecode and GCC module file path for later processing
+    // with either fat or thin LTO
+    let mut upstream_modules = Vec::new();
+    if cgcx.lto != Lto::ThinLocal {
+        // Make sure we actually can run LTO
+        for crate_type in cgcx.crate_types.iter() {
+            if !crate_type_allows_lto(*crate_type) {
+                diag_handler.emit_err(LtoDisallowed);
+                return Err(FatalError);
+            } else if *crate_type == CrateType::Dylib {
+                if !cgcx.opts.unstable_opts.dylib_lto {
+                    diag_handler.emit_err(LtoDylib);
+                    return Err(FatalError);
+                }
+            }
+        }
+
+        if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto {
+            diag_handler.emit_err(DynamicLinkingWithLTO);
+            return Err(FatalError);
+        }
+
+        for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() {
+            let exported_symbols =
+                cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
+            {
+                let _timer =
+                    cgcx.prof.generic_activity("GCC_lto_generate_symbols_below_threshold");
+                symbols_below_threshold
+                    .extend(exported_symbols[&cnum].iter().filter_map(symbol_filter));
+            }
+
+            let archive_data = unsafe {
+                Mmap::map(File::open(&path).expect("couldn't open rlib"))
+                    .expect("couldn't map rlib")
+            };
+            let archive = ArchiveFile::parse(&*archive_data).expect("wanted an rlib");
+            let obj_files = archive
+                .members()
+                .filter_map(|child| {
+                    child.ok().and_then(|c| {
+                        std::str::from_utf8(c.name()).ok().map(|name| (name.trim(), c))
+                    })
+                })
+                .filter(|&(name, _)| looks_like_rust_object_file(name));
+            for (name, child) in obj_files {
+                info!("adding bitcode from {}", name);
+                let path = tmp_path.path().join(name);
+                match save_as_file(child.data(&*archive_data).expect("corrupt rlib"), &path) {
+                    Ok(()) => {
+                        let buffer = ModuleBuffer::new(path);
+                        let module = SerializedModule::Local(buffer);
+                        upstream_modules.push((module, CString::new(name).unwrap()));
+                    }
+                    Err(e) => {
+                        diag_handler.emit_err(e);
+                        return Err(FatalError);
+                    }
+                }
+            }
+        }
+    }
+
+    Ok(LtoData {
+        //symbols_below_threshold,
+        upstream_modules,
+        tmp_path,
+    })
+}
+
+fn save_as_file(obj: &[u8], path: &Path) -> Result<(), LtoBitcodeFromRlib> {
+    fs::write(path, obj)
+        .map_err(|error| LtoBitcodeFromRlib {
+            gcc_err: format!("write object file to temp dir: {}", error)
+        })
+}
+
+/// Performs fat LTO by merging all modules into a single one and returning it
+/// for further optimization.
+pub(crate) fn run_fat(
+    cgcx: &CodegenContext<GccCodegenBackend>,
+    modules: Vec<FatLtoInput<GccCodegenBackend>>,
+    cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
+) -> Result<LtoModuleCodegen<GccCodegenBackend>, FatalError> {
+    let diag_handler = cgcx.create_diag_handler();
+    let lto_data = prepare_lto(cgcx, &diag_handler)?;
+    /*let symbols_below_threshold =
+        lto_data.symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();*/
+    fat_lto(cgcx, &diag_handler, modules, cached_modules, lto_data.upstream_modules, lto_data.tmp_path,
+        //&symbols_below_threshold,
+    )
+}
+
+fn fat_lto(cgcx: &CodegenContext<GccCodegenBackend>, _diag_handler: &Handler, modules: Vec<FatLtoInput<GccCodegenBackend>>, cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>, mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>, tmp_path: TempDir,
+    //symbols_below_threshold: &[*const libc::c_char],
+) -> Result<LtoModuleCodegen<GccCodegenBackend>, FatalError> {
+    let _timer = cgcx.prof.generic_activity("GCC_fat_lto_build_monolithic_module");
+    info!("going for a fat lto");
+
+    // Sort out all our lists of incoming modules into two lists.
+    //
+    // * `serialized_modules` (also and argument to this function) contains all
+    //   modules that are serialized in-memory.
+    // * `in_memory` contains modules which are already parsed and in-memory,
+    //   such as from multi-CGU builds.
+    //
+    // All of `cached_modules` (cached from previous incremental builds) can
+    // immediately go onto the `serialized_modules` modules list and then we can
+    // split the `modules` array into these two lists.
+    let mut in_memory = Vec::new();
+    serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| {
+        info!("pushing cached module {:?}", wp.cgu_name);
+        (buffer, CString::new(wp.cgu_name).unwrap())
+    }));
+    for module in modules {
+        match module {
+            FatLtoInput::InMemory(m) => in_memory.push(m),
+            FatLtoInput::Serialized { name, buffer } => {
+                info!("pushing serialized module {:?}", name);
+                let buffer = SerializedModule::Local(buffer);
+                serialized_modules.push((buffer, CString::new(name).unwrap()));
+            }
+        }
+    }
+
+    // Find the "costliest" module and merge everything into that codegen unit.
+    // All the other modules will be serialized and reparsed into the new
+    // context, so this hopefully avoids serializing and parsing the largest
+    // codegen unit.
+    //
+    // Additionally use a regular module as the base here to ensure that various
+    // file copy operations in the backend work correctly. The only other kind
+    // of module here should be an allocator one, and if your crate is smaller
+    // than the allocator module then the size doesn't really matter anyway.
+    let costliest_module = in_memory
+        .iter()
+        .enumerate()
+        .filter(|&(_, module)| module.kind == ModuleKind::Regular)
+        .map(|(i, _module)| {
+            //let cost = unsafe { llvm::LLVMRustModuleCost(module.module_llvm.llmod()) };
+            // TODO(antoyo): compute the cost of a module if GCC allows this.
+            (0, i)
+        })
+        .max();
+
+    // If we found a costliest module, we're good to go. Otherwise all our
+    // inputs were serialized which could happen in the case, for example, that
+    // all our inputs were incrementally reread from the cache and we're just
+    // re-executing the LTO passes. If that's the case deserialize the first
+    // module and create a linker with it.
+    let mut module: ModuleCodegen<GccContext> = match costliest_module {
+        Some((_cost, i)) => in_memory.remove(i),
+        None => {
+            unimplemented!("Incremental");
+            /*assert!(!serialized_modules.is_empty(), "must have at least one serialized module");
+            let (buffer, name) = serialized_modules.remove(0);
+            info!("no in-memory regular modules to choose from, parsing {:?}", name);
+            ModuleCodegen {
+                module_llvm: GccContext::parse(cgcx, &name, buffer.data(), diag_handler)?,
+                name: name.into_string().unwrap(),
+                kind: ModuleKind::Regular,
+            }*/
+        }
+    };
+    let mut serialized_bitcode = Vec::new();
+    {
+        info!("using {:?} as a base module", module.name);
+
+        // We cannot load and merge GCC contexts in memory like cg_llvm is doing.
+        // Instead, we combine the object files into a single object file.
+        for module in in_memory {
+            let path = tmp_path.path().to_path_buf().join(&module.name);
+            let path = path.to_str().expect("path");
+            let context = &module.module_llvm.context;
+            let config = cgcx.config(module.kind);
+            // NOTE: we need to set the optimization level here in order for LTO to do its job.
+            context.set_optimization_level(to_gcc_opt_level(config.opt_level));
+            context.add_command_line_option("-flto=auto");
+            context.add_command_line_option("-flto-partition=one");
+            context.compile_to_file(OutputKind::ObjectFile, path);
+            let buffer = ModuleBuffer::new(PathBuf::from(path));
+            let llmod_id = CString::new(&module.name[..]).unwrap();
+            serialized_modules.push((SerializedModule::Local(buffer), llmod_id));
+        }
+        // Sort the modules to ensure we produce deterministic results.
+        serialized_modules.sort_by(|module1, module2| module1.1.cmp(&module2.1));
+
+        // We add the object files and save in should_combine_object_files that we should combine
+        // them into a single object file when compiling later.
+        for (bc_decoded, name) in serialized_modules {
+            let _timer = cgcx
+                .prof
+                .generic_activity_with_arg_recorder("GCC_fat_lto_link_module", |recorder| {
+                    recorder.record_arg(format!("{:?}", name))
+                });
+            info!("linking {:?}", name);
+            match bc_decoded {
+                SerializedModule::Local(ref module_buffer) => {
+                    module.module_llvm.should_combine_object_files = true;
+                    module.module_llvm.context.add_driver_option(module_buffer.0.to_str().expect("path"));
+                },
+                SerializedModule::FromRlib(_) => unimplemented!("from rlib"),
+                SerializedModule::FromUncompressedFile(_) => unimplemented!("from uncompressed file"),
+            }
+            serialized_bitcode.push(bc_decoded);
+        }
+        save_temp_bitcode(cgcx, &module, "lto.input");
+
+        // Internalize everything below threshold to help strip out more modules and such.
+        /*unsafe {
+            let ptr = symbols_below_threshold.as_ptr();
+            llvm::LLVMRustRunRestrictionPass(
+                llmod,
+                ptr as *const *const libc::c_char,
+                symbols_below_threshold.len() as libc::size_t,
+            );*/
+            save_temp_bitcode(cgcx, &module, "lto.after-restriction");
+        //}
+    }
+
+    // NOTE: save the temporary directory used by LTO so that it gets deleted after linking instead
+    // of now.
+    module.module_llvm.temp_dir = Some(tmp_path);
+
+    Ok(LtoModuleCodegen::Fat { module, _serialized_bitcode: serialized_bitcode })
+}
+
+pub struct ModuleBuffer(PathBuf);
+
+impl ModuleBuffer {
+    pub fn new(path: PathBuf) -> ModuleBuffer {
+        ModuleBuffer(path)
+    }
+}
+
+impl ModuleBufferMethods for ModuleBuffer {
+    fn data(&self) -> &[u8] {
+        unimplemented!("data not needed for GCC codegen");
+    }
+}
diff --git a/src/back/mod.rs b/src/back/mod.rs
index d692799d7642..10187eab0d7e 100644
--- a/src/back/mod.rs
+++ b/src/back/mod.rs
@@ -1 +1,2 @@
+pub mod lto;
 pub mod write;
diff --git a/src/back/write.rs b/src/back/write.rs
index 5f54ac4ebc69..04772d7707ab 100644
--- a/src/back/write.rs
+++ b/src/back/write.rs
@@ -2,27 +2,71 @@ use std::{env, fs};
 
 use gccjit::OutputKind;
 use rustc_codegen_ssa::{CompiledModule, ModuleCodegen};
-use rustc_codegen_ssa::back::write::{CodegenContext, EmitObj, ModuleConfig};
+use rustc_codegen_ssa::back::link::ensure_removed;
+use rustc_codegen_ssa::back::write::{BitcodeSection, CodegenContext, EmitObj, ModuleConfig};
 use rustc_errors::Handler;
+use rustc_fs_util::link_or_copy;
 use rustc_session::config::OutputType;
 use rustc_span::fatal_error::FatalError;
 use rustc_target::spec::SplitDebuginfo;
 
 use crate::{GccCodegenBackend, GccContext};
+use crate::errors::CopyBitcode;
 
-pub(crate) unsafe fn codegen(cgcx: &CodegenContext<GccCodegenBackend>, _diag_handler: &Handler, module: ModuleCodegen<GccContext>, config: &ModuleConfig) -> Result<CompiledModule, FatalError> {
-    let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_codegen", &*module.name);
+pub(crate) unsafe fn codegen(cgcx: &CodegenContext<GccCodegenBackend>, diag_handler: &Handler, module: ModuleCodegen<GccContext>, config: &ModuleConfig) -> Result<CompiledModule, FatalError> {
+    let _timer = cgcx.prof.generic_activity_with_arg("GCC_module_codegen", &*module.name);
     {
         let context = &module.module_llvm.context;
 
         let module_name = module.name.clone();
+
+        let should_combine_object_files = module.module_llvm.should_combine_object_files;
+
         let module_name = Some(&module_name[..]);
 
-        let _bc_out = cgcx.output_filenames.temp_path(OutputType::Bitcode, module_name);
+        // NOTE: Only generate object files with GIMPLE when this environment variable is set for
+        // now because this requires a particular setup (same gcc/lto1/lto-wrapper commit as libgccjit).
+        let fat_lto = env::var("EMBED_LTO_BITCODE").as_deref() == Ok("1");
+
+        let bc_out = cgcx.output_filenames.temp_path(OutputType::Bitcode, module_name);
         let obj_out = cgcx.output_filenames.temp_path(OutputType::Object, module_name);
 
-        if config.bitcode_needed() {
+        if config.bitcode_needed() && fat_lto {
+            let _timer = cgcx
+                .prof
+                .generic_activity_with_arg("GCC_module_codegen_make_bitcode", &*module.name);
+
             // TODO(antoyo)
+            /*if let Some(bitcode_filename) = bc_out.file_name() {
+                cgcx.prof.artifact_size(
+                    "llvm_bitcode",
+                    bitcode_filename.to_string_lossy(),
+                    data.len() as u64,
+                );
+            }*/
+
+            if config.emit_bc || config.emit_obj == EmitObj::Bitcode {
+                let _timer = cgcx
+                    .prof
+                    .generic_activity_with_arg("GCC_module_codegen_emit_bitcode", &*module.name);
+                context.add_command_line_option("-flto=auto");
+                context.add_command_line_option("-flto-partition=one");
+                context.compile_to_file(OutputKind::ObjectFile, bc_out.to_str().expect("path to str"));
+            }
+
+            if config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full) {
+                let _timer = cgcx
+                    .prof
+                    .generic_activity_with_arg("GCC_module_codegen_embed_bitcode", &*module.name);
+                // TODO(antoyo): maybe we should call embed_bitcode to have the proper iOS fixes?
+                //embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, data);
+
+                context.add_command_line_option("-flto=auto");
+                context.add_command_line_option("-flto-partition=one");
+                context.add_command_line_option("-ffat-lto-objects");
+                // TODO(antoyo): Send -plugin/usr/lib/gcc/x86_64-pc-linux-gnu/11.1.0/liblto_plugin.so to linker (this should be done when specifying the appropriate rustc cli argument).
+                context.compile_to_file(OutputKind::ObjectFile, bc_out.to_str().expect("path to str"));
+            }
         }
 
         if config.emit_ir {
@@ -32,7 +76,7 @@ pub(crate) unsafe fn codegen(cgcx: &CodegenContext<GccCodegenBackend>, _diag_han
         if config.emit_asm {
             let _timer = cgcx
                 .prof
-                .generic_activity_with_arg("LLVM_module_codegen_emit_asm", &*module.name);
+                .generic_activity_with_arg("GCC_module_codegen_emit_asm", &*module.name);
             let path = cgcx.output_filenames.temp_path(OutputType::Assembly, module_name);
             context.compile_to_file(OutputKind::Assembler, path.to_str().expect("path to str"));
         }
@@ -41,7 +85,7 @@ pub(crate) unsafe fn codegen(cgcx: &CodegenContext<GccCodegenBackend>, _diag_han
             EmitObj::ObjectCode(_) => {
                 let _timer = cgcx
                     .prof
-                    .generic_activity_with_arg("LLVM_module_codegen_emit_obj", &*module.name);
+                    .generic_activity_with_arg("GCC_module_codegen_emit_obj", &*module.name);
                 if env::var("CG_GCCJIT_DUMP_MODULE_NAMES").as_deref() == Ok("1") {
                     println!("Module {}", module.name);
                 }
@@ -60,11 +104,36 @@ pub(crate) unsafe fn codegen(cgcx: &CodegenContext<GccCodegenBackend>, _diag_han
                     context.set_debug_info(true);
                     context.dump_to_file(path, true);
                 }
-                context.compile_to_file(OutputKind::ObjectFile, obj_out.to_str().expect("path to str"));
+                if should_combine_object_files && fat_lto {
+                    context.add_command_line_option("-flto=auto");
+                    context.add_command_line_option("-flto-partition=one");
+
+                    context.add_driver_option("-Wl,-r");
+                    // NOTE: we need -nostdlib, otherwise, we get the following error:
+                    // /usr/bin/ld: cannot find -lgcc_s: No such file or directory
+                    context.add_driver_option("-nostdlib");
+                    // NOTE: without -fuse-linker-plugin, we get the following error:
+                    // lto1: internal compiler error: decompressed stream: Destination buffer is too small
+                    context.add_driver_option("-fuse-linker-plugin");
+
+                    // NOTE: this doesn't actually generate an executable. With the above flags, it combines the .o files together in another .o.
+                    context.compile_to_file(OutputKind::Executable, obj_out.to_str().expect("path to str"));
+                }
+                else {
+                    context.compile_to_file(OutputKind::ObjectFile, obj_out.to_str().expect("path to str"));
+                }
             }
 
             EmitObj::Bitcode => {
-                // TODO(antoyo)
+                debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out);
+                if let Err(err) = link_or_copy(&bc_out, &obj_out) {
+                    diag_handler.emit_err(CopyBitcode { err });
+                }
+
+                if !config.emit_bc {
+                    debug!("removing_bitcode {:?}", bc_out);
+                    ensure_removed(diag_handler, &bc_out);
+                }
             }
 
             EmitObj::None => {}
@@ -82,3 +151,18 @@ pub(crate) unsafe fn codegen(cgcx: &CodegenContext<GccCodegenBackend>, _diag_han
 pub(crate) fn link(_cgcx: &CodegenContext<GccCodegenBackend>, _diag_handler: &Handler, mut _modules: Vec<ModuleCodegen<GccContext>>) -> Result<ModuleCodegen<GccContext>, FatalError> {
     unimplemented!();
 }
+
+pub(crate) fn save_temp_bitcode(cgcx: &CodegenContext<GccCodegenBackend>, _module: &ModuleCodegen<GccContext>, _name: &str) {
+    if !cgcx.save_temps {
+        return;
+    }
+    unimplemented!();
+    /*unsafe {
+        let ext = format!("{}.bc", name);
+        let cgu = Some(&module.name[..]);
+        let path = cgcx.output_filenames.temp_path_ext(&ext, cgu);
+        let cstr = path_to_c_string(&path);
+        let llmod = module.module_llvm.llmod();
+        llvm::LLVMWriteBitcodeToFile(llmod, cstr.as_ptr());
+    }*/
+}
diff --git a/src/base.rs b/src/base.rs
index 9e614ca4ace0..61da38f4b0db 100644
--- a/src/base.rs
+++ b/src/base.rs
@@ -1,3 +1,4 @@
+use std::collections::HashSet;
 use std::env;
 use std::time::Instant;
 
@@ -18,6 +19,7 @@ use rustc_codegen_ssa::traits::DebugInfoMethods;
 use rustc_session::config::DebugInfo;
 use rustc_span::Symbol;
 
+use crate::{LockedTargetInfo, gcc_util};
 use crate::GccContext;
 use crate::builder::Builder;
 use crate::context::CodegenCx;
@@ -50,6 +52,7 @@ pub fn global_linkage_to_gcc(linkage: Linkage) -> GlobalKind {
 pub fn linkage_to_gcc(linkage: Linkage) -> FunctionType {
     match linkage {
         Linkage::External => FunctionType::Exported,
+        // TODO(antoyo): set the attribute externally_visible.
         Linkage::AvailableExternally => FunctionType::Extern,
         Linkage::LinkOnceAny => unimplemented!(),
         Linkage::LinkOnceODR => unimplemented!(),
@@ -63,7 +66,7 @@ pub fn linkage_to_gcc(linkage: Linkage) -> FunctionType {
     }
 }
 
-pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol, supports_128bit_integers: bool) -> (ModuleCodegen<GccContext>, u64) {
+pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol, target_info: LockedTargetInfo) -> (ModuleCodegen<GccContext>, u64) {
     let prof_timer = tcx.prof.generic_activity("codegen_module");
     let start_time = Instant::now();
 
@@ -71,7 +74,7 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol, supports_128bit_i
     let (module, _) = tcx.dep_graph.with_task(
         dep_node,
         tcx,
-        (cgu_name, supports_128bit_integers),
+        (cgu_name, target_info),
         module_codegen,
         Some(dep_graph::hash_result),
     );
@@ -82,38 +85,28 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol, supports_128bit_i
     // the time we needed for codegenning it.
     let cost = time_to_codegen.as_secs() * 1_000_000_000 + time_to_codegen.subsec_nanos() as u64;
 
-    fn module_codegen(tcx: TyCtxt<'_>, (cgu_name, supports_128bit_integers): (Symbol, bool)) -> ModuleCodegen<GccContext> {
+    fn module_codegen(tcx: TyCtxt<'_>, (cgu_name, target_info): (Symbol, LockedTargetInfo)) -> ModuleCodegen<GccContext> {
         let cgu = tcx.codegen_unit(cgu_name);
         // Instantiate monomorphizations without filling out definitions yet...
-        //let llvm_module = ModuleLlvm::new(tcx, &cgu_name.as_str());
         let context = Context::default();
 
         context.add_command_line_option("-fexceptions");
         context.add_driver_option("-fexceptions");
 
+        let disabled_features: HashSet<_> = tcx.sess.opts.cg.target_feature.split(',')
+            .filter(|feature| feature.starts_with('-'))
+            .map(|string| &string[1..])
+            .collect();
+
         // TODO(antoyo): only set on x86 platforms.
         context.add_command_line_option("-masm=intel");
-        // TODO(antoyo): only add the following cli argument if the feature is supported.
-        context.add_command_line_option("-msse2");
-        context.add_command_line_option("-mavx2");
-        // FIXME(antoyo): the following causes an illegal instruction on vmovdqu64 in std_example on my CPU.
-        // Only add if the CPU supports it.
-        context.add_command_line_option("-msha");
-        context.add_command_line_option("-mpclmul");
-        context.add_command_line_option("-mfma");
-        context.add_command_line_option("-mfma4");
-        context.add_command_line_option("-m64");
-        context.add_command_line_option("-mbmi");
-        context.add_command_line_option("-mgfni");
-        //context.add_command_line_option("-mavxvnni"); // The CI doesn't support this option.
-        context.add_command_line_option("-mf16c");
-        context.add_command_line_option("-maes");
-        context.add_command_line_option("-mxsavec");
-        context.add_command_line_option("-mbmi2");
-        context.add_command_line_option("-mrtm");
-        context.add_command_line_option("-mvaes");
-        context.add_command_line_option("-mvpclmulqdq");
-        context.add_command_line_option("-mavx");
+
+        if !disabled_features.contains("avx") {
+            // NOTE: we always enable AVX because the equivalent of llvm.x86.sse2.cmp.pd in GCC for
+            // SSE2 is multiple builtins, so we use the AVX __builtin_ia32_cmppd instead.
+            // FIXME(antoyo): use the proper builtins for llvm.x86.sse2.cmp.pd and similar.
+            context.add_command_line_option("-mavx");
+        }
 
         for arg in &tcx.sess.opts.cg.llvm_args {
             context.add_command_line_option(arg);
@@ -127,6 +120,16 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol, supports_128bit_i
         // NOTE: Rust relies on LLVM doing wrapping on overflow.
         context.add_command_line_option("-fwrapv");
 
+        if tcx.sess.opts.cg.relocation_model == Some(rustc_target::spec::RelocModel::Static) {
+            context.add_command_line_option("-mcmodel=kernel");
+            context.add_command_line_option("-fno-pie");
+        }
+
+        let target_cpu = gcc_util::target_cpu(tcx.sess);
+        if target_cpu != "generic" {
+            context.add_command_line_option(&format!("-march={}", target_cpu));
+        }
+
         if tcx.sess.opts.unstable_opts.function_sections.unwrap_or(tcx.sess.target.function_sections) {
             context.add_command_line_option("-ffunction-sections");
             context.add_command_line_option("-fdata-sections");
@@ -135,8 +138,14 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol, supports_128bit_i
         if env::var("CG_GCCJIT_DUMP_RTL").as_deref() == Ok("1") {
             context.add_command_line_option("-fdump-rtl-vregs");
         }
+        if env::var("CG_GCCJIT_DUMP_RTL_ALL").as_deref() == Ok("1") {
+            context.add_command_line_option("-fdump-rtl-all");
+        }
         if env::var("CG_GCCJIT_DUMP_TREE_ALL").as_deref() == Ok("1") {
-            context.add_command_line_option("-fdump-tree-all");
+            context.add_command_line_option("-fdump-tree-all-eh");
+        }
+        if env::var("CG_GCCJIT_DUMP_IPA_ALL").as_deref() == Ok("1") {
+            context.add_command_line_option("-fdump-ipa-all-eh");
         }
         if env::var("CG_GCCJIT_DUMP_CODE").as_deref() == Ok("1") {
             context.set_dump_code_on_compile(true);
@@ -152,11 +161,15 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol, supports_128bit_i
             context.set_keep_intermediates(true);
         }
 
+        if env::var("CG_GCCJIT_VERBOSE").as_deref() == Ok("1") {
+            context.add_driver_option("-v");
+        }
+
         // NOTE: The codegen generates unrechable blocks.
         context.set_allow_unreachable_blocks(true);
 
         {
-            let cx = CodegenCx::new(&context, cgu, tcx, supports_128bit_integers);
+            let cx = CodegenCx::new(&context, cgu, tcx, target_info.supports_128bit_int());
 
             let mono_items = cgu.items_in_deterministic_order(tcx);
             for &(mono_item, data) in &mono_items {
@@ -181,7 +194,9 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol, supports_128bit_i
         ModuleCodegen {
             name: cgu_name.to_string(),
             module_llvm: GccContext {
-                context
+                context,
+                should_combine_object_files: false,
+                temp_dir: None,
             },
             kind: ModuleKind::Regular,
         }
diff --git a/src/builder.rs b/src/builder.rs
index ecc293aee237..b78418089342 100644
--- a/src/builder.rs
+++ b/src/builder.rs
@@ -247,16 +247,9 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
     }
 
     fn check_store(&mut self, val: RValue<'gcc>, ptr: RValue<'gcc>) -> RValue<'gcc> {
-        let dest_ptr_ty = self.cx.val_ty(ptr).make_pointer(); // TODO(antoyo): make sure make_pointer() is okay here.
         let stored_ty = self.cx.val_ty(val);
         let stored_ptr_ty = self.cx.type_ptr_to(stored_ty);
-
-        if dest_ptr_ty == stored_ptr_ty {
-            ptr
-        }
-        else {
-            self.bitcast(ptr, stored_ptr_ty)
-        }
+        self.bitcast(ptr, stored_ptr_ty)
     }
 
     pub fn current_func(&self) -> Function<'gcc> {
@@ -500,7 +493,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
     }
 
     #[cfg(not(feature="master"))]
-    fn invoke(&mut self, typ: Type<'gcc>, fn_attrs: &CodegenFnAttrs, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, func: RValue<'gcc>, args: &[RValue<'gcc>], then: Block<'gcc>, catch: Block<'gcc>, _funclet: Option<&Funclet>) -> RValue<'gcc> {
+    fn invoke(&mut self, typ: Type<'gcc>, fn_attrs: Option<&CodegenFnAttrs>, fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, func: RValue<'gcc>, args: &[RValue<'gcc>], then: Block<'gcc>, catch: Block<'gcc>, _funclet: Option<&Funclet>) -> RValue<'gcc> {
         let call_site = self.call(typ, fn_attrs, None, func, args, None);
         let condition = self.context.new_rvalue_from_int(self.bool_type, 1);
         self.llbb().end_with_conditional(None, condition, then, catch);
@@ -663,7 +656,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
     }
 
     fn unchecked_sadd(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
-        a + b
+        self.gcc_add(a, b)
     }
 
     fn unchecked_uadd(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
@@ -671,7 +664,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
     }
 
     fn unchecked_ssub(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
-        a - b
+        self.gcc_sub(a, b)
     }
 
     fn unchecked_usub(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
@@ -680,11 +673,11 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
     }
 
     fn unchecked_smul(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
-        a * b
+        self.gcc_mul(a, b)
     }
 
     fn unchecked_umul(&mut self, a: RValue<'gcc>, b: RValue<'gcc>) -> RValue<'gcc> {
-        a * b
+        self.gcc_mul(a, b)
     }
 
     fn fadd_fast(&mut self, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> RValue<'gcc> {
@@ -916,7 +909,9 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
             .add_eval(None, self.context.new_call(None, atomic_store, &[ptr, value, ordering]));
     }
 
-    fn gep(&mut self, _typ: Type<'gcc>, ptr: RValue<'gcc>, indices: &[RValue<'gcc>]) -> RValue<'gcc> {
+    fn gep(&mut self, typ: Type<'gcc>, ptr: RValue<'gcc>, indices: &[RValue<'gcc>]) -> RValue<'gcc> {
+        // NOTE: due to opaque pointers now being used, we need to cast here.
+        let ptr = self.context.new_cast(None, ptr, typ.make_pointer());
         let ptr_type = ptr.get_type();
         let mut pointee_type = ptr.get_type();
         // NOTE: we cannot use array indexing here like in inbounds_gep because array indexing is
@@ -927,6 +922,12 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         // require dereferencing the pointer.
         for index in indices {
             pointee_type = pointee_type.get_pointee().expect("pointee type");
+            #[cfg(feature="master")]
+            let pointee_size = {
+                let size = self.cx.context.new_sizeof(pointee_type);
+                self.context.new_cast(None, size, index.get_type())
+            };
+            #[cfg(not(feature="master"))]
             let pointee_size = self.context.new_rvalue_from_int(index.get_type(), pointee_type.get_size() as i32);
             result = result + self.gcc_int_cast(*index * pointee_size, self.sizet_type);
         }
diff --git a/src/declare.rs b/src/declare.rs
index 493626c3cf5d..e673d0af4c7e 100644
--- a/src/declare.rs
+++ b/src/declare.rs
@@ -1,4 +1,6 @@
 use gccjit::{Function, FunctionType, GlobalKind, LValue, RValue, Type};
+#[cfg(feature="master")]
+use gccjit::{FnAttribute, ToRValue};
 use rustc_codegen_ssa::traits::BaseTypeMethods;
 use rustc_middle::ty::Ty;
 use rustc_span::Symbol;
@@ -114,6 +116,44 @@ fn declare_raw_fn<'gcc>(cx: &CodegenCx<'gcc, '_>, name: &str, _callconv: () /*ll
                 .collect();
             let func = cx.context.new_function(None, cx.linkage.get(), return_type, &params, mangle_name(name), variadic);
             cx.functions.borrow_mut().insert(name.to_string(), func);
+
+            #[cfg(feature="master")]
+            if name == "rust_eh_personality" {
+                // NOTE: GCC will sometimes change the personality function set on a function from
+                // rust_eh_personality to __gcc_personality_v0 as an optimization.
+                // As such, we need to create a weak alias from __gcc_personality_v0 to
+                // rust_eh_personality in order to avoid a linker error.
+                // This needs to be weak in order to still allow using the standard
+                // __gcc_personality_v0 when the linking to it.
+                // Since aliases don't work (maybe because of a bug in LTO partitioning?), we
+                // create a wrapper function that calls rust_eh_personality.
+
+                let params: Vec<_> = param_types.into_iter().enumerate()
+                    .map(|(index, param)| cx.context.new_parameter(None, *param, &format!("param{}", index))) // TODO(antoyo): set name.
+                    .collect();
+                let gcc_func = cx.context.new_function(None, FunctionType::Exported, return_type, &params, "__gcc_personality_v0", variadic);
+
+                // We need a normal extern function for the crates that access rust_eh_personality
+                // without defining it, otherwise we'll get a compiler error.
+                //
+                // For the crate defining it, that needs to be a weak alias instead.
+                gcc_func.add_attribute(FnAttribute::Weak);
+
+                let block = gcc_func.new_block("start");
+                let mut args = vec![];
+                for param in &params {
+                    args.push(param.to_rvalue());
+                }
+                let call = cx.context.new_call(None, func, &args);
+                if return_type == cx.type_void() {
+                    block.add_eval(None, call);
+                    block.end_with_void_return(None);
+                }
+                else {
+                    block.end_with_return(None, call);
+                }
+            }
+
             func
         };
 
diff --git a/src/errors.rs b/src/errors.rs
index 693367192b1f..4bf3b71f503d 100644
--- a/src/errors.rs
+++ b/src/errors.rs
@@ -1,8 +1,36 @@
-use rustc_errors::{DiagnosticArgValue, IntoDiagnosticArg};
-use rustc_macros::Diagnostic;
+use rustc_errors::{
+    DiagnosticArgValue, DiagnosticBuilder, ErrorGuaranteed, Handler, IntoDiagnostic, IntoDiagnosticArg,
+};
+use rustc_macros::{Diagnostic, Subdiagnostic};
 use rustc_span::Span;
 use std::borrow::Cow;
 
+use crate::fluent_generated as fluent;
+
+#[derive(Diagnostic)]
+#[diag(codegen_gcc_unknown_ctarget_feature_prefix)]
+#[note]
+pub(crate) struct UnknownCTargetFeaturePrefix<'a> {
+    pub feature: &'a str,
+}
+
+#[derive(Diagnostic)]
+#[diag(codegen_gcc_unknown_ctarget_feature)]
+#[note]
+pub(crate) struct UnknownCTargetFeature<'a> {
+    pub feature: &'a str,
+    #[subdiagnostic]
+    pub rust_feature: PossibleFeature<'a>,
+}
+
+#[derive(Subdiagnostic)]
+pub(crate) enum PossibleFeature<'a> {
+    #[help(codegen_gcc_possible_feature)]
+    Some { rust_feature: &'a str },
+    #[help(codegen_gcc_consider_filing_feature_request)]
+    None,
+}
+
 struct ExitCode(Option<i32>);
 
 impl IntoDiagnosticArg for ExitCode {
@@ -40,3 +68,58 @@ pub(crate) struct TiedTargetFeatures {
     pub span: Span,
     pub features: String,
 }
+
+#[derive(Diagnostic)]
+#[diag(codegen_gcc_copy_bitcode)]
+pub(crate) struct CopyBitcode {
+    pub err: std::io::Error,
+}
+
+#[derive(Diagnostic)]
+#[diag(codegen_gcc_dynamic_linking_with_lto)]
+#[note]
+pub(crate) struct DynamicLinkingWithLTO;
+
+#[derive(Diagnostic)]
+#[diag(codegen_gcc_load_bitcode)]
+pub(crate) struct LoadBitcode {
+    name: String,
+}
+
+#[derive(Diagnostic)]
+#[diag(codegen_gcc_lto_disallowed)]
+pub(crate) struct LtoDisallowed;
+
+#[derive(Diagnostic)]
+#[diag(codegen_gcc_lto_dylib)]
+pub(crate) struct LtoDylib;
+
+#[derive(Diagnostic)]
+#[diag(codegen_gcc_lto_bitcode_from_rlib)]
+pub(crate) struct LtoBitcodeFromRlib {
+    pub gcc_err: String,
+}
+
+pub(crate) struct TargetFeatureDisableOrEnable<'a> {
+    pub features: &'a [&'a str],
+    pub span: Option<Span>,
+    pub missing_features: Option<MissingFeatures>,
+}
+
+#[derive(Subdiagnostic)]
+#[help(codegen_gcc_missing_features)]
+pub(crate) struct MissingFeatures;
+
+impl IntoDiagnostic<'_, ErrorGuaranteed> for TargetFeatureDisableOrEnable<'_> {
+    fn into_diagnostic(self, sess: &'_ Handler) -> DiagnosticBuilder<'_, ErrorGuaranteed> {
+        let mut diag = sess.struct_err(fluent::codegen_gcc_target_feature_disable_or_enable);
+        if let Some(span) = self.span {
+            diag.set_span(span);
+        };
+        if let Some(missing_features) = self.missing_features {
+            diag.subdiagnostic(missing_features);
+        }
+        diag.set_arg("features", self.features.join(", "));
+        diag
+    }
+}
diff --git a/src/gcc_util.rs b/src/gcc_util.rs
new file mode 100644
index 000000000000..0514c9988e0f
--- /dev/null
+++ b/src/gcc_util.rs
@@ -0,0 +1,223 @@
+#[cfg(feature="master")]
+use gccjit::Context;
+use smallvec::{smallvec, SmallVec};
+
+use rustc_codegen_ssa::target_features::{
+    supported_target_features, tied_target_features, RUSTC_SPECIFIC_FEATURES,
+};
+use rustc_data_structures::fx::FxHashMap;
+use rustc_middle::bug;
+use rustc_session::Session;
+
+use crate::errors::{PossibleFeature, TargetFeatureDisableOrEnable, UnknownCTargetFeature, UnknownCTargetFeaturePrefix};
+
+/// The list of GCC features computed from CLI flags (`-Ctarget-cpu`, `-Ctarget-feature`,
+/// `--target` and similar).
+pub(crate) fn global_gcc_features(sess: &Session, diagnostics: bool) -> Vec<String> {
+    // Features that come earlier are overridden by conflicting features later in the string.
+    // Typically we'll want more explicit settings to override the implicit ones, so:
+    //
+    // * Features from -Ctarget-cpu=*; are overridden by [^1]
+    // * Features implied by --target; are overridden by
+    // * Features from -Ctarget-feature; are overridden by
+    // * function specific features.
+    //
+    // [^1]: target-cpu=native is handled here, other target-cpu values are handled implicitly
+    // through GCC march implementation.
+    //
+    // FIXME(nagisa): it isn't clear what's the best interaction between features implied by
+    // `-Ctarget-cpu` and `--target` are. On one hand, you'd expect CLI arguments to always
+    // override anything that's implicit, so e.g. when there's no `--target` flag, features implied
+    // the host target are overridden by `-Ctarget-cpu=*`. On the other hand, what about when both
+    // `--target` and `-Ctarget-cpu=*` are specified? Both then imply some target features and both
+    // flags are specified by the user on the CLI. It isn't as clear-cut which order of precedence
+    // should be taken in cases like these.
+    let mut features = vec![];
+
+    // Features implied by an implicit or explicit `--target`.
+    features.extend(
+        sess.target
+            .features
+            .split(',')
+            .filter(|v| !v.is_empty() && backend_feature_name(v).is_some())
+            .map(String::from),
+    );
+
+    // -Ctarget-features
+    let supported_features = supported_target_features(sess);
+    let mut featsmap = FxHashMap::default();
+    let feats = sess.opts.cg.target_feature
+        .split(',')
+        .filter_map(|s| {
+            let enable_disable = match s.chars().next() {
+                None => return None,
+                Some(c @ ('+' | '-')) => c,
+                Some(_) => {
+                    if diagnostics {
+                        sess.emit_warning(UnknownCTargetFeaturePrefix { feature: s });
+                    }
+                    return None;
+                }
+            };
+
+            let feature = backend_feature_name(s)?;
+            // Warn against use of GCC specific feature names on the CLI.
+            if diagnostics && !supported_features.iter().any(|&(v, _)| v == feature) {
+                let rust_feature = supported_features.iter().find_map(|&(rust_feature, _)| {
+                    let gcc_features = to_gcc_features(sess, rust_feature);
+                    if gcc_features.contains(&feature) && !gcc_features.contains(&rust_feature) {
+                        Some(rust_feature)
+                    } else {
+                        None
+                    }
+                });
+                let unknown_feature =
+                    if let Some(rust_feature) = rust_feature {
+                        UnknownCTargetFeature {
+                            feature,
+                            rust_feature: PossibleFeature::Some { rust_feature },
+                        }
+                    }
+                    else {
+                        UnknownCTargetFeature { feature, rust_feature: PossibleFeature::None }
+                    };
+                sess.emit_warning(unknown_feature);
+            }
+
+            if diagnostics {
+                // FIXME(nagisa): figure out how to not allocate a full hashset here.
+                featsmap.insert(feature, enable_disable == '+');
+            }
+
+            // rustc-specific features do not get passed down to GCC…
+            if RUSTC_SPECIFIC_FEATURES.contains(&feature) {
+                return None;
+            }
+            // ... otherwise though we run through `to_gcc_features` when
+            // passing requests down to GCC. This means that all in-language
+            // features also work on the command line instead of having two
+            // different names when the GCC name and the Rust name differ.
+            Some(to_gcc_features(sess, feature)
+                .iter()
+                .flat_map(|feat| to_gcc_features(sess, feat).into_iter())
+                .map(|feature| {
+                    if enable_disable == '-' {
+                        format!("-{}", feature)
+                    }
+                    else {
+                        feature.to_string()
+                    }
+                })
+                .collect::<Vec<_>>(),
+            )
+        })
+        .flatten();
+    features.extend(feats);
+
+    if diagnostics {
+        if let Some(f) = check_tied_features(sess, &featsmap) {
+            sess.emit_err(TargetFeatureDisableOrEnable {
+                features: f,
+                span: None,
+                missing_features: None,
+            });
+        }
+    }
+
+    features
+}
+
+/// Returns a feature name for the given `+feature` or `-feature` string.
+///
+/// Only allows features that are backend specific (i.e. not [`RUSTC_SPECIFIC_FEATURES`].)
+fn backend_feature_name(s: &str) -> Option<&str> {
+    // features must start with a `+` or `-`.
+    let feature = s.strip_prefix(&['+', '-'][..]).unwrap_or_else(|| {
+        bug!("target feature `{}` must begin with a `+` or `-`", s);
+    });
+    // Rustc-specific feature requests like `+crt-static` or `-crt-static`
+    // are not passed down to GCC.
+    if RUSTC_SPECIFIC_FEATURES.contains(&feature) {
+        return None;
+    }
+    Some(feature)
+}
+
+// To find a list of GCC's names, check https://gcc.gnu.org/onlinedocs/gcc/Function-Attributes.html
+pub fn to_gcc_features<'a>(sess: &Session, s: &'a str) -> SmallVec<[&'a str; 2]> {
+    let arch = if sess.target.arch == "x86_64" { "x86" } else { &*sess.target.arch };
+    match (arch, s) {
+        ("x86", "sse4.2") => smallvec!["sse4.2", "crc32"],
+        ("x86", "pclmulqdq") => smallvec!["pclmul"],
+        ("x86", "rdrand") => smallvec!["rdrnd"],
+        ("x86", "bmi1") => smallvec!["bmi"],
+        ("x86", "cmpxchg16b") => smallvec!["cx16"],
+        ("x86", "avx512vaes") => smallvec!["vaes"],
+        ("x86", "avx512gfni") => smallvec!["gfni"],
+        ("x86", "avx512vpclmulqdq") => smallvec!["vpclmulqdq"],
+        // NOTE: seems like GCC requires 'avx512bw' for 'avx512vbmi2'.
+        ("x86", "avx512vbmi2") => smallvec!["avx512vbmi2", "avx512bw"],
+        // NOTE: seems like GCC requires 'avx512bw' for 'avx512bitalg'.
+        ("x86", "avx512bitalg") => smallvec!["avx512bitalg", "avx512bw"],
+        ("aarch64", "rcpc2") => smallvec!["rcpc-immo"],
+        ("aarch64", "dpb") => smallvec!["ccpp"],
+        ("aarch64", "dpb2") => smallvec!["ccdp"],
+        ("aarch64", "frintts") => smallvec!["fptoint"],
+        ("aarch64", "fcma") => smallvec!["complxnum"],
+        ("aarch64", "pmuv3") => smallvec!["perfmon"],
+        ("aarch64", "paca") => smallvec!["pauth"],
+        ("aarch64", "pacg") => smallvec!["pauth"],
+        // Rust ties fp and neon together. In GCC neon implicitly enables fp,
+        // but we manually enable neon when a feature only implicitly enables fp
+        ("aarch64", "f32mm") => smallvec!["f32mm", "neon"],
+        ("aarch64", "f64mm") => smallvec!["f64mm", "neon"],
+        ("aarch64", "fhm") => smallvec!["fp16fml", "neon"],
+        ("aarch64", "fp16") => smallvec!["fullfp16", "neon"],
+        ("aarch64", "jsconv") => smallvec!["jsconv", "neon"],
+        ("aarch64", "sve") => smallvec!["sve", "neon"],
+        ("aarch64", "sve2") => smallvec!["sve2", "neon"],
+        ("aarch64", "sve2-aes") => smallvec!["sve2-aes", "neon"],
+        ("aarch64", "sve2-sm4") => smallvec!["sve2-sm4", "neon"],
+        ("aarch64", "sve2-sha3") => smallvec!["sve2-sha3", "neon"],
+        ("aarch64", "sve2-bitperm") => smallvec!["sve2-bitperm", "neon"],
+        (_, s) => smallvec![s],
+    }
+}
+
+// Given a map from target_features to whether they are enabled or disabled,
+// ensure only valid combinations are allowed.
+pub fn check_tied_features(sess: &Session, features: &FxHashMap<&str, bool>) -> Option<&'static [&'static str]> {
+    for tied in tied_target_features(sess) {
+        // Tied features must be set to the same value, or not set at all
+        let mut tied_iter = tied.iter();
+        let enabled = features.get(tied_iter.next().unwrap());
+        if tied_iter.any(|feature| enabled != features.get(feature)) {
+            return Some(tied);
+        }
+    }
+    None
+}
+
+fn handle_native(name: &str) -> &str {
+    if name != "native" {
+        return name;
+    }
+
+    #[cfg(feature="master")]
+    {
+        // Get the native arch.
+        let context = Context::default();
+        context.get_target_info().arch().unwrap()
+            .to_str()
+            .unwrap()
+    }
+    #[cfg(not(feature="master"))]
+    unimplemented!();
+}
+
+pub fn target_cpu(sess: &Session) -> &str {
+    match sess.opts.cg.target_cpu {
+        Some(ref name) => handle_native(name),
+        None => handle_native(sess.target.cpu.as_ref()),
+    }
+}
diff --git a/src/int.rs b/src/int.rs
index 0cf1204791d3..58e0dd56f38d 100644
--- a/src/int.rs
+++ b/src/int.rs
@@ -36,7 +36,6 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
             self.cx.context.new_unary_op(None, operation, typ, a)
         }
         else {
-            // TODO(antoyo): use __negdi2 and __negti2 instead?
             let element_type = typ.dyncast_array().expect("element type");
             let values = [
                 self.cx.context.new_unary_op(None, UnaryOp::BitwiseNegate, element_type, self.low(a)),
@@ -52,9 +51,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
             self.cx.context.new_unary_op(None, UnaryOp::Minus, a.get_type(), a)
         }
         else {
-            let param_a = self.context.new_parameter(None, a_type, "a");
-            let func = self.context.new_function(None, FunctionType::Extern, a_type, &[param_a], "__negti2", false);
-            self.context.new_call(None, func, &[a])
+            self.gcc_add(self.gcc_not(a), self.gcc_int(a_type, 1))
         }
     }
 
@@ -353,23 +350,63 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
         (res.dereference(None).to_rvalue(), overflow)
     }
 
-    pub fn gcc_icmp(&self, op: IntPredicate, mut lhs: RValue<'gcc>, mut rhs: RValue<'gcc>) -> RValue<'gcc> {
+    pub fn gcc_icmp(&mut self, op: IntPredicate, mut lhs: RValue<'gcc>, mut rhs: RValue<'gcc>) -> RValue<'gcc> {
         let a_type = lhs.get_type();
         let b_type = rhs.get_type();
         if self.is_non_native_int_type(a_type) || self.is_non_native_int_type(b_type) {
-            let signed = a_type.is_compatible_with(self.i128_type);
-            let sign =
-                if signed {
-                    ""
-                }
-                else {
-                    "u"
-                };
-            let func_name = format!("__{}cmpti2", sign);
-            let param_a = self.context.new_parameter(None, a_type, "a");
-            let param_b = self.context.new_parameter(None, b_type, "b");
-            let func = self.context.new_function(None, FunctionType::Extern, self.int_type, &[param_a, param_b], func_name, false);
-            let cmp = self.context.new_call(None, func, &[lhs, rhs]);
+            // This algorithm is based on compiler-rt's __cmpti2:
+            // https://github.com/llvm-mirror/compiler-rt/blob/f0745e8476f069296a7c71accedd061dce4cdf79/lib/builtins/cmpti2.c#L21
+            let result = self.current_func().new_local(None, self.int_type, "icmp_result");
+            let block1 = self.current_func().new_block("block1");
+            let block2 = self.current_func().new_block("block2");
+            let block3 = self.current_func().new_block("block3");
+            let block4 = self.current_func().new_block("block4");
+            let block5 = self.current_func().new_block("block5");
+            let block6 = self.current_func().new_block("block6");
+            let block7 = self.current_func().new_block("block7");
+            let block8 = self.current_func().new_block("block8");
+            let after = self.current_func().new_block("after");
+
+            let native_int_type = a_type.dyncast_array().expect("get element type");
+            // NOTE: cast low to its unsigned type in order to perform a comparison correctly (e.g.
+            // the sign is only on high).
+            let unsigned_type = native_int_type.to_unsigned(&self.cx);
+
+            let lhs_low = self.context.new_cast(None, self.low(lhs), unsigned_type);
+            let rhs_low = self.context.new_cast(None, self.low(rhs), unsigned_type);
+
+            let condition = self.context.new_comparison(None, ComparisonOp::LessThan, self.high(lhs), self.high(rhs));
+            self.llbb().end_with_conditional(None, condition, block1, block2);
+
+            block1.add_assignment(None, result, self.context.new_rvalue_zero(self.int_type));
+            block1.end_with_jump(None, after);
+
+            let condition = self.context.new_comparison(None, ComparisonOp::GreaterThan, self.high(lhs), self.high(rhs));
+            block2.end_with_conditional(None, condition, block3, block4);
+
+            block3.add_assignment(None, result, self.context.new_rvalue_from_int(self.int_type, 2));
+            block3.end_with_jump(None, after);
+
+            let condition = self.context.new_comparison(None, ComparisonOp::LessThan, lhs_low, rhs_low);
+            block4.end_with_conditional(None, condition, block5, block6);
+
+            block5.add_assignment(None, result, self.context.new_rvalue_zero(self.int_type));
+            block5.end_with_jump(None, after);
+
+            let condition = self.context.new_comparison(None, ComparisonOp::GreaterThan, lhs_low, rhs_low);
+            block6.end_with_conditional(None, condition, block7, block8);
+
+            block7.add_assignment(None, result, self.context.new_rvalue_from_int(self.int_type, 2));
+            block7.end_with_jump(None, after);
+
+            block8.add_assignment(None, result, self.context.new_rvalue_one(self.int_type));
+            block8.end_with_jump(None, after);
+
+            // NOTE: since jumps were added in a place rustc does not expect, the current block in the
+            // state need to be updated.
+            self.switch_to_block(after);
+
+            let cmp = result.to_rvalue();
             let (op, limit) =
                 match op {
                     IntPredicate::IntEQ => {
@@ -546,7 +583,12 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
     }
 
     pub fn gcc_uint(&self, typ: Type<'gcc>, int: u64) -> RValue<'gcc> {
-        if self.is_native_int_type_or_bool(typ) {
+        if typ.is_u128(self) {
+            // FIXME(antoyo): libgccjit cannot create 128-bit values yet.
+            let num = self.context.new_rvalue_from_long(self.u64_type, int as i64);
+            self.gcc_int_cast(num, typ)
+        }
+        else if self.is_native_int_type_or_bool(typ) {
             self.context.new_rvalue_from_long(typ, u64::try_from(int).expect("u64::try_from") as i64)
         }
         else {
@@ -572,6 +614,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
             }
         }
         else if typ.is_i128(self) {
+            // FIXME(antoyo): libgccjit cannot create 128-bit values yet.
             let num = self.context.new_rvalue_from_long(self.u64_type, num as u64 as i64);
             self.gcc_int_cast(num, typ)
         }
diff --git a/src/intrinsic/archs.rs b/src/intrinsic/archs.rs
index 438eab78943a..e01299d32fda 100644
--- a/src/intrinsic/archs.rs
+++ b/src/intrinsic/archs.rs
@@ -2254,6 +2254,42 @@ match name {
     "llvm.hexagon.prefetch" => "__builtin_HEXAGON_prefetch",
     "llvm.hexagon.vmemcpy" => "__builtin_hexagon_vmemcpy",
     "llvm.hexagon.vmemset" => "__builtin_hexagon_vmemset",
+    // loongarch
+    "llvm.loongarch.asrtgt.d" => "__builtin_loongarch_asrtgt_d",
+    "llvm.loongarch.asrtle.d" => "__builtin_loongarch_asrtle_d",
+    "llvm.loongarch.break" => "__builtin_loongarch_break",
+    "llvm.loongarch.cacop.d" => "__builtin_loongarch_cacop_d",
+    "llvm.loongarch.cacop.w" => "__builtin_loongarch_cacop_w",
+    "llvm.loongarch.cpucfg" => "__builtin_loongarch_cpucfg",
+    "llvm.loongarch.crc.w.b.w" => "__builtin_loongarch_crc_w_b_w",
+    "llvm.loongarch.crc.w.d.w" => "__builtin_loongarch_crc_w_d_w",
+    "llvm.loongarch.crc.w.h.w" => "__builtin_loongarch_crc_w_h_w",
+    "llvm.loongarch.crc.w.w.w" => "__builtin_loongarch_crc_w_w_w",
+    "llvm.loongarch.crcc.w.b.w" => "__builtin_loongarch_crcc_w_b_w",
+    "llvm.loongarch.crcc.w.d.w" => "__builtin_loongarch_crcc_w_d_w",
+    "llvm.loongarch.crcc.w.h.w" => "__builtin_loongarch_crcc_w_h_w",
+    "llvm.loongarch.crcc.w.w.w" => "__builtin_loongarch_crcc_w_w_w",
+    "llvm.loongarch.csrrd.d" => "__builtin_loongarch_csrrd_d",
+    "llvm.loongarch.csrrd.w" => "__builtin_loongarch_csrrd_w",
+    "llvm.loongarch.csrwr.d" => "__builtin_loongarch_csrwr_d",
+    "llvm.loongarch.csrwr.w" => "__builtin_loongarch_csrwr_w",
+    "llvm.loongarch.csrxchg.d" => "__builtin_loongarch_csrxchg_d",
+    "llvm.loongarch.csrxchg.w" => "__builtin_loongarch_csrxchg_w",
+    "llvm.loongarch.dbar" => "__builtin_loongarch_dbar",
+    "llvm.loongarch.ibar" => "__builtin_loongarch_ibar",
+    "llvm.loongarch.iocsrrd.b" => "__builtin_loongarch_iocsrrd_b",
+    "llvm.loongarch.iocsrrd.d" => "__builtin_loongarch_iocsrrd_d",
+    "llvm.loongarch.iocsrrd.h" => "__builtin_loongarch_iocsrrd_h",
+    "llvm.loongarch.iocsrrd.w" => "__builtin_loongarch_iocsrrd_w",
+    "llvm.loongarch.iocsrwr.b" => "__builtin_loongarch_iocsrwr_b",
+    "llvm.loongarch.iocsrwr.d" => "__builtin_loongarch_iocsrwr_d",
+    "llvm.loongarch.iocsrwr.h" => "__builtin_loongarch_iocsrwr_h",
+    "llvm.loongarch.iocsrwr.w" => "__builtin_loongarch_iocsrwr_w",
+    "llvm.loongarch.lddir.d" => "__builtin_loongarch_lddir_d",
+    "llvm.loongarch.ldpte.d" => "__builtin_loongarch_ldpte_d",
+    "llvm.loongarch.movfcsr2gr" => "__builtin_loongarch_movfcsr2gr",
+    "llvm.loongarch.movgr2fcsr" => "__builtin_loongarch_movgr2fcsr",
+    "llvm.loongarch.syscall" => "__builtin_loongarch_syscall",
     // mips
     "llvm.mips.absq.s.ph" => "__builtin_mips_absq_s_ph",
     "llvm.mips.absq.s.qb" => "__builtin_mips_absq_s_qb",
@@ -2954,6 +2990,8 @@ match name {
     "llvm.nvvm.barrier0.and" => "__nvvm_bar0_and",
     "llvm.nvvm.barrier0.or" => "__nvvm_bar0_or",
     "llvm.nvvm.barrier0.popc" => "__nvvm_bar0_popc",
+    "llvm.nvvm.bf2h.rn" => "__nvvm_bf2h_rn",
+    "llvm.nvvm.bf2h.rn.ftz" => "__nvvm_bf2h_rn_ftz",
     "llvm.nvvm.bitcast.d2ll" => "__nvvm_bitcast_d2ll",
     "llvm.nvvm.bitcast.f2i" => "__nvvm_bitcast_f2i",
     "llvm.nvvm.bitcast.i2f" => "__nvvm_bitcast_i2f",
@@ -3016,8 +3054,6 @@ match name {
     "llvm.nvvm.div.rz.ftz.f" => "__nvvm_div_rz_ftz_f",
     "llvm.nvvm.ex2.approx.d" => "__nvvm_ex2_approx_d",
     "llvm.nvvm.ex2.approx.f" => "__nvvm_ex2_approx_f",
-    "llvm.nvvm.ex2.approx.f16" => "__nvvm_ex2_approx_f16",
-    "llvm.nvvm.ex2.approx.f16x2" => "__nvvm_ex2_approx_f16x2",
     "llvm.nvvm.ex2.approx.ftz.f" => "__nvvm_ex2_approx_ftz_f",
     "llvm.nvvm.f2bf16.rn" => "__nvvm_f2bf16_rn",
     "llvm.nvvm.f2bf16.rn.relu" => "__nvvm_f2bf16_rn_relu",
@@ -3079,11 +3115,17 @@ match name {
     "llvm.nvvm.fma.rn.bf16x2" => "__nvvm_fma_rn_bf16x2",
     "llvm.nvvm.fma.rn.d" => "__nvvm_fma_rn_d",
     "llvm.nvvm.fma.rn.f" => "__nvvm_fma_rn_f",
-    "llvm.nvvm.fma.rn.f16" => "__nvvm_fma_rn_f16",
-    "llvm.nvvm.fma.rn.f16x2" => "__nvvm_fma_rn_f16x2",
+    "llvm.nvvm.fma.rn.ftz.bf16" => "__nvvm_fma_rn_ftz_bf16",
+    "llvm.nvvm.fma.rn.ftz.bf16x2" => "__nvvm_fma_rn_ftz_bf16x2",
     "llvm.nvvm.fma.rn.ftz.f" => "__nvvm_fma_rn_ftz_f",
+    "llvm.nvvm.fma.rn.ftz.relu.bf16" => "__nvvm_fma_rn_ftz_relu_bf16",
+    "llvm.nvvm.fma.rn.ftz.relu.bf16x2" => "__nvvm_fma_rn_ftz_relu_bf16x2",
+    "llvm.nvvm.fma.rn.ftz.sat.bf16" => "__nvvm_fma_rn_ftz_sat_bf16",
+    "llvm.nvvm.fma.rn.ftz.sat.bf16x2" => "__nvvm_fma_rn_ftz_sat_bf16x2",
     "llvm.nvvm.fma.rn.relu.bf16" => "__nvvm_fma_rn_relu_bf16",
     "llvm.nvvm.fma.rn.relu.bf16x2" => "__nvvm_fma_rn_relu_bf16x2",
+    "llvm.nvvm.fma.rn.sat.bf16" => "__nvvm_fma_rn_sat_bf16",
+    "llvm.nvvm.fma.rn.sat.bf16x2" => "__nvvm_fma_rn_sat_bf16x2",
     "llvm.nvvm.fma.rp.d" => "__nvvm_fma_rp_d",
     "llvm.nvvm.fma.rp.f" => "__nvvm_fma_rp_f",
     "llvm.nvvm.fma.rp.ftz.f" => "__nvvm_fma_rp_ftz_f",
@@ -3094,11 +3136,17 @@ match name {
     "llvm.nvvm.fmax.bf16x2" => "__nvvm_fmax_bf16x2",
     "llvm.nvvm.fmax.d" => "__nvvm_fmax_d",
     "llvm.nvvm.fmax.f" => "__nvvm_fmax_f",
-    "llvm.nvvm.fmax.f16" => "__nvvm_fmax_f16",
-    "llvm.nvvm.fmax.f16x2" => "__nvvm_fmax_f16x2",
+    "llvm.nvvm.fmax.ftz.bf16" => "__nvvm_fmax_ftz_bf16",
+    "llvm.nvvm.fmax.ftz.bf16x2" => "__nvvm_fmax_ftz_bf16x2",
     "llvm.nvvm.fmax.ftz.f" => "__nvvm_fmax_ftz_f",
+    "llvm.nvvm.fmax.ftz.nan.bf16" => "__nvvm_fmax_ftz_nan_bf16",
+    "llvm.nvvm.fmax.ftz.nan.bf16x2" => "__nvvm_fmax_ftz_nan_bf16x2",
     "llvm.nvvm.fmax.ftz.nan.f" => "__nvvm_fmax_ftz_nan_f",
+    "llvm.nvvm.fmax.ftz.nan.xorsign.abs.bf16" => "__nvvm_fmax_ftz_nan_xorsign_abs_bf16",
+    "llvm.nvvm.fmax.ftz.nan.xorsign.abs.bf16x2" => "__nvvm_fmax_ftz_nan_xorsign_abs_bf16x2",
     "llvm.nvvm.fmax.ftz.nan.xorsign.abs.f" => "__nvvm_fmax_ftz_nan_xorsign_abs_f",
+    "llvm.nvvm.fmax.ftz.xorsign.abs.bf16" => "__nvvm_fmax_ftz_xorsign_abs_bf16",
+    "llvm.nvvm.fmax.ftz.xorsign.abs.bf16x2" => "__nvvm_fmax_ftz_xorsign_abs_bf16x2",
     "llvm.nvvm.fmax.ftz.xorsign.abs.f" => "__nvvm_fmax_ftz_xorsign_abs_f",
     "llvm.nvvm.fmax.nan.bf16" => "__nvvm_fmax_nan_bf16",
     "llvm.nvvm.fmax.nan.bf16x2" => "__nvvm_fmax_nan_bf16x2",
@@ -3113,11 +3161,17 @@ match name {
     "llvm.nvvm.fmin.bf16x2" => "__nvvm_fmin_bf16x2",
     "llvm.nvvm.fmin.d" => "__nvvm_fmin_d",
     "llvm.nvvm.fmin.f" => "__nvvm_fmin_f",
-    "llvm.nvvm.fmin.f16" => "__nvvm_fmin_f16",
-    "llvm.nvvm.fmin.f16x2" => "__nvvm_fmin_f16x2",
+    "llvm.nvvm.fmin.ftz.bf16" => "__nvvm_fmin_ftz_bf16",
+    "llvm.nvvm.fmin.ftz.bf16x2" => "__nvvm_fmin_ftz_bf16x2",
     "llvm.nvvm.fmin.ftz.f" => "__nvvm_fmin_ftz_f",
+    "llvm.nvvm.fmin.ftz.nan.bf16" => "__nvvm_fmin_ftz_nan_bf16",
+    "llvm.nvvm.fmin.ftz.nan.bf16x2" => "__nvvm_fmin_ftz_nan_bf16x2",
     "llvm.nvvm.fmin.ftz.nan.f" => "__nvvm_fmin_ftz_nan_f",
+    "llvm.nvvm.fmin.ftz.nan.xorsign.abs.bf16" => "__nvvm_fmin_ftz_nan_xorsign_abs_bf16",
+    "llvm.nvvm.fmin.ftz.nan.xorsign.abs.bf16x2" => "__nvvm_fmin_ftz_nan_xorsign_abs_bf16x2",
     "llvm.nvvm.fmin.ftz.nan.xorsign.abs.f" => "__nvvm_fmin_ftz_nan_xorsign_abs_f",
+    "llvm.nvvm.fmin.ftz.xorsign.abs.bf16" => "__nvvm_fmin_ftz_xorsign_abs_bf16",
+    "llvm.nvvm.fmin.ftz.xorsign.abs.bf16x2" => "__nvvm_fmin_ftz_xorsign_abs_bf16x2",
     "llvm.nvvm.fmin.ftz.xorsign.abs.f" => "__nvvm_fmin_ftz_xorsign_abs_f",
     "llvm.nvvm.fmin.nan.bf16" => "__nvvm_fmin_nan_bf16",
     "llvm.nvvm.fmin.nan.bf16x2" => "__nvvm_fmin_nan_bf16x2",
@@ -4213,6 +4267,28 @@ match name {
     "llvm.r600.read.tgid.x" => "__builtin_r600_read_tgid_x",
     "llvm.r600.read.tgid.y" => "__builtin_r600_read_tgid_y",
     "llvm.r600.read.tgid.z" => "__builtin_r600_read_tgid_z",
+    // riscv
+    "llvm.riscv.aes32dsi" => "__builtin_riscv_aes32dsi",
+    "llvm.riscv.aes32dsmi" => "__builtin_riscv_aes32dsmi",
+    "llvm.riscv.aes32esi" => "__builtin_riscv_aes32esi",
+    "llvm.riscv.aes32esmi" => "__builtin_riscv_aes32esmi",
+    "llvm.riscv.aes64ds" => "__builtin_riscv_aes64ds",
+    "llvm.riscv.aes64dsm" => "__builtin_riscv_aes64dsm",
+    "llvm.riscv.aes64es" => "__builtin_riscv_aes64es",
+    "llvm.riscv.aes64esm" => "__builtin_riscv_aes64esm",
+    "llvm.riscv.aes64im" => "__builtin_riscv_aes64im",
+    "llvm.riscv.aes64ks1i" => "__builtin_riscv_aes64ks1i",
+    "llvm.riscv.aes64ks2" => "__builtin_riscv_aes64ks2",
+    "llvm.riscv.sha512sig0" => "__builtin_riscv_sha512sig0",
+    "llvm.riscv.sha512sig0h" => "__builtin_riscv_sha512sig0h",
+    "llvm.riscv.sha512sig0l" => "__builtin_riscv_sha512sig0l",
+    "llvm.riscv.sha512sig1" => "__builtin_riscv_sha512sig1",
+    "llvm.riscv.sha512sig1h" => "__builtin_riscv_sha512sig1h",
+    "llvm.riscv.sha512sig1l" => "__builtin_riscv_sha512sig1l",
+    "llvm.riscv.sha512sum0" => "__builtin_riscv_sha512sum0",
+    "llvm.riscv.sha512sum0r" => "__builtin_riscv_sha512sum0r",
+    "llvm.riscv.sha512sum1" => "__builtin_riscv_sha512sum1",
+    "llvm.riscv.sha512sum1r" => "__builtin_riscv_sha512sum1r",
     // s390
     "llvm.s390.efpc" => "__builtin_s390_efpc",
     "llvm.s390.etnd" => "__builtin_tx_nesting_depth",
@@ -5912,6 +5988,18 @@ match name {
     "llvm.x86.avx2.vpdpbuud.256" => "__builtin_ia32_vpdpbuud256",
     "llvm.x86.avx2.vpdpbuuds.128" => "__builtin_ia32_vpdpbuuds128",
     "llvm.x86.avx2.vpdpbuuds.256" => "__builtin_ia32_vpdpbuuds256",
+    "llvm.x86.avx2.vpdpwsud.128" => "__builtin_ia32_vpdpwsud128",
+    "llvm.x86.avx2.vpdpwsud.256" => "__builtin_ia32_vpdpwsud256",
+    "llvm.x86.avx2.vpdpwsuds.128" => "__builtin_ia32_vpdpwsuds128",
+    "llvm.x86.avx2.vpdpwsuds.256" => "__builtin_ia32_vpdpwsuds256",
+    "llvm.x86.avx2.vpdpwusd.128" => "__builtin_ia32_vpdpwusd128",
+    "llvm.x86.avx2.vpdpwusd.256" => "__builtin_ia32_vpdpwusd256",
+    "llvm.x86.avx2.vpdpwusds.128" => "__builtin_ia32_vpdpwusds128",
+    "llvm.x86.avx2.vpdpwusds.256" => "__builtin_ia32_vpdpwusds256",
+    "llvm.x86.avx2.vpdpwuud.128" => "__builtin_ia32_vpdpwuud128",
+    "llvm.x86.avx2.vpdpwuud.256" => "__builtin_ia32_vpdpwuud256",
+    "llvm.x86.avx2.vpdpwuuds.128" => "__builtin_ia32_vpdpwuuds128",
+    "llvm.x86.avx2.vpdpwuuds.256" => "__builtin_ia32_vpdpwuuds256",
     "llvm.x86.avx2.vperm2i128" => "__builtin_ia32_permti256",
     "llvm.x86.avx512.add.pd.512" => "__builtin_ia32_addpd512",
     "llvm.x86.avx512.add.ps.512" => "__builtin_ia32_addps512",
@@ -7909,6 +7997,16 @@ match name {
     "llvm.x86.vgf2p8mulb.128" => "__builtin_ia32_vgf2p8mulb_v16qi",
     "llvm.x86.vgf2p8mulb.256" => "__builtin_ia32_vgf2p8mulb_v32qi",
     "llvm.x86.vgf2p8mulb.512" => "__builtin_ia32_vgf2p8mulb_v64qi",
+    "llvm.x86.vsha512msg1" => "__builtin_ia32_vsha512msg1",
+    "llvm.x86.vsha512msg2" => "__builtin_ia32_vsha512msg2",
+    "llvm.x86.vsha512rnds2" => "__builtin_ia32_vsha512rnds2",
+    "llvm.x86.vsm3msg1" => "__builtin_ia32_vsm3msg1",
+    "llvm.x86.vsm3msg2" => "__builtin_ia32_vsm3msg2",
+    "llvm.x86.vsm3rnds2" => "__builtin_ia32_vsm3rnds2",
+    "llvm.x86.vsm4key4128" => "__builtin_ia32_vsm4key4128",
+    "llvm.x86.vsm4key4256" => "__builtin_ia32_vsm4key4256",
+    "llvm.x86.vsm4rnds4128" => "__builtin_ia32_vsm4rnds4128",
+    "llvm.x86.vsm4rnds4256" => "__builtin_ia32_vsm4rnds4256",
     "llvm.x86.wbinvd" => "__builtin_ia32_wbinvd",
     "llvm.x86.wbnoinvd" => "__builtin_ia32_wbnoinvd",
     "llvm.x86.wrfsbase.32" => "__builtin_ia32_wrfsbase32",
diff --git a/src/intrinsic/llvm.rs b/src/intrinsic/llvm.rs
index f28348380d7b..5996623bdc58 100644
--- a/src/intrinsic/llvm.rs
+++ b/src/intrinsic/llvm.rs
@@ -236,11 +236,17 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(builder: &Builder<'a, 'gcc
                 let arg2 = builder.context.new_cast(None, arg2, arg2_type);
                 args = vec![new_args[0], arg2].into();
             },
+            // These builtins are sent one more argument than needed.
             "__builtin_prefetch" => {
                 let mut new_args = args.to_vec();
                 new_args.pop();
                 args = new_args.into();
             },
+            // The GCC version returns one value of the tuple through a pointer.
+            "__builtin_ia32_rdrand64_step" => {
+                let arg = builder.current_func().new_local(None, builder.ulonglong_type, "return_rdrand_arg");
+                args = vec![arg.get_address(None)].into();
+            },
             _ => (),
         }
     }
@@ -361,6 +367,19 @@ pub fn adjust_intrinsic_return_value<'a, 'gcc, 'tcx>(builder: &Builder<'a, 'gcc,
             // builtin twice, we overwrite the return value with a dummy value.
             return_value = builder.context.new_rvalue_zero(builder.int_type);
         },
+        "__builtin_ia32_rdrand64_step" => {
+            let random_number = args[0].dereference(None).to_rvalue();
+            let success_variable = builder.current_func().new_local(None, return_value.get_type(), "success");
+            builder.llbb().add_assignment(None, success_variable, return_value);
+
+            let field1 = builder.context.new_field(None, random_number.get_type(), "random_number");
+            let field2 = builder.context.new_field(None, return_value.get_type(), "success");
+            let struct_type = builder.context.new_struct_type(None, "rdrand_result", &[field1, field2]);
+            return_value = builder.context.new_struct_constructor(None, struct_type.as_type(), None, &[
+                random_number,
+                success_variable.to_rvalue(),
+            ]);
+        },
         _ => (),
     }
 
@@ -613,6 +632,7 @@ pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function
         "llvm.fshr.v8i16" => "__builtin_ia32_vpshrdv_v8hi",
         "llvm.x86.fma.vfmadd.sd" => "__builtin_ia32_vfmaddsd3",
         "llvm.x86.fma.vfmadd.ss" => "__builtin_ia32_vfmaddss3",
+        "llvm.x86.rdrand.64" => "__builtin_ia32_rdrand64_step",
 
         // The above doc points to unknown builtins for the following, so override them:
         "llvm.x86.avx2.gather.d.d" => "__builtin_ia32_gathersiv4si",
diff --git a/src/intrinsic/mod.rs b/src/intrinsic/mod.rs
index 68a087a1d7f7..9caed459a292 100644
--- a/src/intrinsic/mod.rs
+++ b/src/intrinsic/mod.rs
@@ -10,9 +10,9 @@ use rustc_codegen_ssa::base::wants_msvc_seh;
 use rustc_codegen_ssa::common::IntPredicate;
 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
 use rustc_codegen_ssa::mir::place::PlaceRef;
-use rustc_codegen_ssa::traits::{ArgAbiMethods, BaseTypeMethods, BuilderMethods, ConstMethods, IntrinsicCallMethods};
+use rustc_codegen_ssa::traits::{ArgAbiMethods, BuilderMethods, ConstMethods, IntrinsicCallMethods};
 #[cfg(feature="master")]
-use rustc_codegen_ssa::traits::MiscMethods;
+use rustc_codegen_ssa::traits::{BaseTypeMethods, MiscMethods};
 use rustc_codegen_ssa::errors::InvalidMonomorphization;
 use rustc_middle::bug;
 use rustc_middle::ty::{self, Instance, Ty};
diff --git a/src/lib.rs b/src/lib.rs
index ce7e31682f10..fe2339305604 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -2,6 +2,12 @@
  * TODO(antoyo): implement equality in libgccjit based on https://zpz.github.io/blog/overloading-equality-operator-in-cpp-class-hierarchy/ (for type equality?)
  * TODO(antoyo): support #[inline] attributes.
  * TODO(antoyo): support LTO (gcc's equivalent to Full LTO is -flto -flto-partition=one — https://documentation.suse.com/sbp/all/html/SBP-GCC-10/index.html).
+ * For Thin LTO, this might be helpful:
+ * In gcc 4.6 -fwhopr was removed and became default with -flto. The non-whopr path can still be executed via -flto-partition=none.
+ *
+ * Maybe some missing optizations enabled by rustc's LTO is in there: https://gcc.gnu.org/onlinedocs/gcc/Optimize-Options.html
+ * Like -fipa-icf (should be already enabled) and maybe -fdevirtualize-at-ltrans.
+ * TODO: disable debug info always being emitted. Perhaps this slows down things?
  *
  * TODO(antoyo): remove the patches.
  */
@@ -28,6 +34,7 @@ extern crate rustc_codegen_ssa;
 extern crate rustc_data_structures;
 extern crate rustc_errors;
 extern crate rustc_fluent_macro;
+extern crate rustc_fs_util;
 extern crate rustc_hir;
 extern crate rustc_macros;
 extern crate rustc_metadata;
@@ -35,7 +42,8 @@ extern crate rustc_middle;
 extern crate rustc_session;
 extern crate rustc_span;
 extern crate rustc_target;
-extern crate tempfile;
+#[macro_use]
+extern crate tracing;
 
 // This prevents duplicating functions and statics that are already part of the host rustc process.
 #[allow(unused_extern_crates)]
@@ -57,6 +65,7 @@ mod coverageinfo;
 mod debuginfo;
 mod declare;
 mod errors;
+mod gcc_util;
 mod int;
 mod intrinsic;
 mod mono_item;
@@ -64,18 +73,27 @@ mod type_;
 mod type_of;
 
 use std::any::Any;
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
+use std::sync::Mutex;
+#[cfg(not(feature="master"))]
+use std::sync::atomic::AtomicBool;
+#[cfg(not(feature="master"))]
+use std::sync::atomic::Ordering;
 
-use crate::errors::LTONotSupported;
-use gccjit::{Context, OptimizationLevel, CType};
+use gccjit::{Context, OptimizationLevel};
+#[cfg(feature="master")]
+use gccjit::TargetInfo;
+#[cfg(not(feature="master"))]
+use gccjit::CType;
+use errors::LTONotSupported;
 use rustc_ast::expand::allocator::AllocatorKind;
 use rustc_codegen_ssa::{CodegenResults, CompiledModule, ModuleCodegen};
 use rustc_codegen_ssa::base::codegen_crate;
 use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput, ModuleConfig, TargetMachineFactoryFn};
 use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule};
 use rustc_codegen_ssa::target_features::supported_target_features;
-use rustc_codegen_ssa::traits::{CodegenBackend, ExtraBackendMethods, ModuleBufferMethods, ThinBufferMethods, WriteBackendMethods};
 use rustc_data_structures::fx::FxIndexMap;
+use rustc_codegen_ssa::traits::{CodegenBackend, ExtraBackendMethods, ThinBufferMethods, WriteBackendMethods};
 use rustc_errors::{DiagnosticMessage, ErrorGuaranteed, Handler, SubdiagnosticMessage};
 use rustc_fluent_macro::fluent_messages;
 use rustc_metadata::EncodedMetadata;
@@ -88,6 +106,9 @@ use rustc_span::Symbol;
 use rustc_span::fatal_error::FatalError;
 use tempfile::TempDir;
 
+use crate::back::lto::ModuleBuffer;
+use crate::gcc_util::target_cpu;
+
 fluent_messages! { "../messages.ftl" }
 
 pub struct PrintOnPanic<F: Fn() -> String>(pub F);
@@ -100,9 +121,41 @@ impl<F: Fn() -> String> Drop for PrintOnPanic<F> {
     }
 }
 
+#[cfg(not(feature="master"))]
+#[derive(Debug)]
+pub struct TargetInfo {
+    supports_128bit_integers: AtomicBool,
+}
+
+#[cfg(not(feature="master"))]
+impl TargetInfo {
+    fn cpu_supports(&self, _feature: &str) -> bool {
+        false
+    }
+
+    fn supports_128bit_int(&self) -> bool {
+        self.supports_128bit_integers.load(Ordering::SeqCst)
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct LockedTargetInfo {
+    info: Arc<Mutex<TargetInfo>>,
+}
+
+impl LockedTargetInfo {
+    fn cpu_supports(&self, feature: &str) -> bool {
+        self.info.lock().expect("lock").cpu_supports(feature)
+    }
+
+    fn supports_128bit_int(&self) -> bool {
+        self.info.lock().expect("lock").supports_128bit_int()
+    }
+}
+
 #[derive(Clone)]
 pub struct GccCodegenBackend {
-    supports_128bit_integers: Arc<Mutex<bool>>,
+    target_info: LockedTargetInfo,
 }
 
 impl CodegenBackend for GccCodegenBackend {
@@ -111,25 +164,41 @@ impl CodegenBackend for GccCodegenBackend {
     }
 
     fn init(&self, sess: &Session) {
+        #[cfg(feature="master")]
+        {
+            let target_cpu = target_cpu(sess);
+
+            // Get the second TargetInfo with the correct CPU features by setting the arch.
+            let context = Context::default();
+            if target_cpu != "generic" {
+                context.add_command_line_option(&format!("-march={}", target_cpu));
+            }
+
+            *self.target_info.info.lock().expect("lock") = context.get_target_info();
+        }
+
         #[cfg(feature="master")]
         gccjit::set_global_personality_function_name(b"rust_eh_personality\0");
-        if sess.lto() != Lto::No {
+        if sess.lto() == Lto::Thin {
             sess.emit_warning(LTONotSupported {});
         }
 
-        let temp_dir = TempDir::new().expect("cannot create temporary directory");
-        let temp_file = temp_dir.into_path().join("result.asm");
-        let check_context = Context::default();
-        check_context.set_print_errors_to_stderr(false);
-        let _int128_ty = check_context.new_c_type(CType::UInt128t);
-        // NOTE: we cannot just call compile() as this would require other files than libgccjit.so.
-        check_context.compile_to_file(gccjit::OutputKind::Assembler, temp_file.to_str().expect("path to str"));
-        *self.supports_128bit_integers.lock().expect("lock") = check_context.get_last_error() == Ok(None);
+        #[cfg(not(feature="master"))]
+        {
+            let temp_dir = TempDir::new().expect("cannot create temporary directory");
+            let temp_file = temp_dir.into_path().join("result.asm");
+            let check_context = Context::default();
+            check_context.set_print_errors_to_stderr(false);
+            let _int128_ty = check_context.new_c_type(CType::UInt128t);
+            // NOTE: we cannot just call compile() as this would require other files than libgccjit.so.
+            check_context.compile_to_file(gccjit::OutputKind::Assembler, temp_file.to_str().expect("path to str"));
+            self.target_info.info.lock().expect("lock").supports_128bit_integers.store(check_context.get_last_error() == Ok(None), Ordering::SeqCst);
+        }
     }
 
     fn provide(&self, providers: &mut Providers) {
-        // FIXME(antoyo) compute list of enabled features from cli flags
-        providers.global_backend_features = |_tcx, ()| vec![];
+        providers.global_backend_features =
+            |tcx, ()| gcc_util::global_gcc_features(tcx.sess, true)
     }
 
     fn codegen_crate<'tcx>(&self, tcx: TyCtxt<'tcx>, metadata: EncodedMetadata, need_metadata_module: bool) -> Box<dyn Any> {
@@ -160,7 +229,7 @@ impl CodegenBackend for GccCodegenBackend {
     }
 
     fn target_features(&self, sess: &Session, allow_unstable: bool) -> Vec<Symbol> {
-        target_features(sess, allow_unstable)
+        target_features(sess, allow_unstable, &self.target_info)
     }
 }
 
@@ -168,13 +237,18 @@ impl ExtraBackendMethods for GccCodegenBackend {
     fn codegen_allocator<'tcx>(&self, tcx: TyCtxt<'tcx>, module_name: &str, kind: AllocatorKind, alloc_error_handler_kind: AllocatorKind) -> Self::Module {
         let mut mods = GccContext {
             context: Context::default(),
+            should_combine_object_files: false,
+            temp_dir: None,
         };
+
+        // TODO(antoyo): only set for x86.
+        mods.context.add_command_line_option("-masm=intel");
         unsafe { allocator::codegen(tcx, &mut mods, module_name, kind, alloc_error_handler_kind); }
         mods
     }
 
     fn compile_codegen_unit(&self, tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen<Self::Module>, u64) {
-        base::compile_codegen_unit(tcx, cgu_name, *self.supports_128bit_integers.lock().expect("lock"))
+        base::compile_codegen_unit(tcx, cgu_name, self.target_info.clone())
     }
 
     fn target_machine_factory(&self, _sess: &Session, _opt_level: OptLevel, _features: &[String]) -> TargetMachineFactoryFn<Self> {
@@ -185,14 +259,6 @@ impl ExtraBackendMethods for GccCodegenBackend {
     }
 }
 
-pub struct ModuleBuffer;
-
-impl ModuleBufferMethods for ModuleBuffer {
-    fn data(&self) -> &[u8] {
-        unimplemented!();
-    }
-}
-
 pub struct ThinBuffer;
 
 impl ThinBufferMethods for ThinBuffer {
@@ -203,6 +269,9 @@ impl ThinBufferMethods for ThinBuffer {
 
 pub struct GccContext {
     context: Context<'static>,
+    should_combine_object_files: bool,
+    // Temporary directory used by LTO. We keep it here so that it's not removed before linking.
+    temp_dir: Option<TempDir>,
 }
 
 unsafe impl Send for GccContext {}
@@ -217,18 +286,8 @@ impl WriteBackendMethods for GccCodegenBackend {
     type ThinData = ();
     type ThinBuffer = ThinBuffer;
 
-    fn run_fat_lto(_cgcx: &CodegenContext<Self>, mut modules: Vec<FatLtoInput<Self>>, _cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>) -> Result<LtoModuleCodegen<Self>, FatalError> {
-        // TODO(antoyo): implement LTO by sending -flto to libgccjit and adding the appropriate gcc linker plugins.
-        // NOTE: implemented elsewhere.
-        // TODO(antoyo): what is implemented elsewhere ^ ?
-        let module =
-            match modules.remove(0) {
-                FatLtoInput::InMemory(module) => module,
-                FatLtoInput::Serialized { .. } => {
-                    unimplemented!();
-                }
-            };
-        Ok(LtoModuleCodegen::Fat { module, _serialized_bitcode: vec![] })
+    fn run_fat_lto(cgcx: &CodegenContext<Self>, modules: Vec<FatLtoInput<Self>>, cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>) -> Result<LtoModuleCodegen<Self>, FatalError> {
+        back::lto::run_fat(cgcx, modules, cached_modules)
     }
 
     fn run_thin_lto(_cgcx: &CodegenContext<Self>, _modules: Vec<(String, Self::ThinBuffer)>, _cached_modules: Vec<(SerializedModule<Self::ModuleBuffer>, WorkProduct)>) -> Result<(Vec<LtoModuleCodegen<Self>>, Vec<WorkProduct>), FatalError> {
@@ -277,8 +336,19 @@ impl WriteBackendMethods for GccCodegenBackend {
 /// This is the entrypoint for a hot plugged rustc_codegen_gccjit
 #[no_mangle]
 pub fn __rustc_codegen_backend() -> Box<dyn CodegenBackend> {
+    #[cfg(feature="master")]
+    let info = {
+        // Check whether the target supports 128-bit integers.
+        let context = Context::default();
+        Arc::new(Mutex::new(context.get_target_info()))
+    };
+    #[cfg(not(feature="master"))]
+    let info = Arc::new(Mutex::new(TargetInfo {
+        supports_128bit_integers: AtomicBool::new(false),
+    }));
+
     Box::new(GccCodegenBackend {
-        supports_128bit_integers: Arc::new(Mutex::new(false)),
+        target_info: LockedTargetInfo { info },
     })
 }
 
@@ -297,22 +367,7 @@ fn to_gcc_opt_level(optlevel: Option<OptLevel>) -> OptimizationLevel {
     }
 }
 
-fn handle_native(name: &str) -> &str {
-    if name != "native" {
-        return name;
-    }
-
-    unimplemented!();
-}
-
-pub fn target_cpu(sess: &Session) -> &str {
-    match sess.opts.cg.target_cpu {
-        Some(ref name) => handle_native(name),
-        None => handle_native(sess.target.cpu.as_ref()),
-    }
-}
-
-pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> {
+pub fn target_features(sess: &Session, allow_unstable: bool, target_info: &LockedTargetInfo) -> Vec<Symbol> {
     supported_target_features(sess)
         .iter()
         .filter_map(
@@ -321,26 +376,13 @@ pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec<Symbol> {
             },
         )
         .filter(|_feature| {
-            // TODO(antoyo): implement a way to get enabled feature in libgccjit.
-            // Probably using the equivalent of __builtin_cpu_supports.
-            // TODO(antoyo): maybe use whatever outputs the following command:
-            // gcc -march=native -Q --help=target
-            #[cfg(feature="master")]
-            {
-                // NOTE: the CPU in the CI doesn't support sse4a, so disable it to make the stdarch tests pass in the CI.
-                (_feature.contains("sse") || _feature.contains("avx")) && !_feature.contains("avx512") && !_feature.contains("sse4a")
-            }
-            #[cfg(not(feature="master"))]
-            {
-                false
-            }
+            target_info.cpu_supports(_feature)
             /*
                adx, aes, avx, avx2, avx512bf16, avx512bitalg, avx512bw, avx512cd, avx512dq, avx512er, avx512f, avx512ifma,
                avx512pf, avx512vbmi, avx512vbmi2, avx512vl, avx512vnni, avx512vp2intersect, avx512vpopcntdq,
                bmi1, bmi2, cmpxchg16b, ermsb, f16c, fma, fxsr, gfni, lzcnt, movbe, pclmulqdq, popcnt, rdrand, rdseed, rtm,
                sha, sse, sse2, sse3, sse4.1, sse4.2, sse4a, ssse3, tbm, vaes, vpclmulqdq, xsave, xsavec, xsaveopt, xsaves
              */
-            //false
         })
         .map(|feature| Symbol::intern(feature))
         .collect()
diff --git a/src/type_of.rs b/src/type_of.rs
index cc467801beba..c2eab295acd2 100644
--- a/src/type_of.rs
+++ b/src/type_of.rs
@@ -182,6 +182,7 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
     /// of that field's type - this is useful for taking the address of
     /// that field and ensuring the struct has the right alignment.
     fn gcc_type<'gcc>(&self, cx: &CodegenCx<'gcc, 'tcx>) -> Type<'gcc> {
+        use crate::rustc_middle::ty::layout::FnAbiOf;
         // This must produce the same result for `repr(transparent)` wrappers as for the inner type!
         // In other words, this should generally not look at the type at all, but only at the
         // layout.
@@ -191,7 +192,14 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
             if let Some(&ty) = cx.scalar_types.borrow().get(&self.ty) {
                 return ty;
             }
-            let ty = self.scalar_gcc_type_at(cx, scalar, Size::ZERO);
+            let ty =
+                match *self.ty.kind() {
+                    // NOTE: we cannot remove this match like in the LLVM codegen because the call
+                    // to fn_ptr_backend_type handle the on-stack attribute.
+                    // TODO(antoyo): find a less hackish way to hande the on-stack attribute.
+                    ty::FnPtr(sig) => cx.fn_ptr_backend_type(&cx.fn_abi_of_fn_ptr(sig, ty::List::empty())),
+                    _ => self.scalar_gcc_type_at(cx, scalar, Size::ZERO),
+                };
             cx.scalar_types.borrow_mut().insert(self.ty, ty);
             return ty;
         }
diff --git a/test.sh b/test.sh
index b462e5d156b2..e4cbd6fbcaff 100755
--- a/test.sh
+++ b/test.sh
@@ -3,6 +3,7 @@
 # TODO(antoyo): rewrite to cargo-make (or just) or something like that to only rebuild the sysroot when needed?
 
 set -e
+#set -x
 
 if [ -f ./gcc_path ]; then
     export GCC_PATH=$(cat gcc_path)
@@ -219,6 +220,7 @@ change-id = 115898
 [rust]
 codegen-backends = []
 deny-warnings = false
+verbose-tests = true
 
 [build]
 cargo = "$(rustup which cargo)"
@@ -345,15 +347,19 @@ function test_rustc() {
 
     git checkout -- tests/ui/issues/auxiliary/issue-3136-a.rs # contains //~ERROR, but shouldn't be removed
 
-    rm -r tests/ui/{abi*,extern/,unsized-locals/,proc-macro/,threads-sendsync/,thinlto/,borrowck/,chalkify/bugs/,test*,*lto*.rs,consts/const-float-bits-reject-conv.rs,consts/issue-miri-1910.rs} || true
-    rm tests/ui/mir/mir_heavy_promoted.rs # this tests is oom-killed in the CI.
-    for test in $(rg --files-with-matches "thread|lto" tests/ui); do
+    rm -r tests/ui/{abi*,extern/,unsized-locals/,proc-macro/,threads-sendsync/,thinlto/,borrowck/,chalkify/bugs/,test*,consts/const-float-bits-reject-conv.rs,consts/issue-miri-1910.rs} || true
+    rm tests/ui/mir/mir_heavy_promoted.rs # this test is oom-killed in the CI.
+    # Tests generating errors.
+    rm tests/ui/consts/const-eval/nonnull_as_ref_ub.rs tests/ui/consts/issue-94675.rs
+    for test in $(rg --files-with-matches "thread" tests/ui); do
       rm $test
     done
-    git checkout tests/ui/lto/auxiliary/dylib.rs
     git checkout tests/ui/type-alias-impl-trait/auxiliary/cross_crate_ice.rs
     git checkout tests/ui/type-alias-impl-trait/auxiliary/cross_crate_ice2.rs
     git checkout tests/ui/macros/rfc-2011-nicer-assert-messages/auxiliary/common.rs
+    git checkout tests/ui/imports/ambiguous-1.rs
+    git checkout tests/ui/imports/ambiguous-4-extern.rs
+    git checkout tests/ui/entry-point/auxiliary/bad_main_functions.rs
 
     RUSTC_ARGS="$TEST_FLAGS -Csymbol-mangling-version=v0 -Zcodegen-backend="$(pwd)"/../target/"$CHANNEL"/librustc_codegen_gcc."$dylib_ext" --sysroot "$(pwd)"/../build_sysroot/sysroot"
 
diff --git a/tests/run/abort1.rs b/tests/run/abort1.rs
index 25041d93e748..44297e12779b 100644
--- a/tests/run/abort1.rs
+++ b/tests/run/abort1.rs
@@ -3,7 +3,8 @@
 // Run-time:
 //   status: signal
 
-#![feature(auto_traits, lang_items, no_core, start, intrinsics)]
+#![feature(auto_traits, lang_items, no_core, start, intrinsics, rustc_attrs)]
+#![allow(internal_features)]
 
 #![no_std]
 #![no_core]
diff --git a/tests/run/abort2.rs b/tests/run/abort2.rs
index e7443c8dbe5b..ce816927123d 100644
--- a/tests/run/abort2.rs
+++ b/tests/run/abort2.rs
@@ -3,7 +3,8 @@
 // Run-time:
 //   status: signal
 
-#![feature(auto_traits, lang_items, no_core, start, intrinsics)]
+#![feature(auto_traits, lang_items, no_core, start, intrinsics, rustc_attrs)]
+#![allow(internal_features)]
 
 #![no_std]
 #![no_core]
diff --git a/tests/run/array.rs b/tests/run/array.rs
index 49b28d98f2fe..afd0eed82004 100644
--- a/tests/run/array.rs
+++ b/tests/run/array.rs
@@ -7,7 +7,8 @@
 //     5
 //     10
 
-#![feature(arbitrary_self_types, auto_traits, lang_items, no_core, start, intrinsics)]
+#![feature(arbitrary_self_types, auto_traits, lang_items, no_core, start, intrinsics, rustc_attrs)]
+#![allow(internal_features)]
 
 #![no_std]
 #![no_core]
diff --git a/tests/run/asm.rs b/tests/run/asm.rs
index 38c1eac7adf6..507b65ca049e 100644
--- a/tests/run/asm.rs
+++ b/tests/run/asm.rs
@@ -124,7 +124,7 @@ fn main() {
     // check const (ATT syntax)
     let mut x: u64 = 42;
     unsafe {
-        asm!("add {}, {}",
+        asm!("add ${}, {}",
             const 1,
             inout(reg) x,
             options(att_syntax)
diff --git a/tests/run/assign.rs b/tests/run/assign.rs
index 427c1a250339..5b0db2da294d 100644
--- a/tests/run/assign.rs
+++ b/tests/run/assign.rs
@@ -5,8 +5,8 @@
 //     7 8
 //     10
 
-#![allow(unused_attributes)]
-#![feature(auto_traits, lang_items, no_core, start, intrinsics, track_caller)]
+#![allow(internal_features, unused_attributes)]
+#![feature(auto_traits, lang_items, no_core, start, intrinsics, rustc_attrs, track_caller)]
 
 #![no_std]
 #![no_core]
diff --git a/tests/run/closure.rs b/tests/run/closure.rs
index 8daa681abf7d..4ce528f86800 100644
--- a/tests/run/closure.rs
+++ b/tests/run/closure.rs
@@ -9,7 +9,8 @@
 //     Both args: 11
 
 #![feature(arbitrary_self_types, auto_traits, lang_items, no_core, start, intrinsics,
-    unboxed_closures)]
+    unboxed_closures, rustc_attrs)]
+#![allow(internal_features)]
 
 #![no_std]
 #![no_core]
diff --git a/tests/run/condition.rs b/tests/run/condition.rs
index b7a13081deae..1b3ae6dc004a 100644
--- a/tests/run/condition.rs
+++ b/tests/run/condition.rs
@@ -5,7 +5,8 @@
 //   stdout: true
 //     1
 
-#![feature(arbitrary_self_types, auto_traits, lang_items, no_core, start, intrinsics)]
+#![feature(arbitrary_self_types, auto_traits, lang_items, no_core, start, intrinsics, rustc_attrs)]
+#![allow(internal_features)]
 
 #![no_std]
 #![no_core]
diff --git a/tests/run/empty_main.rs b/tests/run/empty_main.rs
index c02cfd2a85f0..2d78ef12aa72 100644
--- a/tests/run/empty_main.rs
+++ b/tests/run/empty_main.rs
@@ -4,6 +4,7 @@
 //   status: 0
 
 #![feature(auto_traits, lang_items, no_core, start)]
+#![allow(internal_features)]
 
 #![no_std]
 #![no_core]
diff --git a/tests/run/exit.rs b/tests/run/exit.rs
index 956e53dd4aa6..bf1cbeef3020 100644
--- a/tests/run/exit.rs
+++ b/tests/run/exit.rs
@@ -4,6 +4,7 @@
 //   status: 2
 
 #![feature(auto_traits, lang_items, no_core, start, intrinsics)]
+#![allow(internal_features)]
 
 #![no_std]
 #![no_core]
diff --git a/tests/run/exit_code.rs b/tests/run/exit_code.rs
index eeab35209512..be7a233efdaa 100644
--- a/tests/run/exit_code.rs
+++ b/tests/run/exit_code.rs
@@ -4,6 +4,7 @@
 //   status: 1
 
 #![feature(auto_traits, lang_items, no_core, start)]
+#![allow(internal_features)]
 
 #![no_std]
 #![no_core]
diff --git a/tests/run/fun_ptr.rs b/tests/run/fun_ptr.rs
index 8a196f774c82..960303597726 100644
--- a/tests/run/fun_ptr.rs
+++ b/tests/run/fun_ptr.rs
@@ -4,7 +4,8 @@
 //   status: 0
 //   stdout: 1
 
-#![feature(arbitrary_self_types, auto_traits, lang_items, no_core, start, intrinsics)]
+#![feature(arbitrary_self_types, auto_traits, lang_items, no_core, start, intrinsics, rustc_attrs)]
+#![allow(internal_features)]
 
 #![no_std]
 #![no_core]
diff --git a/tests/run/gep.rs b/tests/run/gep.rs
new file mode 100644
index 000000000000..c3d1672cff57
--- /dev/null
+++ b/tests/run/gep.rs
@@ -0,0 +1,10 @@
+// Compiler:
+//
+// Run-time:
+//   status: 0
+
+fn main() {
+    let mut value = (1, 1);
+    let ptr = &mut value as *mut (i32, i32);
+    println!("{:?}", ptr.wrapping_offset(10));
+}
diff --git a/tests/run/int_overflow.rs b/tests/run/int_overflow.rs
index c3fcb3c0a2a0..08fa087fccdf 100644
--- a/tests/run/int_overflow.rs
+++ b/tests/run/int_overflow.rs
@@ -4,8 +4,8 @@
 //   stdout: Success
 //   status: signal
 
-#![allow(unused_attributes)]
-#![feature(auto_traits, lang_items, no_core, start, intrinsics)]
+#![allow(internal_features, unused_attributes)]
+#![feature(auto_traits, lang_items, no_core, start, intrinsics, rustc_attrs)]
 
 #![no_std]
 #![no_core]
diff --git a/tests/run/mut_ref.rs b/tests/run/mut_ref.rs
index 2a2ea8b8bf0a..194e55a3deae 100644
--- a/tests/run/mut_ref.rs
+++ b/tests/run/mut_ref.rs
@@ -7,8 +7,8 @@
 //     6
 //     11
 
-#![allow(unused_attributes)]
-#![feature(auto_traits, lang_items, no_core, start, intrinsics, track_caller)]
+#![allow(internal_features, unused_attributes)]
+#![feature(auto_traits, lang_items, no_core, start, intrinsics, rustc_attrs, track_caller)]
 
 #![no_std]
 #![no_core]
diff --git a/tests/run/operations.rs b/tests/run/operations.rs
index 67b9f241dbbb..2d781670873f 100644
--- a/tests/run/operations.rs
+++ b/tests/run/operations.rs
@@ -5,8 +5,8 @@
 //     39
 //     10
 
-#![allow(unused_attributes)]
-#![feature(auto_traits, lang_items, no_core, start, intrinsics, arbitrary_self_types)]
+#![allow(internal_features, unused_attributes)]
+#![feature(auto_traits, lang_items, no_core, start, intrinsics, arbitrary_self_types, rustc_attrs)]
 
 #![no_std]
 #![no_core]
diff --git a/tests/run/ptr_cast.rs b/tests/run/ptr_cast.rs
index da8a8295d564..09d77abe27cf 100644
--- a/tests/run/ptr_cast.rs
+++ b/tests/run/ptr_cast.rs
@@ -4,7 +4,8 @@
 //   status: 0
 //   stdout: 1
 
-#![feature(arbitrary_self_types, auto_traits, lang_items, no_core, start, intrinsics)]
+#![feature(arbitrary_self_types, auto_traits, lang_items, no_core, start, intrinsics, rustc_attrs)]
+#![allow(internal_features)]
 
 #![no_std]
 #![no_core]
diff --git a/tests/run/return-tuple.rs b/tests/run/return-tuple.rs
index 6fa10dca06f6..8d40deb8c85e 100644
--- a/tests/run/return-tuple.rs
+++ b/tests/run/return-tuple.rs
@@ -7,6 +7,7 @@
 //     42
 
 #![feature(auto_traits, lang_items, no_core, start, intrinsics)]
+#![allow(internal_features)]
 
 #![no_std]
 #![no_core]
diff --git a/tests/run/slice.rs b/tests/run/slice.rs
index 96f1c4792e58..1262c86c8109 100644
--- a/tests/run/slice.rs
+++ b/tests/run/slice.rs
@@ -4,7 +4,8 @@
 //   status: 0
 //   stdout: 5
 
-#![feature(arbitrary_self_types, auto_traits, lang_items, no_core, start, intrinsics)]
+#![feature(arbitrary_self_types, auto_traits, lang_items, no_core, start, intrinsics, rustc_attrs)]
+#![allow(internal_features)]
 
 #![no_std]
 #![no_core]
diff --git a/tests/run/static.rs b/tests/run/static.rs
index 19201f1df266..0b933754c293 100644
--- a/tests/run/static.rs
+++ b/tests/run/static.rs
@@ -9,7 +9,8 @@
 //      12
 //      1
 
-#![feature(auto_traits, lang_items, no_core, start, intrinsics)]
+#![feature(auto_traits, lang_items, no_core, start, intrinsics, rustc_attrs)]
+#![allow(internal_features)]
 
 #![no_std]
 #![no_core]
diff --git a/tests/run/structs.rs b/tests/run/structs.rs
index 6c8884855ac3..d6455667400c 100644
--- a/tests/run/structs.rs
+++ b/tests/run/structs.rs
@@ -6,6 +6,7 @@
 //     2
 
 #![feature(auto_traits, lang_items, no_core, start, intrinsics)]
+#![allow(internal_features)]
 
 #![no_std]
 #![no_core]
diff --git a/tests/run/tuple.rs b/tests/run/tuple.rs
index 0b670bf26742..8a7d85ae867e 100644
--- a/tests/run/tuple.rs
+++ b/tests/run/tuple.rs
@@ -5,6 +5,7 @@
 //   stdout: 3
 
 #![feature(auto_traits, lang_items, no_core, start, intrinsics)]
+#![allow(internal_features)]
 
 #![no_std]
 #![no_core]
diff --git a/tools/generate_intrinsics.py b/tools/generate_intrinsics.py
index 83abe145e64f..90fb7bfad27c 100644
--- a/tools/generate_intrinsics.py
+++ b/tools/generate_intrinsics.py
@@ -46,10 +46,10 @@ def convert_to_string(content):
 
 
 def extract_instrinsics_from_llvm(llvm_path, intrinsics):
-    p = subprocess.Popen(
-        ["llvm-tblgen", "llvm/IR/Intrinsics.td"],
-        cwd=os.path.join(llvm_path, "llvm/include"),
-        stdout=subprocess.PIPE)
+    command = ["llvm-tblgen", "llvm/IR/Intrinsics.td"]
+    cwd = os.path.join(llvm_path, "llvm/include")
+    print("=> Running command `{}` from `{}`".format(command, cwd))
+    p = subprocess.Popen(command, cwd=cwd, stdout=subprocess.PIPE)
     output, err = p.communicate()
     lines = convert_to_string(output).splitlines()
     pos = 0
diff --git a/y.sh b/y.sh
new file mode 100755
index 000000000000..188109743e3d
--- /dev/null
+++ b/y.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -e
+echo "[BUILD] build system" 1>&2
+cd build_system
+cargo build --release
+cd ..
+./build_system/target/release/y $@

From 22e6f6caafd79948c3d9edaa5e1507570e809d75 Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Mon, 9 Oct 2023 16:03:05 -0400
Subject: [PATCH 011/297] Fix checks

---
 src/base.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/base.rs b/src/base.rs
index 61da38f4b0db..b081e9ff2fdb 100644
--- a/src/base.rs
+++ b/src/base.rs
@@ -120,7 +120,7 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol, target_info: Lock
         // NOTE: Rust relies on LLVM doing wrapping on overflow.
         context.add_command_line_option("-fwrapv");
 
-        if tcx.sess.opts.cg.relocation_model == Some(rustc_target::spec::RelocModel::Static) {
+        if tcx.sess.relocation_model() == rustc_target::spec::RelocModel::Static {
             context.add_command_line_option("-mcmodel=kernel");
             context.add_command_line_option("-fno-pie");
         }

From ba103e34c040eecf5756b2539b0ebf743b3c1058 Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Mon, 9 Oct 2023 16:45:46 -0400
Subject: [PATCH 012/297] Use IntoDynSyncSend

---
 src/lib.rs | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/src/lib.rs b/src/lib.rs
index fe2339305604..9c18fc4a0dc7 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -73,6 +73,7 @@ mod type_;
 mod type_of;
 
 use std::any::Any;
+use std::fmt::Debug;
 use std::sync::Arc;
 use std::sync::Mutex;
 #[cfg(not(feature="master"))]
@@ -93,6 +94,7 @@ use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput, ModuleConfig,
 use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule};
 use rustc_codegen_ssa::target_features::supported_target_features;
 use rustc_data_structures::fx::FxIndexMap;
+use rustc_data_structures::sync::IntoDynSyncSend;
 use rustc_codegen_ssa::traits::{CodegenBackend, ExtraBackendMethods, ThinBufferMethods, WriteBackendMethods};
 use rustc_errors::{DiagnosticMessage, ErrorGuaranteed, Handler, SubdiagnosticMessage};
 use rustc_fluent_macro::fluent_messages;
@@ -138,9 +140,15 @@ impl TargetInfo {
     }
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone)]
 pub struct LockedTargetInfo {
-    info: Arc<Mutex<TargetInfo>>,
+    info: Arc<Mutex<IntoDynSyncSend<TargetInfo>>>,
+}
+
+impl Debug for LockedTargetInfo {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        self.info.lock().expect("lock").fmt(formatter)
+    }
 }
 
 impl LockedTargetInfo {
@@ -174,7 +182,7 @@ impl CodegenBackend for GccCodegenBackend {
                 context.add_command_line_option(&format!("-march={}", target_cpu));
             }
 
-            *self.target_info.info.lock().expect("lock") = context.get_target_info();
+            **self.target_info.info.lock().expect("lock") = context.get_target_info();
         }
 
         #[cfg(feature="master")]
@@ -340,12 +348,12 @@ pub fn __rustc_codegen_backend() -> Box<dyn CodegenBackend> {
     let info = {
         // Check whether the target supports 128-bit integers.
         let context = Context::default();
-        Arc::new(Mutex::new(context.get_target_info()))
+        Arc::new(Mutex::new(IntoDynSyncSend(context.get_target_info())))
     };
     #[cfg(not(feature="master"))]
-    let info = Arc::new(Mutex::new(TargetInfo {
+    let info = Arc::new(Mutex::new(IntoDynSyncSend(TargetInfo {
         supports_128bit_integers: AtomicBool::new(false),
-    }));
+    })));
 
     Box::new(GccCodegenBackend {
         target_info: LockedTargetInfo { info },

From cf8c391fc174c3e3264d910d5bbb4107b0565715 Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Tue, 10 Oct 2023 19:19:38 -0400
Subject: [PATCH 013/297] Add comment

---
 Readme.md | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/Readme.md b/Readme.md
index f001c83b08d3..6fad0707fbe9 100644
--- a/Readme.md
+++ b/Readme.md
@@ -286,6 +286,16 @@ git checkout sync_branch_name
 git merge master
 ```
 
+To send the changes to the rust repo:
+
+```bash
+cd ../rust
+git pull origin master
+git checkbout -b subtree-update_cg_gcc_YYYY-MM-DD
+PATH="$HOME/bin:$PATH" ~/bin/git-subtree pull --prefix=compiler/rustc_codegen_gcc/ https://github.com/rust-lang/rustc_codegen_gcc.git master
+git push
+```
+
 TODO: write a script that does the above.
 
 https://rust-lang.zulipchat.com/#narrow/stream/301329-t-devtools/topic/subtree.20madness/near/258877725

From 6988d2294defdbe02dfca1287613b649d21259a1 Mon Sep 17 00:00:00 2001
From: Gijs Burghoorn <g.burghoorn@gmail.com>
Date: Wed, 4 Oct 2023 13:45:30 +0200
Subject: [PATCH 014/297] Stabilize Ratified RISC-V Target Features

As shortly discussed on Zulip
(https://rust-lang.zulipchat.com/#narrow/stream/250483-t-compiler.2Frisc-v/topic/Stabilization.20of.20RISC-V.20Target.20Features/near/394793704),
this commit stabilizes the ratified RISC-V instruction bases and
extensions.

Specifically, this commit stabilizes the:
* Atomic Instructions (A) on v2.0
* Compressed Instructions (C) on v2.0
* Integer Multiplication and Division (M) on v2.0
* Bit Manipulations (B) on v1.0 listed as `zba`, `zbc`, `zbs`
* Scalar Cryptography (Zk) v1.0.1 listed as `zk`, `zkn`, `zknd`, `zkne`, `zknh`, `zkr`, `zks`, `zksed`, `zksh`, `zkt`, `zbkb`, `zbkc` `zkbx`
---
 .../rustc_codegen_ssa/src/target_features.rs  | 40 +++++++++----------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/compiler/rustc_codegen_ssa/src/target_features.rs b/compiler/rustc_codegen_ssa/src/target_features.rs
index baf6b19d3f97..e57e0ee25244 100644
--- a/compiler/rustc_codegen_ssa/src/target_features.rs
+++ b/compiler/rustc_codegen_ssa/src/target_features.rs
@@ -244,38 +244,38 @@ const MIPS_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
 
 const RISCV_ALLOWED_FEATURES: &[(&str, Option<Symbol>)] = &[
     // tidy-alphabetical-start
-    ("a", Some(sym::riscv_target_feature)),
-    ("c", Some(sym::riscv_target_feature)),
+    ("a", None),
+    ("c", None),
     ("d", Some(sym::riscv_target_feature)),
     ("e", Some(sym::riscv_target_feature)),
     ("f", Some(sym::riscv_target_feature)),
-    ("m", Some(sym::riscv_target_feature)),
+    ("m", None),
     ("relax", Some(sym::riscv_target_feature)),
     ("unaligned-scalar-mem", Some(sym::riscv_target_feature)),
     ("v", Some(sym::riscv_target_feature)),
-    ("zba", Some(sym::riscv_target_feature)),
-    ("zbb", Some(sym::riscv_target_feature)),
-    ("zbc", Some(sym::riscv_target_feature)),
-    ("zbkb", Some(sym::riscv_target_feature)),
-    ("zbkc", Some(sym::riscv_target_feature)),
-    ("zbkx", Some(sym::riscv_target_feature)),
-    ("zbs", Some(sym::riscv_target_feature)),
+    ("zba", None),
+    ("zbb", None),
+    ("zbc", None),
+    ("zbkb", None),
+    ("zbkc", None),
+    ("zbkx", None),
+    ("zbs", None),
     ("zdinx", Some(sym::riscv_target_feature)),
     ("zfh", Some(sym::riscv_target_feature)),
     ("zfhmin", Some(sym::riscv_target_feature)),
     ("zfinx", Some(sym::riscv_target_feature)),
     ("zhinx", Some(sym::riscv_target_feature)),
     ("zhinxmin", Some(sym::riscv_target_feature)),
-    ("zk", Some(sym::riscv_target_feature)),
-    ("zkn", Some(sym::riscv_target_feature)),
-    ("zknd", Some(sym::riscv_target_feature)),
-    ("zkne", Some(sym::riscv_target_feature)),
-    ("zknh", Some(sym::riscv_target_feature)),
-    ("zkr", Some(sym::riscv_target_feature)),
-    ("zks", Some(sym::riscv_target_feature)),
-    ("zksed", Some(sym::riscv_target_feature)),
-    ("zksh", Some(sym::riscv_target_feature)),
-    ("zkt", Some(sym::riscv_target_feature)),
+    ("zk", None),
+    ("zkn", None),
+    ("zknd", None),
+    ("zkne", None),
+    ("zknh", None),
+    ("zkr", None),
+    ("zks", None),
+    ("zksed", None),
+    ("zksh", None),
+    ("zkt", None),
     // tidy-alphabetical-end
 ];
 

From 9030b704214d4b63f56498ee2af4c190a5287ac6 Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Wed, 11 Oct 2023 19:38:03 -0400
Subject: [PATCH 015/297] Update libgccjit

---
 Cargo.lock | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 85675fc40c34..b8e2e5d80804 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -74,7 +74,7 @@ dependencies = [
 [[package]]
 name = "gccjit"
 version = "1.0.0"
-source = "git+https://github.com/antoyo/gccjit.rs#0b158c68bf7e46732869d90550a98e886dee8858"
+source = "git+https://github.com/antoyo/gccjit.rs#c52a218f5529321285b4489e5562a00e5428e033"
 dependencies = [
  "gccjit_sys",
 ]
@@ -82,7 +82,7 @@ dependencies = [
 [[package]]
 name = "gccjit_sys"
 version = "0.0.1"
-source = "git+https://github.com/antoyo/gccjit.rs#0b158c68bf7e46732869d90550a98e886dee8858"
+source = "git+https://github.com/antoyo/gccjit.rs#c52a218f5529321285b4489e5562a00e5428e033"
 dependencies = [
  "libc",
 ]

From 100dfced2067925df97cf203d696559be5c828b9 Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Wed, 11 Oct 2023 20:55:32 -0400
Subject: [PATCH 016/297] Fix #[inline(always)] attribute

---
 src/attributes.rs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/attributes.rs b/src/attributes.rs
index 971e019a4f60..6159971cfaa8 100644
--- a/src/attributes.rs
+++ b/src/attributes.rs
@@ -53,6 +53,9 @@ pub fn from_fn_attrs<'gcc, 'tcx>(
                 codegen_fn_attrs.inline
             };
         if let Some(attr) = inline_attr(cx, inline) {
+            if let FnAttribute::AlwaysInline = attr {
+                func.add_attribute(FnAttribute::Inline);
+            }
             func.add_attribute(attr);
         }
 

From 89f9da2056d88ab96faf95901687cb772d2dc10c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eduardo=20S=C3=A1nchez=20Mu=C3=B1oz?= <eduardosm-dev@e64.io>
Date: Tue, 10 Oct 2023 18:29:07 +0200
Subject: [PATCH 017/297] Bump stdarch submodule

---
 library/stdarch | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/library/stdarch b/library/stdarch
index 333e9e997718..f4528dd6e85d 160000
--- a/library/stdarch
+++ b/library/stdarch
@@ -1 +1 @@
-Subproject commit 333e9e9977188d0748327e9b5be0f3f412063174
+Subproject commit f4528dd6e85d97bb802240d7cd048b6e1bf72540

From b57a1570920623d57c9e47b668896fe288d4f6e9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eduardo=20S=C3=A1nchez=20Mu=C3=B1oz?= <eduardosm-dev@e64.io>
Date: Tue, 10 Oct 2023 18:35:28 +0200
Subject: [PATCH 018/297] Remove from cranelift codegen LLVM intrinsics that
 are no longer needed

---
 .../src/intrinsics/llvm_x86.rs                | 35 -------------------
 1 file changed, 35 deletions(-)

diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
index 0c9a94e1c231..559c64bb13bd 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
@@ -32,41 +32,6 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, CValue::by_val(res, fx.layout_of(fx.tcx.types.i64)));
         }
 
-        // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8`
-        "llvm.x86.sse2.pmovmskb.128"
-        | "llvm.x86.avx2.pmovmskb"
-        | "llvm.x86.sse.movmsk.ps"
-        | "llvm.x86.sse2.movmsk.pd" => {
-            intrinsic_args!(fx, args => (a); intrinsic);
-
-            let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
-            let lane_ty = fx.clif_type(lane_ty).unwrap();
-            assert!(lane_count <= 32);
-
-            let mut res = fx.bcx.ins().iconst(types::I32, 0);
-
-            for lane in (0..lane_count).rev() {
-                let a_lane = a.value_lane(fx, lane).load_scalar(fx);
-
-                // cast float to int
-                let a_lane = match lane_ty {
-                    types::F32 => codegen_bitcast(fx, types::I32, a_lane),
-                    types::F64 => codegen_bitcast(fx, types::I64, a_lane),
-                    _ => a_lane,
-                };
-
-                // extract sign bit of an int
-                let a_lane_sign = fx.bcx.ins().ushr_imm(a_lane, i64::from(lane_ty.bits() - 1));
-
-                // shift sign bit into result
-                let a_lane_sign = clif_intcast(fx, a_lane_sign, types::I32, false);
-                res = fx.bcx.ins().ishl_imm(res, 1);
-                res = fx.bcx.ins().bor(res, a_lane_sign);
-            }
-
-            let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32));
-            ret.write_cvalue(fx, res);
-        }
         "llvm.x86.sse.cmp.ps" | "llvm.x86.sse2.cmp.pd" => {
             let (x, y, kind) = match args {
                 [x, y, kind] => (x, y, kind),

From 337af7caf76e14051721943fb67484ad8a6cff35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eduardo=20S=C3=A1nchez=20Mu=C3=B1oz?= <eduardosm-dev@e64.io>
Date: Tue, 10 Oct 2023 18:43:42 +0200
Subject: [PATCH 019/297] Remove from miri LLVM intrinsics that are no longer
 needed

---
 src/tools/miri/src/shims/x86/sse.rs  |  19 ---
 src/tools/miri/src/shims/x86/sse2.rs | 176 +--------------------------
 src/tools/miri/src/shims/x86/sse3.rs |  26 ----
 3 files changed, 1 insertion(+), 220 deletions(-)

diff --git a/src/tools/miri/src/shims/x86/sse.rs b/src/tools/miri/src/shims/x86/sse.rs
index 6f0b76059f10..831228b7a26c 100644
--- a/src/tools/miri/src/shims/x86/sse.rs
+++ b/src/tools/miri/src/shims/x86/sse.rs
@@ -209,25 +209,6 @@ pub(super) trait EvalContextExt<'mir, 'tcx: 'mir>:
                     )?;
                 }
             }
-            // Used to implement the _mm_movemask_ps function.
-            // Returns a scalar integer where the i-th bit is the highest
-            // bit of the i-th component of `op`.
-            // https://www.felixcloutier.com/x86/movmskps
-            "movmsk.ps" => {
-                let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
-                let (op, op_len) = this.operand_to_simd(op)?;
-
-                let mut res = 0;
-                for i in 0..op_len {
-                    let op = this.read_scalar(&this.project_index(&op, i)?)?;
-                    let op = op.to_u32()?;
-
-                    // Extract the highest bit of `op` and place it in the `i`-th bit of `res`
-                    res |= (op >> 31) << i;
-                }
-
-                this.write_scalar(Scalar::from_u32(res), dest)?;
-            }
             _ => return Ok(EmulateForeignItemResult::NotSupported),
         }
         Ok(EmulateForeignItemResult::NeedsJumping)
diff --git a/src/tools/miri/src/shims/x86/sse2.rs b/src/tools/miri/src/shims/x86/sse2.rs
index c6a847b5cf82..3f2b9f5f0adf 100644
--- a/src/tools/miri/src/shims/x86/sse2.rs
+++ b/src/tools/miri/src/shims/x86/sse2.rs
@@ -1,8 +1,4 @@
-use rustc_apfloat::{
-    ieee::{Double, Single},
-    Float as _,
-};
-use rustc_middle::mir;
+use rustc_apfloat::ieee::Double;
 use rustc_middle::ty::layout::LayoutOf as _;
 use rustc_middle::ty::Ty;
 use rustc_span::Symbol;
@@ -39,49 +35,6 @@ pub(super) trait EvalContextExt<'mir, 'tcx: 'mir>:
         // Intrinsincs sufixed with "epiX" or "epuX" operate with X-bit signed or unsigned
         // vectors.
         match unprefixed_name {
-            // Used to implement the _mm_avg_epu8 and _mm_avg_epu16 functions.
-            // Averages packed unsigned 8/16-bit integers in `left` and `right`.
-            "pavg.b" | "pavg.w" => {
-                let [left, right] =
-                    this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
-
-                let (left, left_len) = this.operand_to_simd(left)?;
-                let (right, right_len) = this.operand_to_simd(right)?;
-                let (dest, dest_len) = this.place_to_simd(dest)?;
-
-                assert_eq!(dest_len, left_len);
-                assert_eq!(dest_len, right_len);
-
-                for i in 0..dest_len {
-                    let left = this.read_immediate(&this.project_index(&left, i)?)?;
-                    let right = this.read_immediate(&this.project_index(&right, i)?)?;
-                    let dest = this.project_index(&dest, i)?;
-
-                    // Widen the operands to avoid overflow
-                    let twice_wide = this.layout_of(this.get_twice_wide_int_ty(left.layout.ty))?;
-                    let left = this.int_to_int_or_float(&left, twice_wide)?;
-                    let right = this.int_to_int_or_float(&right, twice_wide)?;
-
-                    // Calculate left + right + 1
-                    let added = this.wrapping_binary_op(mir::BinOp::Add, &left, &right)?;
-                    let added = this.wrapping_binary_op(
-                        mir::BinOp::Add,
-                        &added,
-                        &ImmTy::from_uint(1u32, twice_wide),
-                    )?;
-
-                    // Calculate (left + right + 1) / 2
-                    let divided = this.wrapping_binary_op(
-                        mir::BinOp::Div,
-                        &added,
-                        &ImmTy::from_uint(2u32, twice_wide),
-                    )?;
-
-                    // Narrow back to the original type
-                    let res = this.int_to_int_or_float(&divided, dest.layout)?;
-                    this.write_immediate(*res, &dest)?;
-                }
-            }
             // Used to implement the _mm_madd_epi16 function.
             // Multiplies packed signed 16-bit integers in `left` and `right`, producing
             // intermediate signed 32-bit integers. Horizontally add adjacent pairs of
@@ -118,70 +71,6 @@ pub(super) trait EvalContextExt<'mir, 'tcx: 'mir>:
                     this.write_scalar(Scalar::from_i32(res), &dest)?;
                 }
             }
-            // Used to implement the _mm_mulhi_epi16 and _mm_mulhi_epu16 functions.
-            "pmulh.w" | "pmulhu.w" => {
-                let [left, right] =
-                    this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
-
-                let (left, left_len) = this.operand_to_simd(left)?;
-                let (right, right_len) = this.operand_to_simd(right)?;
-                let (dest, dest_len) = this.place_to_simd(dest)?;
-
-                assert_eq!(dest_len, left_len);
-                assert_eq!(dest_len, right_len);
-
-                for i in 0..dest_len {
-                    let left = this.read_immediate(&this.project_index(&left, i)?)?;
-                    let right = this.read_immediate(&this.project_index(&right, i)?)?;
-                    let dest = this.project_index(&dest, i)?;
-
-                    // Widen the operands to avoid overflow
-                    let twice_wide = this.layout_of(this.get_twice_wide_int_ty(left.layout.ty))?;
-                    let left = this.int_to_int_or_float(&left, twice_wide)?;
-                    let right = this.int_to_int_or_float(&right, twice_wide)?;
-
-                    // Multiply
-                    let multiplied = this.wrapping_binary_op(mir::BinOp::Mul, &left, &right)?;
-                    // Keep the high half
-                    let high = this.wrapping_binary_op(
-                        mir::BinOp::Shr,
-                        &multiplied,
-                        &ImmTy::from_uint(dest.layout.size.bits(), twice_wide),
-                    )?;
-
-                    // Narrow back to the original type
-                    let res = this.int_to_int_or_float(&high, dest.layout)?;
-                    this.write_immediate(*res, &dest)?;
-                }
-            }
-            // Used to implement the _mm_mul_epu32 function.
-            // Multiplies the the low unsigned 32-bit integers from each packed
-            // 64-bit element and stores the result as 64-bit unsigned integers.
-            "pmulu.dq" => {
-                let [left, right] =
-                    this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
-
-                let (left, left_len) = this.operand_to_simd(left)?;
-                let (right, right_len) = this.operand_to_simd(right)?;
-                let (dest, dest_len) = this.place_to_simd(dest)?;
-
-                // left and right are u32x4, dest is u64x2
-                assert_eq!(left_len, 4);
-                assert_eq!(right_len, 4);
-                assert_eq!(dest_len, 2);
-
-                for i in 0..dest_len {
-                    let op_i = i.checked_mul(2).unwrap();
-                    let left = this.read_scalar(&this.project_index(&left, op_i)?)?.to_u32()?;
-                    let right = this.read_scalar(&this.project_index(&right, op_i)?)?.to_u32()?;
-                    let dest = this.project_index(&dest, i)?;
-
-                    // The multiplication will not overflow because stripping the
-                    // operands are expanded from 32-bit to 64-bit.
-                    let res = u64::from(left).checked_mul(u64::from(right)).unwrap();
-                    this.write_scalar(Scalar::from_u64(res), &dest)?;
-                }
-            }
             // Used to implement the _mm_sad_epu8 function.
             // Computes the absolute differences of packed unsigned 8-bit integers in `a`
             // and `b`, then horizontally sum each consecutive 8 differences to produce
@@ -370,25 +259,6 @@ pub(super) trait EvalContextExt<'mir, 'tcx: 'mir>:
                     this.write_scalar(Scalar::from_u64(res), &dest)?;
                 }
             }
-            // Used to implement the _mm_cvtepi32_ps function.
-            // Converts packed i32 to packed f32.
-            // FIXME: Can we get rid of this intrinsic and just use simd_as?
-            "cvtdq2ps" => {
-                let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
-
-                let (op, op_len) = this.operand_to_simd(op)?;
-                let (dest, dest_len) = this.place_to_simd(dest)?;
-
-                assert_eq!(dest_len, op_len);
-
-                for i in 0..dest_len {
-                    let op = this.read_scalar(&this.project_index(&op, i)?)?.to_i32()?;
-                    let dest = this.project_index(&dest, i)?;
-
-                    let res = Scalar::from_f32(Single::from_i128(op.into()).value);
-                    this.write_scalar(res, &dest)?;
-                }
-            }
             // Used to implement the _mm_cvtps_epi32 and _mm_cvttps_epi32 functions.
             // Converts packed f32 to packed i32.
             "cvtps2dq" | "cvttps2dq" => {
@@ -652,31 +522,6 @@ pub(super) trait EvalContextExt<'mir, 'tcx: 'mir>:
                 };
                 this.write_scalar(Scalar::from_i32(i32::from(res)), dest)?;
             }
-            // Used to implement the _mm_cvtpd_ps and _mm_cvtps_pd functions.
-            // Converts packed f32/f64 to packed f64/f32.
-            "cvtpd2ps" | "cvtps2pd" => {
-                let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
-
-                let (op, op_len) = this.operand_to_simd(op)?;
-                let (dest, dest_len) = this.place_to_simd(dest)?;
-
-                // For cvtpd2ps: op is f64x2, dest is f32x4
-                // For cvtps2pd: op is f32x4, dest is f64x2
-                // In either case, the two first values are converted
-                for i in 0..op_len.min(dest_len) {
-                    let op = this.read_immediate(&this.project_index(&op, i)?)?;
-                    let dest = this.project_index(&dest, i)?;
-
-                    let res = this.float_to_float_or_int(&op, dest.layout)?;
-                    this.write_immediate(*res, &dest)?;
-                }
-                // For f32 -> f64, ignore the remaining
-                // For f64 -> f32, fill the remaining with zeros
-                for i in op_len..dest_len {
-                    let dest = this.project_index(&dest, i)?;
-                    this.write_scalar(Scalar::from_int(0, dest.layout.size), &dest)?;
-                }
-            }
             // Used to implement the _mm_cvtpd_epi32 and _mm_cvttpd_epi32 functions.
             // Converts packed f64 to packed i32.
             "cvtpd2dq" | "cvttpd2dq" => {
@@ -772,25 +617,6 @@ pub(super) trait EvalContextExt<'mir, 'tcx: 'mir>:
                     )?;
                 }
             }
-            // Used to implement the _mm_movemask_pd function.
-            // Returns a scalar integer where the i-th bit is the highest
-            // bit of the i-th component of `op`.
-            // https://www.felixcloutier.com/x86/movmskpd
-            "movmsk.pd" => {
-                let [op] = this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
-                let (op, op_len) = this.operand_to_simd(op)?;
-
-                let mut res = 0;
-                for i in 0..op_len {
-                    let op = this.read_scalar(&this.project_index(&op, i)?)?;
-                    let op = op.to_u64()?;
-
-                    // Extract the highest bit of `op` and place it in the `i`-th bit of `res`
-                    res |= (op >> 63) << i;
-                }
-
-                this.write_scalar(Scalar::from_u32(res.try_into().unwrap()), dest)?;
-            }
             // Used to implement the `_mm_pause` function.
             // The intrinsic is used to hint the processor that the code is in a spin-loop.
             "pause" => {
diff --git a/src/tools/miri/src/shims/x86/sse3.rs b/src/tools/miri/src/shims/x86/sse3.rs
index a41de5dbf7ee..20a4b560749a 100644
--- a/src/tools/miri/src/shims/x86/sse3.rs
+++ b/src/tools/miri/src/shims/x86/sse3.rs
@@ -23,32 +23,6 @@ pub(super) trait EvalContextExt<'mir, 'tcx: 'mir>:
         let unprefixed_name = link_name.as_str().strip_prefix("llvm.x86.sse3.").unwrap();
 
         match unprefixed_name {
-            // Used to implement the _mm_addsub_ps and _mm_addsub_pd functions.
-            // Alternatingly add and subtract floating point (f32 or f64) from
-            // `left` and `right`
-            "addsub.ps" | "addsub.pd" => {
-                let [left, right] =
-                    this.check_shim(abi, Abi::C { unwind: false }, link_name, args)?;
-
-                let (left, left_len) = this.operand_to_simd(left)?;
-                let (right, right_len) = this.operand_to_simd(right)?;
-                let (dest, dest_len) = this.place_to_simd(dest)?;
-
-                assert_eq!(dest_len, left_len);
-                assert_eq!(dest_len, right_len);
-
-                for i in 0..dest_len {
-                    let left = this.read_immediate(&this.project_index(&left, i)?)?;
-                    let right = this.read_immediate(&this.project_index(&right, i)?)?;
-                    let dest = this.project_index(&dest, i)?;
-
-                    // Even elements are subtracted and odd elements are added.
-                    let op = if i % 2 == 0 { mir::BinOp::Sub } else { mir::BinOp::Add };
-                    let res = this.wrapping_binary_op(op, &left, &right)?;
-
-                    this.write_immediate(*res, &dest)?;
-                }
-            }
             // Used to implement the _mm_h{add,sub}_p{s,d} functions.
             // Horizontally add/subtract adjacent floating point values
             // in `left` and `right`.

From 35e2f4e0af4bfce8802239ac498255ee9b02fbb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eduardo=20S=C3=A1nchez=20Mu=C3=B1oz?= <eduardosm-dev@e64.io>
Date: Tue, 10 Oct 2023 18:44:53 +0200
Subject: [PATCH 020/297] Fix identation of a `rustfmt::skip`ed statement

---
 src/tools/miri/tests/pass/intrinsics-x86-sse2.rs | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/tools/miri/tests/pass/intrinsics-x86-sse2.rs b/src/tools/miri/tests/pass/intrinsics-x86-sse2.rs
index 2c7665bc7363..e636d6c8aaf8 100644
--- a/src/tools/miri/tests/pass/intrinsics-x86-sse2.rs
+++ b/src/tools/miri/tests/pass/intrinsics-x86-sse2.rs
@@ -117,12 +117,12 @@ mod tests {
         #[target_feature(enable = "sse2")]
         unsafe fn test_mm_sad_epu8() {
             #[rustfmt::skip]
-        let a = _mm_setr_epi8(
-            255u8 as i8, 254u8 as i8, 253u8 as i8, 252u8 as i8,
-            1, 2, 3, 4,
-            155u8 as i8, 154u8 as i8, 153u8 as i8, 152u8 as i8,
-            1, 2, 3, 4,
-        );
+            let a = _mm_setr_epi8(
+                255u8 as i8, 254u8 as i8, 253u8 as i8, 252u8 as i8,
+                1, 2, 3, 4,
+                155u8 as i8, 154u8 as i8, 153u8 as i8, 152u8 as i8,
+                1, 2, 3, 4,
+            );
             let b = _mm_setr_epi8(0, 0, 0, 0, 2, 1, 2, 1, 1, 1, 1, 1, 1, 2, 1, 2);
             let r = _mm_sad_epu8(a, b);
             let e = _mm_setr_epi64x(1020, 614);

From 40cc89137b22099699d40230973b124e55ca21c6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Eduardo=20S=C3=A1nchez=20Mu=C3=B1oz?= <eduardosm-dev@e64.io>
Date: Tue, 10 Oct 2023 18:35:28 +0200
Subject: [PATCH 021/297] Remove from cranelift codegen LLVM intrinsics that
 are no longer needed

---
 src/intrinsics/llvm_x86.rs | 35 -----------------------------------
 1 file changed, 35 deletions(-)

diff --git a/src/intrinsics/llvm_x86.rs b/src/intrinsics/llvm_x86.rs
index 0c9a94e1c231..559c64bb13bd 100644
--- a/src/intrinsics/llvm_x86.rs
+++ b/src/intrinsics/llvm_x86.rs
@@ -32,41 +32,6 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, CValue::by_val(res, fx.layout_of(fx.tcx.types.i64)));
         }
 
-        // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8`
-        "llvm.x86.sse2.pmovmskb.128"
-        | "llvm.x86.avx2.pmovmskb"
-        | "llvm.x86.sse.movmsk.ps"
-        | "llvm.x86.sse2.movmsk.pd" => {
-            intrinsic_args!(fx, args => (a); intrinsic);
-
-            let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
-            let lane_ty = fx.clif_type(lane_ty).unwrap();
-            assert!(lane_count <= 32);
-
-            let mut res = fx.bcx.ins().iconst(types::I32, 0);
-
-            for lane in (0..lane_count).rev() {
-                let a_lane = a.value_lane(fx, lane).load_scalar(fx);
-
-                // cast float to int
-                let a_lane = match lane_ty {
-                    types::F32 => codegen_bitcast(fx, types::I32, a_lane),
-                    types::F64 => codegen_bitcast(fx, types::I64, a_lane),
-                    _ => a_lane,
-                };
-
-                // extract sign bit of an int
-                let a_lane_sign = fx.bcx.ins().ushr_imm(a_lane, i64::from(lane_ty.bits() - 1));
-
-                // shift sign bit into result
-                let a_lane_sign = clif_intcast(fx, a_lane_sign, types::I32, false);
-                res = fx.bcx.ins().ishl_imm(res, 1);
-                res = fx.bcx.ins().bor(res, a_lane_sign);
-            }
-
-            let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32));
-            ret.write_cvalue(fx, res);
-        }
         "llvm.x86.sse.cmp.ps" | "llvm.x86.sse2.cmp.pd" => {
             let (x, y, kind) = match args {
                 [x, y, kind] => (x, y, kind),

From 91405ab74a4db7993855071511a02bb25a66a628 Mon Sep 17 00:00:00 2001
From: ltdk <usr@ltdk.xyz>
Date: Wed, 6 Sep 2023 23:24:23 -0400
Subject: [PATCH 022/297] Clean up unchecked_math, separate out
 unchecked_shifts

---
 library/core/src/lib.rs                       |  3 +-
 library/core/src/num/int_macros.rs            | 43 +++++++++++++++----
 library/core/src/num/uint_macros.rs           | 18 ++++----
 .../tests/fail/intrinsics/unchecked_shl.rs    |  2 +-
 .../tests/fail/intrinsics/unchecked_shr.rs    |  2 +-
 tests/codegen/unchecked_shifts.rs             |  2 +-
 tests/mir-opt/inline/unchecked_shifts.rs      |  2 +-
 7 files changed, 49 insertions(+), 23 deletions(-)

diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs
index 906421327cbc..881be8914fe9 100644
--- a/library/core/src/lib.rs
+++ b/library/core/src/lib.rs
@@ -136,7 +136,6 @@
 #![feature(const_hash)]
 #![feature(const_heap)]
 #![feature(const_index_range_slice_index)]
-#![feature(const_inherent_unchecked_arith)]
 #![feature(const_int_unchecked_arith)]
 #![feature(const_intrinsic_forget)]
 #![feature(const_ipv4)]
@@ -190,6 +189,8 @@
 #![feature(str_split_inclusive_remainder)]
 #![feature(str_split_remainder)]
 #![feature(strict_provenance)]
+#![feature(unchecked_math)]
+#![feature(unchecked_shifts)]
 #![feature(utf16_extra)]
 #![feature(utf16_extra_const)]
 #![feature(variant_count)]
diff --git a/library/core/src/num/int_macros.rs b/library/core/src/num/int_macros.rs
index 3cbb55af3bc6..fd01f1b26101 100644
--- a/library/core/src/num/int_macros.rs
+++ b/library/core/src/num/int_macros.rs
@@ -471,7 +471,7 @@ macro_rules! int_impl {
         )]
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
-        #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")]
+        #[rustc_const_unstable(feature = "unchecked_math", issue = "85122")]
         #[inline(always)]
         #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
         pub const unsafe fn unchecked_add(self, rhs: Self) -> Self {
@@ -539,7 +539,7 @@ macro_rules! int_impl {
         )]
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
-        #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")]
+        #[rustc_const_unstable(feature = "unchecked_math", issue = "85122")]
         #[inline(always)]
         #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
         pub const unsafe fn unchecked_sub(self, rhs: Self) -> Self {
@@ -607,7 +607,7 @@ macro_rules! int_impl {
         )]
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
-        #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")]
+        #[rustc_const_unstable(feature = "unchecked_math", issue = "85122")]
         #[inline(always)]
         #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
         pub const unsafe fn unchecked_mul(self, rhs: Self) -> Self {
@@ -740,6 +740,31 @@ macro_rules! int_impl {
             if unlikely!(b) {None} else {Some(a)}
         }
 
+        /// Unchecked negation. Computes `-self`, assuming overflow cannot occur.
+        ///
+        /// # Safety
+        ///
+        /// This results in undefined behavior when
+        #[doc = concat!("`self == ", stringify!($SelfT), "::MIN`,")]
+        /// i.e. when [`checked_neg`] would return `None`.
+        ///
+        #[doc = concat!("[`checked_neg`]: ", stringify!($SelfT), "::checked_neg")]
+        #[unstable(
+            feature = "unchecked_neg",
+            reason = "niche optimization path",
+            issue = "85122",
+        )]
+        #[must_use = "this returns the result of the operation, \
+                      without modifying the original"]
+        #[rustc_const_unstable(feature = "unchecked_neg", issue = "85122")]
+        #[inline(always)]
+        #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
+        pub const unsafe fn unchecked_neg(self) -> Self {
+            // SAFETY: the caller must uphold the safety contract for
+            // `unchecked_neg`.
+            unsafe { intrinsics::unchecked_sub(0, self) }
+        }
+
         /// Checked shift left. Computes `self << rhs`, returning `None` if `rhs` is larger
         /// than or equal to the number of bits in `self`.
         ///
@@ -772,13 +797,13 @@ macro_rules! int_impl {
         ///
         #[doc = concat!("[`checked_shl`]: ", stringify!($SelfT), "::checked_shl")]
         #[unstable(
-            feature = "unchecked_math",
+            feature = "unchecked_shifts",
             reason = "niche optimization path",
             issue = "85122",
         )]
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
-        #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")]
+        #[rustc_const_unstable(feature = "unchecked_shifts", issue = "85122")]
         #[inline(always)]
         #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
         pub const unsafe fn unchecked_shl(self, rhs: u32) -> Self {
@@ -820,13 +845,13 @@ macro_rules! int_impl {
         ///
         #[doc = concat!("[`checked_shr`]: ", stringify!($SelfT), "::checked_shr")]
         #[unstable(
-            feature = "unchecked_math",
+            feature = "unchecked_shifts",
             reason = "niche optimization path",
             issue = "85122",
         )]
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
-        #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")]
+        #[rustc_const_unstable(feature = "unchecked_shifts", issue = "85122")]
         #[inline(always)]
         #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
         pub const unsafe fn unchecked_shr(self, rhs: u32) -> Self {
@@ -1404,7 +1429,7 @@ macro_rules! int_impl {
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
         #[inline(always)]
-        #[rustc_allow_const_fn_unstable(const_inherent_unchecked_arith)]
+        #[rustc_allow_const_fn_unstable(unchecked_shifts)]
         pub const fn wrapping_shl(self, rhs: u32) -> Self {
             // SAFETY: the masking by the bitsize of the type ensures that we do not shift
             // out of bounds
@@ -1434,7 +1459,7 @@ macro_rules! int_impl {
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
         #[inline(always)]
-        #[rustc_allow_const_fn_unstable(const_inherent_unchecked_arith)]
+        #[rustc_allow_const_fn_unstable(unchecked_shifts)]
         pub const fn wrapping_shr(self, rhs: u32) -> Self {
             // SAFETY: the masking by the bitsize of the type ensures that we do not shift
             // out of bounds
diff --git a/library/core/src/num/uint_macros.rs b/library/core/src/num/uint_macros.rs
index a9c5312a1c02..11a53aaf122e 100644
--- a/library/core/src/num/uint_macros.rs
+++ b/library/core/src/num/uint_macros.rs
@@ -479,7 +479,7 @@ macro_rules! uint_impl {
         )]
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
-        #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")]
+        #[rustc_const_unstable(feature = "unchecked_math", issue = "85122")]
         #[inline(always)]
         #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
         pub const unsafe fn unchecked_add(self, rhs: Self) -> Self {
@@ -548,7 +548,7 @@ macro_rules! uint_impl {
         )]
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
-        #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")]
+        #[rustc_const_unstable(feature = "unchecked_math", issue = "85122")]
         #[inline(always)]
         #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
         pub const unsafe fn unchecked_sub(self, rhs: Self) -> Self {
@@ -595,7 +595,7 @@ macro_rules! uint_impl {
         )]
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
-        #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")]
+        #[rustc_const_unstable(feature = "unchecked_math", issue = "85122")]
         #[inline(always)]
         #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
         pub const unsafe fn unchecked_mul(self, rhs: Self) -> Self {
@@ -926,13 +926,13 @@ macro_rules! uint_impl {
         ///
         #[doc = concat!("[`checked_shl`]: ", stringify!($SelfT), "::checked_shl")]
         #[unstable(
-            feature = "unchecked_math",
+            feature = "unchecked_shifts",
             reason = "niche optimization path",
             issue = "85122",
         )]
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
-        #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")]
+        #[rustc_const_unstable(feature = "unchecked_shifts", issue = "85122")]
         #[inline(always)]
         #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
         pub const unsafe fn unchecked_shl(self, rhs: u32) -> Self {
@@ -974,13 +974,13 @@ macro_rules! uint_impl {
         ///
         #[doc = concat!("[`checked_shr`]: ", stringify!($SelfT), "::checked_shr")]
         #[unstable(
-            feature = "unchecked_math",
+            feature = "unchecked_shifts",
             reason = "niche optimization path",
             issue = "85122",
         )]
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
-        #[rustc_const_unstable(feature = "const_inherent_unchecked_arith", issue = "85122")]
+        #[rustc_const_unstable(feature = "unchecked_shifts", issue = "85122")]
         #[inline(always)]
         #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces
         pub const unsafe fn unchecked_shr(self, rhs: u32) -> Self {
@@ -1418,7 +1418,7 @@ macro_rules! uint_impl {
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
         #[inline(always)]
-        #[rustc_allow_const_fn_unstable(const_inherent_unchecked_arith)]
+        #[rustc_allow_const_fn_unstable(unchecked_shifts)]
         pub const fn wrapping_shl(self, rhs: u32) -> Self {
             // SAFETY: the masking by the bitsize of the type ensures that we do not shift
             // out of bounds
@@ -1451,7 +1451,7 @@ macro_rules! uint_impl {
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
         #[inline(always)]
-        #[rustc_allow_const_fn_unstable(const_inherent_unchecked_arith)]
+        #[rustc_allow_const_fn_unstable(unchecked_shifts)]
         pub const fn wrapping_shr(self, rhs: u32) -> Self {
             // SAFETY: the masking by the bitsize of the type ensures that we do not shift
             // out of bounds
diff --git a/src/tools/miri/tests/fail/intrinsics/unchecked_shl.rs b/src/tools/miri/tests/fail/intrinsics/unchecked_shl.rs
index 4554d0cb82ba..abb345938fa0 100644
--- a/src/tools/miri/tests/fail/intrinsics/unchecked_shl.rs
+++ b/src/tools/miri/tests/fail/intrinsics/unchecked_shl.rs
@@ -1,4 +1,4 @@
-#![feature(unchecked_math)]
+#![feature(unchecked_shifts)]
 
 fn main() {
     unsafe {
diff --git a/src/tools/miri/tests/fail/intrinsics/unchecked_shr.rs b/src/tools/miri/tests/fail/intrinsics/unchecked_shr.rs
index fe2e85be6986..cdc10185e470 100644
--- a/src/tools/miri/tests/fail/intrinsics/unchecked_shr.rs
+++ b/src/tools/miri/tests/fail/intrinsics/unchecked_shr.rs
@@ -1,4 +1,4 @@
-#![feature(unchecked_math)]
+#![feature(unchecked_shifts)]
 
 fn main() {
     unsafe {
diff --git a/tests/codegen/unchecked_shifts.rs b/tests/codegen/unchecked_shifts.rs
index d5f53bedd543..aca9bec77dfe 100644
--- a/tests/codegen/unchecked_shifts.rs
+++ b/tests/codegen/unchecked_shifts.rs
@@ -2,7 +2,7 @@
 // ignore-debug (because unchecked is checked in debug)
 
 #![crate_type = "lib"]
-#![feature(unchecked_math)]
+#![feature(unchecked_shifts)]
 
 // CHECK-LABEL: @unchecked_shl_unsigned_same
 #[no_mangle]
diff --git a/tests/mir-opt/inline/unchecked_shifts.rs b/tests/mir-opt/inline/unchecked_shifts.rs
index 22f84e44a641..ffb7b079ce24 100644
--- a/tests/mir-opt/inline/unchecked_shifts.rs
+++ b/tests/mir-opt/inline/unchecked_shifts.rs
@@ -1,6 +1,6 @@
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 #![crate_type = "lib"]
-#![feature(unchecked_math)]
+#![feature(unchecked_shifts)]
 
 // ignore-debug: the debug assertions prevent the inlining we are testing for
 // compile-flags: -Zmir-opt-level=2 -Zinline-mir

From e3998b2d4633705eb4da278dcf5127f273e388ea Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Thu, 12 Oct 2023 17:06:29 -0400
Subject: [PATCH 023/297] Handle unsigned comparison for signed integers

---
 src/int.rs | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/src/int.rs b/src/int.rs
index 58e0dd56f38d..5719f6a8cf50 100644
--- a/src/int.rs
+++ b/src/int.rs
@@ -415,6 +415,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
                     IntPredicate::IntNE => {
                         return self.context.new_comparison(None, ComparisonOp::NotEquals, cmp, self.context.new_rvalue_one(self.int_type));
                     },
+                    // TODO(antoyo): cast to u128 for unsigned comparison. See below.
                     IntPredicate::IntUGT => (ComparisonOp::Equals, 2),
                     IntPredicate::IntUGE => (ComparisonOp::GreaterThanEquals, 1),
                     IntPredicate::IntULT => (ComparisonOp::Equals, 0),
@@ -444,6 +445,18 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
                     rhs = self.context.new_cast(None, rhs, a_type);
                 }
             }
+            match op {
+                IntPredicate::IntUGT | IntPredicate::IntUGE | IntPredicate::IntULT | IntPredicate::IntULE => {
+                    if !a_type.is_vector() {
+                        let unsigned_type = a_type.to_unsigned(&self.cx);
+                        lhs = self.context.new_cast(None, lhs, unsigned_type);
+                        rhs = self.context.new_cast(None, rhs, unsigned_type);
+                    }
+                },
+                // TODO(antoyo): we probably need to handle signed comparison for unsigned
+                // integers.
+                _ => (),
+            }
             self.context.new_comparison(None, op.to_gcc_comparison(), lhs, rhs)
         }
     }

From 184d5ef1077869c58479d0c3fcab404b99090ed3 Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Thu, 21 Sep 2023 04:11:08 +0000
Subject: [PATCH 024/297] Consider alias bounds when considering lliveness for
 alias types in NLL

---
 .../src/type_check/liveness/trace.rs          | 97 +++++++++++++++----
 .../ui/borrowck/alias-liveness/rpit-static.rs | 14 +++
 .../borrowck/alias-liveness/rpitit-static.rs  | 12 +++
 3 files changed, 103 insertions(+), 20 deletions(-)
 create mode 100644 tests/ui/borrowck/alias-liveness/rpit-static.rs
 create mode 100644 tests/ui/borrowck/alias-liveness/rpitit-static.rs

diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
index 21da05c32dd8..432c7a120719 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
@@ -5,10 +5,13 @@ use rustc_index::interval::IntervalSet;
 use rustc_infer::infer::canonical::QueryRegionConstraints;
 use rustc_middle::mir::{BasicBlock, Body, ConstraintCategory, Local, Location};
 use rustc_middle::traits::query::DropckOutlivesResult;
-use rustc_middle::ty::{RegionVid, Ty, TyCtxt, TypeVisitable, TypeVisitableExt};
+use rustc_middle::ty::{
+    self, RegionVid, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor,
+};
 use rustc_span::DUMMY_SP;
 use rustc_trait_selection::traits::query::type_op::outlives::DropckOutlives;
 use rustc_trait_selection::traits::query::type_op::{TypeOp, TypeOpOutput};
+use std::ops::ControlFlow;
 use std::rc::Rc;
 
 use rustc_mir_dataflow::impls::MaybeInitializedPlaces;
@@ -601,34 +604,88 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
             values::location_set_str(elements, live_at.iter()),
         );
 
-        let tcx = typeck.tcx();
-        let borrowck_context = &mut typeck.borrowck_context;
-
         // When using `-Zpolonius=next`, we want to record the loans that flow into this value's
         // regions as being live at the given `live_at` points: this will be used to compute the
         // location where a loan goes out of scope.
-        let num_loans = borrowck_context.borrow_set.len();
-        let mut value_loans = HybridBitSet::new_empty(num_loans);
+        let num_loans = typeck.borrowck_context.borrow_set.len();
+        let value_loans = &mut HybridBitSet::new_empty(num_loans);
 
-        tcx.for_each_free_region(&value, |live_region| {
-            let live_region_vid = borrowck_context.universal_regions.to_region_vid(live_region);
-
-            borrowck_context
-                .constraints
-                .liveness_constraints
-                .add_elements(live_region_vid, live_at);
-
-            // There can only be inflowing loans for this region when we are using
-            // `-Zpolonius=next`.
-            if let Some(inflowing) = inflowing_loans.row(live_region_vid) {
-                value_loans.union(inflowing);
+        struct MakeAllRegionsLive<'a, 'b, 'tcx> {
+            typeck: &'b mut TypeChecker<'a, 'tcx>,
+            live_at: &'b IntervalSet<PointIndex>,
+            value_loans: &'b mut HybridBitSet<BorrowIndex>,
+            inflowing_loans: &'b SparseBitMatrix<RegionVid, BorrowIndex>,
+        }
+        impl<'tcx> MakeAllRegionsLive<'_, '_, 'tcx> {
+            fn make_alias_live(&mut self, t: Ty<'tcx>) -> ControlFlow<!> {
+                let ty::Alias(_kind, alias_ty) = t.kind() else {
+                    bug!();
+                };
+                let tcx = self.typeck.infcx.tcx;
+                let mut outlives_bounds = tcx
+                    .item_bounds(alias_ty.def_id)
+                    .iter_instantiated(tcx, alias_ty.args)
+                    .filter_map(|clause| {
+                        if let Some(outlives) = clause.as_type_outlives_clause()
+                        && outlives.skip_binder().0 == t
+                    {
+                        Some(outlives.skip_binder().1)
+                    } else {
+                        None
+                    }
+                    });
+                if let Some(r) = outlives_bounds.next()
+                    && !r.is_late_bound()
+                    && outlives_bounds.all(|other_r| {
+                        other_r == r
+                    })
+                {
+                    r.visit_with(self)
+                } else {
+                    t.super_visit_with(self)
+                }
             }
-        });
+        }
+        impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for MakeAllRegionsLive<'_, '_, 'tcx> {
+            type BreakTy = !;
+
+            fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
+                if r.is_late_bound() {
+                    return ControlFlow::Continue(());
+                }
+                let live_region_vid =
+                    self.typeck.borrowck_context.universal_regions.to_region_vid(r);
+
+                self.typeck
+                    .borrowck_context
+                    .constraints
+                    .liveness_constraints
+                    .add_elements(live_region_vid, self.live_at);
+
+                // There can only be inflowing loans for this region when we are using
+                // `-Zpolonius=next`.
+                if let Some(inflowing) = self.inflowing_loans.row(live_region_vid) {
+                    self.value_loans.union(inflowing);
+                }
+                ControlFlow::Continue(())
+            }
+
+            fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
+                if !t.has_free_regions() {
+                    ControlFlow::Continue(())
+                } else if let ty::Alias(..) = t.kind() {
+                    self.make_alias_live(t)
+                } else {
+                    t.super_visit_with(self)
+                }
+            }
+        }
+        value.visit_with(&mut MakeAllRegionsLive { typeck, live_at, value_loans, inflowing_loans });
 
         // Record the loans reaching the value.
         if !value_loans.is_empty() {
             for point in live_at.iter() {
-                borrowck_context.live_loans.union_row(point, &value_loans);
+                typeck.borrowck_context.live_loans.union_row(point, value_loans);
             }
         }
     }
diff --git a/tests/ui/borrowck/alias-liveness/rpit-static.rs b/tests/ui/borrowck/alias-liveness/rpit-static.rs
new file mode 100644
index 000000000000..8637d5fb720e
--- /dev/null
+++ b/tests/ui/borrowck/alias-liveness/rpit-static.rs
@@ -0,0 +1,14 @@
+// check-pass
+
+trait Captures<'a> {}
+impl<T> Captures<'_> for T {}
+
+fn foo(x: &i32) -> impl Sized + Captures<'_> + 'static {}
+
+fn main() {
+    let y;
+    {
+        let x = 1;
+        y = foo(&x);
+    }
+}
diff --git a/tests/ui/borrowck/alias-liveness/rpitit-static.rs b/tests/ui/borrowck/alias-liveness/rpitit-static.rs
new file mode 100644
index 000000000000..ed7b496a42ba
--- /dev/null
+++ b/tests/ui/borrowck/alias-liveness/rpitit-static.rs
@@ -0,0 +1,12 @@
+// check-pass
+
+trait Foo {
+    fn rpitit(&mut self) -> impl Sized + 'static;
+}
+
+fn test<T: Foo>(mut t: T) {
+    let a = t.rpitit();
+    let b = t.rpitit();
+}
+
+fn main() {}

From e425d85518b4e3c2dc6edfccab638db26380198a Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Thu, 21 Sep 2023 18:02:22 +0000
Subject: [PATCH 025/297] Consider param-env candidates, too

---
 .../src/type_check/liveness/trace.rs          | 22 ++++++++++++++++++-
 .../ui/borrowck/alias-liveness/gat-static.rs  | 20 +++++++++++++++++
 2 files changed, 41 insertions(+), 1 deletion(-)
 create mode 100644 tests/ui/borrowck/alias-liveness/gat-static.rs

diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
index 432c7a120719..d0078f71cd81 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
@@ -3,6 +3,8 @@ use rustc_data_structures::graph::WithSuccessors;
 use rustc_index::bit_set::{HybridBitSet, SparseBitMatrix};
 use rustc_index::interval::IntervalSet;
 use rustc_infer::infer::canonical::QueryRegionConstraints;
+use rustc_infer::infer::outlives::test_type_match;
+use rustc_infer::infer::region_constraints::VerifyIfEq;
 use rustc_middle::mir::{BasicBlock, Body, ConstraintCategory, Local, Location};
 use rustc_middle::traits::query::DropckOutlivesResult;
 use rustc_middle::ty::{
@@ -622,6 +624,7 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
                     bug!();
                 };
                 let tcx = self.typeck.infcx.tcx;
+                let param_env = self.typeck.param_env;
                 let mut outlives_bounds = tcx
                     .item_bounds(alias_ty.def_id)
                     .iter_instantiated(tcx, alias_ty.args)
@@ -633,7 +636,24 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
                     } else {
                         None
                     }
-                    });
+                    })
+                    .chain(param_env.caller_bounds().iter().filter_map(|clause| {
+                        let outlives = clause.as_type_outlives_clause()?;
+                        if let Some(outlives) = outlives.no_bound_vars()
+                            && outlives.0 == t
+                        {
+                            Some(outlives.1)
+                        } else {
+                            test_type_match::extract_verify_if_eq(
+                                tcx,
+                                param_env,
+                                &outlives.map_bound(|ty::OutlivesPredicate(ty, bound)| {
+                                    VerifyIfEq { ty, bound }
+                                }),
+                                t,
+                            )
+                        }
+                    }));
                 if let Some(r) = outlives_bounds.next()
                     && !r.is_late_bound()
                     && outlives_bounds.all(|other_r| {
diff --git a/tests/ui/borrowck/alias-liveness/gat-static.rs b/tests/ui/borrowck/alias-liveness/gat-static.rs
new file mode 100644
index 000000000000..1ab40fe828c3
--- /dev/null
+++ b/tests/ui/borrowck/alias-liveness/gat-static.rs
@@ -0,0 +1,20 @@
+// check-pass
+
+trait Foo {
+    type Assoc<'a>
+    where
+        Self: 'a;
+
+    fn assoc(&mut self) -> Self::Assoc<'_>;
+}
+
+fn test<T>(mut t: T)
+where
+    T: Foo,
+    for<'a> T::Assoc<'a>: 'static,
+{
+    let a = t.assoc();
+    let b = t.assoc();
+}
+
+fn main() {}

From 371d8a8215597dbdb048552d7bc6962d2a95909b Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Thu, 21 Sep 2023 05:51:40 +0000
Subject: [PATCH 026/297] Consider static specially

---
 .../src/type_check/liveness/trace.rs          | 28 +++++++++++++------
 .../ui/borrowck/alias-liveness/rtn-static.rs  | 17 +++++++++++
 .../borrowck/alias-liveness/rtn-static.stderr | 11 ++++++++
 3 files changed, 48 insertions(+), 8 deletions(-)
 create mode 100644 tests/ui/borrowck/alias-liveness/rtn-static.rs
 create mode 100644 tests/ui/borrowck/alias-liveness/rtn-static.stderr

diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
index d0078f71cd81..e0eed812b6ad 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
@@ -619,13 +619,21 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
             inflowing_loans: &'b SparseBitMatrix<RegionVid, BorrowIndex>,
         }
         impl<'tcx> MakeAllRegionsLive<'_, '_, 'tcx> {
+            /// We can prove that an alias is live two ways:
+            /// 1. All the components are live.
+            /// 2. There is a known outlives bound or where-clause, and that
+            ///    region is live.
+            /// We search through the item bounds and where clauses for
+            /// either `'static` or a unique outlives region, and if one is
+            /// found, we just need to prove that that region is still live.
+            /// If one is not found, then we continue to walk through the alias.
             fn make_alias_live(&mut self, t: Ty<'tcx>) -> ControlFlow<!> {
                 let ty::Alias(_kind, alias_ty) = t.kind() else {
-                    bug!();
+                    bug!("`make_alias_live` only takes alias types");
                 };
                 let tcx = self.typeck.infcx.tcx;
                 let param_env = self.typeck.param_env;
-                let mut outlives_bounds = tcx
+                let outlives_bounds: Vec<_> = tcx
                     .item_bounds(alias_ty.def_id)
                     .iter_instantiated(tcx, alias_ty.args)
                     .filter_map(|clause| {
@@ -653,12 +661,16 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
                                 t,
                             )
                         }
-                    }));
-                if let Some(r) = outlives_bounds.next()
-                    && !r.is_late_bound()
-                    && outlives_bounds.all(|other_r| {
-                        other_r == r
-                    })
+                    }))
+                    .collect();
+                // If we find `'static`, then we know the alias doesn't capture *any* regions.
+                // Otherwise, all of the outlives regions should be equal -- if they're not,
+                // we don't really know how to proceed, so we continue recursing through the
+                // alias.
+                if outlives_bounds.contains(&tcx.lifetimes.re_static) {
+                    ControlFlow::Continue(())
+                } else if let Some(r) = outlives_bounds.first()
+                    && outlives_bounds[1..].iter().all(|other_r| other_r == r)
                 {
                     r.visit_with(self)
                 } else {
diff --git a/tests/ui/borrowck/alias-liveness/rtn-static.rs b/tests/ui/borrowck/alias-liveness/rtn-static.rs
new file mode 100644
index 000000000000..c850f0122e5a
--- /dev/null
+++ b/tests/ui/borrowck/alias-liveness/rtn-static.rs
@@ -0,0 +1,17 @@
+// check-pass
+
+#![feature(return_type_notation)]
+//~^ WARN the feature `return_type_notation` is incomplete
+
+trait Foo {
+    fn borrow(&mut self) -> impl Sized + '_;
+}
+
+// Test that the `'_` item bound in `borrow` does not cause us to
+// overlook the `'static` RTN bound.
+fn test<T: Foo<borrow(): 'static>>(mut t: T) {
+    let x = t.borrow();
+    let x = t.borrow();
+}
+
+fn main() {}
diff --git a/tests/ui/borrowck/alias-liveness/rtn-static.stderr b/tests/ui/borrowck/alias-liveness/rtn-static.stderr
new file mode 100644
index 000000000000..e9202db2c790
--- /dev/null
+++ b/tests/ui/borrowck/alias-liveness/rtn-static.stderr
@@ -0,0 +1,11 @@
+warning: the feature `return_type_notation` is incomplete and may not be safe to use and/or cause compiler crashes
+  --> $DIR/rtn-static.rs:3:12
+   |
+LL | #![feature(return_type_notation)]
+   |            ^^^^^^^^^^^^^^^^^^^^
+   |
+   = note: see issue #109417 <https://github.com/rust-lang/rust/issues/109417> for more information
+   = note: `#[warn(incomplete_features)]` on by default
+
+warning: 1 warning emitted
+

From 3733316c6b5bc2d66c2b5ce4a426038315956dff Mon Sep 17 00:00:00 2001
From: Chris Denton <chris@chrisdenton.dev>
Date: Mon, 16 Oct 2023 18:27:12 +0100
Subject: [PATCH 027/297] Create `windows/api.rs` for safer FFI

---
 library/std/src/sys/windows/api.rs            | 157 ++++++++++++++++++
 library/std/src/sys/windows/c/windows_sys.lst |   2 +
 library/std/src/sys/windows/c/windows_sys.rs  |  21 +++
 library/std/src/sys/windows/fs.rs             |  56 ++-----
 library/std/src/sys/windows/mod.rs            |  16 +-
 library/std/src/sys/windows/os.rs             |   6 +-
 library/std/src/sys/windows/stack_overflow.rs |   4 +-
 library/std/src/sys/windows/stdio.rs          |   3 +-
 8 files changed, 212 insertions(+), 53 deletions(-)
 create mode 100644 library/std/src/sys/windows/api.rs

diff --git a/library/std/src/sys/windows/api.rs b/library/std/src/sys/windows/api.rs
new file mode 100644
index 000000000000..e9f0bbfbe2e7
--- /dev/null
+++ b/library/std/src/sys/windows/api.rs
@@ -0,0 +1,157 @@
+//! # Safe(r) wrappers around Windows API functions.
+//!
+//! This module contains fairly thin wrappers around Windows API functions,
+//! aimed at centralising safety instead of having unsafe blocks spread
+//! throughout higher level code. This makes it much easier to audit FFI safety.
+//!
+//! Not all functions can be made completely safe without more context but in
+//! such cases we should still endeavour to reduce the caller's burden of safety
+//! as much as possible.
+//!
+//! ## Guidelines for wrappers
+//!
+//! Items here should be named similarly to their raw Windows API name, except
+//! that they follow Rust's case conventions. E.g. function names are
+//! lower_snake_case. The idea here is that it should be easy for a Windows
+//! C/C++ programmer to identify the underlying function that's being wrapped
+//! while not looking too out of place in Rust code.
+//!
+//! Every use of an `unsafe` block must have a related SAFETY comment, even if
+//! it's trivially safe (for example, see `get_last_error`). Public unsafe
+//! functions must document what the caller has to do to call them safely.
+//!
+//! Avoid unchecked `as` casts. For integers, either assert that the integer
+//! is in range or use `try_into` instead. For pointers, prefer to use
+//! `ptr.cast::<Type>()` when possible.
+//!
+//! This module must only depend on core and not on std types as the eventual
+//! hope is to have std depend on sys and not the other way around.
+//! However, some amount of glue code may currently be necessary so such code
+//! should go in sys/windows/mod.rs rather than here. See `IoResult` as an example.
+
+use core::ffi::c_void;
+use core::ptr::addr_of;
+
+use super::c;
+
+/// Helper method for getting the size of `T` as a u32.
+/// Errors at compile time if the size would overflow.
+///
+/// While a type larger than u32::MAX is unlikely, it is possible if only because of a bug.
+/// However, one key motivation for this function is to avoid the temptation to
+/// use frequent `as` casts. This is risky because they are too powerful.
+/// For example, the following will compile today:
+///
+/// `std::mem::size_of::<u64> as u32`
+///
+/// Note that `size_of` is never actually called, instead a function pointer is
+/// converted to a `u32`. Clippy would warn about this but, alas, it's not run
+/// on the standard library.
+const fn win32_size_of<T: Sized>() -> u32 {
+    // Const assert that the size is less than u32::MAX.
+    // Uses a trait to workaround restriction on using generic types in inner items.
+    trait Win32SizeOf: Sized {
+        const WIN32_SIZE_OF: u32 = {
+            let size = core::mem::size_of::<Self>();
+            assert!(size <= u32::MAX as usize);
+            size as u32
+        };
+    }
+    impl<T: Sized> Win32SizeOf for T {}
+
+    T::WIN32_SIZE_OF
+}
+
+/// The `SetFileInformationByHandle` function takes a generic parameter by
+/// making the user specify the type (class), a pointer to the data and its
+/// size. This trait allows attaching that information to a Rust type so that
+/// [`set_file_information_by_handle`] can be called safely.
+///
+/// This trait is designed so that it can support variable sized types.
+/// However, currently Rust's std only uses fixed sized structures.
+///
+/// # Safety
+///
+/// * `as_ptr` must return a pointer to memory that is readable up to `size` bytes.
+/// * `CLASS` must accurately reflect the type pointed to by `as_ptr`. E.g.
+/// the `FILE_BASIC_INFO` structure has the class `FileBasicInfo`.
+pub unsafe trait SetFileInformation {
+    /// The type of information to set.
+    const CLASS: i32;
+    /// A pointer to the file information to set.
+    fn as_ptr(&self) -> *const c_void;
+    /// The size of the type pointed to by `as_ptr`.
+    fn size(&self) -> u32;
+}
+/// Helper trait for implementing `SetFileInformation` for statically sized types.
+unsafe trait SizedSetFileInformation: Sized {
+    const CLASS: i32;
+}
+unsafe impl<T: SizedSetFileInformation> SetFileInformation for T {
+    const CLASS: i32 = T::CLASS;
+    fn as_ptr(&self) -> *const c_void {
+        addr_of!(*self).cast::<c_void>()
+    }
+    fn size(&self) -> u32 {
+        win32_size_of::<Self>()
+    }
+}
+
+// SAFETY: FILE_BASIC_INFO, FILE_END_OF_FILE_INFO, FILE_ALLOCATION_INFO,
+// FILE_DISPOSITION_INFO, FILE_DISPOSITION_INFO_EX and FILE_IO_PRIORITY_HINT_INFO
+// are all plain `repr(C)` structs that only contain primitive types.
+// The given information classes correctly match with the struct.
+unsafe impl SizedSetFileInformation for c::FILE_BASIC_INFO {
+    const CLASS: i32 = c::FileBasicInfo;
+}
+unsafe impl SizedSetFileInformation for c::FILE_END_OF_FILE_INFO {
+    const CLASS: i32 = c::FileEndOfFileInfo;
+}
+unsafe impl SizedSetFileInformation for c::FILE_ALLOCATION_INFO {
+    const CLASS: i32 = c::FileAllocationInfo;
+}
+unsafe impl SizedSetFileInformation for c::FILE_DISPOSITION_INFO {
+    const CLASS: i32 = c::FileDispositionInfo;
+}
+unsafe impl SizedSetFileInformation for c::FILE_DISPOSITION_INFO_EX {
+    const CLASS: i32 = c::FileDispositionInfoEx;
+}
+unsafe impl SizedSetFileInformation for c::FILE_IO_PRIORITY_HINT_INFO {
+    const CLASS: i32 = c::FileIoPriorityHintInfo;
+}
+
+#[inline]
+pub fn set_file_information_by_handle<T: SetFileInformation>(
+    handle: c::HANDLE,
+    info: &T,
+) -> Result<(), WinError> {
+    unsafe fn set_info(
+        handle: c::HANDLE,
+        class: i32,
+        info: *const c_void,
+        size: u32,
+    ) -> Result<(), WinError> {
+        let result = c::SetFileInformationByHandle(handle, class, info, size);
+        (result != 0).then_some(()).ok_or_else(|| get_last_error())
+    }
+    // SAFETY: The `SetFileInformation` trait ensures that this is safe.
+    unsafe { set_info(handle, T::CLASS, info.as_ptr(), info.size()) }
+}
+
+/// Gets the error from the last function.
+/// This must be called immediately after the function that sets the error to
+/// avoid the risk of another function overwriting it.
+pub fn get_last_error() -> WinError {
+    // SAFETY: This just returns a thread-local u32 and has no other effects.
+    unsafe { WinError { code: c::GetLastError() } }
+}
+
+/// An error code as returned by [`get_last_error`].
+///
+/// This is usually a 16-bit Win32 error code but may be a 32-bit HRESULT or NTSTATUS.
+/// Check the documentation of the Windows API function being called for expected errors.
+#[derive(Clone, Copy, PartialEq, Eq)]
+#[repr(transparent)]
+pub struct WinError {
+    pub code: u32,
+}
diff --git a/library/std/src/sys/windows/c/windows_sys.lst b/library/std/src/sys/windows/c/windows_sys.lst
index dad4a4223b2a..bc5bfe80fb10 100644
--- a/library/std/src/sys/windows/c/windows_sys.lst
+++ b/library/std/src/sys/windows/c/windows_sys.lst
@@ -2224,6 +2224,7 @@ Windows.Win32.Storage.FileSystem.FILE_ACCESS_RIGHTS
 Windows.Win32.Storage.FileSystem.FILE_ADD_FILE
 Windows.Win32.Storage.FileSystem.FILE_ADD_SUBDIRECTORY
 Windows.Win32.Storage.FileSystem.FILE_ALL_ACCESS
+Windows.Win32.Storage.FileSystem.FILE_ALLOCATION_INFO
 Windows.Win32.Storage.FileSystem.FILE_APPEND_DATA
 Windows.Win32.Storage.FileSystem.FILE_ATTRIBUTE_ARCHIVE
 Windows.Win32.Storage.FileSystem.FILE_ATTRIBUTE_COMPRESSED
@@ -2284,6 +2285,7 @@ Windows.Win32.Storage.FileSystem.FILE_GENERIC_READ
 Windows.Win32.Storage.FileSystem.FILE_GENERIC_WRITE
 Windows.Win32.Storage.FileSystem.FILE_ID_BOTH_DIR_INFO
 Windows.Win32.Storage.FileSystem.FILE_INFO_BY_HANDLE_CLASS
+Windows.Win32.Storage.FileSystem.FILE_IO_PRIORITY_HINT_INFO
 Windows.Win32.Storage.FileSystem.FILE_LIST_DIRECTORY
 Windows.Win32.Storage.FileSystem.FILE_NAME_NORMALIZED
 Windows.Win32.Storage.FileSystem.FILE_NAME_OPENED
diff --git a/library/std/src/sys/windows/c/windows_sys.rs b/library/std/src/sys/windows/c/windows_sys.rs
index 20b44966a39f..d66dfda6b8e0 100644
--- a/library/std/src/sys/windows/c/windows_sys.rs
+++ b/library/std/src/sys/windows/c/windows_sys.rs
@@ -3107,6 +3107,16 @@ impl ::core::clone::Clone for FILETIME {
 pub type FILE_ACCESS_RIGHTS = u32;
 pub const FILE_ADD_FILE: FILE_ACCESS_RIGHTS = 2u32;
 pub const FILE_ADD_SUBDIRECTORY: FILE_ACCESS_RIGHTS = 4u32;
+#[repr(C)]
+pub struct FILE_ALLOCATION_INFO {
+    pub AllocationSize: i64,
+}
+impl ::core::marker::Copy for FILE_ALLOCATION_INFO {}
+impl ::core::clone::Clone for FILE_ALLOCATION_INFO {
+    fn clone(&self) -> Self {
+        *self
+    }
+}
 pub const FILE_ALL_ACCESS: FILE_ACCESS_RIGHTS = 2032127u32;
 pub const FILE_APPEND_DATA: FILE_ACCESS_RIGHTS = 4u32;
 pub const FILE_ATTRIBUTE_ARCHIVE: FILE_FLAGS_AND_ATTRIBUTES = 32u32;
@@ -3248,6 +3258,16 @@ impl ::core::clone::Clone for FILE_ID_BOTH_DIR_INFO {
     }
 }
 pub type FILE_INFO_BY_HANDLE_CLASS = i32;
+#[repr(C)]
+pub struct FILE_IO_PRIORITY_HINT_INFO {
+    pub PriorityHint: PRIORITY_HINT,
+}
+impl ::core::marker::Copy for FILE_IO_PRIORITY_HINT_INFO {}
+impl ::core::clone::Clone for FILE_IO_PRIORITY_HINT_INFO {
+    fn clone(&self) -> Self {
+        *self
+    }
+}
 pub const FILE_LIST_DIRECTORY: FILE_ACCESS_RIGHTS = 1u32;
 pub const FILE_NAME_NORMALIZED: GETFINALPATHNAMEBYHANDLE_FLAGS = 0u32;
 pub const FILE_NAME_OPENED: GETFINALPATHNAMEBYHANDLE_FLAGS = 8u32;
@@ -3753,6 +3773,7 @@ pub const PIPE_SERVER_END: NAMED_PIPE_MODE = 1u32;
 pub const PIPE_TYPE_BYTE: NAMED_PIPE_MODE = 0u32;
 pub const PIPE_TYPE_MESSAGE: NAMED_PIPE_MODE = 4u32;
 pub const PIPE_WAIT: NAMED_PIPE_MODE = 0u32;
+pub type PRIORITY_HINT = i32;
 pub type PROCESSOR_ARCHITECTURE = u16;
 pub type PROCESS_CREATION_FLAGS = u32;
 #[repr(C)]
diff --git a/library/std/src/sys/windows/fs.rs b/library/std/src/sys/windows/fs.rs
index 6ded683aade1..3bd8a30969ce 100644
--- a/library/std/src/sys/windows/fs.rs
+++ b/library/std/src/sys/windows/fs.rs
@@ -19,7 +19,7 @@ use crate::thread;
 use core::ffi::c_void;
 
 use super::path::maybe_verbatim;
-use super::to_u16s;
+use super::{api, to_u16s, IoResult};
 
 pub struct File {
     handle: Handle,
@@ -123,7 +123,7 @@ impl Iterator for ReadDir {
             let mut wfd = mem::zeroed();
             loop {
                 if c::FindNextFileW(self.handle.0, &mut wfd) == 0 {
-                    if c::GetLastError() == c::ERROR_NO_MORE_FILES {
+                    if api::get_last_error().code == c::ERROR_NO_MORE_FILES {
                         return None;
                     } else {
                         return Some(Err(Error::last_os_error()));
@@ -318,17 +318,8 @@ impl File {
     }
 
     pub fn truncate(&self, size: u64) -> io::Result<()> {
-        let mut info = c::FILE_END_OF_FILE_INFO { EndOfFile: size as c::LARGE_INTEGER };
-        let size = mem::size_of_val(&info);
-        cvt(unsafe {
-            c::SetFileInformationByHandle(
-                self.handle.as_raw_handle(),
-                c::FileEndOfFileInfo,
-                &mut info as *mut _ as *mut _,
-                size as c::DWORD,
-            )
-        })?;
-        Ok(())
+        let info = c::FILE_END_OF_FILE_INFO { EndOfFile: size as i64 };
+        api::set_file_information_by_handle(self.handle.as_raw_handle(), &info).io_result()
     }
 
     #[cfg(not(target_vendor = "uwp"))]
@@ -565,23 +556,14 @@ impl File {
     }
 
     pub fn set_permissions(&self, perm: FilePermissions) -> io::Result<()> {
-        let mut info = c::FILE_BASIC_INFO {
+        let info = c::FILE_BASIC_INFO {
             CreationTime: 0,
             LastAccessTime: 0,
             LastWriteTime: 0,
             ChangeTime: 0,
             FileAttributes: perm.attrs,
         };
-        let size = mem::size_of_val(&info);
-        cvt(unsafe {
-            c::SetFileInformationByHandle(
-                self.handle.as_raw_handle(),
-                c::FileBasicInfo,
-                &mut info as *mut _ as *mut _,
-                size as c::DWORD,
-            )
-        })?;
-        Ok(())
+        api::set_file_information_by_handle(self.handle.as_raw_handle(), &info).io_result()
     }
 
     pub fn set_times(&self, times: FileTimes) -> io::Result<()> {
@@ -641,38 +623,20 @@ impl File {
     /// If the operation is not supported for this filesystem or OS version
     /// then errors will be `ERROR_NOT_SUPPORTED` or `ERROR_INVALID_PARAMETER`.
     fn posix_delete(&self) -> io::Result<()> {
-        let mut info = c::FILE_DISPOSITION_INFO_EX {
+        let info = c::FILE_DISPOSITION_INFO_EX {
             Flags: c::FILE_DISPOSITION_FLAG_DELETE
                 | c::FILE_DISPOSITION_FLAG_POSIX_SEMANTICS
                 | c::FILE_DISPOSITION_FLAG_IGNORE_READONLY_ATTRIBUTE,
         };
-        let size = mem::size_of_val(&info);
-        cvt(unsafe {
-            c::SetFileInformationByHandle(
-                self.handle.as_raw_handle(),
-                c::FileDispositionInfoEx,
-                &mut info as *mut _ as *mut _,
-                size as c::DWORD,
-            )
-        })?;
-        Ok(())
+        api::set_file_information_by_handle(self.handle.as_raw_handle(), &info).io_result()
     }
 
     /// Delete a file using win32 semantics. The file won't actually be deleted
     /// until all file handles are closed. However, marking a file for deletion
     /// will prevent anyone from opening a new handle to the file.
     fn win32_delete(&self) -> io::Result<()> {
-        let mut info = c::FILE_DISPOSITION_INFO { DeleteFile: c::TRUE as _ };
-        let size = mem::size_of_val(&info);
-        cvt(unsafe {
-            c::SetFileInformationByHandle(
-                self.handle.as_raw_handle(),
-                c::FileDispositionInfo,
-                &mut info as *mut _ as *mut _,
-                size as c::DWORD,
-            )
-        })?;
-        Ok(())
+        let info = c::FILE_DISPOSITION_INFO { DeleteFile: c::TRUE as _ };
+        api::set_file_information_by_handle(self.handle.as_raw_handle(), &info).io_result()
     }
 
     /// Fill the given buffer with as many directory entries as will fit.
diff --git a/library/std/src/sys/windows/mod.rs b/library/std/src/sys/windows/mod.rs
index b609ad2472c6..c4e56e13be32 100644
--- a/library/std/src/sys/windows/mod.rs
+++ b/library/std/src/sys/windows/mod.rs
@@ -44,6 +44,18 @@ cfg_if::cfg_if! {
     }
 }
 
+mod api;
+
+/// Map a Result<T, WinError> to io::Result<T>.
+trait IoResult<T> {
+    fn io_result(self) -> crate::io::Result<T>;
+}
+impl<T> IoResult<T> for Result<T, api::WinError> {
+    fn io_result(self) -> crate::io::Result<T> {
+        self.map_err(|e| crate::io::Error::from_raw_os_error(e.code as i32))
+    }
+}
+
 // SAFETY: must be called only once during runtime initialization.
 // NOTE: this is not guaranteed to run, for example when Rust code is called externally.
 pub unsafe fn init(_argc: isize, _argv: *const *const u8, _sigpipe: u8) {
@@ -241,11 +253,11 @@ where
             // not an actual error.
             c::SetLastError(0);
             let k = match f1(buf.as_mut_ptr().cast::<u16>(), n as c::DWORD) {
-                0 if c::GetLastError() == 0 => 0,
+                0 if api::get_last_error().code == 0 => 0,
                 0 => return Err(crate::io::Error::last_os_error()),
                 n => n,
             } as usize;
-            if k == n && c::GetLastError() == c::ERROR_INSUFFICIENT_BUFFER {
+            if k == n && api::get_last_error().code == c::ERROR_INSUFFICIENT_BUFFER {
                 n = n.saturating_mul(2).min(c::DWORD::MAX as usize);
             } else if k > n {
                 n = k;
diff --git a/library/std/src/sys/windows/os.rs b/library/std/src/sys/windows/os.rs
index 58afca088ef9..8cc905101de7 100644
--- a/library/std/src/sys/windows/os.rs
+++ b/library/std/src/sys/windows/os.rs
@@ -17,10 +17,10 @@ use crate::ptr;
 use crate::slice;
 use crate::sys::{c, cvt};
 
-use super::to_u16s;
+use super::{api, to_u16s};
 
 pub fn errno() -> i32 {
-    unsafe { c::GetLastError() as i32 }
+    api::get_last_error().code as i32
 }
 
 /// Gets a detailed string description for the given error number.
@@ -336,7 +336,7 @@ fn home_dir_crt() -> Option<PathBuf> {
         super::fill_utf16_buf(
             |buf, mut sz| {
                 match c::GetUserProfileDirectoryW(token, buf, &mut sz) {
-                    0 if c::GetLastError() != c::ERROR_INSUFFICIENT_BUFFER => 0,
+                    0 if api::get_last_error().code != c::ERROR_INSUFFICIENT_BUFFER => 0,
                     0 => sz,
                     _ => sz - 1, // sz includes the null terminator
                 }
diff --git a/library/std/src/sys/windows/stack_overflow.rs b/library/std/src/sys/windows/stack_overflow.rs
index 0caf0a317a4a..627763da8561 100644
--- a/library/std/src/sys/windows/stack_overflow.rs
+++ b/library/std/src/sys/windows/stack_overflow.rs
@@ -3,6 +3,8 @@
 use crate::sys::c;
 use crate::thread;
 
+use super::api;
+
 pub struct Handler;
 
 impl Handler {
@@ -10,7 +12,7 @@ impl Handler {
         // This API isn't available on XP, so don't panic in that case and just
         // pray it works out ok.
         if c::SetThreadStackGuarantee(&mut 0x5000) == 0
-            && c::GetLastError() as u32 != c::ERROR_CALL_NOT_IMPLEMENTED as u32
+            && api::get_last_error().code != c::ERROR_CALL_NOT_IMPLEMENTED
         {
             panic!("failed to reserve stack space for exception handling");
         }
diff --git a/library/std/src/sys/windows/stdio.rs b/library/std/src/sys/windows/stdio.rs
index 3fcaaa508e3c..a9ff909aa679 100644
--- a/library/std/src/sys/windows/stdio.rs
+++ b/library/std/src/sys/windows/stdio.rs
@@ -9,6 +9,7 @@ use crate::str;
 use crate::sys::c;
 use crate::sys::cvt;
 use crate::sys::handle::Handle;
+use crate::sys::windows::api;
 use core::str::utf8_char_width;
 
 #[cfg(test)]
@@ -369,7 +370,7 @@ fn read_u16s(handle: c::HANDLE, buf: &mut [MaybeUninit<u16>]) -> io::Result<usiz
 
         // ReadConsoleW returns success with ERROR_OPERATION_ABORTED for Ctrl-C or Ctrl-Break.
         // Explicitly check for that case here and try again.
-        if amount == 0 && unsafe { c::GetLastError() } == c::ERROR_OPERATION_ABORTED {
+        if amount == 0 && api::get_last_error().code == c::ERROR_OPERATION_ABORTED {
             continue;
         }
         break;

From e1cb8187e1b532bb006bc85f842161fed8837563 Mon Sep 17 00:00:00 2001
From: Michael Howell <michael@notriddle.com>
Date: Mon, 16 Oct 2023 15:22:17 -0700
Subject: [PATCH 028/297] docs: add Rust logo to more compiler crates

c6e6ecb1afea9695a42d0f148ce153536b279eb5 added it to some of the
compiler's crates, but avoided adding it to all of them to reduce
bit-rot. This commit adds to more.
---
 src/lib.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/lib.rs b/src/lib.rs
index 9c18fc4a0dc7..6aa5b6bd1ff1 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -12,6 +12,8 @@
  * TODO(antoyo): remove the patches.
  */
 
+#![cfg_attr(not(bootstrap), doc(rust_logo))]
+#![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
 #![feature(
     rustc_private,
     decl_macro,

From 10059704852424b735201ef9443b36361821a6c1 Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Sat, 14 Oct 2023 15:25:58 +0100
Subject: [PATCH 029/297] Use a visitor that could be reused by opaque type
 capture check later

---
 .../src/type_check/liveness/trace.rs          | 112 ++--------------
 .../rustc_infer/src/infer/opaque_types.rs     |   2 +
 .../src/infer/outlives/for_liveness.rs        | 121 ++++++++++++++++++
 .../rustc_infer/src/infer/outlives/mod.rs     |   1 +
 4 files changed, 137 insertions(+), 99 deletions(-)
 create mode 100644 compiler/rustc_infer/src/infer/outlives/for_liveness.rs

diff --git a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
index e0eed812b6ad..3281cd7a52b3 100644
--- a/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
+++ b/compiler/rustc_borrowck/src/type_check/liveness/trace.rs
@@ -3,17 +3,13 @@ use rustc_data_structures::graph::WithSuccessors;
 use rustc_index::bit_set::{HybridBitSet, SparseBitMatrix};
 use rustc_index::interval::IntervalSet;
 use rustc_infer::infer::canonical::QueryRegionConstraints;
-use rustc_infer::infer::outlives::test_type_match;
-use rustc_infer::infer::region_constraints::VerifyIfEq;
+use rustc_infer::infer::outlives::for_liveness;
 use rustc_middle::mir::{BasicBlock, Body, ConstraintCategory, Local, Location};
 use rustc_middle::traits::query::DropckOutlivesResult;
-use rustc_middle::ty::{
-    self, RegionVid, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitableExt, TypeVisitor,
-};
+use rustc_middle::ty::{RegionVid, Ty, TyCtxt, TypeVisitable, TypeVisitableExt};
 use rustc_span::DUMMY_SP;
 use rustc_trait_selection::traits::query::type_op::outlives::DropckOutlives;
 use rustc_trait_selection::traits::query::type_op::{TypeOp, TypeOpOutput};
-use std::ops::ControlFlow;
 use std::rc::Rc;
 
 use rustc_mir_dataflow::impls::MaybeInitializedPlaces;
@@ -612,107 +608,25 @@ impl<'tcx> LivenessContext<'_, '_, '_, 'tcx> {
         let num_loans = typeck.borrowck_context.borrow_set.len();
         let value_loans = &mut HybridBitSet::new_empty(num_loans);
 
-        struct MakeAllRegionsLive<'a, 'b, 'tcx> {
-            typeck: &'b mut TypeChecker<'a, 'tcx>,
-            live_at: &'b IntervalSet<PointIndex>,
-            value_loans: &'b mut HybridBitSet<BorrowIndex>,
-            inflowing_loans: &'b SparseBitMatrix<RegionVid, BorrowIndex>,
-        }
-        impl<'tcx> MakeAllRegionsLive<'_, '_, 'tcx> {
-            /// We can prove that an alias is live two ways:
-            /// 1. All the components are live.
-            /// 2. There is a known outlives bound or where-clause, and that
-            ///    region is live.
-            /// We search through the item bounds and where clauses for
-            /// either `'static` or a unique outlives region, and if one is
-            /// found, we just need to prove that that region is still live.
-            /// If one is not found, then we continue to walk through the alias.
-            fn make_alias_live(&mut self, t: Ty<'tcx>) -> ControlFlow<!> {
-                let ty::Alias(_kind, alias_ty) = t.kind() else {
-                    bug!("`make_alias_live` only takes alias types");
-                };
-                let tcx = self.typeck.infcx.tcx;
-                let param_env = self.typeck.param_env;
-                let outlives_bounds: Vec<_> = tcx
-                    .item_bounds(alias_ty.def_id)
-                    .iter_instantiated(tcx, alias_ty.args)
-                    .filter_map(|clause| {
-                        if let Some(outlives) = clause.as_type_outlives_clause()
-                        && outlives.skip_binder().0 == t
-                    {
-                        Some(outlives.skip_binder().1)
-                    } else {
-                        None
-                    }
-                    })
-                    .chain(param_env.caller_bounds().iter().filter_map(|clause| {
-                        let outlives = clause.as_type_outlives_clause()?;
-                        if let Some(outlives) = outlives.no_bound_vars()
-                            && outlives.0 == t
-                        {
-                            Some(outlives.1)
-                        } else {
-                            test_type_match::extract_verify_if_eq(
-                                tcx,
-                                param_env,
-                                &outlives.map_bound(|ty::OutlivesPredicate(ty, bound)| {
-                                    VerifyIfEq { ty, bound }
-                                }),
-                                t,
-                            )
-                        }
-                    }))
-                    .collect();
-                // If we find `'static`, then we know the alias doesn't capture *any* regions.
-                // Otherwise, all of the outlives regions should be equal -- if they're not,
-                // we don't really know how to proceed, so we continue recursing through the
-                // alias.
-                if outlives_bounds.contains(&tcx.lifetimes.re_static) {
-                    ControlFlow::Continue(())
-                } else if let Some(r) = outlives_bounds.first()
-                    && outlives_bounds[1..].iter().all(|other_r| other_r == r)
-                {
-                    r.visit_with(self)
-                } else {
-                    t.super_visit_with(self)
-                }
-            }
-        }
-        impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for MakeAllRegionsLive<'_, '_, 'tcx> {
-            type BreakTy = !;
+        value.visit_with(&mut for_liveness::FreeRegionsVisitor {
+            tcx: typeck.tcx(),
+            param_env: typeck.param_env,
+            op: |r| {
+                let live_region_vid = typeck.borrowck_context.universal_regions.to_region_vid(r);
 
-            fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
-                if r.is_late_bound() {
-                    return ControlFlow::Continue(());
-                }
-                let live_region_vid =
-                    self.typeck.borrowck_context.universal_regions.to_region_vid(r);
-
-                self.typeck
+                typeck
                     .borrowck_context
                     .constraints
                     .liveness_constraints
-                    .add_elements(live_region_vid, self.live_at);
+                    .add_elements(live_region_vid, live_at);
 
                 // There can only be inflowing loans for this region when we are using
                 // `-Zpolonius=next`.
-                if let Some(inflowing) = self.inflowing_loans.row(live_region_vid) {
-                    self.value_loans.union(inflowing);
+                if let Some(inflowing) = inflowing_loans.row(live_region_vid) {
+                    value_loans.union(inflowing);
                 }
-                ControlFlow::Continue(())
-            }
-
-            fn visit_ty(&mut self, t: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
-                if !t.has_free_regions() {
-                    ControlFlow::Continue(())
-                } else if let ty::Alias(..) = t.kind() {
-                    self.make_alias_live(t)
-                } else {
-                    t.super_visit_with(self)
-                }
-            }
-        }
-        value.visit_with(&mut MakeAllRegionsLive { typeck, live_at, value_loans, inflowing_loans });
+            },
+        });
 
         // Record the loans reaching the value.
         if !value_loans.is_empty() {
diff --git a/compiler/rustc_infer/src/infer/opaque_types.rs b/compiler/rustc_infer/src/infer/opaque_types.rs
index 09df93fcc2fd..cbd9abc137b2 100644
--- a/compiler/rustc_infer/src/infer/opaque_types.rs
+++ b/compiler/rustc_infer/src/infer/opaque_types.rs
@@ -380,6 +380,8 @@ impl<'tcx> InferCtxt<'tcx> {
                 .collect(),
         );
 
+        // FIXME(#42940): This should use the `FreeRegionsVisitor`, but that's
+        // not currently sound until we have existential regions.
         concrete_ty.visit_with(&mut ConstrainOpaqueTypeRegionVisitor {
             tcx: self.tcx,
             op: |r| self.member_constraint(opaque_type_key, span, concrete_ty, r, &choice_regions),
diff --git a/compiler/rustc_infer/src/infer/outlives/for_liveness.rs b/compiler/rustc_infer/src/infer/outlives/for_liveness.rs
new file mode 100644
index 000000000000..ebc51b98be53
--- /dev/null
+++ b/compiler/rustc_infer/src/infer/outlives/for_liveness.rs
@@ -0,0 +1,121 @@
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitor};
+
+use std::ops::ControlFlow;
+
+use crate::infer::outlives::test_type_match;
+use crate::infer::region_constraints::VerifyIfEq;
+
+/// Visits free regions in the type that are relevant for liveness computation.
+/// These regions are passed to `OP`.
+///
+/// Specifically, we visit all of the regions of types recursively, except if
+/// the type is an alias, we look at the outlives bounds in the param-env
+/// and alias's item bounds. If there is a unique outlives bound, then visit
+/// that instead. If there is not a unique but there is a `'static` outlives
+/// bound, then don't visit anything. Otherwise, walk through the opaque's
+/// regions structurally.
+pub struct FreeRegionsVisitor<'tcx, OP: FnMut(ty::Region<'tcx>)> {
+    pub tcx: TyCtxt<'tcx>,
+    pub param_env: ty::ParamEnv<'tcx>,
+    pub op: OP,
+}
+
+impl<'tcx, OP> TypeVisitor<TyCtxt<'tcx>> for FreeRegionsVisitor<'tcx, OP>
+where
+    OP: FnMut(ty::Region<'tcx>),
+{
+    fn visit_binder<T: TypeVisitable<TyCtxt<'tcx>>>(
+        &mut self,
+        t: &ty::Binder<'tcx, T>,
+    ) -> ControlFlow<Self::BreakTy> {
+        t.super_visit_with(self);
+        ControlFlow::Continue(())
+    }
+
+    fn visit_region(&mut self, r: ty::Region<'tcx>) -> ControlFlow<Self::BreakTy> {
+        match *r {
+            // ignore bound regions, keep visiting
+            ty::ReLateBound(_, _) => ControlFlow::Continue(()),
+            _ => {
+                (self.op)(r);
+                ControlFlow::Continue(())
+            }
+        }
+    }
+
+    fn visit_ty(&mut self, ty: Ty<'tcx>) -> ControlFlow<Self::BreakTy> {
+        // We're only interested in types involving regions
+        if !ty.flags().intersects(ty::TypeFlags::HAS_FREE_REGIONS) {
+            return ControlFlow::Continue(());
+        }
+
+        match ty.kind() {
+            // We can prove that an alias is live two ways:
+            // 1. All the components are live.
+            //
+            // 2. There is a known outlives bound or where-clause, and that
+            //    region is live.
+            //
+            // We search through the item bounds and where clauses for
+            // either `'static` or a unique outlives region, and if one is
+            // found, we just need to prove that that region is still live.
+            // If one is not found, then we continue to walk through the alias.
+            ty::Alias(kind, ty::AliasTy { def_id, args, .. }) => {
+                let tcx = self.tcx;
+                let param_env = self.param_env;
+                let outlives_bounds: Vec<_> = tcx
+                    .item_bounds(def_id)
+                    .iter_instantiated(tcx, args)
+                    .chain(param_env.caller_bounds())
+                    .filter_map(|clause| {
+                        let outlives = clause.as_type_outlives_clause()?;
+                        if let Some(outlives) = outlives.no_bound_vars()
+                            && outlives.0 == ty
+                        {
+                            Some(outlives.1)
+                        } else {
+                            test_type_match::extract_verify_if_eq(
+                                tcx,
+                                param_env,
+                                &outlives.map_bound(|ty::OutlivesPredicate(ty, bound)| {
+                                    VerifyIfEq { ty, bound }
+                                }),
+                                ty,
+                            )
+                        }
+                    })
+                    .collect();
+                // If we find `'static`, then we know the alias doesn't capture *any* regions.
+                // Otherwise, all of the outlives regions should be equal -- if they're not,
+                // we don't really know how to proceed, so we continue recursing through the
+                // alias.
+                if outlives_bounds.contains(&tcx.lifetimes.re_static) {
+                    // no
+                } else if let Some(r) = outlives_bounds.first()
+                    && outlives_bounds[1..].iter().all(|other_r| other_r == r)
+                {
+                    assert!(r.type_flags().intersects(ty::TypeFlags::HAS_FREE_REGIONS));
+                    r.visit_with(self)?;
+                } else {
+                    // Skip lifetime parameters that are not captures.
+                    let variances = match kind {
+                        ty::Opaque => Some(self.tcx.variances_of(*def_id)),
+                        _ => None,
+                    };
+
+                    for (idx, s) in args.iter().enumerate() {
+                        if variances.map(|variances| variances[idx]) != Some(ty::Variance::Bivariant) {
+                            s.visit_with(self)?;
+                        }
+                    }
+                }
+            }
+
+            _ => {
+                ty.super_visit_with(self)?;
+            }
+        }
+
+        ControlFlow::Continue(())
+    }
+}
diff --git a/compiler/rustc_infer/src/infer/outlives/mod.rs b/compiler/rustc_infer/src/infer/outlives/mod.rs
index cb92fc6ddb64..0987915f4fdb 100644
--- a/compiler/rustc_infer/src/infer/outlives/mod.rs
+++ b/compiler/rustc_infer/src/infer/outlives/mod.rs
@@ -9,6 +9,7 @@ use rustc_middle::ty;
 
 pub mod components;
 pub mod env;
+pub mod for_liveness;
 pub mod obligations;
 pub mod test_type_match;
 pub mod verify;

From f0e385d6b71a8e4b2cef49a968fa9f630d01d2e6 Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Sat, 14 Oct 2023 15:28:44 +0100
Subject: [PATCH 030/297] Flesh out tests more

---
 tests/ui/borrowck/alias-liveness/gat-static.rs  | 11 ++++++++++-
 .../higher-ranked-outlives-for-capture.rs       | 16 ++++++++++++++++
 .../higher-ranked-outlives-for-capture.stderr   | 16 ++++++++++++++++
 .../ui/borrowck/alias-liveness/higher-ranked.rs | 16 ++++++++++++++++
 .../borrowck/alias-liveness/opaque-capture.rs   | 17 +++++++++++++++++
 .../alias-liveness/opaque-type-param.rs         | 14 ++++++++++++++
 .../alias-liveness/opaque-type-param.stderr     | 13 +++++++++++++
 tests/ui/borrowck/alias-liveness/rpit-static.rs | 16 ++++++++++++----
 .../ui/borrowck/alias-liveness/rpitit-static.rs |  8 +++++++-
 tests/ui/borrowck/alias-liveness/rtn-static.rs  |  8 +++++++-
 .../impl-trait/bivariant-lifetime-liveness.rs   | 15 +++++++++++++++
 11 files changed, 143 insertions(+), 7 deletions(-)
 create mode 100644 tests/ui/borrowck/alias-liveness/higher-ranked-outlives-for-capture.rs
 create mode 100644 tests/ui/borrowck/alias-liveness/higher-ranked-outlives-for-capture.stderr
 create mode 100644 tests/ui/borrowck/alias-liveness/higher-ranked.rs
 create mode 100644 tests/ui/borrowck/alias-liveness/opaque-capture.rs
 create mode 100644 tests/ui/borrowck/alias-liveness/opaque-type-param.rs
 create mode 100644 tests/ui/borrowck/alias-liveness/opaque-type-param.stderr
 create mode 100644 tests/ui/impl-trait/bivariant-lifetime-liveness.rs

diff --git a/tests/ui/borrowck/alias-liveness/gat-static.rs b/tests/ui/borrowck/alias-liveness/gat-static.rs
index 1ab40fe828c3..92153124af93 100644
--- a/tests/ui/borrowck/alias-liveness/gat-static.rs
+++ b/tests/ui/borrowck/alias-liveness/gat-static.rs
@@ -8,7 +8,7 @@ trait Foo {
     fn assoc(&mut self) -> Self::Assoc<'_>;
 }
 
-fn test<T>(mut t: T)
+fn overlapping_mut<T>(mut t: T)
 where
     T: Foo,
     for<'a> T::Assoc<'a>: 'static,
@@ -17,4 +17,13 @@ where
     let b = t.assoc();
 }
 
+fn live_past_borrow<T>(mut t: T)
+where
+    T: Foo,
+    for<'a> T::Assoc<'a>: 'static {
+    let x = t.assoc();
+    drop(t);
+    drop(x);
+}
+
 fn main() {}
diff --git a/tests/ui/borrowck/alias-liveness/higher-ranked-outlives-for-capture.rs b/tests/ui/borrowck/alias-liveness/higher-ranked-outlives-for-capture.rs
new file mode 100644
index 000000000000..1f26c7babf21
--- /dev/null
+++ b/tests/ui/borrowck/alias-liveness/higher-ranked-outlives-for-capture.rs
@@ -0,0 +1,16 @@
+// known-bug: #42940
+
+trait Captures<'a> {}
+impl<T> Captures<'_> for T {}
+
+trait Outlives<'a>: 'a {}
+impl<'a, T: 'a> Outlives<'a> for T {}
+
+// Test that we treat `for<'a> Opaque: 'a` as `Opaque: 'static`
+fn test<'o>(v: &'o Vec<i32>) -> impl Captures<'o> + for<'a> Outlives<'a> {}
+
+fn statik() -> impl Sized {
+    test(&vec![])
+}
+
+fn main() {}
diff --git a/tests/ui/borrowck/alias-liveness/higher-ranked-outlives-for-capture.stderr b/tests/ui/borrowck/alias-liveness/higher-ranked-outlives-for-capture.stderr
new file mode 100644
index 000000000000..58a42d8afe4d
--- /dev/null
+++ b/tests/ui/borrowck/alias-liveness/higher-ranked-outlives-for-capture.stderr
@@ -0,0 +1,16 @@
+error[E0716]: temporary value dropped while borrowed
+  --> $DIR/higher-ranked-outlives-for-capture.rs:13:11
+   |
+LL |     test(&vec![])
+   |     ------^^^^^^-
+   |     |     |
+   |     |     creates a temporary value which is freed while still in use
+   |     argument requires that borrow lasts for `'static`
+LL | }
+   | - temporary value is freed at the end of this statement
+   |
+   = note: this error originates in the macro `vec` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0716`.
diff --git a/tests/ui/borrowck/alias-liveness/higher-ranked.rs b/tests/ui/borrowck/alias-liveness/higher-ranked.rs
new file mode 100644
index 000000000000..afd0d3b31e3f
--- /dev/null
+++ b/tests/ui/borrowck/alias-liveness/higher-ranked.rs
@@ -0,0 +1,16 @@
+// check-pass
+
+trait Captures<'a> {}
+impl<T> Captures<'_> for T {}
+
+trait Outlives<'a>: 'a {}
+impl<'a, T: 'a> Outlives<'a> for T {}
+
+// Test that we treat `for<'a> Opaque: 'a` as `Opaque: 'static`
+fn test<'o>(v: &'o Vec<i32>) -> impl Captures<'o> + for<'a> Outlives<'a> {}
+
+fn opaque_doesnt_use_temporary() {
+    let a = test(&vec![]);
+}
+
+fn main() {}
diff --git a/tests/ui/borrowck/alias-liveness/opaque-capture.rs b/tests/ui/borrowck/alias-liveness/opaque-capture.rs
new file mode 100644
index 000000000000..f4ca2728bdbe
--- /dev/null
+++ b/tests/ui/borrowck/alias-liveness/opaque-capture.rs
@@ -0,0 +1,17 @@
+// check-pass
+
+// Check that opaques capturing early and late-bound vars correctly mark
+// regions required to be live using the item bounds.
+
+trait Captures<'a> {}
+impl<T> Captures<'_> for T {}
+
+fn captures_temp_late<'a>(x: &'a Vec<i32>) -> impl Sized + Captures<'a> + 'static {}
+fn captures_temp_early<'a: 'a>(x: &'a Vec<i32>) -> impl Sized + Captures<'a> + 'static {}
+
+fn test() {
+    let x = captures_temp_early(&vec![]);
+    let y = captures_temp_late(&vec![]);
+}
+
+fn main() {}
diff --git a/tests/ui/borrowck/alias-liveness/opaque-type-param.rs b/tests/ui/borrowck/alias-liveness/opaque-type-param.rs
new file mode 100644
index 000000000000..a292463b2ac4
--- /dev/null
+++ b/tests/ui/borrowck/alias-liveness/opaque-type-param.rs
@@ -0,0 +1,14 @@
+// known-bug: #42940
+
+trait Trait {}
+impl Trait for () {}
+
+fn foo<'a>(s: &'a str) -> impl Trait + 'static {
+    bar(s)
+}
+
+fn bar<P: AsRef<str>>(s: P) -> impl Trait + 'static {
+    ()
+}
+
+fn main() {}
diff --git a/tests/ui/borrowck/alias-liveness/opaque-type-param.stderr b/tests/ui/borrowck/alias-liveness/opaque-type-param.stderr
new file mode 100644
index 000000000000..e1fbbc14f44e
--- /dev/null
+++ b/tests/ui/borrowck/alias-liveness/opaque-type-param.stderr
@@ -0,0 +1,13 @@
+error[E0700]: hidden type for `impl Trait + 'static` captures lifetime that does not appear in bounds
+  --> $DIR/opaque-type-param.rs:7:5
+   |
+LL | fn foo<'a>(s: &'a str) -> impl Trait + 'static {
+   |        --                 -------------------- opaque type defined here
+   |        |
+   |        hidden type `impl Trait + 'static` captures the lifetime `'a` as defined here
+LL |     bar(s)
+   |     ^^^^^^
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0700`.
diff --git a/tests/ui/borrowck/alias-liveness/rpit-static.rs b/tests/ui/borrowck/alias-liveness/rpit-static.rs
index 8637d5fb720e..45da3edb8780 100644
--- a/tests/ui/borrowck/alias-liveness/rpit-static.rs
+++ b/tests/ui/borrowck/alias-liveness/rpit-static.rs
@@ -3,12 +3,20 @@
 trait Captures<'a> {}
 impl<T> Captures<'_> for T {}
 
-fn foo(x: &i32) -> impl Sized + Captures<'_> + 'static {}
+fn foo(x: &mut i32) -> impl Sized + Captures<'_> + 'static {}
 
-fn main() {
+fn overlapping_mut() {
+    let i = &mut 1;
+    let x = foo(i);
+    let y = foo(i);
+}
+
+fn live_past_borrow() {
     let y;
     {
-        let x = 1;
-        y = foo(&x);
+        let x = &mut 1;
+        y = foo(x);
     }
 }
+
+fn main() {}
diff --git a/tests/ui/borrowck/alias-liveness/rpitit-static.rs b/tests/ui/borrowck/alias-liveness/rpitit-static.rs
index ed7b496a42ba..2cc68d2bf3d8 100644
--- a/tests/ui/borrowck/alias-liveness/rpitit-static.rs
+++ b/tests/ui/borrowck/alias-liveness/rpitit-static.rs
@@ -4,7 +4,13 @@ trait Foo {
     fn rpitit(&mut self) -> impl Sized + 'static;
 }
 
-fn test<T: Foo>(mut t: T) {
+fn live_past_borrow<T: Foo>(mut t: T) {
+    let x = t.rpitit();
+    drop(t);
+    drop(x);
+}
+
+fn overlapping_mut<T: Foo>(mut t: T) {
     let a = t.rpitit();
     let b = t.rpitit();
 }
diff --git a/tests/ui/borrowck/alias-liveness/rtn-static.rs b/tests/ui/borrowck/alias-liveness/rtn-static.rs
index c850f0122e5a..1f136b8b998b 100644
--- a/tests/ui/borrowck/alias-liveness/rtn-static.rs
+++ b/tests/ui/borrowck/alias-liveness/rtn-static.rs
@@ -7,9 +7,15 @@ trait Foo {
     fn borrow(&mut self) -> impl Sized + '_;
 }
 
+fn live_past_borrow<T: Foo<borrow(): 'static>>(mut t: T) {
+    let x = t.borrow();
+    drop(t);
+    drop(x);
+}
+
 // Test that the `'_` item bound in `borrow` does not cause us to
 // overlook the `'static` RTN bound.
-fn test<T: Foo<borrow(): 'static>>(mut t: T) {
+fn overlapping_mut<T: Foo<borrow(): 'static>>(mut t: T) {
     let x = t.borrow();
     let x = t.borrow();
 }
diff --git a/tests/ui/impl-trait/bivariant-lifetime-liveness.rs b/tests/ui/impl-trait/bivariant-lifetime-liveness.rs
new file mode 100644
index 000000000000..fe99fe3f3406
--- /dev/null
+++ b/tests/ui/impl-trait/bivariant-lifetime-liveness.rs
@@ -0,0 +1,15 @@
+// check-pass
+// issue: 116794
+
+// Uncaptured lifetimes should not be required to be live.
+
+struct Invariant<T>(*mut T);
+
+fn opaque<'a: 'a>(_: &'a str) -> Invariant<impl Sized> {
+    Invariant(&mut ())
+}
+
+fn main() {
+    let x = opaque(&String::new());
+    drop(x);
+}

From e2f32c72a7a6ad3b1ba971416d86c6eca1096a36 Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Tue, 17 Oct 2023 07:57:56 -0400
Subject: [PATCH 031/297] Ignore failing test

This test only fails with non-native 128-bit integers, but it will also
fail with native 128-bit integers if we copy/paste it so that it's
executed twice. Interestingly, wrapping the test in a loop won't make it
fail.
So, it could be due to stack space or unwinding in release mode.
Also, the test only fails with -O2:
../cargo.sh rustc --bin test-rust -- -O

It doesn't fail with -O3.
---
 doc/tests.md         | 5 +++++
 failing-ui-tests.txt | 1 +
 2 files changed, 6 insertions(+)
 create mode 100644 doc/tests.md

diff --git a/doc/tests.md b/doc/tests.md
new file mode 100644
index 000000000000..3ac993bc2fd8
--- /dev/null
+++ b/doc/tests.md
@@ -0,0 +1,5 @@
+# Tests
+
+## Show the rustc command for UI tests
+
+Add ` --test-args "--verbose"` to `./x.py test`.
diff --git a/failing-ui-tests.txt b/failing-ui-tests.txt
index ed56a11a1709..771da5812957 100644
--- a/failing-ui-tests.txt
+++ b/failing-ui-tests.txt
@@ -71,3 +71,4 @@ tests/ui/lto/all-crates.rs
 tests/ui/async-await/deep-futures-are-freeze.rs
 tests/ui/closures/capture-unsized-by-ref.rs
 tests/ui/generator/resume-after-return.rs
+tests/ui/macros/rfc-2011-nicer-assert-messages/all-expr-kinds.rs

From 096f14d37488ec5f2855cda241da5c795d738429 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez <guillaume1.gomez@gmail.com>
Date: Mon, 4 Sep 2023 20:29:50 +0200
Subject: [PATCH 032/297] Add support for NonNull function attribute

---
 src/abi.rs           | 54 ++++++++++++++++++++++++++++----------------
 src/declare.rs       |  6 ++++-
 src/intrinsic/mod.rs |  2 +-
 src/type_of.rs       |  3 ++-
 4 files changed, 43 insertions(+), 22 deletions(-)

diff --git a/src/abi.rs b/src/abi.rs
index 35bb0b6e5f4e..a2825773bd39 100644
--- a/src/abi.rs
+++ b/src/abi.rs
@@ -1,4 +1,4 @@
-use gccjit::{ToLValue, ToRValue, Type};
+use gccjit::{FnAttribute, ToLValue, ToRValue, Type};
 use rustc_codegen_ssa::traits::{AbiBuilderMethods, BaseTypeMethods};
 use rustc_data_structures::fx::FxHashSet;
 use rustc_middle::bug;
@@ -98,12 +98,12 @@ impl GccType for Reg {
 
 pub trait FnAbiGccExt<'gcc, 'tcx> {
     // TODO(antoyo): return a function pointer type instead?
-    fn gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> (Type<'gcc>, Vec<Type<'gcc>>, bool, FxHashSet<usize>);
+    fn gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> (Type<'gcc>, Vec<Type<'gcc>>, bool, FxHashSet<usize>, Vec<FnAttribute<'gcc>>);
     fn ptr_to_gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> Type<'gcc>;
 }
 
 impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
-    fn gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> (Type<'gcc>, Vec<Type<'gcc>>, bool, FxHashSet<usize>) {
+    fn gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> (Type<'gcc>, Vec<Type<'gcc>>, bool, FxHashSet<usize>, Vec<FnAttribute<'gcc>>) {
         let mut on_stack_param_indices = FxHashSet::default();
 
         // This capacity calculation is approximate.
@@ -121,19 +121,23 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     cx.type_void()
                 }
             };
+        let mut non_null_args = Vec::new();
 
         #[cfg(feature = "master")]
-        let apply_attrs = |ty: Type<'gcc>, attrs: &ArgAttributes| {
-            if cx.sess().opts.optimize != config::OptLevel::No
-                && attrs.regular.contains(rustc_target::abi::call::ArgAttribute::NoAlias)
-            {
-                ty.make_restrict()
-            } else {
-                ty
+        let mut apply_attrs = |mut ty: Type<'gcc>, attrs: &ArgAttributes, arg_index: usize| {
+            if cx.sess().opts.optimize == config::OptLevel::No {
+                return ty;
             }
+            if attrs.regular.contains(rustc_target::abi::call::ArgAttribute::NoAlias) {
+                ty = ty.make_restrict()
+            }
+            if attrs.regular.contains(rustc_target::abi::call::ArgAttribute::NonNull) {
+                non_null_args.push(arg_index as i32 + 1);
+            }
+            ty
         };
         #[cfg(not(feature = "master"))]
-        let apply_attrs = |ty: Type<'gcc>, _attrs: &ArgAttributes| {
+        let apply_attrs = |ty: Type<'gcc>, _attrs: &ArgAttributes, _arg_index: usize| {
             ty
         };
 
@@ -141,8 +145,9 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             let arg_ty = match arg.mode {
                 PassMode::Ignore => continue,
                 PassMode::Pair(a, b) => {
-                    argument_tys.push(apply_attrs(arg.layout.scalar_pair_element_gcc_type(cx, 0), &a));
-                    argument_tys.push(apply_attrs(arg.layout.scalar_pair_element_gcc_type(cx, 1), &b));
+                    let arg_pos = argument_tys.len();
+                    argument_tys.push(apply_attrs(arg.layout.scalar_pair_element_gcc_type(cx, 0), &a, arg_pos));
+                    argument_tys.push(apply_attrs(arg.layout.scalar_pair_element_gcc_type(cx, 1), &b, arg_pos + 1));
                     continue;
                 }
                 PassMode::Cast { ref cast, pad_i32 } => {
@@ -151,30 +156,41 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                         argument_tys.push(Reg::i32().gcc_type(cx));
                     }
                     let ty = cast.gcc_type(cx);
-                    apply_attrs(ty, &cast.attrs)
+                    apply_attrs(ty, &cast.attrs, argument_tys.len())
                 }
                 PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: true } => {
                     // This is a "byval" argument, so we don't apply the `restrict` attribute on it.
                     on_stack_param_indices.insert(argument_tys.len());
                     arg.memory_ty(cx)
                 },
-                PassMode::Direct(attrs) => apply_attrs(arg.layout.immediate_gcc_type(cx), &attrs),
+                PassMode::Direct(attrs) => apply_attrs(arg.layout.immediate_gcc_type(cx), &attrs, argument_tys.len()),
                 PassMode::Indirect { attrs, meta_attrs: None, on_stack: false } => {
-                    apply_attrs(cx.type_ptr_to(arg.memory_ty(cx)), &attrs)
+                    apply_attrs(cx.type_ptr_to(arg.memory_ty(cx)), &attrs, argument_tys.len())
                 }
                 PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => {
                     assert!(!on_stack);
-                    apply_attrs(apply_attrs(cx.type_ptr_to(arg.memory_ty(cx)), &attrs), &meta_attrs)
+                    let ty = apply_attrs(cx.type_ptr_to(arg.memory_ty(cx)), &attrs, argument_tys.len());
+                    apply_attrs(ty, &meta_attrs, argument_tys.len())
                 }
             };
             argument_tys.push(arg_ty);
         }
 
-        (return_ty, argument_tys, self.c_variadic, on_stack_param_indices)
+        #[cfg(feature = "master")]
+        let fn_attrs = if non_null_args.is_empty() {
+            Vec::new()
+        } else {
+            vec![FnAttribute::NonNull(non_null_args)]
+        };
+        #[cfg(not(feature = "master"))]
+        let fn_attrs = Vec::new();
+
+        (return_ty, argument_tys, self.c_variadic, on_stack_param_indices, fn_attrs)
     }
 
     fn ptr_to_gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> Type<'gcc> {
-        let (return_type, params, variadic, on_stack_param_indices) = self.gcc_type(cx);
+        // FIXME: Should we do something with `fn_attrs`?
+        let (return_type, params, variadic, on_stack_param_indices, _fn_attrs) = self.gcc_type(cx);
         let pointer_type = cx.context.new_function_pointer_type(None, return_type, &params, variadic);
         cx.on_stack_params.borrow_mut().insert(pointer_type.dyncast_function_ptr_type().expect("function ptr type"), on_stack_param_indices);
         pointer_type
diff --git a/src/declare.rs b/src/declare.rs
index e673d0af4c7e..409f112ca739 100644
--- a/src/declare.rs
+++ b/src/declare.rs
@@ -80,9 +80,13 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
     }
 
     pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Function<'gcc> {
-        let (return_type, params, variadic, on_stack_param_indices) = fn_abi.gcc_type(self);
+        let (return_type, params, variadic, on_stack_param_indices, fn_attrs) = fn_abi.gcc_type(self);
         let func = declare_raw_fn(self, name, () /*fn_abi.llvm_cconv()*/, return_type, &params, variadic);
         self.on_stack_function_params.borrow_mut().insert(func, on_stack_param_indices);
+        // We need to handle `nonnull` here where we still have access to function args.
+        for fn_attr in fn_attrs {
+            func.add_attribute(fn_attr);
+        }
         func
     }
 
diff --git a/src/intrinsic/mod.rs b/src/intrinsic/mod.rs
index 9caed459a292..4d0670d802d5 100644
--- a/src/intrinsic/mod.rs
+++ b/src/intrinsic/mod.rs
@@ -1197,7 +1197,7 @@ fn get_rust_try_fn<'a, 'gcc, 'tcx>(cx: &'a CodegenCx<'gcc, 'tcx>, codegen: &mut
 #[cfg(feature="master")]
 fn gen_fn<'a, 'gcc, 'tcx>(cx: &'a CodegenCx<'gcc, 'tcx>, name: &str, rust_fn_sig: ty::PolyFnSig<'tcx>, codegen: &mut dyn FnMut(Builder<'a, 'gcc, 'tcx>)) -> (Type<'gcc>, Function<'gcc>) {
     let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty());
-    let (typ, _, _, _) = fn_abi.gcc_type(cx);
+    let (typ, _, _, _, _) = fn_abi.gcc_type(cx);
     // FIXME(eddyb) find a nicer way to do this.
     cx.linkage.set(FunctionType::Internal);
     let func = cx.declare_fn(name, fn_abi);
diff --git a/src/type_of.rs b/src/type_of.rs
index c2eab295acd2..4563e32301f5 100644
--- a/src/type_of.rs
+++ b/src/type_of.rs
@@ -372,7 +372,8 @@ impl<'gcc, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
     }
 
     fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Type<'gcc> {
-        let (return_type, param_types, variadic, _) = fn_abi.gcc_type(self);
+        // // FIXME: Should we do something with `fn_attrs`?
+        let (return_type, param_types, variadic, _, _fn_attrs) = fn_abi.gcc_type(self);
         self.context.new_function_pointer_type(None, return_type, &param_types, variadic)
     }
 }

From 0348a5f17adbb922cdb2a7b75bfee4045fe1adf8 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez <guillaume1.gomez@gmail.com>
Date: Tue, 17 Oct 2023 21:55:36 +0200
Subject: [PATCH 033/297] Improve code readability

---
 src/abi.rs           | 34 +++++++++++++++++++++++++++-------
 src/declare.rs       | 15 ++++++++++-----
 src/intrinsic/mod.rs |  4 ++--
 src/type_of.rs       | 13 +++++++++----
 4 files changed, 48 insertions(+), 18 deletions(-)

diff --git a/src/abi.rs b/src/abi.rs
index a2825773bd39..5600f1ba8a9d 100644
--- a/src/abi.rs
+++ b/src/abi.rs
@@ -96,14 +96,22 @@ impl GccType for Reg {
     }
 }
 
+pub struct FnAbiGcc<'gcc> {
+    pub return_type: Type<'gcc>,
+    pub arguments_type: Vec<Type<'gcc>>,
+    pub is_c_variadic: bool,
+    pub on_stack_param_indices: FxHashSet<usize>,
+    pub fn_attributes: Vec<FnAttribute<'gcc>>,
+}
+
 pub trait FnAbiGccExt<'gcc, 'tcx> {
     // TODO(antoyo): return a function pointer type instead?
-    fn gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> (Type<'gcc>, Vec<Type<'gcc>>, bool, FxHashSet<usize>, Vec<FnAttribute<'gcc>>);
+    fn gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> FnAbiGcc<'gcc>;
     fn ptr_to_gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> Type<'gcc>;
 }
 
 impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
-    fn gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> (Type<'gcc>, Vec<Type<'gcc>>, bool, FxHashSet<usize>, Vec<FnAttribute<'gcc>>) {
+    fn gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> FnAbiGcc<'gcc> {
         let mut on_stack_param_indices = FxHashSet::default();
 
         // This capacity calculation is approximate.
@@ -111,7 +119,7 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             self.args.len() + if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 }
         );
 
-        let return_ty =
+        let return_type =
             match self.ret.mode {
                 PassMode::Ignore => cx.type_void(),
                 PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_gcc_type(cx),
@@ -185,13 +193,25 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         #[cfg(not(feature = "master"))]
         let fn_attrs = Vec::new();
 
-        (return_ty, argument_tys, self.c_variadic, on_stack_param_indices, fn_attrs)
+        FnAbiGcc {
+            return_type,
+            arguments_type: argument_tys,
+            is_c_variadic: self.c_variadic,
+            on_stack_param_indices,
+            fn_attributes: fn_attrs,
+        }
     }
 
     fn ptr_to_gcc_type(&self, cx: &CodegenCx<'gcc, 'tcx>) -> Type<'gcc> {
-        // FIXME: Should we do something with `fn_attrs`?
-        let (return_type, params, variadic, on_stack_param_indices, _fn_attrs) = self.gcc_type(cx);
-        let pointer_type = cx.context.new_function_pointer_type(None, return_type, &params, variadic);
+        // FIXME(antoyo): Should we do something with `FnAbiGcc::fn_attributes`?
+        let FnAbiGcc {
+            return_type,
+            arguments_type,
+            is_c_variadic,
+            on_stack_param_indices,
+            ..
+        } = self.gcc_type(cx);
+        let pointer_type = cx.context.new_function_pointer_type(None, return_type, &arguments_type, is_c_variadic);
         cx.on_stack_params.borrow_mut().insert(pointer_type.dyncast_function_ptr_type().expect("function ptr type"), on_stack_param_indices);
         pointer_type
     }
diff --git a/src/declare.rs b/src/declare.rs
index 409f112ca739..0b583c074dd2 100644
--- a/src/declare.rs
+++ b/src/declare.rs
@@ -6,7 +6,7 @@ use rustc_middle::ty::Ty;
 use rustc_span::Symbol;
 use rustc_target::abi::call::FnAbi;
 
-use crate::abi::FnAbiGccExt;
+use crate::abi::{FnAbiGcc, FnAbiGccExt};
 use crate::context::CodegenCx;
 use crate::intrinsic::llvm;
 
@@ -80,11 +80,16 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
     }
 
     pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Function<'gcc> {
-        let (return_type, params, variadic, on_stack_param_indices, fn_attrs) = fn_abi.gcc_type(self);
-        let func = declare_raw_fn(self, name, () /*fn_abi.llvm_cconv()*/, return_type, &params, variadic);
+        let FnAbiGcc {
+            return_type,
+            arguments_type,
+            is_c_variadic,
+            on_stack_param_indices,
+            fn_attributes,
+        } = fn_abi.gcc_type(self);
+        let func = declare_raw_fn(self, name, () /*fn_abi.llvm_cconv()*/, return_type, &arguments_type, is_c_variadic);
         self.on_stack_function_params.borrow_mut().insert(func, on_stack_param_indices);
-        // We need to handle `nonnull` here where we still have access to function args.
-        for fn_attr in fn_attrs {
+        for fn_attr in fn_attributes {
             func.add_attribute(fn_attr);
         }
         func
diff --git a/src/intrinsic/mod.rs b/src/intrinsic/mod.rs
index 4d0670d802d5..eaee1d453c53 100644
--- a/src/intrinsic/mod.rs
+++ b/src/intrinsic/mod.rs
@@ -1197,7 +1197,7 @@ fn get_rust_try_fn<'a, 'gcc, 'tcx>(cx: &'a CodegenCx<'gcc, 'tcx>, codegen: &mut
 #[cfg(feature="master")]
 fn gen_fn<'a, 'gcc, 'tcx>(cx: &'a CodegenCx<'gcc, 'tcx>, name: &str, rust_fn_sig: ty::PolyFnSig<'tcx>, codegen: &mut dyn FnMut(Builder<'a, 'gcc, 'tcx>)) -> (Type<'gcc>, Function<'gcc>) {
     let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty());
-    let (typ, _, _, _, _) = fn_abi.gcc_type(cx);
+    let return_type = fn_abi.gcc_type(cx).return_type;
     // FIXME(eddyb) find a nicer way to do this.
     cx.linkage.set(FunctionType::Internal);
     let func = cx.declare_fn(name, fn_abi);
@@ -1207,5 +1207,5 @@ fn gen_fn<'a, 'gcc, 'tcx>(cx: &'a CodegenCx<'gcc, 'tcx>, name: &str, rust_fn_sig
     let block = Builder::append_block(cx, func_val, "entry-block");
     let bx = Builder::build(cx, block);
     codegen(bx);
-    (typ, func)
+    (return_type, func)
 }
diff --git a/src/type_of.rs b/src/type_of.rs
index 4563e32301f5..1189e96e3087 100644
--- a/src/type_of.rs
+++ b/src/type_of.rs
@@ -9,7 +9,7 @@ use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_target::abi::{self, Abi, Align, F32, F64, FieldsShape, Int, Integer, Pointer, PointeeInfo, Size, TyAbiInterface, Variants};
 use rustc_target::abi::call::{CastTarget, FnAbi, Reg};
 
-use crate::abi::{FnAbiGccExt, GccType};
+use crate::abi::{FnAbiGcc, FnAbiGccExt, GccType};
 use crate::context::CodegenCx;
 use crate::type_::struct_fields;
 
@@ -372,8 +372,13 @@ impl<'gcc, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
     }
 
     fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Type<'gcc> {
-        // // FIXME: Should we do something with `fn_attrs`?
-        let (return_type, param_types, variadic, _, _fn_attrs) = fn_abi.gcc_type(self);
-        self.context.new_function_pointer_type(None, return_type, &param_types, variadic)
+        // FIXME(antoyo): Should we do something with `FnAbiGcc::fn_attributes`?
+        let FnAbiGcc {
+            return_type,
+            arguments_type,
+            is_c_variadic,
+            ..
+        } = fn_abi.gcc_type(self);
+        self.context.new_function_pointer_type(None, return_type, &arguments_type, is_c_variadic)
     }
 }

From 64abf5862ffb5b32f1555642550eb18f383fdc3a Mon Sep 17 00:00:00 2001
From: Andy Sadler <andrewsadler122@gmail.com>
Date: Thu, 5 Oct 2023 19:23:46 -0500
Subject: [PATCH 034/297] optimize popcount implementation

In the current implementation, the gcc backend of rustc currently emits the
following for a function that implements popcount for a u32 (x86_64 targeting
AVX2, using standard unix calling convention):

    popcount:
        mov     eax, edi
        and     edi, 1431655765
        shr     eax
        and     eax, 1431655765
        add     edi, eax
        mov     edx, edi
        and     edi, 858993459
        shr     edx, 2
        and     edx, 858993459
        add     edx, edi
        mov     eax, edx
        and     edx, 252645135
        shr     eax, 4
        and     eax, 252645135
        add     eax, edx
        mov     edx, eax
        and     eax, 16711935
        shr     edx, 8
        and     edx, 16711935
        add     edx, eax
        movzx   eax, dx
        shr     edx, 16
        add     eax, edx
        ret

Rather than using this implementation, gcc could be told to use Wenger's
algorithm.  This would give the same function the following implementation:

    popcount:
        xor eax, eax
        xor edx, edx
        popcnt eax, edi
        test edi, edi
        cmove eax, edx
        ret

This patch implements the popcount operation in terms of Wenger's algorithm in
all cases.

Signed-off-by: Andy Sadler <andrewsadler122@gmail.com>
---
 src/intrinsic/mod.rs | 90 +++++++++++++++++---------------------------
 1 file changed, 34 insertions(+), 56 deletions(-)

diff --git a/src/intrinsic/mod.rs b/src/intrinsic/mod.rs
index 9caed459a292..f0437bf4cc83 100644
--- a/src/intrinsic/mod.rs
+++ b/src/intrinsic/mod.rs
@@ -4,7 +4,7 @@ mod simd;
 #[cfg(feature="master")]
 use std::iter;
 
-use gccjit::{ComparisonOp, Function, RValue, ToRValue, Type, UnaryOp, FunctionType};
+use gccjit::{BinaryOp, ComparisonOp, Function, RValue, ToRValue, Type, UnaryOp, FunctionType};
 use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::base::wants_msvc_seh;
 use rustc_codegen_ssa::common::IntPredicate;
@@ -820,74 +820,52 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
             };
 
         if value_type.is_u128(&self.cx) {
-            // TODO(antoyo): implement in the normal algorithm below to have a more efficient
-            // implementation (that does not require a call to __popcountdi2).
-            let popcount = self.context.get_builtin_function("__builtin_popcountll");
             let sixty_four = self.gcc_int(value_type, 64);
             let right_shift = self.gcc_lshr(value, sixty_four);
             let high = self.gcc_int_cast(right_shift, self.cx.ulonglong_type);
-            let high = self.context.new_call(None, popcount, &[high]);
+            let high = self.pop_count(high);
             let low = self.gcc_int_cast(value, self.cx.ulonglong_type);
-            let low = self.context.new_call(None, popcount, &[low]);
+            let low = self.pop_count(low);
             let res = high + low;
             return self.gcc_int_cast(res, result_type);
         }
 
-        // First step.
-        let mask = self.context.new_rvalue_from_long(value_type, 0x5555555555555555);
-        let left = value & mask;
-        let shifted = value >> self.context.new_rvalue_from_int(value_type, 1);
-        let right = shifted & mask;
-        let value = left + right;
+        // Use Wenger's algorithm for population count, gcc's seems to play better with it
+        // for (int counter = 0; value != 0; counter++) {
+        //     value &= value - 1;
+        // }
+        let func = self.current_func.borrow().expect("func");
+        let loop_head = func.new_block("head");
+        let loop_body = func.new_block("body");
+        let loop_tail = func.new_block("tail");
 
-        // Second step.
-        let mask = self.context.new_rvalue_from_long(value_type, 0x3333333333333333);
-        let left = value & mask;
-        let shifted = value >> self.context.new_rvalue_from_int(value_type, 2);
-        let right = shifted & mask;
-        let value = left + right;
+        let counter_type = self.int_type;
+        let counter = self.current_func().new_local(None, counter_type, "popcount_counter");
+        let val = self.current_func().new_local(None, value_type, "popcount_value");
+        let zero = self.context.new_rvalue_zero(counter_type);
+        self.llbb().add_assignment(None, counter, zero);
+        self.llbb().add_assignment(None, val, value);
+        self.br(loop_head);
 
-        // Third step.
-        let mask = self.context.new_rvalue_from_long(value_type, 0x0F0F0F0F0F0F0F0F);
-        let left = value & mask;
-        let shifted = value >> self.context.new_rvalue_from_int(value_type, 4);
-        let right = shifted & mask;
-        let value = left + right;
+        // check if value isn't zero
+        self.switch_to_block(loop_head);
+        let zero = self.context.new_rvalue_zero(value_type);
+        let cond = self.context.new_comparison(None, ComparisonOp::NotEquals, val.to_rvalue(), zero);
+        self.cond_br(cond, loop_body, loop_tail);
 
-        if value_type.is_u8(&self.cx) {
-            return self.context.new_cast(None, value, result_type);
-        }
+        // val &= val - 1;
+        self.switch_to_block(loop_body);
+        let sub = val.to_rvalue() - self.context.new_rvalue_one(value_type);
+        loop_body.add_assignment_op(None, val, BinaryOp::BitwiseAnd, sub);
 
-        // Fourth step.
-        let mask = self.context.new_rvalue_from_long(value_type, 0x00FF00FF00FF00FF);
-        let left = value & mask;
-        let shifted = value >> self.context.new_rvalue_from_int(value_type, 8);
-        let right = shifted & mask;
-        let value = left + right;
+        // counter += 1
+        let one = self.context.new_rvalue_one(counter_type);
+        loop_body.add_assignment_op(None, counter, BinaryOp::Plus, one);
+        self.br(loop_head);
 
-        if value_type.is_u16(&self.cx) {
-            return self.context.new_cast(None, value, result_type);
-        }
-
-        // Fifth step.
-        let mask = self.context.new_rvalue_from_long(value_type, 0x0000FFFF0000FFFF);
-        let left = value & mask;
-        let shifted = value >> self.context.new_rvalue_from_int(value_type, 16);
-        let right = shifted & mask;
-        let value = left + right;
-
-        if value_type.is_u32(&self.cx) {
-            return self.context.new_cast(None, value, result_type);
-        }
-
-        // Sixth step.
-        let mask = self.context.new_rvalue_from_long(value_type, 0x00000000FFFFFFFF);
-        let left = value & mask;
-        let shifted = value >> self.context.new_rvalue_from_int(value_type, 32);
-        let right = shifted & mask;
-        let value = left + right;
-
-        self.context.new_cast(None, value, result_type)
+        // end of loop
+        self.switch_to_block(loop_tail);
+        self.context.new_cast(None, counter.to_rvalue(), result_type)
     }
 
     // Algorithm from: https://blog.regehr.org/archives/1063

From e5fa9f869287f85ddc6b3d72457d8c56669638f6 Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Sat, 7 Oct 2023 15:13:55 -0400
Subject: [PATCH 035/297] Use the correct alignment for integer types

---
 src/common.rs  | 16 ++++++++--------
 src/context.rs | 43 +++++++++++++++++++++++++------------------
 2 files changed, 33 insertions(+), 26 deletions(-)

diff --git a/src/common.rs b/src/common.rs
index 5f54cb16d8e2..93fe27e547ae 100644
--- a/src/common.rs
+++ b/src/common.rs
@@ -424,35 +424,35 @@ impl<'gcc, 'tcx> TypeReflection<'gcc, 'tcx> for Type<'gcc> {
     }
 
     fn is_i8(&self, cx: &CodegenCx<'gcc, 'tcx>) -> bool {
-        self.unqualified() == cx.i8_type
+        self.is_compatible_with(cx.i8_type)
     }
 
     fn is_u8(&self, cx: &CodegenCx<'gcc, 'tcx>) -> bool {
-        self.unqualified() == cx.u8_type
+        self.is_compatible_with(cx.u8_type)
     }
 
     fn is_i16(&self, cx: &CodegenCx<'gcc, 'tcx>) -> bool {
-        self.unqualified() == cx.i16_type
+        self.is_compatible_with(cx.i16_type)
     }
 
     fn is_u16(&self, cx: &CodegenCx<'gcc, 'tcx>) -> bool {
-        self.unqualified() == cx.u16_type
+        self.is_compatible_with(cx.u16_type)
     }
 
     fn is_i32(&self, cx: &CodegenCx<'gcc, 'tcx>) -> bool {
-        self.unqualified() == cx.i32_type
+        self.is_compatible_with(cx.i32_type)
     }
 
     fn is_u32(&self, cx: &CodegenCx<'gcc, 'tcx>) -> bool {
-        self.unqualified() == cx.u32_type
+        self.is_compatible_with(cx.u32_type)
     }
 
     fn is_i64(&self, cx: &CodegenCx<'gcc, 'tcx>) -> bool {
-        self.unqualified() == cx.i64_type
+        self.is_compatible_with(cx.i64_type)
     }
 
     fn is_u64(&self, cx: &CodegenCx<'gcc, 'tcx>) -> bool {
-        self.unqualified() == cx.u64_type
+        self.is_compatible_with(cx.u64_type)
     }
 
     fn is_i128(&self, cx: &CodegenCx<'gcc, 'tcx>) -> bool {
diff --git a/src/context.rs b/src/context.rs
index dcebd92a61c6..b01ac7b57afa 100644
--- a/src/context.rs
+++ b/src/context.rs
@@ -129,19 +129,25 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
     pub fn new(context: &'gcc Context<'gcc>, codegen_unit: &'tcx CodegenUnit<'tcx>, tcx: TyCtxt<'tcx>, supports_128bit_integers: bool) -> Self {
         let check_overflow = tcx.sess.overflow_checks();
 
-        let i8_type = context.new_c_type(CType::Int8t);
-        let i16_type = context.new_c_type(CType::Int16t);
-        let i32_type = context.new_c_type(CType::Int32t);
-        let i64_type = context.new_c_type(CType::Int64t);
-        let u8_type = context.new_c_type(CType::UInt8t);
-        let u16_type = context.new_c_type(CType::UInt16t);
-        let u32_type = context.new_c_type(CType::UInt32t);
-        let u64_type = context.new_c_type(CType::UInt64t);
+        let create_type = |ctype, rust_type| {
+            let layout = tcx.layout_of(ParamEnv::reveal_all().and(rust_type)).unwrap();
+            let align = layout.align.abi.bytes();
+            context.new_c_type(ctype).get_aligned(align)
+        };
+
+        let i8_type = create_type(CType::Int8t, tcx.types.i8);
+        let i16_type = create_type(CType::Int16t, tcx.types.i16);
+        let i32_type = create_type(CType::Int32t, tcx.types.i32);
+        let i64_type = create_type(CType::Int64t, tcx.types.i64);
+        let u8_type = create_type(CType::UInt8t, tcx.types.u8);
+        let u16_type = create_type(CType::UInt16t, tcx.types.u16);
+        let u32_type = create_type(CType::UInt32t, tcx.types.u32);
+        let u64_type = create_type(CType::UInt64t, tcx.types.u64);
 
         let (i128_type, u128_type) =
             if supports_128bit_integers {
-                let i128_type = context.new_c_type(CType::Int128t).get_aligned(8); // TODO(antoyo): should the alignment be hard-coded?;
-                let u128_type = context.new_c_type(CType::UInt128t).get_aligned(8); // TODO(antoyo): should the alignment be hard-coded?;
+                let i128_type = create_type(CType::Int128t, tcx.types.i128);
+                let u128_type = create_type(CType::UInt128t, tcx.types.u128);
                 (i128_type, u128_type)
             }
             else {
@@ -265,15 +271,16 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
     }
 
     pub fn is_native_int_type(&self, typ: Type<'gcc>) -> bool {
+        // TODO: cache those types to not query libgccjit everytime this is called.
         let types = [
-            self.u8_type,
-            self.u16_type,
-            self.u32_type,
-            self.u64_type,
-            self.i8_type,
-            self.i16_type,
-            self.i32_type,
-            self.i64_type,
+            self.context.new_c_type(CType::UInt8t),
+            self.context.new_c_type(CType::UInt16t),
+            self.context.new_c_type(CType::UInt32t),
+            self.context.new_c_type(CType::UInt64t),
+            self.context.new_c_type(CType::Int8t),
+            self.context.new_c_type(CType::Int16t),
+            self.context.new_c_type(CType::Int32t),
+            self.context.new_c_type(CType::Int64t),
         ];
 
         for native_type in types {

From 9d5e0ba1f51f61ba8ccaa6c37eef25ac497ea70c Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Tue, 17 Oct 2023 19:43:20 -0400
Subject: [PATCH 036/297] Fixes including fixing compilation for
 --no-default-features

---
 src/abi.rs     |  9 ++++++---
 src/context.rs | 33 +++++++++++++++++++++++----------
 src/declare.rs |  2 ++
 3 files changed, 31 insertions(+), 13 deletions(-)

diff --git a/src/abi.rs b/src/abi.rs
index 5600f1ba8a9d..f601cd95f2a6 100644
--- a/src/abi.rs
+++ b/src/abi.rs
@@ -1,4 +1,6 @@
-use gccjit::{FnAttribute, ToLValue, ToRValue, Type};
+#[cfg(feature = "master")]
+use gccjit::FnAttribute;
+use gccjit::{ToLValue, ToRValue, Type};
 use rustc_codegen_ssa::traits::{AbiBuilderMethods, BaseTypeMethods};
 use rustc_data_structures::fx::FxHashSet;
 use rustc_middle::bug;
@@ -101,6 +103,7 @@ pub struct FnAbiGcc<'gcc> {
     pub arguments_type: Vec<Type<'gcc>>,
     pub is_c_variadic: bool,
     pub on_stack_param_indices: FxHashSet<usize>,
+    #[cfg(feature = "master")]
     pub fn_attributes: Vec<FnAttribute<'gcc>>,
 }
 
@@ -129,6 +132,7 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
                     cx.type_void()
                 }
             };
+        #[cfg(feature = "master")]
         let mut non_null_args = Vec::new();
 
         #[cfg(feature = "master")]
@@ -190,14 +194,13 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
         } else {
             vec![FnAttribute::NonNull(non_null_args)]
         };
-        #[cfg(not(feature = "master"))]
-        let fn_attrs = Vec::new();
 
         FnAbiGcc {
             return_type,
             arguments_type: argument_tys,
             is_c_variadic: self.c_variadic,
             on_stack_param_indices,
+            #[cfg(feature = "master")]
             fn_attributes: fn_attrs,
         }
     }
diff --git a/src/context.rs b/src/context.rs
index b01ac7b57afa..243556a0e523 100644
--- a/src/context.rs
+++ b/src/context.rs
@@ -132,7 +132,21 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
         let create_type = |ctype, rust_type| {
             let layout = tcx.layout_of(ParamEnv::reveal_all().and(rust_type)).unwrap();
             let align = layout.align.abi.bytes();
-            context.new_c_type(ctype).get_aligned(align)
+            #[cfg(feature="master")]
+            {
+                context.new_c_type(ctype).get_aligned(align)
+            }
+            #[cfg(not(feature="master"))]
+            {
+                // Since libgccjit 12 doesn't contain the fix to compare aligned integer types,
+                // only align u128 and i128.
+                if layout.ty.int_size_and_signed(tcx).0.bytes() == 16 {
+                    context.new_c_type(ctype).get_aligned(align)
+                }
+                else {
+                    context.new_c_type(ctype)
+                }
+            }
         };
 
         let i8_type = create_type(CType::Int8t, tcx.types.i8);
@@ -271,16 +285,15 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
     }
 
     pub fn is_native_int_type(&self, typ: Type<'gcc>) -> bool {
-        // TODO: cache those types to not query libgccjit everytime this is called.
         let types = [
-            self.context.new_c_type(CType::UInt8t),
-            self.context.new_c_type(CType::UInt16t),
-            self.context.new_c_type(CType::UInt32t),
-            self.context.new_c_type(CType::UInt64t),
-            self.context.new_c_type(CType::Int8t),
-            self.context.new_c_type(CType::Int16t),
-            self.context.new_c_type(CType::Int32t),
-            self.context.new_c_type(CType::Int64t),
+            self.u8_type,
+            self.u16_type,
+            self.u32_type,
+            self.u64_type,
+            self.i8_type,
+            self.i16_type,
+            self.i32_type,
+            self.i64_type,
         ];
 
         for native_type in types {
diff --git a/src/declare.rs b/src/declare.rs
index 0b583c074dd2..247454fa58e1 100644
--- a/src/declare.rs
+++ b/src/declare.rs
@@ -85,10 +85,12 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
             arguments_type,
             is_c_variadic,
             on_stack_param_indices,
+            #[cfg(feature="master")]
             fn_attributes,
         } = fn_abi.gcc_type(self);
         let func = declare_raw_fn(self, name, () /*fn_abi.llvm_cconv()*/, return_type, &arguments_type, is_c_variadic);
         self.on_stack_function_params.borrow_mut().insert(func, on_stack_param_indices);
+        #[cfg(feature="master")]
         for fn_attr in fn_attributes {
             func.add_attribute(fn_attr);
         }

From 99bc37e075573ae8b49893d2bb62f4dfb0595067 Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Sun, 21 May 2023 21:20:46 -0400
Subject: [PATCH 037/297] Refactor CI scripts to have a different file for
 libgccjit 12

---
 .github/workflows/ci.yml       |  26 ++-----
 .github/workflows/failures.yml | 120 +++++++++++++++++++++++++++++++
 .github/workflows/gcc12.yml    | 124 +++++++++++++++++++++++++++++++++
 .ignore                        |  10 +++
 build_system/src/build.rs      |   4 +-
 5 files changed, 261 insertions(+), 23 deletions(-)
 create mode 100644 .github/workflows/failures.yml
 create mode 100644 .github/workflows/gcc12.yml
 create mode 100644 .ignore

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f075c744e457..a38120e26f8d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -19,9 +19,8 @@ jobs:
       fail-fast: false
       matrix:
         libgccjit_version:
-          - { gcc: "libgccjit.so", extra: "", env_extra: "", artifacts_branch: "master" }
-          - { gcc: "libgccjit_without_int128.so", extra: "", env_extra: "", artifacts_branch: "master-without-128bit-integers" }
-          - { gcc: "libgccjit12.so", extra: "--no-default-features", env_extra: "TEST_FLAGS='-Cpanic=abort -Zpanic-abort-tests'", artifacts_branch: "gcc12" }
+          - { gcc: "libgccjit.so", artifacts_branch: "master" }
+          - { gcc: "libgccjit_without_int128.so", artifacts_branch: "master-without-128bit-integers" }
         commands: [
           "--mini-tests",
           "--std-tests",
@@ -33,7 +32,6 @@ jobs:
           "--extended-regex-tests",
           "--test-successful-rustc --nb-parts 2 --current-part 0",
           "--test-successful-rustc --nb-parts 2 --current-part 1",
-          "--test-failing-rustc",
         ]
 
     steps:
@@ -48,12 +46,7 @@ jobs:
       # `llvm-14-tools` is needed to install the `FileCheck` binary which is used for asm tests.
       run: sudo apt-get install ninja-build ripgrep llvm-14-tools
 
-    - name: Install libgccjit12
-      if: matrix.libgccjit_version.gcc == 'libgccjit12.so'
-      run: sudo apt-get install libgccjit-12-dev
-
     - name: Download artifact
-      if: matrix.libgccjit_version.gcc != 'libgccjit12.so'
       uses: dawidd6/action-download-artifact@v2
       with:
           workflow: main.yml
@@ -65,11 +58,6 @@ jobs:
           search_artifacts: true # Because, instead, the action only check the last job ran and that won't work since we want multiple artifacts.
 
     - name: Setup path to libgccjit
-      if: matrix.libgccjit_version.gcc == 'libgccjit12.so'
-      run: echo /usr/lib/gcc/x86_64-linux-gnu/12 > gcc_path
-
-    - name: Setup path to libgccjit
-      if: matrix.libgccjit_version.gcc != 'libgccjit12.so'
       run: |
           sudo dpkg --force-overwrite -i gcc-13/gcc-13.deb
           echo /usr/lib/ > gcc_path
@@ -119,8 +107,8 @@ jobs:
     - name: Build
       run: |
         ./y.sh prepare --only-libcore
-        ${{ matrix.libgccjit_version.env_extra }} ./y.sh build ${{ matrix.libgccjit_version.extra }}
-        ${{ matrix.libgccjit_version.env_extra }} cargo test ${{ matrix.libgccjit_version.extra }}
+        ./y.sh build
+        cargo test
         ./clean_all.sh
 
     - name: Prepare dependencies
@@ -136,16 +124,12 @@ jobs:
         command: build
         args: --release
 
-    - name: Add more failing tests for GCC 12
-      if: ${{ matrix.libgccjit_version.gcc == 'libgccjit12.so' }}
-      run: cat failing-ui-tests12.txt >> failing-ui-tests.txt
-
     - name: Add more failing tests because the sysroot is not compiled with LTO
       run: cat failing-non-lto-tests.txt >> failing-ui-tests.txt
 
     - name: Run tests
       run: |
-        ${{ matrix.libgccjit_version.env_extra }} ./test.sh --release --clean --build-sysroot ${{ matrix.commands }} ${{ matrix.libgccjit_version.extra }}
+        ./test.sh --release --clean --build-sysroot ${{ matrix.commands }}
 
   duplicates:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/failures.yml b/.github/workflows/failures.yml
new file mode 100644
index 000000000000..e04648ab6306
--- /dev/null
+++ b/.github/workflows/failures.yml
@@ -0,0 +1,120 @@
+# TODO: refactor to avoid duplication with the ci.yml file.
+name: Failures
+
+on:
+  - pull_request
+
+permissions:
+  contents: read
+
+env:
+  # Enable backtraces for easier debugging
+  RUST_BACKTRACE: 1
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        libgccjit_version:
+          - { gcc: "libgccjit.so", artifacts_branch: "master" }
+          - { gcc: "libgccjit_without_int128.so", artifacts_branch: "master-without-128bit-integers" }
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - uses: actions/checkout@v3
+      with:
+        repository: llvm/llvm-project
+        path: llvm
+
+    - name: Install packages
+      # `llvm-14-tools` is needed to install the `FileCheck` binary which is used for asm tests.
+      run: sudo apt-get install ninja-build ripgrep llvm-14-tools
+
+    - name: Download artifact
+      uses: dawidd6/action-download-artifact@v2
+      with:
+          workflow: main.yml
+          name: gcc-13
+          path: gcc-13
+          repo: antoyo/gcc
+          branch: ${{ matrix.libgccjit_version.artifacts_branch }}
+          event: push
+          search_artifacts: true # Because, instead, the action only check the last job ran and that won't work since we want multiple artifacts.
+
+    - name: Setup path to libgccjit
+      run: |
+          sudo dpkg --force-overwrite -i gcc-13/gcc-13.deb
+          echo /usr/lib/ > gcc_path
+
+    - name: Set env
+      run: |
+        echo "LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
+        echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
+
+    - name: Set RUST_COMPILER_RT_ROOT
+      run: echo "RUST_COMPILER_RT_ROOT="${{ env.workspace }}/llvm/compiler-rt >> $GITHUB_ENV
+
+    - name: Cache cargo installed crates
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/bin
+        key: cargo-installed-crates2-ubuntu-latest
+
+    - name: Cache cargo registry
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/registry
+        key: ${{ runner.os }}-cargo-registry2-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo index
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/git
+        key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v3
+      with:
+        path: target
+        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain') }}
+
+    #- name: Cache rust repository
+      #uses: actions/cache@v3
+      #id: cache-rust-repository
+      #with:
+        #path: rust
+        #key: ${{ runner.os }}-packages-${{ hashFiles('rust/.git/HEAD') }}
+
+    - name: Build
+      run: |
+        ./y.sh prepare --only-libcore
+        ./y.sh build
+        cargo test
+        ./clean_all.sh
+
+    - name: Prepare dependencies
+      run: |
+        git config --global user.email "user@example.com"
+        git config --global user.name "User"
+        ./y.sh prepare
+
+    # Compile is a separate step, as the actions-rs/cargo action supports error annotations
+    - name: Compile
+      uses: actions-rs/cargo@v1.0.3
+      with:
+        command: build
+        args: --release
+
+    - name: Add more failing tests because the sysroot is not compiled with LTO
+      run: cat failing-non-lto-tests.txt >> failing-ui-tests.txt
+
+    - name: Run tests
+      id: tests
+      run: |
+        ./test.sh --release --clean --build-sysroot --test-failing-rustc | tee output_log
+        rg "test result" output_log >> $GITHUB_STEP_SUMMARY
diff --git a/.github/workflows/gcc12.yml b/.github/workflows/gcc12.yml
new file mode 100644
index 000000000000..59e0c5ad234d
--- /dev/null
+++ b/.github/workflows/gcc12.yml
@@ -0,0 +1,124 @@
+name: CI libgccjit 12
+
+on:
+  - push
+  - pull_request
+
+permissions:
+  contents: read
+
+env:
+  # Enable backtraces for easier debugging
+  RUST_BACKTRACE: 1
+  TEST_FLAGS: "-Cpanic=abort -Zpanic-abort-tests"
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        commands: [
+          "--mini-tests",
+          # TODO(antoyo): re-enable those commands when the build with libgccjit 12 is fixed.
+          #"--std-tests",
+          # FIXME: re-enable asm tests when GCC can emit in the right syntax.
+          # "--asm-tests",
+          #"--test-libcore",
+          #"--extended-rand-tests",
+          #"--extended-regex-example-tests",
+          #"--extended-regex-tests",
+          #"--test-successful-rustc --nb-parts 2 --current-part 0",
+          #"--test-successful-rustc --nb-parts 2 --current-part 1",
+          #"--test-failing-rustc",
+        ]
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - uses: actions/checkout@v3
+      with:
+        repository: llvm/llvm-project
+        path: llvm
+
+    - name: Install packages
+      # `llvm-14-tools` is needed to install the `FileCheck` binary which is used for asm tests.
+      run: sudo apt-get install ninja-build ripgrep llvm-14-tools libgccjit-12-dev
+
+    - name: Setup path to libgccjit
+      run: echo /usr/lib/gcc/x86_64-linux-gnu/12 > gcc_path
+
+    - name: Set env
+      run: |
+        echo "LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
+        echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
+
+    - name: Set RUST_COMPILER_RT_ROOT
+      run: echo "RUST_COMPILER_RT_ROOT="${{ env.workspace }}/llvm/compiler-rt >> $GITHUB_ENV
+
+    - name: Cache cargo installed crates
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/bin
+        key: cargo-installed-crates2-ubuntu-latest
+
+    - name: Cache cargo registry
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/registry
+        key: ${{ runner.os }}-cargo-registry2-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo index
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/git
+        key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v3
+      with:
+        path: target
+        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain') }}
+
+    #- name: Cache rust repository
+      ## We only clone the rust repository for rustc tests
+      #if: ${{ contains(matrix.commands, 'rustc') }}
+      #uses: actions/cache@v3
+      #id: cache-rust-repository
+      #with:
+        #path: rust
+        #key: ${{ runner.os }}-packages-${{ hashFiles('rust/.git/HEAD') }}
+
+    - name: Build
+      run: |
+        ./y.sh prepare --only-libcore
+        # TODO(antoyo): build the sysroot when the build with libgccjit 12 is fixed.
+        #./y.sh build --no-default-features
+        # TODO(antoyo): run the tests when we can build the sysroot with libgccjit 12.
+        #cargo test --no-default-features
+        ./clean_all.sh
+
+    - name: Prepare dependencies
+      run: |
+        git config --global user.email "user@example.com"
+        git config --global user.name "User"
+        ./y.sh prepare
+
+    # Compile is a separate step, as the actions-rs/cargo action supports error annotations
+    - name: Compile
+      uses: actions-rs/cargo@v1.0.3
+      with:
+        command: build
+        args: --release
+
+    - name: Add more failing tests for GCC 12
+      run: cat failing-ui-tests12.txt >> failing-ui-tests.txt
+
+    - name: Run tests
+      run: |
+        # TODO(antoyo): add --build-sysroot when the build with libgccjit 12 is fixed.
+        # FIXME(antoyo): we need to set GCC_EXEC_PREFIX so that the linker can find the linker plugin.
+        # Not sure why it's not found otherwise.
+        GCC_EXEC_PREFIX=/usr/lib/gcc/ ./test.sh --release --clean ${{ matrix.commands }} --no-default-features
diff --git a/.ignore b/.ignore
new file mode 100644
index 000000000000..d8d189e5c7c6
--- /dev/null
+++ b/.ignore
@@ -0,0 +1,10 @@
+!/build_sysroot/sysroot_src
+!/simple-raytracer
+!/regex
+!/rand
+!/test-backend
+!/gcc_path
+!/benchmarks
+!*gimple*
+!*asm*
+!.github
diff --git a/build_system/src/build.rs b/build_system/src/build.rs
index e2819c37ad9b..0428c6b2cdae 100644
--- a/build_system/src/build.rs
+++ b/build_system/src/build.rs
@@ -1,6 +1,6 @@
 use crate::config::set_config;
 use crate::utils::{
-    get_gcc_path, run_command, run_command_with_env, run_command_with_output_and_env, walk_dir,
+    get_gcc_path, run_command, run_command_with_output_and_env, walk_dir,
 };
 use std::collections::HashMap;
 use std::ffi::OsStr;
@@ -200,7 +200,7 @@ fn build_codegen(args: &BuildArg) -> Result<(), String> {
     for feature in &ref_features {
         command.push(feature);
     }
-    run_command_with_env(&command, None, Some(&env))?;
+    run_command_with_output_and_env(&command, None, Some(&env))?;
 
     let config = set_config(&mut env, &[], Some(&args.gcc_path))?;
 

From c7679c4831204dc2ac2f835fe5f0851baf45e9df Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Wed, 18 Oct 2023 19:04:47 -0400
Subject: [PATCH 038/297] Revome llvm-14-tools from failures CI

---
 .github/workflows/failures.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/failures.yml b/.github/workflows/failures.yml
index e04648ab6306..b2835cd36993 100644
--- a/.github/workflows/failures.yml
+++ b/.github/workflows/failures.yml
@@ -31,8 +31,7 @@ jobs:
         path: llvm
 
     - name: Install packages
-      # `llvm-14-tools` is needed to install the `FileCheck` binary which is used for asm tests.
-      run: sudo apt-get install ninja-build ripgrep llvm-14-tools
+      run: sudo apt-get install ninja-build ripgrep
 
     - name: Download artifact
       uses: dawidd6/action-download-artifact@v2

From 81c1f39a86295df471d5d7341a15c4bff44c2545 Mon Sep 17 00:00:00 2001
From: Andy Sadler <andrewsadler122@gmail.com>
Date: Tue, 17 Oct 2023 19:38:17 -0500
Subject: [PATCH 039/297] optimize u128/i128 popcounts further

Don't fall back on breaking apart the popcount operation if 128-bit
integers are natively supported.

Signed-off-by: Andy Sadler <andrewsadler122@gmail.com>
---
 src/intrinsic/mod.rs | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/src/intrinsic/mod.rs b/src/intrinsic/mod.rs
index 32cc724bb199..69927b28cd5f 100644
--- a/src/intrinsic/mod.rs
+++ b/src/intrinsic/mod.rs
@@ -4,7 +4,7 @@ mod simd;
 #[cfg(feature="master")]
 use std::iter;
 
-use gccjit::{BinaryOp, ComparisonOp, Function, RValue, ToRValue, Type, UnaryOp, FunctionType};
+use gccjit::{ComparisonOp, Function, RValue, ToRValue, Type, UnaryOp, FunctionType};
 use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::base::wants_msvc_seh;
 use rustc_codegen_ssa::common::IntPredicate;
@@ -819,7 +819,9 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
                 value
             };
 
-        if value_type.is_u128(&self.cx) {
+        // only break apart 128-bit ints if they're not natively supported
+        // TODO(antoyo): remove this if/when native 128-bit integers land in libgccjit
+        if value_type.is_u128(&self.cx) && !self.cx.supports_128bit_integers {
             let sixty_four = self.gcc_int(value_type, 64);
             let right_shift = self.gcc_lshr(value, sixty_four);
             let high = self.gcc_int_cast(right_shift, self.cx.ulonglong_type);
@@ -842,30 +844,33 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
         let counter_type = self.int_type;
         let counter = self.current_func().new_local(None, counter_type, "popcount_counter");
         let val = self.current_func().new_local(None, value_type, "popcount_value");
-        let zero = self.context.new_rvalue_zero(counter_type);
+        let zero = self.gcc_zero(counter_type);
         self.llbb().add_assignment(None, counter, zero);
         self.llbb().add_assignment(None, val, value);
         self.br(loop_head);
 
         // check if value isn't zero
         self.switch_to_block(loop_head);
-        let zero = self.context.new_rvalue_zero(value_type);
-        let cond = self.context.new_comparison(None, ComparisonOp::NotEquals, val.to_rvalue(), zero);
+        let zero = self.gcc_zero(value_type);
+        let cond = self.gcc_icmp(IntPredicate::IntNE, val.to_rvalue(), zero);
         self.cond_br(cond, loop_body, loop_tail);
 
         // val &= val - 1;
         self.switch_to_block(loop_body);
-        let sub = val.to_rvalue() - self.context.new_rvalue_one(value_type);
-        loop_body.add_assignment_op(None, val, BinaryOp::BitwiseAnd, sub);
+        let one = self.gcc_int(value_type, 1);
+        let sub = self.gcc_sub(val.to_rvalue(), one);
+        let op = self.gcc_and(val.to_rvalue(), sub);
+        loop_body.add_assignment(None, val, op);
 
         // counter += 1
-        let one = self.context.new_rvalue_one(counter_type);
-        loop_body.add_assignment_op(None, counter, BinaryOp::Plus, one);
+        let one = self.gcc_int(counter_type, 1);
+        let op = self.gcc_add(counter.to_rvalue(), one);
+        loop_body.add_assignment(None, counter, op);
         self.br(loop_head);
 
         // end of loop
         self.switch_to_block(loop_tail);
-        self.context.new_cast(None, counter.to_rvalue(), result_type)
+        self.gcc_int_cast(counter.to_rvalue(), result_type)
     }
 
     // Algorithm from: https://blog.regehr.org/archives/1063

From 67b28ac34b82242f02060fb14a1e1d22cccdb7b4 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Thu, 19 Oct 2023 16:06:43 +0000
Subject: [PATCH 040/297] s/Generator/Coroutine/

---
 example/std_example.rs | 2 +-
 src/type_of.rs         | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/example/std_example.rs b/example/std_example.rs
index 18f2ddcde126..5bcc2c3315c1 100644
--- a/example/std_example.rs
+++ b/example/std_example.rs
@@ -3,7 +3,7 @@
 #[cfg(feature="master")]
 use std::arch::x86_64::*;
 use std::io::Write;
-use std::ops::Generator;
+use std::ops::Coroutine;
 
 extern {
     pub fn printf(format: *const i8, ...) -> i32;
diff --git a/src/type_of.rs b/src/type_of.rs
index c2eab295acd2..357ce6226592 100644
--- a/src/type_of.rs
+++ b/src/type_of.rs
@@ -87,7 +87,7 @@ fn uncached_gcc_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, layout: TyAndLayout
         // FIXME(eddyb) producing readable type names for trait objects can result
         // in problematically distinct types due to HRTB and subtyping (see #47638).
         // ty::Dynamic(..) |
-        ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Generator(..) | ty::Str
+        ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Coroutine(..) | ty::Str
             if !cx.sess().fewer_names() =>
         {
             let mut name = with_no_trimmed_paths!(layout.ty.to_string());
@@ -98,10 +98,10 @@ fn uncached_gcc_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, layout: TyAndLayout
                     write!(&mut name, "::{}", def.variant(index).name).unwrap();
                 }
             }
-            if let (&ty::Generator(_, _, _), &Variants::Single { index }) =
+            if let (&ty::Coroutine(_, _, _), &Variants::Single { index }) =
                 (layout.ty.kind(), &layout.variants)
             {
-                write!(&mut name, "::{}", ty::GeneratorArgs::variant_name(index)).unwrap();
+                write!(&mut name, "::{}", ty::CoroutineArgs::variant_name(index)).unwrap();
             }
             Some(name)
         }

From a69046793849bd8bb28bd7cb3ecae0d8c5bed3c4 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Thu, 19 Oct 2023 21:46:28 +0000
Subject: [PATCH 041/297] s/generator/coroutine/

---
 example/std_example.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/example/std_example.rs b/example/std_example.rs
index 5bcc2c3315c1..dc0aad04a78f 100644
--- a/example/std_example.rs
+++ b/example/std_example.rs
@@ -1,4 +1,4 @@
-#![feature(core_intrinsics, generators, generator_trait, is_sorted)]
+#![feature(core_intrinsics, coroutines, coroutine_trait, is_sorted)]
 
 #[cfg(feature="master")]
 use std::arch::x86_64::*;

From 7425c560d3e53eb34fbdf8979981566ab8b344d1 Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Wed, 6 Sep 2023 19:01:04 -0400
Subject: [PATCH 042/297] Add comment

---
 .github/workflows/ci.yml                      |   8 -
 .github/workflows/failures.yml                |   8 -
 .github/workflows/gcc12.yml                   |   8 -
 .github/workflows/m68k.yml                    | 139 ++++++++++++++++
 .github/workflows/release.yml                 |   8 -
 .github/workflows/stdarch.yml                 |   8 -
 Readme.md                                     |  37 +++--
 build_sysroot/build_sysroot.sh                |   2 +-
 build_system/src/build.rs                     |  36 ++---
 build_system/src/config.rs                    |  36 +++--
 build_system/src/prepare.rs                   |  19 ++-
 config.sh                                     |  25 ++-
 ...001-Disable-libstd-and-libtest-dylib.patch |  39 +++++
 example/alloc_system.rs                       |   1 +
 example/mini_core_hello_world.rs              |   6 +-
 example/std_example.rs                        |  13 ++
 src/base.rs                                   |   7 +-
 src/context.rs                                |  28 ++--
 src/gcc_util.rs                               |   9 +-
 src/int.rs                                    | 152 ++++++++++++------
 src/intrinsic/mod.rs                          |  20 +--
 src/lib.rs                                    |   5 +-
 src/type_.rs                                  |   4 +-
 test.sh                                       |  36 ++++-
 tests/lang_tests_common.rs                    |  48 +++++-
 tests/run/asm.rs                              |  14 +-
 tests/run/empty_main.rs                       |   2 +-
 tests/run/int_overflow.rs                     | 138 ++--------------
 28 files changed, 523 insertions(+), 333 deletions(-)
 create mode 100644 .github/workflows/m68k.yml
 create mode 100644 cross_patches/0001-Disable-libstd-and-libtest-dylib.patch

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a38120e26f8d..65e7a697ab0d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -37,11 +37,6 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - uses: actions/checkout@v3
-      with:
-        repository: llvm/llvm-project
-        path: llvm
-
     - name: Install packages
       # `llvm-14-tools` is needed to install the `FileCheck` binary which is used for asm tests.
       run: sudo apt-get install ninja-build ripgrep llvm-14-tools
@@ -68,9 +63,6 @@ jobs:
         echo "LD_LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
         echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
 
-    - name: Set RUST_COMPILER_RT_ROOT
-      run: echo "RUST_COMPILER_RT_ROOT="${{ env.workspace }}/llvm/compiler-rt >> $GITHUB_ENV
-
     - name: Cache cargo installed crates
       uses: actions/cache@v3
       with:
diff --git a/.github/workflows/failures.yml b/.github/workflows/failures.yml
index b2835cd36993..e6a9716d18cc 100644
--- a/.github/workflows/failures.yml
+++ b/.github/workflows/failures.yml
@@ -25,11 +25,6 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - uses: actions/checkout@v3
-      with:
-        repository: llvm/llvm-project
-        path: llvm
-
     - name: Install packages
       run: sudo apt-get install ninja-build ripgrep
 
@@ -55,9 +50,6 @@ jobs:
         echo "LD_LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
         echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
 
-    - name: Set RUST_COMPILER_RT_ROOT
-      run: echo "RUST_COMPILER_RT_ROOT="${{ env.workspace }}/llvm/compiler-rt >> $GITHUB_ENV
-
     - name: Cache cargo installed crates
       uses: actions/cache@v3
       with:
diff --git a/.github/workflows/gcc12.yml b/.github/workflows/gcc12.yml
index 59e0c5ad234d..295f43acb385 100644
--- a/.github/workflows/gcc12.yml
+++ b/.github/workflows/gcc12.yml
@@ -37,11 +37,6 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - uses: actions/checkout@v3
-      with:
-        repository: llvm/llvm-project
-        path: llvm
-
     - name: Install packages
       # `llvm-14-tools` is needed to install the `FileCheck` binary which is used for asm tests.
       run: sudo apt-get install ninja-build ripgrep llvm-14-tools libgccjit-12-dev
@@ -55,9 +50,6 @@ jobs:
         echo "LD_LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
         echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
 
-    - name: Set RUST_COMPILER_RT_ROOT
-      run: echo "RUST_COMPILER_RT_ROOT="${{ env.workspace }}/llvm/compiler-rt >> $GITHUB_ENV
-
     - name: Cache cargo installed crates
       uses: actions/cache@v3
       with:
diff --git a/.github/workflows/m68k.yml b/.github/workflows/m68k.yml
new file mode 100644
index 000000000000..55ee0a212142
--- /dev/null
+++ b/.github/workflows/m68k.yml
@@ -0,0 +1,139 @@
+# TODO: check if qemu-user-static-binfmt is needed (perhaps to run some tests since it probably calls exec).
+
+name: m68k CI
+
+on:
+  - push
+  - pull_request
+
+permissions:
+  contents: read
+
+env:
+  # Enable backtraces for easier debugging
+  RUST_BACKTRACE: 1
+  # TODO: remove when confish.sh is removed.
+  OVERWRITE_TARGET_TRIPLE: m68k-unknown-linux-gnu
+
+jobs:
+  build:
+    runs-on: ubuntu-22.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        commands: [
+          "--mini-tests",
+          "--std-tests",
+          # TODO(antoyo): fix those on m68k.
+          #"--test-libcore",
+          #"--extended-rand-tests",
+          #"--extended-regex-example-tests",
+          #"--extended-regex-tests",
+          #"--test-successful-rustc --nb-parts 2 --current-part 0",
+          #"--test-successful-rustc --nb-parts 2 --current-part 1",
+          #"--test-failing-rustc",
+        ]
+
+    steps:
+    - name: Install packages
+      run: |
+        sudo apt-get update
+        sudo apt-get install qemu qemu-user-static
+
+    - uses: actions/checkout@v3
+
+    - name: Download GCC artifact
+      uses: dawidd6/action-download-artifact@v2
+      with:
+          workflow: m68k.yml
+          name: gcc-m68k-13
+          repo: cross-cg-gcc-tools/cross-gcc
+          branch: master
+          event: push
+
+    - name: Download VM artifact
+      uses: dawidd6/action-download-artifact@v2
+      with:
+          workflow: m68k.yml
+          name: debian-m68k
+          repo: cross-cg-gcc-tools/vms
+          branch: master
+          event: push
+
+    - name: Setup path to libgccjit
+      run: |
+          sudo dpkg -i gcc-m68k-13.deb
+          echo /usr/lib/ > gcc_path
+
+    - name: Set env
+      run: |
+        echo "LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
+        echo "LD_LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
+        echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
+
+    - name: Cache cargo installed crates
+      uses: actions/cache@v3
+      with:
+        path: ~/.cargo/bin
+        key: cargo-installed-crates2-ubuntu-latest
+
+    #- name: Cache cargo registry
+      #uses: actions/cache@v3
+      #with:
+        #path: ~/.cargo/registry
+        #key: ${{ runner.os }}-cargo-registry2-${{ hashFiles('**/Cargo.lock') }}
+
+    #- name: Cache cargo index
+      #uses: actions/cache@v3
+      #with:
+        #path: ~/.cargo/git
+        #key: ${{ runner.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}
+
+    - name: Cache cargo target dir
+      uses: actions/cache@v3
+      with:
+        path: target
+        key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain') }}
+
+    #- name: Cache rust repository
+      ## We only clone the rust repository for rustc tests
+      #if: ${{ contains(matrix.commands, 'rustc') }}
+      #uses: actions/cache@v3
+      #id: cache-rust-repository
+      #with:
+        #path: rust
+        #key: ${{ runner.os }}-packages-${{ hashFiles('rust/.git/HEAD') }}
+
+    - name: Prepare VM
+      run: |
+        mkdir vm
+        sudo mount debian-m68k.img vm
+        sudo cp $(which qemu-m68k-static) vm/usr/bin/
+
+    - name: Build
+      run: |
+        ./y.sh prepare --only-libcore --cross
+        ./y.sh build --target-triple m68k-unknown-linux-gnu
+        CG_GCC_TEST_TARGET=m68k-unknown-linux-gnu cargo test
+        ./clean_all.sh
+
+    - name: Prepare dependencies
+      run: |
+        git config --global user.email "user@example.com"
+        git config --global user.name "User"
+        ./y.sh prepare --cross
+
+    # Compile is a separate step, as the actions-rs/cargo action supports error annotations
+    - name: Compile
+      uses: actions-rs/cargo@v1.0.3
+      with:
+        command: build
+        args: --release
+
+    - name: Add more failing tests because the sysroot is not compiled with LTO
+      run: cat failing-non-lto-tests.txt >> failing-ui-tests.txt
+
+    - name: Run tests
+      run: |
+        ./test.sh --release --clean --build-sysroot ${{ matrix.commands }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index bd0415040e7e..ae1134177a77 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -26,11 +26,6 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - uses: actions/checkout@v3
-      with:
-        repository: llvm/llvm-project
-        path: llvm
-
     - name: Install packages
       run: sudo apt-get install ninja-build ripgrep
 
@@ -56,9 +51,6 @@ jobs:
         echo "LD_LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
         echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
 
-    - name: Set RUST_COMPILER_RT_ROOT
-      run: echo "RUST_COMPILER_RT_ROOT="${{ env.workspace }}/llvm/compiler-rt >> $GITHUB_ENV
-
     - name: Cache cargo installed crates
       uses: actions/cache@v3
       with:
diff --git a/.github/workflows/stdarch.yml b/.github/workflows/stdarch.yml
index 6c28326823cc..28ac3cb65422 100644
--- a/.github/workflows/stdarch.yml
+++ b/.github/workflows/stdarch.yml
@@ -26,11 +26,6 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - uses: actions/checkout@v3
-      with:
-        repository: llvm/llvm-project
-        path: llvm
-
     - name: Install packages
       run: sudo apt-get install ninja-build ripgrep
 
@@ -70,9 +65,6 @@ jobs:
         echo "LD_LIBRARY_PATH=$(cat gcc_path)" >> $GITHUB_ENV
         echo "workspace="$GITHUB_WORKSPACE >> $GITHUB_ENV
 
-    - name: Set RUST_COMPILER_RT_ROOT
-      run: echo "RUST_COMPILER_RT_ROOT="${{ env.workspace }}/llvm/compiler-rt >> $GITHUB_ENV
-
     - name: Cache cargo installed crates
       uses: actions/cache@v3
       with:
diff --git a/Readme.md b/Readme.md
index 6fad0707fbe9..2207bd35edb4 100644
--- a/Readme.md
+++ b/Readme.md
@@ -55,13 +55,6 @@ $ make check-jit RUNTESTFLAGS="-v -v -v jit.exp=jit.dg/test-asm.cc"
 $ dirname $(readlink -f `find . -name libgccjit.so`) > gcc_path
 ```
 
-You also need to set RUST_COMPILER_RT_ROOT:
-
-```bash
-$ git clone https://github.com/llvm/llvm-project llvm --depth 1 --single-branch
-$ export RUST_COMPILER_RT_ROOT="$PWD/llvm/compiler-rt"
-```
-
 Then you can run commands like this:
 
 ```bash
@@ -91,9 +84,17 @@ $ CHANNEL="release" $CG_GCCJIT_DIR/cargo.sh run
 
 If you compiled cg_gccjit in debug mode (aka you didn't pass `--release` to `./test.sh`) you should use `CHANNEL="debug"` instead or omit `CHANNEL="release"` completely.
 
+### LTO
+
 To use LTO, you need to set the variable `FAT_LTO=1` and `EMBED_LTO_BITCODE=1` in addition to setting `lto = "fat"` in the `Cargo.toml`.
 Don't set `FAT_LTO` when compiling the sysroot, though: only set `EMBED_LTO_BITCODE=1`.
 
+Failing to set `EMBED_LTO_BITCODE` will give you the following error:
+
+```
+error: failed to copy bitcode to object file: No such file or directory (os error 2)
+```
+
 ### Rustc
 
 > You should prefer using the Cargo method.
@@ -313,16 +314,20 @@ generate it in [gimple.md](./doc/gimple.md).
 
 #### Building libgccjit
 
- * Follow these instructions: https://preshing.com/20141119/how-to-build-a-gcc-cross-compiler/ with the following changes:
- * Configure gcc with `../gcc/configure --enable-host-shared --disable-multilib --enable-languages=c,jit,c++ --disable-bootstrap --enable-checking=release --prefix=/opt/m68k-gcc/ --target=m68k-linux --without-headers`.
- * Some shells, like fish, don't define the environment variable `$MACHTYPE`.
- * Add `CFLAGS="-Wno-error=attributes -g -O2"` at the end of the configure command for building glibc (`CFLAGS="-Wno-error=attributes -Wno-error=array-parameter -Wno-error=stringop-overflow -Wno-error=array-bounds -g -O2"` for glibc 2.31, which is useful for Debian).
+ * Follow the instructions on [this repo](https://github.com/cross-cg-gcc-tools/cross-gcc).
 
 #### Configuring rustc_codegen_gcc
 
- * Set `TARGET_TRIPLE="m68k-unknown-linux-gnu"` in config.sh.
- * Since rustc doesn't support this architecture yet, set it back to `TARGET_TRIPLE="mips-unknown-linux-gnu"` (or another target having the same attributes). Alternatively, create a [target specification file](https://book.avr-rust.com/005.1-the-target-specification-json-file.html) (note that the `arch` specified in this file must be supported by the rust compiler).
- * Set `linker='-Clinker=m68k-linux-gcc'`.
+ * Run `./y.sh prepare --cross` so that the sysroot is patched for the cross-compiling case.
  * Set the path to the cross-compiling libgccjit in `gcc_path`.
- * Comment the line: `context.add_command_line_option("-masm=intel");` in src/base.rs.
- * (might not be necessary) Disable the compilation of libstd.so (and possibly libcore.so?): Remove dylib from build_sysroot/sysroot_src/library/std/Cargo.toml.
+ * Make sure you have the linker for your target (for instance `m68k-unknown-linux-gnu-gcc`) in your `$PATH`. Currently, the linker name is hardcoded as being `$TARGET-gcc`. Specify the target when building the sysroot: `./y.sh build --target-triple m68k-unknown-linux-gnu`.
+ * Build your project by specifying the target: `OVERWRITE_TARGET_TRIPLE=m68k-unknown-linux-gnu ../cargo.sh build --target m68k-unknown-linux-gnu`.
+ * If the target is not yet supported by the Rust compiler, create a [target specification file](https://docs.rust-embedded.org/embedonomicon/custom-target.html) (note that the `arch` specified in this file must be supported by the rust compiler).
+
+If you get the following error:
+
+```
+/usr/bin/ld: unrecognised emulation mode: m68kelf
+```
+
+Make sure you set `gcc_path` to the install directory.
diff --git a/build_sysroot/build_sysroot.sh b/build_sysroot/build_sysroot.sh
index 851e9895ce2b..116fd36e7a7b 100755
--- a/build_sysroot/build_sysroot.sh
+++ b/build_sysroot/build_sysroot.sh
@@ -22,7 +22,7 @@ if [[ "$1" == "--release" ]]; then
     RUSTFLAGS="$RUSTFLAGS -Zmir-opt-level=3" cargo build --target $TARGET_TRIPLE --release
 else
     sysroot_channel='debug'
-    cargo build --target $TARGET_TRIPLE --features compiler_builtins/c
+    cargo build --target $TARGET_TRIPLE
 fi
 
 # Copy files to sysroot
diff --git a/build_system/src/build.rs b/build_system/src/build.rs
index 0428c6b2cdae..b013ca80705d 100644
--- a/build_system/src/build.rs
+++ b/build_system/src/build.rs
@@ -11,7 +11,7 @@ use std::path::Path;
 struct BuildArg {
     codegen_release_channel: bool,
     sysroot_release_channel: bool,
-    features: Vec<String>,
+    flags: Vec<String>,
     gcc_path: String,
 }
 
@@ -30,12 +30,12 @@ impl BuildArg {
                 "--release" => build_arg.codegen_release_channel = true,
                 "--release-sysroot" => build_arg.sysroot_release_channel = true,
                 "--no-default-features" => {
-                    build_arg.features.push("--no-default-features".to_string());
+                    build_arg.flags.push("--no-default-features".to_string());
                 }
                 "--features" => {
                     if let Some(arg) = args.next() {
-                        build_arg.features.push("--features".to_string());
-                        build_arg.features.push(arg.as_str().into());
+                        build_arg.flags.push("--features".to_string());
+                        build_arg.flags.push(arg.as_str().into());
                     } else {
                         return Err(
                             "Expected a value after `--features`, found nothing".to_string()
@@ -46,6 +46,15 @@ impl BuildArg {
                     Self::usage();
                     return Ok(None);
                 }
+                "--target-triple" => {
+                    if args.next().is_some() {
+                        // Handled in config.rs.
+                    } else {
+                        return Err(
+                            "Expected a value after `--target-triple`, found nothing".to_string()
+                        );
+                    }
+                }
                 arg => return Err(format!("Unknown argument `{}`", arg)),
             }
         }
@@ -61,6 +70,7 @@ impl BuildArg {
     --release-sysroot      : Build sysroot in release mode
     --no-default-features  : Add `--no-default-features` flag
     --features [arg]       : Add a new feature [arg]
+    --target-triple [arg]  : Set the target triple to [arg]
     --help                 : Show this help
 "#
         )
@@ -147,8 +157,6 @@ fn build_sysroot(
                 &"build",
                 &"--target",
                 &target_triple,
-                &"--features",
-                &"compiler_builtins/c",
             ],
             None,
             Some(env),
@@ -175,16 +183,6 @@ fn build_sysroot(
 fn build_codegen(args: &BuildArg) -> Result<(), String> {
     let mut env = HashMap::new();
 
-    let current_dir =
-        std::env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?;
-    if let Ok(rt_root) = std::env::var("RUST_COMPILER_RT_ROOT") {
-        env.insert("RUST_COMPILER_RT_ROOT".to_string(), rt_root);
-    } else {
-        env.insert(
-            "RUST_COMPILER_RT_ROOT".to_string(),
-            format!("{}", current_dir.join("llvm/compiler-rt").display()),
-        );
-    }
     env.insert("LD_LIBRARY_PATH".to_string(), args.gcc_path.clone());
     env.insert("LIBRARY_PATH".to_string(), args.gcc_path.clone());
 
@@ -196,9 +194,9 @@ fn build_codegen(args: &BuildArg) -> Result<(), String> {
     } else {
         env.insert("CHANNEL".to_string(), "debug".to_string());
     }
-    let ref_features = args.features.iter().map(|s| s.as_str()).collect::<Vec<_>>();
-    for feature in &ref_features {
-        command.push(feature);
+    let flags = args.flags.iter().map(|s| s.as_str()).collect::<Vec<_>>();
+    for flag in &flags {
+        command.push(flag);
     }
     run_command_with_output_and_env(&command, None, Some(&env))?;
 
diff --git a/build_system/src/config.rs b/build_system/src/config.rs
index 4f2e33f0f998..0f77943476fe 100644
--- a/build_system/src/config.rs
+++ b/build_system/src/config.rs
@@ -5,7 +5,6 @@ use std::env as std_env;
 pub struct ConfigInfo {
     pub target_triple: String,
     pub rustc_command: Vec<String>,
-    pub run_wrapper: Option<&'static str>,
 }
 
 // Returns the beginning for the command line of rustc.
@@ -30,22 +29,28 @@ pub fn set_config(
     };
     let host_triple = get_rustc_host_triple()?;
     let mut linker = None;
-    let mut target_triple = host_triple.as_str();
-    let mut run_wrapper = None;
-    // FIXME: handle this with a command line flag?
-    // let mut target_triple = "m68k-unknown-linux-gnu";
+    let mut target_triple = host_triple.clone();
+
+    // We skip binary name and the command.
+    let mut args = std::env::args().skip(2);
+
+    while let Some(arg) = args.next() {
+        match arg.as_str() {
+            "--target-triple" => {
+                if let Some(arg) = args.next() {
+                    target_triple = arg;
+                } else {
+                    return Err(
+                        "Expected a value after `--target-triple`, found nothing".to_string()
+                    );
+                }
+            },
+            _ => (),
+        }
+    }
 
     if host_triple != target_triple {
-        if target_triple == "m68k-unknown-linux-gnu" {
-            target_triple = "mips-unknown-linux-gnu";
-            linker = Some("-Clinker=m68k-linux-gcc");
-        } else if target_triple == "aarch64-unknown-linux-gnu" {
-            // We are cross-compiling for aarch64. Use the correct linker and run tests in qemu.
-            linker = Some("-Clinker=aarch64-linux-gnu-gcc");
-            run_wrapper = Some("qemu-aarch64 -L /usr/aarch64-linux-gnu");
-        } else {
-            return Err(format!("unknown non-native platform `{}`", target_triple));
-        }
+        linker = Some(format!("-Clinker={}-gcc", target_triple));
     }
     let current_dir =
         std_env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?;
@@ -120,6 +125,5 @@ pub fn set_config(
     Ok(ConfigInfo {
         target_triple: target_triple.to_string(),
         rustc_command,
-        run_wrapper,
     })
 }
diff --git a/build_system/src/prepare.rs b/build_system/src/prepare.rs
index b258ddf36648..d5d034c419c6 100644
--- a/build_system/src/prepare.rs
+++ b/build_system/src/prepare.rs
@@ -4,7 +4,7 @@ use crate::utils::{cargo_install, git_clone, run_command, run_command_with_outpu
 use std::fs;
 use std::path::Path;
 
-fn prepare_libcore(sysroot_path: &Path) -> Result<(), String> {
+fn prepare_libcore(sysroot_path: &Path, cross_compile: bool) -> Result<(), String> {
     let rustc_path = match get_rustc_path() {
         Some(path) => path,
         None => return Err("`rustc` path not found".to_string()),
@@ -87,6 +87,12 @@ fn prepare_libcore(sysroot_path: &Path) -> Result<(), String> {
             Ok(())
         },
     )?;
+    if cross_compile {
+        walk_dir("cross_patches", |_| Ok(()), |file_path: &Path| {
+            patches.push(file_path.to_path_buf());
+            Ok(())
+        })?;
+    }
     patches.sort();
     for file_path in patches {
         println!("[GIT] apply `{}`", file_path.display());
@@ -156,16 +162,19 @@ where
 }
 
 struct PrepareArg {
+    cross_compile: bool,
     only_libcore: bool,
 }
 
 impl PrepareArg {
     fn new() -> Result<Option<Self>, String> {
         let mut only_libcore = false;
+        let mut cross_compile = false;
 
         for arg in std::env::args().skip(2) {
             match arg.as_str() {
                 "--only-libcore" => only_libcore = true,
+                "--cross" => cross_compile = true,
                 "--help" => {
                     Self::usage();
                     return Ok(None);
@@ -173,7 +182,10 @@ impl PrepareArg {
                 a => return Err(format!("Unknown argument `{a}`")),
             }
         }
-        Ok(Some(Self { only_libcore }))
+        Ok(Some(Self {
+            cross_compile,
+            only_libcore,
+        }))
     }
 
     fn usage() {
@@ -182,6 +194,7 @@ impl PrepareArg {
 `prepare` command help:
 
     --only-libcore  : Only setup libcore and don't clone other repositories
+    --cross         : Apply the patches needed to do cross-compilation
     --help          : Show this help
 "#
         )
@@ -194,7 +207,7 @@ pub fn run() -> Result<(), String> {
         None => return Ok(()),
     };
     let sysroot_path = Path::new("build_sysroot");
-    prepare_libcore(sysroot_path)?;
+    prepare_libcore(sysroot_path, args.cross_compile)?;
 
     if !args.only_libcore {
         cargo_install("hyperfine")?;
diff --git a/config.sh b/config.sh
index c686df0c72a2..99ee9b054c6f 100644
--- a/config.sh
+++ b/config.sh
@@ -20,22 +20,21 @@ else
 fi
 
 HOST_TRIPLE=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ")
-TARGET_TRIPLE=$HOST_TRIPLE
-#TARGET_TRIPLE="m68k-unknown-linux-gnu"
+# TODO: remove $OVERWRITE_TARGET_TRIPLE when config.sh is removed.
+TARGET_TRIPLE="${OVERWRITE_TARGET_TRIPLE:-$HOST_TRIPLE}"
 
 linker=''
 RUN_WRAPPER=''
 if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
-   if [[ "$TARGET_TRIPLE" == "m68k-unknown-linux-gnu" ]]; then
-       TARGET_TRIPLE="mips-unknown-linux-gnu"
-       linker='-Clinker=m68k-linux-gcc'
-   elif [[ "$TARGET_TRIPLE" == "aarch64-unknown-linux-gnu" ]]; then
-      # We are cross-compiling for aarch64. Use the correct linker and run tests in qemu.
-      linker='-Clinker=aarch64-linux-gnu-gcc'
-      RUN_WRAPPER='qemu-aarch64 -L /usr/aarch64-linux-gnu'
-   else
-      echo "Unknown non-native platform"
-   fi
+    RUN_WRAPPER=run_in_vm
+    if [[ "$TARGET_TRIPLE" == "m68k-unknown-linux-gnu" ]]; then
+        linker='-Clinker=m68k-unknown-linux-gnu-gcc'
+    elif [[ "$TARGET_TRIPLE" == "aarch64-unknown-linux-gnu" ]]; then
+        # We are cross-compiling for aarch64. Use the correct linker and run tests in qemu.
+        linker='-Clinker=aarch64-linux-gnu-gcc'
+    else
+        echo "Unknown non-native platform"
+    fi
 fi
 
 # Since we don't support ThinLTO, disable LTO completely when not trying to do LTO.
@@ -60,4 +59,4 @@ export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH
 # NOTE: To avoid the -fno-inline errors, use /opt/gcc/bin/gcc instead of cc.
 # To do so, add a symlink for cc to /opt/gcc/bin/gcc in our PATH.
 # Another option would be to add the following Rust flag: -Clinker=/opt/gcc/bin/gcc
-export PATH="/opt/gcc/bin:$PATH"
+export PATH="/opt/gcc/bin:/opt/m68k-unknown-linux-gnu/bin:$PATH"
diff --git a/cross_patches/0001-Disable-libstd-and-libtest-dylib.patch b/cross_patches/0001-Disable-libstd-and-libtest-dylib.patch
new file mode 100644
index 000000000000..74d9c208a05a
--- /dev/null
+++ b/cross_patches/0001-Disable-libstd-and-libtest-dylib.patch
@@ -0,0 +1,39 @@
+From 966beefe08be6045bfcca26079b76a7a80413080 Mon Sep 17 00:00:00 2001
+From: None <none@example.com>
+Date: Thu, 28 Sep 2023 17:37:38 -0400
+Subject: [PATCH] Disable libstd and libtest dylib
+
+---
+ library/std/Cargo.toml  | 2 +-
+ library/test/Cargo.toml | 2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/library/std/Cargo.toml b/library/std/Cargo.toml
+index 5b21355..cb0c49b 100644
+--- a/library/std/Cargo.toml
++++ b/library/std/Cargo.toml
+@@ -9,7 +9,7 @@ description = "The Rust Standard Library"
+ edition = "2021"
+ 
+ [lib]
+-crate-type = ["dylib", "rlib"]
++crate-type = ["rlib"]
+ 
+ [dependencies]
+ alloc = { path = "../alloc", public = true }
+diff --git a/library/test/Cargo.toml b/library/test/Cargo.toml
+index 91a1abd..a58c160 100644
+--- a/library/test/Cargo.toml
++++ b/library/test/Cargo.toml
+@@ -4,7 +4,7 @@ version = "0.0.0"
+ edition = "2021"
+ 
+ [lib]
+-crate-type = ["dylib", "rlib"]
++crate-type = ["rlib"]
+ 
+ [dependencies]
+ getopts = { version = "0.2.21", features = ['rustc-dep-of-std'] }
+-- 
+2.42.0
+
diff --git a/example/alloc_system.rs b/example/alloc_system.rs
index 56ff84e4bdfb..201e4c73675c 100644
--- a/example/alloc_system.rs
+++ b/example/alloc_system.rs
@@ -9,6 +9,7 @@
 // add fast paths for low alignment values.
 #[cfg(any(target_arch = "x86",
               target_arch = "arm",
+              target_arch = "m68k",
               target_arch = "mips",
               target_arch = "mips32r6",
               target_arch = "powerpc",
diff --git a/example/mini_core_hello_world.rs b/example/mini_core_hello_world.rs
index c3aea5718154..40a1ad22c0e1 100644
--- a/example/mini_core_hello_world.rs
+++ b/example/mini_core_hello_world.rs
@@ -152,7 +152,8 @@ fn main() {
     let slice = &[0, 1] as &[i32];
     let slice_ptr = slice as *const [i32] as *const i32;
 
-    assert_eq!(slice_ptr as usize % 4, 0);
+    let align = intrinsics::min_align_of::<*const i32>();
+    assert_eq!(slice_ptr as usize % align, 0);
 
     //return;
 
@@ -186,7 +187,10 @@ fn main() {
         let a: &dyn SomeTrait = &"abc\0";
         a.object_safe();
 
+        #[cfg(target_arch="x86_64")]
         assert_eq!(intrinsics::size_of_val(a) as u8, 16);
+        #[cfg(target_arch="m68k")]
+        assert_eq!(intrinsics::size_of_val(a) as u8, 8);
         assert_eq!(intrinsics::size_of_val(&0u32) as u8, 4);
 
         assert_eq!(intrinsics::min_align_of::<u16>() as u8, 2);
diff --git a/example/std_example.rs b/example/std_example.rs
index 18f2ddcde126..2d57866c1d17 100644
--- a/example/std_example.rs
+++ b/example/std_example.rs
@@ -1,6 +1,7 @@
 #![feature(core_intrinsics, generators, generator_trait, is_sorted)]
 
 #[cfg(feature="master")]
+#[cfg(target_arch="x86_64")]
 use std::arch::x86_64::*;
 use std::io::Write;
 use std::ops::Generator;
@@ -95,6 +96,7 @@ fn main() {
 
     println!("{:?}", std::intrinsics::caller_location());
 
+    #[cfg(target_arch="x86_64")]
     #[cfg(feature="master")]
     unsafe {
         test_simd();
@@ -108,6 +110,7 @@ fn main() {
 }
 
 #[cfg(feature="master")]
+#[cfg(target_arch="x86_64")]
 #[target_feature(enable = "sse2")]
 unsafe fn test_simd() {
     let x = _mm_setzero_si128();
@@ -136,6 +139,7 @@ unsafe fn test_simd() {
 }
 
 #[cfg(feature="master")]
+#[cfg(target_arch="x86_64")]
 #[target_feature(enable = "sse2")]
 unsafe fn test_mm_slli_si128() {
     #[rustfmt::skip]
@@ -164,6 +168,7 @@ unsafe fn test_mm_slli_si128() {
 
 
 #[cfg(feature="master")]
+#[cfg(target_arch="x86_64")]
 #[target_feature(enable = "sse2")]
 unsafe fn test_mm_movemask_epi8() {
     #[rustfmt::skip]
@@ -178,6 +183,7 @@ unsafe fn test_mm_movemask_epi8() {
 }
 
 #[cfg(feature="master")]
+#[cfg(target_arch="x86_64")]
 #[target_feature(enable = "avx2")]
 unsafe fn test_mm256_movemask_epi8() {
     let a = _mm256_set1_epi8(-1);
@@ -187,6 +193,7 @@ unsafe fn test_mm256_movemask_epi8() {
 }
 
 #[cfg(feature="master")]
+#[cfg(target_arch="x86_64")]
 #[target_feature(enable = "sse2")]
 unsafe fn test_mm_add_epi8() {
     let a = _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
@@ -203,6 +210,7 @@ unsafe fn test_mm_add_epi8() {
 }
 
 #[cfg(feature="master")]
+#[cfg(target_arch="x86_64")]
 #[target_feature(enable = "sse2")]
 unsafe fn test_mm_add_pd() {
     let a = _mm_setr_pd(1.0, 2.0);
@@ -212,6 +220,7 @@ unsafe fn test_mm_add_pd() {
 }
 
 #[cfg(feature="master")]
+#[cfg(target_arch="x86_64")]
 fn assert_eq_m128i(x: std::arch::x86_64::__m128i, y: std::arch::x86_64::__m128i) {
     unsafe {
         assert_eq!(std::mem::transmute::<_, [u8; 16]>(x), std::mem::transmute::<_, [u8; 16]>(y));
@@ -219,6 +228,7 @@ fn assert_eq_m128i(x: std::arch::x86_64::__m128i, y: std::arch::x86_64::__m128i)
 }
 
 #[cfg(feature="master")]
+#[cfg(target_arch="x86_64")]
 #[target_feature(enable = "sse2")]
 pub unsafe fn assert_eq_m128d(a: __m128d, b: __m128d) {
     if _mm_movemask_pd(_mm_cmpeq_pd(a, b)) != 0b11 {
@@ -227,6 +237,7 @@ pub unsafe fn assert_eq_m128d(a: __m128d, b: __m128d) {
 }
 
 #[cfg(feature="master")]
+#[cfg(target_arch="x86_64")]
 #[target_feature(enable = "sse2")]
 unsafe fn test_mm_cvtsi128_si64() {
     let r = _mm_cvtsi128_si64(std::mem::transmute::<[i64; 2], _>([5, 0]));
@@ -234,6 +245,7 @@ unsafe fn test_mm_cvtsi128_si64() {
 }
 
 #[cfg(feature="master")]
+#[cfg(target_arch="x86_64")]
 #[target_feature(enable = "sse4.1")]
 unsafe fn test_mm_cvtepi8_epi16() {
     let a = _mm_set1_epi8(10);
@@ -247,6 +259,7 @@ unsafe fn test_mm_cvtepi8_epi16() {
 }
 
 #[cfg(feature="master")]
+#[cfg(target_arch="x86_64")]
 #[target_feature(enable = "sse4.1")]
 unsafe fn test_mm_extract_epi8() {
     #[rustfmt::skip]
diff --git a/src/base.rs b/src/base.rs
index 61da38f4b0db..3152357fe492 100644
--- a/src/base.rs
+++ b/src/base.rs
@@ -98,10 +98,11 @@ pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol, target_info: Lock
             .map(|string| &string[1..])
             .collect();
 
-        // TODO(antoyo): only set on x86 platforms.
-        context.add_command_line_option("-masm=intel");
+        if tcx.sess.target.arch == "x86" || tcx.sess.target.arch == "x86_64" {
+            context.add_command_line_option("-masm=intel");
+        }
 
-        if !disabled_features.contains("avx") {
+        if !disabled_features.contains("avx") && tcx.sess.target.arch == "x86_64" {
             // NOTE: we always enable AVX because the equivalent of llvm.x86.sse2.cmp.pd in GCC for
             // SSE2 is multiple builtins, so we use the AVX __builtin_ia32_cmppd instead.
             // FIXME(antoyo): use the proper builtins for llvm.x86.sse2.cmp.pd and similar.
diff --git a/src/context.rs b/src/context.rs
index 243556a0e523..a043660ea632 100644
--- a/src/context.rs
+++ b/src/context.rs
@@ -20,6 +20,7 @@ use rustc_target::abi::{call::FnAbi, HasDataLayout, PointeeInfo, Size, TargetDat
 use rustc_target::spec::{HasTargetSpec, Target, TlsModel};
 
 use crate::callee::get_fn;
+use crate::common::SignType;
 
 #[derive(Clone)]
 pub struct FuncSig<'gcc> {
@@ -165,13 +166,21 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
                 (i128_type, u128_type)
             }
             else {
-                let i128_type = context.new_array_type(None, i64_type, 2);
-                let u128_type = context.new_array_type(None, u64_type, 2);
+                /*let layout = tcx.layout_of(ParamEnv::reveal_all().and(tcx.types.i128)).unwrap();
+                let i128_align = layout.align.abi.bytes();
+                let layout = tcx.layout_of(ParamEnv::reveal_all().and(tcx.types.u128)).unwrap();
+                let u128_align = layout.align.abi.bytes();*/
+
+                // TODO(antoyo): re-enable the alignment when libgccjit fixed the issue in
+                // gcc_jit_context_new_array_constructor (it should not use reinterpret_cast).
+                let i128_type = context.new_array_type(None, i64_type, 2)/*.get_aligned(i128_align)*/;
+                let u128_type = context.new_array_type(None, u64_type, 2)/*.get_aligned(u128_align)*/;
                 (i128_type, u128_type)
             };
 
         let tls_model = to_gcc_tls_mode(tcx.sess.tls_model());
 
+        // TODO(antoyo): set alignment on those types as well.
         let float_type = context.new_type::<f32>();
         let double_type = context.new_type::<f64>();
 
@@ -187,14 +196,10 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
         let ulonglong_type = context.new_c_type(CType::ULongLong);
         let sizet_type = context.new_c_type(CType::SizeT);
 
-        let isize_type = context.new_c_type(CType::LongLong);
-        let usize_type = context.new_c_type(CType::ULongLong);
+        let usize_type = sizet_type;
+        let isize_type = usize_type;
         let bool_type = context.new_type::<bool>();
 
-        // TODO(antoyo): only have those assertions on x86_64.
-        assert_eq!(isize_type.get_size(), i64_type.get_size());
-        assert_eq!(usize_type.get_size(), u64_type.get_size());
-
         let mut functions = FxHashMap::default();
         let builtins = [
             "__builtin_unreachable", "abort", "__builtin_expect", "__builtin_add_overflow", "__builtin_mul_overflow",
@@ -212,7 +217,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
             functions.insert(builtin.to_string(), context.get_builtin_function(builtin));
         }
 
-        Self {
+        let mut cx = Self {
             check_overflow,
             codegen_unit,
             context,
@@ -274,7 +279,10 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
             pointee_infos: Default::default(),
             structs_as_pointer: Default::default(),
             cleanup_blocks: Default::default(),
-        }
+        };
+        // TODO(antoyo): instead of doing this, add SsizeT to libgccjit.
+        cx.isize_type = usize_type.to_signed(&cx);
+        cx
     }
 
     pub fn rvalue_as_function(&self, value: RValue<'gcc>) -> Function<'gcc> {
diff --git a/src/gcc_util.rs b/src/gcc_util.rs
index 0514c9988e0f..1248fdcd2599 100644
--- a/src/gcc_util.rs
+++ b/src/gcc_util.rs
@@ -198,9 +198,16 @@ pub fn check_tied_features(sess: &Session, features: &FxHashMap<&str, bool>) ->
     None
 }
 
+fn arch_to_gcc(name: &str) -> &str {
+    match name {
+        "M68020" => "68020",
+         _ => name,
+    }
+}
+
 fn handle_native(name: &str) -> &str {
     if name != "native" {
-        return name;
+        return arch_to_gcc(name);
     }
 
     #[cfg(feature="master")]
diff --git a/src/int.rs b/src/int.rs
index 5719f6a8cf50..ea8550d20f36 100644
--- a/src/int.rs
+++ b/src/int.rs
@@ -7,7 +7,9 @@ use std::convert::TryFrom;
 use gccjit::{ComparisonOp, FunctionType, RValue, ToRValue, Type, UnaryOp, BinaryOp};
 use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
 use rustc_codegen_ssa::traits::{BackendTypes, BaseTypeMethods, BuilderMethods, OverflowOp};
-use rustc_middle::ty::Ty;
+use rustc_middle::ty::{ParamEnv, Ty};
+use rustc_target::abi::{Endian, call::{ArgAbi, ArgAttributes, Conv, FnAbi, PassMode}};
+use rustc_target::spec;
 
 use crate::builder::ToGccComp;
 use crate::{builder::Builder, common::{SignType, TypeReflection}, context::CodegenCx};
@@ -37,11 +39,10 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
         }
         else {
             let element_type = typ.dyncast_array().expect("element type");
-            let values = [
+            self.from_low_high_rvalues(typ,
                 self.cx.context.new_unary_op(None, UnaryOp::BitwiseNegate, element_type, self.low(a)),
                 self.cx.context.new_unary_op(None, UnaryOp::BitwiseNegate, element_type, self.high(a)),
-            ];
-            self.cx.context.new_array_constructor(None, typ, &values)
+            )
         }
     }
 
@@ -100,7 +101,6 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
             let condition = self.gcc_icmp(IntPredicate::IntNE, self.gcc_and(b, sixty_four), zero);
             self.llbb().end_with_conditional(None, condition, then_block, else_block);
 
-            // TODO(antoyo): take endianness into account.
             let shift_value = self.gcc_sub(b, sixty_four);
             let high = self.high(a);
             let sign =
@@ -110,11 +110,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
                 else {
                     zero
                 };
-            let values = [
-                high >> shift_value,
-                sign,
-            ];
-            let array_value = self.context.new_array_constructor(None, a_type, &values);
+            let array_value = self.from_low_high_rvalues(a_type, high >> shift_value, sign);
             then_block.add_assignment(None, result, array_value);
             then_block.end_with_jump(None, after_block);
 
@@ -130,11 +126,10 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
             let casted_low = self.context.new_cast(None, self.low(a), unsigned_type);
             let shifted_low = casted_low >> self.context.new_cast(None, b, unsigned_type);
             let shifted_low = self.context.new_cast(None, shifted_low, native_int_type);
-            let values = [
+            let array_value = self.from_low_high_rvalues(a_type,
                 (high << shift_value) | shifted_low,
                 high >> b,
-            ];
-            let array_value = self.context.new_array_constructor(None, a_type, &values);
+            );
             actual_else_block.add_assignment(None, result, array_value);
             actual_else_block.end_with_jump(None, after_block);
 
@@ -314,18 +309,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
                                         _ => unreachable!(),
                                     },
                             };
-                        let a_type = lhs.get_type();
-                        let b_type = rhs.get_type();
-                        let param_a = self.context.new_parameter(None, a_type, "a");
-                        let param_b = self.context.new_parameter(None, b_type, "b");
-                        let result_field = self.context.new_field(None, a_type, "result");
-                        let overflow_field = self.context.new_field(None, self.bool_type, "overflow");
-                        let return_type = self.context.new_struct_type(None, "result_overflow", &[result_field, overflow_field]);
-                        let func = self.context.new_function(None, FunctionType::Extern, return_type.as_type(), &[param_a, param_b], func_name, false);
-                        let result = self.context.new_call(None, func, &[lhs, rhs]);
-                        let overflow = result.access_field(None, overflow_field);
-                        let int_result = result.access_field(None, result_field);
-                        return (int_result, overflow);
+                        return self.operation_with_overflow(func_name, lhs, rhs);
                     },
                     _ => {
                         match oop {
@@ -350,6 +334,54 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
         (res.dereference(None).to_rvalue(), overflow)
     }
 
+    pub fn operation_with_overflow(&self, func_name: &str, lhs: RValue<'gcc>, rhs: RValue<'gcc>) -> (RValue<'gcc>, RValue<'gcc>) {
+        let a_type = lhs.get_type();
+        let b_type = rhs.get_type();
+        let param_a = self.context.new_parameter(None, a_type, "a");
+        let param_b = self.context.new_parameter(None, b_type, "b");
+        let result_field = self.context.new_field(None, a_type, "result");
+        let overflow_field = self.context.new_field(None, self.bool_type, "overflow");
+
+        let ret_ty = Ty::new_tup(self.tcx, &[self.tcx.types.i128, self.tcx.types.bool]);
+        let layout = self.tcx.layout_of(ParamEnv::reveal_all().and(ret_ty)).unwrap();
+
+        let arg_abi = ArgAbi {
+            layout,
+            mode: PassMode::Direct(ArgAttributes::new()),
+        };
+        let mut fn_abi = FnAbi {
+            args: vec![arg_abi.clone(), arg_abi.clone()].into_boxed_slice(),
+            ret: arg_abi,
+            c_variadic: false,
+            fixed_count: 2,
+            conv: Conv::C,
+            can_unwind: false,
+        };
+        fn_abi.adjust_for_foreign_abi(self.cx, spec::abi::Abi::C {
+            unwind: false,
+        }).unwrap();
+
+        let indirect = matches!(fn_abi.ret.mode, PassMode::Indirect { .. });
+
+        let return_type = self.context.new_struct_type(None, "result_overflow", &[result_field, overflow_field]);
+        let result =
+            if indirect {
+                let return_value = self.current_func().new_local(None, return_type.as_type(), "return_value");
+                let return_param_type = return_type.as_type().make_pointer();
+                let return_param = self.context.new_parameter(None, return_param_type, "return_value");
+                let func = self.context.new_function(None, FunctionType::Extern, self.type_void(), &[return_param, param_a, param_b], func_name, false);
+                self.llbb().add_eval(None, self.context.new_call(None, func, &[return_value.get_address(None), lhs, rhs]));
+                return_value.to_rvalue()
+            }
+            else {
+                let func = self.context.new_function(None, FunctionType::Extern, return_type.as_type(), &[param_a, param_b], func_name, false);
+                self.context.new_call(None, func, &[lhs, rhs])
+            };
+        let overflow = result.access_field(None, overflow_field);
+        let int_result = result.access_field(None, result_field);
+        return (int_result, overflow);
+    }
+
     pub fn gcc_icmp(&mut self, op: IntPredicate, mut lhs: RValue<'gcc>, mut rhs: RValue<'gcc>) -> RValue<'gcc> {
         let a_type = lhs.get_type();
         let b_type = rhs.get_type();
@@ -468,11 +500,10 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
             a ^ b
         }
         else {
-            let values = [
+            self.from_low_high_rvalues(a_type,
                 self.low(a) ^ self.low(b),
                 self.high(a) ^ self.high(b),
-            ];
-            self.context.new_array_constructor(None, a_type, &values)
+            )
         }
     }
 
@@ -518,12 +549,10 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
             let condition = self.gcc_icmp(IntPredicate::IntNE, self.gcc_and(b, sixty_four), zero);
             self.llbb().end_with_conditional(None, condition, then_block, else_block);
 
-            // TODO(antoyo): take endianness into account.
-            let values = [
+            let array_value = self.from_low_high_rvalues(a_type,
                 zero,
                 self.low(a) << (b - sixty_four),
-            ];
-            let array_value = self.context.new_array_constructor(None, a_type, &values);
+            );
             then_block.add_assignment(None, result, array_value);
             then_block.end_with_jump(None, after_block);
 
@@ -534,16 +563,16 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
             b0_block.end_with_jump(None, after_block);
 
             // NOTE: cast low to its unsigned type in order to perform a logical right shift.
+            // TODO(antoyo): adjust this ^ comment.
             let unsigned_type = native_int_type.to_unsigned(&self.cx);
             let casted_low = self.context.new_cast(None, self.low(a), unsigned_type);
             let shift_value = self.context.new_cast(None, sixty_four - b, unsigned_type);
             let high_low = self.context.new_cast(None, casted_low >> shift_value, native_int_type);
-            let values = [
+
+            let array_value = self.from_low_high_rvalues(a_type,
                 self.low(a) << b,
                 (self.high(a) << b) | high_low,
-            ];
-
-            let array_value = self.context.new_array_constructor(None, a_type, &values);
+            );
             actual_else_block.add_assignment(None, result, array_value);
             actual_else_block.end_with_jump(None, after_block);
 
@@ -559,16 +588,16 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
         let arg_type = arg.get_type();
         if !self.is_native_int_type(arg_type) {
             let native_int_type = arg_type.dyncast_array().expect("get element type");
-            let lsb = self.context.new_array_access(None, arg, self.context.new_rvalue_from_int(self.int_type, 0)).to_rvalue();
+            let lsb = self.low(arg);
             let swapped_lsb = self.gcc_bswap(lsb, width / 2);
             let swapped_lsb = self.context.new_cast(None, swapped_lsb, native_int_type);
-            let msb = self.context.new_array_access(None, arg, self.context.new_rvalue_from_int(self.int_type, 1)).to_rvalue();
+            let msb = self.high(arg);
             let swapped_msb = self.gcc_bswap(msb, width / 2);
             let swapped_msb = self.context.new_cast(None, swapped_msb, native_int_type);
 
             // NOTE: we also need to swap the two elements here, in addition to swapping inside
             // the elements themselves like done above.
-            return self.context.new_array_constructor(None, arg_type, &[swapped_msb, swapped_lsb]);
+            return self.from_low_high_rvalues(arg_type, swapped_msb, swapped_lsb);
         }
 
         // TODO(antoyo): check if it's faster to use string literals and a
@@ -672,11 +701,10 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
         else {
             assert!(!a_native && !b_native, "both types should either be native or non-native for or operation");
             let native_int_type = a_type.dyncast_array().expect("get element type");
-            let values = [
+            self.from_low_high_rvalues(a_type,
                 self.context.new_binary_op(None, operation, native_int_type, self.low(a), self.low(b)),
                 self.context.new_binary_op(None, operation, native_int_type, self.high(a), self.high(b)),
-            ];
-            self.context.new_array_constructor(None, a_type, &values)
+            )
         }
     }
 
@@ -700,11 +728,10 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
             let zero = self.context.new_rvalue_zero(value_type);
             let is_negative = self.context.new_comparison(None, ComparisonOp::LessThan, value, zero);
             let is_negative = self.gcc_int_cast(is_negative, dest_element_type);
-            let values = [
+            self.from_low_high_rvalues(dest_typ,
                 self.context.new_cast(None, value, dest_element_type),
                 self.context.new_unary_op(None, UnaryOp::Minus, dest_element_type, is_negative),
-            ];
-            self.context.new_array_constructor(None, dest_typ, &values)
+            )
         }
         else {
             // Since u128 and i128 are the only types that can be unsupported, we know the type of
@@ -782,20 +809,47 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
     }
 
     fn high(&self, value: RValue<'gcc>) -> RValue<'gcc> {
-        self.context.new_array_access(None, value, self.context.new_rvalue_from_int(self.int_type, 1))
+        let index =
+            match self.sess().target.options.endian {
+                Endian::Little => 1,
+                Endian::Big => 0,
+            };
+        self.context.new_array_access(None, value, self.context.new_rvalue_from_int(self.int_type, index))
             .to_rvalue()
     }
 
     fn low(&self, value: RValue<'gcc>) -> RValue<'gcc> {
-        self.context.new_array_access(None, value, self.context.new_rvalue_from_int(self.int_type, 0))
+        let index =
+            match self.sess().target.options.endian {
+                Endian::Little => 0,
+                Endian::Big => 1,
+            };
+        self.context.new_array_access(None, value, self.context.new_rvalue_from_int(self.int_type, index))
             .to_rvalue()
     }
 
+    fn from_low_high_rvalues(&self, typ: Type<'gcc>, low: RValue<'gcc>, high: RValue<'gcc>) -> RValue<'gcc> {
+        let (first, last) =
+            match self.sess().target.options.endian {
+                Endian::Little => (low, high),
+                Endian::Big => (high, low),
+            };
+
+        let values = [first, last];
+        self.context.new_array_constructor(None, typ, &values)
+    }
+
     fn from_low_high(&self, typ: Type<'gcc>, low: i64, high: i64) -> RValue<'gcc> {
+        let (first, last) =
+            match self.sess().target.options.endian {
+                Endian::Little => (low, high),
+                Endian::Big => (high, low),
+            };
+
         let native_int_type = typ.dyncast_array().expect("get element type");
         let values = [
-            self.context.new_rvalue_from_long(native_int_type, low),
-            self.context.new_rvalue_from_long(native_int_type, high),
+            self.context.new_rvalue_from_long(native_int_type, first),
+            self.context.new_rvalue_from_long(native_int_type, last),
         ];
         self.context.new_array_constructor(None, typ, &values)
     }
diff --git a/src/intrinsic/mod.rs b/src/intrinsic/mod.rs
index 69927b28cd5f..bfe27c0552f0 100644
--- a/src/intrinsic/mod.rs
+++ b/src/intrinsic/mod.rs
@@ -930,15 +930,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
                             128 => "__rust_i128_addo",
                             _ => unreachable!(),
                         };
-                    let param_a = self.context.new_parameter(None, result_type, "a");
-                    let param_b = self.context.new_parameter(None, result_type, "b");
-                    let result_field = self.context.new_field(None, result_type, "result");
-                    let overflow_field = self.context.new_field(None, self.bool_type, "overflow");
-                    let return_type = self.context.new_struct_type(None, "result_overflow", &[result_field, overflow_field]);
-                    let func = self.context.new_function(None, FunctionType::Extern, return_type.as_type(), &[param_a, param_b], func_name, false);
-                    let result = self.context.new_call(None, func, &[lhs, rhs]);
-                    let overflow = result.access_field(None, overflow_field);
-                    let int_result = result.access_field(None, result_field);
+                    let (int_result, overflow) = self.operation_with_overflow(func_name, lhs, rhs);
                     self.llbb().add_assignment(None, res, int_result);
                     overflow
                 };
@@ -1000,15 +992,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
                             128 => "__rust_i128_subo",
                             _ => unreachable!(),
                         };
-                    let param_a = self.context.new_parameter(None, result_type, "a");
-                    let param_b = self.context.new_parameter(None, result_type, "b");
-                    let result_field = self.context.new_field(None, result_type, "result");
-                    let overflow_field = self.context.new_field(None, self.bool_type, "overflow");
-                    let return_type = self.context.new_struct_type(None, "result_overflow", &[result_field, overflow_field]);
-                    let func = self.context.new_function(None, FunctionType::Extern, return_type.as_type(), &[param_a, param_b], func_name, false);
-                    let result = self.context.new_call(None, func, &[lhs, rhs]);
-                    let overflow = result.access_field(None, overflow_field);
-                    let int_result = result.access_field(None, result_field);
+                    let (int_result, overflow) = self.operation_with_overflow(func_name, lhs, rhs);
                     self.llbb().add_assignment(None, res, int_result);
                     overflow
                 };
diff --git a/src/lib.rs b/src/lib.rs
index fe2339305604..2355cd1f6960 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -241,8 +241,9 @@ impl ExtraBackendMethods for GccCodegenBackend {
             temp_dir: None,
         };
 
-        // TODO(antoyo): only set for x86.
-        mods.context.add_command_line_option("-masm=intel");
+        if tcx.sess.target.arch == "x86" || tcx.sess.target.arch == "x86_64" {
+            mods.context.add_command_line_option("-masm=intel");
+        }
         unsafe { allocator::codegen(tcx, &mut mods, module_name, kind, alloc_error_handler_kind); }
         mods
     }
diff --git a/src/type_.rs b/src/type_.rs
index 31899740514a..4914792c7b12 100644
--- a/src/type_.rs
+++ b/src/type_.rs
@@ -119,11 +119,11 @@ impl<'gcc, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
     }
 
     fn type_f32(&self) -> Type<'gcc> {
-        self.context.new_type::<f32>()
+        self.float_type
     }
 
     fn type_f64(&self) -> Type<'gcc> {
-        self.context.new_type::<f64>()
+        self.double_type
     }
 
     fn type_func(&self, params: &[Type<'gcc>], return_type: Type<'gcc>) -> Type<'gcc> {
diff --git a/test.sh b/test.sh
index e4cbd6fbcaff..2eceee7c1e9d 100755
--- a/test.sh
+++ b/test.sh
@@ -151,7 +151,11 @@ function clean() {
 
 function mini_tests() {
     echo "[BUILD] mini_core"
-    $RUSTC example/mini_core.rs --crate-name mini_core --crate-type lib,dylib --target $TARGET_TRIPLE
+    crate_types="lib,dylib"
+    if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
+        crate_types="lib"
+    fi
+    $RUSTC example/mini_core.rs --crate-name mini_core --crate-type $crate_types --target $TARGET_TRIPLE
 
     echo "[BUILD] example"
     $RUSTC example/example.rs --crate-type lib --target $TARGET_TRIPLE
@@ -166,6 +170,23 @@ function build_sysroot() {
     time ./build_sysroot/build_sysroot.sh $sysroot_channel
 }
 
+# TODO(GuillaumeGomez): when rewriting in Rust, refactor with the code in tests/lang_tests_common.rs if possible.
+function run_in_vm() {
+    vm_parent_dir=${CG_GCC_VM_DIR:-$(pwd)}
+    vm_dir=vm
+    exe=$1
+    exe_filename=$(basename $exe)
+    vm_home_dir=$vm_parent_dir/$vm_dir/home
+    vm_exe_path=$vm_home_dir/$exe_filename
+    inside_vm_exe_path=/home/$exe_filename
+    sudo cp $exe $vm_exe_path
+
+    shift
+    pushd $vm_parent_dir
+    sudo chroot $vm_dir qemu-m68k-static $inside_vm_exe_path $@
+    popd
+}
+
 function std_tests() {
     echo "[AOT] arbitrary_self_types_pointers_and_wrappers"
     $RUSTC example/arbitrary_self_types_pointers_and_wrappers.rs --crate-name arbitrary_self_types_pointers_and_wrappers --crate-type bin --target $TARGET_TRIPLE
@@ -174,9 +195,12 @@ function std_tests() {
     echo "[AOT] alloc_system"
     $RUSTC example/alloc_system.rs --crate-type lib --target "$TARGET_TRIPLE"
 
-    echo "[AOT] alloc_example"
-    $RUSTC example/alloc_example.rs --crate-type bin --target $TARGET_TRIPLE
-    $RUN_WRAPPER ./target/out/alloc_example
+    # FIXME: doesn't work on m68k.
+    if [[ "$HOST_TRIPLE" == "$TARGET_TRIPLE" ]]; then
+        echo "[AOT] alloc_example"
+        $RUSTC example/alloc_example.rs --crate-type bin --target $TARGET_TRIPLE
+        $RUN_WRAPPER ./target/out/alloc_example
+    fi
 
     echo "[AOT] dst_field_align"
     # FIXME(antoyo): Re-add -Zmir-opt-level=2 once rust-lang/rust#67529 is fixed.
@@ -225,7 +249,7 @@ verbose-tests = true
 [build]
 cargo = "$(rustup which cargo)"
 local-rebuild = true
-rustc = "$HOME/.rustup/toolchains/$rust_toolchain-$TARGET_TRIPLE/bin/rustc"
+rustc = "$HOME/.rustup/toolchains/$rust_toolchain-$HOST_TRIPLE/bin/rustc"
 
 [target.x86_64-unknown-linux-gnu]
 llvm-filecheck = "`which FileCheck-10 || which FileCheck-11 || which FileCheck-12 || which FileCheck-13 || which FileCheck-14`"
@@ -393,7 +417,7 @@ function test_rustc() {
     fi
 
     echo "[TEST] rustc test suite"
-    COMPILETEST_FORCE_STAGE0=1 ./x.py test --run always --stage 0 tests/ui/ --rustc-args "$RUSTC_ARGS"
+    COMPILETEST_FORCE_STAGE0=1 ./x.py test --run always --stage 0 tests/ui/ --rustc-args "$RUSTC_ARGS" # --target $TARGET_TRIPLE
 }
 
 function test_failing_rustc() {
diff --git a/tests/lang_tests_common.rs b/tests/lang_tests_common.rs
index 06de26f7efc9..940c7cfd2662 100644
--- a/tests/lang_tests_common.rs
+++ b/tests/lang_tests_common.rs
@@ -2,7 +2,7 @@
 use std::{
     env::{self, current_dir},
     path::PathBuf,
-    process::Command,
+    process::{self, Command},
 };
 
 use lang_tester::LangTester;
@@ -50,6 +50,19 @@ pub fn main_inner(profile: Profile) {
                 "-o", exe.to_str().expect("to_str"),
                 path.to_str().expect("to_str"),
             ]);
+
+            // TODO(antoyo): find a way to send this via a cli argument.
+            let test_target = std::env::var("CG_GCC_TEST_TARGET");
+            if let Ok(ref target) = test_target {
+                compiler.args(&["--target", &target]);
+                let linker = format!("{}-gcc", target);
+                compiler.args(&[format!("-Clinker={}", linker)]);
+                let mut env_path = std::env::var("PATH").unwrap_or_default();
+                // TODO(antoyo): find a better way to add the PATH necessary locally.
+                env_path = format!("/opt/m68k-unknown-linux-gnu/bin:{}", env_path);
+                compiler.env("PATH", env_path);
+            }
+
             if let Some(flags) = option_env!("TEST_FLAGS") {
                 for flag in flags.split_whitespace() {
                     compiler.arg(&flag);
@@ -65,8 +78,37 @@ pub fn main_inner(profile: Profile) {
                 }
             }
             // Test command 2: run `tempdir/x`.
-            let runtime = Command::new(exe);
-            vec![("Compiler", compiler), ("Run-time", runtime)]
+            if test_target.is_ok() {
+                let vm_parent_dir = std::env::var("CG_GCC_VM_DIR")
+                    .map(|dir| PathBuf::from(dir))
+                    .unwrap_or_else(|_| std::env::current_dir().unwrap());
+                let vm_dir = "vm";
+                let exe_filename = exe.file_name().unwrap();
+                let vm_home_dir = vm_parent_dir.join(vm_dir).join("home");
+                let vm_exe_path = vm_home_dir.join(exe_filename);
+                // FIXME(antoyo): panicking here makes the test pass.
+                let inside_vm_exe_path = PathBuf::from("/home").join(&exe_filename);
+                let mut copy = Command::new("sudo");
+                copy.arg("cp");
+                copy.args(&[&exe, &vm_exe_path]);
+
+                let mut runtime = Command::new("sudo");
+                runtime.args(&["chroot", vm_dir, "qemu-m68k-static"]);
+                runtime.arg(inside_vm_exe_path);
+                runtime.current_dir(vm_parent_dir);
+                vec![
+                    ("Compiler", compiler),
+                    ("Copy", copy),
+                    ("Run-time", runtime),
+                ]
+            }
+            else {
+                let runtime = Command::new(exe);
+                vec![
+                    ("Compiler", compiler),
+                    ("Run-time", runtime),
+                ]
+            }
         })
         .run();
 }
diff --git a/tests/run/asm.rs b/tests/run/asm.rs
index 507b65ca049e..56f2aac3d0a1 100644
--- a/tests/run/asm.rs
+++ b/tests/run/asm.rs
@@ -5,8 +5,10 @@
 
 #![feature(asm_const)]
 
+#[cfg(target_arch="x86_64")]
 use std::arch::{asm, global_asm};
 
+#[cfg(target_arch="x86_64")]
 global_asm!(
     "
     .global add_asm
@@ -20,6 +22,7 @@ extern "C" {
     fn add_asm(a: i64, b: i64) -> i64;
 }
 
+#[cfg(target_arch="x86_64")]
 pub unsafe fn mem_cpy(dst: *mut u8, src: *const u8, len: usize) {
     asm!(
         "rep movsb",
@@ -30,7 +33,8 @@ pub unsafe fn mem_cpy(dst: *mut u8, src: *const u8, len: usize) {
     );
 }
 
-fn main() {
+#[cfg(target_arch="x86_64")]
+fn asm() {
     unsafe {
         asm!("nop");
     }
@@ -173,3 +177,11 @@ fn main() {
     }
     assert_eq!(array1, array2);
 }
+
+#[cfg(not(target_arch="x86_64"))]
+fn asm() {
+}
+
+fn main() {
+    asm();
+}
diff --git a/tests/run/empty_main.rs b/tests/run/empty_main.rs
index 2d78ef12aa72..e66a859ad698 100644
--- a/tests/run/empty_main.rs
+++ b/tests/run/empty_main.rs
@@ -35,6 +35,6 @@ pub(crate) unsafe auto trait Freeze {}
  */
 
 #[start]
-fn main(mut argc: isize, _argv: *const *const u8) -> isize {
+fn main(_argc: isize, _argv: *const *const u8) -> isize {
     0
 }
diff --git a/tests/run/int_overflow.rs b/tests/run/int_overflow.rs
index 08fa087fccdf..78872159f62d 100644
--- a/tests/run/int_overflow.rs
+++ b/tests/run/int_overflow.rs
@@ -4,138 +4,20 @@
 //   stdout: Success
 //   status: signal
 
-#![allow(internal_features, unused_attributes)]
-#![feature(auto_traits, lang_items, no_core, start, intrinsics, rustc_attrs)]
+fn main() {
+    std::panic::set_hook(Box::new(|_| {
+        println!("Success");
+        std::process::abort();
+    }));
 
-#![no_std]
-#![no_core]
-
-/*
- * Core
- */
-
-// Because we don't have core yet.
-#[lang = "sized"]
-pub trait Sized {}
-
-#[lang = "copy"]
-trait Copy {
-}
-
-impl Copy for isize {}
-impl Copy for *mut i32 {}
-impl Copy for usize {}
-impl Copy for i32 {}
-impl Copy for u8 {}
-impl Copy for i8 {}
-
-#[lang = "receiver"]
-trait Receiver {
-}
-
-#[lang = "freeze"]
-pub(crate) unsafe auto trait Freeze {}
-
-#[lang = "panic_location"]
-struct PanicLocation {
-    file: &'static str,
-    line: u32,
-    column: u32,
-}
-
-mod libc {
-    #[link(name = "c")]
-    extern "C" {
-        pub fn puts(s: *const u8) -> i32;
-        pub fn fflush(stream: *mut i32) -> i32;
-
-        pub static stdout: *mut i32;
-    }
-}
-
-mod intrinsics {
-    extern "rust-intrinsic" {
-        #[rustc_safe_intrinsic]
-        pub fn abort() -> !;
-    }
-}
-
-#[lang = "panic"]
-#[track_caller]
-#[no_mangle]
-pub fn panic(_msg: &'static str) -> ! {
-    unsafe {
-        // Panicking is expected iff overflow checking is enabled.
-        #[cfg(debug_assertions)]
-        libc::puts("Success\0" as *const str as *const u8);
-        libc::fflush(libc::stdout);
-        intrinsics::abort();
-    }
-}
-
-#[lang = "add"]
-trait Add<RHS = Self> {
-    type Output;
-
-    fn add(self, rhs: RHS) -> Self::Output;
-}
-
-impl Add for u8 {
-    type Output = Self;
-
-    fn add(self, rhs: Self) -> Self {
-        self + rhs
-    }
-}
-
-impl Add for i8 {
-    type Output = Self;
-
-    fn add(self, rhs: Self) -> Self {
-        self + rhs
-    }
-}
-
-impl Add for i32 {
-    type Output = Self;
-
-    fn add(self, rhs: Self) -> Self {
-        self + rhs
-    }
-}
-
-impl Add for usize {
-    type Output = Self;
-
-    fn add(self, rhs: Self) -> Self {
-        self + rhs
-    }
-}
-
-impl Add for isize {
-    type Output = Self;
-
-    fn add(self, rhs: Self) -> Self {
-        self + rhs
-    }
-}
-
-/*
- * Code
- */
-
-#[start]
-fn main(mut argc: isize, _argv: *const *const u8) -> isize {
-    let int = 9223372036854775807isize;
-    let int = int + argc;  // overflow
+    let arg_count = std::env::args().count();
+    let int = isize::MAX;
+    let _int = int + arg_count as isize;  // overflow
 
     // If overflow checking is disabled, we should reach here.
     #[cfg(not(debug_assertions))]
     unsafe {
-        libc::puts("Success\0" as *const str as *const u8);
-        libc::fflush(libc::stdout);
-        intrinsics::abort();
+        println!("Success");
+        std::process::abort();
     }
-
-    int
 }

From 2820568c2d57a28305541476fee41ea926ed55d9 Mon Sep 17 00:00:00 2001
From: Ben Kimock <kimockb@gmail.com>
Date: Sun, 22 Oct 2023 00:29:45 -0400
Subject: [PATCH 043/297] Add #[inline] to some recalcitrant ops::range methods

---
 library/core/src/ops/range.rs | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/library/core/src/ops/range.rs b/library/core/src/ops/range.rs
index b41600958876..b419a738fbe3 100644
--- a/library/core/src/ops/range.rs
+++ b/library/core/src/ops/range.rs
@@ -115,6 +115,7 @@ impl<Idx: PartialOrd<Idx>> Range<Idx> {
     /// assert!(!(0.0..f32::NAN).contains(&0.5));
     /// assert!(!(f32::NAN..1.0).contains(&0.5));
     /// ```
+    #[inline]
     #[stable(feature = "range_contains", since = "1.35.0")]
     pub fn contains<U>(&self, item: &U) -> bool
     where
@@ -141,6 +142,7 @@ impl<Idx: PartialOrd<Idx>> Range<Idx> {
     /// assert!( (3.0..f32::NAN).is_empty());
     /// assert!( (f32::NAN..5.0).is_empty());
     /// ```
+    #[inline]
     #[stable(feature = "range_is_empty", since = "1.47.0")]
     pub fn is_empty(&self) -> bool {
         !(self.start < self.end)
@@ -213,6 +215,7 @@ impl<Idx: PartialOrd<Idx>> RangeFrom<Idx> {
     /// assert!(!(0.0..).contains(&f32::NAN));
     /// assert!(!(f32::NAN..).contains(&0.5));
     /// ```
+    #[inline]
     #[stable(feature = "range_contains", since = "1.35.0")]
     pub fn contains<U>(&self, item: &U) -> bool
     where
@@ -294,6 +297,7 @@ impl<Idx: PartialOrd<Idx>> RangeTo<Idx> {
     /// assert!(!(..1.0).contains(&f32::NAN));
     /// assert!(!(..f32::NAN).contains(&0.5));
     /// ```
+    #[inline]
     #[stable(feature = "range_contains", since = "1.35.0")]
     pub fn contains<U>(&self, item: &U) -> bool
     where
@@ -500,6 +504,7 @@ impl<Idx: PartialOrd<Idx>> RangeInclusive<Idx> {
     /// // Precise field values are unspecified here
     /// assert!(!r.contains(&3) && !r.contains(&5));
     /// ```
+    #[inline]
     #[stable(feature = "range_contains", since = "1.35.0")]
     pub fn contains<U>(&self, item: &U) -> bool
     where
@@ -613,6 +618,7 @@ impl<Idx: PartialOrd<Idx>> RangeToInclusive<Idx> {
     /// assert!(!(..=1.0).contains(&f32::NAN));
     /// assert!(!(..=f32::NAN).contains(&0.5));
     /// ```
+    #[inline]
     #[stable(feature = "range_contains", since = "1.35.0")]
     pub fn contains<U>(&self, item: &U) -> bool
     where
@@ -808,6 +814,7 @@ pub trait RangeBounds<T: ?Sized> {
     /// assert!(!(0.0..1.0).contains(&f32::NAN));
     /// assert!(!(0.0..f32::NAN).contains(&0.5));
     /// assert!(!(f32::NAN..1.0).contains(&0.5));
+    #[inline]
     #[stable(feature = "range_contains", since = "1.35.0")]
     fn contains<U>(&self, item: &U) -> bool
     where

From a63624a67b16d7a6a4e2ae5676f665f2a2dcebb7 Mon Sep 17 00:00:00 2001
From: onur-ozkan <work@onurozkan.dev>
Date: Sun, 22 Oct 2023 10:30:51 +0300
Subject: [PATCH 044/297] re-enable stage0 compiler version check

Signed-off-by: onur-ozkan <work@onurozkan.dev>
---
 src/bootstrap/src/core/config/config.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs
index 7ca1bbad9684..904026bf9b90 100644
--- a/src/bootstrap/src/core/config/config.rs
+++ b/src/bootstrap/src/core/config/config.rs
@@ -1275,8 +1275,7 @@ impl Config {
         }
 
         config.initial_rustc = if let Some(rustc) = build.rustc {
-            // FIXME(#115065): re-enable this check
-            // config.check_build_rustc_version(&rustc);
+            config.check_build_rustc_version(&rustc);
             PathBuf::from(rustc)
         } else {
             config.download_beta_toolchain();

From 9586bb8064b905116ff6a5a730f631b8b5bfc527 Mon Sep 17 00:00:00 2001
From: onur-ozkan <work@onurozkan.dev>
Date: Sun, 22 Oct 2023 10:40:00 +0300
Subject: [PATCH 045/297] add bootstrap flag `--skip-stage0-validation`

This change introduces the --skip-stage0-validation flag,
which permits the use of any desired version of the stage0
compiler without verifying its version.

Signed-off-by: onur-ozkan <work@onurozkan.dev>
---
 src/bootstrap/src/core/config/config.rs |  4 +++-
 src/bootstrap/src/core/config/flags.rs  |  3 +++
 src/etc/completions/x.py.fish           | 15 +++++++++++++
 src/etc/completions/x.py.ps1            | 15 +++++++++++++
 src/etc/completions/x.py.sh             | 30 ++++++++++++-------------
 src/etc/completions/x.py.zsh            | 15 +++++++++++++
 6 files changed, 66 insertions(+), 16 deletions(-)

diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs
index 904026bf9b90..e677b05d2fe2 100644
--- a/src/bootstrap/src/core/config/config.rs
+++ b/src/bootstrap/src/core/config/config.rs
@@ -1275,7 +1275,9 @@ impl Config {
         }
 
         config.initial_rustc = if let Some(rustc) = build.rustc {
-            config.check_build_rustc_version(&rustc);
+            if !flags.skip_stage0_validation {
+                config.check_build_rustc_version(&rustc);
+            }
             PathBuf::from(rustc)
         } else {
             config.download_beta_toolchain();
diff --git a/src/bootstrap/src/core/config/flags.rs b/src/bootstrap/src/core/config/flags.rs
index dea553035449..64af114f998e 100644
--- a/src/bootstrap/src/core/config/flags.rs
+++ b/src/bootstrap/src/core/config/flags.rs
@@ -155,6 +155,9 @@ pub struct Flags {
     /// Enable BOLT link flags
     #[arg(global(true), long)]
     pub enable_bolt_settings: bool,
+    /// Skip stage0 compiler validation
+    #[arg(global(true), long)]
+    pub skip_stage0_validation: bool,
     /// Additional reproducible artifacts that should be added to the reproducible artifacts archive.
     #[arg(global(true), long)]
     pub reproducible_artifact: Vec<String>,
diff --git a/src/etc/completions/x.py.fish b/src/etc/completions/x.py.fish
index 85fcc37fd816..48796988868e 100644
--- a/src/etc/completions/x.py.fish
+++ b/src/etc/completions/x.py.fish
@@ -28,6 +28,7 @@ complete -c x.py -n "__fish_use_subcommand" -l dry-run -d 'dry run; don\'t build
 complete -c x.py -n "__fish_use_subcommand" -l json-output -d 'use message-format=json'
 complete -c x.py -n "__fish_use_subcommand" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_use_subcommand" -l enable-bolt-settings -d 'Enable BOLT link flags'
+complete -c x.py -n "__fish_use_subcommand" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
 complete -c x.py -n "__fish_use_subcommand" -s h -l help -d 'Print help'
 complete -c x.py -n "__fish_use_subcommand" -f -a "build" -d 'Compile either the compiler or libraries'
 complete -c x.py -n "__fish_use_subcommand" -f -a "check" -d 'Compile either the compiler or libraries, using cargo check'
@@ -73,6 +74,7 @@ complete -c x.py -n "__fish_seen_subcommand_from build" -l dry-run -d 'dry run;
 complete -c x.py -n "__fish_seen_subcommand_from build" -l json-output -d 'use message-format=json'
 complete -c x.py -n "__fish_seen_subcommand_from build" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from build" -l enable-bolt-settings -d 'Enable BOLT link flags'
+complete -c x.py -n "__fish_seen_subcommand_from build" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
 complete -c x.py -n "__fish_seen_subcommand_from build" -s h -l help -d 'Print help (see more with \'--help\')'
 complete -c x.py -n "__fish_seen_subcommand_from check" -l config -d 'TOML configuration file for build' -r -F
 complete -c x.py -n "__fish_seen_subcommand_from check" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"
@@ -105,6 +107,7 @@ complete -c x.py -n "__fish_seen_subcommand_from check" -l dry-run -d 'dry run;
 complete -c x.py -n "__fish_seen_subcommand_from check" -l json-output -d 'use message-format=json'
 complete -c x.py -n "__fish_seen_subcommand_from check" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from check" -l enable-bolt-settings -d 'Enable BOLT link flags'
+complete -c x.py -n "__fish_seen_subcommand_from check" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
 complete -c x.py -n "__fish_seen_subcommand_from check" -s h -l help -d 'Print help (see more with \'--help\')'
 complete -c x.py -n "__fish_seen_subcommand_from clippy" -s A -d 'clippy lints to allow' -r
 complete -c x.py -n "__fish_seen_subcommand_from clippy" -s D -d 'clippy lints to deny' -r
@@ -141,6 +144,7 @@ complete -c x.py -n "__fish_seen_subcommand_from clippy" -l dry-run -d 'dry run;
 complete -c x.py -n "__fish_seen_subcommand_from clippy" -l json-output -d 'use message-format=json'
 complete -c x.py -n "__fish_seen_subcommand_from clippy" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from clippy" -l enable-bolt-settings -d 'Enable BOLT link flags'
+complete -c x.py -n "__fish_seen_subcommand_from clippy" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
 complete -c x.py -n "__fish_seen_subcommand_from clippy" -s h -l help -d 'Print help (see more with \'--help\')'
 complete -c x.py -n "__fish_seen_subcommand_from fix" -l config -d 'TOML configuration file for build' -r -F
 complete -c x.py -n "__fish_seen_subcommand_from fix" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"
@@ -172,6 +176,7 @@ complete -c x.py -n "__fish_seen_subcommand_from fix" -l dry-run -d 'dry run; do
 complete -c x.py -n "__fish_seen_subcommand_from fix" -l json-output -d 'use message-format=json'
 complete -c x.py -n "__fish_seen_subcommand_from fix" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from fix" -l enable-bolt-settings -d 'Enable BOLT link flags'
+complete -c x.py -n "__fish_seen_subcommand_from fix" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
 complete -c x.py -n "__fish_seen_subcommand_from fix" -s h -l help -d 'Print help (see more with \'--help\')'
 complete -c x.py -n "__fish_seen_subcommand_from fmt" -l config -d 'TOML configuration file for build' -r -F
 complete -c x.py -n "__fish_seen_subcommand_from fmt" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"
@@ -204,6 +209,7 @@ complete -c x.py -n "__fish_seen_subcommand_from fmt" -l dry-run -d 'dry run; do
 complete -c x.py -n "__fish_seen_subcommand_from fmt" -l json-output -d 'use message-format=json'
 complete -c x.py -n "__fish_seen_subcommand_from fmt" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from fmt" -l enable-bolt-settings -d 'Enable BOLT link flags'
+complete -c x.py -n "__fish_seen_subcommand_from fmt" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
 complete -c x.py -n "__fish_seen_subcommand_from fmt" -s h -l help -d 'Print help (see more with \'--help\')'
 complete -c x.py -n "__fish_seen_subcommand_from doc" -l config -d 'TOML configuration file for build' -r -F
 complete -c x.py -n "__fish_seen_subcommand_from doc" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"
@@ -237,6 +243,7 @@ complete -c x.py -n "__fish_seen_subcommand_from doc" -l dry-run -d 'dry run; do
 complete -c x.py -n "__fish_seen_subcommand_from doc" -l json-output -d 'use message-format=json'
 complete -c x.py -n "__fish_seen_subcommand_from doc" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from doc" -l enable-bolt-settings -d 'Enable BOLT link flags'
+complete -c x.py -n "__fish_seen_subcommand_from doc" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
 complete -c x.py -n "__fish_seen_subcommand_from doc" -s h -l help -d 'Print help (see more with \'--help\')'
 complete -c x.py -n "__fish_seen_subcommand_from test" -l skip -d 'skips tests matching SUBSTRING, if supported by test tool. May be passed multiple times' -r -F
 complete -c x.py -n "__fish_seen_subcommand_from test" -l test-args -d 'extra arguments to be passed for the test tool being used (e.g. libtest, compiletest or rustdoc)' -r
@@ -281,6 +288,7 @@ complete -c x.py -n "__fish_seen_subcommand_from test" -l dry-run -d 'dry run; d
 complete -c x.py -n "__fish_seen_subcommand_from test" -l json-output -d 'use message-format=json'
 complete -c x.py -n "__fish_seen_subcommand_from test" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from test" -l enable-bolt-settings -d 'Enable BOLT link flags'
+complete -c x.py -n "__fish_seen_subcommand_from test" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
 complete -c x.py -n "__fish_seen_subcommand_from test" -s h -l help -d 'Print help (see more with \'--help\')'
 complete -c x.py -n "__fish_seen_subcommand_from bench" -l test-args -r
 complete -c x.py -n "__fish_seen_subcommand_from bench" -l config -d 'TOML configuration file for build' -r -F
@@ -313,6 +321,7 @@ complete -c x.py -n "__fish_seen_subcommand_from bench" -l dry-run -d 'dry run;
 complete -c x.py -n "__fish_seen_subcommand_from bench" -l json-output -d 'use message-format=json'
 complete -c x.py -n "__fish_seen_subcommand_from bench" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from bench" -l enable-bolt-settings -d 'Enable BOLT link flags'
+complete -c x.py -n "__fish_seen_subcommand_from bench" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
 complete -c x.py -n "__fish_seen_subcommand_from bench" -s h -l help -d 'Print help'
 complete -c x.py -n "__fish_seen_subcommand_from clean" -l stage -d 'Clean a specific stage without touching other artifacts. By default, every stage is cleaned if this option is not used' -r
 complete -c x.py -n "__fish_seen_subcommand_from clean" -l config -d 'TOML configuration file for build' -r -F
@@ -345,6 +354,7 @@ complete -c x.py -n "__fish_seen_subcommand_from clean" -l dry-run -d 'dry run;
 complete -c x.py -n "__fish_seen_subcommand_from clean" -l json-output -d 'use message-format=json'
 complete -c x.py -n "__fish_seen_subcommand_from clean" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from clean" -l enable-bolt-settings -d 'Enable BOLT link flags'
+complete -c x.py -n "__fish_seen_subcommand_from clean" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
 complete -c x.py -n "__fish_seen_subcommand_from clean" -s h -l help -d 'Print help'
 complete -c x.py -n "__fish_seen_subcommand_from dist" -l config -d 'TOML configuration file for build' -r -F
 complete -c x.py -n "__fish_seen_subcommand_from dist" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"
@@ -376,6 +386,7 @@ complete -c x.py -n "__fish_seen_subcommand_from dist" -l dry-run -d 'dry run; d
 complete -c x.py -n "__fish_seen_subcommand_from dist" -l json-output -d 'use message-format=json'
 complete -c x.py -n "__fish_seen_subcommand_from dist" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from dist" -l enable-bolt-settings -d 'Enable BOLT link flags'
+complete -c x.py -n "__fish_seen_subcommand_from dist" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
 complete -c x.py -n "__fish_seen_subcommand_from dist" -s h -l help -d 'Print help'
 complete -c x.py -n "__fish_seen_subcommand_from install" -l config -d 'TOML configuration file for build' -r -F
 complete -c x.py -n "__fish_seen_subcommand_from install" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"
@@ -407,6 +418,7 @@ complete -c x.py -n "__fish_seen_subcommand_from install" -l dry-run -d 'dry run
 complete -c x.py -n "__fish_seen_subcommand_from install" -l json-output -d 'use message-format=json'
 complete -c x.py -n "__fish_seen_subcommand_from install" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from install" -l enable-bolt-settings -d 'Enable BOLT link flags'
+complete -c x.py -n "__fish_seen_subcommand_from install" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
 complete -c x.py -n "__fish_seen_subcommand_from install" -s h -l help -d 'Print help'
 complete -c x.py -n "__fish_seen_subcommand_from run" -l args -d 'arguments for the tool' -r
 complete -c x.py -n "__fish_seen_subcommand_from run" -l config -d 'TOML configuration file for build' -r -F
@@ -439,6 +451,7 @@ complete -c x.py -n "__fish_seen_subcommand_from run" -l dry-run -d 'dry run; do
 complete -c x.py -n "__fish_seen_subcommand_from run" -l json-output -d 'use message-format=json'
 complete -c x.py -n "__fish_seen_subcommand_from run" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from run" -l enable-bolt-settings -d 'Enable BOLT link flags'
+complete -c x.py -n "__fish_seen_subcommand_from run" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
 complete -c x.py -n "__fish_seen_subcommand_from run" -s h -l help -d 'Print help (see more with \'--help\')'
 complete -c x.py -n "__fish_seen_subcommand_from setup" -l config -d 'TOML configuration file for build' -r -F
 complete -c x.py -n "__fish_seen_subcommand_from setup" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"
@@ -470,6 +483,7 @@ complete -c x.py -n "__fish_seen_subcommand_from setup" -l dry-run -d 'dry run;
 complete -c x.py -n "__fish_seen_subcommand_from setup" -l json-output -d 'use message-format=json'
 complete -c x.py -n "__fish_seen_subcommand_from setup" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from setup" -l enable-bolt-settings -d 'Enable BOLT link flags'
+complete -c x.py -n "__fish_seen_subcommand_from setup" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
 complete -c x.py -n "__fish_seen_subcommand_from setup" -s h -l help -d 'Print help (see more with \'--help\')'
 complete -c x.py -n "__fish_seen_subcommand_from suggest" -l config -d 'TOML configuration file for build' -r -F
 complete -c x.py -n "__fish_seen_subcommand_from suggest" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)"
@@ -502,4 +516,5 @@ complete -c x.py -n "__fish_seen_subcommand_from suggest" -l dry-run -d 'dry run
 complete -c x.py -n "__fish_seen_subcommand_from suggest" -l json-output -d 'use message-format=json'
 complete -c x.py -n "__fish_seen_subcommand_from suggest" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc'
 complete -c x.py -n "__fish_seen_subcommand_from suggest" -l enable-bolt-settings -d 'Enable BOLT link flags'
+complete -c x.py -n "__fish_seen_subcommand_from suggest" -l skip-stage0-validation -d 'Skip stage0 compiler validation'
 complete -c x.py -n "__fish_seen_subcommand_from suggest" -s h -l help -d 'Print help (see more with \'--help\')'
diff --git a/src/etc/completions/x.py.ps1 b/src/etc/completions/x.py.ps1
index 07e1b0ace9da..2fed1be72692 100644
--- a/src/etc/completions/x.py.ps1
+++ b/src/etc/completions/x.py.ps1
@@ -54,6 +54,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
+            [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help')
             [CompletionResult]::new('build', 'build', [CompletionResultType]::ParameterValue, 'Compile either the compiler or libraries')
@@ -106,6 +107,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
+            [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             break
@@ -145,6 +147,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
+            [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             break
@@ -188,6 +191,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
+            [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             break
@@ -226,6 +230,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
+            [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             break
@@ -265,6 +270,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
+            [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             break
@@ -305,6 +311,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
+            [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             break
@@ -356,6 +363,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
+            [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             break
@@ -395,6 +403,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
+            [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help')
             break
@@ -434,6 +443,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
+            [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help')
             break
@@ -472,6 +482,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
+            [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help')
             break
@@ -510,6 +521,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
+            [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help')
             break
@@ -549,6 +561,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
+            [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             break
@@ -587,6 +600,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
+            [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             break
@@ -626,6 +640,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
             [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json')
             [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc')
             [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags')
+            [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation')
             [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')')
             break
diff --git a/src/etc/completions/x.py.sh b/src/etc/completions/x.py.sh
index 241bc058e7be..f22d7e3e131d 100644
--- a/src/etc/completions/x.py.sh
+++ b/src/etc/completions/x.py.sh
@@ -61,7 +61,7 @@ _x.py() {
 
     case "${cmd}" in
         x.py)
-            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --reproducible-artifact --set --help [PATHS]... [ARGS]... build check clippy fix fmt doc test bench clean dist install run setup suggest"
+            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]... build check clippy fix fmt doc test bench clean dist install run setup suggest"
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -171,7 +171,7 @@ _x.py() {
             return 0
             ;;
         x.py__bench)
-            opts="-v -i -j -h --test-args --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --test-args --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -285,7 +285,7 @@ _x.py() {
             return 0
             ;;
         x.py__build)
-            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -395,7 +395,7 @@ _x.py() {
             return 0
             ;;
         x.py__check)
-            opts="-v -i -j -h --all-targets --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --all-targets --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -505,7 +505,7 @@ _x.py() {
             return 0
             ;;
         x.py__clean)
-            opts="-v -i -j -h --all --stage --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --all --stage --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -615,7 +615,7 @@ _x.py() {
             return 0
             ;;
         x.py__clippy)
-            opts="-A -D -W -F -v -i -j -h --fix --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-A -D -W -F -v -i -j -h --fix --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -741,7 +741,7 @@ _x.py() {
             return 0
             ;;
         x.py__dist)
-            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -851,7 +851,7 @@ _x.py() {
             return 0
             ;;
         x.py__doc)
-            opts="-v -i -j -h --open --json --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --open --json --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -961,7 +961,7 @@ _x.py() {
             return 0
             ;;
         x.py__fix)
-            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -1071,7 +1071,7 @@ _x.py() {
             return 0
             ;;
         x.py__fmt)
-            opts="-v -i -j -h --check --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --check --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -1181,7 +1181,7 @@ _x.py() {
             return 0
             ;;
         x.py__install)
-            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -1291,7 +1291,7 @@ _x.py() {
             return 0
             ;;
         x.py__run)
-            opts="-v -i -j -h --args --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --args --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -1405,7 +1405,7 @@ _x.py() {
             return 0
             ;;
         x.py__setup)
-            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --reproducible-artifact --set --help [<PROFILE>|hook|vscode|link] [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [<PROFILE>|hook|vscode|link] [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -1515,7 +1515,7 @@ _x.py() {
             return 0
             ;;
         x.py__suggest)
-            opts="-v -i -j -h --run --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --run --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
@@ -1625,7 +1625,7 @@ _x.py() {
             return 0
             ;;
         x.py__test)
-            opts="-v -i -j -h --no-fail-fast --skip --test-args --rustc-args --no-doc --doc --bless --extra-checks --force-rerun --only-modified --compare-mode --pass --run --rustfix-coverage --verbose --incremental --config --build-dir --build --host --target --exclude --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --reproducible-artifact --set --help [PATHS]... [ARGS]..."
+            opts="-v -i -j -h --no-fail-fast --skip --test-args --rustc-args --no-doc --doc --bless --extra-checks --force-rerun --only-modified --compare-mode --pass --run --rustfix-coverage --verbose --incremental --config --build-dir --build --host --target --exclude --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..."
             if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
                 COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
                 return 0
diff --git a/src/etc/completions/x.py.zsh b/src/etc/completions/x.py.zsh
index 89959701a062..1e5a7b5aa896 100644
--- a/src/etc/completions/x.py.zsh
+++ b/src/etc/completions/x.py.zsh
@@ -48,6 +48,7 @@ _x.py() {
 '--json-output[use message-format=json]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
+'--skip-stage0-validation[Skip stage0 compiler validation]' \
 '-h[Print help]' \
 '--help[Print help]' \
 '::paths -- paths for the subcommand:_files' \
@@ -96,6 +97,7 @@ _arguments "${_arguments_options[@]}" \
 '--json-output[use message-format=json]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
+'--skip-stage0-validation[Skip stage0 compiler validation]' \
 '-h[Print help (see more with '\''--help'\'')]' \
 '--help[Print help (see more with '\''--help'\'')]' \
 '*::paths -- paths for the subcommand:_files' \
@@ -137,6 +139,7 @@ _arguments "${_arguments_options[@]}" \
 '--json-output[use message-format=json]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
+'--skip-stage0-validation[Skip stage0 compiler validation]' \
 '-h[Print help (see more with '\''--help'\'')]' \
 '--help[Print help (see more with '\''--help'\'')]' \
 '*::paths -- paths for the subcommand:_files' \
@@ -182,6 +185,7 @@ _arguments "${_arguments_options[@]}" \
 '--json-output[use message-format=json]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
+'--skip-stage0-validation[Skip stage0 compiler validation]' \
 '-h[Print help (see more with '\''--help'\'')]' \
 '--help[Print help (see more with '\''--help'\'')]' \
 '*::paths -- paths for the subcommand:_files' \
@@ -222,6 +226,7 @@ _arguments "${_arguments_options[@]}" \
 '--json-output[use message-format=json]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
+'--skip-stage0-validation[Skip stage0 compiler validation]' \
 '-h[Print help (see more with '\''--help'\'')]' \
 '--help[Print help (see more with '\''--help'\'')]' \
 '*::paths -- paths for the subcommand:_files' \
@@ -263,6 +268,7 @@ _arguments "${_arguments_options[@]}" \
 '--json-output[use message-format=json]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
+'--skip-stage0-validation[Skip stage0 compiler validation]' \
 '-h[Print help (see more with '\''--help'\'')]' \
 '--help[Print help (see more with '\''--help'\'')]' \
 '*::paths -- paths for the subcommand:_files' \
@@ -305,6 +311,7 @@ _arguments "${_arguments_options[@]}" \
 '--json-output[use message-format=json]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
+'--skip-stage0-validation[Skip stage0 compiler validation]' \
 '-h[Print help (see more with '\''--help'\'')]' \
 '--help[Print help (see more with '\''--help'\'')]' \
 '*::paths -- paths for the subcommand:_files' \
@@ -358,6 +365,7 @@ _arguments "${_arguments_options[@]}" \
 '--json-output[use message-format=json]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
+'--skip-stage0-validation[Skip stage0 compiler validation]' \
 '-h[Print help (see more with '\''--help'\'')]' \
 '--help[Print help (see more with '\''--help'\'')]' \
 '*::paths -- paths for the subcommand:_files' \
@@ -399,6 +407,7 @@ _arguments "${_arguments_options[@]}" \
 '--json-output[use message-format=json]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
+'--skip-stage0-validation[Skip stage0 compiler validation]' \
 '-h[Print help]' \
 '--help[Print help]' \
 '*::paths -- paths for the subcommand:_files' \
@@ -440,6 +449,7 @@ _arguments "${_arguments_options[@]}" \
 '--json-output[use message-format=json]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
+'--skip-stage0-validation[Skip stage0 compiler validation]' \
 '-h[Print help]' \
 '--help[Print help]' \
 '*::paths -- paths for the subcommand:_files' \
@@ -480,6 +490,7 @@ _arguments "${_arguments_options[@]}" \
 '--json-output[use message-format=json]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
+'--skip-stage0-validation[Skip stage0 compiler validation]' \
 '-h[Print help]' \
 '--help[Print help]' \
 '*::paths -- paths for the subcommand:_files' \
@@ -520,6 +531,7 @@ _arguments "${_arguments_options[@]}" \
 '--json-output[use message-format=json]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
+'--skip-stage0-validation[Skip stage0 compiler validation]' \
 '-h[Print help]' \
 '--help[Print help]' \
 '*::paths -- paths for the subcommand:_files' \
@@ -561,6 +573,7 @@ _arguments "${_arguments_options[@]}" \
 '--json-output[use message-format=json]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
+'--skip-stage0-validation[Skip stage0 compiler validation]' \
 '-h[Print help (see more with '\''--help'\'')]' \
 '--help[Print help (see more with '\''--help'\'')]' \
 '*::paths -- paths for the subcommand:_files' \
@@ -601,6 +614,7 @@ _arguments "${_arguments_options[@]}" \
 '--json-output[use message-format=json]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
+'--skip-stage0-validation[Skip stage0 compiler validation]' \
 '-h[Print help (see more with '\''--help'\'')]' \
 '--help[Print help (see more with '\''--help'\'')]' \
 '::profile -- Either the profile for `config.toml` or another setup action. May be omitted to set up interactively:_files' \
@@ -643,6 +657,7 @@ _arguments "${_arguments_options[@]}" \
 '--json-output[use message-format=json]' \
 '--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \
 '--enable-bolt-settings[Enable BOLT link flags]' \
+'--skip-stage0-validation[Skip stage0 compiler validation]' \
 '-h[Print help (see more with '\''--help'\'')]' \
 '--help[Print help (see more with '\''--help'\'')]' \
 '*::paths -- paths for the subcommand:_files' \

From c094ba0be45b1d11fd906776b57ec5785b1ef284 Mon Sep 17 00:00:00 2001
From: Raoul Strackx <raoul.strackx@fortanix.com>
Date: Mon, 23 Oct 2023 14:23:28 +0200
Subject: [PATCH 046/297] Fix closure-inherit-target-feature test for SGX
 platform

---
 tests/assembly/closure-inherit-target-feature.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/assembly/closure-inherit-target-feature.rs b/tests/assembly/closure-inherit-target-feature.rs
index 248026034526..7acda76e25f0 100644
--- a/tests/assembly/closure-inherit-target-feature.rs
+++ b/tests/assembly/closure-inherit-target-feature.rs
@@ -1,4 +1,5 @@
 // only-x86_64
+// ignore-sgx Tests incompatible with LVI mitigations
 // assembly-output: emit-asm
 // make sure the feature is not enabled at compile-time
 // compile-flags: -C opt-level=3 -C target-feature=-sse4.1 -C llvm-args=-x86-asm-syntax=intel

From 04fdcd3228972a60a4d7e3504b08e7dc46dcc5cf Mon Sep 17 00:00:00 2001
From: Wesley Wiser <wesleywiser@microsoft.com>
Date: Mon, 23 Oct 2023 10:59:56 -0400
Subject: [PATCH 047/297] Update backtrace submodule

---
 library/backtrace | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/library/backtrace b/library/backtrace
index 99faef833f89..e9da96eb452a 160000
--- a/library/backtrace
+++ b/library/backtrace
@@ -1 +1 @@
-Subproject commit 99faef833f890fe89f1a959d89b951954118828b
+Subproject commit e9da96eb452aa65e79e2342be700544afe509440

From a302610016bd8299aff79c38f5a6b95ec1c112ba Mon Sep 17 00:00:00 2001
From: bjorn3 <17426603+bjorn3@users.noreply.github.com>
Date: Tue, 24 Oct 2023 12:22:23 +0000
Subject: [PATCH 048/297] Merge commit
 '93a5433f17ab5ed48cc88f1e69b0713b16183373' into sync_cg_clif-2023-10-24

---
 .vscode/settings.json                         |   2 +-
 Cargo.lock                                    |  52 +++----
 Cargo.toml                                    |  12 +-
 Readme.md                                     |   6 +-
 example/mini_core_hello_world.rs              |  11 ++
 ...h-gets-miscompiled-with-llvm-sysroot.patch |  25 ----
 src/abi/mod.rs                                |  35 ++---
 src/abi/pass_mode.rs                          |  11 +-
 src/cast.rs                                   |   6 +-
 src/common.rs                                 |  19 +++
 src/driver/aot.rs                             |  23 ++-
 src/inline_asm.rs                             |  22 +--
 src/intrinsics/llvm_x86.rs                    | 137 ++++++++++++++++++
 src/value_and_place.rs                        |  37 ++---
 14 files changed, 260 insertions(+), 138 deletions(-)
 delete mode 100644 patches/0001-regex-Ignore-test-which-gets-miscompiled-with-llvm-sysroot.patch

diff --git a/.vscode/settings.json b/.vscode/settings.json
index 60cb51d56636..834a1362caf3 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -33,7 +33,7 @@
             ]
         },
         {
-            "sysroot_src": "./download/sysroot/sysroot_src/library",
+            "sysroot_src": "./build/stdlib/library",
             "crates": [
                 {
                     "root_module": "./example/std_example.rs",
diff --git a/Cargo.lock b/Cargo.lock
index 8079913cb0ff..c716d5011731 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -45,18 +45,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e5e1df0da8488dd03b34afc134ba84b754d61862cc465932a9e5d07952f661e"
+checksum = "c1512c3bb6b13018e7109fc3ac964bc87b329eaf3a77825d337558d0c7f6f1be"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77a17ca4e699a0aaf49a0c88f6311a864f321048aa63f6b787cab20eb5f93f10"
+checksum = "16cb8fb9220a6ea7a226705a273ab905309ee546267bdf34948d57932d7f0396"
 dependencies = [
  "bumpalo",
  "cranelift-bforest",
@@ -75,39 +75,39 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "022f2793cdade1d37a1f755ac42938a3f832f533eac6cafc8b26b209544c3c06"
+checksum = "ab3a8d3b0d4745b183da5ea0792b13d79f5c23d6e69ac04761728e2532b56649"
 dependencies = [
  "cranelift-codegen-shared",
 ]
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4d72dbb83c2ad788dec4ad0843070973cb48c35a3ca19b1e7437ac40834fd9c"
+checksum = "524141c8e68f2abc2043de4c2b31f6d9dd42432738c246431d0572a1422a4a84"
 
 [[package]]
 name = "cranelift-control"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae07cf26dcc90d546826d747ac63b6c40c916f34b03e92a6ae0422c28d771b8a"
+checksum = "97513b57c961c713789a03886a57b43e14ebcd204cbaa8ae50ca6c70a8e716b3"
 dependencies = [
  "arbitrary",
 ]
 
 [[package]]
 name = "cranelift-entity"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2fe6b7e49820893691aea497f36257e9d6f52061d8c4758d61d802d5f101a3d"
+checksum = "e3f23d3cf3afa7e45f239702612c76d87964f652a55e28d13ed6d7e20f3479dd"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44f497576ca3674581581601b6a55ccc1b43447217648c880e5bce70db3cf659"
+checksum = "554cd4947ec9209b58bf9ae5bf83581b5ddf9128bd967208e334b504a57db54e"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -117,15 +117,15 @@ dependencies = [
 
 [[package]]
 name = "cranelift-isle"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b96aa02eac00fffee13b0cd37d17874ccdb3d5458983041accd825ef78ce6454"
+checksum = "6c1892a439696b6413cb54083806f5fd9fc431768b8de74864b3d9e8b93b124f"
 
 [[package]]
 name = "cranelift-jit"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1d6e0e308c873eefc185745a6b21daec2a10f7554c9fb67e334c2d7d756d979"
+checksum = "32209252fb38acaf1662ccd0397907bbe0e92bdb13b6ddbfd2f74e437f83e685"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -143,9 +143,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1aa8ebb06eced4e478c3f94f1d65d4e7c93493f4640057912b27a3e34b84841"
+checksum = "bf42656f5f6df7bfafc4dd7b63a1888b0627c07b43b2cb9aa54e13843fed39eb"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -154,9 +154,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2870170ca44054b202c737626607b87be6e35655084bd94a6ff807a5812ba7df"
+checksum = "e0c2d3badd4b9690865f5bb68a71fa94de592fa2df3f3d11a5a062c60c0a107a"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -165,9 +165,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.101.0"
+version = "0.101.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20647761742d17dabac8205da958910ede78599550e06418a16711a3ee2fc897"
+checksum = "88eca54bbecea3170035168357306e9c779d4a63d8bf036c9e16bd21fdaa69b5"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -374,9 +374,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
 [[package]]
 name = "wasmtime-jit-icache-coherence"
-version = "14.0.0"
+version = "14.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3a5dda53ad6993f9b0a2d65fb49e0348a7232a27a8794064122870d6ee19eb2"
+checksum = "9aaf2fa8fd2d6b65abae9b92edfe69254cc5d6b166e342364036c3e347de8da9"
 dependencies = [
  "cfg-if",
  "libc",
diff --git a/Cargo.toml b/Cargo.toml
index 5ad5e0e8140c..f2ce714e8ff1 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,12 +8,12 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { version = "0.101", features = ["unwind", "all-arch"] }
-cranelift-frontend = { version = "0.101" }
-cranelift-module = { version = "0.101" }
-cranelift-native = { version = "0.101" }
-cranelift-jit = { version = "0.101", optional = true }
-cranelift-object = { version = "0.101" }
+cranelift-codegen = { version = "0.101.1", features = ["unwind", "all-arch"] }
+cranelift-frontend = { version = "0.101.1" }
+cranelift-module = { version = "0.101.1" }
+cranelift-native = { version = "0.101.1" }
+cranelift-jit = { version = "0.101.1", optional = true }
+cranelift-object = { version = "0.101.1" }
 target-lexicon = "0.12.0"
 gimli = { version = "0.28", default-features = false, features = ["write"]}
 object = { version = "0.32", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
diff --git a/Readme.md b/Readme.md
index 6f2027be96de..5664cbe7d4fa 100644
--- a/Readme.md
+++ b/Readme.md
@@ -8,7 +8,7 @@ If not please open an issue.
 ## Building and testing
 
 ```bash
-$ git clone https://github.com/bjorn3/rustc_codegen_cranelift
+$ git clone https://github.com/rust-lang/rustc_codegen_cranelift
 $ cd rustc_codegen_cranelift
 $ ./y.sh prepare
 $ ./y.sh build
@@ -29,7 +29,7 @@ Extract the `dist` directory in the archive anywhere you want.
 If you want to use `cargo clif build` instead of having to specify the full path to the `cargo-clif` executable, you can add the `bin` subdirectory of the extracted `dist` directory to your `PATH`.
 (tutorial [for Windows](https://stackoverflow.com/a/44272417), and [for Linux/MacOS](https://unix.stackexchange.com/questions/26047/how-to-correctly-add-a-path-to-path/26059#26059)).
 
-[releases]: https://github.com/bjorn3/rustc_codegen_cranelift/releases/tag/dev
+[releases]: https://github.com/rust-lang/rustc_codegen_cranelift/releases/tag/dev
 
 ## Usage
 
@@ -78,7 +78,7 @@ configuration options.
 
 * Inline assembly ([no cranelift support](https://github.com/bytecodealliance/wasmtime/issues/1041))
     * On UNIX there is support for invoking an external assembler for `global_asm!` and `asm!`.
-* SIMD ([tracked here](https://github.com/bjorn3/rustc_codegen_cranelift/issues/171), `std::simd` fully works, `std::arch` is partially supported)
+* SIMD ([tracked here](https://github.com/rust-lang/rustc_codegen_cranelift/issues/171), `std::simd` fully works, `std::arch` is partially supported)
 * Unwinding on panics ([no cranelift support](https://github.com/bytecodealliance/wasmtime/issues/1677), `-Cpanic=abort` is enabled by default)
 
 ## License
diff --git a/example/mini_core_hello_world.rs b/example/mini_core_hello_world.rs
index 91de04d97702..afc51a47f140 100644
--- a/example/mini_core_hello_world.rs
+++ b/example/mini_core_hello_world.rs
@@ -353,6 +353,17 @@ fn main() {
 
     let f = V([0.0, 1.0]);
     let _a = f.0[0];
+
+    stack_val_align();
+}
+
+#[inline(never)]
+fn stack_val_align() {
+    #[repr(align(8192))]
+    struct Foo(u8);
+
+    let a = Foo(0);
+    assert_eq!(&a as *const Foo as usize % 8192, 0);
 }
 
 #[cfg(all(
diff --git a/patches/0001-regex-Ignore-test-which-gets-miscompiled-with-llvm-sysroot.patch b/patches/0001-regex-Ignore-test-which-gets-miscompiled-with-llvm-sysroot.patch
deleted file mode 100644
index e6ebdcec783a..000000000000
--- a/patches/0001-regex-Ignore-test-which-gets-miscompiled-with-llvm-sysroot.patch
+++ /dev/null
@@ -1,25 +0,0 @@
-From 5d4afb8d807d181038b6a004d17ed055a8d191b2 Mon Sep 17 00:00:00 2001
-From: bjorn3 <17426603+bjorn3@users.noreply.github.com>
-Date: Mon, 2 Oct 2023 13:59:00 +0000
-Subject: [PATCH] Ignore test which gets miscompiled with llvm sysroot
-
----
- regex-automata/src/util/pool.rs | 2 ++
- 1 file changed, 2 insertions(+)
-
-diff --git a/regex-automata/src/util/pool.rs b/regex-automata/src/util/pool.rs
-index c03d7b0..28b233b 100644
---- a/regex-automata/src/util/pool.rs
-+++ b/regex-automata/src/util/pool.rs
-@@ -1081,6 +1081,8 @@ mod tests {
-     // into the pool. This in turn resulted in this test producing a data race.
-     #[cfg(feature = "std")]
-     #[test]
-+    // FIXME(rustc_codegen_cranelift#1395) miscompilation of thread::scope with LLVM sysroot
-+    #[ignore]
-     fn thread_owner_sync() {
-         let pool = Pool::new(|| vec!['a']);
-         {
--- 
-2.34.1
-
diff --git a/src/abi/mod.rs b/src/abi/mod.rs
index c75ad852f82d..c4572e035258 100644
--- a/src/abi/mod.rs
+++ b/src/abi/mod.rs
@@ -120,32 +120,25 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         args: &[Value],
     ) -> Cow<'_, [Value]> {
         if self.tcx.sess.target.is_like_windows {
-            let (mut params, mut args): (Vec<_>, Vec<_>) =
-                params
-                    .into_iter()
-                    .zip(args)
-                    .map(|(param, &arg)| {
-                        if param.value_type == types::I128 {
-                            let arg_ptr = Pointer::stack_slot(self.bcx.create_sized_stack_slot(
-                                StackSlotData { kind: StackSlotKind::ExplicitSlot, size: 16 },
-                            ));
-                            arg_ptr.store(self, arg, MemFlags::trusted());
-                            (AbiParam::new(self.pointer_type), arg_ptr.get_addr(self))
-                        } else {
-                            (param, arg)
-                        }
-                    })
-                    .unzip();
+            let (mut params, mut args): (Vec<_>, Vec<_>) = params
+                .into_iter()
+                .zip(args)
+                .map(|(param, &arg)| {
+                    if param.value_type == types::I128 {
+                        let arg_ptr = self.create_stack_slot(16, 16);
+                        arg_ptr.store(self, arg, MemFlags::trusted());
+                        (AbiParam::new(self.pointer_type), arg_ptr.get_addr(self))
+                    } else {
+                        (param, arg)
+                    }
+                })
+                .unzip();
 
             let indirect_ret_val = returns.len() == 1 && returns[0].value_type == types::I128;
 
             if indirect_ret_val {
                 params.insert(0, AbiParam::new(self.pointer_type));
-                let ret_ptr =
-                    Pointer::stack_slot(self.bcx.create_sized_stack_slot(StackSlotData {
-                        kind: StackSlotKind::ExplicitSlot,
-                        size: 16,
-                    }));
+                let ret_ptr = self.create_stack_slot(16, 16);
                 args.insert(0, ret_ptr.get_addr(self));
                 self.lib_call_unadjusted(name, params, vec![], &args);
                 return Cow::Owned(vec![ret_ptr.load(self, types::I128, MemFlags::trusted())]);
diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs
index 7c9f8c1051cb..065226700291 100644
--- a/src/abi/pass_mode.rs
+++ b/src/abi/pass_mode.rs
@@ -189,16 +189,13 @@ pub(super) fn from_casted_value<'tcx>(
     let abi_params = cast_target_to_abi_params(cast);
     let abi_param_size: u32 = abi_params.iter().map(|param| param.value_type.bytes()).sum();
     let layout_size = u32::try_from(layout.size.bytes()).unwrap();
-    let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
-        kind: StackSlotKind::ExplicitSlot,
-        // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
-        // specify stack slot alignment.
+    let ptr = fx.create_stack_slot(
         // Stack slot size may be bigger for example `[u8; 3]` which is packed into an `i32`.
         // It may also be smaller for example when the type is a wrapper around an integer with a
         // larger alignment than the integer.
-        size: (std::cmp::max(abi_param_size, layout_size) + 15) / 16 * 16,
-    });
-    let ptr = Pointer::stack_slot(stack_slot);
+        std::cmp::max(abi_param_size, layout_size),
+        u32::try_from(layout.align.pref.bytes()).unwrap(),
+    );
     let mut offset = 0;
     let mut block_params_iter = block_params.iter().copied();
     for param in abi_params {
diff --git a/src/cast.rs b/src/cast.rs
index 7e027c5f1b30..0b5cb1547fc6 100644
--- a/src/cast.rs
+++ b/src/cast.rs
@@ -104,11 +104,7 @@ pub(crate) fn clif_int_or_float_cast(
                     &[from],
                 )[0];
                 // FIXME(bytecodealliance/wasmtime#6104) use bitcast instead of store to get from i64x2 to i128
-                let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
-                    kind: StackSlotKind::ExplicitSlot,
-                    size: 16,
-                });
-                let ret_ptr = Pointer::stack_slot(stack_slot);
+                let ret_ptr = fx.create_stack_slot(16, 16);
                 ret_ptr.store(fx, ret, MemFlags::trusted());
                 ret_ptr.load(fx, types::I128, MemFlags::trusted())
             } else {
diff --git a/src/common.rs b/src/common.rs
index 7a3ae6ebf52f..9771f44f62cf 100644
--- a/src/common.rs
+++ b/src/common.rs
@@ -383,6 +383,25 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         })
     }
 
+    pub(crate) fn create_stack_slot(&mut self, size: u32, align: u32) -> Pointer {
+        if align <= 16 {
+            let stack_slot = self.bcx.create_sized_stack_slot(StackSlotData {
+                kind: StackSlotKind::ExplicitSlot,
+                // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
+                // specify stack slot alignment.
+                size: (size + 15) / 16 * 16,
+            });
+            Pointer::stack_slot(stack_slot)
+        } else {
+            // Alignment is too big to handle using the above hack. Dynamically realign a stack slot
+            // instead. This wastes some space for the realignment.
+            let base_ptr = self.create_stack_slot(size + align, 16).get_addr(self);
+            let misalign_offset = self.bcx.ins().urem_imm(base_ptr, i64::from(align));
+            let realign_offset = self.bcx.ins().irsub_imm(misalign_offset, i64::from(align));
+            Pointer::new(self.bcx.ins().iadd(base_ptr, realign_offset))
+        }
+    }
+
     pub(crate) fn set_debug_loc(&mut self, source_info: mir::SourceInfo) {
         if let Some(debug_context) = &mut self.cx.debug_context {
             let (file, line, column) =
diff --git a/src/driver/aot.rs b/src/driver/aot.rs
index d3a7decc5430..11229dd421ec 100644
--- a/src/driver/aot.rs
+++ b/src/driver/aot.rs
@@ -361,12 +361,26 @@ pub(crate) fn run_aot(
     metadata: EncodedMetadata,
     need_metadata_module: bool,
 ) -> Box<OngoingCodegen> {
+    // FIXME handle `-Ctarget-cpu=native`
+    let target_cpu = match tcx.sess.opts.cg.target_cpu {
+        Some(ref name) => name,
+        None => tcx.sess.target.cpu.as_ref(),
+    }
+    .to_owned();
+
     let cgus = if tcx.sess.opts.output_types.should_codegen() {
         tcx.collect_and_partition_mono_items(()).1
     } else {
         // If only `--emit metadata` is used, we shouldn't perform any codegen.
         // Also `tcx.collect_and_partition_mono_items` may panic in that case.
-        &[]
+        return Box::new(OngoingCodegen {
+            modules: vec![],
+            allocator_module: None,
+            metadata_module: None,
+            metadata,
+            crate_info: CrateInfo::new(tcx, target_cpu),
+            concurrency_limiter: ConcurrencyLimiter::new(tcx.sess, 0),
+        });
     };
 
     if tcx.dep_graph.is_fully_enabled() {
@@ -481,13 +495,6 @@ pub(crate) fn run_aot(
         None
     };
 
-    // FIXME handle `-Ctarget-cpu=native`
-    let target_cpu = match tcx.sess.opts.cg.target_cpu {
-        Some(ref name) => name,
-        None => tcx.sess.target.cpu.as_ref(),
-    }
-    .to_owned();
-
     Box::new(OngoingCodegen {
         modules,
         allocator_module,
diff --git a/src/inline_asm.rs b/src/inline_asm.rs
index ed0772342548..0517c609337b 100644
--- a/src/inline_asm.rs
+++ b/src/inline_asm.rs
@@ -878,13 +878,7 @@ fn call_inline_asm<'tcx>(
     inputs: Vec<(Size, Value)>,
     outputs: Vec<(Size, CPlace<'tcx>)>,
 ) {
-    let stack_slot = fx.bcx.func.create_sized_stack_slot(StackSlotData {
-        kind: StackSlotKind::ExplicitSlot,
-        size: u32::try_from(slot_size.bytes()).unwrap(),
-    });
-    if fx.clif_comments.enabled() {
-        fx.add_comment(stack_slot, "inline asm scratch slot");
-    }
+    let stack_slot = fx.create_stack_slot(u32::try_from(slot_size.bytes()).unwrap(), 16);
 
     let inline_asm_func = fx
         .module
@@ -904,15 +898,23 @@ fn call_inline_asm<'tcx>(
     }
 
     for (offset, value) in inputs {
-        fx.bcx.ins().stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap());
+        stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).store(
+            fx,
+            value,
+            MemFlags::trusted(),
+        );
     }
 
-    let stack_slot_addr = fx.bcx.ins().stack_addr(fx.pointer_type, stack_slot, 0);
+    let stack_slot_addr = stack_slot.get_addr(fx);
     fx.bcx.ins().call(inline_asm_func, &[stack_slot_addr]);
 
     for (offset, place) in outputs {
         let ty = fx.clif_type(place.layout().ty).unwrap();
-        let value = fx.bcx.ins().stack_load(ty, stack_slot, i32::try_from(offset.bytes()).unwrap());
+        let value = stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).load(
+            fx,
+            ty,
+            MemFlags::trusted(),
+        );
         place.write_cvalue(fx, CValue::by_val(value, place.layout()));
     }
 }
diff --git a/src/intrinsics/llvm_x86.rs b/src/intrinsics/llvm_x86.rs
index 0c9a94e1c231..35f144d7dad4 100644
--- a/src/intrinsics/llvm_x86.rs
+++ b/src/intrinsics/llvm_x86.rs
@@ -310,6 +310,143 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
             let val = CValue::by_val_pair(cb_out, c, layout);
             ret.write_cvalue(fx, val);
         }
+        "llvm.x86.sse2.pavg.b" | "llvm.x86.sse2.pavg.w" => {
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            // FIXME use vector instructions when possible
+            simd_pair_for_each_lane(
+                fx,
+                a,
+                b,
+                ret,
+                &|fx, _lane_ty, _res_lane_ty, a_lane, b_lane| {
+                    // (a + b + 1) >> 1
+                    let lane_ty = fx.bcx.func.dfg.value_type(a_lane);
+                    let a_lane = fx.bcx.ins().uextend(lane_ty.double_width().unwrap(), a_lane);
+                    let b_lane = fx.bcx.ins().uextend(lane_ty.double_width().unwrap(), b_lane);
+                    let sum = fx.bcx.ins().iadd(a_lane, b_lane);
+                    let num_plus_one = fx.bcx.ins().iadd_imm(sum, 1);
+                    let res = fx.bcx.ins().ushr_imm(num_plus_one, 1);
+                    fx.bcx.ins().ireduce(lane_ty, res)
+                },
+            );
+        }
+        "llvm.x86.sse2.psra.w" => {
+            intrinsic_args!(fx, args => (a, count); intrinsic);
+
+            let count_lane = count.force_stack(fx).0.load(fx, types::I64, MemFlags::trusted());
+            let lane_ty = fx.clif_type(a.layout().ty.simd_size_and_type(fx.tcx).1).unwrap();
+            let max_count = fx.bcx.ins().iconst(types::I64, i64::from(lane_ty.bits() - 1));
+            let saturated_count = fx.bcx.ins().umin(count_lane, max_count);
+
+            // FIXME use vector instructions when possible
+            simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, a_lane| {
+                fx.bcx.ins().sshr(a_lane, saturated_count)
+            });
+        }
+        "llvm.x86.sse2.psad.bw" => {
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            assert_eq!(a.layout(), b.layout());
+            let layout = a.layout();
+
+            let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+            assert_eq!(lane_ty, fx.tcx.types.u8);
+            assert_eq!(ret_lane_ty, fx.tcx.types.u64);
+            assert_eq!(lane_count, ret_lane_count * 8);
+
+            let ret_lane_layout = fx.layout_of(fx.tcx.types.u64);
+            for out_lane_idx in 0..lane_count / 8 {
+                let mut lane_diff_acc = fx.bcx.ins().iconst(types::I64, 0);
+
+                for lane_idx in out_lane_idx * 8..out_lane_idx * 8 + 1 {
+                    let a_lane = a.value_lane(fx, lane_idx).load_scalar(fx);
+                    let b_lane = b.value_lane(fx, lane_idx).load_scalar(fx);
+
+                    let lane_diff = fx.bcx.ins().isub(a_lane, b_lane);
+                    let abs_lane_diff = fx.bcx.ins().iabs(lane_diff);
+                    let abs_lane_diff = fx.bcx.ins().uextend(types::I64, abs_lane_diff);
+                    lane_diff_acc = fx.bcx.ins().iadd(lane_diff_acc, abs_lane_diff);
+                }
+
+                let res_lane = CValue::by_val(lane_diff_acc, ret_lane_layout);
+
+                ret.place_lane(fx, out_lane_idx).write_cvalue(fx, res_lane);
+            }
+        }
+        "llvm.x86.ssse3.pmadd.ub.sw.128" => {
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
+            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+            assert_eq!(lane_ty, fx.tcx.types.u8);
+            assert_eq!(ret_lane_ty, fx.tcx.types.i16);
+            assert_eq!(lane_count, ret_lane_count * 2);
+
+            let ret_lane_layout = fx.layout_of(fx.tcx.types.i16);
+            for out_lane_idx in 0..lane_count / 2 {
+                let a_lane0 = a.value_lane(fx, out_lane_idx * 2).load_scalar(fx);
+                let a_lane0 = fx.bcx.ins().uextend(types::I16, a_lane0);
+                let b_lane0 = b.value_lane(fx, out_lane_idx * 2).load_scalar(fx);
+                let b_lane0 = fx.bcx.ins().sextend(types::I16, b_lane0);
+
+                let a_lane1 = a.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx);
+                let a_lane1 = fx.bcx.ins().uextend(types::I16, a_lane1);
+                let b_lane1 = b.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx);
+                let b_lane1 = fx.bcx.ins().sextend(types::I16, b_lane1);
+
+                let mul0: Value = fx.bcx.ins().imul(a_lane0, b_lane0);
+                let mul1 = fx.bcx.ins().imul(a_lane1, b_lane1);
+
+                let (val, has_overflow) = fx.bcx.ins().sadd_overflow(mul0, mul1);
+
+                let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, mul1, 0);
+
+                let min = fx.bcx.ins().iconst(types::I16, i64::from(i16::MIN as u16));
+                let max = fx.bcx.ins().iconst(types::I16, i64::from(i16::MAX as u16));
+
+                let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min);
+                let res_lane = fx.bcx.ins().select(has_overflow, sat_val, val);
+
+                let res_lane = CValue::by_val(res_lane, ret_lane_layout);
+
+                ret.place_lane(fx, out_lane_idx).write_cvalue(fx, res_lane);
+            }
+        }
+        "llvm.x86.sse2.pmadd.wd" => {
+            intrinsic_args!(fx, args => (a, b); intrinsic);
+
+            assert_eq!(a.layout(), b.layout());
+            let layout = a.layout();
+
+            let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
+            assert_eq!(lane_ty, fx.tcx.types.i16);
+            assert_eq!(ret_lane_ty, fx.tcx.types.i32);
+            assert_eq!(lane_count, ret_lane_count * 2);
+
+            let ret_lane_layout = fx.layout_of(fx.tcx.types.i32);
+            for out_lane_idx in 0..lane_count / 2 {
+                let a_lane0 = a.value_lane(fx, out_lane_idx * 2).load_scalar(fx);
+                let a_lane0 = fx.bcx.ins().uextend(types::I32, a_lane0);
+                let b_lane0 = b.value_lane(fx, out_lane_idx * 2).load_scalar(fx);
+                let b_lane0 = fx.bcx.ins().sextend(types::I32, b_lane0);
+
+                let a_lane1 = a.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx);
+                let a_lane1 = fx.bcx.ins().uextend(types::I32, a_lane1);
+                let b_lane1 = b.value_lane(fx, out_lane_idx * 2 + 1).load_scalar(fx);
+                let b_lane1 = fx.bcx.ins().sextend(types::I32, b_lane1);
+
+                let mul0: Value = fx.bcx.ins().imul(a_lane0, b_lane0);
+                let mul1 = fx.bcx.ins().imul(a_lane1, b_lane1);
+
+                let res_lane = fx.bcx.ins().iadd(mul0, mul1);
+                let res_lane = CValue::by_val(res_lane, ret_lane_layout);
+
+                ret.place_lane(fx, out_lane_idx).write_cvalue(fx, res_lane);
+            }
+        }
         _ => {
             fx.tcx
                 .sess
diff --git a/src/value_and_place.rs b/src/value_and_place.rs
index 3a6a6c9e3f50..5f0aa6c5581d 100644
--- a/src/value_and_place.rs
+++ b/src/value_and_place.rs
@@ -132,18 +132,11 @@ impl<'tcx> CValue<'tcx> {
                 (ptr.get_addr(fx), vtable)
             }
             CValueInner::ByValPair(data, vtable) => {
-                let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
-                    kind: StackSlotKind::ExplicitSlot,
-                    // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
-                    // specify stack slot alignment.
-                    size: (u32::try_from(fx.target_config.pointer_type().bytes()).unwrap() + 15)
-                        / 16
-                        * 16,
-                });
-                let data_ptr = Pointer::stack_slot(stack_slot);
-                let mut flags = MemFlags::new();
-                flags.set_notrap();
-                data_ptr.store(fx, data, flags);
+                let data_ptr = fx.create_stack_slot(
+                    u32::try_from(fx.target_config.pointer_type().bytes()).unwrap(),
+                    u32::try_from(fx.target_config.pointer_type().bytes()).unwrap(),
+                );
+                data_ptr.store(fx, data, MemFlags::trusted());
 
                 (data_ptr.get_addr(fx), vtable)
             }
@@ -372,13 +365,11 @@ impl<'tcx> CPlace<'tcx> {
                 .fatal(format!("values of type {} are too big to store on the stack", layout.ty));
         }
 
-        let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
-            kind: StackSlotKind::ExplicitSlot,
-            // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
-            // specify stack slot alignment.
-            size: (u32::try_from(layout.size.bytes()).unwrap() + 15) / 16 * 16,
-        });
-        CPlace { inner: CPlaceInner::Addr(Pointer::stack_slot(stack_slot), None), layout }
+        let stack_slot = fx.create_stack_slot(
+            u32::try_from(layout.size.bytes()).unwrap(),
+            u32::try_from(layout.align.pref.bytes()).unwrap(),
+        );
+        CPlace { inner: CPlaceInner::Addr(stack_slot, None), layout }
     }
 
     pub(crate) fn new_var(
@@ -543,13 +534,7 @@ impl<'tcx> CPlace<'tcx> {
                 _ if src_ty.is_vector() && dst_ty.is_vector() => codegen_bitcast(fx, dst_ty, data),
                 _ if src_ty.is_vector() || dst_ty.is_vector() => {
                     // FIXME(bytecodealliance/wasmtime#6104) do something more efficient for transmutes between vectors and integers.
-                    let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
-                        kind: StackSlotKind::ExplicitSlot,
-                        // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
-                        // specify stack slot alignment.
-                        size: (src_ty.bytes() + 15) / 16 * 16,
-                    });
-                    let ptr = Pointer::stack_slot(stack_slot);
+                    let ptr = fx.create_stack_slot(src_ty.bytes(), src_ty.bytes());
                     ptr.store(fx, data, MemFlags::trusted());
                     ptr.load(fx, dst_ty, MemFlags::trusted())
                 }

From fcd336b3de942192d9723a3e523c4b0cbe616d3d Mon Sep 17 00:00:00 2001
From: Guillaume Gomez <guillaume1.gomez@gmail.com>
Date: Tue, 24 Oct 2023 17:15:18 +0200
Subject: [PATCH 049/297] Add basics for `test` command in build system

---
 build_system/src/main.rs |  5 +++++
 build_system/src/test.rs | 15 +++++++++++++++
 2 files changed, 20 insertions(+)
 create mode 100644 build_system/src/test.rs

diff --git a/build_system/src/main.rs b/build_system/src/main.rs
index 332a14ff0a28..bff82b6e3e57 100644
--- a/build_system/src/main.rs
+++ b/build_system/src/main.rs
@@ -5,6 +5,7 @@ mod build;
 mod config;
 mod prepare;
 mod rustc_info;
+mod test;
 mod utils;
 
 macro_rules! arg_error {
@@ -23,6 +24,7 @@ Available commands for build_system:
 
     prepare  : Run prepare command
     build    : Run build command
+    test     : Run test command
     --help   : Show this message"
     );
 }
@@ -30,6 +32,7 @@ Available commands for build_system:
 pub enum Command {
     Prepare,
     Build,
+    Test,
 }
 
 fn main() {
@@ -40,6 +43,7 @@ fn main() {
     let command = match env::args().nth(1).as_deref() {
         Some("prepare") => Command::Prepare,
         Some("build") => Command::Build,
+        Some("test") => Command::Test,
         Some("--help") => {
             usage();
             process::exit(0);
@@ -55,6 +59,7 @@ fn main() {
     if let Err(e) = match command {
         Command::Prepare => prepare::run(),
         Command::Build => build::run(),
+        Command::Test => test::run(),
     } {
         eprintln!("Command failed to run: {e:?}");
         process::exit(1);
diff --git a/build_system/src/test.rs b/build_system/src/test.rs
new file mode 100644
index 000000000000..4c8c63e59ab7
--- /dev/null
+++ b/build_system/src/test.rs
@@ -0,0 +1,15 @@
+use crate::utils::run_command_with_output;
+
+fn get_args<'a>(args: &mut Vec<&'a dyn AsRef<std::ffi::OsStr>>, extra_args: &'a Vec<String>) {
+    for extra_arg in extra_args {
+        args.push(extra_arg);
+    }
+}
+
+pub fn run() -> Result<(), String> {
+    let mut args: Vec<&dyn AsRef<std::ffi::OsStr>> = vec![&"bash", &"test.sh"];
+    let extra_args = std::env::args().skip(2).collect::<Vec<_>>();
+    get_args(&mut args, &extra_args);
+    let current_dir = std::env::current_dir().map_err(|error| format!("`current_dir` failed: {:?}", error))?;
+    run_command_with_output(args.as_slice(), Some(&current_dir))
+}

From b7bea6e46fa1f00d96a779cc75f5a652cf5cdc5d Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Fri, 20 Oct 2023 14:19:28 +1100
Subject: [PATCH 050/297] Fix a bug in tidy's alphabetical checking.

Currently, if a `tidy-alphabetical-end` marker appears on the last line
of a file, tidy will erroneously issue a "reach end of file expecting
`tidy-alphabetical-end`" error. This is because it uses `take_while`
within `check_section`, which consumes the line with the end marker, and
then after `check_section` returns `check` peeks for at least one more
line, which won't be there is the marker was on the last line.

This commit fixes the problem, by removing the use of `take_while`, and
doing the "reached end of file" test within `check_section` without
using `peek`.

It also renames `{START,END}_COMMENT` as `{START,END}_MARKER`, which is
a more appropriate name.
---
 src/tools/tidy/src/alphabetical.rs | 31 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/src/tools/tidy/src/alphabetical.rs b/src/tools/tidy/src/alphabetical.rs
index 3e60915c224c..dde5bef5a5f7 100644
--- a/src/tools/tidy/src/alphabetical.rs
+++ b/src/tools/tidy/src/alphabetical.rs
@@ -1,6 +1,6 @@
 //! Checks that a list of items is in alphabetical order
 //!
-//! To use, use the following annotation in the code:
+//! Use the following marker in the code:
 //! ```rust
 //! // tidy-alphabetical-start
 //! fn aaa() {}
@@ -30,27 +30,25 @@ fn is_close_bracket(c: char) -> bool {
 }
 
 // Don't let tidy check this here :D
-const START_COMMENT: &str = concat!("tidy-alphabetical", "-start");
-const END_COMMENT: &str = "tidy-alphabetical-end";
+const START_MARKER: &str = concat!("tidy-alphabetical", "-start");
+const END_MARKER: &str = "tidy-alphabetical-end";
 
 fn check_section<'a>(
     file: impl Display,
     lines: impl Iterator<Item = (usize, &'a str)>,
     bad: &mut bool,
 ) {
-    let content_lines = lines.take_while(|(_, line)| !line.contains(END_COMMENT));
-
     let mut prev_line = String::new();
     let mut first_indent = None;
     let mut in_split_line = None;
 
-    for (line_idx, line) in content_lines {
-        if line.contains(START_COMMENT) {
-            tidy_error!(
-                bad,
-                "{file}:{} found `{START_COMMENT}` expecting `{END_COMMENT}`",
-                line_idx
-            )
+    for (line_idx, line) in lines {
+        if line.contains(START_MARKER) {
+            tidy_error!(bad, "{file}:{} found `{START_MARKER}` expecting `{END_MARKER}`", line_idx)
+        }
+
+        if line.contains(END_MARKER) {
+            return;
         }
 
         let indent = first_indent.unwrap_or_else(|| {
@@ -92,19 +90,18 @@ fn check_section<'a>(
 
         prev_line = line;
     }
+
+    tidy_error!(bad, "{file}: reached end of file expecting `{END_MARKER}`")
 }
 
 pub fn check(path: &Path, bad: &mut bool) {
     walk(path, |path, _is_dir| filter_dirs(path), &mut |entry, contents| {
         let file = &entry.path().display();
 
-        let mut lines = contents.lines().enumerate().peekable();
+        let mut lines = contents.lines().enumerate();
         while let Some((_, line)) = lines.next() {
-            if line.contains(START_COMMENT) {
+            if line.contains(START_MARKER) {
                 check_section(file, &mut lines, bad);
-                if lines.peek().is_none() {
-                    tidy_error!(bad, "{file}: reached end of file expecting `{END_COMMENT}`")
-                }
             }
         }
     });

From 40797eef5ea3f8d50c98fbb81b0c005f7a358539 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 23 Oct 2023 08:53:16 +1100
Subject: [PATCH 051/297] tidy: skip lines starting with `#` in alphabetical
 check.

These are comment lines in `Cargo.toml` files.

But exclude lines starting with `#!` from the skipping, because we want
to check them. (Rust `#![feature(...)]` lines.)

Also allow empty lines, which are occasionally useful.
---
 src/tools/tidy/src/alphabetical.rs | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/tools/tidy/src/alphabetical.rs b/src/tools/tidy/src/alphabetical.rs
index dde5bef5a5f7..98a8f4a99801 100644
--- a/src/tools/tidy/src/alphabetical.rs
+++ b/src/tools/tidy/src/alphabetical.rs
@@ -10,9 +10,10 @@
 //! ```
 //!
 //! The following lines are ignored:
+//! - Empty lines
 //! - Lines that are indented with more or less spaces than the first line
-//! - Lines starting with `//`, `#[`, `)`, `]`, `}` if the comment has the same indentation as
-//!   the first line
+//! - Lines starting with `//`, `#` (except those starting with `#!`), `)`, `]`, `}` if the comment
+//!   has the same indentation as the first line
 //!
 //! If a line ends with an opening bracket, the line is ignored and the next line will have
 //! its extra indentation ignored.
@@ -43,6 +44,10 @@ fn check_section<'a>(
     let mut in_split_line = None;
 
     for (line_idx, line) in lines {
+        if line.is_empty() {
+            continue;
+        }
+
         if line.contains(START_MARKER) {
             tidy_error!(bad, "{file}:{} found `{START_MARKER}` expecting `{END_MARKER}`", line_idx)
         }
@@ -71,7 +76,7 @@ fn check_section<'a>(
         let trimmed_line = line.trim_start_matches(' ');
 
         if trimmed_line.starts_with("//")
-            || trimmed_line.starts_with("#[")
+            || (trimmed_line.starts_with("#") && !trimmed_line.starts_with("#!"))
             || trimmed_line.starts_with(is_close_bracket)
         {
             continue;

From 0b6e1332b1c1258d24a52fdefd95e3fa2e88f69f Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Tue, 24 Oct 2023 17:40:45 -0400
Subject: [PATCH 052/297] Fix to be able to use a target specification JSON
 file and document the process

---
 Readme.md                  |  7 ++++++-
 build_system/src/build.rs  | 23 ++++++++++++++++-------
 build_system/src/config.rs | 22 +++++++++++++++++++++-
 3 files changed, 43 insertions(+), 9 deletions(-)

diff --git a/Readme.md b/Readme.md
index 2207bd35edb4..1bad1e71137c 100644
--- a/Readme.md
+++ b/Readme.md
@@ -322,7 +322,12 @@ generate it in [gimple.md](./doc/gimple.md).
  * Set the path to the cross-compiling libgccjit in `gcc_path`.
  * Make sure you have the linker for your target (for instance `m68k-unknown-linux-gnu-gcc`) in your `$PATH`. Currently, the linker name is hardcoded as being `$TARGET-gcc`. Specify the target when building the sysroot: `./y.sh build --target-triple m68k-unknown-linux-gnu`.
  * Build your project by specifying the target: `OVERWRITE_TARGET_TRIPLE=m68k-unknown-linux-gnu ../cargo.sh build --target m68k-unknown-linux-gnu`.
- * If the target is not yet supported by the Rust compiler, create a [target specification file](https://docs.rust-embedded.org/embedonomicon/custom-target.html) (note that the `arch` specified in this file must be supported by the rust compiler).
+
+If the target is not yet supported by the Rust compiler, create a [target specification file](https://docs.rust-embedded.org/embedonomicon/custom-target.html) (note that the `arch` specified in this file must be supported by the rust compiler).
+Then, you can use it the following way:
+
+ * Add the target specification file using `--target` as an **absolute** path to build the sysroot: `./y.sh build --target-triple m68k-unknown-linux-gnu --target $(pwd)/m68k-unknown-linux-gnu.json`
+ * Build your project by specifying the target specification file: `OVERWRITE_TARGET_TRIPLE=m68k-unknown-linux-gnu ../cargo.sh build --target path/to/m68k-unknown-linux-gnu.json`.
 
 If you get the following error:
 
diff --git a/build_system/src/build.rs b/build_system/src/build.rs
index b013ca80705d..c71954e4d113 100644
--- a/build_system/src/build.rs
+++ b/build_system/src/build.rs
@@ -1,4 +1,4 @@
-use crate::config::set_config;
+use crate::config::{set_config, ConfigInfo};
 use crate::utils::{
     get_gcc_path, run_command, run_command_with_output_and_env, walk_dir,
 };
@@ -55,6 +55,15 @@ impl BuildArg {
                         );
                     }
                 }
+                "--target" => {
+                    if args.next().is_some() {
+                        // Handled in config.rs.
+                    } else {
+                        return Err(
+                            "Expected a value after `--target`, found nothing".to_string()
+                        );
+                    }
+                }
                 arg => return Err(format!("Unknown argument `{}`", arg)),
             }
         }
@@ -80,7 +89,7 @@ impl BuildArg {
 fn build_sysroot(
     env: &mut HashMap<String, String>,
     release_mode: bool,
-    target_triple: &str,
+    config: &ConfigInfo,
 ) -> Result<(), String> {
     std::env::set_current_dir("build_sysroot")
         .map_err(|error| format!("Failed to go to `build_sysroot` directory: {:?}", error))?;
@@ -143,7 +152,7 @@ fn build_sysroot(
                 &"cargo",
                 &"build",
                 &"--target",
-                &target_triple,
+                &config.target,
                 &"--release",
             ],
             None,
@@ -156,7 +165,7 @@ fn build_sysroot(
                 &"cargo",
                 &"build",
                 &"--target",
-                &target_triple,
+                &config.target,
             ],
             None,
             Some(env),
@@ -165,14 +174,14 @@ fn build_sysroot(
     };
 
     // Copy files to sysroot
-    let sysroot_path = format!("sysroot/lib/rustlib/{}/lib/", target_triple);
+    let sysroot_path = format!("sysroot/lib/rustlib/{}/lib/", config.target_triple);
     fs::create_dir_all(&sysroot_path)
         .map_err(|error| format!("Failed to create directory `{}`: {:?}", sysroot_path, error))?;
     let copier = |dir_to_copy: &Path| {
         run_command(&[&"cp", &"-r", &dir_to_copy, &sysroot_path], None).map(|_| ())
     };
     walk_dir(
-        &format!("target/{}/{}/deps", target_triple, channel),
+        &format!("target/{}/{}/deps", config.target_triple, channel),
         copier,
         copier,
     )?;
@@ -216,7 +225,7 @@ fn build_codegen(args: &BuildArg) -> Result<(), String> {
     build_sysroot(
         &mut env,
         args.sysroot_release_channel,
-        &config.target_triple,
+        &config,
     )?;
     Ok(())
 }
diff --git a/build_system/src/config.rs b/build_system/src/config.rs
index 0f77943476fe..64d9bd73e01a 100644
--- a/build_system/src/config.rs
+++ b/build_system/src/config.rs
@@ -3,6 +3,7 @@ use std::collections::HashMap;
 use std::env as std_env;
 
 pub struct ConfigInfo {
+    pub target: String,
     pub target_triple: String,
     pub rustc_command: Vec<String>,
 }
@@ -30,25 +31,43 @@ pub fn set_config(
     let host_triple = get_rustc_host_triple()?;
     let mut linker = None;
     let mut target_triple = host_triple.clone();
+    let mut target = target_triple.clone();
 
     // We skip binary name and the command.
     let mut args = std::env::args().skip(2);
 
+    let mut set_target_triple = false;
+    let mut set_target = false;
     while let Some(arg) = args.next() {
         match arg.as_str() {
             "--target-triple" => {
                 if let Some(arg) = args.next() {
                     target_triple = arg;
+                    set_target_triple = true;
                 } else {
                     return Err(
                         "Expected a value after `--target-triple`, found nothing".to_string()
                     );
                 }
             },
+            "--target" => {
+                if let Some(arg) = args.next() {
+                    target = arg;
+                    set_target = true;
+                } else {
+                    return Err(
+                        "Expected a value after `--target`, found nothing".to_string()
+                    );
+                }
+            },
             _ => (),
         }
     }
 
+    if set_target_triple && !set_target {
+        target = target_triple.clone();
+    }
+
     if host_triple != target_triple {
         linker = Some(format!("-Clinker={}-gcc", target_triple));
     }
@@ -123,7 +142,8 @@ pub fn set_config(
         "target/out".to_string(),
     ]);
     Ok(ConfigInfo {
-        target_triple: target_triple.to_string(),
+        target,
+        target_triple,
         rustc_command,
     })
 }

From a79a2c729df5086a64c2aaf6c77c0ce582cb1499 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Wed, 25 Oct 2023 10:45:56 +1100
Subject: [PATCH 053/297] tidy: some minor improvements.

- Tweak some comments.
- No need to do the `concat!` trick on `START_MARKER` because it's
  immediately followed by `END_MARKER`.
- Fix an off-by-one error in the line number for an error message.
- When a second start marker is found without an intervening end marker,
  after giving an error, treat it as though it ends the section. It's
  hard to know exactly what to do in this case, but it makes unit
  testing this case a little simpler (in the next commit).
- If an end marker occurs without a preceding start marker, issue an
  error.
---
 src/tools/tidy/src/alphabetical.rs | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/src/tools/tidy/src/alphabetical.rs b/src/tools/tidy/src/alphabetical.rs
index 98a8f4a99801..32fb16b0a421 100644
--- a/src/tools/tidy/src/alphabetical.rs
+++ b/src/tools/tidy/src/alphabetical.rs
@@ -14,11 +14,13 @@
 //! - Lines that are indented with more or less spaces than the first line
 //! - Lines starting with `//`, `#` (except those starting with `#!`), `)`, `]`, `}` if the comment
 //!   has the same indentation as the first line
+//! - Lines starting with a closing delimiter (`)`, `[`, `}`) are ignored.
 //!
-//! If a line ends with an opening bracket, the line is ignored and the next line will have
-//! its extra indentation ignored.
+//! If a line ends with an opening delimiter, we effectively join the following line to it before
+//! checking it. E.g. `foo(\nbar)` is treated like `foo(bar)`.
 
-use std::{fmt::Display, path::Path};
+use std::fmt::Display;
+use std::path::Path;
 
 use crate::walk::{filter_dirs, walk};
 
@@ -30,8 +32,7 @@ fn is_close_bracket(c: char) -> bool {
     matches!(c, ')' | ']' | '}')
 }
 
-// Don't let tidy check this here :D
-const START_MARKER: &str = concat!("tidy-alphabetical", "-start");
+const START_MARKER: &str = "tidy-alphabetical-start";
 const END_MARKER: &str = "tidy-alphabetical-end";
 
 fn check_section<'a>(
@@ -43,13 +44,14 @@ fn check_section<'a>(
     let mut first_indent = None;
     let mut in_split_line = None;
 
-    for (line_idx, line) in lines {
+    for (idx, line) in lines {
         if line.is_empty() {
             continue;
         }
 
         if line.contains(START_MARKER) {
-            tidy_error!(bad, "{file}:{} found `{START_MARKER}` expecting `{END_MARKER}`", line_idx)
+            tidy_error!(bad, "{file}:{} found `{START_MARKER}` expecting `{END_MARKER}`", idx + 1);
+            return;
         }
 
         if line.contains(END_MARKER) {
@@ -63,6 +65,7 @@ fn check_section<'a>(
         });
 
         let line = if let Some(prev_split_line) = in_split_line {
+            // Join the split lines.
             in_split_line = None;
             format!("{prev_split_line}{}", line.trim_start())
         } else {
@@ -90,7 +93,7 @@ fn check_section<'a>(
         let prev_line_trimmed_lowercase = prev_line.trim_start_matches(' ').to_lowercase();
 
         if trimmed_line.to_lowercase() < prev_line_trimmed_lowercase {
-            tidy_error!(bad, "{file}:{}: line not in alphabetical order", line_idx + 1,);
+            tidy_error!(bad, "{file}:{}: line not in alphabetical order", idx + 1);
         }
 
         prev_line = line;
@@ -104,7 +107,15 @@ pub fn check(path: &Path, bad: &mut bool) {
         let file = &entry.path().display();
 
         let mut lines = contents.lines().enumerate();
-        while let Some((_, line)) = lines.next() {
+        while let Some((idx, line)) = lines.next() {
+            if line.contains(END_MARKER) {
+                tidy_error!(
+                    bad,
+                    "{file}:{} found `{END_MARKER}` expecting `{START_MARKER}`",
+                    idx + 1
+                )
+            }
+
             if line.contains(START_MARKER) {
                 check_section(file, &mut lines, bad);
             }

From a93d1b73c67f8d08d759a9a880a3bc0081d6ae16 Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Tue, 24 Oct 2023 17:43:23 -0400
Subject: [PATCH 054/297] Fix volatile_load

---
 src/builder.rs        |  5 ++---
 src/intrinsic/mod.rs  | 18 ++++++++++++------
 tests/run/volatile.rs | 26 ++++++++++++++++++++++++++
 3 files changed, 40 insertions(+), 9 deletions(-)
 create mode 100644 tests/run/volatile.rs

diff --git a/src/builder.rs b/src/builder.rs
index b78418089342..b8a8c144dc90 100644
--- a/src/builder.rs
+++ b/src/builder.rs
@@ -751,9 +751,8 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
         loaded_value.to_rvalue()
     }
 
-    fn volatile_load(&mut self, _ty: Type<'gcc>, ptr: RValue<'gcc>) -> RValue<'gcc> {
-        // TODO(antoyo): use ty.
-        let ptr = self.context.new_cast(None, ptr, ptr.get_type().make_volatile());
+    fn volatile_load(&mut self, ty: Type<'gcc>, ptr: RValue<'gcc>) -> RValue<'gcc> {
+        let ptr = self.context.new_cast(None, ptr, ty.make_volatile().make_pointer());
         ptr.dereference(None).to_rvalue()
     }
 
diff --git a/src/intrinsic/mod.rs b/src/intrinsic/mod.rs
index bfe27c0552f0..ba1cae03f3e4 100644
--- a/src/intrinsic/mod.rs
+++ b/src/intrinsic/mod.rs
@@ -4,7 +4,9 @@ mod simd;
 #[cfg(feature="master")]
 use std::iter;
 
-use gccjit::{ComparisonOp, Function, RValue, ToRValue, Type, UnaryOp, FunctionType};
+#[cfg(feature="master")]
+use gccjit::FunctionType;
+use gccjit::{ComparisonOp, Function, RValue, ToRValue, Type, UnaryOp};
 use rustc_codegen_ssa::MemFlags;
 use rustc_codegen_ssa::base::wants_msvc_seh;
 use rustc_codegen_ssa::common::IntPredicate;
@@ -143,11 +145,15 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
 
                 sym::volatile_load | sym::unaligned_volatile_load => {
                     let tp_ty = fn_args.type_at(0);
-                    let mut ptr = args[0].immediate();
-                    if let PassMode::Cast { cast: ty, .. } = &fn_abi.ret.mode {
-                        ptr = self.pointercast(ptr, self.type_ptr_to(ty.gcc_type(self)));
-                    }
-                    let load = self.volatile_load(ptr.get_type(), ptr);
+                    let ptr = args[0].immediate();
+                    let load =
+                        if let PassMode::Cast { cast: ty, pad_i32: _ } = &fn_abi.ret.mode {
+                            let gcc_ty = ty.gcc_type(self);
+                            self.volatile_load(gcc_ty, ptr)
+                        }
+                        else {
+                            self.volatile_load(self.layout_of(tp_ty).gcc_type(self), ptr)
+                        };
                     // TODO(antoyo): set alignment.
                     self.to_immediate(load, self.layout_of(tp_ty))
                 }
diff --git a/tests/run/volatile.rs b/tests/run/volatile.rs
new file mode 100644
index 000000000000..8b0433125936
--- /dev/null
+++ b/tests/run/volatile.rs
@@ -0,0 +1,26 @@
+// Compiler:
+//
+// Run-time:
+//   status: 0
+
+use std::mem::MaybeUninit;
+
+#[derive(Debug)]
+struct Struct {
+    pointer: *const (),
+    func: unsafe fn(*const ()),
+}
+
+fn func(ptr: *const ()) {
+}
+
+fn main() {
+    let mut x = MaybeUninit::<&Struct>::uninit();
+    x.write(&Struct {
+        pointer: std::ptr::null(),
+        func,
+    });
+    let x = unsafe { x.assume_init() };
+    let value = unsafe { (x as *const Struct).read_volatile() };
+    println!("{:?}", value);
+}

From a0b0ace061e797e26dc764b5a3e41f7440864a8c Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Wed, 25 Oct 2023 10:51:29 +1100
Subject: [PATCH 055/297] tidy: add unit tests for alphabetical markers.

This requires introducing `tidy_error_ext!` as an alternative to
`tidy_error!`. It is passed a closure that is called for an error. This
lets tests capture tidy error messages in a buffer instead of printing
them to stderr.

It also requires pulling part of `check` out into a new function
`check_lines`, so that tests can pass in an iterator over some strings
instead of a file.
---
 src/tools/tidy/src/alphabetical.rs       |  59 ++++---
 src/tools/tidy/src/alphabetical/tests.rs | 188 +++++++++++++++++++++++
 src/tools/tidy/src/lib.rs                |  16 +-
 3 files changed, 238 insertions(+), 25 deletions(-)
 create mode 100644 src/tools/tidy/src/alphabetical/tests.rs

diff --git a/src/tools/tidy/src/alphabetical.rs b/src/tools/tidy/src/alphabetical.rs
index 32fb16b0a421..150a9594350e 100644
--- a/src/tools/tidy/src/alphabetical.rs
+++ b/src/tools/tidy/src/alphabetical.rs
@@ -24,6 +24,9 @@ use std::path::Path;
 
 use crate::walk::{filter_dirs, walk};
 
+#[cfg(test)]
+mod tests;
+
 fn indentation(line: &str) -> usize {
     line.find(|c| c != ' ').unwrap_or(0)
 }
@@ -38,6 +41,7 @@ const END_MARKER: &str = "tidy-alphabetical-end";
 fn check_section<'a>(
     file: impl Display,
     lines: impl Iterator<Item = (usize, &'a str)>,
+    err: &mut dyn FnMut(&str) -> std::io::Result<()>,
     bad: &mut bool,
 ) {
     let mut prev_line = String::new();
@@ -50,7 +54,12 @@ fn check_section<'a>(
         }
 
         if line.contains(START_MARKER) {
-            tidy_error!(bad, "{file}:{} found `{START_MARKER}` expecting `{END_MARKER}`", idx + 1);
+            tidy_error_ext!(
+                err,
+                bad,
+                "{file}:{} found `{START_MARKER}` expecting `{END_MARKER}`",
+                idx + 1
+            );
             return;
         }
 
@@ -93,32 +102,44 @@ fn check_section<'a>(
         let prev_line_trimmed_lowercase = prev_line.trim_start_matches(' ').to_lowercase();
 
         if trimmed_line.to_lowercase() < prev_line_trimmed_lowercase {
-            tidy_error!(bad, "{file}:{}: line not in alphabetical order", idx + 1);
+            tidy_error_ext!(err, bad, "{file}:{}: line not in alphabetical order", idx + 1);
         }
 
         prev_line = line;
     }
 
-    tidy_error!(bad, "{file}: reached end of file expecting `{END_MARKER}`")
+    tidy_error_ext!(err, bad, "{file}: reached end of file expecting `{END_MARKER}`")
+}
+
+fn check_lines<'a>(
+    file: &impl Display,
+    mut lines: impl Iterator<Item = (usize, &'a str)>,
+    err: &mut dyn FnMut(&str) -> std::io::Result<()>,
+    bad: &mut bool,
+) {
+    while let Some((idx, line)) = lines.next() {
+        if line.contains(END_MARKER) {
+            tidy_error_ext!(
+                err,
+                bad,
+                "{file}:{} found `{END_MARKER}` expecting `{START_MARKER}`",
+                idx + 1
+            )
+        }
+
+        if line.contains(START_MARKER) {
+            check_section(file, &mut lines, err, bad);
+        }
+    }
 }
 
 pub fn check(path: &Path, bad: &mut bool) {
-    walk(path, |path, _is_dir| filter_dirs(path), &mut |entry, contents| {
+    let skip =
+        |path: &_, _is_dir| filter_dirs(path) || path.ends_with("tidy/src/alphabetical/tests.rs");
+
+    walk(path, skip, &mut |entry, contents| {
         let file = &entry.path().display();
-
-        let mut lines = contents.lines().enumerate();
-        while let Some((idx, line)) = lines.next() {
-            if line.contains(END_MARKER) {
-                tidy_error!(
-                    bad,
-                    "{file}:{} found `{END_MARKER}` expecting `{START_MARKER}`",
-                    idx + 1
-                )
-            }
-
-            if line.contains(START_MARKER) {
-                check_section(file, &mut lines, bad);
-            }
-        }
+        let lines = contents.lines().enumerate();
+        check_lines(file, lines, &mut crate::tidy_error, bad)
     });
 }
diff --git a/src/tools/tidy/src/alphabetical/tests.rs b/src/tools/tidy/src/alphabetical/tests.rs
new file mode 100644
index 000000000000..560e0284b89a
--- /dev/null
+++ b/src/tools/tidy/src/alphabetical/tests.rs
@@ -0,0 +1,188 @@
+use super::*;
+use std::io::Write;
+use std::str::from_utf8;
+
+fn test(lines: &str, name: &str, expected_msg: &str, expected_bad: bool) {
+    let mut actual_msg = Vec::new();
+    let mut actual_bad = false;
+    let mut err = |args: &_| {
+        write!(&mut actual_msg, "{args}")?;
+        Ok(())
+    };
+    check_lines(&name, lines.lines().enumerate(), &mut err, &mut actual_bad);
+    assert_eq!(expected_msg, from_utf8(&actual_msg).unwrap());
+    assert_eq!(expected_bad, actual_bad);
+}
+
+fn good(lines: &str) {
+    test(lines, "good", "", false);
+}
+
+fn bad(lines: &str, expected_msg: &str) {
+    test(lines, "bad", expected_msg, true);
+}
+
+#[test]
+fn test_no_markers() {
+    let lines = "\
+        def
+        abc
+        xyz
+    ";
+    good(lines);
+}
+
+#[test]
+fn test_rust_good() {
+    let lines = "\
+        // tidy-alphabetical-start
+        abc
+        def
+        xyz
+        // tidy-alphabetical-end"; // important: end marker on last line
+    good(lines);
+}
+
+#[test]
+fn test_complex_good() {
+    let lines = "\
+        zzz
+
+        // tidy-alphabetical-start
+        abc
+        // Rust comments are ok
+        def
+        # TOML comments are ok
+        xyz
+        // tidy-alphabetical-end
+
+        # tidy-alphabetical-start
+        foo(abc);
+        // blank lines are ok
+
+        // split line gets joined
+        foo(
+            def
+        );
+
+        foo(xyz);
+        # tidy-alphabetical-end
+
+        % tidy-alphabetical-start
+        abc
+            ignored_due_to_different_indent
+        def
+        % tidy-alphabetical-end
+
+        aaa
+    ";
+    good(lines);
+}
+
+#[test]
+fn test_rust_bad() {
+    let lines = "\
+        // tidy-alphabetical-start
+        abc
+        xyz
+        def
+        // tidy-alphabetical-end
+    ";
+    bad(lines, "bad:4: line not in alphabetical order");
+}
+
+#[test]
+fn test_toml_bad() {
+    let lines = "\
+        # tidy-alphabetical-start
+        abc
+        xyz
+        def
+        # tidy-alphabetical-end
+    ";
+    bad(lines, "bad:4: line not in alphabetical order");
+}
+
+#[test]
+fn test_features_bad() {
+    // Even though lines starting with `#` are treated as comments, lines
+    // starting with `#!` are an exception.
+    let lines = "\
+        tidy-alphabetical-start
+        #![feature(abc)]
+        #![feature(xyz)]
+        #![feature(def)]
+        tidy-alphabetical-end
+    ";
+    bad(lines, "bad:4: line not in alphabetical order");
+}
+
+#[test]
+fn test_indent_bad() {
+    // All lines are indented the same amount, and so are checked.
+    let lines = "\
+        $ tidy-alphabetical-start
+            abc
+            xyz
+            def
+        $ tidy-alphabetical-end
+    ";
+    bad(lines, "bad:4: line not in alphabetical order");
+}
+
+#[test]
+fn test_split_bad() {
+    let lines = "\
+        || tidy-alphabetical-start
+        foo(abc)
+        foo(
+            xyz
+        )
+        foo(
+            def
+        )
+        && tidy-alphabetical-end
+    ";
+    bad(lines, "bad:7: line not in alphabetical order");
+}
+
+#[test]
+fn test_double_start() {
+    let lines = "\
+        tidy-alphabetical-start
+        abc
+        tidy-alphabetical-start
+    ";
+    bad(lines, "bad:3 found `tidy-alphabetical-start` expecting `tidy-alphabetical-end`");
+}
+
+#[test]
+fn test_missing_start() {
+    let lines = "\
+        abc
+        tidy-alphabetical-end
+        abc
+    ";
+    bad(lines, "bad:2 found `tidy-alphabetical-end` expecting `tidy-alphabetical-start`");
+}
+
+#[test]
+fn test_missing_end() {
+    let lines = "\
+        tidy-alphabetical-start
+        abc
+    ";
+    bad(lines, "bad: reached end of file expecting `tidy-alphabetical-end`");
+}
+
+#[test]
+fn test_double_end() {
+    let lines = "\
+        tidy-alphabetical-start
+        abc
+        tidy-alphabetical-end
+        def
+        tidy-alphabetical-end
+    ";
+    bad(lines, "bad:5 found `tidy-alphabetical-end` expecting `tidy-alphabetical-start`");
+}
diff --git a/src/tools/tidy/src/lib.rs b/src/tools/tidy/src/lib.rs
index fc69c1432227..eb0a2fda290d 100644
--- a/src/tools/tidy/src/lib.rs
+++ b/src/tools/tidy/src/lib.rs
@@ -3,8 +3,6 @@
 //! This library contains the tidy lints and exposes it
 //! to be used by tools.
 
-use std::fmt::Display;
-
 use termcolor::WriteColor;
 
 /// A helper macro to `unwrap` a result except also print out details like:
@@ -31,16 +29,22 @@ macro_rules! t {
 
 macro_rules! tidy_error {
     ($bad:expr, $($fmt:tt)*) => ({
-        $crate::tidy_error($bad, format_args!($($fmt)*)).expect("failed to output error");
+        $crate::tidy_error(&format_args!($($fmt)*).to_string()).expect("failed to output error");
+        *$bad = true;
     });
 }
 
-fn tidy_error(bad: &mut bool, args: impl Display) -> std::io::Result<()> {
+macro_rules! tidy_error_ext {
+    ($tidy_error:path, $bad:expr, $($fmt:tt)*) => ({
+        $tidy_error(&format_args!($($fmt)*).to_string()).expect("failed to output error");
+        *$bad = true;
+    });
+}
+
+fn tidy_error(args: &str) -> std::io::Result<()> {
     use std::io::Write;
     use termcolor::{Color, ColorChoice, ColorSpec, StandardStream};
 
-    *bad = true;
-
     let mut stderr = StandardStream::stdout(ColorChoice::Auto);
     stderr.set_color(ColorSpec::new().set_fg(Some(Color::Red)))?;
 

From d28405972ff8e83ac5e1be53c72eb44a43f983a8 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 16 Sep 2023 17:13:30 +0000
Subject: [PATCH 056/297] Do not remove unused definitions inside GVN.

---
 compiler/rustc_mir_transform/src/gvn.rs       |  12 -
 compiler/rustc_mir_transform/src/lib.rs       |   1 +
 compiler/rustc_mir_transform/src/simplify.rs  |   2 +
 .../const_debuginfo.main.ConstDebugInfo.diff  |  75 ++-
 .../gvn.arithmetic.GVN.panic-abort.diff       | 107 ++--
 .../gvn.arithmetic.GVN.panic-unwind.diff      | 107 ++--
 ...vn.arithmetic_checked.GVN.panic-abort.diff | 109 ++--
 ...n.arithmetic_checked.GVN.panic-unwind.diff | 109 ++--
 .../gvn.arithmetic_float.GVN.panic-abort.diff |  66 +--
 ...gvn.arithmetic_float.GVN.panic-unwind.diff |  66 +--
 tests/mir-opt/gvn.cast.GVN.panic-abort.diff   | 156 +++---
 tests/mir-opt/gvn.cast.GVN.panic-unwind.diff  | 156 +++---
 .../gvn.dereferences.GVN.panic-abort.diff     |  14 +-
 .../gvn.dereferences.GVN.panic-unwind.diff    |  14 +-
 ...gvn.multiple_branches.GVN.panic-abort.diff | 136 ++---
 ...vn.multiple_branches.GVN.panic-unwind.diff | 136 ++---
 .../gvn.repeated_index.GVN.panic-abort.diff   |   7 +-
 .../gvn.repeated_index.GVN.panic-unwind.diff  |   7 +-
 tests/mir-opt/gvn.slices.GVN.panic-abort.diff |  41 +-
 .../mir-opt/gvn.slices.GVN.panic-unwind.diff  |  41 +-
 ...xpression_elimination.GVN.panic-abort.diff | 493 ++++++++++--------
 ...pression_elimination.GVN.panic-unwind.diff | 493 ++++++++++--------
 .../gvn.wrap_unwrap.GVN.panic-abort.diff      |   8 +-
 .../gvn.wrap_unwrap.GVN.panic-unwind.diff     |   8 +-
 ...implifyComparisonIntegral.panic-abort.diff |  47 +-
 ...mplifyComparisonIntegral.panic-unwind.diff |  47 +-
 ...ondition-after-const-prop.panic-abort.diff |   8 +-
 ...ndition-after-const-prop.panic-unwind.diff |   8 +-
 28 files changed, 1348 insertions(+), 1126 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index eece7c3e8341..e7f36e5881a5 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -118,16 +118,11 @@ fn propagate_ssa<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         let data = &mut body.basic_blocks.as_mut_preserves_cfg()[bb];
         state.visit_basic_block_data(bb, data);
     }
-    let any_replacement = state.any_replacement;
 
     // For each local that is reused (`y` above), we remove its storage statements do avoid any
     // difficulty. Those locals are SSA, so should be easy to optimize by LLVM without storage
     // statements.
     StorageRemover { tcx, reused_locals: state.reused_locals }.visit_body_preserves_cfg(body);
-
-    if any_replacement {
-        crate::simplify::remove_unused_definitions(body);
-    }
 }
 
 newtype_index! {
@@ -190,7 +185,6 @@ struct VnState<'body, 'tcx> {
     ssa: &'body SsaLocals,
     dominators: &'body Dominators<BasicBlock>,
     reused_locals: BitSet<Local>,
-    any_replacement: bool,
 }
 
 impl<'body, 'tcx> VnState<'body, 'tcx> {
@@ -212,7 +206,6 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             ssa,
             dominators,
             reused_locals: BitSet::new_empty(local_decls.len()),
-            any_replacement: false,
         }
     }
 
@@ -324,14 +317,12 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         {
             *place = local.into();
             self.reused_locals.insert(local);
-            self.any_replacement = true;
         } else if place_ref.local != place.local
             || place_ref.projection.len() < place.projection.len()
         {
             // By the invariant on `place_ref`.
             *place = place_ref.project_deeper(&[], self.tcx);
             self.reused_locals.insert(place_ref.local);
-            self.any_replacement = true;
         }
 
         Some(value)
@@ -349,7 +340,6 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                 let value = self.simplify_place_value(place, location)?;
                 if let Some(const_) = self.try_as_constant(value) {
                     *operand = Operand::Constant(Box::new(const_));
-                    self.any_replacement = true;
                 }
                 Some(value)
             }
@@ -502,13 +492,11 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, 'tcx> {
         {
             if let Some(const_) = self.try_as_constant(value) {
                 *rvalue = Rvalue::Use(Operand::Constant(Box::new(const_)));
-                self.any_replacement = true;
             } else if let Some(local) = self.try_as_local(value, location)
                 && *rvalue != Rvalue::Use(Operand::Move(local.into()))
             {
                 *rvalue = Rvalue::Use(Operand::Copy(local.into()));
                 self.reused_locals.insert(local);
-                self.any_replacement = true;
             }
         }
     }
diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs
index 9aaa54110bdf..4ec91a55f1d2 100644
--- a/compiler/rustc_mir_transform/src/lib.rs
+++ b/compiler/rustc_mir_transform/src/lib.rs
@@ -570,6 +570,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
             &separate_const_switch::SeparateConstSwitch,
             &const_prop::ConstProp,
             &gvn::GVN,
+            &simplify::SimplifyLocals::AfterGVN,
             &dataflow_const_prop::DataflowConstProp,
             &const_debuginfo::ConstDebugInfo,
             &o1(simplify_branches::SimplifyConstCondition::AfterConstProp),
diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs
index 88c89e106fd5..0a1c011147ae 100644
--- a/compiler/rustc_mir_transform/src/simplify.rs
+++ b/compiler/rustc_mir_transform/src/simplify.rs
@@ -366,6 +366,7 @@ pub fn remove_dead_blocks(body: &mut Body<'_>) {
 
 pub enum SimplifyLocals {
     BeforeConstProp,
+    AfterGVN,
     Final,
 }
 
@@ -373,6 +374,7 @@ impl<'tcx> MirPass<'tcx> for SimplifyLocals {
     fn name(&self) -> &'static str {
         match &self {
             SimplifyLocals::BeforeConstProp => "SimplifyLocals-before-const-prop",
+            SimplifyLocals::AfterGVN => "SimplifyLocals-after-value-numbering",
             SimplifyLocals::Final => "SimplifyLocals-final",
         }
     }
diff --git a/tests/mir-opt/const_debuginfo.main.ConstDebugInfo.diff b/tests/mir-opt/const_debuginfo.main.ConstDebugInfo.diff
index 313e5dddbbba..f5d822520a70 100644
--- a/tests/mir-opt/const_debuginfo.main.ConstDebugInfo.diff
+++ b/tests/mir-opt/const_debuginfo.main.ConstDebugInfo.diff
@@ -4,12 +4,6 @@
   fn main() -> () {
       let mut _0: ();
       let _1: u8;
-      let mut _5: u8;
-      let mut _6: u8;
-      let mut _7: u8;
-      let mut _8: u8;
-      let mut _12: u32;
-      let mut _13: u32;
       scope 1 {
 -         debug x => _1;
 +         debug x => const 1_u8;
@@ -25,34 +19,34 @@
                   scope 4 {
 -                     debug sum => _4;
 +                     debug sum => const 6_u8;
-                      let _9: &str;
+                      let _5: &str;
                       scope 5 {
--                         debug s => _9;
+-                         debug s => _5;
 +                         debug s => const "hello, world!";
-                          let _14: bool;
-                          let _15: bool;
-                          let _16: u32;
+                          let _8: bool;
+                          let _9: bool;
+                          let _10: u32;
                           scope 6 {
--                             debug ((f: (bool, bool, u32)).0: bool) => _14;
--                             debug ((f: (bool, bool, u32)).1: bool) => _15;
--                             debug ((f: (bool, bool, u32)).2: u32) => _16;
+-                             debug ((f: (bool, bool, u32)).0: bool) => _8;
+-                             debug ((f: (bool, bool, u32)).1: bool) => _9;
+-                             debug ((f: (bool, bool, u32)).2: u32) => _10;
 +                             debug ((f: (bool, bool, u32)).0: bool) => const true;
 +                             debug ((f: (bool, bool, u32)).1: bool) => const false;
 +                             debug ((f: (bool, bool, u32)).2: u32) => const 123_u32;
-                              let _10: std::option::Option<u16>;
+                              let _6: std::option::Option<u16>;
                               scope 7 {
--                                 debug o => _10;
+-                                 debug o => _6;
 +                                 debug o => const Option::<u16>::Some(99_u16);
-                                  let _17: u32;
-                                  let _18: u32;
+                                  let _11: u32;
+                                  let _12: u32;
                                   scope 8 {
--                                     debug ((p: Point).0: u32) => _17;
--                                     debug ((p: Point).1: u32) => _18;
+-                                     debug ((p: Point).0: u32) => _11;
+-                                     debug ((p: Point).1: u32) => _12;
 +                                     debug ((p: Point).0: u32) => const 32_u32;
 +                                     debug ((p: Point).1: u32) => const 32_u32;
-                                      let _11: u32;
+                                      let _7: u32;
                                       scope 9 {
--                                         debug a => _11;
+-                                         debug a => _7;
 +                                         debug a => const 64_u32;
                                       }
                                   }
@@ -69,30 +63,27 @@
           _2 = const 2_u8;
           _3 = const 3_u8;
           StorageLive(_4);
-          StorageLive(_5);
-          _5 = const 3_u8;
           _4 = const 6_u8;
-          StorageDead(_5);
+          StorageLive(_5);
+          _5 = const "hello, world!";
+          StorageLive(_8);
           StorageLive(_9);
-          _9 = const "hello, world!";
-          StorageLive(_14);
-          StorageLive(_15);
-          StorageLive(_16);
-          _14 = const true;
-          _15 = const false;
-          _16 = const 123_u32;
           StorageLive(_10);
-          _10 = const Option::<u16>::Some(99_u16);
-          _17 = const 32_u32;
-          _18 = const 32_u32;
-          StorageLive(_11);
-          _11 = const 64_u32;
-          StorageDead(_11);
-          StorageDead(_10);
-          StorageDead(_14);
-          StorageDead(_15);
-          StorageDead(_16);
+          _8 = const true;
+          _9 = const false;
+          _10 = const 123_u32;
+          StorageLive(_6);
+          _6 = const Option::<u16>::Some(99_u16);
+          _11 = const 32_u32;
+          _12 = const 32_u32;
+          StorageLive(_7);
+          _7 = const 64_u32;
+          StorageDead(_7);
+          StorageDead(_6);
+          StorageDead(_8);
           StorageDead(_9);
+          StorageDead(_10);
+          StorageDead(_5);
           StorageDead(_4);
           return;
       }
diff --git a/tests/mir-opt/gvn.arithmetic.GVN.panic-abort.diff b/tests/mir-opt/gvn.arithmetic.GVN.panic-abort.diff
index 3f5173c189e1..17ba0d3cef2d 100644
--- a/tests/mir-opt/gvn.arithmetic.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.arithmetic.GVN.panic-abort.diff
@@ -67,11 +67,11 @@
       bb0: {
           StorageLive(_2);
           StorageLive(_3);
--         StorageLive(_4);
--         _4 = _1;
+          StorageLive(_4);
+          _4 = _1;
 -         _3 = Add(move _4, const 0_u64);
--         StorageDead(_4);
 +         _3 = Add(_1, const 0_u64);
+          StorageDead(_4);
           _2 = opaque::<u64>(move _3) -> [return: bb1, unwind unreachable];
       }
   
@@ -80,11 +80,11 @@
           StorageDead(_2);
           StorageLive(_5);
           StorageLive(_6);
--         StorageLive(_7);
--         _7 = _1;
+          StorageLive(_7);
+          _7 = _1;
 -         _6 = Sub(move _7, const 0_u64);
--         StorageDead(_7);
 +         _6 = Sub(_1, const 0_u64);
+          StorageDead(_7);
           _5 = opaque::<u64>(move _6) -> [return: bb2, unwind unreachable];
       }
   
@@ -93,11 +93,11 @@
           StorageDead(_5);
           StorageLive(_8);
           StorageLive(_9);
--         StorageLive(_10);
--         _10 = _1;
+          StorageLive(_10);
+          _10 = _1;
 -         _9 = Mul(move _10, const 0_u64);
--         StorageDead(_10);
 +         _9 = Mul(_1, const 0_u64);
+          StorageDead(_10);
           _8 = opaque::<u64>(move _9) -> [return: bb3, unwind unreachable];
       }
   
@@ -106,11 +106,11 @@
           StorageDead(_8);
           StorageLive(_11);
           StorageLive(_12);
--         StorageLive(_13);
--         _13 = _1;
+          StorageLive(_13);
+          _13 = _1;
 -         _12 = Mul(move _13, const 1_u64);
--         StorageDead(_13);
 +         _12 = Mul(_1, const 1_u64);
+          StorageDead(_13);
           _11 = opaque::<u64>(move _12) -> [return: bb4, unwind unreachable];
       }
   
@@ -119,8 +119,8 @@
           StorageDead(_11);
           StorageLive(_14);
           StorageLive(_15);
--         StorageLive(_16);
--         _16 = _1;
+          StorageLive(_16);
+          _16 = _1;
           _17 = Eq(const 0_u64, const 0_u64);
 -         assert(!move _17, "attempt to divide `{}` by zero", _16) -> [success: bb5, unwind unreachable];
 +         assert(!_17, "attempt to divide `{}` by zero", _1) -> [success: bb5, unwind unreachable];
@@ -128,8 +128,8 @@
   
       bb5: {
 -         _15 = Div(move _16, const 0_u64);
--         StorageDead(_16);
 +         _15 = Div(_1, const 0_u64);
+          StorageDead(_16);
           _14 = opaque::<u64>(move _15) -> [return: bb6, unwind unreachable];
       }
   
@@ -138,8 +138,8 @@
           StorageDead(_14);
           StorageLive(_18);
           StorageLive(_19);
--         StorageLive(_20);
--         _20 = _1;
+          StorageLive(_20);
+          _20 = _1;
           _21 = Eq(const 1_u64, const 0_u64);
 -         assert(!move _21, "attempt to divide `{}` by zero", _20) -> [success: bb7, unwind unreachable];
 +         assert(!_21, "attempt to divide `{}` by zero", _1) -> [success: bb7, unwind unreachable];
@@ -147,8 +147,8 @@
   
       bb7: {
 -         _19 = Div(move _20, const 1_u64);
--         StorageDead(_20);
 +         _19 = Div(_1, const 1_u64);
+          StorageDead(_20);
           _18 = opaque::<u64>(move _19) -> [return: bb8, unwind unreachable];
       }
   
@@ -157,8 +157,8 @@
           StorageDead(_18);
           StorageLive(_22);
           StorageLive(_23);
--         StorageLive(_24);
--         _24 = _1;
+          StorageLive(_24);
+          _24 = _1;
 -         _25 = Eq(_24, const 0_u64);
 -         assert(!move _25, "attempt to divide `{}` by zero", const 0_u64) -> [success: bb9, unwind unreachable];
 +         _25 = Eq(_1, const 0_u64);
@@ -167,8 +167,8 @@
   
       bb9: {
 -         _23 = Div(const 0_u64, move _24);
--         StorageDead(_24);
 +         _23 = Div(const 0_u64, _1);
+          StorageDead(_24);
           _22 = opaque::<u64>(move _23) -> [return: bb10, unwind unreachable];
       }
   
@@ -177,17 +177,18 @@
           StorageDead(_22);
           StorageLive(_26);
           StorageLive(_27);
--         StorageLive(_28);
--         _28 = _1;
+          StorageLive(_28);
+          _28 = _1;
 -         _29 = Eq(_28, const 0_u64);
 -         assert(!move _29, "attempt to divide `{}` by zero", const 1_u64) -> [success: bb11, unwind unreachable];
++         _29 = _25;
 +         assert(!_25, "attempt to divide `{}` by zero", const 1_u64) -> [success: bb11, unwind unreachable];
       }
   
       bb11: {
 -         _27 = Div(const 1_u64, move _28);
--         StorageDead(_28);
 +         _27 = Div(const 1_u64, _1);
+          StorageDead(_28);
           _26 = opaque::<u64>(move _27) -> [return: bb12, unwind unreachable];
       }
   
@@ -196,17 +197,18 @@
           StorageDead(_26);
           StorageLive(_30);
           StorageLive(_31);
--         StorageLive(_32);
--         _32 = _1;
+          StorageLive(_32);
+          _32 = _1;
 -         _33 = Eq(const 0_u64, const 0_u64);
 -         assert(!move _33, "attempt to calculate the remainder of `{}` with a divisor of zero", _32) -> [success: bb13, unwind unreachable];
++         _33 = _17;
 +         assert(!_17, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb13, unwind unreachable];
       }
   
       bb13: {
 -         _31 = Rem(move _32, const 0_u64);
--         StorageDead(_32);
 +         _31 = Rem(_1, const 0_u64);
+          StorageDead(_32);
           _30 = opaque::<u64>(move _31) -> [return: bb14, unwind unreachable];
       }
   
@@ -215,17 +217,18 @@
           StorageDead(_30);
           StorageLive(_34);
           StorageLive(_35);
--         StorageLive(_36);
--         _36 = _1;
+          StorageLive(_36);
+          _36 = _1;
 -         _37 = Eq(const 1_u64, const 0_u64);
 -         assert(!move _37, "attempt to calculate the remainder of `{}` with a divisor of zero", _36) -> [success: bb15, unwind unreachable];
++         _37 = _21;
 +         assert(!_21, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb15, unwind unreachable];
       }
   
       bb15: {
 -         _35 = Rem(move _36, const 1_u64);
--         StorageDead(_36);
 +         _35 = Rem(_1, const 1_u64);
+          StorageDead(_36);
           _34 = opaque::<u64>(move _35) -> [return: bb16, unwind unreachable];
       }
   
@@ -234,17 +237,18 @@
           StorageDead(_34);
           StorageLive(_38);
           StorageLive(_39);
--         StorageLive(_40);
--         _40 = _1;
+          StorageLive(_40);
+          _40 = _1;
 -         _41 = Eq(_40, const 0_u64);
 -         assert(!move _41, "attempt to calculate the remainder of `{}` with a divisor of zero", const 0_u64) -> [success: bb17, unwind unreachable];
++         _41 = _25;
 +         assert(!_25, "attempt to calculate the remainder of `{}` with a divisor of zero", const 0_u64) -> [success: bb17, unwind unreachable];
       }
   
       bb17: {
 -         _39 = Rem(const 0_u64, move _40);
--         StorageDead(_40);
 +         _39 = Rem(const 0_u64, _1);
+          StorageDead(_40);
           _38 = opaque::<u64>(move _39) -> [return: bb18, unwind unreachable];
       }
   
@@ -253,17 +257,18 @@
           StorageDead(_38);
           StorageLive(_42);
           StorageLive(_43);
--         StorageLive(_44);
--         _44 = _1;
+          StorageLive(_44);
+          _44 = _1;
 -         _45 = Eq(_44, const 0_u64);
 -         assert(!move _45, "attempt to calculate the remainder of `{}` with a divisor of zero", const 1_u64) -> [success: bb19, unwind unreachable];
++         _45 = _25;
 +         assert(!_25, "attempt to calculate the remainder of `{}` with a divisor of zero", const 1_u64) -> [success: bb19, unwind unreachable];
       }
   
       bb19: {
 -         _43 = Rem(const 1_u64, move _44);
--         StorageDead(_44);
 +         _43 = Rem(const 1_u64, _1);
+          StorageDead(_44);
           _42 = opaque::<u64>(move _43) -> [return: bb20, unwind unreachable];
       }
   
@@ -272,11 +277,11 @@
           StorageDead(_42);
           StorageLive(_46);
           StorageLive(_47);
--         StorageLive(_48);
--         _48 = _1;
+          StorageLive(_48);
+          _48 = _1;
 -         _47 = BitAnd(move _48, const 0_u64);
--         StorageDead(_48);
 +         _47 = BitAnd(_1, const 0_u64);
+          StorageDead(_48);
           _46 = opaque::<u64>(move _47) -> [return: bb21, unwind unreachable];
       }
   
@@ -285,11 +290,11 @@
           StorageDead(_46);
           StorageLive(_49);
           StorageLive(_50);
--         StorageLive(_51);
--         _51 = _1;
+          StorageLive(_51);
+          _51 = _1;
 -         _50 = BitOr(move _51, const 0_u64);
--         StorageDead(_51);
 +         _50 = BitOr(_1, const 0_u64);
+          StorageDead(_51);
           _49 = opaque::<u64>(move _50) -> [return: bb22, unwind unreachable];
       }
   
@@ -298,11 +303,11 @@
           StorageDead(_49);
           StorageLive(_52);
           StorageLive(_53);
--         StorageLive(_54);
--         _54 = _1;
+          StorageLive(_54);
+          _54 = _1;
 -         _53 = BitXor(move _54, const 0_u64);
--         StorageDead(_54);
 +         _53 = BitXor(_1, const 0_u64);
+          StorageDead(_54);
           _52 = opaque::<u64>(move _53) -> [return: bb23, unwind unreachable];
       }
   
@@ -311,11 +316,11 @@
           StorageDead(_52);
           StorageLive(_55);
           StorageLive(_56);
--         StorageLive(_57);
--         _57 = _1;
+          StorageLive(_57);
+          _57 = _1;
 -         _56 = Shr(move _57, const 0_i32);
--         StorageDead(_57);
 +         _56 = Shr(_1, const 0_i32);
+          StorageDead(_57);
           _55 = opaque::<u64>(move _56) -> [return: bb24, unwind unreachable];
       }
   
@@ -324,11 +329,11 @@
           StorageDead(_55);
           StorageLive(_58);
           StorageLive(_59);
--         StorageLive(_60);
--         _60 = _1;
+          StorageLive(_60);
+          _60 = _1;
 -         _59 = Shl(move _60, const 0_i32);
--         StorageDead(_60);
 +         _59 = Shl(_1, const 0_i32);
+          StorageDead(_60);
           _58 = opaque::<u64>(move _59) -> [return: bb25, unwind unreachable];
       }
   
diff --git a/tests/mir-opt/gvn.arithmetic.GVN.panic-unwind.diff b/tests/mir-opt/gvn.arithmetic.GVN.panic-unwind.diff
index 38da21d91d41..f14fd409bea1 100644
--- a/tests/mir-opt/gvn.arithmetic.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.arithmetic.GVN.panic-unwind.diff
@@ -67,11 +67,11 @@
       bb0: {
           StorageLive(_2);
           StorageLive(_3);
--         StorageLive(_4);
--         _4 = _1;
+          StorageLive(_4);
+          _4 = _1;
 -         _3 = Add(move _4, const 0_u64);
--         StorageDead(_4);
 +         _3 = Add(_1, const 0_u64);
+          StorageDead(_4);
           _2 = opaque::<u64>(move _3) -> [return: bb1, unwind continue];
       }
   
@@ -80,11 +80,11 @@
           StorageDead(_2);
           StorageLive(_5);
           StorageLive(_6);
--         StorageLive(_7);
--         _7 = _1;
+          StorageLive(_7);
+          _7 = _1;
 -         _6 = Sub(move _7, const 0_u64);
--         StorageDead(_7);
 +         _6 = Sub(_1, const 0_u64);
+          StorageDead(_7);
           _5 = opaque::<u64>(move _6) -> [return: bb2, unwind continue];
       }
   
@@ -93,11 +93,11 @@
           StorageDead(_5);
           StorageLive(_8);
           StorageLive(_9);
--         StorageLive(_10);
--         _10 = _1;
+          StorageLive(_10);
+          _10 = _1;
 -         _9 = Mul(move _10, const 0_u64);
--         StorageDead(_10);
 +         _9 = Mul(_1, const 0_u64);
+          StorageDead(_10);
           _8 = opaque::<u64>(move _9) -> [return: bb3, unwind continue];
       }
   
@@ -106,11 +106,11 @@
           StorageDead(_8);
           StorageLive(_11);
           StorageLive(_12);
--         StorageLive(_13);
--         _13 = _1;
+          StorageLive(_13);
+          _13 = _1;
 -         _12 = Mul(move _13, const 1_u64);
--         StorageDead(_13);
 +         _12 = Mul(_1, const 1_u64);
+          StorageDead(_13);
           _11 = opaque::<u64>(move _12) -> [return: bb4, unwind continue];
       }
   
@@ -119,8 +119,8 @@
           StorageDead(_11);
           StorageLive(_14);
           StorageLive(_15);
--         StorageLive(_16);
--         _16 = _1;
+          StorageLive(_16);
+          _16 = _1;
           _17 = Eq(const 0_u64, const 0_u64);
 -         assert(!move _17, "attempt to divide `{}` by zero", _16) -> [success: bb5, unwind continue];
 +         assert(!_17, "attempt to divide `{}` by zero", _1) -> [success: bb5, unwind continue];
@@ -128,8 +128,8 @@
   
       bb5: {
 -         _15 = Div(move _16, const 0_u64);
--         StorageDead(_16);
 +         _15 = Div(_1, const 0_u64);
+          StorageDead(_16);
           _14 = opaque::<u64>(move _15) -> [return: bb6, unwind continue];
       }
   
@@ -138,8 +138,8 @@
           StorageDead(_14);
           StorageLive(_18);
           StorageLive(_19);
--         StorageLive(_20);
--         _20 = _1;
+          StorageLive(_20);
+          _20 = _1;
           _21 = Eq(const 1_u64, const 0_u64);
 -         assert(!move _21, "attempt to divide `{}` by zero", _20) -> [success: bb7, unwind continue];
 +         assert(!_21, "attempt to divide `{}` by zero", _1) -> [success: bb7, unwind continue];
@@ -147,8 +147,8 @@
   
       bb7: {
 -         _19 = Div(move _20, const 1_u64);
--         StorageDead(_20);
 +         _19 = Div(_1, const 1_u64);
+          StorageDead(_20);
           _18 = opaque::<u64>(move _19) -> [return: bb8, unwind continue];
       }
   
@@ -157,8 +157,8 @@
           StorageDead(_18);
           StorageLive(_22);
           StorageLive(_23);
--         StorageLive(_24);
--         _24 = _1;
+          StorageLive(_24);
+          _24 = _1;
 -         _25 = Eq(_24, const 0_u64);
 -         assert(!move _25, "attempt to divide `{}` by zero", const 0_u64) -> [success: bb9, unwind continue];
 +         _25 = Eq(_1, const 0_u64);
@@ -167,8 +167,8 @@
   
       bb9: {
 -         _23 = Div(const 0_u64, move _24);
--         StorageDead(_24);
 +         _23 = Div(const 0_u64, _1);
+          StorageDead(_24);
           _22 = opaque::<u64>(move _23) -> [return: bb10, unwind continue];
       }
   
@@ -177,17 +177,18 @@
           StorageDead(_22);
           StorageLive(_26);
           StorageLive(_27);
--         StorageLive(_28);
--         _28 = _1;
+          StorageLive(_28);
+          _28 = _1;
 -         _29 = Eq(_28, const 0_u64);
 -         assert(!move _29, "attempt to divide `{}` by zero", const 1_u64) -> [success: bb11, unwind continue];
++         _29 = _25;
 +         assert(!_25, "attempt to divide `{}` by zero", const 1_u64) -> [success: bb11, unwind continue];
       }
   
       bb11: {
 -         _27 = Div(const 1_u64, move _28);
--         StorageDead(_28);
 +         _27 = Div(const 1_u64, _1);
+          StorageDead(_28);
           _26 = opaque::<u64>(move _27) -> [return: bb12, unwind continue];
       }
   
@@ -196,17 +197,18 @@
           StorageDead(_26);
           StorageLive(_30);
           StorageLive(_31);
--         StorageLive(_32);
--         _32 = _1;
+          StorageLive(_32);
+          _32 = _1;
 -         _33 = Eq(const 0_u64, const 0_u64);
 -         assert(!move _33, "attempt to calculate the remainder of `{}` with a divisor of zero", _32) -> [success: bb13, unwind continue];
++         _33 = _17;
 +         assert(!_17, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb13, unwind continue];
       }
   
       bb13: {
 -         _31 = Rem(move _32, const 0_u64);
--         StorageDead(_32);
 +         _31 = Rem(_1, const 0_u64);
+          StorageDead(_32);
           _30 = opaque::<u64>(move _31) -> [return: bb14, unwind continue];
       }
   
@@ -215,17 +217,18 @@
           StorageDead(_30);
           StorageLive(_34);
           StorageLive(_35);
--         StorageLive(_36);
--         _36 = _1;
+          StorageLive(_36);
+          _36 = _1;
 -         _37 = Eq(const 1_u64, const 0_u64);
 -         assert(!move _37, "attempt to calculate the remainder of `{}` with a divisor of zero", _36) -> [success: bb15, unwind continue];
++         _37 = _21;
 +         assert(!_21, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb15, unwind continue];
       }
   
       bb15: {
 -         _35 = Rem(move _36, const 1_u64);
--         StorageDead(_36);
 +         _35 = Rem(_1, const 1_u64);
+          StorageDead(_36);
           _34 = opaque::<u64>(move _35) -> [return: bb16, unwind continue];
       }
   
@@ -234,17 +237,18 @@
           StorageDead(_34);
           StorageLive(_38);
           StorageLive(_39);
--         StorageLive(_40);
--         _40 = _1;
+          StorageLive(_40);
+          _40 = _1;
 -         _41 = Eq(_40, const 0_u64);
 -         assert(!move _41, "attempt to calculate the remainder of `{}` with a divisor of zero", const 0_u64) -> [success: bb17, unwind continue];
++         _41 = _25;
 +         assert(!_25, "attempt to calculate the remainder of `{}` with a divisor of zero", const 0_u64) -> [success: bb17, unwind continue];
       }
   
       bb17: {
 -         _39 = Rem(const 0_u64, move _40);
--         StorageDead(_40);
 +         _39 = Rem(const 0_u64, _1);
+          StorageDead(_40);
           _38 = opaque::<u64>(move _39) -> [return: bb18, unwind continue];
       }
   
@@ -253,17 +257,18 @@
           StorageDead(_38);
           StorageLive(_42);
           StorageLive(_43);
--         StorageLive(_44);
--         _44 = _1;
+          StorageLive(_44);
+          _44 = _1;
 -         _45 = Eq(_44, const 0_u64);
 -         assert(!move _45, "attempt to calculate the remainder of `{}` with a divisor of zero", const 1_u64) -> [success: bb19, unwind continue];
++         _45 = _25;
 +         assert(!_25, "attempt to calculate the remainder of `{}` with a divisor of zero", const 1_u64) -> [success: bb19, unwind continue];
       }
   
       bb19: {
 -         _43 = Rem(const 1_u64, move _44);
--         StorageDead(_44);
 +         _43 = Rem(const 1_u64, _1);
+          StorageDead(_44);
           _42 = opaque::<u64>(move _43) -> [return: bb20, unwind continue];
       }
   
@@ -272,11 +277,11 @@
           StorageDead(_42);
           StorageLive(_46);
           StorageLive(_47);
--         StorageLive(_48);
--         _48 = _1;
+          StorageLive(_48);
+          _48 = _1;
 -         _47 = BitAnd(move _48, const 0_u64);
--         StorageDead(_48);
 +         _47 = BitAnd(_1, const 0_u64);
+          StorageDead(_48);
           _46 = opaque::<u64>(move _47) -> [return: bb21, unwind continue];
       }
   
@@ -285,11 +290,11 @@
           StorageDead(_46);
           StorageLive(_49);
           StorageLive(_50);
--         StorageLive(_51);
--         _51 = _1;
+          StorageLive(_51);
+          _51 = _1;
 -         _50 = BitOr(move _51, const 0_u64);
--         StorageDead(_51);
 +         _50 = BitOr(_1, const 0_u64);
+          StorageDead(_51);
           _49 = opaque::<u64>(move _50) -> [return: bb22, unwind continue];
       }
   
@@ -298,11 +303,11 @@
           StorageDead(_49);
           StorageLive(_52);
           StorageLive(_53);
--         StorageLive(_54);
--         _54 = _1;
+          StorageLive(_54);
+          _54 = _1;
 -         _53 = BitXor(move _54, const 0_u64);
--         StorageDead(_54);
 +         _53 = BitXor(_1, const 0_u64);
+          StorageDead(_54);
           _52 = opaque::<u64>(move _53) -> [return: bb23, unwind continue];
       }
   
@@ -311,11 +316,11 @@
           StorageDead(_52);
           StorageLive(_55);
           StorageLive(_56);
--         StorageLive(_57);
--         _57 = _1;
+          StorageLive(_57);
+          _57 = _1;
 -         _56 = Shr(move _57, const 0_i32);
--         StorageDead(_57);
 +         _56 = Shr(_1, const 0_i32);
+          StorageDead(_57);
           _55 = opaque::<u64>(move _56) -> [return: bb24, unwind continue];
       }
   
@@ -324,11 +329,11 @@
           StorageDead(_55);
           StorageLive(_58);
           StorageLive(_59);
--         StorageLive(_60);
--         _60 = _1;
+          StorageLive(_60);
+          _60 = _1;
 -         _59 = Shl(move _60, const 0_i32);
--         StorageDead(_60);
 +         _59 = Shl(_1, const 0_i32);
+          StorageDead(_60);
           _58 = opaque::<u64>(move _59) -> [return: bb25, unwind continue];
       }
   
diff --git a/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-abort.diff b/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-abort.diff
index 0c342799e079..e586e4ac8898 100644
--- a/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-abort.diff
@@ -75,8 +75,8 @@
       bb0: {
           StorageLive(_2);
           StorageLive(_3);
--         StorageLive(_4);
--         _4 = _1;
+          StorageLive(_4);
+          _4 = _1;
 -         _5 = CheckedAdd(_4, const 0_u64);
 -         assert(!move (_5.1: bool), "attempt to compute `{} + {}`, which would overflow", move _4, const 0_u64) -> [success: bb1, unwind unreachable];
 +         _5 = CheckedAdd(_1, const 0_u64);
@@ -85,7 +85,7 @@
   
       bb1: {
           _3 = move (_5.0: u64);
--         StorageDead(_4);
+          StorageDead(_4);
           _2 = opaque::<u64>(move _3) -> [return: bb2, unwind unreachable];
       }
   
@@ -94,8 +94,8 @@
           StorageDead(_2);
           StorageLive(_6);
           StorageLive(_7);
--         StorageLive(_8);
--         _8 = _1;
+          StorageLive(_8);
+          _8 = _1;
 -         _9 = CheckedSub(_8, const 0_u64);
 -         assert(!move (_9.1: bool), "attempt to compute `{} - {}`, which would overflow", move _8, const 0_u64) -> [success: bb3, unwind unreachable];
 +         _9 = CheckedSub(_1, const 0_u64);
@@ -104,7 +104,7 @@
   
       bb3: {
           _7 = move (_9.0: u64);
--         StorageDead(_8);
+          StorageDead(_8);
           _6 = opaque::<u64>(move _7) -> [return: bb4, unwind unreachable];
       }
   
@@ -113,8 +113,8 @@
           StorageDead(_6);
           StorageLive(_10);
           StorageLive(_11);
--         StorageLive(_12);
--         _12 = _1;
+          StorageLive(_12);
+          _12 = _1;
 -         _13 = CheckedMul(_12, const 0_u64);
 -         assert(!move (_13.1: bool), "attempt to compute `{} * {}`, which would overflow", move _12, const 0_u64) -> [success: bb5, unwind unreachable];
 +         _13 = CheckedMul(_1, const 0_u64);
@@ -123,7 +123,7 @@
   
       bb5: {
           _11 = move (_13.0: u64);
--         StorageDead(_12);
+          StorageDead(_12);
           _10 = opaque::<u64>(move _11) -> [return: bb6, unwind unreachable];
       }
   
@@ -132,8 +132,8 @@
           StorageDead(_10);
           StorageLive(_14);
           StorageLive(_15);
--         StorageLive(_16);
--         _16 = _1;
+          StorageLive(_16);
+          _16 = _1;
 -         _17 = CheckedMul(_16, const 1_u64);
 -         assert(!move (_17.1: bool), "attempt to compute `{} * {}`, which would overflow", move _16, const 1_u64) -> [success: bb7, unwind unreachable];
 +         _17 = CheckedMul(_1, const 1_u64);
@@ -142,7 +142,7 @@
   
       bb7: {
           _15 = move (_17.0: u64);
--         StorageDead(_16);
+          StorageDead(_16);
           _14 = opaque::<u64>(move _15) -> [return: bb8, unwind unreachable];
       }
   
@@ -151,8 +151,8 @@
           StorageDead(_14);
           StorageLive(_18);
           StorageLive(_19);
--         StorageLive(_20);
--         _20 = _1;
+          StorageLive(_20);
+          _20 = _1;
           _21 = Eq(const 0_u64, const 0_u64);
 -         assert(!move _21, "attempt to divide `{}` by zero", _20) -> [success: bb9, unwind unreachable];
 +         assert(!_21, "attempt to divide `{}` by zero", _1) -> [success: bb9, unwind unreachable];
@@ -160,8 +160,8 @@
   
       bb9: {
 -         _19 = Div(move _20, const 0_u64);
--         StorageDead(_20);
 +         _19 = Div(_1, const 0_u64);
+          StorageDead(_20);
           _18 = opaque::<u64>(move _19) -> [return: bb10, unwind unreachable];
       }
   
@@ -170,8 +170,8 @@
           StorageDead(_18);
           StorageLive(_22);
           StorageLive(_23);
--         StorageLive(_24);
--         _24 = _1;
+          StorageLive(_24);
+          _24 = _1;
           _25 = Eq(const 1_u64, const 0_u64);
 -         assert(!move _25, "attempt to divide `{}` by zero", _24) -> [success: bb11, unwind unreachable];
 +         assert(!_25, "attempt to divide `{}` by zero", _1) -> [success: bb11, unwind unreachable];
@@ -179,8 +179,8 @@
   
       bb11: {
 -         _23 = Div(move _24, const 1_u64);
--         StorageDead(_24);
 +         _23 = Div(_1, const 1_u64);
+          StorageDead(_24);
           _22 = opaque::<u64>(move _23) -> [return: bb12, unwind unreachable];
       }
   
@@ -189,8 +189,8 @@
           StorageDead(_22);
           StorageLive(_26);
           StorageLive(_27);
--         StorageLive(_28);
--         _28 = _1;
+          StorageLive(_28);
+          _28 = _1;
 -         _29 = Eq(_28, const 0_u64);
 -         assert(!move _29, "attempt to divide `{}` by zero", const 0_u64) -> [success: bb13, unwind unreachable];
 +         _29 = Eq(_1, const 0_u64);
@@ -199,8 +199,8 @@
   
       bb13: {
 -         _27 = Div(const 0_u64, move _28);
--         StorageDead(_28);
 +         _27 = Div(const 0_u64, _1);
+          StorageDead(_28);
           _26 = opaque::<u64>(move _27) -> [return: bb14, unwind unreachable];
       }
   
@@ -209,17 +209,18 @@
           StorageDead(_26);
           StorageLive(_30);
           StorageLive(_31);
--         StorageLive(_32);
--         _32 = _1;
+          StorageLive(_32);
+          _32 = _1;
 -         _33 = Eq(_32, const 0_u64);
 -         assert(!move _33, "attempt to divide `{}` by zero", const 1_u64) -> [success: bb15, unwind unreachable];
++         _33 = _29;
 +         assert(!_29, "attempt to divide `{}` by zero", const 1_u64) -> [success: bb15, unwind unreachable];
       }
   
       bb15: {
 -         _31 = Div(const 1_u64, move _32);
--         StorageDead(_32);
 +         _31 = Div(const 1_u64, _1);
+          StorageDead(_32);
           _30 = opaque::<u64>(move _31) -> [return: bb16, unwind unreachable];
       }
   
@@ -228,17 +229,18 @@
           StorageDead(_30);
           StorageLive(_34);
           StorageLive(_35);
--         StorageLive(_36);
--         _36 = _1;
+          StorageLive(_36);
+          _36 = _1;
 -         _37 = Eq(const 0_u64, const 0_u64);
 -         assert(!move _37, "attempt to calculate the remainder of `{}` with a divisor of zero", _36) -> [success: bb17, unwind unreachable];
++         _37 = _21;
 +         assert(!_21, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb17, unwind unreachable];
       }
   
       bb17: {
 -         _35 = Rem(move _36, const 0_u64);
--         StorageDead(_36);
 +         _35 = Rem(_1, const 0_u64);
+          StorageDead(_36);
           _34 = opaque::<u64>(move _35) -> [return: bb18, unwind unreachable];
       }
   
@@ -247,17 +249,18 @@
           StorageDead(_34);
           StorageLive(_38);
           StorageLive(_39);
--         StorageLive(_40);
--         _40 = _1;
+          StorageLive(_40);
+          _40 = _1;
 -         _41 = Eq(const 1_u64, const 0_u64);
 -         assert(!move _41, "attempt to calculate the remainder of `{}` with a divisor of zero", _40) -> [success: bb19, unwind unreachable];
++         _41 = _25;
 +         assert(!_25, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb19, unwind unreachable];
       }
   
       bb19: {
 -         _39 = Rem(move _40, const 1_u64);
--         StorageDead(_40);
 +         _39 = Rem(_1, const 1_u64);
+          StorageDead(_40);
           _38 = opaque::<u64>(move _39) -> [return: bb20, unwind unreachable];
       }
   
@@ -266,17 +269,18 @@
           StorageDead(_38);
           StorageLive(_42);
           StorageLive(_43);
--         StorageLive(_44);
--         _44 = _1;
+          StorageLive(_44);
+          _44 = _1;
 -         _45 = Eq(_44, const 0_u64);
 -         assert(!move _45, "attempt to calculate the remainder of `{}` with a divisor of zero", const 0_u64) -> [success: bb21, unwind unreachable];
++         _45 = _29;
 +         assert(!_29, "attempt to calculate the remainder of `{}` with a divisor of zero", const 0_u64) -> [success: bb21, unwind unreachable];
       }
   
       bb21: {
 -         _43 = Rem(const 0_u64, move _44);
--         StorageDead(_44);
 +         _43 = Rem(const 0_u64, _1);
+          StorageDead(_44);
           _42 = opaque::<u64>(move _43) -> [return: bb22, unwind unreachable];
       }
   
@@ -285,17 +289,18 @@
           StorageDead(_42);
           StorageLive(_46);
           StorageLive(_47);
--         StorageLive(_48);
--         _48 = _1;
+          StorageLive(_48);
+          _48 = _1;
 -         _49 = Eq(_48, const 0_u64);
 -         assert(!move _49, "attempt to calculate the remainder of `{}` with a divisor of zero", const 1_u64) -> [success: bb23, unwind unreachable];
++         _49 = _29;
 +         assert(!_29, "attempt to calculate the remainder of `{}` with a divisor of zero", const 1_u64) -> [success: bb23, unwind unreachable];
       }
   
       bb23: {
 -         _47 = Rem(const 1_u64, move _48);
--         StorageDead(_48);
 +         _47 = Rem(const 1_u64, _1);
+          StorageDead(_48);
           _46 = opaque::<u64>(move _47) -> [return: bb24, unwind unreachable];
       }
   
@@ -304,11 +309,11 @@
           StorageDead(_46);
           StorageLive(_50);
           StorageLive(_51);
--         StorageLive(_52);
--         _52 = _1;
+          StorageLive(_52);
+          _52 = _1;
 -         _51 = BitAnd(move _52, const 0_u64);
--         StorageDead(_52);
 +         _51 = BitAnd(_1, const 0_u64);
+          StorageDead(_52);
           _50 = opaque::<u64>(move _51) -> [return: bb25, unwind unreachable];
       }
   
@@ -317,11 +322,11 @@
           StorageDead(_50);
           StorageLive(_53);
           StorageLive(_54);
--         StorageLive(_55);
--         _55 = _1;
+          StorageLive(_55);
+          _55 = _1;
 -         _54 = BitOr(move _55, const 0_u64);
--         StorageDead(_55);
 +         _54 = BitOr(_1, const 0_u64);
+          StorageDead(_55);
           _53 = opaque::<u64>(move _54) -> [return: bb26, unwind unreachable];
       }
   
@@ -330,11 +335,11 @@
           StorageDead(_53);
           StorageLive(_56);
           StorageLive(_57);
--         StorageLive(_58);
--         _58 = _1;
+          StorageLive(_58);
+          _58 = _1;
 -         _57 = BitXor(move _58, const 0_u64);
--         StorageDead(_58);
 +         _57 = BitXor(_1, const 0_u64);
+          StorageDead(_58);
           _56 = opaque::<u64>(move _57) -> [return: bb27, unwind unreachable];
       }
   
@@ -343,8 +348,8 @@
           StorageDead(_56);
           StorageLive(_59);
           StorageLive(_60);
--         StorageLive(_61);
--         _61 = _1;
+          StorageLive(_61);
+          _61 = _1;
           _62 = const 0_i32 as u32 (IntToInt);
 -         _63 = Lt(move _62, const 64_u32);
 -         assert(move _63, "attempt to shift right by `{}`, which would overflow", const 0_i32) -> [success: bb28, unwind unreachable];
@@ -354,8 +359,8 @@
   
       bb28: {
 -         _60 = Shr(move _61, const 0_i32);
--         StorageDead(_61);
 +         _60 = Shr(_1, const 0_i32);
+          StorageDead(_61);
           _59 = opaque::<u64>(move _60) -> [return: bb29, unwind unreachable];
       }
   
@@ -364,18 +369,20 @@
           StorageDead(_59);
           StorageLive(_64);
           StorageLive(_65);
--         StorageLive(_66);
--         _66 = _1;
+          StorageLive(_66);
+          _66 = _1;
 -         _67 = const 0_i32 as u32 (IntToInt);
 -         _68 = Lt(move _67, const 64_u32);
 -         assert(move _68, "attempt to shift left by `{}`, which would overflow", const 0_i32) -> [success: bb30, unwind unreachable];
++         _67 = _62;
++         _68 = _63;
 +         assert(_63, "attempt to shift left by `{}`, which would overflow", const 0_i32) -> [success: bb30, unwind unreachable];
       }
   
       bb30: {
 -         _65 = Shl(move _66, const 0_i32);
--         StorageDead(_66);
 +         _65 = Shl(_1, const 0_i32);
+          StorageDead(_66);
           _64 = opaque::<u64>(move _65) -> [return: bb31, unwind unreachable];
       }
   
diff --git a/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-unwind.diff b/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-unwind.diff
index 7813c29b962d..f58a9116b8fe 100644
--- a/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-unwind.diff
@@ -75,8 +75,8 @@
       bb0: {
           StorageLive(_2);
           StorageLive(_3);
--         StorageLive(_4);
--         _4 = _1;
+          StorageLive(_4);
+          _4 = _1;
 -         _5 = CheckedAdd(_4, const 0_u64);
 -         assert(!move (_5.1: bool), "attempt to compute `{} + {}`, which would overflow", move _4, const 0_u64) -> [success: bb1, unwind continue];
 +         _5 = CheckedAdd(_1, const 0_u64);
@@ -85,7 +85,7 @@
   
       bb1: {
           _3 = move (_5.0: u64);
--         StorageDead(_4);
+          StorageDead(_4);
           _2 = opaque::<u64>(move _3) -> [return: bb2, unwind continue];
       }
   
@@ -94,8 +94,8 @@
           StorageDead(_2);
           StorageLive(_6);
           StorageLive(_7);
--         StorageLive(_8);
--         _8 = _1;
+          StorageLive(_8);
+          _8 = _1;
 -         _9 = CheckedSub(_8, const 0_u64);
 -         assert(!move (_9.1: bool), "attempt to compute `{} - {}`, which would overflow", move _8, const 0_u64) -> [success: bb3, unwind continue];
 +         _9 = CheckedSub(_1, const 0_u64);
@@ -104,7 +104,7 @@
   
       bb3: {
           _7 = move (_9.0: u64);
--         StorageDead(_8);
+          StorageDead(_8);
           _6 = opaque::<u64>(move _7) -> [return: bb4, unwind continue];
       }
   
@@ -113,8 +113,8 @@
           StorageDead(_6);
           StorageLive(_10);
           StorageLive(_11);
--         StorageLive(_12);
--         _12 = _1;
+          StorageLive(_12);
+          _12 = _1;
 -         _13 = CheckedMul(_12, const 0_u64);
 -         assert(!move (_13.1: bool), "attempt to compute `{} * {}`, which would overflow", move _12, const 0_u64) -> [success: bb5, unwind continue];
 +         _13 = CheckedMul(_1, const 0_u64);
@@ -123,7 +123,7 @@
   
       bb5: {
           _11 = move (_13.0: u64);
--         StorageDead(_12);
+          StorageDead(_12);
           _10 = opaque::<u64>(move _11) -> [return: bb6, unwind continue];
       }
   
@@ -132,8 +132,8 @@
           StorageDead(_10);
           StorageLive(_14);
           StorageLive(_15);
--         StorageLive(_16);
--         _16 = _1;
+          StorageLive(_16);
+          _16 = _1;
 -         _17 = CheckedMul(_16, const 1_u64);
 -         assert(!move (_17.1: bool), "attempt to compute `{} * {}`, which would overflow", move _16, const 1_u64) -> [success: bb7, unwind continue];
 +         _17 = CheckedMul(_1, const 1_u64);
@@ -142,7 +142,7 @@
   
       bb7: {
           _15 = move (_17.0: u64);
--         StorageDead(_16);
+          StorageDead(_16);
           _14 = opaque::<u64>(move _15) -> [return: bb8, unwind continue];
       }
   
@@ -151,8 +151,8 @@
           StorageDead(_14);
           StorageLive(_18);
           StorageLive(_19);
--         StorageLive(_20);
--         _20 = _1;
+          StorageLive(_20);
+          _20 = _1;
           _21 = Eq(const 0_u64, const 0_u64);
 -         assert(!move _21, "attempt to divide `{}` by zero", _20) -> [success: bb9, unwind continue];
 +         assert(!_21, "attempt to divide `{}` by zero", _1) -> [success: bb9, unwind continue];
@@ -160,8 +160,8 @@
   
       bb9: {
 -         _19 = Div(move _20, const 0_u64);
--         StorageDead(_20);
 +         _19 = Div(_1, const 0_u64);
+          StorageDead(_20);
           _18 = opaque::<u64>(move _19) -> [return: bb10, unwind continue];
       }
   
@@ -170,8 +170,8 @@
           StorageDead(_18);
           StorageLive(_22);
           StorageLive(_23);
--         StorageLive(_24);
--         _24 = _1;
+          StorageLive(_24);
+          _24 = _1;
           _25 = Eq(const 1_u64, const 0_u64);
 -         assert(!move _25, "attempt to divide `{}` by zero", _24) -> [success: bb11, unwind continue];
 +         assert(!_25, "attempt to divide `{}` by zero", _1) -> [success: bb11, unwind continue];
@@ -179,8 +179,8 @@
   
       bb11: {
 -         _23 = Div(move _24, const 1_u64);
--         StorageDead(_24);
 +         _23 = Div(_1, const 1_u64);
+          StorageDead(_24);
           _22 = opaque::<u64>(move _23) -> [return: bb12, unwind continue];
       }
   
@@ -189,8 +189,8 @@
           StorageDead(_22);
           StorageLive(_26);
           StorageLive(_27);
--         StorageLive(_28);
--         _28 = _1;
+          StorageLive(_28);
+          _28 = _1;
 -         _29 = Eq(_28, const 0_u64);
 -         assert(!move _29, "attempt to divide `{}` by zero", const 0_u64) -> [success: bb13, unwind continue];
 +         _29 = Eq(_1, const 0_u64);
@@ -199,8 +199,8 @@
   
       bb13: {
 -         _27 = Div(const 0_u64, move _28);
--         StorageDead(_28);
 +         _27 = Div(const 0_u64, _1);
+          StorageDead(_28);
           _26 = opaque::<u64>(move _27) -> [return: bb14, unwind continue];
       }
   
@@ -209,17 +209,18 @@
           StorageDead(_26);
           StorageLive(_30);
           StorageLive(_31);
--         StorageLive(_32);
--         _32 = _1;
+          StorageLive(_32);
+          _32 = _1;
 -         _33 = Eq(_32, const 0_u64);
 -         assert(!move _33, "attempt to divide `{}` by zero", const 1_u64) -> [success: bb15, unwind continue];
++         _33 = _29;
 +         assert(!_29, "attempt to divide `{}` by zero", const 1_u64) -> [success: bb15, unwind continue];
       }
   
       bb15: {
 -         _31 = Div(const 1_u64, move _32);
--         StorageDead(_32);
 +         _31 = Div(const 1_u64, _1);
+          StorageDead(_32);
           _30 = opaque::<u64>(move _31) -> [return: bb16, unwind continue];
       }
   
@@ -228,17 +229,18 @@
           StorageDead(_30);
           StorageLive(_34);
           StorageLive(_35);
--         StorageLive(_36);
--         _36 = _1;
+          StorageLive(_36);
+          _36 = _1;
 -         _37 = Eq(const 0_u64, const 0_u64);
 -         assert(!move _37, "attempt to calculate the remainder of `{}` with a divisor of zero", _36) -> [success: bb17, unwind continue];
++         _37 = _21;
 +         assert(!_21, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb17, unwind continue];
       }
   
       bb17: {
 -         _35 = Rem(move _36, const 0_u64);
--         StorageDead(_36);
 +         _35 = Rem(_1, const 0_u64);
+          StorageDead(_36);
           _34 = opaque::<u64>(move _35) -> [return: bb18, unwind continue];
       }
   
@@ -247,17 +249,18 @@
           StorageDead(_34);
           StorageLive(_38);
           StorageLive(_39);
--         StorageLive(_40);
--         _40 = _1;
+          StorageLive(_40);
+          _40 = _1;
 -         _41 = Eq(const 1_u64, const 0_u64);
 -         assert(!move _41, "attempt to calculate the remainder of `{}` with a divisor of zero", _40) -> [success: bb19, unwind continue];
++         _41 = _25;
 +         assert(!_25, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb19, unwind continue];
       }
   
       bb19: {
 -         _39 = Rem(move _40, const 1_u64);
--         StorageDead(_40);
 +         _39 = Rem(_1, const 1_u64);
+          StorageDead(_40);
           _38 = opaque::<u64>(move _39) -> [return: bb20, unwind continue];
       }
   
@@ -266,17 +269,18 @@
           StorageDead(_38);
           StorageLive(_42);
           StorageLive(_43);
--         StorageLive(_44);
--         _44 = _1;
+          StorageLive(_44);
+          _44 = _1;
 -         _45 = Eq(_44, const 0_u64);
 -         assert(!move _45, "attempt to calculate the remainder of `{}` with a divisor of zero", const 0_u64) -> [success: bb21, unwind continue];
++         _45 = _29;
 +         assert(!_29, "attempt to calculate the remainder of `{}` with a divisor of zero", const 0_u64) -> [success: bb21, unwind continue];
       }
   
       bb21: {
 -         _43 = Rem(const 0_u64, move _44);
--         StorageDead(_44);
 +         _43 = Rem(const 0_u64, _1);
+          StorageDead(_44);
           _42 = opaque::<u64>(move _43) -> [return: bb22, unwind continue];
       }
   
@@ -285,17 +289,18 @@
           StorageDead(_42);
           StorageLive(_46);
           StorageLive(_47);
--         StorageLive(_48);
--         _48 = _1;
+          StorageLive(_48);
+          _48 = _1;
 -         _49 = Eq(_48, const 0_u64);
 -         assert(!move _49, "attempt to calculate the remainder of `{}` with a divisor of zero", const 1_u64) -> [success: bb23, unwind continue];
++         _49 = _29;
 +         assert(!_29, "attempt to calculate the remainder of `{}` with a divisor of zero", const 1_u64) -> [success: bb23, unwind continue];
       }
   
       bb23: {
 -         _47 = Rem(const 1_u64, move _48);
--         StorageDead(_48);
 +         _47 = Rem(const 1_u64, _1);
+          StorageDead(_48);
           _46 = opaque::<u64>(move _47) -> [return: bb24, unwind continue];
       }
   
@@ -304,11 +309,11 @@
           StorageDead(_46);
           StorageLive(_50);
           StorageLive(_51);
--         StorageLive(_52);
--         _52 = _1;
+          StorageLive(_52);
+          _52 = _1;
 -         _51 = BitAnd(move _52, const 0_u64);
--         StorageDead(_52);
 +         _51 = BitAnd(_1, const 0_u64);
+          StorageDead(_52);
           _50 = opaque::<u64>(move _51) -> [return: bb25, unwind continue];
       }
   
@@ -317,11 +322,11 @@
           StorageDead(_50);
           StorageLive(_53);
           StorageLive(_54);
--         StorageLive(_55);
--         _55 = _1;
+          StorageLive(_55);
+          _55 = _1;
 -         _54 = BitOr(move _55, const 0_u64);
--         StorageDead(_55);
 +         _54 = BitOr(_1, const 0_u64);
+          StorageDead(_55);
           _53 = opaque::<u64>(move _54) -> [return: bb26, unwind continue];
       }
   
@@ -330,11 +335,11 @@
           StorageDead(_53);
           StorageLive(_56);
           StorageLive(_57);
--         StorageLive(_58);
--         _58 = _1;
+          StorageLive(_58);
+          _58 = _1;
 -         _57 = BitXor(move _58, const 0_u64);
--         StorageDead(_58);
 +         _57 = BitXor(_1, const 0_u64);
+          StorageDead(_58);
           _56 = opaque::<u64>(move _57) -> [return: bb27, unwind continue];
       }
   
@@ -343,8 +348,8 @@
           StorageDead(_56);
           StorageLive(_59);
           StorageLive(_60);
--         StorageLive(_61);
--         _61 = _1;
+          StorageLive(_61);
+          _61 = _1;
           _62 = const 0_i32 as u32 (IntToInt);
 -         _63 = Lt(move _62, const 64_u32);
 -         assert(move _63, "attempt to shift right by `{}`, which would overflow", const 0_i32) -> [success: bb28, unwind continue];
@@ -354,8 +359,8 @@
   
       bb28: {
 -         _60 = Shr(move _61, const 0_i32);
--         StorageDead(_61);
 +         _60 = Shr(_1, const 0_i32);
+          StorageDead(_61);
           _59 = opaque::<u64>(move _60) -> [return: bb29, unwind continue];
       }
   
@@ -364,18 +369,20 @@
           StorageDead(_59);
           StorageLive(_64);
           StorageLive(_65);
--         StorageLive(_66);
--         _66 = _1;
+          StorageLive(_66);
+          _66 = _1;
 -         _67 = const 0_i32 as u32 (IntToInt);
 -         _68 = Lt(move _67, const 64_u32);
 -         assert(move _68, "attempt to shift left by `{}`, which would overflow", const 0_i32) -> [success: bb30, unwind continue];
++         _67 = _62;
++         _68 = _63;
 +         assert(_63, "attempt to shift left by `{}`, which would overflow", const 0_i32) -> [success: bb30, unwind continue];
       }
   
       bb30: {
 -         _65 = Shl(move _66, const 0_i32);
--         StorageDead(_66);
 +         _65 = Shl(_1, const 0_i32);
+          StorageDead(_66);
           _64 = opaque::<u64>(move _65) -> [return: bb31, unwind continue];
       }
   
diff --git a/tests/mir-opt/gvn.arithmetic_float.GVN.panic-abort.diff b/tests/mir-opt/gvn.arithmetic_float.GVN.panic-abort.diff
index 7d5ac8353fe4..b332100eaf03 100644
--- a/tests/mir-opt/gvn.arithmetic_float.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.arithmetic_float.GVN.panic-abort.diff
@@ -37,11 +37,11 @@
       bb0: {
           StorageLive(_2);
           StorageLive(_3);
--         StorageLive(_4);
--         _4 = _1;
+          StorageLive(_4);
+          _4 = _1;
 -         _3 = Add(move _4, const 0f64);
--         StorageDead(_4);
 +         _3 = Add(_1, const 0f64);
+          StorageDead(_4);
           _2 = opaque::<f64>(move _3) -> [return: bb1, unwind unreachable];
       }
   
@@ -50,11 +50,11 @@
           StorageDead(_2);
           StorageLive(_5);
           StorageLive(_6);
--         StorageLive(_7);
--         _7 = _1;
+          StorageLive(_7);
+          _7 = _1;
 -         _6 = Sub(move _7, const 0f64);
--         StorageDead(_7);
 +         _6 = Sub(_1, const 0f64);
+          StorageDead(_7);
           _5 = opaque::<f64>(move _6) -> [return: bb2, unwind unreachable];
       }
   
@@ -63,11 +63,11 @@
           StorageDead(_5);
           StorageLive(_8);
           StorageLive(_9);
--         StorageLive(_10);
--         _10 = _1;
+          StorageLive(_10);
+          _10 = _1;
 -         _9 = Mul(move _10, const 0f64);
--         StorageDead(_10);
 +         _9 = Mul(_1, const 0f64);
+          StorageDead(_10);
           _8 = opaque::<f64>(move _9) -> [return: bb3, unwind unreachable];
       }
   
@@ -76,11 +76,11 @@
           StorageDead(_8);
           StorageLive(_11);
           StorageLive(_12);
--         StorageLive(_13);
--         _13 = _1;
+          StorageLive(_13);
+          _13 = _1;
 -         _12 = Div(move _13, const 0f64);
--         StorageDead(_13);
 +         _12 = Div(_1, const 0f64);
+          StorageDead(_13);
           _11 = opaque::<f64>(move _12) -> [return: bb4, unwind unreachable];
       }
   
@@ -89,11 +89,11 @@
           StorageDead(_11);
           StorageLive(_14);
           StorageLive(_15);
--         StorageLive(_16);
--         _16 = _1;
+          StorageLive(_16);
+          _16 = _1;
 -         _15 = Div(const 0f64, move _16);
--         StorageDead(_16);
 +         _15 = Div(const 0f64, _1);
+          StorageDead(_16);
           _14 = opaque::<f64>(move _15) -> [return: bb5, unwind unreachable];
       }
   
@@ -102,11 +102,11 @@
           StorageDead(_14);
           StorageLive(_17);
           StorageLive(_18);
--         StorageLive(_19);
--         _19 = _1;
+          StorageLive(_19);
+          _19 = _1;
 -         _18 = Rem(move _19, const 0f64);
--         StorageDead(_19);
 +         _18 = Rem(_1, const 0f64);
+          StorageDead(_19);
           _17 = opaque::<f64>(move _18) -> [return: bb6, unwind unreachable];
       }
   
@@ -115,11 +115,11 @@
           StorageDead(_17);
           StorageLive(_20);
           StorageLive(_21);
--         StorageLive(_22);
--         _22 = _1;
+          StorageLive(_22);
+          _22 = _1;
 -         _21 = Rem(const 0f64, move _22);
--         StorageDead(_22);
 +         _21 = Rem(const 0f64, _1);
+          StorageDead(_22);
           _20 = opaque::<f64>(move _21) -> [return: bb7, unwind unreachable];
       }
   
@@ -128,14 +128,14 @@
           StorageDead(_20);
           StorageLive(_23);
           StorageLive(_24);
--         StorageLive(_25);
--         _25 = _1;
--         StorageLive(_26);
--         _26 = _1;
+          StorageLive(_25);
+          _25 = _1;
+          StorageLive(_26);
+          _26 = _1;
 -         _24 = Eq(move _25, move _26);
--         StorageDead(_26);
--         StorageDead(_25);
 +         _24 = Eq(_1, _1);
+          StorageDead(_26);
+          StorageDead(_25);
           _23 = opaque::<bool>(move _24) -> [return: bb8, unwind unreachable];
       }
   
@@ -144,14 +144,14 @@
           StorageDead(_23);
           StorageLive(_27);
           StorageLive(_28);
--         StorageLive(_29);
--         _29 = _1;
--         StorageLive(_30);
--         _30 = _1;
+          StorageLive(_29);
+          _29 = _1;
+          StorageLive(_30);
+          _30 = _1;
 -         _28 = Ne(move _29, move _30);
--         StorageDead(_30);
--         StorageDead(_29);
 +         _28 = Ne(_1, _1);
+          StorageDead(_30);
+          StorageDead(_29);
           _27 = opaque::<bool>(move _28) -> [return: bb9, unwind unreachable];
       }
   
diff --git a/tests/mir-opt/gvn.arithmetic_float.GVN.panic-unwind.diff b/tests/mir-opt/gvn.arithmetic_float.GVN.panic-unwind.diff
index 36c26dc66051..28664cb0ac8c 100644
--- a/tests/mir-opt/gvn.arithmetic_float.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.arithmetic_float.GVN.panic-unwind.diff
@@ -37,11 +37,11 @@
       bb0: {
           StorageLive(_2);
           StorageLive(_3);
--         StorageLive(_4);
--         _4 = _1;
+          StorageLive(_4);
+          _4 = _1;
 -         _3 = Add(move _4, const 0f64);
--         StorageDead(_4);
 +         _3 = Add(_1, const 0f64);
+          StorageDead(_4);
           _2 = opaque::<f64>(move _3) -> [return: bb1, unwind continue];
       }
   
@@ -50,11 +50,11 @@
           StorageDead(_2);
           StorageLive(_5);
           StorageLive(_6);
--         StorageLive(_7);
--         _7 = _1;
+          StorageLive(_7);
+          _7 = _1;
 -         _6 = Sub(move _7, const 0f64);
--         StorageDead(_7);
 +         _6 = Sub(_1, const 0f64);
+          StorageDead(_7);
           _5 = opaque::<f64>(move _6) -> [return: bb2, unwind continue];
       }
   
@@ -63,11 +63,11 @@
           StorageDead(_5);
           StorageLive(_8);
           StorageLive(_9);
--         StorageLive(_10);
--         _10 = _1;
+          StorageLive(_10);
+          _10 = _1;
 -         _9 = Mul(move _10, const 0f64);
--         StorageDead(_10);
 +         _9 = Mul(_1, const 0f64);
+          StorageDead(_10);
           _8 = opaque::<f64>(move _9) -> [return: bb3, unwind continue];
       }
   
@@ -76,11 +76,11 @@
           StorageDead(_8);
           StorageLive(_11);
           StorageLive(_12);
--         StorageLive(_13);
--         _13 = _1;
+          StorageLive(_13);
+          _13 = _1;
 -         _12 = Div(move _13, const 0f64);
--         StorageDead(_13);
 +         _12 = Div(_1, const 0f64);
+          StorageDead(_13);
           _11 = opaque::<f64>(move _12) -> [return: bb4, unwind continue];
       }
   
@@ -89,11 +89,11 @@
           StorageDead(_11);
           StorageLive(_14);
           StorageLive(_15);
--         StorageLive(_16);
--         _16 = _1;
+          StorageLive(_16);
+          _16 = _1;
 -         _15 = Div(const 0f64, move _16);
--         StorageDead(_16);
 +         _15 = Div(const 0f64, _1);
+          StorageDead(_16);
           _14 = opaque::<f64>(move _15) -> [return: bb5, unwind continue];
       }
   
@@ -102,11 +102,11 @@
           StorageDead(_14);
           StorageLive(_17);
           StorageLive(_18);
--         StorageLive(_19);
--         _19 = _1;
+          StorageLive(_19);
+          _19 = _1;
 -         _18 = Rem(move _19, const 0f64);
--         StorageDead(_19);
 +         _18 = Rem(_1, const 0f64);
+          StorageDead(_19);
           _17 = opaque::<f64>(move _18) -> [return: bb6, unwind continue];
       }
   
@@ -115,11 +115,11 @@
           StorageDead(_17);
           StorageLive(_20);
           StorageLive(_21);
--         StorageLive(_22);
--         _22 = _1;
+          StorageLive(_22);
+          _22 = _1;
 -         _21 = Rem(const 0f64, move _22);
--         StorageDead(_22);
 +         _21 = Rem(const 0f64, _1);
+          StorageDead(_22);
           _20 = opaque::<f64>(move _21) -> [return: bb7, unwind continue];
       }
   
@@ -128,14 +128,14 @@
           StorageDead(_20);
           StorageLive(_23);
           StorageLive(_24);
--         StorageLive(_25);
--         _25 = _1;
--         StorageLive(_26);
--         _26 = _1;
+          StorageLive(_25);
+          _25 = _1;
+          StorageLive(_26);
+          _26 = _1;
 -         _24 = Eq(move _25, move _26);
--         StorageDead(_26);
--         StorageDead(_25);
 +         _24 = Eq(_1, _1);
+          StorageDead(_26);
+          StorageDead(_25);
           _23 = opaque::<bool>(move _24) -> [return: bb8, unwind continue];
       }
   
@@ -144,14 +144,14 @@
           StorageDead(_23);
           StorageLive(_27);
           StorageLive(_28);
--         StorageLive(_29);
--         _29 = _1;
--         StorageLive(_30);
--         _30 = _1;
+          StorageLive(_29);
+          _29 = _1;
+          StorageLive(_30);
+          _30 = _1;
 -         _28 = Ne(move _29, move _30);
--         StorageDead(_30);
--         StorageDead(_29);
 +         _28 = Ne(_1, _1);
+          StorageDead(_30);
+          StorageDead(_29);
           _27 = opaque::<bool>(move _28) -> [return: bb9, unwind continue];
       }
   
diff --git a/tests/mir-opt/gvn.cast.GVN.panic-abort.diff b/tests/mir-opt/gvn.cast.GVN.panic-abort.diff
index 513fe60b65d9..37b0d92931f6 100644
--- a/tests/mir-opt/gvn.cast.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.cast.GVN.panic-abort.diff
@@ -105,18 +105,22 @@
   
       bb0: {
 -         StorageLive(_1);
++         nop;
           _1 = const 1_i64;
 -         StorageLive(_2);
++         nop;
           _2 = const 1_u64;
 -         StorageLive(_3);
++         nop;
           _3 = const 1f64;
           StorageLive(_4);
           StorageLive(_5);
--         StorageLive(_6);
+          StorageLive(_6);
 -         _6 = _1;
 -         _5 = move _6 as u8 (IntToInt);
--         StorageDead(_6);
++         _6 = const 1_i64;
 +         _5 = const 1_i64 as u8 (IntToInt);
+          StorageDead(_6);
           _4 = opaque::<u8>(move _5) -> [return: bb1, unwind unreachable];
       }
   
@@ -125,11 +129,12 @@
           StorageDead(_4);
           StorageLive(_7);
           StorageLive(_8);
--         StorageLive(_9);
+          StorageLive(_9);
 -         _9 = _1;
 -         _8 = move _9 as u16 (IntToInt);
--         StorageDead(_9);
++         _9 = const 1_i64;
 +         _8 = const 1_i64 as u16 (IntToInt);
+          StorageDead(_9);
           _7 = opaque::<u16>(move _8) -> [return: bb2, unwind unreachable];
       }
   
@@ -138,11 +143,12 @@
           StorageDead(_7);
           StorageLive(_10);
           StorageLive(_11);
--         StorageLive(_12);
+          StorageLive(_12);
 -         _12 = _1;
 -         _11 = move _12 as u32 (IntToInt);
--         StorageDead(_12);
++         _12 = const 1_i64;
 +         _11 = const 1_i64 as u32 (IntToInt);
+          StorageDead(_12);
           _10 = opaque::<u32>(move _11) -> [return: bb3, unwind unreachable];
       }
   
@@ -151,11 +157,12 @@
           StorageDead(_10);
           StorageLive(_13);
           StorageLive(_14);
--         StorageLive(_15);
+          StorageLive(_15);
 -         _15 = _1;
 -         _14 = move _15 as u64 (IntToInt);
--         StorageDead(_15);
++         _15 = const 1_i64;
 +         _14 = const 1_i64 as u64 (IntToInt);
+          StorageDead(_15);
           _13 = opaque::<u64>(move _14) -> [return: bb4, unwind unreachable];
       }
   
@@ -164,11 +171,12 @@
           StorageDead(_13);
           StorageLive(_16);
           StorageLive(_17);
--         StorageLive(_18);
+          StorageLive(_18);
 -         _18 = _1;
 -         _17 = move _18 as i8 (IntToInt);
--         StorageDead(_18);
++         _18 = const 1_i64;
 +         _17 = const 1_i64 as i8 (IntToInt);
+          StorageDead(_18);
           _16 = opaque::<i8>(move _17) -> [return: bb5, unwind unreachable];
       }
   
@@ -177,11 +185,12 @@
           StorageDead(_16);
           StorageLive(_19);
           StorageLive(_20);
--         StorageLive(_21);
+          StorageLive(_21);
 -         _21 = _1;
 -         _20 = move _21 as i16 (IntToInt);
--         StorageDead(_21);
++         _21 = const 1_i64;
 +         _20 = const 1_i64 as i16 (IntToInt);
+          StorageDead(_21);
           _19 = opaque::<i16>(move _20) -> [return: bb6, unwind unreachable];
       }
   
@@ -190,11 +199,12 @@
           StorageDead(_19);
           StorageLive(_22);
           StorageLive(_23);
--         StorageLive(_24);
+          StorageLive(_24);
 -         _24 = _1;
 -         _23 = move _24 as i32 (IntToInt);
--         StorageDead(_24);
++         _24 = const 1_i64;
 +         _23 = const 1_i64 as i32 (IntToInt);
+          StorageDead(_24);
           _22 = opaque::<i32>(move _23) -> [return: bb7, unwind unreachable];
       }
   
@@ -202,22 +212,24 @@
           StorageDead(_23);
           StorageDead(_22);
           StorageLive(_25);
--         StorageLive(_26);
+          StorageLive(_26);
 -         _26 = _1;
 -         _25 = opaque::<i64>(move _26) -> [return: bb8, unwind unreachable];
++         _26 = const 1_i64;
 +         _25 = opaque::<i64>(const 1_i64) -> [return: bb8, unwind unreachable];
       }
   
       bb8: {
--         StorageDead(_26);
+          StorageDead(_26);
           StorageDead(_25);
           StorageLive(_27);
           StorageLive(_28);
--         StorageLive(_29);
+          StorageLive(_29);
 -         _29 = _1;
 -         _28 = move _29 as f32 (IntToFloat);
--         StorageDead(_29);
++         _29 = const 1_i64;
 +         _28 = const 1_i64 as f32 (IntToFloat);
+          StorageDead(_29);
           _27 = opaque::<f32>(move _28) -> [return: bb9, unwind unreachable];
       }
   
@@ -226,11 +238,12 @@
           StorageDead(_27);
           StorageLive(_30);
           StorageLive(_31);
--         StorageLive(_32);
+          StorageLive(_32);
 -         _32 = _1;
 -         _31 = move _32 as f64 (IntToFloat);
--         StorageDead(_32);
++         _32 = const 1_i64;
 +         _31 = const 1_i64 as f64 (IntToFloat);
+          StorageDead(_32);
           _30 = opaque::<f64>(move _31) -> [return: bb10, unwind unreachable];
       }
   
@@ -239,11 +252,12 @@
           StorageDead(_30);
           StorageLive(_33);
           StorageLive(_34);
--         StorageLive(_35);
+          StorageLive(_35);
 -         _35 = _2;
 -         _34 = move _35 as u8 (IntToInt);
--         StorageDead(_35);
++         _35 = const 1_u64;
 +         _34 = const 1_u64 as u8 (IntToInt);
+          StorageDead(_35);
           _33 = opaque::<u8>(move _34) -> [return: bb11, unwind unreachable];
       }
   
@@ -252,11 +266,12 @@
           StorageDead(_33);
           StorageLive(_36);
           StorageLive(_37);
--         StorageLive(_38);
+          StorageLive(_38);
 -         _38 = _2;
 -         _37 = move _38 as u16 (IntToInt);
--         StorageDead(_38);
++         _38 = const 1_u64;
 +         _37 = const 1_u64 as u16 (IntToInt);
+          StorageDead(_38);
           _36 = opaque::<u16>(move _37) -> [return: bb12, unwind unreachable];
       }
   
@@ -265,11 +280,12 @@
           StorageDead(_36);
           StorageLive(_39);
           StorageLive(_40);
--         StorageLive(_41);
+          StorageLive(_41);
 -         _41 = _2;
 -         _40 = move _41 as u32 (IntToInt);
--         StorageDead(_41);
++         _41 = const 1_u64;
 +         _40 = const 1_u64 as u32 (IntToInt);
+          StorageDead(_41);
           _39 = opaque::<u32>(move _40) -> [return: bb13, unwind unreachable];
       }
   
@@ -277,22 +293,24 @@
           StorageDead(_40);
           StorageDead(_39);
           StorageLive(_42);
--         StorageLive(_43);
+          StorageLive(_43);
 -         _43 = _2;
 -         _42 = opaque::<u64>(move _43) -> [return: bb14, unwind unreachable];
++         _43 = const 1_u64;
 +         _42 = opaque::<u64>(const 1_u64) -> [return: bb14, unwind unreachable];
       }
   
       bb14: {
--         StorageDead(_43);
+          StorageDead(_43);
           StorageDead(_42);
           StorageLive(_44);
           StorageLive(_45);
--         StorageLive(_46);
+          StorageLive(_46);
 -         _46 = _2;
 -         _45 = move _46 as i8 (IntToInt);
--         StorageDead(_46);
++         _46 = const 1_u64;
 +         _45 = const 1_u64 as i8 (IntToInt);
+          StorageDead(_46);
           _44 = opaque::<i8>(move _45) -> [return: bb15, unwind unreachable];
       }
   
@@ -301,11 +319,12 @@
           StorageDead(_44);
           StorageLive(_47);
           StorageLive(_48);
--         StorageLive(_49);
+          StorageLive(_49);
 -         _49 = _2;
 -         _48 = move _49 as i16 (IntToInt);
--         StorageDead(_49);
++         _49 = const 1_u64;
 +         _48 = const 1_u64 as i16 (IntToInt);
+          StorageDead(_49);
           _47 = opaque::<i16>(move _48) -> [return: bb16, unwind unreachable];
       }
   
@@ -314,11 +333,12 @@
           StorageDead(_47);
           StorageLive(_50);
           StorageLive(_51);
--         StorageLive(_52);
+          StorageLive(_52);
 -         _52 = _2;
 -         _51 = move _52 as i32 (IntToInt);
--         StorageDead(_52);
++         _52 = const 1_u64;
 +         _51 = const 1_u64 as i32 (IntToInt);
+          StorageDead(_52);
           _50 = opaque::<i32>(move _51) -> [return: bb17, unwind unreachable];
       }
   
@@ -327,11 +347,12 @@
           StorageDead(_50);
           StorageLive(_53);
           StorageLive(_54);
--         StorageLive(_55);
+          StorageLive(_55);
 -         _55 = _2;
 -         _54 = move _55 as i64 (IntToInt);
--         StorageDead(_55);
++         _55 = const 1_u64;
 +         _54 = const 1_u64 as i64 (IntToInt);
+          StorageDead(_55);
           _53 = opaque::<i64>(move _54) -> [return: bb18, unwind unreachable];
       }
   
@@ -340,11 +361,12 @@
           StorageDead(_53);
           StorageLive(_56);
           StorageLive(_57);
--         StorageLive(_58);
+          StorageLive(_58);
 -         _58 = _2;
 -         _57 = move _58 as f32 (IntToFloat);
--         StorageDead(_58);
++         _58 = const 1_u64;
 +         _57 = const 1_u64 as f32 (IntToFloat);
+          StorageDead(_58);
           _56 = opaque::<f32>(move _57) -> [return: bb19, unwind unreachable];
       }
   
@@ -353,11 +375,12 @@
           StorageDead(_56);
           StorageLive(_59);
           StorageLive(_60);
--         StorageLive(_61);
+          StorageLive(_61);
 -         _61 = _2;
 -         _60 = move _61 as f64 (IntToFloat);
--         StorageDead(_61);
++         _61 = const 1_u64;
 +         _60 = const 1_u64 as f64 (IntToFloat);
+          StorageDead(_61);
           _59 = opaque::<f64>(move _60) -> [return: bb20, unwind unreachable];
       }
   
@@ -366,11 +389,12 @@
           StorageDead(_59);
           StorageLive(_62);
           StorageLive(_63);
--         StorageLive(_64);
+          StorageLive(_64);
 -         _64 = _3;
 -         _63 = move _64 as u8 (FloatToInt);
--         StorageDead(_64);
++         _64 = const 1f64;
 +         _63 = const 1f64 as u8 (FloatToInt);
+          StorageDead(_64);
           _62 = opaque::<u8>(move _63) -> [return: bb21, unwind unreachable];
       }
   
@@ -379,11 +403,12 @@
           StorageDead(_62);
           StorageLive(_65);
           StorageLive(_66);
--         StorageLive(_67);
+          StorageLive(_67);
 -         _67 = _3;
 -         _66 = move _67 as u16 (FloatToInt);
--         StorageDead(_67);
++         _67 = const 1f64;
 +         _66 = const 1f64 as u16 (FloatToInt);
+          StorageDead(_67);
           _65 = opaque::<u16>(move _66) -> [return: bb22, unwind unreachable];
       }
   
@@ -392,11 +417,12 @@
           StorageDead(_65);
           StorageLive(_68);
           StorageLive(_69);
--         StorageLive(_70);
+          StorageLive(_70);
 -         _70 = _3;
 -         _69 = move _70 as u32 (FloatToInt);
--         StorageDead(_70);
++         _70 = const 1f64;
 +         _69 = const 1f64 as u32 (FloatToInt);
+          StorageDead(_70);
           _68 = opaque::<u32>(move _69) -> [return: bb23, unwind unreachable];
       }
   
@@ -405,11 +431,12 @@
           StorageDead(_68);
           StorageLive(_71);
           StorageLive(_72);
--         StorageLive(_73);
+          StorageLive(_73);
 -         _73 = _3;
 -         _72 = move _73 as u64 (FloatToInt);
--         StorageDead(_73);
++         _73 = const 1f64;
 +         _72 = const 1f64 as u64 (FloatToInt);
+          StorageDead(_73);
           _71 = opaque::<u64>(move _72) -> [return: bb24, unwind unreachable];
       }
   
@@ -418,11 +445,12 @@
           StorageDead(_71);
           StorageLive(_74);
           StorageLive(_75);
--         StorageLive(_76);
+          StorageLive(_76);
 -         _76 = _3;
 -         _75 = move _76 as i8 (FloatToInt);
--         StorageDead(_76);
++         _76 = const 1f64;
 +         _75 = const 1f64 as i8 (FloatToInt);
+          StorageDead(_76);
           _74 = opaque::<i8>(move _75) -> [return: bb25, unwind unreachable];
       }
   
@@ -431,11 +459,12 @@
           StorageDead(_74);
           StorageLive(_77);
           StorageLive(_78);
--         StorageLive(_79);
+          StorageLive(_79);
 -         _79 = _3;
 -         _78 = move _79 as i16 (FloatToInt);
--         StorageDead(_79);
++         _79 = const 1f64;
 +         _78 = const 1f64 as i16 (FloatToInt);
+          StorageDead(_79);
           _77 = opaque::<i16>(move _78) -> [return: bb26, unwind unreachable];
       }
   
@@ -444,11 +473,12 @@
           StorageDead(_77);
           StorageLive(_80);
           StorageLive(_81);
--         StorageLive(_82);
+          StorageLive(_82);
 -         _82 = _3;
 -         _81 = move _82 as i32 (FloatToInt);
--         StorageDead(_82);
++         _82 = const 1f64;
 +         _81 = const 1f64 as i32 (FloatToInt);
+          StorageDead(_82);
           _80 = opaque::<i32>(move _81) -> [return: bb27, unwind unreachable];
       }
   
@@ -457,11 +487,12 @@
           StorageDead(_80);
           StorageLive(_83);
           StorageLive(_84);
--         StorageLive(_85);
+          StorageLive(_85);
 -         _85 = _3;
 -         _84 = move _85 as i64 (FloatToInt);
--         StorageDead(_85);
++         _85 = const 1f64;
 +         _84 = const 1f64 as i64 (FloatToInt);
+          StorageDead(_85);
           _83 = opaque::<i64>(move _84) -> [return: bb28, unwind unreachable];
       }
   
@@ -470,11 +501,12 @@
           StorageDead(_83);
           StorageLive(_86);
           StorageLive(_87);
--         StorageLive(_88);
+          StorageLive(_88);
 -         _88 = _3;
 -         _87 = move _88 as f32 (FloatToFloat);
--         StorageDead(_88);
++         _88 = const 1f64;
 +         _87 = const 1f64 as f32 (FloatToFloat);
+          StorageDead(_88);
           _86 = opaque::<f32>(move _87) -> [return: bb29, unwind unreachable];
       }
   
@@ -482,19 +514,23 @@
           StorageDead(_87);
           StorageDead(_86);
           StorageLive(_89);
--         StorageLive(_90);
+          StorageLive(_90);
 -         _90 = _3;
 -         _89 = opaque::<f64>(move _90) -> [return: bb30, unwind unreachable];
++         _90 = const 1f64;
 +         _89 = opaque::<f64>(const 1f64) -> [return: bb30, unwind unreachable];
       }
   
       bb30: {
--         StorageDead(_90);
+          StorageDead(_90);
           StorageDead(_89);
           _0 = const ();
 -         StorageDead(_3);
 -         StorageDead(_2);
 -         StorageDead(_1);
++         nop;
++         nop;
++         nop;
           return;
       }
   }
diff --git a/tests/mir-opt/gvn.cast.GVN.panic-unwind.diff b/tests/mir-opt/gvn.cast.GVN.panic-unwind.diff
index 33192ed8de03..fbdec455188a 100644
--- a/tests/mir-opt/gvn.cast.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.cast.GVN.panic-unwind.diff
@@ -105,18 +105,22 @@
   
       bb0: {
 -         StorageLive(_1);
++         nop;
           _1 = const 1_i64;
 -         StorageLive(_2);
++         nop;
           _2 = const 1_u64;
 -         StorageLive(_3);
++         nop;
           _3 = const 1f64;
           StorageLive(_4);
           StorageLive(_5);
--         StorageLive(_6);
+          StorageLive(_6);
 -         _6 = _1;
 -         _5 = move _6 as u8 (IntToInt);
--         StorageDead(_6);
++         _6 = const 1_i64;
 +         _5 = const 1_i64 as u8 (IntToInt);
+          StorageDead(_6);
           _4 = opaque::<u8>(move _5) -> [return: bb1, unwind continue];
       }
   
@@ -125,11 +129,12 @@
           StorageDead(_4);
           StorageLive(_7);
           StorageLive(_8);
--         StorageLive(_9);
+          StorageLive(_9);
 -         _9 = _1;
 -         _8 = move _9 as u16 (IntToInt);
--         StorageDead(_9);
++         _9 = const 1_i64;
 +         _8 = const 1_i64 as u16 (IntToInt);
+          StorageDead(_9);
           _7 = opaque::<u16>(move _8) -> [return: bb2, unwind continue];
       }
   
@@ -138,11 +143,12 @@
           StorageDead(_7);
           StorageLive(_10);
           StorageLive(_11);
--         StorageLive(_12);
+          StorageLive(_12);
 -         _12 = _1;
 -         _11 = move _12 as u32 (IntToInt);
--         StorageDead(_12);
++         _12 = const 1_i64;
 +         _11 = const 1_i64 as u32 (IntToInt);
+          StorageDead(_12);
           _10 = opaque::<u32>(move _11) -> [return: bb3, unwind continue];
       }
   
@@ -151,11 +157,12 @@
           StorageDead(_10);
           StorageLive(_13);
           StorageLive(_14);
--         StorageLive(_15);
+          StorageLive(_15);
 -         _15 = _1;
 -         _14 = move _15 as u64 (IntToInt);
--         StorageDead(_15);
++         _15 = const 1_i64;
 +         _14 = const 1_i64 as u64 (IntToInt);
+          StorageDead(_15);
           _13 = opaque::<u64>(move _14) -> [return: bb4, unwind continue];
       }
   
@@ -164,11 +171,12 @@
           StorageDead(_13);
           StorageLive(_16);
           StorageLive(_17);
--         StorageLive(_18);
+          StorageLive(_18);
 -         _18 = _1;
 -         _17 = move _18 as i8 (IntToInt);
--         StorageDead(_18);
++         _18 = const 1_i64;
 +         _17 = const 1_i64 as i8 (IntToInt);
+          StorageDead(_18);
           _16 = opaque::<i8>(move _17) -> [return: bb5, unwind continue];
       }
   
@@ -177,11 +185,12 @@
           StorageDead(_16);
           StorageLive(_19);
           StorageLive(_20);
--         StorageLive(_21);
+          StorageLive(_21);
 -         _21 = _1;
 -         _20 = move _21 as i16 (IntToInt);
--         StorageDead(_21);
++         _21 = const 1_i64;
 +         _20 = const 1_i64 as i16 (IntToInt);
+          StorageDead(_21);
           _19 = opaque::<i16>(move _20) -> [return: bb6, unwind continue];
       }
   
@@ -190,11 +199,12 @@
           StorageDead(_19);
           StorageLive(_22);
           StorageLive(_23);
--         StorageLive(_24);
+          StorageLive(_24);
 -         _24 = _1;
 -         _23 = move _24 as i32 (IntToInt);
--         StorageDead(_24);
++         _24 = const 1_i64;
 +         _23 = const 1_i64 as i32 (IntToInt);
+          StorageDead(_24);
           _22 = opaque::<i32>(move _23) -> [return: bb7, unwind continue];
       }
   
@@ -202,22 +212,24 @@
           StorageDead(_23);
           StorageDead(_22);
           StorageLive(_25);
--         StorageLive(_26);
+          StorageLive(_26);
 -         _26 = _1;
 -         _25 = opaque::<i64>(move _26) -> [return: bb8, unwind continue];
++         _26 = const 1_i64;
 +         _25 = opaque::<i64>(const 1_i64) -> [return: bb8, unwind continue];
       }
   
       bb8: {
--         StorageDead(_26);
+          StorageDead(_26);
           StorageDead(_25);
           StorageLive(_27);
           StorageLive(_28);
--         StorageLive(_29);
+          StorageLive(_29);
 -         _29 = _1;
 -         _28 = move _29 as f32 (IntToFloat);
--         StorageDead(_29);
++         _29 = const 1_i64;
 +         _28 = const 1_i64 as f32 (IntToFloat);
+          StorageDead(_29);
           _27 = opaque::<f32>(move _28) -> [return: bb9, unwind continue];
       }
   
@@ -226,11 +238,12 @@
           StorageDead(_27);
           StorageLive(_30);
           StorageLive(_31);
--         StorageLive(_32);
+          StorageLive(_32);
 -         _32 = _1;
 -         _31 = move _32 as f64 (IntToFloat);
--         StorageDead(_32);
++         _32 = const 1_i64;
 +         _31 = const 1_i64 as f64 (IntToFloat);
+          StorageDead(_32);
           _30 = opaque::<f64>(move _31) -> [return: bb10, unwind continue];
       }
   
@@ -239,11 +252,12 @@
           StorageDead(_30);
           StorageLive(_33);
           StorageLive(_34);
--         StorageLive(_35);
+          StorageLive(_35);
 -         _35 = _2;
 -         _34 = move _35 as u8 (IntToInt);
--         StorageDead(_35);
++         _35 = const 1_u64;
 +         _34 = const 1_u64 as u8 (IntToInt);
+          StorageDead(_35);
           _33 = opaque::<u8>(move _34) -> [return: bb11, unwind continue];
       }
   
@@ -252,11 +266,12 @@
           StorageDead(_33);
           StorageLive(_36);
           StorageLive(_37);
--         StorageLive(_38);
+          StorageLive(_38);
 -         _38 = _2;
 -         _37 = move _38 as u16 (IntToInt);
--         StorageDead(_38);
++         _38 = const 1_u64;
 +         _37 = const 1_u64 as u16 (IntToInt);
+          StorageDead(_38);
           _36 = opaque::<u16>(move _37) -> [return: bb12, unwind continue];
       }
   
@@ -265,11 +280,12 @@
           StorageDead(_36);
           StorageLive(_39);
           StorageLive(_40);
--         StorageLive(_41);
+          StorageLive(_41);
 -         _41 = _2;
 -         _40 = move _41 as u32 (IntToInt);
--         StorageDead(_41);
++         _41 = const 1_u64;
 +         _40 = const 1_u64 as u32 (IntToInt);
+          StorageDead(_41);
           _39 = opaque::<u32>(move _40) -> [return: bb13, unwind continue];
       }
   
@@ -277,22 +293,24 @@
           StorageDead(_40);
           StorageDead(_39);
           StorageLive(_42);
--         StorageLive(_43);
+          StorageLive(_43);
 -         _43 = _2;
 -         _42 = opaque::<u64>(move _43) -> [return: bb14, unwind continue];
++         _43 = const 1_u64;
 +         _42 = opaque::<u64>(const 1_u64) -> [return: bb14, unwind continue];
       }
   
       bb14: {
--         StorageDead(_43);
+          StorageDead(_43);
           StorageDead(_42);
           StorageLive(_44);
           StorageLive(_45);
--         StorageLive(_46);
+          StorageLive(_46);
 -         _46 = _2;
 -         _45 = move _46 as i8 (IntToInt);
--         StorageDead(_46);
++         _46 = const 1_u64;
 +         _45 = const 1_u64 as i8 (IntToInt);
+          StorageDead(_46);
           _44 = opaque::<i8>(move _45) -> [return: bb15, unwind continue];
       }
   
@@ -301,11 +319,12 @@
           StorageDead(_44);
           StorageLive(_47);
           StorageLive(_48);
--         StorageLive(_49);
+          StorageLive(_49);
 -         _49 = _2;
 -         _48 = move _49 as i16 (IntToInt);
--         StorageDead(_49);
++         _49 = const 1_u64;
 +         _48 = const 1_u64 as i16 (IntToInt);
+          StorageDead(_49);
           _47 = opaque::<i16>(move _48) -> [return: bb16, unwind continue];
       }
   
@@ -314,11 +333,12 @@
           StorageDead(_47);
           StorageLive(_50);
           StorageLive(_51);
--         StorageLive(_52);
+          StorageLive(_52);
 -         _52 = _2;
 -         _51 = move _52 as i32 (IntToInt);
--         StorageDead(_52);
++         _52 = const 1_u64;
 +         _51 = const 1_u64 as i32 (IntToInt);
+          StorageDead(_52);
           _50 = opaque::<i32>(move _51) -> [return: bb17, unwind continue];
       }
   
@@ -327,11 +347,12 @@
           StorageDead(_50);
           StorageLive(_53);
           StorageLive(_54);
--         StorageLive(_55);
+          StorageLive(_55);
 -         _55 = _2;
 -         _54 = move _55 as i64 (IntToInt);
--         StorageDead(_55);
++         _55 = const 1_u64;
 +         _54 = const 1_u64 as i64 (IntToInt);
+          StorageDead(_55);
           _53 = opaque::<i64>(move _54) -> [return: bb18, unwind continue];
       }
   
@@ -340,11 +361,12 @@
           StorageDead(_53);
           StorageLive(_56);
           StorageLive(_57);
--         StorageLive(_58);
+          StorageLive(_58);
 -         _58 = _2;
 -         _57 = move _58 as f32 (IntToFloat);
--         StorageDead(_58);
++         _58 = const 1_u64;
 +         _57 = const 1_u64 as f32 (IntToFloat);
+          StorageDead(_58);
           _56 = opaque::<f32>(move _57) -> [return: bb19, unwind continue];
       }
   
@@ -353,11 +375,12 @@
           StorageDead(_56);
           StorageLive(_59);
           StorageLive(_60);
--         StorageLive(_61);
+          StorageLive(_61);
 -         _61 = _2;
 -         _60 = move _61 as f64 (IntToFloat);
--         StorageDead(_61);
++         _61 = const 1_u64;
 +         _60 = const 1_u64 as f64 (IntToFloat);
+          StorageDead(_61);
           _59 = opaque::<f64>(move _60) -> [return: bb20, unwind continue];
       }
   
@@ -366,11 +389,12 @@
           StorageDead(_59);
           StorageLive(_62);
           StorageLive(_63);
--         StorageLive(_64);
+          StorageLive(_64);
 -         _64 = _3;
 -         _63 = move _64 as u8 (FloatToInt);
--         StorageDead(_64);
++         _64 = const 1f64;
 +         _63 = const 1f64 as u8 (FloatToInt);
+          StorageDead(_64);
           _62 = opaque::<u8>(move _63) -> [return: bb21, unwind continue];
       }
   
@@ -379,11 +403,12 @@
           StorageDead(_62);
           StorageLive(_65);
           StorageLive(_66);
--         StorageLive(_67);
+          StorageLive(_67);
 -         _67 = _3;
 -         _66 = move _67 as u16 (FloatToInt);
--         StorageDead(_67);
++         _67 = const 1f64;
 +         _66 = const 1f64 as u16 (FloatToInt);
+          StorageDead(_67);
           _65 = opaque::<u16>(move _66) -> [return: bb22, unwind continue];
       }
   
@@ -392,11 +417,12 @@
           StorageDead(_65);
           StorageLive(_68);
           StorageLive(_69);
--         StorageLive(_70);
+          StorageLive(_70);
 -         _70 = _3;
 -         _69 = move _70 as u32 (FloatToInt);
--         StorageDead(_70);
++         _70 = const 1f64;
 +         _69 = const 1f64 as u32 (FloatToInt);
+          StorageDead(_70);
           _68 = opaque::<u32>(move _69) -> [return: bb23, unwind continue];
       }
   
@@ -405,11 +431,12 @@
           StorageDead(_68);
           StorageLive(_71);
           StorageLive(_72);
--         StorageLive(_73);
+          StorageLive(_73);
 -         _73 = _3;
 -         _72 = move _73 as u64 (FloatToInt);
--         StorageDead(_73);
++         _73 = const 1f64;
 +         _72 = const 1f64 as u64 (FloatToInt);
+          StorageDead(_73);
           _71 = opaque::<u64>(move _72) -> [return: bb24, unwind continue];
       }
   
@@ -418,11 +445,12 @@
           StorageDead(_71);
           StorageLive(_74);
           StorageLive(_75);
--         StorageLive(_76);
+          StorageLive(_76);
 -         _76 = _3;
 -         _75 = move _76 as i8 (FloatToInt);
--         StorageDead(_76);
++         _76 = const 1f64;
 +         _75 = const 1f64 as i8 (FloatToInt);
+          StorageDead(_76);
           _74 = opaque::<i8>(move _75) -> [return: bb25, unwind continue];
       }
   
@@ -431,11 +459,12 @@
           StorageDead(_74);
           StorageLive(_77);
           StorageLive(_78);
--         StorageLive(_79);
+          StorageLive(_79);
 -         _79 = _3;
 -         _78 = move _79 as i16 (FloatToInt);
--         StorageDead(_79);
++         _79 = const 1f64;
 +         _78 = const 1f64 as i16 (FloatToInt);
+          StorageDead(_79);
           _77 = opaque::<i16>(move _78) -> [return: bb26, unwind continue];
       }
   
@@ -444,11 +473,12 @@
           StorageDead(_77);
           StorageLive(_80);
           StorageLive(_81);
--         StorageLive(_82);
+          StorageLive(_82);
 -         _82 = _3;
 -         _81 = move _82 as i32 (FloatToInt);
--         StorageDead(_82);
++         _82 = const 1f64;
 +         _81 = const 1f64 as i32 (FloatToInt);
+          StorageDead(_82);
           _80 = opaque::<i32>(move _81) -> [return: bb27, unwind continue];
       }
   
@@ -457,11 +487,12 @@
           StorageDead(_80);
           StorageLive(_83);
           StorageLive(_84);
--         StorageLive(_85);
+          StorageLive(_85);
 -         _85 = _3;
 -         _84 = move _85 as i64 (FloatToInt);
--         StorageDead(_85);
++         _85 = const 1f64;
 +         _84 = const 1f64 as i64 (FloatToInt);
+          StorageDead(_85);
           _83 = opaque::<i64>(move _84) -> [return: bb28, unwind continue];
       }
   
@@ -470,11 +501,12 @@
           StorageDead(_83);
           StorageLive(_86);
           StorageLive(_87);
--         StorageLive(_88);
+          StorageLive(_88);
 -         _88 = _3;
 -         _87 = move _88 as f32 (FloatToFloat);
--         StorageDead(_88);
++         _88 = const 1f64;
 +         _87 = const 1f64 as f32 (FloatToFloat);
+          StorageDead(_88);
           _86 = opaque::<f32>(move _87) -> [return: bb29, unwind continue];
       }
   
@@ -482,19 +514,23 @@
           StorageDead(_87);
           StorageDead(_86);
           StorageLive(_89);
--         StorageLive(_90);
+          StorageLive(_90);
 -         _90 = _3;
 -         _89 = opaque::<f64>(move _90) -> [return: bb30, unwind continue];
++         _90 = const 1f64;
 +         _89 = opaque::<f64>(const 1f64) -> [return: bb30, unwind continue];
       }
   
       bb30: {
--         StorageDead(_90);
+          StorageDead(_90);
           StorageDead(_89);
           _0 = const ();
 -         StorageDead(_3);
 -         StorageDead(_2);
 -         StorageDead(_1);
++         nop;
++         nop;
++         nop;
           return;
       }
   }
diff --git a/tests/mir-opt/gvn.dereferences.GVN.panic-abort.diff b/tests/mir-opt/gvn.dereferences.GVN.panic-abort.diff
index ee320cf67870..46bf13985daf 100644
--- a/tests/mir-opt/gvn.dereferences.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.dereferences.GVN.panic-abort.diff
@@ -116,6 +116,7 @@
           _18 = &(*_1);
           StorageLive(_19);
 -         StorageLive(_20);
++         nop;
           _20 = (*_18);
 -         _19 = opaque::<u32>(move _20) -> [return: bb7, unwind unreachable];
 +         _19 = opaque::<u32>(_20) -> [return: bb7, unwind unreachable];
@@ -123,16 +124,18 @@
   
       bb7: {
 -         StorageDead(_20);
++         nop;
           StorageDead(_19);
           StorageLive(_21);
--         StorageLive(_22);
+          StorageLive(_22);
 -         _22 = (*_18);
 -         _21 = opaque::<u32>(move _22) -> [return: bb8, unwind unreachable];
++         _22 = _20;
 +         _21 = opaque::<u32>(_20) -> [return: bb8, unwind unreachable];
       }
   
       bb8: {
--         StorageDead(_22);
+          StorageDead(_22);
           StorageDead(_21);
           StorageLive(_23);
           StorageLive(_24);
@@ -163,6 +166,7 @@
           StorageDead(_27);
           StorageLive(_29);
 -         StorageLive(_30);
++         nop;
           _30 = ((*_3).0: u32);
 -         _29 = opaque::<u32>(move _30) -> [return: bb12, unwind unreachable];
 +         _29 = opaque::<u32>(_30) -> [return: bb12, unwind unreachable];
@@ -170,16 +174,18 @@
   
       bb12: {
 -         StorageDead(_30);
++         nop;
           StorageDead(_29);
           StorageLive(_31);
--         StorageLive(_32);
+          StorageLive(_32);
 -         _32 = ((*_3).0: u32);
 -         _31 = opaque::<u32>(move _32) -> [return: bb13, unwind unreachable];
++         _32 = _30;
 +         _31 = opaque::<u32>(_30) -> [return: bb13, unwind unreachable];
       }
   
       bb13: {
--         StorageDead(_32);
+          StorageDead(_32);
           StorageDead(_31);
           _0 = const ();
           StorageDead(_18);
diff --git a/tests/mir-opt/gvn.dereferences.GVN.panic-unwind.diff b/tests/mir-opt/gvn.dereferences.GVN.panic-unwind.diff
index f627b4d59887..3e731ead859e 100644
--- a/tests/mir-opt/gvn.dereferences.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.dereferences.GVN.panic-unwind.diff
@@ -116,6 +116,7 @@
           _18 = &(*_1);
           StorageLive(_19);
 -         StorageLive(_20);
++         nop;
           _20 = (*_18);
 -         _19 = opaque::<u32>(move _20) -> [return: bb7, unwind continue];
 +         _19 = opaque::<u32>(_20) -> [return: bb7, unwind continue];
@@ -123,16 +124,18 @@
   
       bb7: {
 -         StorageDead(_20);
++         nop;
           StorageDead(_19);
           StorageLive(_21);
--         StorageLive(_22);
+          StorageLive(_22);
 -         _22 = (*_18);
 -         _21 = opaque::<u32>(move _22) -> [return: bb8, unwind continue];
++         _22 = _20;
 +         _21 = opaque::<u32>(_20) -> [return: bb8, unwind continue];
       }
   
       bb8: {
--         StorageDead(_22);
+          StorageDead(_22);
           StorageDead(_21);
           StorageLive(_23);
           StorageLive(_24);
@@ -163,6 +166,7 @@
           StorageDead(_27);
           StorageLive(_29);
 -         StorageLive(_30);
++         nop;
           _30 = ((*_3).0: u32);
 -         _29 = opaque::<u32>(move _30) -> [return: bb12, unwind continue];
 +         _29 = opaque::<u32>(_30) -> [return: bb12, unwind continue];
@@ -170,16 +174,18 @@
   
       bb12: {
 -         StorageDead(_30);
++         nop;
           StorageDead(_29);
           StorageLive(_31);
--         StorageLive(_32);
+          StorageLive(_32);
 -         _32 = ((*_3).0: u32);
 -         _31 = opaque::<u32>(move _32) -> [return: bb13, unwind continue];
++         _32 = _30;
 +         _31 = opaque::<u32>(_30) -> [return: bb13, unwind continue];
       }
   
       bb13: {
--         StorageDead(_32);
+          StorageDead(_32);
           StorageDead(_31);
           _0 = const ();
           StorageDead(_18);
diff --git a/tests/mir-opt/gvn.multiple_branches.GVN.panic-abort.diff b/tests/mir-opt/gvn.multiple_branches.GVN.panic-abort.diff
index 0a66900283b6..29ca1ba5902e 100644
--- a/tests/mir-opt/gvn.multiple_branches.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.multiple_branches.GVN.panic-abort.diff
@@ -39,9 +39,9 @@
       let mut _34: u8;
   
       bb0: {
--         StorageLive(_4);
--         StorageLive(_5);
--         _5 = _1;
+          StorageLive(_4);
+          StorageLive(_5);
+          _5 = _1;
 -         switchInt(move _5) -> [0: bb4, otherwise: bb1];
 +         switchInt(_1) -> [0: bb4, otherwise: bb1];
       }
@@ -49,121 +49,130 @@
       bb1: {
           StorageLive(_6);
 -         StorageLive(_7);
--         StorageLive(_8);
--         _8 = _2;
--         StorageLive(_9);
--         _9 = _3;
++         nop;
+          StorageLive(_8);
+          _8 = _2;
+          StorageLive(_9);
+          _9 = _3;
 -         _7 = Add(move _8, move _9);
--         StorageDead(_9);
--         StorageDead(_8);
--         _6 = opaque::<u8>(move _7) -> [return: bb2, unwind unreachable];
 +         _7 = Add(_2, _3);
+          StorageDead(_9);
+          StorageDead(_8);
+-         _6 = opaque::<u8>(move _7) -> [return: bb2, unwind unreachable];
 +         _6 = opaque::<u8>(_7) -> [return: bb2, unwind unreachable];
       }
   
       bb2: {
 -         StorageDead(_7);
++         nop;
           StorageDead(_6);
           StorageLive(_10);
--         StorageLive(_11);
--         StorageLive(_12);
--         _12 = _2;
--         StorageLive(_13);
--         _13 = _3;
+          StorageLive(_11);
+          StorageLive(_12);
+          _12 = _2;
+          StorageLive(_13);
+          _13 = _3;
 -         _11 = Add(move _12, move _13);
--         StorageDead(_13);
--         StorageDead(_12);
++         _11 = _7;
+          StorageDead(_13);
+          StorageDead(_12);
 -         _10 = opaque::<u8>(move _11) -> [return: bb3, unwind unreachable];
 +         _10 = opaque::<u8>(_7) -> [return: bb3, unwind unreachable];
       }
   
       bb3: {
--         StorageDead(_11);
+          StorageDead(_11);
           StorageDead(_10);
--         _4 = const ();
+          _4 = const ();
           goto -> bb7;
       }
   
       bb4: {
           StorageLive(_14);
 -         StorageLive(_15);
--         StorageLive(_16);
--         _16 = _2;
--         StorageLive(_17);
--         _17 = _3;
++         nop;
+          StorageLive(_16);
+          _16 = _2;
+          StorageLive(_17);
+          _17 = _3;
 -         _15 = Add(move _16, move _17);
--         StorageDead(_17);
--         StorageDead(_16);
--         _14 = opaque::<u8>(move _15) -> [return: bb5, unwind unreachable];
 +         _15 = Add(_2, _3);
+          StorageDead(_17);
+          StorageDead(_16);
+-         _14 = opaque::<u8>(move _15) -> [return: bb5, unwind unreachable];
 +         _14 = opaque::<u8>(_15) -> [return: bb5, unwind unreachable];
       }
   
       bb5: {
 -         StorageDead(_15);
++         nop;
           StorageDead(_14);
           StorageLive(_18);
--         StorageLive(_19);
--         StorageLive(_20);
--         _20 = _2;
--         StorageLive(_21);
--         _21 = _3;
+          StorageLive(_19);
+          StorageLive(_20);
+          _20 = _2;
+          StorageLive(_21);
+          _21 = _3;
 -         _19 = Add(move _20, move _21);
--         StorageDead(_21);
--         StorageDead(_20);
++         _19 = _15;
+          StorageDead(_21);
+          StorageDead(_20);
 -         _18 = opaque::<u8>(move _19) -> [return: bb6, unwind unreachable];
 +         _18 = opaque::<u8>(_15) -> [return: bb6, unwind unreachable];
       }
   
       bb6: {
--         StorageDead(_19);
+          StorageDead(_19);
           StorageDead(_18);
--         _4 = const ();
+          _4 = const ();
           goto -> bb7;
       }
   
       bb7: {
--         StorageDead(_5);
--         StorageDead(_4);
+          StorageDead(_5);
+          StorageDead(_4);
           StorageLive(_22);
 -         StorageLive(_23);
--         StorageLive(_24);
--         _24 = _2;
--         StorageLive(_25);
--         _25 = _3;
++         nop;
+          StorageLive(_24);
+          _24 = _2;
+          StorageLive(_25);
+          _25 = _3;
 -         _23 = Add(move _24, move _25);
--         StorageDead(_25);
--         StorageDead(_24);
--         _22 = opaque::<u8>(move _23) -> [return: bb8, unwind unreachable];
 +         _23 = Add(_2, _3);
+          StorageDead(_25);
+          StorageDead(_24);
+-         _22 = opaque::<u8>(move _23) -> [return: bb8, unwind unreachable];
 +         _22 = opaque::<u8>(_23) -> [return: bb8, unwind unreachable];
       }
   
       bb8: {
 -         StorageDead(_23);
++         nop;
           StorageDead(_22);
--         StorageLive(_26);
--         _26 = _1;
+          StorageLive(_26);
+          _26 = _1;
 -         switchInt(move _26) -> [0: bb11, otherwise: bb9];
 +         switchInt(_1) -> [0: bb11, otherwise: bb9];
       }
   
       bb9: {
           StorageLive(_27);
--         StorageLive(_28);
--         StorageLive(_29);
--         _29 = _2;
--         StorageLive(_30);
--         _30 = _3;
+          StorageLive(_28);
+          StorageLive(_29);
+          _29 = _2;
+          StorageLive(_30);
+          _30 = _3;
 -         _28 = Add(move _29, move _30);
--         StorageDead(_30);
--         StorageDead(_29);
++         _28 = _23;
+          StorageDead(_30);
+          StorageDead(_29);
 -         _27 = opaque::<u8>(move _28) -> [return: bb10, unwind unreachable];
 +         _27 = opaque::<u8>(_23) -> [return: bb10, unwind unreachable];
       }
   
       bb10: {
--         StorageDead(_28);
+          StorageDead(_28);
           StorageDead(_27);
           _0 = const ();
           goto -> bb13;
@@ -171,27 +180,28 @@
   
       bb11: {
           StorageLive(_31);
--         StorageLive(_32);
--         StorageLive(_33);
--         _33 = _2;
--         StorageLive(_34);
--         _34 = _3;
+          StorageLive(_32);
+          StorageLive(_33);
+          _33 = _2;
+          StorageLive(_34);
+          _34 = _3;
 -         _32 = Add(move _33, move _34);
--         StorageDead(_34);
--         StorageDead(_33);
++         _32 = _23;
+          StorageDead(_34);
+          StorageDead(_33);
 -         _31 = opaque::<u8>(move _32) -> [return: bb12, unwind unreachable];
 +         _31 = opaque::<u8>(_23) -> [return: bb12, unwind unreachable];
       }
   
       bb12: {
--         StorageDead(_32);
+          StorageDead(_32);
           StorageDead(_31);
           _0 = const ();
           goto -> bb13;
       }
   
       bb13: {
--         StorageDead(_26);
+          StorageDead(_26);
           return;
       }
   }
diff --git a/tests/mir-opt/gvn.multiple_branches.GVN.panic-unwind.diff b/tests/mir-opt/gvn.multiple_branches.GVN.panic-unwind.diff
index 0199f2720a98..5394dc8be8a0 100644
--- a/tests/mir-opt/gvn.multiple_branches.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.multiple_branches.GVN.panic-unwind.diff
@@ -39,9 +39,9 @@
       let mut _34: u8;
   
       bb0: {
--         StorageLive(_4);
--         StorageLive(_5);
--         _5 = _1;
+          StorageLive(_4);
+          StorageLive(_5);
+          _5 = _1;
 -         switchInt(move _5) -> [0: bb4, otherwise: bb1];
 +         switchInt(_1) -> [0: bb4, otherwise: bb1];
       }
@@ -49,121 +49,130 @@
       bb1: {
           StorageLive(_6);
 -         StorageLive(_7);
--         StorageLive(_8);
--         _8 = _2;
--         StorageLive(_9);
--         _9 = _3;
++         nop;
+          StorageLive(_8);
+          _8 = _2;
+          StorageLive(_9);
+          _9 = _3;
 -         _7 = Add(move _8, move _9);
--         StorageDead(_9);
--         StorageDead(_8);
--         _6 = opaque::<u8>(move _7) -> [return: bb2, unwind continue];
 +         _7 = Add(_2, _3);
+          StorageDead(_9);
+          StorageDead(_8);
+-         _6 = opaque::<u8>(move _7) -> [return: bb2, unwind continue];
 +         _6 = opaque::<u8>(_7) -> [return: bb2, unwind continue];
       }
   
       bb2: {
 -         StorageDead(_7);
++         nop;
           StorageDead(_6);
           StorageLive(_10);
--         StorageLive(_11);
--         StorageLive(_12);
--         _12 = _2;
--         StorageLive(_13);
--         _13 = _3;
+          StorageLive(_11);
+          StorageLive(_12);
+          _12 = _2;
+          StorageLive(_13);
+          _13 = _3;
 -         _11 = Add(move _12, move _13);
--         StorageDead(_13);
--         StorageDead(_12);
++         _11 = _7;
+          StorageDead(_13);
+          StorageDead(_12);
 -         _10 = opaque::<u8>(move _11) -> [return: bb3, unwind continue];
 +         _10 = opaque::<u8>(_7) -> [return: bb3, unwind continue];
       }
   
       bb3: {
--         StorageDead(_11);
+          StorageDead(_11);
           StorageDead(_10);
--         _4 = const ();
+          _4 = const ();
           goto -> bb7;
       }
   
       bb4: {
           StorageLive(_14);
 -         StorageLive(_15);
--         StorageLive(_16);
--         _16 = _2;
--         StorageLive(_17);
--         _17 = _3;
++         nop;
+          StorageLive(_16);
+          _16 = _2;
+          StorageLive(_17);
+          _17 = _3;
 -         _15 = Add(move _16, move _17);
--         StorageDead(_17);
--         StorageDead(_16);
--         _14 = opaque::<u8>(move _15) -> [return: bb5, unwind continue];
 +         _15 = Add(_2, _3);
+          StorageDead(_17);
+          StorageDead(_16);
+-         _14 = opaque::<u8>(move _15) -> [return: bb5, unwind continue];
 +         _14 = opaque::<u8>(_15) -> [return: bb5, unwind continue];
       }
   
       bb5: {
 -         StorageDead(_15);
++         nop;
           StorageDead(_14);
           StorageLive(_18);
--         StorageLive(_19);
--         StorageLive(_20);
--         _20 = _2;
--         StorageLive(_21);
--         _21 = _3;
+          StorageLive(_19);
+          StorageLive(_20);
+          _20 = _2;
+          StorageLive(_21);
+          _21 = _3;
 -         _19 = Add(move _20, move _21);
--         StorageDead(_21);
--         StorageDead(_20);
++         _19 = _15;
+          StorageDead(_21);
+          StorageDead(_20);
 -         _18 = opaque::<u8>(move _19) -> [return: bb6, unwind continue];
 +         _18 = opaque::<u8>(_15) -> [return: bb6, unwind continue];
       }
   
       bb6: {
--         StorageDead(_19);
+          StorageDead(_19);
           StorageDead(_18);
--         _4 = const ();
+          _4 = const ();
           goto -> bb7;
       }
   
       bb7: {
--         StorageDead(_5);
--         StorageDead(_4);
+          StorageDead(_5);
+          StorageDead(_4);
           StorageLive(_22);
 -         StorageLive(_23);
--         StorageLive(_24);
--         _24 = _2;
--         StorageLive(_25);
--         _25 = _3;
++         nop;
+          StorageLive(_24);
+          _24 = _2;
+          StorageLive(_25);
+          _25 = _3;
 -         _23 = Add(move _24, move _25);
--         StorageDead(_25);
--         StorageDead(_24);
--         _22 = opaque::<u8>(move _23) -> [return: bb8, unwind continue];
 +         _23 = Add(_2, _3);
+          StorageDead(_25);
+          StorageDead(_24);
+-         _22 = opaque::<u8>(move _23) -> [return: bb8, unwind continue];
 +         _22 = opaque::<u8>(_23) -> [return: bb8, unwind continue];
       }
   
       bb8: {
 -         StorageDead(_23);
++         nop;
           StorageDead(_22);
--         StorageLive(_26);
--         _26 = _1;
+          StorageLive(_26);
+          _26 = _1;
 -         switchInt(move _26) -> [0: bb11, otherwise: bb9];
 +         switchInt(_1) -> [0: bb11, otherwise: bb9];
       }
   
       bb9: {
           StorageLive(_27);
--         StorageLive(_28);
--         StorageLive(_29);
--         _29 = _2;
--         StorageLive(_30);
--         _30 = _3;
+          StorageLive(_28);
+          StorageLive(_29);
+          _29 = _2;
+          StorageLive(_30);
+          _30 = _3;
 -         _28 = Add(move _29, move _30);
--         StorageDead(_30);
--         StorageDead(_29);
++         _28 = _23;
+          StorageDead(_30);
+          StorageDead(_29);
 -         _27 = opaque::<u8>(move _28) -> [return: bb10, unwind continue];
 +         _27 = opaque::<u8>(_23) -> [return: bb10, unwind continue];
       }
   
       bb10: {
--         StorageDead(_28);
+          StorageDead(_28);
           StorageDead(_27);
           _0 = const ();
           goto -> bb13;
@@ -171,27 +180,28 @@
   
       bb11: {
           StorageLive(_31);
--         StorageLive(_32);
--         StorageLive(_33);
--         _33 = _2;
--         StorageLive(_34);
--         _34 = _3;
+          StorageLive(_32);
+          StorageLive(_33);
+          _33 = _2;
+          StorageLive(_34);
+          _34 = _3;
 -         _32 = Add(move _33, move _34);
--         StorageDead(_34);
--         StorageDead(_33);
++         _32 = _23;
+          StorageDead(_34);
+          StorageDead(_33);
 -         _31 = opaque::<u8>(move _32) -> [return: bb12, unwind continue];
 +         _31 = opaque::<u8>(_23) -> [return: bb12, unwind continue];
       }
   
       bb12: {
--         StorageDead(_32);
+          StorageDead(_32);
           StorageDead(_31);
           _0 = const ();
           goto -> bb13;
       }
   
       bb13: {
--         StorageDead(_26);
+          StorageDead(_26);
           return;
       }
   }
diff --git a/tests/mir-opt/gvn.repeated_index.GVN.panic-abort.diff b/tests/mir-opt/gvn.repeated_index.GVN.panic-abort.diff
index 4c29523d6b28..6e542d7f964c 100644
--- a/tests/mir-opt/gvn.repeated_index.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.repeated_index.GVN.panic-abort.diff
@@ -23,11 +23,11 @@
   
       bb0: {
           StorageLive(_3);
--         StorageLive(_4);
--         _4 = _1;
+          StorageLive(_4);
+          _4 = _1;
 -         _3 = [move _4; N];
--         StorageDead(_4);
 +         _3 = [_1; N];
+          StorageDead(_4);
           StorageLive(_5);
           StorageLive(_6);
           StorageLive(_7);
@@ -55,6 +55,7 @@
 -         _13 = Len(_3);
 -         _14 = Lt(_12, _13);
 -         assert(move _14, "index out of bounds: the length is {} but the index is {}", move _13, _12) -> [success: bb3, unwind unreachable];
++         _13 = _8;
 +         _14 = Lt(_2, _8);
 +         assert(move _14, "index out of bounds: the length is {} but the index is {}", _8, _2) -> [success: bb3, unwind unreachable];
       }
diff --git a/tests/mir-opt/gvn.repeated_index.GVN.panic-unwind.diff b/tests/mir-opt/gvn.repeated_index.GVN.panic-unwind.diff
index e44f54cf3cf3..a07cd49ec7c6 100644
--- a/tests/mir-opt/gvn.repeated_index.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.repeated_index.GVN.panic-unwind.diff
@@ -23,11 +23,11 @@
   
       bb0: {
           StorageLive(_3);
--         StorageLive(_4);
--         _4 = _1;
+          StorageLive(_4);
+          _4 = _1;
 -         _3 = [move _4; N];
--         StorageDead(_4);
 +         _3 = [_1; N];
+          StorageDead(_4);
           StorageLive(_5);
           StorageLive(_6);
           StorageLive(_7);
@@ -55,6 +55,7 @@
 -         _13 = Len(_3);
 -         _14 = Lt(_12, _13);
 -         assert(move _14, "index out of bounds: the length is {} but the index is {}", move _13, _12) -> [success: bb3, unwind continue];
++         _13 = _8;
 +         _14 = Lt(_2, _8);
 +         assert(move _14, "index out of bounds: the length is {} but the index is {}", _8, _2) -> [success: bb3, unwind continue];
       }
diff --git a/tests/mir-opt/gvn.slices.GVN.panic-abort.diff b/tests/mir-opt/gvn.slices.GVN.panic-abort.diff
index de3d28d05759..d924c70d617d 100644
--- a/tests/mir-opt/gvn.slices.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.slices.GVN.panic-abort.diff
@@ -85,30 +85,32 @@
   
       bb0: {
 -         StorageLive(_1);
++         nop;
           _1 = const "my favourite slice";
           StorageLive(_2);
--         StorageLive(_3);
--         _3 = _1;
+          StorageLive(_3);
+          _3 = _1;
 -         _2 = opaque::<&str>(move _3) -> [return: bb1, unwind unreachable];
 +         _2 = opaque::<&str>(_1) -> [return: bb1, unwind unreachable];
       }
   
       bb1: {
--         StorageDead(_3);
+          StorageDead(_3);
           StorageDead(_2);
           StorageLive(_4);
           _4 = _1;
           StorageLive(_5);
--         StorageLive(_6);
+          StorageLive(_6);
 -         _6 = _4;
 -         _5 = opaque::<&str>(move _6) -> [return: bb2, unwind unreachable];
++         _6 = _1;
 +         _5 = opaque::<&str>(_1) -> [return: bb2, unwind unreachable];
       }
   
       bb2: {
--         StorageDead(_6);
+          StorageDead(_6);
           StorageDead(_5);
--         StorageLive(_7);
+          StorageLive(_7);
           StorageLive(_8);
           StorageLive(_9);
           StorageLive(_10);
@@ -149,22 +151,23 @@
       bb5: {
           StorageDead(_19);
           StorageDead(_18);
--         _7 = const ();
+          _7 = const ();
           StorageDead(_17);
           StorageDead(_16);
           StorageDead(_15);
           StorageDead(_13);
           StorageDead(_10);
           StorageDead(_8);
--         StorageDead(_7);
+          StorageDead(_7);
 -         StorageLive(_29);
++         nop;
           StorageLive(_30);
           _30 = &(*_1);
           _29 = move _30 as &[u8] (Transmute);
           StorageDead(_30);
           StorageLive(_31);
--         StorageLive(_32);
--         _32 = _29;
+          StorageLive(_32);
+          _32 = _29;
 -         _31 = opaque::<&[u8]>(move _32) -> [return: bb7, unwind unreachable];
 +         _31 = opaque::<&[u8]>(_29) -> [return: bb7, unwind unreachable];
       }
@@ -173,10 +176,12 @@
           StorageDead(_19);
           StorageDead(_18);
 -         StorageLive(_21);
++         nop;
           _21 = core::panicking::AssertKind::Eq;
           StorageLive(_22);
--         StorageLive(_23);
+          StorageLive(_23);
 -         _23 = move _21;
++         _23 = _21;
           StorageLive(_24);
           StorageLive(_25);
           _25 = &(*_15);
@@ -192,9 +197,9 @@
       }
   
       bb7: {
--         StorageDead(_32);
+          StorageDead(_32);
           StorageDead(_31);
--         StorageLive(_33);
+          StorageLive(_33);
           StorageLive(_34);
           StorageLive(_35);
           StorageLive(_36);
@@ -235,18 +240,20 @@
       bb10: {
           StorageDead(_45);
           StorageDead(_44);
--         _33 = const ();
+          _33 = const ();
           StorageDead(_43);
           StorageDead(_42);
           StorageDead(_41);
           StorageDead(_39);
           StorageDead(_36);
           StorageDead(_34);
--         StorageDead(_33);
+          StorageDead(_33);
           _0 = const ();
 -         StorageDead(_29);
++         nop;
           StorageDead(_4);
 -         StorageDead(_1);
++         nop;
           return;
       }
   
@@ -254,10 +261,12 @@
           StorageDead(_45);
           StorageDead(_44);
 -         StorageLive(_47);
++         nop;
           _47 = core::panicking::AssertKind::Eq;
           StorageLive(_48);
--         StorageLive(_49);
+          StorageLive(_49);
 -         _49 = move _47;
++         _49 = _47;
           StorageLive(_50);
           StorageLive(_51);
           _51 = &(*_41);
diff --git a/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff b/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff
index f22bb25436f0..63225aa0a101 100644
--- a/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff
@@ -85,30 +85,32 @@
   
       bb0: {
 -         StorageLive(_1);
++         nop;
           _1 = const "my favourite slice";
           StorageLive(_2);
--         StorageLive(_3);
--         _3 = _1;
+          StorageLive(_3);
+          _3 = _1;
 -         _2 = opaque::<&str>(move _3) -> [return: bb1, unwind continue];
 +         _2 = opaque::<&str>(_1) -> [return: bb1, unwind continue];
       }
   
       bb1: {
--         StorageDead(_3);
+          StorageDead(_3);
           StorageDead(_2);
           StorageLive(_4);
           _4 = _1;
           StorageLive(_5);
--         StorageLive(_6);
+          StorageLive(_6);
 -         _6 = _4;
 -         _5 = opaque::<&str>(move _6) -> [return: bb2, unwind continue];
++         _6 = _1;
 +         _5 = opaque::<&str>(_1) -> [return: bb2, unwind continue];
       }
   
       bb2: {
--         StorageDead(_6);
+          StorageDead(_6);
           StorageDead(_5);
--         StorageLive(_7);
+          StorageLive(_7);
           StorageLive(_8);
           StorageLive(_9);
           StorageLive(_10);
@@ -149,22 +151,23 @@
       bb5: {
           StorageDead(_19);
           StorageDead(_18);
--         _7 = const ();
+          _7 = const ();
           StorageDead(_17);
           StorageDead(_16);
           StorageDead(_15);
           StorageDead(_13);
           StorageDead(_10);
           StorageDead(_8);
--         StorageDead(_7);
+          StorageDead(_7);
 -         StorageLive(_29);
++         nop;
           StorageLive(_30);
           _30 = &(*_1);
           _29 = move _30 as &[u8] (Transmute);
           StorageDead(_30);
           StorageLive(_31);
--         StorageLive(_32);
--         _32 = _29;
+          StorageLive(_32);
+          _32 = _29;
 -         _31 = opaque::<&[u8]>(move _32) -> [return: bb7, unwind continue];
 +         _31 = opaque::<&[u8]>(_29) -> [return: bb7, unwind continue];
       }
@@ -173,10 +176,12 @@
           StorageDead(_19);
           StorageDead(_18);
 -         StorageLive(_21);
++         nop;
           _21 = core::panicking::AssertKind::Eq;
           StorageLive(_22);
--         StorageLive(_23);
+          StorageLive(_23);
 -         _23 = move _21;
++         _23 = _21;
           StorageLive(_24);
           StorageLive(_25);
           _25 = &(*_15);
@@ -192,9 +197,9 @@
       }
   
       bb7: {
--         StorageDead(_32);
+          StorageDead(_32);
           StorageDead(_31);
--         StorageLive(_33);
+          StorageLive(_33);
           StorageLive(_34);
           StorageLive(_35);
           StorageLive(_36);
@@ -235,18 +240,20 @@
       bb10: {
           StorageDead(_45);
           StorageDead(_44);
--         _33 = const ();
+          _33 = const ();
           StorageDead(_43);
           StorageDead(_42);
           StorageDead(_41);
           StorageDead(_39);
           StorageDead(_36);
           StorageDead(_34);
--         StorageDead(_33);
+          StorageDead(_33);
           _0 = const ();
 -         StorageDead(_29);
++         nop;
           StorageDead(_4);
 -         StorageDead(_1);
++         nop;
           return;
       }
   
@@ -254,10 +261,12 @@
           StorageDead(_45);
           StorageDead(_44);
 -         StorageLive(_47);
++         nop;
           _47 = core::panicking::AssertKind::Eq;
           StorageLive(_48);
--         StorageLive(_49);
+          StorageLive(_49);
 -         _49 = move _47;
++         _49 = _47;
           StorageLive(_50);
           StorageLive(_51);
           _51 = &(*_41);
diff --git a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff
index bf866e2f4d22..6c8513e32816 100644
--- a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff
@@ -197,61 +197,68 @@
       bb0: {
           StorageLive(_4);
 -         StorageLive(_5);
--         StorageLive(_6);
--         _6 = _1;
--         StorageLive(_7);
--         _7 = _2;
++         nop;
+          StorageLive(_6);
+          _6 = _1;
+          StorageLive(_7);
+          _7 = _2;
 -         _5 = Add(move _6, move _7);
--         StorageDead(_7);
--         StorageDead(_6);
--         _4 = opaque::<u64>(move _5) -> [return: bb1, unwind unreachable];
 +         _5 = Add(_1, _2);
+          StorageDead(_7);
+          StorageDead(_6);
+-         _4 = opaque::<u64>(move _5) -> [return: bb1, unwind unreachable];
 +         _4 = opaque::<u64>(_5) -> [return: bb1, unwind unreachable];
       }
   
       bb1: {
 -         StorageDead(_5);
++         nop;
           StorageDead(_4);
           StorageLive(_8);
 -         StorageLive(_9);
--         StorageLive(_10);
--         _10 = _1;
--         StorageLive(_11);
--         _11 = _2;
++         nop;
+          StorageLive(_10);
+          _10 = _1;
+          StorageLive(_11);
+          _11 = _2;
 -         _9 = Mul(move _10, move _11);
--         StorageDead(_11);
--         StorageDead(_10);
--         _8 = opaque::<u64>(move _9) -> [return: bb2, unwind unreachable];
 +         _9 = Mul(_1, _2);
+          StorageDead(_11);
+          StorageDead(_10);
+-         _8 = opaque::<u64>(move _9) -> [return: bb2, unwind unreachable];
 +         _8 = opaque::<u64>(_9) -> [return: bb2, unwind unreachable];
       }
   
       bb2: {
 -         StorageDead(_9);
++         nop;
           StorageDead(_8);
           StorageLive(_12);
 -         StorageLive(_13);
--         StorageLive(_14);
--         _14 = _1;
--         StorageLive(_15);
--         _15 = _2;
++         nop;
+          StorageLive(_14);
+          _14 = _1;
+          StorageLive(_15);
+          _15 = _2;
 -         _13 = Sub(move _14, move _15);
--         StorageDead(_15);
--         StorageDead(_14);
--         _12 = opaque::<u64>(move _13) -> [return: bb3, unwind unreachable];
 +         _13 = Sub(_1, _2);
+          StorageDead(_15);
+          StorageDead(_14);
+-         _12 = opaque::<u64>(move _13) -> [return: bb3, unwind unreachable];
 +         _12 = opaque::<u64>(_13) -> [return: bb3, unwind unreachable];
       }
   
       bb3: {
 -         StorageDead(_13);
++         nop;
           StorageDead(_12);
           StorageLive(_16);
 -         StorageLive(_17);
--         StorageLive(_18);
--         _18 = _1;
--         StorageLive(_19);
--         _19 = _2;
++         nop;
+          StorageLive(_18);
+          _18 = _1;
+          StorageLive(_19);
+          _19 = _2;
 -         _20 = Eq(_19, const 0_u64);
 -         assert(!move _20, "attempt to divide `{}` by zero", _18) -> [success: bb4, unwind unreachable];
 +         _20 = Eq(_2, const 0_u64);
@@ -260,131 +267,145 @@
   
       bb4: {
 -         _17 = Div(move _18, move _19);
--         StorageDead(_19);
--         StorageDead(_18);
--         _16 = opaque::<u64>(move _17) -> [return: bb5, unwind unreachable];
 +         _17 = Div(_1, _2);
+          StorageDead(_19);
+          StorageDead(_18);
+-         _16 = opaque::<u64>(move _17) -> [return: bb5, unwind unreachable];
 +         _16 = opaque::<u64>(_17) -> [return: bb5, unwind unreachable];
       }
   
       bb5: {
 -         StorageDead(_17);
++         nop;
           StorageDead(_16);
           StorageLive(_21);
 -         StorageLive(_22);
--         StorageLive(_23);
--         _23 = _1;
--         StorageLive(_24);
--         _24 = _2;
++         nop;
+          StorageLive(_23);
+          _23 = _1;
+          StorageLive(_24);
+          _24 = _2;
 -         _25 = Eq(_24, const 0_u64);
 -         assert(!move _25, "attempt to calculate the remainder of `{}` with a divisor of zero", _23) -> [success: bb6, unwind unreachable];
++         _25 = _20;
 +         assert(!_20, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb6, unwind unreachable];
       }
   
       bb6: {
 -         _22 = Rem(move _23, move _24);
--         StorageDead(_24);
--         StorageDead(_23);
--         _21 = opaque::<u64>(move _22) -> [return: bb7, unwind unreachable];
 +         _22 = Rem(_1, _2);
+          StorageDead(_24);
+          StorageDead(_23);
+-         _21 = opaque::<u64>(move _22) -> [return: bb7, unwind unreachable];
 +         _21 = opaque::<u64>(_22) -> [return: bb7, unwind unreachable];
       }
   
       bb7: {
 -         StorageDead(_22);
++         nop;
           StorageDead(_21);
           StorageLive(_26);
 -         StorageLive(_27);
--         StorageLive(_28);
--         _28 = _1;
--         StorageLive(_29);
--         _29 = _2;
++         nop;
+          StorageLive(_28);
+          _28 = _1;
+          StorageLive(_29);
+          _29 = _2;
 -         _27 = BitAnd(move _28, move _29);
--         StorageDead(_29);
--         StorageDead(_28);
--         _26 = opaque::<u64>(move _27) -> [return: bb8, unwind unreachable];
 +         _27 = BitAnd(_1, _2);
+          StorageDead(_29);
+          StorageDead(_28);
+-         _26 = opaque::<u64>(move _27) -> [return: bb8, unwind unreachable];
 +         _26 = opaque::<u64>(_27) -> [return: bb8, unwind unreachable];
       }
   
       bb8: {
 -         StorageDead(_27);
++         nop;
           StorageDead(_26);
           StorageLive(_30);
 -         StorageLive(_31);
--         StorageLive(_32);
--         _32 = _1;
--         StorageLive(_33);
--         _33 = _2;
++         nop;
+          StorageLive(_32);
+          _32 = _1;
+          StorageLive(_33);
+          _33 = _2;
 -         _31 = BitOr(move _32, move _33);
--         StorageDead(_33);
--         StorageDead(_32);
--         _30 = opaque::<u64>(move _31) -> [return: bb9, unwind unreachable];
 +         _31 = BitOr(_1, _2);
+          StorageDead(_33);
+          StorageDead(_32);
+-         _30 = opaque::<u64>(move _31) -> [return: bb9, unwind unreachable];
 +         _30 = opaque::<u64>(_31) -> [return: bb9, unwind unreachable];
       }
   
       bb9: {
 -         StorageDead(_31);
++         nop;
           StorageDead(_30);
           StorageLive(_34);
 -         StorageLive(_35);
--         StorageLive(_36);
--         _36 = _1;
--         StorageLive(_37);
--         _37 = _2;
++         nop;
+          StorageLive(_36);
+          _36 = _1;
+          StorageLive(_37);
+          _37 = _2;
 -         _35 = BitXor(move _36, move _37);
--         StorageDead(_37);
--         StorageDead(_36);
--         _34 = opaque::<u64>(move _35) -> [return: bb10, unwind unreachable];
 +         _35 = BitXor(_1, _2);
+          StorageDead(_37);
+          StorageDead(_36);
+-         _34 = opaque::<u64>(move _35) -> [return: bb10, unwind unreachable];
 +         _34 = opaque::<u64>(_35) -> [return: bb10, unwind unreachable];
       }
   
       bb10: {
 -         StorageDead(_35);
++         nop;
           StorageDead(_34);
           StorageLive(_38);
 -         StorageLive(_39);
--         StorageLive(_40);
--         _40 = _1;
--         StorageLive(_41);
--         _41 = _2;
++         nop;
+          StorageLive(_40);
+          _40 = _1;
+          StorageLive(_41);
+          _41 = _2;
 -         _39 = Shl(move _40, move _41);
--         StorageDead(_41);
--         StorageDead(_40);
--         _38 = opaque::<u64>(move _39) -> [return: bb11, unwind unreachable];
 +         _39 = Shl(_1, _2);
+          StorageDead(_41);
+          StorageDead(_40);
+-         _38 = opaque::<u64>(move _39) -> [return: bb11, unwind unreachable];
 +         _38 = opaque::<u64>(_39) -> [return: bb11, unwind unreachable];
       }
   
       bb11: {
 -         StorageDead(_39);
++         nop;
           StorageDead(_38);
           StorageLive(_42);
 -         StorageLive(_43);
--         StorageLive(_44);
--         _44 = _1;
--         StorageLive(_45);
--         _45 = _2;
++         nop;
+          StorageLive(_44);
+          _44 = _1;
+          StorageLive(_45);
+          _45 = _2;
 -         _43 = Shr(move _44, move _45);
--         StorageDead(_45);
--         StorageDead(_44);
--         _42 = opaque::<u64>(move _43) -> [return: bb12, unwind unreachable];
 +         _43 = Shr(_1, _2);
+          StorageDead(_45);
+          StorageDead(_44);
+-         _42 = opaque::<u64>(move _43) -> [return: bb12, unwind unreachable];
 +         _42 = opaque::<u64>(_43) -> [return: bb12, unwind unreachable];
       }
   
       bb12: {
 -         StorageDead(_43);
++         nop;
           StorageDead(_42);
           StorageLive(_46);
           StorageLive(_47);
--         StorageLive(_48);
--         _48 = _1;
+          StorageLive(_48);
+          _48 = _1;
 -         _47 = move _48 as u32 (IntToInt);
--         StorageDead(_48);
 +         _47 = _1 as u32 (IntToInt);
+          StorageDead(_48);
           _46 = opaque::<u32>(move _47) -> [return: bb13, unwind unreachable];
       }
   
@@ -393,11 +414,11 @@
           StorageDead(_46);
           StorageLive(_49);
           StorageLive(_50);
--         StorageLive(_51);
--         _51 = _1;
+          StorageLive(_51);
+          _51 = _1;
 -         _50 = move _51 as f32 (IntToFloat);
--         StorageDead(_51);
 +         _50 = _1 as f32 (IntToFloat);
+          StorageDead(_51);
           _49 = opaque::<f32>(move _50) -> [return: bb14, unwind unreachable];
       }
   
@@ -406,25 +427,29 @@
           StorageDead(_49);
           StorageLive(_52);
 -         StorageLive(_53);
--         StorageLive(_54);
--         _54 = _1;
++         nop;
+          StorageLive(_54);
+          _54 = _1;
 -         _53 = S::<u64>(move _54);
--         StorageDead(_54);
--         _52 = opaque::<S<u64>>(move _53) -> [return: bb15, unwind unreachable];
 +         _53 = S::<u64>(_1);
+          StorageDead(_54);
+-         _52 = opaque::<S<u64>>(move _53) -> [return: bb15, unwind unreachable];
 +         _52 = opaque::<S<u64>>(_53) -> [return: bb15, unwind unreachable];
       }
   
       bb15: {
 -         StorageDead(_53);
++         nop;
           StorageDead(_52);
           StorageLive(_55);
 -         StorageLive(_56);
--         StorageLive(_57);
--         StorageLive(_58);
--         _58 = _1;
++         nop;
+          StorageLive(_57);
+          StorageLive(_58);
+          _58 = _1;
 -         _57 = S::<u64>(move _58);
--         StorageDead(_58);
++         _57 = _53;
+          StorageDead(_58);
 -         _56 = (_57.0: u64);
 -         _55 = opaque::<u64>(move _56) -> [return: bb16, unwind unreachable];
 +         _56 = (_53.0: u64);
@@ -433,24 +458,26 @@
   
       bb16: {
 -         StorageDead(_56);
--         StorageDead(_57);
++         nop;
+          StorageDead(_57);
           StorageDead(_55);
           StorageLive(_59);
           StorageLive(_60);
--         StorageLive(_61);
--         StorageLive(_62);
--         _62 = _1;
--         StorageLive(_63);
--         _63 = _2;
+          StorageLive(_61);
+          StorageLive(_62);
+          _62 = _1;
+          StorageLive(_63);
+          _63 = _2;
 -         _61 = Add(move _62, move _63);
--         StorageDead(_63);
--         StorageDead(_62);
++         _61 = _5;
+          StorageDead(_63);
+          StorageDead(_62);
           StorageLive(_64);
           _64 = _3;
 -         _60 = Add(move _61, move _64);
 +         _60 = Add(_5, move _64);
           StorageDead(_64);
--         StorageDead(_61);
+          StorageDead(_61);
           _59 = opaque::<u64>(move _60) -> [return: bb17, unwind unreachable];
       }
   
@@ -459,20 +486,21 @@
           StorageDead(_59);
           StorageLive(_65);
           StorageLive(_66);
--         StorageLive(_67);
--         StorageLive(_68);
--         _68 = _1;
--         StorageLive(_69);
--         _69 = _2;
+          StorageLive(_67);
+          StorageLive(_68);
+          _68 = _1;
+          StorageLive(_69);
+          _69 = _2;
 -         _67 = Mul(move _68, move _69);
--         StorageDead(_69);
--         StorageDead(_68);
++         _67 = _9;
+          StorageDead(_69);
+          StorageDead(_68);
           StorageLive(_70);
           _70 = _3;
 -         _66 = Add(move _67, move _70);
 +         _66 = Add(_9, move _70);
           StorageDead(_70);
--         StorageDead(_67);
+          StorageDead(_67);
           _65 = opaque::<u64>(move _66) -> [return: bb18, unwind unreachable];
       }
   
@@ -481,20 +509,21 @@
           StorageDead(_65);
           StorageLive(_71);
           StorageLive(_72);
--         StorageLive(_73);
--         StorageLive(_74);
--         _74 = _1;
--         StorageLive(_75);
--         _75 = _2;
+          StorageLive(_73);
+          StorageLive(_74);
+          _74 = _1;
+          StorageLive(_75);
+          _75 = _2;
 -         _73 = Sub(move _74, move _75);
--         StorageDead(_75);
--         StorageDead(_74);
++         _73 = _13;
+          StorageDead(_75);
+          StorageDead(_74);
           StorageLive(_76);
           _76 = _3;
 -         _72 = Add(move _73, move _76);
 +         _72 = Add(_13, move _76);
           StorageDead(_76);
--         StorageDead(_73);
+          StorageDead(_73);
           _71 = opaque::<u64>(move _72) -> [return: bb19, unwind unreachable];
       }
   
@@ -503,26 +532,28 @@
           StorageDead(_71);
           StorageLive(_77);
           StorageLive(_78);
--         StorageLive(_79);
--         StorageLive(_80);
--         _80 = _1;
--         StorageLive(_81);
--         _81 = _2;
+          StorageLive(_79);
+          StorageLive(_80);
+          _80 = _1;
+          StorageLive(_81);
+          _81 = _2;
 -         _82 = Eq(_81, const 0_u64);
 -         assert(!move _82, "attempt to divide `{}` by zero", _80) -> [success: bb20, unwind unreachable];
++         _82 = _20;
 +         assert(!_20, "attempt to divide `{}` by zero", _1) -> [success: bb20, unwind unreachable];
       }
   
       bb20: {
 -         _79 = Div(move _80, move _81);
--         StorageDead(_81);
--         StorageDead(_80);
++         _79 = _17;
+          StorageDead(_81);
+          StorageDead(_80);
           StorageLive(_83);
           _83 = _3;
 -         _78 = Add(move _79, move _83);
 +         _78 = Add(_17, move _83);
           StorageDead(_83);
--         StorageDead(_79);
+          StorageDead(_79);
           _77 = opaque::<u64>(move _78) -> [return: bb21, unwind unreachable];
       }
   
@@ -531,26 +562,28 @@
           StorageDead(_77);
           StorageLive(_84);
           StorageLive(_85);
--         StorageLive(_86);
--         StorageLive(_87);
--         _87 = _1;
--         StorageLive(_88);
--         _88 = _2;
+          StorageLive(_86);
+          StorageLive(_87);
+          _87 = _1;
+          StorageLive(_88);
+          _88 = _2;
 -         _89 = Eq(_88, const 0_u64);
 -         assert(!move _89, "attempt to calculate the remainder of `{}` with a divisor of zero", _87) -> [success: bb22, unwind unreachable];
++         _89 = _20;
 +         assert(!_20, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb22, unwind unreachable];
       }
   
       bb22: {
 -         _86 = Rem(move _87, move _88);
--         StorageDead(_88);
--         StorageDead(_87);
++         _86 = _22;
+          StorageDead(_88);
+          StorageDead(_87);
           StorageLive(_90);
           _90 = _3;
 -         _85 = Add(move _86, move _90);
 +         _85 = Add(_22, move _90);
           StorageDead(_90);
--         StorageDead(_86);
+          StorageDead(_86);
           _84 = opaque::<u64>(move _85) -> [return: bb23, unwind unreachable];
       }
   
@@ -559,20 +592,21 @@
           StorageDead(_84);
           StorageLive(_91);
           StorageLive(_92);
--         StorageLive(_93);
--         StorageLive(_94);
--         _94 = _1;
--         StorageLive(_95);
--         _95 = _2;
+          StorageLive(_93);
+          StorageLive(_94);
+          _94 = _1;
+          StorageLive(_95);
+          _95 = _2;
 -         _93 = BitAnd(move _94, move _95);
--         StorageDead(_95);
--         StorageDead(_94);
++         _93 = _27;
+          StorageDead(_95);
+          StorageDead(_94);
           StorageLive(_96);
           _96 = _3;
 -         _92 = Add(move _93, move _96);
 +         _92 = Add(_27, move _96);
           StorageDead(_96);
--         StorageDead(_93);
+          StorageDead(_93);
           _91 = opaque::<u64>(move _92) -> [return: bb24, unwind unreachable];
       }
   
@@ -581,20 +615,21 @@
           StorageDead(_91);
           StorageLive(_97);
           StorageLive(_98);
--         StorageLive(_99);
--         StorageLive(_100);
--         _100 = _1;
--         StorageLive(_101);
--         _101 = _2;
+          StorageLive(_99);
+          StorageLive(_100);
+          _100 = _1;
+          StorageLive(_101);
+          _101 = _2;
 -         _99 = BitOr(move _100, move _101);
--         StorageDead(_101);
--         StorageDead(_100);
++         _99 = _31;
+          StorageDead(_101);
+          StorageDead(_100);
           StorageLive(_102);
           _102 = _3;
 -         _98 = Add(move _99, move _102);
 +         _98 = Add(_31, move _102);
           StorageDead(_102);
--         StorageDead(_99);
+          StorageDead(_99);
           _97 = opaque::<u64>(move _98) -> [return: bb25, unwind unreachable];
       }
   
@@ -603,20 +638,21 @@
           StorageDead(_97);
           StorageLive(_103);
           StorageLive(_104);
--         StorageLive(_105);
--         StorageLive(_106);
--         _106 = _1;
--         StorageLive(_107);
--         _107 = _2;
+          StorageLive(_105);
+          StorageLive(_106);
+          _106 = _1;
+          StorageLive(_107);
+          _107 = _2;
 -         _105 = BitXor(move _106, move _107);
--         StorageDead(_107);
--         StorageDead(_106);
++         _105 = _35;
+          StorageDead(_107);
+          StorageDead(_106);
           StorageLive(_108);
           _108 = _3;
 -         _104 = Add(move _105, move _108);
 +         _104 = Add(_35, move _108);
           StorageDead(_108);
--         StorageDead(_105);
+          StorageDead(_105);
           _103 = opaque::<u64>(move _104) -> [return: bb26, unwind unreachable];
       }
   
@@ -625,20 +661,21 @@
           StorageDead(_103);
           StorageLive(_109);
           StorageLive(_110);
--         StorageLive(_111);
--         StorageLive(_112);
--         _112 = _1;
--         StorageLive(_113);
--         _113 = _2;
+          StorageLive(_111);
+          StorageLive(_112);
+          _112 = _1;
+          StorageLive(_113);
+          _113 = _2;
 -         _111 = Shl(move _112, move _113);
--         StorageDead(_113);
--         StorageDead(_112);
++         _111 = _39;
+          StorageDead(_113);
+          StorageDead(_112);
           StorageLive(_114);
           _114 = _3;
 -         _110 = Add(move _111, move _114);
 +         _110 = Add(_39, move _114);
           StorageDead(_114);
--         StorageDead(_111);
+          StorageDead(_111);
           _109 = opaque::<u64>(move _110) -> [return: bb27, unwind unreachable];
       }
   
@@ -647,20 +684,21 @@
           StorageDead(_109);
           StorageLive(_115);
           StorageLive(_116);
--         StorageLive(_117);
--         StorageLive(_118);
--         _118 = _1;
--         StorageLive(_119);
--         _119 = _2;
+          StorageLive(_117);
+          StorageLive(_118);
+          _118 = _1;
+          StorageLive(_119);
+          _119 = _2;
 -         _117 = Shr(move _118, move _119);
--         StorageDead(_119);
--         StorageDead(_118);
++         _117 = _43;
+          StorageDead(_119);
+          StorageDead(_118);
           StorageLive(_120);
           _120 = _3;
 -         _116 = Add(move _117, move _120);
 +         _116 = Add(_43, move _120);
           StorageDead(_120);
--         StorageDead(_117);
+          StorageDead(_117);
           _115 = opaque::<u64>(move _116) -> [return: bb28, unwind unreachable];
       }
   
@@ -668,68 +706,77 @@
           StorageDead(_116);
           StorageDead(_115);
           StorageLive(_121);
--         StorageLive(_122);
--         StorageLive(_123);
--         _123 = _1;
+          StorageLive(_122);
+          StorageLive(_123);
+          _123 = _1;
 -         _122 = S::<u64>(move _123);
--         StorageDead(_123);
++         _122 = _53;
+          StorageDead(_123);
 -         _121 = opaque::<S<u64>>(move _122) -> [return: bb29, unwind unreachable];
 +         _121 = opaque::<S<u64>>(_53) -> [return: bb29, unwind unreachable];
       }
   
       bb29: {
--         StorageDead(_122);
+          StorageDead(_122);
           StorageDead(_121);
           StorageLive(_124);
--         StorageLive(_125);
--         StorageLive(_126);
--         StorageLive(_127);
--         _127 = _1;
+          StorageLive(_125);
+          StorageLive(_126);
+          StorageLive(_127);
+          _127 = _1;
 -         _126 = S::<u64>(move _127);
--         StorageDead(_127);
++         _126 = _53;
+          StorageDead(_127);
 -         _125 = (_126.0: u64);
 -         _124 = opaque::<u64>(move _125) -> [return: bb30, unwind unreachable];
++         _125 = _56;
 +         _124 = opaque::<u64>(_56) -> [return: bb30, unwind unreachable];
       }
   
       bb30: {
--         StorageDead(_125);
--         StorageDead(_126);
+          StorageDead(_125);
+          StorageDead(_126);
           StorageDead(_124);
           StorageLive(_128);
           _128 = &_3;
           StorageLive(_129);
 -         StorageLive(_130);
 -         StorageLive(_131);
++         nop;
++         nop;
           _131 = (*_128);
--         StorageLive(_132);
--         _132 = _1;
+          StorageLive(_132);
+          _132 = _1;
 -         _130 = Add(move _131, move _132);
--         StorageDead(_132);
++         _130 = Add(_131, _1);
+          StorageDead(_132);
 -         StorageDead(_131);
 -         _129 = opaque::<u64>(move _130) -> [return: bb31, unwind unreachable];
-+         _130 = Add(_131, _1);
++         nop;
 +         _129 = opaque::<u64>(_130) -> [return: bb31, unwind unreachable];
       }
   
       bb31: {
 -         StorageDead(_130);
++         nop;
           StorageDead(_129);
           StorageLive(_133);
--         StorageLive(_134);
--         StorageLive(_135);
+          StorageLive(_134);
+          StorageLive(_135);
 -         _135 = (*_128);
--         StorageLive(_136);
--         _136 = _1;
++         _135 = _131;
+          StorageLive(_136);
+          _136 = _1;
 -         _134 = Add(move _135, move _136);
--         StorageDead(_136);
--         StorageDead(_135);
++         _134 = _130;
+          StorageDead(_136);
+          StorageDead(_135);
 -         _133 = opaque::<u64>(move _134) -> [return: bb32, unwind unreachable];
 +         _133 = opaque::<u64>(_130) -> [return: bb32, unwind unreachable];
       }
   
       bb32: {
--         StorageDead(_134);
+          StorageDead(_134);
           StorageDead(_133);
           StorageLive(_137);
           _137 = &mut _3;
@@ -737,11 +784,11 @@
           StorageLive(_139);
           StorageLive(_140);
           _140 = (*_137);
--         StorageLive(_141);
--         _141 = _1;
+          StorageLive(_141);
+          _141 = _1;
 -         _139 = Add(move _140, move _141);
--         StorageDead(_141);
 +         _139 = Add(move _140, _1);
+          StorageDead(_141);
           StorageDead(_140);
           _138 = opaque::<u64>(move _139) -> [return: bb33, unwind unreachable];
       }
@@ -753,11 +800,11 @@
           StorageLive(_143);
           StorageLive(_144);
           _144 = (*_137);
--         StorageLive(_145);
--         _145 = _1;
+          StorageLive(_145);
+          _145 = _1;
 -         _143 = Add(move _144, move _145);
--         StorageDead(_145);
 +         _143 = Add(move _144, _1);
+          StorageDead(_145);
           StorageDead(_144);
           _142 = opaque::<u64>(move _143) -> [return: bb34, unwind unreachable];
       }
@@ -765,18 +812,18 @@
       bb34: {
           StorageDead(_143);
           StorageDead(_142);
--         StorageLive(_146);
+          StorageLive(_146);
           StorageLive(_147);
           _147 = &raw const _3;
           StorageLive(_148);
           StorageLive(_149);
           StorageLive(_150);
           _150 = (*_147);
--         StorageLive(_151);
--         _151 = _1;
+          StorageLive(_151);
+          _151 = _1;
 -         _149 = Add(move _150, move _151);
--         StorageDead(_151);
 +         _149 = Add(move _150, _1);
+          StorageDead(_151);
           StorageDead(_150);
           _148 = opaque::<u64>(move _149) -> [return: bb35, unwind unreachable];
       }
@@ -788,11 +835,11 @@
           StorageLive(_153);
           StorageLive(_154);
           _154 = (*_147);
--         StorageLive(_155);
--         _155 = _1;
+          StorageLive(_155);
+          _155 = _1;
 -         _153 = Add(move _154, move _155);
--         StorageDead(_155);
 +         _153 = Add(move _154, _1);
+          StorageDead(_155);
           StorageDead(_154);
           _152 = opaque::<u64>(move _153) -> [return: bb36, unwind unreachable];
       }
@@ -806,11 +853,11 @@
           StorageLive(_158);
           StorageLive(_159);
           _159 = (*_156);
--         StorageLive(_160);
--         _160 = _1;
+          StorageLive(_160);
+          _160 = _1;
 -         _158 = Add(move _159, move _160);
--         StorageDead(_160);
 +         _158 = Add(move _159, _1);
+          StorageDead(_160);
           StorageDead(_159);
           _157 = opaque::<u64>(move _158) -> [return: bb37, unwind unreachable];
       }
@@ -822,11 +869,11 @@
           StorageLive(_162);
           StorageLive(_163);
           _163 = (*_156);
--         StorageLive(_164);
--         _164 = _1;
+          StorageLive(_164);
+          _164 = _1;
 -         _162 = Add(move _163, move _164);
--         StorageDead(_164);
 +         _162 = Add(move _163, _1);
+          StorageDead(_164);
           StorageDead(_163);
           _161 = opaque::<u64>(move _162) -> [return: bb38, unwind unreachable];
       }
@@ -834,44 +881,50 @@
       bb38: {
           StorageDead(_162);
           StorageDead(_161);
--         _146 = const ();
+          _146 = const ();
           StorageDead(_156);
           StorageDead(_147);
--         StorageDead(_146);
+          StorageDead(_146);
           StorageLive(_165);
           _165 = &_3;
           StorageLive(_166);
 -         StorageLive(_167);
 -         StorageLive(_168);
++         nop;
++         nop;
           _168 = (*_165);
--         StorageLive(_169);
--         _169 = _1;
+          StorageLive(_169);
+          _169 = _1;
 -         _167 = Add(move _168, move _169);
--         StorageDead(_169);
++         _167 = Add(_168, _1);
+          StorageDead(_169);
 -         StorageDead(_168);
 -         _166 = opaque::<u64>(move _167) -> [return: bb39, unwind unreachable];
-+         _167 = Add(_168, _1);
++         nop;
 +         _166 = opaque::<u64>(_167) -> [return: bb39, unwind unreachable];
       }
   
       bb39: {
 -         StorageDead(_167);
++         nop;
           StorageDead(_166);
           StorageLive(_170);
--         StorageLive(_171);
--         StorageLive(_172);
+          StorageLive(_171);
+          StorageLive(_172);
 -         _172 = (*_165);
--         StorageLive(_173);
--         _173 = _1;
++         _172 = _168;
+          StorageLive(_173);
+          _173 = _1;
 -         _171 = Add(move _172, move _173);
--         StorageDead(_173);
--         StorageDead(_172);
++         _171 = _167;
+          StorageDead(_173);
+          StorageDead(_172);
 -         _170 = opaque::<u64>(move _171) -> [return: bb40, unwind unreachable];
 +         _170 = opaque::<u64>(_167) -> [return: bb40, unwind unreachable];
       }
   
       bb40: {
--         StorageDead(_171);
+          StorageDead(_171);
           StorageDead(_170);
           _0 = const ();
           StorageDead(_165);
diff --git a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff
index 68b052907192..fb2c089827de 100644
--- a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff
@@ -197,61 +197,68 @@
       bb0: {
           StorageLive(_4);
 -         StorageLive(_5);
--         StorageLive(_6);
--         _6 = _1;
--         StorageLive(_7);
--         _7 = _2;
++         nop;
+          StorageLive(_6);
+          _6 = _1;
+          StorageLive(_7);
+          _7 = _2;
 -         _5 = Add(move _6, move _7);
--         StorageDead(_7);
--         StorageDead(_6);
--         _4 = opaque::<u64>(move _5) -> [return: bb1, unwind continue];
 +         _5 = Add(_1, _2);
+          StorageDead(_7);
+          StorageDead(_6);
+-         _4 = opaque::<u64>(move _5) -> [return: bb1, unwind continue];
 +         _4 = opaque::<u64>(_5) -> [return: bb1, unwind continue];
       }
   
       bb1: {
 -         StorageDead(_5);
++         nop;
           StorageDead(_4);
           StorageLive(_8);
 -         StorageLive(_9);
--         StorageLive(_10);
--         _10 = _1;
--         StorageLive(_11);
--         _11 = _2;
++         nop;
+          StorageLive(_10);
+          _10 = _1;
+          StorageLive(_11);
+          _11 = _2;
 -         _9 = Mul(move _10, move _11);
--         StorageDead(_11);
--         StorageDead(_10);
--         _8 = opaque::<u64>(move _9) -> [return: bb2, unwind continue];
 +         _9 = Mul(_1, _2);
+          StorageDead(_11);
+          StorageDead(_10);
+-         _8 = opaque::<u64>(move _9) -> [return: bb2, unwind continue];
 +         _8 = opaque::<u64>(_9) -> [return: bb2, unwind continue];
       }
   
       bb2: {
 -         StorageDead(_9);
++         nop;
           StorageDead(_8);
           StorageLive(_12);
 -         StorageLive(_13);
--         StorageLive(_14);
--         _14 = _1;
--         StorageLive(_15);
--         _15 = _2;
++         nop;
+          StorageLive(_14);
+          _14 = _1;
+          StorageLive(_15);
+          _15 = _2;
 -         _13 = Sub(move _14, move _15);
--         StorageDead(_15);
--         StorageDead(_14);
--         _12 = opaque::<u64>(move _13) -> [return: bb3, unwind continue];
 +         _13 = Sub(_1, _2);
+          StorageDead(_15);
+          StorageDead(_14);
+-         _12 = opaque::<u64>(move _13) -> [return: bb3, unwind continue];
 +         _12 = opaque::<u64>(_13) -> [return: bb3, unwind continue];
       }
   
       bb3: {
 -         StorageDead(_13);
++         nop;
           StorageDead(_12);
           StorageLive(_16);
 -         StorageLive(_17);
--         StorageLive(_18);
--         _18 = _1;
--         StorageLive(_19);
--         _19 = _2;
++         nop;
+          StorageLive(_18);
+          _18 = _1;
+          StorageLive(_19);
+          _19 = _2;
 -         _20 = Eq(_19, const 0_u64);
 -         assert(!move _20, "attempt to divide `{}` by zero", _18) -> [success: bb4, unwind continue];
 +         _20 = Eq(_2, const 0_u64);
@@ -260,131 +267,145 @@
   
       bb4: {
 -         _17 = Div(move _18, move _19);
--         StorageDead(_19);
--         StorageDead(_18);
--         _16 = opaque::<u64>(move _17) -> [return: bb5, unwind continue];
 +         _17 = Div(_1, _2);
+          StorageDead(_19);
+          StorageDead(_18);
+-         _16 = opaque::<u64>(move _17) -> [return: bb5, unwind continue];
 +         _16 = opaque::<u64>(_17) -> [return: bb5, unwind continue];
       }
   
       bb5: {
 -         StorageDead(_17);
++         nop;
           StorageDead(_16);
           StorageLive(_21);
 -         StorageLive(_22);
--         StorageLive(_23);
--         _23 = _1;
--         StorageLive(_24);
--         _24 = _2;
++         nop;
+          StorageLive(_23);
+          _23 = _1;
+          StorageLive(_24);
+          _24 = _2;
 -         _25 = Eq(_24, const 0_u64);
 -         assert(!move _25, "attempt to calculate the remainder of `{}` with a divisor of zero", _23) -> [success: bb6, unwind continue];
++         _25 = _20;
 +         assert(!_20, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb6, unwind continue];
       }
   
       bb6: {
 -         _22 = Rem(move _23, move _24);
--         StorageDead(_24);
--         StorageDead(_23);
--         _21 = opaque::<u64>(move _22) -> [return: bb7, unwind continue];
 +         _22 = Rem(_1, _2);
+          StorageDead(_24);
+          StorageDead(_23);
+-         _21 = opaque::<u64>(move _22) -> [return: bb7, unwind continue];
 +         _21 = opaque::<u64>(_22) -> [return: bb7, unwind continue];
       }
   
       bb7: {
 -         StorageDead(_22);
++         nop;
           StorageDead(_21);
           StorageLive(_26);
 -         StorageLive(_27);
--         StorageLive(_28);
--         _28 = _1;
--         StorageLive(_29);
--         _29 = _2;
++         nop;
+          StorageLive(_28);
+          _28 = _1;
+          StorageLive(_29);
+          _29 = _2;
 -         _27 = BitAnd(move _28, move _29);
--         StorageDead(_29);
--         StorageDead(_28);
--         _26 = opaque::<u64>(move _27) -> [return: bb8, unwind continue];
 +         _27 = BitAnd(_1, _2);
+          StorageDead(_29);
+          StorageDead(_28);
+-         _26 = opaque::<u64>(move _27) -> [return: bb8, unwind continue];
 +         _26 = opaque::<u64>(_27) -> [return: bb8, unwind continue];
       }
   
       bb8: {
 -         StorageDead(_27);
++         nop;
           StorageDead(_26);
           StorageLive(_30);
 -         StorageLive(_31);
--         StorageLive(_32);
--         _32 = _1;
--         StorageLive(_33);
--         _33 = _2;
++         nop;
+          StorageLive(_32);
+          _32 = _1;
+          StorageLive(_33);
+          _33 = _2;
 -         _31 = BitOr(move _32, move _33);
--         StorageDead(_33);
--         StorageDead(_32);
--         _30 = opaque::<u64>(move _31) -> [return: bb9, unwind continue];
 +         _31 = BitOr(_1, _2);
+          StorageDead(_33);
+          StorageDead(_32);
+-         _30 = opaque::<u64>(move _31) -> [return: bb9, unwind continue];
 +         _30 = opaque::<u64>(_31) -> [return: bb9, unwind continue];
       }
   
       bb9: {
 -         StorageDead(_31);
++         nop;
           StorageDead(_30);
           StorageLive(_34);
 -         StorageLive(_35);
--         StorageLive(_36);
--         _36 = _1;
--         StorageLive(_37);
--         _37 = _2;
++         nop;
+          StorageLive(_36);
+          _36 = _1;
+          StorageLive(_37);
+          _37 = _2;
 -         _35 = BitXor(move _36, move _37);
--         StorageDead(_37);
--         StorageDead(_36);
--         _34 = opaque::<u64>(move _35) -> [return: bb10, unwind continue];
 +         _35 = BitXor(_1, _2);
+          StorageDead(_37);
+          StorageDead(_36);
+-         _34 = opaque::<u64>(move _35) -> [return: bb10, unwind continue];
 +         _34 = opaque::<u64>(_35) -> [return: bb10, unwind continue];
       }
   
       bb10: {
 -         StorageDead(_35);
++         nop;
           StorageDead(_34);
           StorageLive(_38);
 -         StorageLive(_39);
--         StorageLive(_40);
--         _40 = _1;
--         StorageLive(_41);
--         _41 = _2;
++         nop;
+          StorageLive(_40);
+          _40 = _1;
+          StorageLive(_41);
+          _41 = _2;
 -         _39 = Shl(move _40, move _41);
--         StorageDead(_41);
--         StorageDead(_40);
--         _38 = opaque::<u64>(move _39) -> [return: bb11, unwind continue];
 +         _39 = Shl(_1, _2);
+          StorageDead(_41);
+          StorageDead(_40);
+-         _38 = opaque::<u64>(move _39) -> [return: bb11, unwind continue];
 +         _38 = opaque::<u64>(_39) -> [return: bb11, unwind continue];
       }
   
       bb11: {
 -         StorageDead(_39);
++         nop;
           StorageDead(_38);
           StorageLive(_42);
 -         StorageLive(_43);
--         StorageLive(_44);
--         _44 = _1;
--         StorageLive(_45);
--         _45 = _2;
++         nop;
+          StorageLive(_44);
+          _44 = _1;
+          StorageLive(_45);
+          _45 = _2;
 -         _43 = Shr(move _44, move _45);
--         StorageDead(_45);
--         StorageDead(_44);
--         _42 = opaque::<u64>(move _43) -> [return: bb12, unwind continue];
 +         _43 = Shr(_1, _2);
+          StorageDead(_45);
+          StorageDead(_44);
+-         _42 = opaque::<u64>(move _43) -> [return: bb12, unwind continue];
 +         _42 = opaque::<u64>(_43) -> [return: bb12, unwind continue];
       }
   
       bb12: {
 -         StorageDead(_43);
++         nop;
           StorageDead(_42);
           StorageLive(_46);
           StorageLive(_47);
--         StorageLive(_48);
--         _48 = _1;
+          StorageLive(_48);
+          _48 = _1;
 -         _47 = move _48 as u32 (IntToInt);
--         StorageDead(_48);
 +         _47 = _1 as u32 (IntToInt);
+          StorageDead(_48);
           _46 = opaque::<u32>(move _47) -> [return: bb13, unwind continue];
       }
   
@@ -393,11 +414,11 @@
           StorageDead(_46);
           StorageLive(_49);
           StorageLive(_50);
--         StorageLive(_51);
--         _51 = _1;
+          StorageLive(_51);
+          _51 = _1;
 -         _50 = move _51 as f32 (IntToFloat);
--         StorageDead(_51);
 +         _50 = _1 as f32 (IntToFloat);
+          StorageDead(_51);
           _49 = opaque::<f32>(move _50) -> [return: bb14, unwind continue];
       }
   
@@ -406,25 +427,29 @@
           StorageDead(_49);
           StorageLive(_52);
 -         StorageLive(_53);
--         StorageLive(_54);
--         _54 = _1;
++         nop;
+          StorageLive(_54);
+          _54 = _1;
 -         _53 = S::<u64>(move _54);
--         StorageDead(_54);
--         _52 = opaque::<S<u64>>(move _53) -> [return: bb15, unwind continue];
 +         _53 = S::<u64>(_1);
+          StorageDead(_54);
+-         _52 = opaque::<S<u64>>(move _53) -> [return: bb15, unwind continue];
 +         _52 = opaque::<S<u64>>(_53) -> [return: bb15, unwind continue];
       }
   
       bb15: {
 -         StorageDead(_53);
++         nop;
           StorageDead(_52);
           StorageLive(_55);
 -         StorageLive(_56);
--         StorageLive(_57);
--         StorageLive(_58);
--         _58 = _1;
++         nop;
+          StorageLive(_57);
+          StorageLive(_58);
+          _58 = _1;
 -         _57 = S::<u64>(move _58);
--         StorageDead(_58);
++         _57 = _53;
+          StorageDead(_58);
 -         _56 = (_57.0: u64);
 -         _55 = opaque::<u64>(move _56) -> [return: bb16, unwind continue];
 +         _56 = (_53.0: u64);
@@ -433,24 +458,26 @@
   
       bb16: {
 -         StorageDead(_56);
--         StorageDead(_57);
++         nop;
+          StorageDead(_57);
           StorageDead(_55);
           StorageLive(_59);
           StorageLive(_60);
--         StorageLive(_61);
--         StorageLive(_62);
--         _62 = _1;
--         StorageLive(_63);
--         _63 = _2;
+          StorageLive(_61);
+          StorageLive(_62);
+          _62 = _1;
+          StorageLive(_63);
+          _63 = _2;
 -         _61 = Add(move _62, move _63);
--         StorageDead(_63);
--         StorageDead(_62);
++         _61 = _5;
+          StorageDead(_63);
+          StorageDead(_62);
           StorageLive(_64);
           _64 = _3;
 -         _60 = Add(move _61, move _64);
 +         _60 = Add(_5, move _64);
           StorageDead(_64);
--         StorageDead(_61);
+          StorageDead(_61);
           _59 = opaque::<u64>(move _60) -> [return: bb17, unwind continue];
       }
   
@@ -459,20 +486,21 @@
           StorageDead(_59);
           StorageLive(_65);
           StorageLive(_66);
--         StorageLive(_67);
--         StorageLive(_68);
--         _68 = _1;
--         StorageLive(_69);
--         _69 = _2;
+          StorageLive(_67);
+          StorageLive(_68);
+          _68 = _1;
+          StorageLive(_69);
+          _69 = _2;
 -         _67 = Mul(move _68, move _69);
--         StorageDead(_69);
--         StorageDead(_68);
++         _67 = _9;
+          StorageDead(_69);
+          StorageDead(_68);
           StorageLive(_70);
           _70 = _3;
 -         _66 = Add(move _67, move _70);
 +         _66 = Add(_9, move _70);
           StorageDead(_70);
--         StorageDead(_67);
+          StorageDead(_67);
           _65 = opaque::<u64>(move _66) -> [return: bb18, unwind continue];
       }
   
@@ -481,20 +509,21 @@
           StorageDead(_65);
           StorageLive(_71);
           StorageLive(_72);
--         StorageLive(_73);
--         StorageLive(_74);
--         _74 = _1;
--         StorageLive(_75);
--         _75 = _2;
+          StorageLive(_73);
+          StorageLive(_74);
+          _74 = _1;
+          StorageLive(_75);
+          _75 = _2;
 -         _73 = Sub(move _74, move _75);
--         StorageDead(_75);
--         StorageDead(_74);
++         _73 = _13;
+          StorageDead(_75);
+          StorageDead(_74);
           StorageLive(_76);
           _76 = _3;
 -         _72 = Add(move _73, move _76);
 +         _72 = Add(_13, move _76);
           StorageDead(_76);
--         StorageDead(_73);
+          StorageDead(_73);
           _71 = opaque::<u64>(move _72) -> [return: bb19, unwind continue];
       }
   
@@ -503,26 +532,28 @@
           StorageDead(_71);
           StorageLive(_77);
           StorageLive(_78);
--         StorageLive(_79);
--         StorageLive(_80);
--         _80 = _1;
--         StorageLive(_81);
--         _81 = _2;
+          StorageLive(_79);
+          StorageLive(_80);
+          _80 = _1;
+          StorageLive(_81);
+          _81 = _2;
 -         _82 = Eq(_81, const 0_u64);
 -         assert(!move _82, "attempt to divide `{}` by zero", _80) -> [success: bb20, unwind continue];
++         _82 = _20;
 +         assert(!_20, "attempt to divide `{}` by zero", _1) -> [success: bb20, unwind continue];
       }
   
       bb20: {
 -         _79 = Div(move _80, move _81);
--         StorageDead(_81);
--         StorageDead(_80);
++         _79 = _17;
+          StorageDead(_81);
+          StorageDead(_80);
           StorageLive(_83);
           _83 = _3;
 -         _78 = Add(move _79, move _83);
 +         _78 = Add(_17, move _83);
           StorageDead(_83);
--         StorageDead(_79);
+          StorageDead(_79);
           _77 = opaque::<u64>(move _78) -> [return: bb21, unwind continue];
       }
   
@@ -531,26 +562,28 @@
           StorageDead(_77);
           StorageLive(_84);
           StorageLive(_85);
--         StorageLive(_86);
--         StorageLive(_87);
--         _87 = _1;
--         StorageLive(_88);
--         _88 = _2;
+          StorageLive(_86);
+          StorageLive(_87);
+          _87 = _1;
+          StorageLive(_88);
+          _88 = _2;
 -         _89 = Eq(_88, const 0_u64);
 -         assert(!move _89, "attempt to calculate the remainder of `{}` with a divisor of zero", _87) -> [success: bb22, unwind continue];
++         _89 = _20;
 +         assert(!_20, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb22, unwind continue];
       }
   
       bb22: {
 -         _86 = Rem(move _87, move _88);
--         StorageDead(_88);
--         StorageDead(_87);
++         _86 = _22;
+          StorageDead(_88);
+          StorageDead(_87);
           StorageLive(_90);
           _90 = _3;
 -         _85 = Add(move _86, move _90);
 +         _85 = Add(_22, move _90);
           StorageDead(_90);
--         StorageDead(_86);
+          StorageDead(_86);
           _84 = opaque::<u64>(move _85) -> [return: bb23, unwind continue];
       }
   
@@ -559,20 +592,21 @@
           StorageDead(_84);
           StorageLive(_91);
           StorageLive(_92);
--         StorageLive(_93);
--         StorageLive(_94);
--         _94 = _1;
--         StorageLive(_95);
--         _95 = _2;
+          StorageLive(_93);
+          StorageLive(_94);
+          _94 = _1;
+          StorageLive(_95);
+          _95 = _2;
 -         _93 = BitAnd(move _94, move _95);
--         StorageDead(_95);
--         StorageDead(_94);
++         _93 = _27;
+          StorageDead(_95);
+          StorageDead(_94);
           StorageLive(_96);
           _96 = _3;
 -         _92 = Add(move _93, move _96);
 +         _92 = Add(_27, move _96);
           StorageDead(_96);
--         StorageDead(_93);
+          StorageDead(_93);
           _91 = opaque::<u64>(move _92) -> [return: bb24, unwind continue];
       }
   
@@ -581,20 +615,21 @@
           StorageDead(_91);
           StorageLive(_97);
           StorageLive(_98);
--         StorageLive(_99);
--         StorageLive(_100);
--         _100 = _1;
--         StorageLive(_101);
--         _101 = _2;
+          StorageLive(_99);
+          StorageLive(_100);
+          _100 = _1;
+          StorageLive(_101);
+          _101 = _2;
 -         _99 = BitOr(move _100, move _101);
--         StorageDead(_101);
--         StorageDead(_100);
++         _99 = _31;
+          StorageDead(_101);
+          StorageDead(_100);
           StorageLive(_102);
           _102 = _3;
 -         _98 = Add(move _99, move _102);
 +         _98 = Add(_31, move _102);
           StorageDead(_102);
--         StorageDead(_99);
+          StorageDead(_99);
           _97 = opaque::<u64>(move _98) -> [return: bb25, unwind continue];
       }
   
@@ -603,20 +638,21 @@
           StorageDead(_97);
           StorageLive(_103);
           StorageLive(_104);
--         StorageLive(_105);
--         StorageLive(_106);
--         _106 = _1;
--         StorageLive(_107);
--         _107 = _2;
+          StorageLive(_105);
+          StorageLive(_106);
+          _106 = _1;
+          StorageLive(_107);
+          _107 = _2;
 -         _105 = BitXor(move _106, move _107);
--         StorageDead(_107);
--         StorageDead(_106);
++         _105 = _35;
+          StorageDead(_107);
+          StorageDead(_106);
           StorageLive(_108);
           _108 = _3;
 -         _104 = Add(move _105, move _108);
 +         _104 = Add(_35, move _108);
           StorageDead(_108);
--         StorageDead(_105);
+          StorageDead(_105);
           _103 = opaque::<u64>(move _104) -> [return: bb26, unwind continue];
       }
   
@@ -625,20 +661,21 @@
           StorageDead(_103);
           StorageLive(_109);
           StorageLive(_110);
--         StorageLive(_111);
--         StorageLive(_112);
--         _112 = _1;
--         StorageLive(_113);
--         _113 = _2;
+          StorageLive(_111);
+          StorageLive(_112);
+          _112 = _1;
+          StorageLive(_113);
+          _113 = _2;
 -         _111 = Shl(move _112, move _113);
--         StorageDead(_113);
--         StorageDead(_112);
++         _111 = _39;
+          StorageDead(_113);
+          StorageDead(_112);
           StorageLive(_114);
           _114 = _3;
 -         _110 = Add(move _111, move _114);
 +         _110 = Add(_39, move _114);
           StorageDead(_114);
--         StorageDead(_111);
+          StorageDead(_111);
           _109 = opaque::<u64>(move _110) -> [return: bb27, unwind continue];
       }
   
@@ -647,20 +684,21 @@
           StorageDead(_109);
           StorageLive(_115);
           StorageLive(_116);
--         StorageLive(_117);
--         StorageLive(_118);
--         _118 = _1;
--         StorageLive(_119);
--         _119 = _2;
+          StorageLive(_117);
+          StorageLive(_118);
+          _118 = _1;
+          StorageLive(_119);
+          _119 = _2;
 -         _117 = Shr(move _118, move _119);
--         StorageDead(_119);
--         StorageDead(_118);
++         _117 = _43;
+          StorageDead(_119);
+          StorageDead(_118);
           StorageLive(_120);
           _120 = _3;
 -         _116 = Add(move _117, move _120);
 +         _116 = Add(_43, move _120);
           StorageDead(_120);
--         StorageDead(_117);
+          StorageDead(_117);
           _115 = opaque::<u64>(move _116) -> [return: bb28, unwind continue];
       }
   
@@ -668,68 +706,77 @@
           StorageDead(_116);
           StorageDead(_115);
           StorageLive(_121);
--         StorageLive(_122);
--         StorageLive(_123);
--         _123 = _1;
+          StorageLive(_122);
+          StorageLive(_123);
+          _123 = _1;
 -         _122 = S::<u64>(move _123);
--         StorageDead(_123);
++         _122 = _53;
+          StorageDead(_123);
 -         _121 = opaque::<S<u64>>(move _122) -> [return: bb29, unwind continue];
 +         _121 = opaque::<S<u64>>(_53) -> [return: bb29, unwind continue];
       }
   
       bb29: {
--         StorageDead(_122);
+          StorageDead(_122);
           StorageDead(_121);
           StorageLive(_124);
--         StorageLive(_125);
--         StorageLive(_126);
--         StorageLive(_127);
--         _127 = _1;
+          StorageLive(_125);
+          StorageLive(_126);
+          StorageLive(_127);
+          _127 = _1;
 -         _126 = S::<u64>(move _127);
--         StorageDead(_127);
++         _126 = _53;
+          StorageDead(_127);
 -         _125 = (_126.0: u64);
 -         _124 = opaque::<u64>(move _125) -> [return: bb30, unwind continue];
++         _125 = _56;
 +         _124 = opaque::<u64>(_56) -> [return: bb30, unwind continue];
       }
   
       bb30: {
--         StorageDead(_125);
--         StorageDead(_126);
+          StorageDead(_125);
+          StorageDead(_126);
           StorageDead(_124);
           StorageLive(_128);
           _128 = &_3;
           StorageLive(_129);
 -         StorageLive(_130);
 -         StorageLive(_131);
++         nop;
++         nop;
           _131 = (*_128);
--         StorageLive(_132);
--         _132 = _1;
+          StorageLive(_132);
+          _132 = _1;
 -         _130 = Add(move _131, move _132);
--         StorageDead(_132);
++         _130 = Add(_131, _1);
+          StorageDead(_132);
 -         StorageDead(_131);
 -         _129 = opaque::<u64>(move _130) -> [return: bb31, unwind continue];
-+         _130 = Add(_131, _1);
++         nop;
 +         _129 = opaque::<u64>(_130) -> [return: bb31, unwind continue];
       }
   
       bb31: {
 -         StorageDead(_130);
++         nop;
           StorageDead(_129);
           StorageLive(_133);
--         StorageLive(_134);
--         StorageLive(_135);
+          StorageLive(_134);
+          StorageLive(_135);
 -         _135 = (*_128);
--         StorageLive(_136);
--         _136 = _1;
++         _135 = _131;
+          StorageLive(_136);
+          _136 = _1;
 -         _134 = Add(move _135, move _136);
--         StorageDead(_136);
--         StorageDead(_135);
++         _134 = _130;
+          StorageDead(_136);
+          StorageDead(_135);
 -         _133 = opaque::<u64>(move _134) -> [return: bb32, unwind continue];
 +         _133 = opaque::<u64>(_130) -> [return: bb32, unwind continue];
       }
   
       bb32: {
--         StorageDead(_134);
+          StorageDead(_134);
           StorageDead(_133);
           StorageLive(_137);
           _137 = &mut _3;
@@ -737,11 +784,11 @@
           StorageLive(_139);
           StorageLive(_140);
           _140 = (*_137);
--         StorageLive(_141);
--         _141 = _1;
+          StorageLive(_141);
+          _141 = _1;
 -         _139 = Add(move _140, move _141);
--         StorageDead(_141);
 +         _139 = Add(move _140, _1);
+          StorageDead(_141);
           StorageDead(_140);
           _138 = opaque::<u64>(move _139) -> [return: bb33, unwind continue];
       }
@@ -753,11 +800,11 @@
           StorageLive(_143);
           StorageLive(_144);
           _144 = (*_137);
--         StorageLive(_145);
--         _145 = _1;
+          StorageLive(_145);
+          _145 = _1;
 -         _143 = Add(move _144, move _145);
--         StorageDead(_145);
 +         _143 = Add(move _144, _1);
+          StorageDead(_145);
           StorageDead(_144);
           _142 = opaque::<u64>(move _143) -> [return: bb34, unwind continue];
       }
@@ -765,18 +812,18 @@
       bb34: {
           StorageDead(_143);
           StorageDead(_142);
--         StorageLive(_146);
+          StorageLive(_146);
           StorageLive(_147);
           _147 = &raw const _3;
           StorageLive(_148);
           StorageLive(_149);
           StorageLive(_150);
           _150 = (*_147);
--         StorageLive(_151);
--         _151 = _1;
+          StorageLive(_151);
+          _151 = _1;
 -         _149 = Add(move _150, move _151);
--         StorageDead(_151);
 +         _149 = Add(move _150, _1);
+          StorageDead(_151);
           StorageDead(_150);
           _148 = opaque::<u64>(move _149) -> [return: bb35, unwind continue];
       }
@@ -788,11 +835,11 @@
           StorageLive(_153);
           StorageLive(_154);
           _154 = (*_147);
--         StorageLive(_155);
--         _155 = _1;
+          StorageLive(_155);
+          _155 = _1;
 -         _153 = Add(move _154, move _155);
--         StorageDead(_155);
 +         _153 = Add(move _154, _1);
+          StorageDead(_155);
           StorageDead(_154);
           _152 = opaque::<u64>(move _153) -> [return: bb36, unwind continue];
       }
@@ -806,11 +853,11 @@
           StorageLive(_158);
           StorageLive(_159);
           _159 = (*_156);
--         StorageLive(_160);
--         _160 = _1;
+          StorageLive(_160);
+          _160 = _1;
 -         _158 = Add(move _159, move _160);
--         StorageDead(_160);
 +         _158 = Add(move _159, _1);
+          StorageDead(_160);
           StorageDead(_159);
           _157 = opaque::<u64>(move _158) -> [return: bb37, unwind continue];
       }
@@ -822,11 +869,11 @@
           StorageLive(_162);
           StorageLive(_163);
           _163 = (*_156);
--         StorageLive(_164);
--         _164 = _1;
+          StorageLive(_164);
+          _164 = _1;
 -         _162 = Add(move _163, move _164);
--         StorageDead(_164);
 +         _162 = Add(move _163, _1);
+          StorageDead(_164);
           StorageDead(_163);
           _161 = opaque::<u64>(move _162) -> [return: bb38, unwind continue];
       }
@@ -834,44 +881,50 @@
       bb38: {
           StorageDead(_162);
           StorageDead(_161);
--         _146 = const ();
+          _146 = const ();
           StorageDead(_156);
           StorageDead(_147);
--         StorageDead(_146);
+          StorageDead(_146);
           StorageLive(_165);
           _165 = &_3;
           StorageLive(_166);
 -         StorageLive(_167);
 -         StorageLive(_168);
++         nop;
++         nop;
           _168 = (*_165);
--         StorageLive(_169);
--         _169 = _1;
+          StorageLive(_169);
+          _169 = _1;
 -         _167 = Add(move _168, move _169);
--         StorageDead(_169);
++         _167 = Add(_168, _1);
+          StorageDead(_169);
 -         StorageDead(_168);
 -         _166 = opaque::<u64>(move _167) -> [return: bb39, unwind continue];
-+         _167 = Add(_168, _1);
++         nop;
 +         _166 = opaque::<u64>(_167) -> [return: bb39, unwind continue];
       }
   
       bb39: {
 -         StorageDead(_167);
++         nop;
           StorageDead(_166);
           StorageLive(_170);
--         StorageLive(_171);
--         StorageLive(_172);
+          StorageLive(_171);
+          StorageLive(_172);
 -         _172 = (*_165);
--         StorageLive(_173);
--         _173 = _1;
++         _172 = _168;
+          StorageLive(_173);
+          _173 = _1;
 -         _171 = Add(move _172, move _173);
--         StorageDead(_173);
--         StorageDead(_172);
++         _171 = _167;
+          StorageDead(_173);
+          StorageDead(_172);
 -         _170 = opaque::<u64>(move _171) -> [return: bb40, unwind continue];
 +         _170 = opaque::<u64>(_167) -> [return: bb40, unwind continue];
       }
   
       bb40: {
--         StorageDead(_171);
+          StorageDead(_171);
           StorageDead(_170);
           _0 = const ();
           StorageDead(_165);
diff --git a/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-abort.diff b/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-abort.diff
index f33845502ad9..2dd51934778c 100644
--- a/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-abort.diff
@@ -15,11 +15,11 @@
   
       bb0: {
           StorageLive(_2);
--         StorageLive(_3);
--         _3 = _1;
+          StorageLive(_3);
+          _3 = _1;
 -         _2 = Option::<T>::Some(move _3);
--         StorageDead(_3);
 +         _2 = Option::<T>::Some(_1);
+          StorageDead(_3);
           _4 = discriminant(_2);
           switchInt(move _4) -> [0: bb1, 1: bb3, otherwise: bb2];
       }
@@ -35,9 +35,11 @@
   
       bb3: {
 -         StorageLive(_5);
++         nop;
           _5 = ((_2 as Some).0: T);
           _0 = _5;
 -         StorageDead(_5);
++         nop;
           StorageDead(_2);
           return;
       }
diff --git a/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-unwind.diff b/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-unwind.diff
index edc05f99fe2f..610d70576c9f 100644
--- a/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-unwind.diff
@@ -15,11 +15,11 @@
   
       bb0: {
           StorageLive(_2);
--         StorageLive(_3);
--         _3 = _1;
+          StorageLive(_3);
+          _3 = _1;
 -         _2 = Option::<T>::Some(move _3);
--         StorageDead(_3);
 +         _2 = Option::<T>::Some(_1);
+          StorageDead(_3);
           _4 = discriminant(_2);
           switchInt(move _4) -> [0: bb1, 1: bb3, otherwise: bb2];
       }
@@ -35,9 +35,11 @@
   
       bb3: {
 -         StorageLive(_5);
++         nop;
           _5 = ((_2 as Some).0: T);
           _0 = _5;
 -         StorageDead(_5);
++         nop;
           StorageDead(_2);
           return;
       }
diff --git a/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-abort.diff b/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-abort.diff
index 9d8f272abea3..b2539f391d1a 100644
--- a/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-abort.diff
+++ b/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-abort.diff
@@ -7,22 +7,18 @@
       let _2: &[T];
       let mut _3: &[T; 3];
       let _4: [T; 3];
-      let mut _5: T;
-      let mut _6: T;
-      let mut _7: T;
-      let mut _8: usize;
-      let mut _9: usize;
-      let mut _10: bool;
-      let mut _14: !;
+      let mut _5: usize;
+      let mut _6: bool;
+      let mut _10: !;
       scope 1 {
           debug v => _2;
-          let _11: &T;
-          let _12: &T;
-          let _13: &T;
+          let _7: &T;
+          let _8: &T;
+          let _9: &T;
           scope 2 {
-              debug v1 => _11;
-              debug v2 => _12;
-              debug v3 => _13;
+              debug v1 => _7;
+              debug v2 => _8;
+              debug v3 => _9;
           }
       }
   
@@ -33,26 +29,25 @@
           _3 = &_4;
           _2 = move _3 as &[T] (PointerCoercion(Unsize));
           StorageDead(_3);
-          _8 = const 3_usize;
-          _9 = const 3_usize;
-          _10 = const true;
+          _5 = const 3_usize;
+          _6 = const true;
           goto -> bb2;
       }
   
       bb1: {
-          _14 = core::panicking::panic(const "internal error: entered unreachable code") -> unwind unreachable;
+          _10 = core::panicking::panic(const "internal error: entered unreachable code") -> unwind unreachable;
       }
   
       bb2: {
-          StorageLive(_11);
-          _11 = &(*_2)[0 of 3];
-          StorageLive(_12);
-          _12 = &(*_2)[1 of 3];
-          StorageLive(_13);
-          _13 = &(*_2)[2 of 3];
-          StorageDead(_13);
-          StorageDead(_12);
-          StorageDead(_11);
+          StorageLive(_7);
+          _7 = &(*_2)[0 of 3];
+          StorageLive(_8);
+          _8 = &(*_2)[1 of 3];
+          StorageLive(_9);
+          _9 = &(*_2)[2 of 3];
+          StorageDead(_9);
+          StorageDead(_8);
+          StorageDead(_7);
           StorageDead(_4);
           return;
       }
diff --git a/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-unwind.diff b/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-unwind.diff
index 738b0b1b3e5a..ff7f12c093ce 100644
--- a/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-unwind.diff
+++ b/tests/mir-opt/issue_76432.test.SimplifyComparisonIntegral.panic-unwind.diff
@@ -7,22 +7,18 @@
       let _2: &[T];
       let mut _3: &[T; 3];
       let _4: [T; 3];
-      let mut _5: T;
-      let mut _6: T;
-      let mut _7: T;
-      let mut _8: usize;
-      let mut _9: usize;
-      let mut _10: bool;
-      let mut _14: !;
+      let mut _5: usize;
+      let mut _6: bool;
+      let mut _10: !;
       scope 1 {
           debug v => _2;
-          let _11: &T;
-          let _12: &T;
-          let _13: &T;
+          let _7: &T;
+          let _8: &T;
+          let _9: &T;
           scope 2 {
-              debug v1 => _11;
-              debug v2 => _12;
-              debug v3 => _13;
+              debug v1 => _7;
+              debug v2 => _8;
+              debug v3 => _9;
           }
       }
   
@@ -33,26 +29,25 @@
           _3 = &_4;
           _2 = move _3 as &[T] (PointerCoercion(Unsize));
           StorageDead(_3);
-          _8 = const 3_usize;
-          _9 = const 3_usize;
-          _10 = const true;
+          _5 = const 3_usize;
+          _6 = const true;
           goto -> bb2;
       }
   
       bb1: {
-          _14 = core::panicking::panic(const "internal error: entered unreachable code") -> unwind continue;
+          _10 = core::panicking::panic(const "internal error: entered unreachable code") -> unwind continue;
       }
   
       bb2: {
-          StorageLive(_11);
-          _11 = &(*_2)[0 of 3];
-          StorageLive(_12);
-          _12 = &(*_2)[1 of 3];
-          StorageLive(_13);
-          _13 = &(*_2)[2 of 3];
-          StorageDead(_13);
-          StorageDead(_12);
-          StorageDead(_11);
+          StorageLive(_7);
+          _7 = &(*_2)[0 of 3];
+          StorageLive(_8);
+          _8 = &(*_2)[1 of 3];
+          StorageLive(_9);
+          _9 = &(*_2)[2 of 3];
+          StorageDead(_9);
+          StorageDead(_8);
+          StorageDead(_7);
           StorageDead(_4);
           return;
       }
diff --git a/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-const-prop.panic-abort.diff b/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-const-prop.panic-abort.diff
index d39571583606..64a435f22457 100644
--- a/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-const-prop.panic-abort.diff
+++ b/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-const-prop.panic-abort.diff
@@ -3,18 +3,15 @@
   
   fn main() -> () {
       let mut _0: ();
-      let mut _1: bool;
-      let _2: ();
+      let _1: ();
   
       bb0: {
-          StorageLive(_1);
-          _1 = const false;
 -         switchInt(const false) -> [0: bb3, otherwise: bb1];
 +         goto -> bb3;
       }
   
       bb1: {
-          _2 = noop() -> [return: bb2, unwind unreachable];
+          _1 = noop() -> [return: bb2, unwind unreachable];
       }
   
       bb2: {
@@ -26,7 +23,6 @@
       }
   
       bb4: {
-          StorageDead(_1);
           return;
       }
   }
diff --git a/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-const-prop.panic-unwind.diff b/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-const-prop.panic-unwind.diff
index 81903c64dbd4..146e00686ed8 100644
--- a/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-const-prop.panic-unwind.diff
+++ b/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-const-prop.panic-unwind.diff
@@ -3,18 +3,15 @@
   
   fn main() -> () {
       let mut _0: ();
-      let mut _1: bool;
-      let _2: ();
+      let _1: ();
   
       bb0: {
-          StorageLive(_1);
-          _1 = const false;
 -         switchInt(const false) -> [0: bb3, otherwise: bb1];
 +         goto -> bb3;
       }
   
       bb1: {
-          _2 = noop() -> [return: bb2, unwind continue];
+          _1 = noop() -> [return: bb2, unwind continue];
       }
   
       bb2: {
@@ -26,7 +23,6 @@
       }
   
       bb4: {
-          StorageDead(_1);
           return;
       }
   }

From afd631cc0c3049d1e862fefa6fc0b778f660bad8 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 23 Sep 2023 15:08:41 +0000
Subject: [PATCH 057/297] Do not visit rvalues twice.

---
 compiler/rustc_mir_transform/src/gvn.rs | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index e7f36e5881a5..2add66ee81bb 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -484,20 +484,23 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, 'tcx> {
     }
 
     fn visit_statement(&mut self, stmt: &mut Statement<'tcx>, location: Location) {
-        self.super_statement(stmt, location);
         if let StatementKind::Assign(box (_, ref mut rvalue)) = stmt.kind
             // Do not try to simplify a constant, it's already in canonical shape.
             && !matches!(rvalue, Rvalue::Use(Operand::Constant(_)))
-            && let Some(value) = self.simplify_rvalue(rvalue, location)
         {
-            if let Some(const_) = self.try_as_constant(value) {
-                *rvalue = Rvalue::Use(Operand::Constant(Box::new(const_)));
-            } else if let Some(local) = self.try_as_local(value, location)
-                && *rvalue != Rvalue::Use(Operand::Move(local.into()))
+            if let Some(value) = self.simplify_rvalue(rvalue, location)
             {
-                *rvalue = Rvalue::Use(Operand::Copy(local.into()));
-                self.reused_locals.insert(local);
+                if let Some(const_) = self.try_as_constant(value) {
+                    *rvalue = Rvalue::Use(Operand::Constant(Box::new(const_)));
+                } else if let Some(local) = self.try_as_local(value, location)
+                    && *rvalue != Rvalue::Use(Operand::Move(local.into()))
+                {
+                    *rvalue = Rvalue::Use(Operand::Copy(local.into()));
+                    self.reused_locals.insert(local);
+                }
             }
+        } else {
+            self.super_statement(stmt, location);
         }
     }
 }

From 38c86b079866d495062632483ec33cf569cb6f27 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Tue, 19 Sep 2023 20:12:48 +0000
Subject: [PATCH 058/297] Evaluate computed values to constants.

---
 .../src/interpret/discriminant.rs             |  11 +-
 .../src/interpret/intrinsics.rs               |   2 +-
 .../rustc_const_eval/src/interpret/operand.rs |  10 +
 .../rustc_const_eval/src/interpret/step.rs    |   2 +-
 compiler/rustc_middle/src/mir/consts.rs       |  12 +
 .../src/dataflow_const_prop.rs                |   8 +-
 compiler/rustc_mir_transform/src/gvn.rs       | 296 +++++++++++++++++-
 .../gvn.arithmetic.GVN.panic-abort.diff       |  18 +-
 .../gvn.arithmetic.GVN.panic-unwind.diff      |  18 +-
 ...vn.arithmetic_checked.GVN.panic-abort.diff |  31 +-
 ...n.arithmetic_checked.GVN.panic-unwind.diff |  31 +-
 tests/mir-opt/gvn.cast.GVN.panic-abort.diff   | 135 ++++----
 tests/mir-opt/gvn.cast.GVN.panic-unwind.diff  | 135 ++++----
 tests/mir-opt/gvn.slices.GVN.panic-abort.diff |  14 +-
 .../mir-opt/gvn.slices.GVN.panic-unwind.diff  |  14 +-
 15 files changed, 546 insertions(+), 191 deletions(-)

diff --git a/compiler/rustc_const_eval/src/interpret/discriminant.rs b/compiler/rustc_const_eval/src/interpret/discriminant.rs
index 8dab45d65ee1..fd1736703743 100644
--- a/compiler/rustc_const_eval/src/interpret/discriminant.rs
+++ b/compiler/rustc_const_eval/src/interpret/discriminant.rs
@@ -1,7 +1,8 @@
 //! Functions for reading and writing discriminants of multi-variant layouts (enums and coroutines).
 
-use rustc_middle::ty::layout::{LayoutOf, PrimitiveExt, TyAndLayout};
-use rustc_middle::{mir, ty};
+use rustc_middle::mir;
+use rustc_middle::ty::layout::{LayoutOf, PrimitiveExt};
+use rustc_middle::ty::{self, Ty};
 use rustc_target::abi::{self, TagEncoding};
 use rustc_target::abi::{VariantIdx, Variants};
 
@@ -244,11 +245,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
     pub fn discriminant_for_variant(
         &self,
-        layout: TyAndLayout<'tcx>,
+        ty: Ty<'tcx>,
         variant: VariantIdx,
     ) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
-        let discr_layout = self.layout_of(layout.ty.discriminant_ty(*self.tcx))?;
-        let discr_value = match layout.ty.discriminant_for_variant(*self.tcx, variant) {
+        let discr_layout = self.layout_of(ty.discriminant_ty(*self.tcx))?;
+        let discr_value = match ty.discriminant_for_variant(*self.tcx, variant) {
             Some(discr) => {
                 // This type actually has discriminants.
                 assert_eq!(discr.ty, discr_layout.ty);
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
index c97207a61ac3..c48857fcf3cd 100644
--- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
@@ -218,7 +218,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             sym::discriminant_value => {
                 let place = self.deref_pointer(&args[0])?;
                 let variant = self.read_discriminant(&place)?;
-                let discr = self.discriminant_for_variant(place.layout, variant)?;
+                let discr = self.discriminant_for_variant(place.layout.ty, variant)?;
                 self.write_immediate(*discr, dest)?;
             }
             sym::exact_div => {
diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs
index 6716888290dc..255dd1eba97f 100644
--- a/compiler/rustc_const_eval/src/interpret/operand.rs
+++ b/compiler/rustc_const_eval/src/interpret/operand.rs
@@ -169,6 +169,16 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> {
         ImmTy { imm: val.into(), layout }
     }
 
+    #[inline]
+    pub fn from_scalar_pair(a: Scalar<Prov>, b: Scalar<Prov>, layout: TyAndLayout<'tcx>) -> Self {
+        debug_assert!(
+            matches!(layout.abi, Abi::ScalarPair(..)),
+            "`ImmTy::from_scalar_pair` on non-scalar-pair layout"
+        );
+        let imm = Immediate::ScalarPair(a, b);
+        ImmTy { imm, layout }
+    }
+
     #[inline(always)]
     pub fn from_immediate(imm: Immediate<Prov>, layout: TyAndLayout<'tcx>) -> Self {
         debug_assert!(
diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs
index 79cbda545f1f..8c34d05042b1 100644
--- a/compiler/rustc_const_eval/src/interpret/step.rs
+++ b/compiler/rustc_const_eval/src/interpret/step.rs
@@ -297,7 +297,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
             Discriminant(place) => {
                 let op = self.eval_place_to_op(place, None)?;
                 let variant = self.read_discriminant(&op)?;
-                let discr = self.discriminant_for_variant(op.layout, variant)?;
+                let discr = self.discriminant_for_variant(op.layout.ty, variant)?;
                 self.write_immediate(*discr, &dest)?;
             }
         }
diff --git a/compiler/rustc_middle/src/mir/consts.rs b/compiler/rustc_middle/src/mir/consts.rs
index 4e429f316e80..9e3ee3f2bd34 100644
--- a/compiler/rustc_middle/src/mir/consts.rs
+++ b/compiler/rustc_middle/src/mir/consts.rs
@@ -172,6 +172,18 @@ impl<'tcx> ConstValue<'tcx> {
         let end = end.try_into().unwrap();
         Some(data.inner().inspect_with_uninit_and_ptr_outside_interpreter(start..end))
     }
+
+    pub fn has_provenance(&self, tcx: TyCtxt<'tcx>, size: Size) -> bool {
+        let (alloc, start, end) = match *self {
+            ConstValue::ZeroSized | ConstValue::Scalar(Scalar::Int(_)) => return false,
+            ConstValue::Scalar(Scalar::Ptr(..)) => return true,
+            ConstValue::Slice { data, meta } => (data, Size::ZERO, Size::from_bytes(meta)),
+            ConstValue::Indirect { alloc_id, offset } => {
+                (tcx.global_alloc(alloc_id).unwrap_memory(), offset, offset + size)
+            }
+        };
+        !alloc.inner().provenance().range_empty(super::AllocRange::from(start..end), &tcx)
+    }
 }
 
 ///////////////////////////////////////////////////////////////////////////
diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
index 2c29978173ff..fd067cb234bd 100644
--- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
+++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
@@ -406,7 +406,8 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> {
                 TrackElem::Variant(idx) => self.ecx.project_downcast(op, idx).ok(),
                 TrackElem::Discriminant => {
                     let variant = self.ecx.read_discriminant(op).ok()?;
-                    let discr_value = self.ecx.discriminant_for_variant(op.layout, variant).ok()?;
+                    let discr_value =
+                        self.ecx.discriminant_for_variant(op.layout.ty, variant).ok()?;
                     Some(discr_value.into())
                 }
                 TrackElem::DerefLen => {
@@ -507,7 +508,8 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> {
             return None;
         }
         let enum_ty_layout = self.tcx.layout_of(self.param_env.and(enum_ty)).ok()?;
-        let discr_value = self.ecx.discriminant_for_variant(enum_ty_layout, variant_index).ok()?;
+        let discr_value =
+            self.ecx.discriminant_for_variant(enum_ty_layout.ty, variant_index).ok()?;
         Some(discr_value.to_scalar())
     }
 
@@ -854,7 +856,7 @@ impl<'tcx> Visitor<'tcx> for OperandCollector<'tcx, '_, '_, '_> {
     }
 }
 
-struct DummyMachine;
+pub(crate) struct DummyMachine;
 
 impl<'mir, 'tcx: 'mir> rustc_const_eval::interpret::Machine<'mir, 'tcx> for DummyMachine {
     rustc_const_eval::interpret::compile_time_machine!(<'mir, 'tcx>);
diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index 2add66ee81bb..c38ebdf512b8 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -53,18 +53,24 @@
 //! _c = *_b // replaced by _c = _a
 //! ```
 
+use rustc_const_eval::interpret::{ImmTy, InterpCx, MemPlaceMeta, OpTy, Projectable, Scalar};
 use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
 use rustc_data_structures::graph::dominators::Dominators;
 use rustc_index::bit_set::BitSet;
 use rustc_index::IndexVec;
 use rustc_macros::newtype_index;
+use rustc_middle::mir::interpret::GlobalAlloc;
 use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
-use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_target::abi::{VariantIdx, FIRST_VARIANT};
+use rustc_middle::ty::layout::LayoutOf;
+use rustc_middle::ty::{self, Ty, TyCtxt, TypeAndMut};
+use rustc_span::DUMMY_SP;
+use rustc_target::abi::{self, Abi, Size, VariantIdx, FIRST_VARIANT};
 
+use crate::dataflow_const_prop::DummyMachine;
 use crate::ssa::{AssignedValue, SsaLocals};
 use crate::MirPass;
+use either::Either;
 
 pub struct GVN;
 
@@ -129,6 +135,12 @@ newtype_index! {
     struct VnIndex {}
 }
 
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+enum AddressKind {
+    Ref(BorrowKind),
+    Address(Mutability),
+}
+
 #[derive(Debug, PartialEq, Eq, Hash)]
 enum Value<'tcx> {
     // Root values.
@@ -145,6 +157,7 @@ enum Value<'tcx> {
     /// The address of a place.
     Address {
         place: Place<'tcx>,
+        kind: AddressKind,
         /// Give each borrow and pointer a different provenance, so we don't merge them.
         provenance: usize,
     },
@@ -172,6 +185,7 @@ enum Value<'tcx> {
 
 struct VnState<'body, 'tcx> {
     tcx: TyCtxt<'tcx>,
+    ecx: InterpCx<'tcx, 'tcx, DummyMachine>,
     param_env: ty::ParamEnv<'tcx>,
     local_decls: &'body LocalDecls<'tcx>,
     /// Value stored in each local.
@@ -179,6 +193,8 @@ struct VnState<'body, 'tcx> {
     /// First local to be assigned that value.
     rev_locals: FxHashMap<VnIndex, Vec<Local>>,
     values: FxIndexSet<Value<'tcx>>,
+    /// Values evaluated as constants if possible.
+    evaluated: IndexVec<VnIndex, Option<OpTy<'tcx>>>,
     /// Counter to generate different values.
     /// This is an option to stop creating opaques during replacement.
     next_opaque: Option<usize>,
@@ -197,11 +213,13 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
     ) -> Self {
         VnState {
             tcx,
+            ecx: InterpCx::new(tcx, DUMMY_SP, param_env, DummyMachine),
             param_env,
             local_decls,
             locals: IndexVec::from_elem(None, local_decls),
             rev_locals: FxHashMap::default(),
             values: FxIndexSet::default(),
+            evaluated: IndexVec::new(),
             next_opaque: Some(0),
             ssa,
             dominators,
@@ -211,8 +229,14 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
 
     #[instrument(level = "trace", skip(self), ret)]
     fn insert(&mut self, value: Value<'tcx>) -> VnIndex {
-        let (index, _) = self.values.insert_full(value);
-        VnIndex::from_usize(index)
+        let (index, new) = self.values.insert_full(value);
+        let index = VnIndex::from_usize(index);
+        if new {
+            let evaluated = self.eval_to_const(index);
+            let _index = self.evaluated.push(evaluated);
+            debug_assert_eq!(index, _index);
+        }
+        index
     }
 
     /// Create a new `Value` for which we have no information at all, except that it is distinct
@@ -227,9 +251,9 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
 
     /// Create a new `Value::Address` distinct from all the others.
     #[instrument(level = "trace", skip(self), ret)]
-    fn new_pointer(&mut self, place: Place<'tcx>) -> Option<VnIndex> {
+    fn new_pointer(&mut self, place: Place<'tcx>, kind: AddressKind) -> Option<VnIndex> {
         let next_opaque = self.next_opaque.as_mut()?;
-        let value = Value::Address { place, provenance: *next_opaque };
+        let value = Value::Address { place, kind, provenance: *next_opaque };
         *next_opaque += 1;
         Some(self.insert(value))
     }
@@ -251,6 +275,176 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         }
     }
 
+    #[instrument(level = "trace", skip(self), ret)]
+    fn eval_to_const(&mut self, value: VnIndex) -> Option<OpTy<'tcx>> {
+        use Value::*;
+        let op = match *self.get(value) {
+            Opaque(_) => return None,
+            // Do not bother evaluating repeat expressions. This would uselessly consume memory.
+            Repeat(..) => return None,
+
+            Constant(ref constant) => self.ecx.eval_mir_constant(constant, None, None).ok()?,
+            Aggregate(ty, variant, ref fields) => {
+                let fields = fields
+                    .iter()
+                    .map(|&f| self.evaluated[f].as_ref())
+                    .collect::<Option<Vec<_>>>()?;
+                let variant = if ty.is_enum() { Some(variant) } else { None };
+                let ty = self.ecx.layout_of(ty).ok()?;
+                let alloc_id = self
+                    .ecx
+                    .intern_with_temp_alloc(ty, |ecx, dest| {
+                        let variant_dest = if let Some(variant) = variant {
+                            ecx.project_downcast(dest, variant)?
+                        } else {
+                            dest.clone()
+                        };
+                        for (field_index, op) in fields.into_iter().enumerate() {
+                            let field_dest = ecx.project_field(&variant_dest, field_index)?;
+                            ecx.copy_op(op, &field_dest, /*allow_transmute*/ false)?;
+                        }
+                        ecx.write_discriminant(variant.unwrap_or(FIRST_VARIANT), dest)
+                    })
+                    .ok()?;
+                let mplace =
+                    self.ecx.raw_const_to_mplace(ConstAlloc { alloc_id, ty: ty.ty }).ok()?;
+                mplace.into()
+            }
+
+            Projection(base, elem) => {
+                let value = self.evaluated[base].as_ref()?;
+                let elem = match elem {
+                    ProjectionElem::Deref => ProjectionElem::Deref,
+                    ProjectionElem::Downcast(name, read_variant) => {
+                        ProjectionElem::Downcast(name, read_variant)
+                    }
+                    ProjectionElem::Field(f, ty) => ProjectionElem::Field(f, ty),
+                    ProjectionElem::ConstantIndex { offset, min_length, from_end } => {
+                        ProjectionElem::ConstantIndex { offset, min_length, from_end }
+                    }
+                    ProjectionElem::Subslice { from, to, from_end } => {
+                        ProjectionElem::Subslice { from, to, from_end }
+                    }
+                    ProjectionElem::OpaqueCast(ty) => ProjectionElem::OpaqueCast(ty),
+                    ProjectionElem::Subtype(ty) => ProjectionElem::Subtype(ty),
+                    // This should have been replaced by a `ConstantIndex` earlier.
+                    ProjectionElem::Index(_) => return None,
+                };
+                self.ecx.project(value, elem).ok()?
+            }
+            Address { place, kind, provenance: _ } => {
+                if !place.is_indirect_first_projection() {
+                    return None;
+                }
+                let local = self.locals[place.local]?;
+                let pointer = self.evaluated[local].as_ref()?;
+                let mut mplace = self.ecx.deref_pointer(pointer).ok()?;
+                for proj in place.projection.iter().skip(1) {
+                    // We have no call stack to associate a local with a value, so we cannot interpret indexing.
+                    if matches!(proj, ProjectionElem::Index(_)) {
+                        return None;
+                    }
+                    mplace = self.ecx.project(&mplace, proj).ok()?;
+                }
+                let pointer = mplace.to_ref(&self.ecx);
+                let ty = match kind {
+                    AddressKind::Ref(bk) => Ty::new_ref(
+                        self.tcx,
+                        self.tcx.lifetimes.re_erased,
+                        ty::TypeAndMut { ty: mplace.layout.ty, mutbl: bk.to_mutbl_lossy() },
+                    ),
+                    AddressKind::Address(mutbl) => {
+                        Ty::new_ptr(self.tcx, TypeAndMut { ty: mplace.layout.ty, mutbl })
+                    }
+                };
+                let layout = self.ecx.layout_of(ty).ok()?;
+                ImmTy::from_immediate(pointer, layout).into()
+            }
+
+            Discriminant(base) => {
+                let base = self.evaluated[base].as_ref()?;
+                let variant = self.ecx.read_discriminant(base).ok()?;
+                let discr_value =
+                    self.ecx.discriminant_for_variant(base.layout.ty, variant).ok()?;
+                discr_value.into()
+            }
+            Len(slice) => {
+                let slice = self.evaluated[slice].as_ref()?;
+                let usize_layout = self.ecx.layout_of(self.tcx.types.usize).unwrap();
+                let len = slice.len(&self.ecx).ok()?;
+                let imm = ImmTy::try_from_uint(len, usize_layout)?;
+                imm.into()
+            }
+            NullaryOp(null_op, ty) => {
+                let layout = self.ecx.layout_of(ty).ok()?;
+                if let NullOp::SizeOf | NullOp::AlignOf = null_op && layout.is_unsized() {
+                    return None;
+                }
+                let val = match null_op {
+                    NullOp::SizeOf => layout.size.bytes(),
+                    NullOp::AlignOf => layout.align.abi.bytes(),
+                    NullOp::OffsetOf(fields) => layout
+                        .offset_of_subfield(&self.ecx, fields.iter().map(|f| f.index()))
+                        .bytes(),
+                };
+                let usize_layout = self.ecx.layout_of(self.tcx.types.usize).unwrap();
+                let imm = ImmTy::try_from_uint(val, usize_layout)?;
+                imm.into()
+            }
+            UnaryOp(un_op, operand) => {
+                let operand = self.evaluated[operand].as_ref()?;
+                let operand = self.ecx.read_immediate(operand).ok()?;
+                let (val, _) = self.ecx.overflowing_unary_op(un_op, &operand).ok()?;
+                val.into()
+            }
+            BinaryOp(bin_op, lhs, rhs) => {
+                let lhs = self.evaluated[lhs].as_ref()?;
+                let lhs = self.ecx.read_immediate(lhs).ok()?;
+                let rhs = self.evaluated[rhs].as_ref()?;
+                let rhs = self.ecx.read_immediate(rhs).ok()?;
+                let (val, _) = self.ecx.overflowing_binary_op(bin_op, &lhs, &rhs).ok()?;
+                val.into()
+            }
+            CheckedBinaryOp(bin_op, lhs, rhs) => {
+                let lhs = self.evaluated[lhs].as_ref()?;
+                let lhs = self.ecx.read_immediate(lhs).ok()?;
+                let rhs = self.evaluated[rhs].as_ref()?;
+                let rhs = self.ecx.read_immediate(rhs).ok()?;
+                let (val, overflowed) = self.ecx.overflowing_binary_op(bin_op, &lhs, &rhs).ok()?;
+                let tuple = Ty::new_tup_from_iter(
+                    self.tcx,
+                    [val.layout.ty, self.tcx.types.bool].into_iter(),
+                );
+                let tuple = self.ecx.layout_of(tuple).ok()?;
+                ImmTy::from_scalar_pair(val.to_scalar(), Scalar::from_bool(overflowed), tuple)
+                    .into()
+            }
+            Cast { kind, value, from: _, to } => match kind {
+                CastKind::IntToInt | CastKind::IntToFloat => {
+                    let value = self.evaluated[value].as_ref()?;
+                    let value = self.ecx.read_immediate(value).ok()?;
+                    let to = self.ecx.layout_of(to).ok()?;
+                    let res = self.ecx.int_to_int_or_float(&value, to).ok()?;
+                    res.into()
+                }
+                CastKind::FloatToFloat | CastKind::FloatToInt => {
+                    let value = self.evaluated[value].as_ref()?;
+                    let value = self.ecx.read_immediate(value).ok()?;
+                    let to = self.ecx.layout_of(to).ok()?;
+                    let res = self.ecx.float_to_float_or_int(&value, to).ok()?;
+                    res.into()
+                }
+                CastKind::Transmute => {
+                    let value = self.evaluated[value].as_ref()?;
+                    let to = self.ecx.layout_of(to).ok()?;
+                    value.offset(Size::ZERO, to, &self.ecx).ok()?
+                }
+                _ => return None,
+            },
+        };
+        Some(op)
+    }
+
     /// Represent the *value* which would be read from `place`, and point `place` to a preexisting
     /// place with the same value (if that already exists).
     #[instrument(level = "trace", skip(self), ret)]
@@ -385,7 +579,12 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                 let ty = rvalue.ty(self.local_decls, self.tcx);
                 Value::Aggregate(ty, variant_index, fields?)
             }
-            Rvalue::Ref(.., place) | Rvalue::AddressOf(_, place) => return self.new_pointer(place),
+            Rvalue::Ref(_, borrow_kind, place) => {
+                return self.new_pointer(place, AddressKind::Ref(borrow_kind));
+            }
+            Rvalue::AddressOf(mutbl, place) => {
+                return self.new_pointer(place, AddressKind::Address(mutbl));
+            }
 
             // Operations.
             Rvalue::Len(ref mut place) => {
@@ -424,43 +623,106 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
     }
 }
 
+fn op_to_prop_const<'tcx>(
+    ecx: &mut InterpCx<'_, 'tcx, DummyMachine>,
+    op: &OpTy<'tcx>,
+) -> Option<ConstValue<'tcx>> {
+    // Do not attempt to propagate unsized locals.
+    if op.layout.is_unsized() {
+        return None;
+    }
+
+    // This constant is a ZST, just return an empty value.
+    if op.layout.is_zst() {
+        return Some(ConstValue::ZeroSized);
+    }
+
+    // Do not synthetize too large constants. Codegen will just memcpy them, which we'd like to avoid.
+    if !matches!(op.layout.abi, Abi::Scalar(..) | Abi::ScalarPair(..)) {
+        return None;
+    }
+
+    // If this constant has scalar ABI, return it as a `ConstValue::Scalar`.
+    if let Abi::Scalar(abi::Scalar::Initialized { .. }) = op.layout.abi
+        && let Ok(scalar) = ecx.read_scalar(op)
+    {
+        return Some(ConstValue::Scalar(scalar));
+    }
+
+    // If this constant is a projection of another, we can return it directly.
+    if let Either::Left(mplace) = op.as_mplace_or_imm()
+        && let MemPlaceMeta::None = mplace.meta()
+    {
+        let pointer = mplace.ptr().into_pointer_or_addr().ok()?;
+        let (alloc_id, offset) = pointer.into_parts();
+        return if matches!(ecx.tcx.global_alloc(alloc_id), GlobalAlloc::Memory(_)) {
+            Some(ConstValue::Indirect { alloc_id, offset })
+        } else {
+            None
+        }
+    }
+
+    // Everything failed: create a new allocation to hold the data.
+    let alloc_id =
+        ecx.intern_with_temp_alloc(op.layout, |ecx, dest| ecx.copy_op(op, dest, false)).ok()?;
+    Some(ConstValue::Indirect { alloc_id, offset: Size::ZERO })
+}
+
 impl<'tcx> VnState<'_, 'tcx> {
     /// If `index` is a `Value::Constant`, return the `Constant` to be put in the MIR.
     fn try_as_constant(&mut self, index: VnIndex) -> Option<ConstOperand<'tcx>> {
+        // This was already constant in MIR, do not change it.
         if let Value::Constant(const_) = *self.get(index) {
             // Some constants may contain pointers. We need to preserve the provenance of these
             // pointers, but not all constants guarantee this:
             // - valtrees purposefully do not;
             // - ConstValue::Slice does not either.
-            match const_ {
+            let const_ok = match const_ {
                 Const::Ty(c) => match c.kind() {
                     ty::ConstKind::Value(valtree) => match valtree {
                         // This is just an integer, keep it.
-                        ty::ValTree::Leaf(_) => {}
-                        ty::ValTree::Branch(_) => return None,
+                        ty::ValTree::Leaf(_) => true,
+                        ty::ValTree::Branch(_) => false,
                     },
                     ty::ConstKind::Param(..)
                     | ty::ConstKind::Unevaluated(..)
-                    | ty::ConstKind::Expr(..) => {}
+                    | ty::ConstKind::Expr(..) => true,
                     // Should not appear in runtime MIR.
                     ty::ConstKind::Infer(..)
                     | ty::ConstKind::Bound(..)
                     | ty::ConstKind::Placeholder(..)
                     | ty::ConstKind::Error(..) => bug!(),
                 },
-                Const::Unevaluated(..) => {}
+                Const::Unevaluated(..) => true,
                 // If the same slice appears twice in the MIR, we cannot guarantee that we will
                 // give the same `AllocId` to the data.
-                Const::Val(ConstValue::Slice { .. }, _) => return None,
+                Const::Val(ConstValue::Slice { .. }, _) => false,
                 Const::Val(
                     ConstValue::ZeroSized | ConstValue::Scalar(_) | ConstValue::Indirect { .. },
                     _,
-                ) => {}
+                ) => true,
+            };
+            if const_ok {
+                return Some(ConstOperand { span: rustc_span::DUMMY_SP, user_ty: None, const_ });
             }
-            Some(ConstOperand { span: rustc_span::DUMMY_SP, user_ty: None, const_ })
-        } else {
-            None
         }
+
+        let op = self.evaluated[index].as_ref()?;
+        if op.layout.is_unsized() {
+            // Do not attempt to propagate unsized locals.
+            return None;
+        }
+
+        let value = op_to_prop_const(&mut self.ecx, op)?;
+
+        // Check that we do not leak a pointer.
+        // Those pointers may lose part of their identity in codegen.
+        if value.has_provenance(self.tcx, op.layout.size) {
+            return None;
+        }
+
+        let const_ = Const::Val(value, op.layout.ty);
+        Some(ConstOperand { span: rustc_span::DUMMY_SP, user_ty: None, const_ })
     }
 
     /// If there is a local which is assigned `index`, and its assignment strictly dominates `loc`,
diff --git a/tests/mir-opt/gvn.arithmetic.GVN.panic-abort.diff b/tests/mir-opt/gvn.arithmetic.GVN.panic-abort.diff
index 17ba0d3cef2d..d524ad242fec 100644
--- a/tests/mir-opt/gvn.arithmetic.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.arithmetic.GVN.panic-abort.diff
@@ -121,9 +121,10 @@
           StorageLive(_15);
           StorageLive(_16);
           _16 = _1;
-          _17 = Eq(const 0_u64, const 0_u64);
+-         _17 = Eq(const 0_u64, const 0_u64);
 -         assert(!move _17, "attempt to divide `{}` by zero", _16) -> [success: bb5, unwind unreachable];
-+         assert(!_17, "attempt to divide `{}` by zero", _1) -> [success: bb5, unwind unreachable];
++         _17 = const true;
++         assert(!const true, "attempt to divide `{}` by zero", _1) -> [success: bb5, unwind unreachable];
       }
   
       bb5: {
@@ -140,9 +141,10 @@
           StorageLive(_19);
           StorageLive(_20);
           _20 = _1;
-          _21 = Eq(const 1_u64, const 0_u64);
+-         _21 = Eq(const 1_u64, const 0_u64);
 -         assert(!move _21, "attempt to divide `{}` by zero", _20) -> [success: bb7, unwind unreachable];
-+         assert(!_21, "attempt to divide `{}` by zero", _1) -> [success: bb7, unwind unreachable];
++         _21 = const false;
++         assert(!const false, "attempt to divide `{}` by zero", _1) -> [success: bb7, unwind unreachable];
       }
   
       bb7: {
@@ -201,8 +203,8 @@
           _32 = _1;
 -         _33 = Eq(const 0_u64, const 0_u64);
 -         assert(!move _33, "attempt to calculate the remainder of `{}` with a divisor of zero", _32) -> [success: bb13, unwind unreachable];
-+         _33 = _17;
-+         assert(!_17, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb13, unwind unreachable];
++         _33 = const true;
++         assert(!const true, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb13, unwind unreachable];
       }
   
       bb13: {
@@ -221,8 +223,8 @@
           _36 = _1;
 -         _37 = Eq(const 1_u64, const 0_u64);
 -         assert(!move _37, "attempt to calculate the remainder of `{}` with a divisor of zero", _36) -> [success: bb15, unwind unreachable];
-+         _37 = _21;
-+         assert(!_21, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb15, unwind unreachable];
++         _37 = const false;
++         assert(!const false, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb15, unwind unreachable];
       }
   
       bb15: {
diff --git a/tests/mir-opt/gvn.arithmetic.GVN.panic-unwind.diff b/tests/mir-opt/gvn.arithmetic.GVN.panic-unwind.diff
index f14fd409bea1..9d69353934c7 100644
--- a/tests/mir-opt/gvn.arithmetic.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.arithmetic.GVN.panic-unwind.diff
@@ -121,9 +121,10 @@
           StorageLive(_15);
           StorageLive(_16);
           _16 = _1;
-          _17 = Eq(const 0_u64, const 0_u64);
+-         _17 = Eq(const 0_u64, const 0_u64);
 -         assert(!move _17, "attempt to divide `{}` by zero", _16) -> [success: bb5, unwind continue];
-+         assert(!_17, "attempt to divide `{}` by zero", _1) -> [success: bb5, unwind continue];
++         _17 = const true;
++         assert(!const true, "attempt to divide `{}` by zero", _1) -> [success: bb5, unwind continue];
       }
   
       bb5: {
@@ -140,9 +141,10 @@
           StorageLive(_19);
           StorageLive(_20);
           _20 = _1;
-          _21 = Eq(const 1_u64, const 0_u64);
+-         _21 = Eq(const 1_u64, const 0_u64);
 -         assert(!move _21, "attempt to divide `{}` by zero", _20) -> [success: bb7, unwind continue];
-+         assert(!_21, "attempt to divide `{}` by zero", _1) -> [success: bb7, unwind continue];
++         _21 = const false;
++         assert(!const false, "attempt to divide `{}` by zero", _1) -> [success: bb7, unwind continue];
       }
   
       bb7: {
@@ -201,8 +203,8 @@
           _32 = _1;
 -         _33 = Eq(const 0_u64, const 0_u64);
 -         assert(!move _33, "attempt to calculate the remainder of `{}` with a divisor of zero", _32) -> [success: bb13, unwind continue];
-+         _33 = _17;
-+         assert(!_17, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb13, unwind continue];
++         _33 = const true;
++         assert(!const true, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb13, unwind continue];
       }
   
       bb13: {
@@ -221,8 +223,8 @@
           _36 = _1;
 -         _37 = Eq(const 1_u64, const 0_u64);
 -         assert(!move _37, "attempt to calculate the remainder of `{}` with a divisor of zero", _36) -> [success: bb15, unwind continue];
-+         _37 = _21;
-+         assert(!_21, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb15, unwind continue];
++         _37 = const false;
++         assert(!const false, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb15, unwind continue];
       }
   
       bb15: {
diff --git a/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-abort.diff b/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-abort.diff
index e586e4ac8898..3efe7abc976f 100644
--- a/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-abort.diff
@@ -153,9 +153,10 @@
           StorageLive(_19);
           StorageLive(_20);
           _20 = _1;
-          _21 = Eq(const 0_u64, const 0_u64);
+-         _21 = Eq(const 0_u64, const 0_u64);
 -         assert(!move _21, "attempt to divide `{}` by zero", _20) -> [success: bb9, unwind unreachable];
-+         assert(!_21, "attempt to divide `{}` by zero", _1) -> [success: bb9, unwind unreachable];
++         _21 = const true;
++         assert(!const true, "attempt to divide `{}` by zero", _1) -> [success: bb9, unwind unreachable];
       }
   
       bb9: {
@@ -172,9 +173,10 @@
           StorageLive(_23);
           StorageLive(_24);
           _24 = _1;
-          _25 = Eq(const 1_u64, const 0_u64);
+-         _25 = Eq(const 1_u64, const 0_u64);
 -         assert(!move _25, "attempt to divide `{}` by zero", _24) -> [success: bb11, unwind unreachable];
-+         assert(!_25, "attempt to divide `{}` by zero", _1) -> [success: bb11, unwind unreachable];
++         _25 = const false;
++         assert(!const false, "attempt to divide `{}` by zero", _1) -> [success: bb11, unwind unreachable];
       }
   
       bb11: {
@@ -233,8 +235,8 @@
           _36 = _1;
 -         _37 = Eq(const 0_u64, const 0_u64);
 -         assert(!move _37, "attempt to calculate the remainder of `{}` with a divisor of zero", _36) -> [success: bb17, unwind unreachable];
-+         _37 = _21;
-+         assert(!_21, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb17, unwind unreachable];
++         _37 = const true;
++         assert(!const true, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb17, unwind unreachable];
       }
   
       bb17: {
@@ -253,8 +255,8 @@
           _40 = _1;
 -         _41 = Eq(const 1_u64, const 0_u64);
 -         assert(!move _41, "attempt to calculate the remainder of `{}` with a divisor of zero", _40) -> [success: bb19, unwind unreachable];
-+         _41 = _25;
-+         assert(!_25, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb19, unwind unreachable];
++         _41 = const false;
++         assert(!const false, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb19, unwind unreachable];
       }
   
       bb19: {
@@ -350,11 +352,12 @@
           StorageLive(_60);
           StorageLive(_61);
           _61 = _1;
-          _62 = const 0_i32 as u32 (IntToInt);
+-         _62 = const 0_i32 as u32 (IntToInt);
 -         _63 = Lt(move _62, const 64_u32);
 -         assert(move _63, "attempt to shift right by `{}`, which would overflow", const 0_i32) -> [success: bb28, unwind unreachable];
-+         _63 = Lt(_62, const 64_u32);
-+         assert(_63, "attempt to shift right by `{}`, which would overflow", const 0_i32) -> [success: bb28, unwind unreachable];
++         _62 = const 0_u32;
++         _63 = const true;
++         assert(const true, "attempt to shift right by `{}`, which would overflow", const 0_i32) -> [success: bb28, unwind unreachable];
       }
   
       bb28: {
@@ -374,9 +377,9 @@
 -         _67 = const 0_i32 as u32 (IntToInt);
 -         _68 = Lt(move _67, const 64_u32);
 -         assert(move _68, "attempt to shift left by `{}`, which would overflow", const 0_i32) -> [success: bb30, unwind unreachable];
-+         _67 = _62;
-+         _68 = _63;
-+         assert(_63, "attempt to shift left by `{}`, which would overflow", const 0_i32) -> [success: bb30, unwind unreachable];
++         _67 = const 0_u32;
++         _68 = const true;
++         assert(const true, "attempt to shift left by `{}`, which would overflow", const 0_i32) -> [success: bb30, unwind unreachable];
       }
   
       bb30: {
diff --git a/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-unwind.diff b/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-unwind.diff
index f58a9116b8fe..9fbb8df79a1d 100644
--- a/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-unwind.diff
@@ -153,9 +153,10 @@
           StorageLive(_19);
           StorageLive(_20);
           _20 = _1;
-          _21 = Eq(const 0_u64, const 0_u64);
+-         _21 = Eq(const 0_u64, const 0_u64);
 -         assert(!move _21, "attempt to divide `{}` by zero", _20) -> [success: bb9, unwind continue];
-+         assert(!_21, "attempt to divide `{}` by zero", _1) -> [success: bb9, unwind continue];
++         _21 = const true;
++         assert(!const true, "attempt to divide `{}` by zero", _1) -> [success: bb9, unwind continue];
       }
   
       bb9: {
@@ -172,9 +173,10 @@
           StorageLive(_23);
           StorageLive(_24);
           _24 = _1;
-          _25 = Eq(const 1_u64, const 0_u64);
+-         _25 = Eq(const 1_u64, const 0_u64);
 -         assert(!move _25, "attempt to divide `{}` by zero", _24) -> [success: bb11, unwind continue];
-+         assert(!_25, "attempt to divide `{}` by zero", _1) -> [success: bb11, unwind continue];
++         _25 = const false;
++         assert(!const false, "attempt to divide `{}` by zero", _1) -> [success: bb11, unwind continue];
       }
   
       bb11: {
@@ -233,8 +235,8 @@
           _36 = _1;
 -         _37 = Eq(const 0_u64, const 0_u64);
 -         assert(!move _37, "attempt to calculate the remainder of `{}` with a divisor of zero", _36) -> [success: bb17, unwind continue];
-+         _37 = _21;
-+         assert(!_21, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb17, unwind continue];
++         _37 = const true;
++         assert(!const true, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb17, unwind continue];
       }
   
       bb17: {
@@ -253,8 +255,8 @@
           _40 = _1;
 -         _41 = Eq(const 1_u64, const 0_u64);
 -         assert(!move _41, "attempt to calculate the remainder of `{}` with a divisor of zero", _40) -> [success: bb19, unwind continue];
-+         _41 = _25;
-+         assert(!_25, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb19, unwind continue];
++         _41 = const false;
++         assert(!const false, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb19, unwind continue];
       }
   
       bb19: {
@@ -350,11 +352,12 @@
           StorageLive(_60);
           StorageLive(_61);
           _61 = _1;
-          _62 = const 0_i32 as u32 (IntToInt);
+-         _62 = const 0_i32 as u32 (IntToInt);
 -         _63 = Lt(move _62, const 64_u32);
 -         assert(move _63, "attempt to shift right by `{}`, which would overflow", const 0_i32) -> [success: bb28, unwind continue];
-+         _63 = Lt(_62, const 64_u32);
-+         assert(_63, "attempt to shift right by `{}`, which would overflow", const 0_i32) -> [success: bb28, unwind continue];
++         _62 = const 0_u32;
++         _63 = const true;
++         assert(const true, "attempt to shift right by `{}`, which would overflow", const 0_i32) -> [success: bb28, unwind continue];
       }
   
       bb28: {
@@ -374,9 +377,9 @@
 -         _67 = const 0_i32 as u32 (IntToInt);
 -         _68 = Lt(move _67, const 64_u32);
 -         assert(move _68, "attempt to shift left by `{}`, which would overflow", const 0_i32) -> [success: bb30, unwind continue];
-+         _67 = _62;
-+         _68 = _63;
-+         assert(_63, "attempt to shift left by `{}`, which would overflow", const 0_i32) -> [success: bb30, unwind continue];
++         _67 = const 0_u32;
++         _68 = const true;
++         assert(const true, "attempt to shift left by `{}`, which would overflow", const 0_i32) -> [success: bb30, unwind continue];
       }
   
       bb30: {
diff --git a/tests/mir-opt/gvn.cast.GVN.panic-abort.diff b/tests/mir-opt/gvn.cast.GVN.panic-abort.diff
index 37b0d92931f6..d43198c99110 100644
--- a/tests/mir-opt/gvn.cast.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.cast.GVN.panic-abort.diff
@@ -119,9 +119,10 @@
 -         _6 = _1;
 -         _5 = move _6 as u8 (IntToInt);
 +         _6 = const 1_i64;
-+         _5 = const 1_i64 as u8 (IntToInt);
++         _5 = const 1_u8;
           StorageDead(_6);
-          _4 = opaque::<u8>(move _5) -> [return: bb1, unwind unreachable];
+-         _4 = opaque::<u8>(move _5) -> [return: bb1, unwind unreachable];
++         _4 = opaque::<u8>(const 1_u8) -> [return: bb1, unwind unreachable];
       }
   
       bb1: {
@@ -133,9 +134,10 @@
 -         _9 = _1;
 -         _8 = move _9 as u16 (IntToInt);
 +         _9 = const 1_i64;
-+         _8 = const 1_i64 as u16 (IntToInt);
++         _8 = const 1_u16;
           StorageDead(_9);
-          _7 = opaque::<u16>(move _8) -> [return: bb2, unwind unreachable];
+-         _7 = opaque::<u16>(move _8) -> [return: bb2, unwind unreachable];
++         _7 = opaque::<u16>(const 1_u16) -> [return: bb2, unwind unreachable];
       }
   
       bb2: {
@@ -147,9 +149,10 @@
 -         _12 = _1;
 -         _11 = move _12 as u32 (IntToInt);
 +         _12 = const 1_i64;
-+         _11 = const 1_i64 as u32 (IntToInt);
++         _11 = const 1_u32;
           StorageDead(_12);
-          _10 = opaque::<u32>(move _11) -> [return: bb3, unwind unreachable];
+-         _10 = opaque::<u32>(move _11) -> [return: bb3, unwind unreachable];
++         _10 = opaque::<u32>(const 1_u32) -> [return: bb3, unwind unreachable];
       }
   
       bb3: {
@@ -161,9 +164,10 @@
 -         _15 = _1;
 -         _14 = move _15 as u64 (IntToInt);
 +         _15 = const 1_i64;
-+         _14 = const 1_i64 as u64 (IntToInt);
++         _14 = const 1_u64;
           StorageDead(_15);
-          _13 = opaque::<u64>(move _14) -> [return: bb4, unwind unreachable];
+-         _13 = opaque::<u64>(move _14) -> [return: bb4, unwind unreachable];
++         _13 = opaque::<u64>(const 1_u64) -> [return: bb4, unwind unreachable];
       }
   
       bb4: {
@@ -175,9 +179,10 @@
 -         _18 = _1;
 -         _17 = move _18 as i8 (IntToInt);
 +         _18 = const 1_i64;
-+         _17 = const 1_i64 as i8 (IntToInt);
++         _17 = const 1_i8;
           StorageDead(_18);
-          _16 = opaque::<i8>(move _17) -> [return: bb5, unwind unreachable];
+-         _16 = opaque::<i8>(move _17) -> [return: bb5, unwind unreachable];
++         _16 = opaque::<i8>(const 1_i8) -> [return: bb5, unwind unreachable];
       }
   
       bb5: {
@@ -189,9 +194,10 @@
 -         _21 = _1;
 -         _20 = move _21 as i16 (IntToInt);
 +         _21 = const 1_i64;
-+         _20 = const 1_i64 as i16 (IntToInt);
++         _20 = const 1_i16;
           StorageDead(_21);
-          _19 = opaque::<i16>(move _20) -> [return: bb6, unwind unreachable];
+-         _19 = opaque::<i16>(move _20) -> [return: bb6, unwind unreachable];
++         _19 = opaque::<i16>(const 1_i16) -> [return: bb6, unwind unreachable];
       }
   
       bb6: {
@@ -203,9 +209,10 @@
 -         _24 = _1;
 -         _23 = move _24 as i32 (IntToInt);
 +         _24 = const 1_i64;
-+         _23 = const 1_i64 as i32 (IntToInt);
++         _23 = const 1_i32;
           StorageDead(_24);
-          _22 = opaque::<i32>(move _23) -> [return: bb7, unwind unreachable];
+-         _22 = opaque::<i32>(move _23) -> [return: bb7, unwind unreachable];
++         _22 = opaque::<i32>(const 1_i32) -> [return: bb7, unwind unreachable];
       }
   
       bb7: {
@@ -228,9 +235,10 @@
 -         _29 = _1;
 -         _28 = move _29 as f32 (IntToFloat);
 +         _29 = const 1_i64;
-+         _28 = const 1_i64 as f32 (IntToFloat);
++         _28 = const 1f32;
           StorageDead(_29);
-          _27 = opaque::<f32>(move _28) -> [return: bb9, unwind unreachable];
+-         _27 = opaque::<f32>(move _28) -> [return: bb9, unwind unreachable];
++         _27 = opaque::<f32>(const 1f32) -> [return: bb9, unwind unreachable];
       }
   
       bb9: {
@@ -242,9 +250,10 @@
 -         _32 = _1;
 -         _31 = move _32 as f64 (IntToFloat);
 +         _32 = const 1_i64;
-+         _31 = const 1_i64 as f64 (IntToFloat);
++         _31 = const 1f64;
           StorageDead(_32);
-          _30 = opaque::<f64>(move _31) -> [return: bb10, unwind unreachable];
+-         _30 = opaque::<f64>(move _31) -> [return: bb10, unwind unreachable];
++         _30 = opaque::<f64>(const 1f64) -> [return: bb10, unwind unreachable];
       }
   
       bb10: {
@@ -256,9 +265,10 @@
 -         _35 = _2;
 -         _34 = move _35 as u8 (IntToInt);
 +         _35 = const 1_u64;
-+         _34 = const 1_u64 as u8 (IntToInt);
++         _34 = const 1_u8;
           StorageDead(_35);
-          _33 = opaque::<u8>(move _34) -> [return: bb11, unwind unreachable];
+-         _33 = opaque::<u8>(move _34) -> [return: bb11, unwind unreachable];
++         _33 = opaque::<u8>(const 1_u8) -> [return: bb11, unwind unreachable];
       }
   
       bb11: {
@@ -270,9 +280,10 @@
 -         _38 = _2;
 -         _37 = move _38 as u16 (IntToInt);
 +         _38 = const 1_u64;
-+         _37 = const 1_u64 as u16 (IntToInt);
++         _37 = const 1_u16;
           StorageDead(_38);
-          _36 = opaque::<u16>(move _37) -> [return: bb12, unwind unreachable];
+-         _36 = opaque::<u16>(move _37) -> [return: bb12, unwind unreachable];
++         _36 = opaque::<u16>(const 1_u16) -> [return: bb12, unwind unreachable];
       }
   
       bb12: {
@@ -284,9 +295,10 @@
 -         _41 = _2;
 -         _40 = move _41 as u32 (IntToInt);
 +         _41 = const 1_u64;
-+         _40 = const 1_u64 as u32 (IntToInt);
++         _40 = const 1_u32;
           StorageDead(_41);
-          _39 = opaque::<u32>(move _40) -> [return: bb13, unwind unreachable];
+-         _39 = opaque::<u32>(move _40) -> [return: bb13, unwind unreachable];
++         _39 = opaque::<u32>(const 1_u32) -> [return: bb13, unwind unreachable];
       }
   
       bb13: {
@@ -309,9 +321,10 @@
 -         _46 = _2;
 -         _45 = move _46 as i8 (IntToInt);
 +         _46 = const 1_u64;
-+         _45 = const 1_u64 as i8 (IntToInt);
++         _45 = const 1_i8;
           StorageDead(_46);
-          _44 = opaque::<i8>(move _45) -> [return: bb15, unwind unreachable];
+-         _44 = opaque::<i8>(move _45) -> [return: bb15, unwind unreachable];
++         _44 = opaque::<i8>(const 1_i8) -> [return: bb15, unwind unreachable];
       }
   
       bb15: {
@@ -323,9 +336,10 @@
 -         _49 = _2;
 -         _48 = move _49 as i16 (IntToInt);
 +         _49 = const 1_u64;
-+         _48 = const 1_u64 as i16 (IntToInt);
++         _48 = const 1_i16;
           StorageDead(_49);
-          _47 = opaque::<i16>(move _48) -> [return: bb16, unwind unreachable];
+-         _47 = opaque::<i16>(move _48) -> [return: bb16, unwind unreachable];
++         _47 = opaque::<i16>(const 1_i16) -> [return: bb16, unwind unreachable];
       }
   
       bb16: {
@@ -337,9 +351,10 @@
 -         _52 = _2;
 -         _51 = move _52 as i32 (IntToInt);
 +         _52 = const 1_u64;
-+         _51 = const 1_u64 as i32 (IntToInt);
++         _51 = const 1_i32;
           StorageDead(_52);
-          _50 = opaque::<i32>(move _51) -> [return: bb17, unwind unreachable];
+-         _50 = opaque::<i32>(move _51) -> [return: bb17, unwind unreachable];
++         _50 = opaque::<i32>(const 1_i32) -> [return: bb17, unwind unreachable];
       }
   
       bb17: {
@@ -351,9 +366,10 @@
 -         _55 = _2;
 -         _54 = move _55 as i64 (IntToInt);
 +         _55 = const 1_u64;
-+         _54 = const 1_u64 as i64 (IntToInt);
++         _54 = const 1_i64;
           StorageDead(_55);
-          _53 = opaque::<i64>(move _54) -> [return: bb18, unwind unreachable];
+-         _53 = opaque::<i64>(move _54) -> [return: bb18, unwind unreachable];
++         _53 = opaque::<i64>(const 1_i64) -> [return: bb18, unwind unreachable];
       }
   
       bb18: {
@@ -365,9 +381,10 @@
 -         _58 = _2;
 -         _57 = move _58 as f32 (IntToFloat);
 +         _58 = const 1_u64;
-+         _57 = const 1_u64 as f32 (IntToFloat);
++         _57 = const 1f32;
           StorageDead(_58);
-          _56 = opaque::<f32>(move _57) -> [return: bb19, unwind unreachable];
+-         _56 = opaque::<f32>(move _57) -> [return: bb19, unwind unreachable];
++         _56 = opaque::<f32>(const 1f32) -> [return: bb19, unwind unreachable];
       }
   
       bb19: {
@@ -379,9 +396,10 @@
 -         _61 = _2;
 -         _60 = move _61 as f64 (IntToFloat);
 +         _61 = const 1_u64;
-+         _60 = const 1_u64 as f64 (IntToFloat);
++         _60 = const 1f64;
           StorageDead(_61);
-          _59 = opaque::<f64>(move _60) -> [return: bb20, unwind unreachable];
+-         _59 = opaque::<f64>(move _60) -> [return: bb20, unwind unreachable];
++         _59 = opaque::<f64>(const 1f64) -> [return: bb20, unwind unreachable];
       }
   
       bb20: {
@@ -393,9 +411,10 @@
 -         _64 = _3;
 -         _63 = move _64 as u8 (FloatToInt);
 +         _64 = const 1f64;
-+         _63 = const 1f64 as u8 (FloatToInt);
++         _63 = const 1_u8;
           StorageDead(_64);
-          _62 = opaque::<u8>(move _63) -> [return: bb21, unwind unreachable];
+-         _62 = opaque::<u8>(move _63) -> [return: bb21, unwind unreachable];
++         _62 = opaque::<u8>(const 1_u8) -> [return: bb21, unwind unreachable];
       }
   
       bb21: {
@@ -407,9 +426,10 @@
 -         _67 = _3;
 -         _66 = move _67 as u16 (FloatToInt);
 +         _67 = const 1f64;
-+         _66 = const 1f64 as u16 (FloatToInt);
++         _66 = const 1_u16;
           StorageDead(_67);
-          _65 = opaque::<u16>(move _66) -> [return: bb22, unwind unreachable];
+-         _65 = opaque::<u16>(move _66) -> [return: bb22, unwind unreachable];
++         _65 = opaque::<u16>(const 1_u16) -> [return: bb22, unwind unreachable];
       }
   
       bb22: {
@@ -421,9 +441,10 @@
 -         _70 = _3;
 -         _69 = move _70 as u32 (FloatToInt);
 +         _70 = const 1f64;
-+         _69 = const 1f64 as u32 (FloatToInt);
++         _69 = const 1_u32;
           StorageDead(_70);
-          _68 = opaque::<u32>(move _69) -> [return: bb23, unwind unreachable];
+-         _68 = opaque::<u32>(move _69) -> [return: bb23, unwind unreachable];
++         _68 = opaque::<u32>(const 1_u32) -> [return: bb23, unwind unreachable];
       }
   
       bb23: {
@@ -435,9 +456,10 @@
 -         _73 = _3;
 -         _72 = move _73 as u64 (FloatToInt);
 +         _73 = const 1f64;
-+         _72 = const 1f64 as u64 (FloatToInt);
++         _72 = const 1_u64;
           StorageDead(_73);
-          _71 = opaque::<u64>(move _72) -> [return: bb24, unwind unreachable];
+-         _71 = opaque::<u64>(move _72) -> [return: bb24, unwind unreachable];
++         _71 = opaque::<u64>(const 1_u64) -> [return: bb24, unwind unreachable];
       }
   
       bb24: {
@@ -449,9 +471,10 @@
 -         _76 = _3;
 -         _75 = move _76 as i8 (FloatToInt);
 +         _76 = const 1f64;
-+         _75 = const 1f64 as i8 (FloatToInt);
++         _75 = const 1_i8;
           StorageDead(_76);
-          _74 = opaque::<i8>(move _75) -> [return: bb25, unwind unreachable];
+-         _74 = opaque::<i8>(move _75) -> [return: bb25, unwind unreachable];
++         _74 = opaque::<i8>(const 1_i8) -> [return: bb25, unwind unreachable];
       }
   
       bb25: {
@@ -463,9 +486,10 @@
 -         _79 = _3;
 -         _78 = move _79 as i16 (FloatToInt);
 +         _79 = const 1f64;
-+         _78 = const 1f64 as i16 (FloatToInt);
++         _78 = const 1_i16;
           StorageDead(_79);
-          _77 = opaque::<i16>(move _78) -> [return: bb26, unwind unreachable];
+-         _77 = opaque::<i16>(move _78) -> [return: bb26, unwind unreachable];
++         _77 = opaque::<i16>(const 1_i16) -> [return: bb26, unwind unreachable];
       }
   
       bb26: {
@@ -477,9 +501,10 @@
 -         _82 = _3;
 -         _81 = move _82 as i32 (FloatToInt);
 +         _82 = const 1f64;
-+         _81 = const 1f64 as i32 (FloatToInt);
++         _81 = const 1_i32;
           StorageDead(_82);
-          _80 = opaque::<i32>(move _81) -> [return: bb27, unwind unreachable];
+-         _80 = opaque::<i32>(move _81) -> [return: bb27, unwind unreachable];
++         _80 = opaque::<i32>(const 1_i32) -> [return: bb27, unwind unreachable];
       }
   
       bb27: {
@@ -491,9 +516,10 @@
 -         _85 = _3;
 -         _84 = move _85 as i64 (FloatToInt);
 +         _85 = const 1f64;
-+         _84 = const 1f64 as i64 (FloatToInt);
++         _84 = const 1_i64;
           StorageDead(_85);
-          _83 = opaque::<i64>(move _84) -> [return: bb28, unwind unreachable];
+-         _83 = opaque::<i64>(move _84) -> [return: bb28, unwind unreachable];
++         _83 = opaque::<i64>(const 1_i64) -> [return: bb28, unwind unreachable];
       }
   
       bb28: {
@@ -505,9 +531,10 @@
 -         _88 = _3;
 -         _87 = move _88 as f32 (FloatToFloat);
 +         _88 = const 1f64;
-+         _87 = const 1f64 as f32 (FloatToFloat);
++         _87 = const 1f32;
           StorageDead(_88);
-          _86 = opaque::<f32>(move _87) -> [return: bb29, unwind unreachable];
+-         _86 = opaque::<f32>(move _87) -> [return: bb29, unwind unreachable];
++         _86 = opaque::<f32>(const 1f32) -> [return: bb29, unwind unreachable];
       }
   
       bb29: {
diff --git a/tests/mir-opt/gvn.cast.GVN.panic-unwind.diff b/tests/mir-opt/gvn.cast.GVN.panic-unwind.diff
index fbdec455188a..08b97e13aa0b 100644
--- a/tests/mir-opt/gvn.cast.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.cast.GVN.panic-unwind.diff
@@ -119,9 +119,10 @@
 -         _6 = _1;
 -         _5 = move _6 as u8 (IntToInt);
 +         _6 = const 1_i64;
-+         _5 = const 1_i64 as u8 (IntToInt);
++         _5 = const 1_u8;
           StorageDead(_6);
-          _4 = opaque::<u8>(move _5) -> [return: bb1, unwind continue];
+-         _4 = opaque::<u8>(move _5) -> [return: bb1, unwind continue];
++         _4 = opaque::<u8>(const 1_u8) -> [return: bb1, unwind continue];
       }
   
       bb1: {
@@ -133,9 +134,10 @@
 -         _9 = _1;
 -         _8 = move _9 as u16 (IntToInt);
 +         _9 = const 1_i64;
-+         _8 = const 1_i64 as u16 (IntToInt);
++         _8 = const 1_u16;
           StorageDead(_9);
-          _7 = opaque::<u16>(move _8) -> [return: bb2, unwind continue];
+-         _7 = opaque::<u16>(move _8) -> [return: bb2, unwind continue];
++         _7 = opaque::<u16>(const 1_u16) -> [return: bb2, unwind continue];
       }
   
       bb2: {
@@ -147,9 +149,10 @@
 -         _12 = _1;
 -         _11 = move _12 as u32 (IntToInt);
 +         _12 = const 1_i64;
-+         _11 = const 1_i64 as u32 (IntToInt);
++         _11 = const 1_u32;
           StorageDead(_12);
-          _10 = opaque::<u32>(move _11) -> [return: bb3, unwind continue];
+-         _10 = opaque::<u32>(move _11) -> [return: bb3, unwind continue];
++         _10 = opaque::<u32>(const 1_u32) -> [return: bb3, unwind continue];
       }
   
       bb3: {
@@ -161,9 +164,10 @@
 -         _15 = _1;
 -         _14 = move _15 as u64 (IntToInt);
 +         _15 = const 1_i64;
-+         _14 = const 1_i64 as u64 (IntToInt);
++         _14 = const 1_u64;
           StorageDead(_15);
-          _13 = opaque::<u64>(move _14) -> [return: bb4, unwind continue];
+-         _13 = opaque::<u64>(move _14) -> [return: bb4, unwind continue];
++         _13 = opaque::<u64>(const 1_u64) -> [return: bb4, unwind continue];
       }
   
       bb4: {
@@ -175,9 +179,10 @@
 -         _18 = _1;
 -         _17 = move _18 as i8 (IntToInt);
 +         _18 = const 1_i64;
-+         _17 = const 1_i64 as i8 (IntToInt);
++         _17 = const 1_i8;
           StorageDead(_18);
-          _16 = opaque::<i8>(move _17) -> [return: bb5, unwind continue];
+-         _16 = opaque::<i8>(move _17) -> [return: bb5, unwind continue];
++         _16 = opaque::<i8>(const 1_i8) -> [return: bb5, unwind continue];
       }
   
       bb5: {
@@ -189,9 +194,10 @@
 -         _21 = _1;
 -         _20 = move _21 as i16 (IntToInt);
 +         _21 = const 1_i64;
-+         _20 = const 1_i64 as i16 (IntToInt);
++         _20 = const 1_i16;
           StorageDead(_21);
-          _19 = opaque::<i16>(move _20) -> [return: bb6, unwind continue];
+-         _19 = opaque::<i16>(move _20) -> [return: bb6, unwind continue];
++         _19 = opaque::<i16>(const 1_i16) -> [return: bb6, unwind continue];
       }
   
       bb6: {
@@ -203,9 +209,10 @@
 -         _24 = _1;
 -         _23 = move _24 as i32 (IntToInt);
 +         _24 = const 1_i64;
-+         _23 = const 1_i64 as i32 (IntToInt);
++         _23 = const 1_i32;
           StorageDead(_24);
-          _22 = opaque::<i32>(move _23) -> [return: bb7, unwind continue];
+-         _22 = opaque::<i32>(move _23) -> [return: bb7, unwind continue];
++         _22 = opaque::<i32>(const 1_i32) -> [return: bb7, unwind continue];
       }
   
       bb7: {
@@ -228,9 +235,10 @@
 -         _29 = _1;
 -         _28 = move _29 as f32 (IntToFloat);
 +         _29 = const 1_i64;
-+         _28 = const 1_i64 as f32 (IntToFloat);
++         _28 = const 1f32;
           StorageDead(_29);
-          _27 = opaque::<f32>(move _28) -> [return: bb9, unwind continue];
+-         _27 = opaque::<f32>(move _28) -> [return: bb9, unwind continue];
++         _27 = opaque::<f32>(const 1f32) -> [return: bb9, unwind continue];
       }
   
       bb9: {
@@ -242,9 +250,10 @@
 -         _32 = _1;
 -         _31 = move _32 as f64 (IntToFloat);
 +         _32 = const 1_i64;
-+         _31 = const 1_i64 as f64 (IntToFloat);
++         _31 = const 1f64;
           StorageDead(_32);
-          _30 = opaque::<f64>(move _31) -> [return: bb10, unwind continue];
+-         _30 = opaque::<f64>(move _31) -> [return: bb10, unwind continue];
++         _30 = opaque::<f64>(const 1f64) -> [return: bb10, unwind continue];
       }
   
       bb10: {
@@ -256,9 +265,10 @@
 -         _35 = _2;
 -         _34 = move _35 as u8 (IntToInt);
 +         _35 = const 1_u64;
-+         _34 = const 1_u64 as u8 (IntToInt);
++         _34 = const 1_u8;
           StorageDead(_35);
-          _33 = opaque::<u8>(move _34) -> [return: bb11, unwind continue];
+-         _33 = opaque::<u8>(move _34) -> [return: bb11, unwind continue];
++         _33 = opaque::<u8>(const 1_u8) -> [return: bb11, unwind continue];
       }
   
       bb11: {
@@ -270,9 +280,10 @@
 -         _38 = _2;
 -         _37 = move _38 as u16 (IntToInt);
 +         _38 = const 1_u64;
-+         _37 = const 1_u64 as u16 (IntToInt);
++         _37 = const 1_u16;
           StorageDead(_38);
-          _36 = opaque::<u16>(move _37) -> [return: bb12, unwind continue];
+-         _36 = opaque::<u16>(move _37) -> [return: bb12, unwind continue];
++         _36 = opaque::<u16>(const 1_u16) -> [return: bb12, unwind continue];
       }
   
       bb12: {
@@ -284,9 +295,10 @@
 -         _41 = _2;
 -         _40 = move _41 as u32 (IntToInt);
 +         _41 = const 1_u64;
-+         _40 = const 1_u64 as u32 (IntToInt);
++         _40 = const 1_u32;
           StorageDead(_41);
-          _39 = opaque::<u32>(move _40) -> [return: bb13, unwind continue];
+-         _39 = opaque::<u32>(move _40) -> [return: bb13, unwind continue];
++         _39 = opaque::<u32>(const 1_u32) -> [return: bb13, unwind continue];
       }
   
       bb13: {
@@ -309,9 +321,10 @@
 -         _46 = _2;
 -         _45 = move _46 as i8 (IntToInt);
 +         _46 = const 1_u64;
-+         _45 = const 1_u64 as i8 (IntToInt);
++         _45 = const 1_i8;
           StorageDead(_46);
-          _44 = opaque::<i8>(move _45) -> [return: bb15, unwind continue];
+-         _44 = opaque::<i8>(move _45) -> [return: bb15, unwind continue];
++         _44 = opaque::<i8>(const 1_i8) -> [return: bb15, unwind continue];
       }
   
       bb15: {
@@ -323,9 +336,10 @@
 -         _49 = _2;
 -         _48 = move _49 as i16 (IntToInt);
 +         _49 = const 1_u64;
-+         _48 = const 1_u64 as i16 (IntToInt);
++         _48 = const 1_i16;
           StorageDead(_49);
-          _47 = opaque::<i16>(move _48) -> [return: bb16, unwind continue];
+-         _47 = opaque::<i16>(move _48) -> [return: bb16, unwind continue];
++         _47 = opaque::<i16>(const 1_i16) -> [return: bb16, unwind continue];
       }
   
       bb16: {
@@ -337,9 +351,10 @@
 -         _52 = _2;
 -         _51 = move _52 as i32 (IntToInt);
 +         _52 = const 1_u64;
-+         _51 = const 1_u64 as i32 (IntToInt);
++         _51 = const 1_i32;
           StorageDead(_52);
-          _50 = opaque::<i32>(move _51) -> [return: bb17, unwind continue];
+-         _50 = opaque::<i32>(move _51) -> [return: bb17, unwind continue];
++         _50 = opaque::<i32>(const 1_i32) -> [return: bb17, unwind continue];
       }
   
       bb17: {
@@ -351,9 +366,10 @@
 -         _55 = _2;
 -         _54 = move _55 as i64 (IntToInt);
 +         _55 = const 1_u64;
-+         _54 = const 1_u64 as i64 (IntToInt);
++         _54 = const 1_i64;
           StorageDead(_55);
-          _53 = opaque::<i64>(move _54) -> [return: bb18, unwind continue];
+-         _53 = opaque::<i64>(move _54) -> [return: bb18, unwind continue];
++         _53 = opaque::<i64>(const 1_i64) -> [return: bb18, unwind continue];
       }
   
       bb18: {
@@ -365,9 +381,10 @@
 -         _58 = _2;
 -         _57 = move _58 as f32 (IntToFloat);
 +         _58 = const 1_u64;
-+         _57 = const 1_u64 as f32 (IntToFloat);
++         _57 = const 1f32;
           StorageDead(_58);
-          _56 = opaque::<f32>(move _57) -> [return: bb19, unwind continue];
+-         _56 = opaque::<f32>(move _57) -> [return: bb19, unwind continue];
++         _56 = opaque::<f32>(const 1f32) -> [return: bb19, unwind continue];
       }
   
       bb19: {
@@ -379,9 +396,10 @@
 -         _61 = _2;
 -         _60 = move _61 as f64 (IntToFloat);
 +         _61 = const 1_u64;
-+         _60 = const 1_u64 as f64 (IntToFloat);
++         _60 = const 1f64;
           StorageDead(_61);
-          _59 = opaque::<f64>(move _60) -> [return: bb20, unwind continue];
+-         _59 = opaque::<f64>(move _60) -> [return: bb20, unwind continue];
++         _59 = opaque::<f64>(const 1f64) -> [return: bb20, unwind continue];
       }
   
       bb20: {
@@ -393,9 +411,10 @@
 -         _64 = _3;
 -         _63 = move _64 as u8 (FloatToInt);
 +         _64 = const 1f64;
-+         _63 = const 1f64 as u8 (FloatToInt);
++         _63 = const 1_u8;
           StorageDead(_64);
-          _62 = opaque::<u8>(move _63) -> [return: bb21, unwind continue];
+-         _62 = opaque::<u8>(move _63) -> [return: bb21, unwind continue];
++         _62 = opaque::<u8>(const 1_u8) -> [return: bb21, unwind continue];
       }
   
       bb21: {
@@ -407,9 +426,10 @@
 -         _67 = _3;
 -         _66 = move _67 as u16 (FloatToInt);
 +         _67 = const 1f64;
-+         _66 = const 1f64 as u16 (FloatToInt);
++         _66 = const 1_u16;
           StorageDead(_67);
-          _65 = opaque::<u16>(move _66) -> [return: bb22, unwind continue];
+-         _65 = opaque::<u16>(move _66) -> [return: bb22, unwind continue];
++         _65 = opaque::<u16>(const 1_u16) -> [return: bb22, unwind continue];
       }
   
       bb22: {
@@ -421,9 +441,10 @@
 -         _70 = _3;
 -         _69 = move _70 as u32 (FloatToInt);
 +         _70 = const 1f64;
-+         _69 = const 1f64 as u32 (FloatToInt);
++         _69 = const 1_u32;
           StorageDead(_70);
-          _68 = opaque::<u32>(move _69) -> [return: bb23, unwind continue];
+-         _68 = opaque::<u32>(move _69) -> [return: bb23, unwind continue];
++         _68 = opaque::<u32>(const 1_u32) -> [return: bb23, unwind continue];
       }
   
       bb23: {
@@ -435,9 +456,10 @@
 -         _73 = _3;
 -         _72 = move _73 as u64 (FloatToInt);
 +         _73 = const 1f64;
-+         _72 = const 1f64 as u64 (FloatToInt);
++         _72 = const 1_u64;
           StorageDead(_73);
-          _71 = opaque::<u64>(move _72) -> [return: bb24, unwind continue];
+-         _71 = opaque::<u64>(move _72) -> [return: bb24, unwind continue];
++         _71 = opaque::<u64>(const 1_u64) -> [return: bb24, unwind continue];
       }
   
       bb24: {
@@ -449,9 +471,10 @@
 -         _76 = _3;
 -         _75 = move _76 as i8 (FloatToInt);
 +         _76 = const 1f64;
-+         _75 = const 1f64 as i8 (FloatToInt);
++         _75 = const 1_i8;
           StorageDead(_76);
-          _74 = opaque::<i8>(move _75) -> [return: bb25, unwind continue];
+-         _74 = opaque::<i8>(move _75) -> [return: bb25, unwind continue];
++         _74 = opaque::<i8>(const 1_i8) -> [return: bb25, unwind continue];
       }
   
       bb25: {
@@ -463,9 +486,10 @@
 -         _79 = _3;
 -         _78 = move _79 as i16 (FloatToInt);
 +         _79 = const 1f64;
-+         _78 = const 1f64 as i16 (FloatToInt);
++         _78 = const 1_i16;
           StorageDead(_79);
-          _77 = opaque::<i16>(move _78) -> [return: bb26, unwind continue];
+-         _77 = opaque::<i16>(move _78) -> [return: bb26, unwind continue];
++         _77 = opaque::<i16>(const 1_i16) -> [return: bb26, unwind continue];
       }
   
       bb26: {
@@ -477,9 +501,10 @@
 -         _82 = _3;
 -         _81 = move _82 as i32 (FloatToInt);
 +         _82 = const 1f64;
-+         _81 = const 1f64 as i32 (FloatToInt);
++         _81 = const 1_i32;
           StorageDead(_82);
-          _80 = opaque::<i32>(move _81) -> [return: bb27, unwind continue];
+-         _80 = opaque::<i32>(move _81) -> [return: bb27, unwind continue];
++         _80 = opaque::<i32>(const 1_i32) -> [return: bb27, unwind continue];
       }
   
       bb27: {
@@ -491,9 +516,10 @@
 -         _85 = _3;
 -         _84 = move _85 as i64 (FloatToInt);
 +         _85 = const 1f64;
-+         _84 = const 1f64 as i64 (FloatToInt);
++         _84 = const 1_i64;
           StorageDead(_85);
-          _83 = opaque::<i64>(move _84) -> [return: bb28, unwind continue];
+-         _83 = opaque::<i64>(move _84) -> [return: bb28, unwind continue];
++         _83 = opaque::<i64>(const 1_i64) -> [return: bb28, unwind continue];
       }
   
       bb28: {
@@ -505,9 +531,10 @@
 -         _88 = _3;
 -         _87 = move _88 as f32 (FloatToFloat);
 +         _88 = const 1f64;
-+         _87 = const 1f64 as f32 (FloatToFloat);
++         _87 = const 1f32;
           StorageDead(_88);
-          _86 = opaque::<f32>(move _87) -> [return: bb29, unwind continue];
+-         _86 = opaque::<f32>(move _87) -> [return: bb29, unwind continue];
++         _86 = opaque::<f32>(const 1f32) -> [return: bb29, unwind continue];
       }
   
       bb29: {
diff --git a/tests/mir-opt/gvn.slices.GVN.panic-abort.diff b/tests/mir-opt/gvn.slices.GVN.panic-abort.diff
index d924c70d617d..98255c42007d 100644
--- a/tests/mir-opt/gvn.slices.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.slices.GVN.panic-abort.diff
@@ -176,12 +176,13 @@
           StorageDead(_19);
           StorageDead(_18);
 -         StorageLive(_21);
+-         _21 = core::panicking::AssertKind::Eq;
 +         nop;
-          _21 = core::panicking::AssertKind::Eq;
++         _21 = const core::panicking::AssertKind::Eq;
           StorageLive(_22);
           StorageLive(_23);
 -         _23 = move _21;
-+         _23 = _21;
++         _23 = const core::panicking::AssertKind::Eq;
           StorageLive(_24);
           StorageLive(_25);
           _25 = &(*_15);
@@ -193,7 +194,7 @@
           StorageLive(_28);
           _28 = Option::<Arguments<'_>>::None;
 -         _22 = core::panicking::assert_failed::<*const u8, *const u8>(move _23, move _24, move _26, move _28) -> unwind unreachable;
-+         _22 = core::panicking::assert_failed::<*const u8, *const u8>(_21, move _24, move _26, move _28) -> unwind unreachable;
++         _22 = core::panicking::assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _24, move _26, move _28) -> unwind unreachable;
       }
   
       bb7: {
@@ -261,12 +262,13 @@
           StorageDead(_45);
           StorageDead(_44);
 -         StorageLive(_47);
+-         _47 = core::panicking::AssertKind::Eq;
 +         nop;
-          _47 = core::panicking::AssertKind::Eq;
++         _47 = const core::panicking::AssertKind::Eq;
           StorageLive(_48);
           StorageLive(_49);
 -         _49 = move _47;
-+         _49 = _47;
++         _49 = const core::panicking::AssertKind::Eq;
           StorageLive(_50);
           StorageLive(_51);
           _51 = &(*_41);
@@ -278,7 +280,7 @@
           StorageLive(_54);
           _54 = Option::<Arguments<'_>>::None;
 -         _48 = core::panicking::assert_failed::<*const u8, *const u8>(move _49, move _50, move _52, move _54) -> unwind unreachable;
-+         _48 = core::panicking::assert_failed::<*const u8, *const u8>(_47, move _50, move _52, move _54) -> unwind unreachable;
++         _48 = core::panicking::assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _50, move _52, move _54) -> unwind unreachable;
       }
   }
   
diff --git a/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff b/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff
index 63225aa0a101..0f79cc409f31 100644
--- a/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff
@@ -176,12 +176,13 @@
           StorageDead(_19);
           StorageDead(_18);
 -         StorageLive(_21);
+-         _21 = core::panicking::AssertKind::Eq;
 +         nop;
-          _21 = core::panicking::AssertKind::Eq;
++         _21 = const core::panicking::AssertKind::Eq;
           StorageLive(_22);
           StorageLive(_23);
 -         _23 = move _21;
-+         _23 = _21;
++         _23 = const core::panicking::AssertKind::Eq;
           StorageLive(_24);
           StorageLive(_25);
           _25 = &(*_15);
@@ -193,7 +194,7 @@
           StorageLive(_28);
           _28 = Option::<Arguments<'_>>::None;
 -         _22 = core::panicking::assert_failed::<*const u8, *const u8>(move _23, move _24, move _26, move _28) -> unwind continue;
-+         _22 = core::panicking::assert_failed::<*const u8, *const u8>(_21, move _24, move _26, move _28) -> unwind continue;
++         _22 = core::panicking::assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _24, move _26, move _28) -> unwind continue;
       }
   
       bb7: {
@@ -261,12 +262,13 @@
           StorageDead(_45);
           StorageDead(_44);
 -         StorageLive(_47);
+-         _47 = core::panicking::AssertKind::Eq;
 +         nop;
-          _47 = core::panicking::AssertKind::Eq;
++         _47 = const core::panicking::AssertKind::Eq;
           StorageLive(_48);
           StorageLive(_49);
 -         _49 = move _47;
-+         _49 = _47;
++         _49 = const core::panicking::AssertKind::Eq;
           StorageLive(_50);
           StorageLive(_51);
           _51 = &(*_41);
@@ -278,7 +280,7 @@
           StorageLive(_54);
           _54 = Option::<Arguments<'_>>::None;
 -         _48 = core::panicking::assert_failed::<*const u8, *const u8>(move _49, move _50, move _52, move _54) -> unwind continue;
-+         _48 = core::panicking::assert_failed::<*const u8, *const u8>(_47, move _50, move _52, move _54) -> unwind continue;
++         _48 = core::panicking::assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _50, move _52, move _54) -> unwind continue;
       }
   }
   

From db9bd9bd8c42aa06d5d822a1f04bc2007601ca43 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 23 Sep 2023 14:55:28 +0000
Subject: [PATCH 059/297] Do not intern too large aggregates.

---
 compiler/rustc_mir_transform/src/gvn.rs | 42 ++++++++++++++-----------
 1 file changed, 24 insertions(+), 18 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index c38ebdf512b8..7800aa637292 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -291,24 +291,30 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                     .collect::<Option<Vec<_>>>()?;
                 let variant = if ty.is_enum() { Some(variant) } else { None };
                 let ty = self.ecx.layout_of(ty).ok()?;
-                let alloc_id = self
-                    .ecx
-                    .intern_with_temp_alloc(ty, |ecx, dest| {
-                        let variant_dest = if let Some(variant) = variant {
-                            ecx.project_downcast(dest, variant)?
-                        } else {
-                            dest.clone()
-                        };
-                        for (field_index, op) in fields.into_iter().enumerate() {
-                            let field_dest = ecx.project_field(&variant_dest, field_index)?;
-                            ecx.copy_op(op, &field_dest, /*allow_transmute*/ false)?;
-                        }
-                        ecx.write_discriminant(variant.unwrap_or(FIRST_VARIANT), dest)
-                    })
-                    .ok()?;
-                let mplace =
-                    self.ecx.raw_const_to_mplace(ConstAlloc { alloc_id, ty: ty.ty }).ok()?;
-                mplace.into()
+                if ty.is_zst() {
+                    ImmTy::uninit(ty).into()
+                } else if matches!(ty.abi, Abi::Scalar(..) | Abi::ScalarPair(..)) {
+                    let alloc_id = self
+                        .ecx
+                        .intern_with_temp_alloc(ty, |ecx, dest| {
+                            let variant_dest = if let Some(variant) = variant {
+                                ecx.project_downcast(dest, variant)?
+                            } else {
+                                dest.clone()
+                            };
+                            for (field_index, op) in fields.into_iter().enumerate() {
+                                let field_dest = ecx.project_field(&variant_dest, field_index)?;
+                                ecx.copy_op(op, &field_dest, /*allow_transmute*/ false)?;
+                            }
+                            ecx.write_discriminant(variant.unwrap_or(FIRST_VARIANT), dest)
+                        })
+                        .ok()?;
+                    let mplace =
+                        self.ecx.raw_const_to_mplace(ConstAlloc { alloc_id, ty: ty.ty }).ok()?;
+                    mplace.into()
+                } else {
+                    return None;
+                }
             }
 
             Projection(base, elem) => {

From 9389373127705dd4ceb460e6e152cfb9126e6be5 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 23 Sep 2023 07:23:14 +0000
Subject: [PATCH 060/297] Do not transmute immediates to non-immediates.

---
 compiler/rustc_mir_transform/src/gvn.rs | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index 7800aa637292..60ff1dd0d782 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -443,6 +443,16 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                 CastKind::Transmute => {
                     let value = self.evaluated[value].as_ref()?;
                     let to = self.ecx.layout_of(to).ok()?;
+                    // `offset` for immediates only supports scalar/scalar-pair ABIs,
+                    // so bail out if the target is not one.
+                    if value.as_mplace_or_imm().is_right() {
+                        match to.abi {
+                            Abi::Scalar(..) | Abi::ScalarPair(..) => {}
+                            _ if to.is_zst() => {}
+                            Abi::Aggregate { .. } if to.fields.count() == 0 => {}
+                            _ => return None,
+                        }
+                    }
                     value.offset(Size::ZERO, to, &self.ecx).ok()?
                 }
                 _ => return None,

From 692e5286479e16b3e9b47ab842b543985e1a6ceb Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 16 Sep 2023 09:16:04 +0000
Subject: [PATCH 061/297] Simplify projections in GVN.

---
 compiler/rustc_mir_transform/src/gvn.rs       | 155 ++++++++++++------
 compiler/rustc_mir_transform/src/lib.rs       |   1 +
 .../gvn.dereferences.GVN.panic-abort.diff     |  18 +-
 .../gvn.dereferences.GVN.panic-unwind.diff    |  18 +-
 .../gvn.references.GVN.panic-abort.diff       |  76 ++++++++-
 .../gvn.references.GVN.panic-unwind.diff      |  98 +++++++++--
 .../gvn.repeated_index.GVN.panic-abort.diff   |   6 +-
 .../gvn.repeated_index.GVN.panic-unwind.diff  |   6 +-
 tests/mir-opt/gvn.rs                          |   7 +
 tests/mir-opt/gvn.slices.GVN.panic-abort.diff |  39 +++--
 .../mir-opt/gvn.slices.GVN.panic-unwind.diff  |  39 +++--
 ...xpression_elimination.GVN.panic-abort.diff |  30 ++--
 ...pression_elimination.GVN.panic-unwind.diff |  30 ++--
 13 files changed, 396 insertions(+), 127 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index 60ff1dd0d782..eaa1c70a5e9b 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -66,6 +66,7 @@ use rustc_middle::ty::layout::LayoutOf;
 use rustc_middle::ty::{self, Ty, TyCtxt, TypeAndMut};
 use rustc_span::DUMMY_SP;
 use rustc_target::abi::{self, Abi, Size, VariantIdx, FIRST_VARIANT};
+use std::borrow::Cow;
 
 use crate::dataflow_const_prop::DummyMachine;
 use crate::ssa::{AssignedValue, SsaLocals};
@@ -461,6 +462,87 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         Some(op)
     }
 
+    fn project(
+        &mut self,
+        place: PlaceRef<'tcx>,
+        value: VnIndex,
+        proj: PlaceElem<'tcx>,
+    ) -> Option<VnIndex> {
+        let proj = match proj {
+            ProjectionElem::Deref => {
+                let ty = place.ty(self.local_decls, self.tcx).ty;
+                if let Some(Mutability::Not) = ty.ref_mutability()
+                    && let Some(pointee_ty) = ty.builtin_deref(true)
+                    && pointee_ty.ty.is_freeze(self.tcx, self.param_env)
+                {
+                    // An immutable borrow `_x` always points to the same value for the
+                    // lifetime of the borrow, so we can merge all instances of `*_x`.
+                    ProjectionElem::Deref
+                } else {
+                    return None;
+                }
+            }
+            ProjectionElem::Downcast(name, index) => ProjectionElem::Downcast(name, index),
+            ProjectionElem::Field(f, ty) => ProjectionElem::Field(f, ty),
+            ProjectionElem::Index(idx) => {
+                let idx = self.locals[idx]?;
+                ProjectionElem::Index(idx)
+            }
+            ProjectionElem::ConstantIndex { offset, min_length, from_end } => {
+                ProjectionElem::ConstantIndex { offset, min_length, from_end }
+            }
+            ProjectionElem::Subslice { from, to, from_end } => {
+                ProjectionElem::Subslice { from, to, from_end }
+            }
+            ProjectionElem::OpaqueCast(ty) => ProjectionElem::OpaqueCast(ty),
+            ProjectionElem::Subtype(ty) => ProjectionElem::Subtype(ty),
+        };
+
+        Some(self.insert(Value::Projection(value, proj)))
+    }
+
+    /// Simplify the projection chain if we know better.
+    #[instrument(level = "trace", skip(self))]
+    fn simplify_place_projection(&mut self, place: &mut Place<'tcx>, location: Location) {
+        // If the projection is indirect, we treat the local as a value, so can replace it with
+        // another local.
+        if place.is_indirect()
+            && let Some(base) = self.locals[place.local]
+            && let Some(new_local) = self.try_as_local(base, location)
+        {
+            place.local = new_local;
+            self.reused_locals.insert(new_local);
+        }
+
+        let mut projection = Cow::Borrowed(&place.projection[..]);
+
+        for i in 0..projection.len() {
+            let elem = projection[i];
+            if let ProjectionElem::Index(idx) = elem
+                && let Some(idx) = self.locals[idx]
+            {
+                if let Some(offset) = self.evaluated[idx].as_ref()
+                    && let Ok(offset) = self.ecx.read_target_usize(offset)
+                {
+                    projection.to_mut()[i] = ProjectionElem::ConstantIndex {
+                        offset,
+                        min_length: offset + 1,
+                        from_end: false,
+                    };
+                } else if let Some(new_idx) = self.try_as_local(idx, location) {
+                    projection.to_mut()[i] = ProjectionElem::Index(new_idx);
+                    self.reused_locals.insert(new_idx);
+                }
+            }
+        }
+
+        if projection.is_owned() {
+            place.projection = self.tcx.mk_place_elems(&projection);
+        }
+
+        trace!(?place);
+    }
+
     /// Represent the *value* which would be read from `place`, and point `place` to a preexisting
     /// place with the same value (if that already exists).
     #[instrument(level = "trace", skip(self), ret)]
@@ -469,6 +551,8 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         place: &mut Place<'tcx>,
         location: Location,
     ) -> Option<VnIndex> {
+        self.simplify_place_projection(place, location);
+
         // Invariant: `place` and `place_ref` point to the same value, even if they point to
         // different memory locations.
         let mut place_ref = place.as_ref();
@@ -483,53 +567,15 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                 place_ref = PlaceRef { local, projection: &place.projection[index..] };
             }
 
-            let proj = match proj {
-                ProjectionElem::Deref => {
-                    let ty = Place::ty_from(
-                        place.local,
-                        &place.projection[..index],
-                        self.local_decls,
-                        self.tcx,
-                    )
-                    .ty;
-                    if let Some(Mutability::Not) = ty.ref_mutability()
-                        && let Some(pointee_ty) = ty.builtin_deref(true)
-                        && pointee_ty.ty.is_freeze(self.tcx, self.param_env)
-                    {
-                        // An immutable borrow `_x` always points to the same value for the
-                        // lifetime of the borrow, so we can merge all instances of `*_x`.
-                        ProjectionElem::Deref
-                    } else {
-                        return None;
-                    }
-                }
-                ProjectionElem::Field(f, ty) => ProjectionElem::Field(f, ty),
-                ProjectionElem::Index(idx) => {
-                    let idx = self.locals[idx]?;
-                    ProjectionElem::Index(idx)
-                }
-                ProjectionElem::ConstantIndex { offset, min_length, from_end } => {
-                    ProjectionElem::ConstantIndex { offset, min_length, from_end }
-                }
-                ProjectionElem::Subslice { from, to, from_end } => {
-                    ProjectionElem::Subslice { from, to, from_end }
-                }
-                ProjectionElem::Downcast(name, index) => ProjectionElem::Downcast(name, index),
-                ProjectionElem::OpaqueCast(ty) => ProjectionElem::OpaqueCast(ty),
-                ProjectionElem::Subtype(ty) => ProjectionElem::Subtype(ty),
-            };
-            value = self.insert(Value::Projection(value, proj));
+            let base = PlaceRef { local: place.local, projection: &place.projection[..index] };
+            value = self.project(base, value, proj)?;
         }
 
-        if let Some(local) = self.try_as_local(value, location)
-            && local != place.local
-        // in case we had no projection to begin with.
-        {
-            *place = local.into();
-            self.reused_locals.insert(local);
-        } else if place_ref.local != place.local
-            || place_ref.projection.len() < place.projection.len()
-        {
+        if let Some(new_local) = self.try_as_local(value, location) {
+            place_ref = PlaceRef { local: new_local, projection: &[] };
+        }
+
+        if place_ref.local != place.local || place_ref.projection.len() < place.projection.len() {
             // By the invariant on `place_ref`.
             *place = place_ref.project_deeper(&[], self.tcx);
             self.reused_locals.insert(place_ref.local);
@@ -545,7 +591,10 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         location: Location,
     ) -> Option<VnIndex> {
         match *operand {
-            Operand::Constant(ref constant) => Some(self.insert(Value::Constant(constant.const_))),
+            Operand::Constant(ref mut constant) => {
+                let const_ = constant.const_.normalize(self.tcx, self.param_env);
+                Some(self.insert(Value::Constant(const_)))
+            }
             Operand::Copy(ref mut place) | Operand::Move(ref mut place) => {
                 let value = self.simplify_place_value(place, location)?;
                 if let Some(const_) = self.try_as_constant(value) {
@@ -595,11 +644,13 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                 let ty = rvalue.ty(self.local_decls, self.tcx);
                 Value::Aggregate(ty, variant_index, fields?)
             }
-            Rvalue::Ref(_, borrow_kind, place) => {
-                return self.new_pointer(place, AddressKind::Ref(borrow_kind));
+            Rvalue::Ref(_, borrow_kind, ref mut place) => {
+                self.simplify_place_projection(place, location);
+                return self.new_pointer(*place, AddressKind::Ref(borrow_kind));
             }
-            Rvalue::AddressOf(mutbl, place) => {
-                return self.new_pointer(place, AddressKind::Address(mutbl));
+            Rvalue::AddressOf(mutbl, ref mut place) => {
+                self.simplify_place_projection(place, location);
+                return self.new_pointer(*place, AddressKind::Address(mutbl));
             }
 
             // Operations.
@@ -757,6 +808,10 @@ impl<'tcx> MutVisitor<'tcx> for VnState<'_, 'tcx> {
         self.tcx
     }
 
+    fn visit_place(&mut self, place: &mut Place<'tcx>, _: PlaceContext, location: Location) {
+        self.simplify_place_projection(place, location);
+    }
+
     fn visit_operand(&mut self, operand: &mut Operand<'tcx>, location: Location) {
         self.simplify_operand(operand, location);
     }
diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs
index 4ec91a55f1d2..bdd5bf702c8a 100644
--- a/compiler/rustc_mir_transform/src/lib.rs
+++ b/compiler/rustc_mir_transform/src/lib.rs
@@ -2,6 +2,7 @@
 #![deny(rustc::untranslatable_diagnostic)]
 #![deny(rustc::diagnostic_outside_of_impl)]
 #![feature(box_patterns)]
+#![feature(cow_is_borrowed)]
 #![feature(decl_macro)]
 #![feature(is_sorted)]
 #![feature(let_chains)]
diff --git a/tests/mir-opt/gvn.dereferences.GVN.panic-abort.diff b/tests/mir-opt/gvn.dereferences.GVN.panic-abort.diff
index 46bf13985daf..a587b1e6b1de 100644
--- a/tests/mir-opt/gvn.dereferences.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.dereferences.GVN.panic-abort.diff
@@ -72,7 +72,8 @@
       bb2: {
           StorageDead(_7);
           StorageDead(_6);
-          StorageLive(_8);
+-         StorageLive(_8);
++         nop;
           _8 = &raw const (*_1);
           StorageLive(_9);
           StorageLive(_10);
@@ -92,7 +93,8 @@
       bb4: {
           StorageDead(_12);
           StorageDead(_11);
-          StorageLive(_13);
+-         StorageLive(_13);
++         nop;
           _13 = &raw mut (*_1);
           StorageLive(_14);
           StorageLive(_15);
@@ -112,7 +114,8 @@
       bb6: {
           StorageDead(_17);
           StorageDead(_16);
-          StorageLive(_18);
+-         StorageLive(_18);
++         nop;
           _18 = &(*_1);
           StorageLive(_19);
 -         StorageLive(_20);
@@ -188,9 +191,12 @@
           StorageDead(_32);
           StorageDead(_31);
           _0 = const ();
-          StorageDead(_18);
-          StorageDead(_13);
-          StorageDead(_8);
+-         StorageDead(_18);
+-         StorageDead(_13);
+-         StorageDead(_8);
++         nop;
++         nop;
++         nop;
           return;
       }
   }
diff --git a/tests/mir-opt/gvn.dereferences.GVN.panic-unwind.diff b/tests/mir-opt/gvn.dereferences.GVN.panic-unwind.diff
index 3e731ead859e..6fdda5e99887 100644
--- a/tests/mir-opt/gvn.dereferences.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.dereferences.GVN.panic-unwind.diff
@@ -72,7 +72,8 @@
       bb2: {
           StorageDead(_7);
           StorageDead(_6);
-          StorageLive(_8);
+-         StorageLive(_8);
++         nop;
           _8 = &raw const (*_1);
           StorageLive(_9);
           StorageLive(_10);
@@ -92,7 +93,8 @@
       bb4: {
           StorageDead(_12);
           StorageDead(_11);
-          StorageLive(_13);
+-         StorageLive(_13);
++         nop;
           _13 = &raw mut (*_1);
           StorageLive(_14);
           StorageLive(_15);
@@ -112,7 +114,8 @@
       bb6: {
           StorageDead(_17);
           StorageDead(_16);
-          StorageLive(_18);
+-         StorageLive(_18);
++         nop;
           _18 = &(*_1);
           StorageLive(_19);
 -         StorageLive(_20);
@@ -188,9 +191,12 @@
           StorageDead(_32);
           StorageDead(_31);
           _0 = const ();
-          StorageDead(_18);
-          StorageDead(_13);
-          StorageDead(_8);
+-         StorageDead(_18);
+-         StorageDead(_13);
+-         StorageDead(_8);
++         nop;
++         nop;
++         nop;
           return;
       }
   }
diff --git a/tests/mir-opt/gvn.references.GVN.panic-abort.diff b/tests/mir-opt/gvn.references.GVN.panic-abort.diff
index b7ad4ab1fd3d..885431725657 100644
--- a/tests/mir-opt/gvn.references.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.references.GVN.panic-abort.diff
@@ -20,6 +20,24 @@
       let mut _15: *mut impl Sized;
       let _16: ();
       let mut _17: *mut impl Sized;
+      let _18: &mut impl Sized;
+      let mut _20: S<&mut impl Sized>;
+      let mut _21: &mut impl Sized;
+      let _22: ();
+      let mut _23: &impl Sized;
+      let _24: ();
+      let mut _25: &mut impl Sized;
+      let _26: ();
+      let mut _27: *const impl Sized;
+      let _28: ();
+      let mut _29: *mut impl Sized;
+      scope 1 {
+          debug r => _18;
+          let _19: &mut impl Sized;
+          scope 2 {
+              debug s => _19;
+          }
+      }
   
       bb0: {
           StorageLive(_2);
@@ -94,11 +112,65 @@
       bb8: {
           StorageDead(_17);
           StorageDead(_16);
-          _0 = const ();
-          drop(_1) -> [return: bb9, unwind unreachable];
+-         StorageLive(_18);
++         nop;
+          _18 = &mut _1;
+-         StorageLive(_19);
++         nop;
+          StorageLive(_20);
+          StorageLive(_21);
+-         _21 = move _18;
+-         _20 = S::<&mut impl Sized>(move _21);
++         _21 = _18;
++         _20 = S::<&mut impl Sized>(_18);
+          StorageDead(_21);
+          _19 = move (_20.0: &mut impl Sized);
+          StorageDead(_20);
+          StorageLive(_22);
+          StorageLive(_23);
+          _23 = &(*_19);
+          _22 = opaque::<&impl Sized>(move _23) -> [return: bb9, unwind unreachable];
       }
   
       bb9: {
+          StorageDead(_23);
+          StorageDead(_22);
+          StorageLive(_24);
+          StorageLive(_25);
+          _25 = &mut (*_19);
+          _24 = opaque::<&mut impl Sized>(move _25) -> [return: bb10, unwind unreachable];
+      }
+  
+      bb10: {
+          StorageDead(_25);
+          StorageDead(_24);
+          StorageLive(_26);
+          StorageLive(_27);
+          _27 = &raw const (*_19);
+          _26 = opaque::<*const impl Sized>(move _27) -> [return: bb11, unwind unreachable];
+      }
+  
+      bb11: {
+          StorageDead(_27);
+          StorageDead(_26);
+          StorageLive(_28);
+          StorageLive(_29);
+          _29 = &raw mut (*_19);
+          _28 = opaque::<*mut impl Sized>(move _29) -> [return: bb12, unwind unreachable];
+      }
+  
+      bb12: {
+          StorageDead(_29);
+          StorageDead(_28);
+          _0 = const ();
+-         StorageDead(_19);
+-         StorageDead(_18);
++         nop;
++         nop;
+          drop(_1) -> [return: bb13, unwind unreachable];
+      }
+  
+      bb13: {
           return;
       }
   }
diff --git a/tests/mir-opt/gvn.references.GVN.panic-unwind.diff b/tests/mir-opt/gvn.references.GVN.panic-unwind.diff
index 08ed4c629a6d..106b453fe800 100644
--- a/tests/mir-opt/gvn.references.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.references.GVN.panic-unwind.diff
@@ -20,12 +20,30 @@
       let mut _15: *mut impl Sized;
       let _16: ();
       let mut _17: *mut impl Sized;
+      let _18: &mut impl Sized;
+      let mut _20: S<&mut impl Sized>;
+      let mut _21: &mut impl Sized;
+      let _22: ();
+      let mut _23: &impl Sized;
+      let _24: ();
+      let mut _25: &mut impl Sized;
+      let _26: ();
+      let mut _27: *const impl Sized;
+      let _28: ();
+      let mut _29: *mut impl Sized;
+      scope 1 {
+          debug r => _18;
+          let _19: &mut impl Sized;
+          scope 2 {
+              debug s => _19;
+          }
+      }
   
       bb0: {
           StorageLive(_2);
           StorageLive(_3);
           _3 = &_1;
-          _2 = opaque::<&impl Sized>(move _3) -> [return: bb1, unwind: bb10];
+          _2 = opaque::<&impl Sized>(move _3) -> [return: bb1, unwind: bb14];
       }
   
       bb1: {
@@ -34,7 +52,7 @@
           StorageLive(_4);
           StorageLive(_5);
           _5 = &_1;
-          _4 = opaque::<&impl Sized>(move _5) -> [return: bb2, unwind: bb10];
+          _4 = opaque::<&impl Sized>(move _5) -> [return: bb2, unwind: bb14];
       }
   
       bb2: {
@@ -43,7 +61,7 @@
           StorageLive(_6);
           StorageLive(_7);
           _7 = &mut _1;
-          _6 = opaque::<&mut impl Sized>(move _7) -> [return: bb3, unwind: bb10];
+          _6 = opaque::<&mut impl Sized>(move _7) -> [return: bb3, unwind: bb14];
       }
   
       bb3: {
@@ -52,7 +70,7 @@
           StorageLive(_8);
           StorageLive(_9);
           _9 = &mut _1;
-          _8 = opaque::<&mut impl Sized>(move _9) -> [return: bb4, unwind: bb10];
+          _8 = opaque::<&mut impl Sized>(move _9) -> [return: bb4, unwind: bb14];
       }
   
       bb4: {
@@ -61,7 +79,7 @@
           StorageLive(_10);
           StorageLive(_11);
           _11 = &raw const _1;
-          _10 = opaque::<*const impl Sized>(move _11) -> [return: bb5, unwind: bb10];
+          _10 = opaque::<*const impl Sized>(move _11) -> [return: bb5, unwind: bb14];
       }
   
       bb5: {
@@ -70,7 +88,7 @@
           StorageLive(_12);
           StorageLive(_13);
           _13 = &raw const _1;
-          _12 = opaque::<*const impl Sized>(move _13) -> [return: bb6, unwind: bb10];
+          _12 = opaque::<*const impl Sized>(move _13) -> [return: bb6, unwind: bb14];
       }
   
       bb6: {
@@ -79,7 +97,7 @@
           StorageLive(_14);
           StorageLive(_15);
           _15 = &raw mut _1;
-          _14 = opaque::<*mut impl Sized>(move _15) -> [return: bb7, unwind: bb10];
+          _14 = opaque::<*mut impl Sized>(move _15) -> [return: bb7, unwind: bb14];
       }
   
       bb7: {
@@ -88,25 +106,79 @@
           StorageLive(_16);
           StorageLive(_17);
           _17 = &raw mut _1;
-          _16 = opaque::<*mut impl Sized>(move _17) -> [return: bb8, unwind: bb10];
+          _16 = opaque::<*mut impl Sized>(move _17) -> [return: bb8, unwind: bb14];
       }
   
       bb8: {
           StorageDead(_17);
           StorageDead(_16);
-          _0 = const ();
-          drop(_1) -> [return: bb9, unwind: bb11];
+-         StorageLive(_18);
++         nop;
+          _18 = &mut _1;
+-         StorageLive(_19);
++         nop;
+          StorageLive(_20);
+          StorageLive(_21);
+-         _21 = move _18;
+-         _20 = S::<&mut impl Sized>(move _21);
++         _21 = _18;
++         _20 = S::<&mut impl Sized>(_18);
+          StorageDead(_21);
+          _19 = move (_20.0: &mut impl Sized);
+          StorageDead(_20);
+          StorageLive(_22);
+          StorageLive(_23);
+          _23 = &(*_19);
+          _22 = opaque::<&impl Sized>(move _23) -> [return: bb9, unwind: bb14];
       }
   
       bb9: {
+          StorageDead(_23);
+          StorageDead(_22);
+          StorageLive(_24);
+          StorageLive(_25);
+          _25 = &mut (*_19);
+          _24 = opaque::<&mut impl Sized>(move _25) -> [return: bb10, unwind: bb14];
+      }
+  
+      bb10: {
+          StorageDead(_25);
+          StorageDead(_24);
+          StorageLive(_26);
+          StorageLive(_27);
+          _27 = &raw const (*_19);
+          _26 = opaque::<*const impl Sized>(move _27) -> [return: bb11, unwind: bb14];
+      }
+  
+      bb11: {
+          StorageDead(_27);
+          StorageDead(_26);
+          StorageLive(_28);
+          StorageLive(_29);
+          _29 = &raw mut (*_19);
+          _28 = opaque::<*mut impl Sized>(move _29) -> [return: bb12, unwind: bb14];
+      }
+  
+      bb12: {
+          StorageDead(_29);
+          StorageDead(_28);
+          _0 = const ();
+-         StorageDead(_19);
+-         StorageDead(_18);
++         nop;
++         nop;
+          drop(_1) -> [return: bb13, unwind: bb15];
+      }
+  
+      bb13: {
           return;
       }
   
-      bb10 (cleanup): {
-          drop(_1) -> [return: bb11, unwind terminate(cleanup)];
+      bb14 (cleanup): {
+          drop(_1) -> [return: bb15, unwind terminate(cleanup)];
       }
   
-      bb11 (cleanup): {
+      bb15 (cleanup): {
           resume;
       }
   }
diff --git a/tests/mir-opt/gvn.repeated_index.GVN.panic-abort.diff b/tests/mir-opt/gvn.repeated_index.GVN.panic-abort.diff
index 6e542d7f964c..91af1ccc33a9 100644
--- a/tests/mir-opt/gvn.repeated_index.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.repeated_index.GVN.panic-abort.diff
@@ -40,7 +40,8 @@
       }
   
       bb1: {
-          _6 = _3[_7];
+-         _6 = _3[_7];
++         _6 = _3[0 of 1];
           _5 = opaque::<T>(move _6) -> [return: bb2, unwind unreachable];
       }
   
@@ -61,7 +62,8 @@
       }
   
       bb3: {
-          _11 = _3[_12];
+-         _11 = _3[_12];
++         _11 = _3[_2];
           _10 = opaque::<T>(move _11) -> [return: bb4, unwind unreachable];
       }
   
diff --git a/tests/mir-opt/gvn.repeated_index.GVN.panic-unwind.diff b/tests/mir-opt/gvn.repeated_index.GVN.panic-unwind.diff
index a07cd49ec7c6..f887094b509a 100644
--- a/tests/mir-opt/gvn.repeated_index.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.repeated_index.GVN.panic-unwind.diff
@@ -40,7 +40,8 @@
       }
   
       bb1: {
-          _6 = _3[_7];
+-         _6 = _3[_7];
++         _6 = _3[0 of 1];
           _5 = opaque::<T>(move _6) -> [return: bb2, unwind continue];
       }
   
@@ -61,7 +62,8 @@
       }
   
       bb3: {
-          _11 = _3[_12];
+-         _11 = _3[_12];
++         _11 = _3[_2];
           _10 = opaque::<T>(move _11) -> [return: bb4, unwind continue];
       }
   
diff --git a/tests/mir-opt/gvn.rs b/tests/mir-opt/gvn.rs
index fd24edc676c4..3633f9c23cd5 100644
--- a/tests/mir-opt/gvn.rs
+++ b/tests/mir-opt/gvn.rs
@@ -192,6 +192,13 @@ fn references(mut x: impl Sized) {
     opaque(&raw const x); // should not reuse a
     opaque(&raw mut x);
     opaque(&raw mut x); // should not reuse a
+
+    let r = &mut x;
+    let s = S(r).0; // Obfuscate `r`.
+    opaque(&*s); // This is `&*r`.
+    opaque(&mut *s); // This is `&*r`.
+    opaque(&raw const *s); // This is `&*r`.
+    opaque(&raw mut *s); // This is `&*r`.
 }
 
 fn dereferences(t: &mut u32, u: &impl Copy, s: &S<u32>) {
diff --git a/tests/mir-opt/gvn.slices.GVN.panic-abort.diff b/tests/mir-opt/gvn.slices.GVN.panic-abort.diff
index 98255c42007d..6168476d6f27 100644
--- a/tests/mir-opt/gvn.slices.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.slices.GVN.panic-abort.diff
@@ -125,7 +125,8 @@
           StorageLive(_12);
           StorageLive(_13);
           StorageLive(_14);
-          _14 = &(*_4);
+-         _14 = &(*_4);
++         _14 = &(*_1);
           _13 = core::str::<impl str>::as_ptr(move _14) -> [return: bb4, unwind unreachable];
       }
   
@@ -135,9 +136,11 @@
           _8 = (move _9, move _12);
           StorageDead(_12);
           StorageDead(_9);
-          StorageLive(_15);
+-         StorageLive(_15);
++         nop;
           _15 = (_8.0: &*const u8);
-          StorageLive(_16);
+-         StorageLive(_16);
++         nop;
           _16 = (_8.1: &*const u8);
           StorageLive(_17);
           StorageLive(_18);
@@ -153,8 +156,10 @@
           StorageDead(_18);
           _7 = const ();
           StorageDead(_17);
-          StorageDead(_16);
-          StorageDead(_15);
+-         StorageDead(_16);
+-         StorageDead(_15);
++         nop;
++         nop;
           StorageDead(_13);
           StorageDead(_10);
           StorageDead(_8);
@@ -184,11 +189,13 @@
 -         _23 = move _21;
 +         _23 = const core::panicking::AssertKind::Eq;
           StorageLive(_24);
-          StorageLive(_25);
+-         StorageLive(_25);
++         nop;
           _25 = &(*_15);
           _24 = &(*_25);
           StorageLive(_26);
-          StorageLive(_27);
+-         StorageLive(_27);
++         nop;
           _27 = &(*_16);
           _26 = &(*_27);
           StorageLive(_28);
@@ -225,9 +232,11 @@
           _34 = (move _35, move _38);
           StorageDead(_38);
           StorageDead(_35);
-          StorageLive(_41);
+-         StorageLive(_41);
++         nop;
           _41 = (_34.0: &*const u8);
-          StorageLive(_42);
+-         StorageLive(_42);
++         nop;
           _42 = (_34.1: &*const u8);
           StorageLive(_43);
           StorageLive(_44);
@@ -243,8 +252,10 @@
           StorageDead(_44);
           _33 = const ();
           StorageDead(_43);
-          StorageDead(_42);
-          StorageDead(_41);
+-         StorageDead(_42);
+-         StorageDead(_41);
++         nop;
++         nop;
           StorageDead(_39);
           StorageDead(_36);
           StorageDead(_34);
@@ -270,11 +281,13 @@
 -         _49 = move _47;
 +         _49 = const core::panicking::AssertKind::Eq;
           StorageLive(_50);
-          StorageLive(_51);
+-         StorageLive(_51);
++         nop;
           _51 = &(*_41);
           _50 = &(*_51);
           StorageLive(_52);
-          StorageLive(_53);
+-         StorageLive(_53);
++         nop;
           _53 = &(*_42);
           _52 = &(*_53);
           StorageLive(_54);
diff --git a/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff b/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff
index 0f79cc409f31..f60038858089 100644
--- a/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff
@@ -125,7 +125,8 @@
           StorageLive(_12);
           StorageLive(_13);
           StorageLive(_14);
-          _14 = &(*_4);
+-         _14 = &(*_4);
++         _14 = &(*_1);
           _13 = core::str::<impl str>::as_ptr(move _14) -> [return: bb4, unwind continue];
       }
   
@@ -135,9 +136,11 @@
           _8 = (move _9, move _12);
           StorageDead(_12);
           StorageDead(_9);
-          StorageLive(_15);
+-         StorageLive(_15);
++         nop;
           _15 = (_8.0: &*const u8);
-          StorageLive(_16);
+-         StorageLive(_16);
++         nop;
           _16 = (_8.1: &*const u8);
           StorageLive(_17);
           StorageLive(_18);
@@ -153,8 +156,10 @@
           StorageDead(_18);
           _7 = const ();
           StorageDead(_17);
-          StorageDead(_16);
-          StorageDead(_15);
+-         StorageDead(_16);
+-         StorageDead(_15);
++         nop;
++         nop;
           StorageDead(_13);
           StorageDead(_10);
           StorageDead(_8);
@@ -184,11 +189,13 @@
 -         _23 = move _21;
 +         _23 = const core::panicking::AssertKind::Eq;
           StorageLive(_24);
-          StorageLive(_25);
+-         StorageLive(_25);
++         nop;
           _25 = &(*_15);
           _24 = &(*_25);
           StorageLive(_26);
-          StorageLive(_27);
+-         StorageLive(_27);
++         nop;
           _27 = &(*_16);
           _26 = &(*_27);
           StorageLive(_28);
@@ -225,9 +232,11 @@
           _34 = (move _35, move _38);
           StorageDead(_38);
           StorageDead(_35);
-          StorageLive(_41);
+-         StorageLive(_41);
++         nop;
           _41 = (_34.0: &*const u8);
-          StorageLive(_42);
+-         StorageLive(_42);
++         nop;
           _42 = (_34.1: &*const u8);
           StorageLive(_43);
           StorageLive(_44);
@@ -243,8 +252,10 @@
           StorageDead(_44);
           _33 = const ();
           StorageDead(_43);
-          StorageDead(_42);
-          StorageDead(_41);
+-         StorageDead(_42);
+-         StorageDead(_41);
++         nop;
++         nop;
           StorageDead(_39);
           StorageDead(_36);
           StorageDead(_34);
@@ -270,11 +281,13 @@
 -         _49 = move _47;
 +         _49 = const core::panicking::AssertKind::Eq;
           StorageLive(_50);
-          StorageLive(_51);
+-         StorageLive(_51);
++         nop;
           _51 = &(*_41);
           _50 = &(*_51);
           StorageLive(_52);
-          StorageLive(_53);
+-         StorageLive(_53);
++         nop;
           _53 = &(*_42);
           _52 = &(*_53);
           StorageLive(_54);
diff --git a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff
index 6c8513e32816..41f29959c953 100644
--- a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff
@@ -737,7 +737,8 @@
           StorageDead(_125);
           StorageDead(_126);
           StorageDead(_124);
-          StorageLive(_128);
+-         StorageLive(_128);
++         nop;
           _128 = &_3;
           StorageLive(_129);
 -         StorageLive(_130);
@@ -778,7 +779,8 @@
       bb32: {
           StorageDead(_134);
           StorageDead(_133);
-          StorageLive(_137);
+-         StorageLive(_137);
++         nop;
           _137 = &mut _3;
           StorageLive(_138);
           StorageLive(_139);
@@ -813,7 +815,8 @@
           StorageDead(_143);
           StorageDead(_142);
           StorageLive(_146);
-          StorageLive(_147);
+-         StorageLive(_147);
++         nop;
           _147 = &raw const _3;
           StorageLive(_148);
           StorageLive(_149);
@@ -847,7 +850,8 @@
       bb36: {
           StorageDead(_153);
           StorageDead(_152);
-          StorageLive(_156);
+-         StorageLive(_156);
++         nop;
           _156 = &raw mut _3;
           StorageLive(_157);
           StorageLive(_158);
@@ -882,10 +886,13 @@
           StorageDead(_162);
           StorageDead(_161);
           _146 = const ();
-          StorageDead(_156);
-          StorageDead(_147);
+-         StorageDead(_156);
+-         StorageDead(_147);
++         nop;
++         nop;
           StorageDead(_146);
-          StorageLive(_165);
+-         StorageLive(_165);
++         nop;
           _165 = &_3;
           StorageLive(_166);
 -         StorageLive(_167);
@@ -927,9 +934,12 @@
           StorageDead(_171);
           StorageDead(_170);
           _0 = const ();
-          StorageDead(_165);
-          StorageDead(_137);
-          StorageDead(_128);
+-         StorageDead(_165);
+-         StorageDead(_137);
+-         StorageDead(_128);
++         nop;
++         nop;
++         nop;
           return;
       }
   }
diff --git a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff
index fb2c089827de..ca928986cf65 100644
--- a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff
@@ -737,7 +737,8 @@
           StorageDead(_125);
           StorageDead(_126);
           StorageDead(_124);
-          StorageLive(_128);
+-         StorageLive(_128);
++         nop;
           _128 = &_3;
           StorageLive(_129);
 -         StorageLive(_130);
@@ -778,7 +779,8 @@
       bb32: {
           StorageDead(_134);
           StorageDead(_133);
-          StorageLive(_137);
+-         StorageLive(_137);
++         nop;
           _137 = &mut _3;
           StorageLive(_138);
           StorageLive(_139);
@@ -813,7 +815,8 @@
           StorageDead(_143);
           StorageDead(_142);
           StorageLive(_146);
-          StorageLive(_147);
+-         StorageLive(_147);
++         nop;
           _147 = &raw const _3;
           StorageLive(_148);
           StorageLive(_149);
@@ -847,7 +850,8 @@
       bb36: {
           StorageDead(_153);
           StorageDead(_152);
-          StorageLive(_156);
+-         StorageLive(_156);
++         nop;
           _156 = &raw mut _3;
           StorageLive(_157);
           StorageLive(_158);
@@ -882,10 +886,13 @@
           StorageDead(_162);
           StorageDead(_161);
           _146 = const ();
-          StorageDead(_156);
-          StorageDead(_147);
+-         StorageDead(_156);
+-         StorageDead(_147);
++         nop;
++         nop;
           StorageDead(_146);
-          StorageLive(_165);
+-         StorageLive(_165);
++         nop;
           _165 = &_3;
           StorageLive(_166);
 -         StorageLive(_167);
@@ -927,9 +934,12 @@
           StorageDead(_171);
           StorageDead(_170);
           _0 = const ();
-          StorageDead(_165);
-          StorageDead(_137);
-          StorageDead(_128);
+-         StorageDead(_165);
+-         StorageDead(_137);
+-         StorageDead(_128);
++         nop;
++         nop;
++         nop;
           return;
       }
   }

From 48d2157a89ffddce3287c57b3e3817a7e093b6c5 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 16 Sep 2023 09:35:46 +0000
Subject: [PATCH 062/297] Simplify aggregate projections.

---
 compiler/rustc_mir_transform/src/gvn.rs       | 41 +++++++++-
 .../gvn.references.GVN.panic-abort.diff       | 21 +++--
 .../gvn.references.GVN.panic-unwind.diff      | 21 +++--
 tests/mir-opt/gvn.slices.GVN.panic-abort.diff | 82 +++++++++++--------
 .../mir-opt/gvn.slices.GVN.panic-unwind.diff  | 82 +++++++++++--------
 ...xpression_elimination.GVN.panic-abort.diff | 14 ++--
 ...pression_elimination.GVN.panic-unwind.diff | 14 ++--
 .../gvn.wrap_unwrap.GVN.panic-abort.diff      | 18 ++--
 .../gvn.wrap_unwrap.GVN.panic-unwind.diff     | 18 ++--
 9 files changed, 192 insertions(+), 119 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index eaa1c70a5e9b..70d0f22a7b90 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -276,6 +276,10 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         }
     }
 
+    fn insert_scalar(&mut self, scalar: Scalar, ty: Ty<'tcx>) -> VnIndex {
+        self.insert(Value::Constant(Const::from_scalar(self.tcx, scalar, ty)))
+    }
+
     #[instrument(level = "trace", skip(self), ret)]
     fn eval_to_const(&mut self, value: VnIndex) -> Option<OpTy<'tcx>> {
         use Value::*;
@@ -483,12 +487,33 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                 }
             }
             ProjectionElem::Downcast(name, index) => ProjectionElem::Downcast(name, index),
-            ProjectionElem::Field(f, ty) => ProjectionElem::Field(f, ty),
+            ProjectionElem::Field(f, ty) => {
+                if let Value::Aggregate(_, _, fields) = self.get(value) {
+                    return Some(fields[f.as_usize()]);
+                } else if let Value::Projection(outer_value, ProjectionElem::Downcast(_, read_variant)) = self.get(value)
+                    && let Value::Aggregate(_, written_variant, fields) = self.get(*outer_value)
+                    && written_variant == read_variant
+                {
+                    return Some(fields[f.as_usize()]);
+                }
+                ProjectionElem::Field(f, ty)
+            }
             ProjectionElem::Index(idx) => {
                 let idx = self.locals[idx]?;
                 ProjectionElem::Index(idx)
             }
             ProjectionElem::ConstantIndex { offset, min_length, from_end } => {
+                match self.get(value) {
+                    Value::Aggregate(ty, _, operands) if ty.is_array() => {
+                        let offset = if from_end {
+                            operands.len() - offset as usize
+                        } else {
+                            offset as usize
+                        };
+                        return operands.get(offset).copied();
+                    }
+                    _ => {}
+                };
                 ProjectionElem::ConstantIndex { offset, min_length, from_end }
             }
             ProjectionElem::Subslice { from, to, from_end } => {
@@ -679,6 +704,9 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             }
             Rvalue::Discriminant(ref mut place) => {
                 let place = self.simplify_place_value(place, location)?;
+                if let Some(discr) = self.simplify_discriminant(place) {
+                    return Some(discr);
+                }
                 Value::Discriminant(place)
             }
 
@@ -688,6 +716,17 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         debug!(?value);
         Some(self.insert(value))
     }
+
+    fn simplify_discriminant(&mut self, place: VnIndex) -> Option<VnIndex> {
+        if let Value::Aggregate(enum_ty, variant, _) = *self.get(place)
+            && enum_ty.is_enum()
+        {
+            let discr = self.ecx.discriminant_for_variant(enum_ty, variant).ok()?;
+            return Some(self.insert_scalar(discr.to_scalar(), discr.layout.ty));
+        }
+
+        None
+    }
 }
 
 fn op_to_prop_const<'tcx>(
diff --git a/tests/mir-opt/gvn.references.GVN.panic-abort.diff b/tests/mir-opt/gvn.references.GVN.panic-abort.diff
index 885431725657..7799c6114452 100644
--- a/tests/mir-opt/gvn.references.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.references.GVN.panic-abort.diff
@@ -115,8 +115,7 @@
 -         StorageLive(_18);
 +         nop;
           _18 = &mut _1;
--         StorageLive(_19);
-+         nop;
+          StorageLive(_19);
           StorageLive(_20);
           StorageLive(_21);
 -         _21 = move _18;
@@ -124,11 +123,13 @@
 +         _21 = _18;
 +         _20 = S::<&mut impl Sized>(_18);
           StorageDead(_21);
-          _19 = move (_20.0: &mut impl Sized);
+-         _19 = move (_20.0: &mut impl Sized);
++         _19 = _18;
           StorageDead(_20);
           StorageLive(_22);
           StorageLive(_23);
-          _23 = &(*_19);
+-         _23 = &(*_19);
++         _23 = &(*_18);
           _22 = opaque::<&impl Sized>(move _23) -> [return: bb9, unwind unreachable];
       }
   
@@ -137,7 +138,8 @@
           StorageDead(_22);
           StorageLive(_24);
           StorageLive(_25);
-          _25 = &mut (*_19);
+-         _25 = &mut (*_19);
++         _25 = &mut (*_18);
           _24 = opaque::<&mut impl Sized>(move _25) -> [return: bb10, unwind unreachable];
       }
   
@@ -146,7 +148,8 @@
           StorageDead(_24);
           StorageLive(_26);
           StorageLive(_27);
-          _27 = &raw const (*_19);
+-         _27 = &raw const (*_19);
++         _27 = &raw const (*_18);
           _26 = opaque::<*const impl Sized>(move _27) -> [return: bb11, unwind unreachable];
       }
   
@@ -155,7 +158,8 @@
           StorageDead(_26);
           StorageLive(_28);
           StorageLive(_29);
-          _29 = &raw mut (*_19);
+-         _29 = &raw mut (*_19);
++         _29 = &raw mut (*_18);
           _28 = opaque::<*mut impl Sized>(move _29) -> [return: bb12, unwind unreachable];
       }
   
@@ -163,9 +167,8 @@
           StorageDead(_29);
           StorageDead(_28);
           _0 = const ();
--         StorageDead(_19);
+          StorageDead(_19);
 -         StorageDead(_18);
-+         nop;
 +         nop;
           drop(_1) -> [return: bb13, unwind unreachable];
       }
diff --git a/tests/mir-opt/gvn.references.GVN.panic-unwind.diff b/tests/mir-opt/gvn.references.GVN.panic-unwind.diff
index 106b453fe800..880e7913fa94 100644
--- a/tests/mir-opt/gvn.references.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.references.GVN.panic-unwind.diff
@@ -115,8 +115,7 @@
 -         StorageLive(_18);
 +         nop;
           _18 = &mut _1;
--         StorageLive(_19);
-+         nop;
+          StorageLive(_19);
           StorageLive(_20);
           StorageLive(_21);
 -         _21 = move _18;
@@ -124,11 +123,13 @@
 +         _21 = _18;
 +         _20 = S::<&mut impl Sized>(_18);
           StorageDead(_21);
-          _19 = move (_20.0: &mut impl Sized);
+-         _19 = move (_20.0: &mut impl Sized);
++         _19 = _18;
           StorageDead(_20);
           StorageLive(_22);
           StorageLive(_23);
-          _23 = &(*_19);
+-         _23 = &(*_19);
++         _23 = &(*_18);
           _22 = opaque::<&impl Sized>(move _23) -> [return: bb9, unwind: bb14];
       }
   
@@ -137,7 +138,8 @@
           StorageDead(_22);
           StorageLive(_24);
           StorageLive(_25);
-          _25 = &mut (*_19);
+-         _25 = &mut (*_19);
++         _25 = &mut (*_18);
           _24 = opaque::<&mut impl Sized>(move _25) -> [return: bb10, unwind: bb14];
       }
   
@@ -146,7 +148,8 @@
           StorageDead(_24);
           StorageLive(_26);
           StorageLive(_27);
-          _27 = &raw const (*_19);
+-         _27 = &raw const (*_19);
++         _27 = &raw const (*_18);
           _26 = opaque::<*const impl Sized>(move _27) -> [return: bb11, unwind: bb14];
       }
   
@@ -155,7 +158,8 @@
           StorageDead(_26);
           StorageLive(_28);
           StorageLive(_29);
-          _29 = &raw mut (*_19);
+-         _29 = &raw mut (*_19);
++         _29 = &raw mut (*_18);
           _28 = opaque::<*mut impl Sized>(move _29) -> [return: bb12, unwind: bb14];
       }
   
@@ -163,9 +167,8 @@
           StorageDead(_29);
           StorageDead(_28);
           _0 = const ();
--         StorageDead(_19);
+          StorageDead(_19);
 -         StorageDead(_18);
-+         nop;
 +         nop;
           drop(_1) -> [return: bb13, unwind: bb15];
       }
diff --git a/tests/mir-opt/gvn.slices.GVN.panic-abort.diff b/tests/mir-opt/gvn.slices.GVN.panic-abort.diff
index 6168476d6f27..9db6e068fa7d 100644
--- a/tests/mir-opt/gvn.slices.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.slices.GVN.panic-abort.diff
@@ -112,7 +112,8 @@
           StorageDead(_5);
           StorageLive(_7);
           StorageLive(_8);
-          StorageLive(_9);
+-         StorageLive(_9);
++         nop;
           StorageLive(_10);
           StorageLive(_11);
           _11 = &(*_1);
@@ -122,7 +123,8 @@
       bb3: {
           StorageDead(_11);
           _9 = &_10;
-          StorageLive(_12);
+-         StorageLive(_12);
++         nop;
           StorageLive(_13);
           StorageLive(_14);
 -         _14 = &(*_4);
@@ -133,20 +135,25 @@
       bb4: {
           StorageDead(_14);
           _12 = &_13;
-          _8 = (move _9, move _12);
-          StorageDead(_12);
-          StorageDead(_9);
--         StorageLive(_15);
+-         _8 = (move _9, move _12);
+-         StorageDead(_12);
+-         StorageDead(_9);
++         _8 = (_9, _12);
 +         nop;
-          _15 = (_8.0: &*const u8);
--         StorageLive(_16);
 +         nop;
-          _16 = (_8.1: &*const u8);
+          StorageLive(_15);
+-         _15 = (_8.0: &*const u8);
++         _15 = _9;
+          StorageLive(_16);
+-         _16 = (_8.1: &*const u8);
++         _16 = _12;
           StorageLive(_17);
           StorageLive(_18);
-          _18 = (*_15);
+-         _18 = (*_15);
++         _18 = (*_9);
           StorageLive(_19);
-          _19 = (*_16);
+-         _19 = (*_16);
++         _19 = (*_12);
           _17 = Eq(move _18, move _19);
           switchInt(move _17) -> [0: bb6, otherwise: bb5];
       }
@@ -156,10 +163,8 @@
           StorageDead(_18);
           _7 = const ();
           StorageDead(_17);
--         StorageDead(_16);
--         StorageDead(_15);
-+         nop;
-+         nop;
+          StorageDead(_16);
+          StorageDead(_15);
           StorageDead(_13);
           StorageDead(_10);
           StorageDead(_8);
@@ -190,13 +195,15 @@
 +         _23 = const core::panicking::AssertKind::Eq;
           StorageLive(_24);
 -         StorageLive(_25);
+-         _25 = &(*_15);
 +         nop;
-          _25 = &(*_15);
++         _25 = &(*_9);
           _24 = &(*_25);
           StorageLive(_26);
 -         StorageLive(_27);
+-         _27 = &(*_16);
 +         nop;
-          _27 = &(*_16);
++         _27 = &(*_12);
           _26 = &(*_27);
           StorageLive(_28);
           _28 = Option::<Arguments<'_>>::None;
@@ -209,7 +216,8 @@
           StorageDead(_31);
           StorageLive(_33);
           StorageLive(_34);
-          StorageLive(_35);
+-         StorageLive(_35);
++         nop;
           StorageLive(_36);
           StorageLive(_37);
           _37 = &(*_1);
@@ -219,7 +227,8 @@
       bb8: {
           StorageDead(_37);
           _35 = &_36;
-          StorageLive(_38);
+-         StorageLive(_38);
++         nop;
           StorageLive(_39);
           StorageLive(_40);
           _40 = &(*_29);
@@ -229,20 +238,25 @@
       bb9: {
           StorageDead(_40);
           _38 = &_39;
-          _34 = (move _35, move _38);
-          StorageDead(_38);
-          StorageDead(_35);
--         StorageLive(_41);
+-         _34 = (move _35, move _38);
+-         StorageDead(_38);
+-         StorageDead(_35);
++         _34 = (_35, _38);
 +         nop;
-          _41 = (_34.0: &*const u8);
--         StorageLive(_42);
 +         nop;
-          _42 = (_34.1: &*const u8);
+          StorageLive(_41);
+-         _41 = (_34.0: &*const u8);
++         _41 = _35;
+          StorageLive(_42);
+-         _42 = (_34.1: &*const u8);
++         _42 = _38;
           StorageLive(_43);
           StorageLive(_44);
-          _44 = (*_41);
+-         _44 = (*_41);
++         _44 = (*_35);
           StorageLive(_45);
-          _45 = (*_42);
+-         _45 = (*_42);
++         _45 = (*_38);
           _43 = Eq(move _44, move _45);
           switchInt(move _43) -> [0: bb11, otherwise: bb10];
       }
@@ -252,10 +266,8 @@
           StorageDead(_44);
           _33 = const ();
           StorageDead(_43);
--         StorageDead(_42);
--         StorageDead(_41);
-+         nop;
-+         nop;
+          StorageDead(_42);
+          StorageDead(_41);
           StorageDead(_39);
           StorageDead(_36);
           StorageDead(_34);
@@ -282,13 +294,15 @@
 +         _49 = const core::panicking::AssertKind::Eq;
           StorageLive(_50);
 -         StorageLive(_51);
+-         _51 = &(*_41);
 +         nop;
-          _51 = &(*_41);
++         _51 = &(*_35);
           _50 = &(*_51);
           StorageLive(_52);
 -         StorageLive(_53);
+-         _53 = &(*_42);
 +         nop;
-          _53 = &(*_42);
++         _53 = &(*_38);
           _52 = &(*_53);
           StorageLive(_54);
           _54 = Option::<Arguments<'_>>::None;
diff --git a/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff b/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff
index f60038858089..ac7a3e746888 100644
--- a/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff
@@ -112,7 +112,8 @@
           StorageDead(_5);
           StorageLive(_7);
           StorageLive(_8);
-          StorageLive(_9);
+-         StorageLive(_9);
++         nop;
           StorageLive(_10);
           StorageLive(_11);
           _11 = &(*_1);
@@ -122,7 +123,8 @@
       bb3: {
           StorageDead(_11);
           _9 = &_10;
-          StorageLive(_12);
+-         StorageLive(_12);
++         nop;
           StorageLive(_13);
           StorageLive(_14);
 -         _14 = &(*_4);
@@ -133,20 +135,25 @@
       bb4: {
           StorageDead(_14);
           _12 = &_13;
-          _8 = (move _9, move _12);
-          StorageDead(_12);
-          StorageDead(_9);
--         StorageLive(_15);
+-         _8 = (move _9, move _12);
+-         StorageDead(_12);
+-         StorageDead(_9);
++         _8 = (_9, _12);
 +         nop;
-          _15 = (_8.0: &*const u8);
--         StorageLive(_16);
 +         nop;
-          _16 = (_8.1: &*const u8);
+          StorageLive(_15);
+-         _15 = (_8.0: &*const u8);
++         _15 = _9;
+          StorageLive(_16);
+-         _16 = (_8.1: &*const u8);
++         _16 = _12;
           StorageLive(_17);
           StorageLive(_18);
-          _18 = (*_15);
+-         _18 = (*_15);
++         _18 = (*_9);
           StorageLive(_19);
-          _19 = (*_16);
+-         _19 = (*_16);
++         _19 = (*_12);
           _17 = Eq(move _18, move _19);
           switchInt(move _17) -> [0: bb6, otherwise: bb5];
       }
@@ -156,10 +163,8 @@
           StorageDead(_18);
           _7 = const ();
           StorageDead(_17);
--         StorageDead(_16);
--         StorageDead(_15);
-+         nop;
-+         nop;
+          StorageDead(_16);
+          StorageDead(_15);
           StorageDead(_13);
           StorageDead(_10);
           StorageDead(_8);
@@ -190,13 +195,15 @@
 +         _23 = const core::panicking::AssertKind::Eq;
           StorageLive(_24);
 -         StorageLive(_25);
+-         _25 = &(*_15);
 +         nop;
-          _25 = &(*_15);
++         _25 = &(*_9);
           _24 = &(*_25);
           StorageLive(_26);
 -         StorageLive(_27);
+-         _27 = &(*_16);
 +         nop;
-          _27 = &(*_16);
++         _27 = &(*_12);
           _26 = &(*_27);
           StorageLive(_28);
           _28 = Option::<Arguments<'_>>::None;
@@ -209,7 +216,8 @@
           StorageDead(_31);
           StorageLive(_33);
           StorageLive(_34);
-          StorageLive(_35);
+-         StorageLive(_35);
++         nop;
           StorageLive(_36);
           StorageLive(_37);
           _37 = &(*_1);
@@ -219,7 +227,8 @@
       bb8: {
           StorageDead(_37);
           _35 = &_36;
-          StorageLive(_38);
+-         StorageLive(_38);
++         nop;
           StorageLive(_39);
           StorageLive(_40);
           _40 = &(*_29);
@@ -229,20 +238,25 @@
       bb9: {
           StorageDead(_40);
           _38 = &_39;
-          _34 = (move _35, move _38);
-          StorageDead(_38);
-          StorageDead(_35);
--         StorageLive(_41);
+-         _34 = (move _35, move _38);
+-         StorageDead(_38);
+-         StorageDead(_35);
++         _34 = (_35, _38);
 +         nop;
-          _41 = (_34.0: &*const u8);
--         StorageLive(_42);
 +         nop;
-          _42 = (_34.1: &*const u8);
+          StorageLive(_41);
+-         _41 = (_34.0: &*const u8);
++         _41 = _35;
+          StorageLive(_42);
+-         _42 = (_34.1: &*const u8);
++         _42 = _38;
           StorageLive(_43);
           StorageLive(_44);
-          _44 = (*_41);
+-         _44 = (*_41);
++         _44 = (*_35);
           StorageLive(_45);
-          _45 = (*_42);
+-         _45 = (*_42);
++         _45 = (*_38);
           _43 = Eq(move _44, move _45);
           switchInt(move _43) -> [0: bb11, otherwise: bb10];
       }
@@ -252,10 +266,8 @@
           StorageDead(_44);
           _33 = const ();
           StorageDead(_43);
--         StorageDead(_42);
--         StorageDead(_41);
-+         nop;
-+         nop;
+          StorageDead(_42);
+          StorageDead(_41);
           StorageDead(_39);
           StorageDead(_36);
           StorageDead(_34);
@@ -282,13 +294,15 @@
 +         _49 = const core::panicking::AssertKind::Eq;
           StorageLive(_50);
 -         StorageLive(_51);
+-         _51 = &(*_41);
 +         nop;
-          _51 = &(*_41);
++         _51 = &(*_35);
           _50 = &(*_51);
           StorageLive(_52);
 -         StorageLive(_53);
+-         _53 = &(*_42);
 +         nop;
-          _53 = &(*_42);
++         _53 = &(*_38);
           _52 = &(*_53);
           StorageLive(_54);
           _54 = Option::<Arguments<'_>>::None;
diff --git a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff
index 41f29959c953..23ba2559448b 100644
--- a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff
@@ -442,8 +442,7 @@
 +         nop;
           StorageDead(_52);
           StorageLive(_55);
--         StorageLive(_56);
-+         nop;
+          StorageLive(_56);
           StorageLive(_57);
           StorageLive(_58);
           _58 = _1;
@@ -452,13 +451,12 @@
           StorageDead(_58);
 -         _56 = (_57.0: u64);
 -         _55 = opaque::<u64>(move _56) -> [return: bb16, unwind unreachable];
-+         _56 = (_53.0: u64);
-+         _55 = opaque::<u64>(_56) -> [return: bb16, unwind unreachable];
++         _56 = _1;
++         _55 = opaque::<u64>(_1) -> [return: bb16, unwind unreachable];
       }
   
       bb16: {
--         StorageDead(_56);
-+         nop;
+          StorageDead(_56);
           StorageDead(_57);
           StorageDead(_55);
           StorageLive(_59);
@@ -729,8 +727,8 @@
           StorageDead(_127);
 -         _125 = (_126.0: u64);
 -         _124 = opaque::<u64>(move _125) -> [return: bb30, unwind unreachable];
-+         _125 = _56;
-+         _124 = opaque::<u64>(_56) -> [return: bb30, unwind unreachable];
++         _125 = _1;
++         _124 = opaque::<u64>(_1) -> [return: bb30, unwind unreachable];
       }
   
       bb30: {
diff --git a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff
index ca928986cf65..062dc6ff5619 100644
--- a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff
@@ -442,8 +442,7 @@
 +         nop;
           StorageDead(_52);
           StorageLive(_55);
--         StorageLive(_56);
-+         nop;
+          StorageLive(_56);
           StorageLive(_57);
           StorageLive(_58);
           _58 = _1;
@@ -452,13 +451,12 @@
           StorageDead(_58);
 -         _56 = (_57.0: u64);
 -         _55 = opaque::<u64>(move _56) -> [return: bb16, unwind continue];
-+         _56 = (_53.0: u64);
-+         _55 = opaque::<u64>(_56) -> [return: bb16, unwind continue];
++         _56 = _1;
++         _55 = opaque::<u64>(_1) -> [return: bb16, unwind continue];
       }
   
       bb16: {
--         StorageDead(_56);
-+         nop;
+          StorageDead(_56);
           StorageDead(_57);
           StorageDead(_55);
           StorageLive(_59);
@@ -729,8 +727,8 @@
           StorageDead(_127);
 -         _125 = (_126.0: u64);
 -         _124 = opaque::<u64>(move _125) -> [return: bb30, unwind continue];
-+         _125 = _56;
-+         _124 = opaque::<u64>(_56) -> [return: bb30, unwind continue];
++         _125 = _1;
++         _124 = opaque::<u64>(_1) -> [return: bb30, unwind continue];
       }
   
       bb30: {
diff --git a/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-abort.diff b/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-abort.diff
index 2dd51934778c..62710ba8fbf5 100644
--- a/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-abort.diff
@@ -20,8 +20,10 @@
 -         _2 = Option::<T>::Some(move _3);
 +         _2 = Option::<T>::Some(_1);
           StorageDead(_3);
-          _4 = discriminant(_2);
-          switchInt(move _4) -> [0: bb1, 1: bb3, otherwise: bb2];
+-         _4 = discriminant(_2);
+-         switchInt(move _4) -> [0: bb1, 1: bb3, otherwise: bb2];
++         _4 = const 1_isize;
++         switchInt(const 1_isize) -> [0: bb1, 1: bb3, otherwise: bb2];
       }
   
       bb1: {
@@ -34,12 +36,12 @@
       }
   
       bb3: {
--         StorageLive(_5);
-+         nop;
-          _5 = ((_2 as Some).0: T);
-          _0 = _5;
--         StorageDead(_5);
-+         nop;
+          StorageLive(_5);
+-         _5 = ((_2 as Some).0: T);
+-         _0 = _5;
++         _5 = _1;
++         _0 = _1;
+          StorageDead(_5);
           StorageDead(_2);
           return;
       }
diff --git a/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-unwind.diff b/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-unwind.diff
index 610d70576c9f..ad46a065b1e3 100644
--- a/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.wrap_unwrap.GVN.panic-unwind.diff
@@ -20,8 +20,10 @@
 -         _2 = Option::<T>::Some(move _3);
 +         _2 = Option::<T>::Some(_1);
           StorageDead(_3);
-          _4 = discriminant(_2);
-          switchInt(move _4) -> [0: bb1, 1: bb3, otherwise: bb2];
+-         _4 = discriminant(_2);
+-         switchInt(move _4) -> [0: bb1, 1: bb3, otherwise: bb2];
++         _4 = const 1_isize;
++         switchInt(const 1_isize) -> [0: bb1, 1: bb3, otherwise: bb2];
       }
   
       bb1: {
@@ -34,12 +36,12 @@
       }
   
       bb3: {
--         StorageLive(_5);
-+         nop;
-          _5 = ((_2 as Some).0: T);
-          _0 = _5;
--         StorageDead(_5);
-+         nop;
+          StorageLive(_5);
+-         _5 = ((_2 as Some).0: T);
+-         _0 = _5;
++         _5 = _1;
++         _0 = _1;
+          StorageDead(_5);
           StorageDead(_2);
           return;
       }

From f110f22060b86dd2873d7ac6d7a4e444c9348b1d Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 16 Sep 2023 09:36:16 +0000
Subject: [PATCH 063/297] Simplify repeat expressions.

---
 compiler/rustc_mir_transform/src/gvn.rs                |  6 ++++++
 tests/mir-opt/gvn.repeated_index.GVN.panic-abort.diff  | 10 ++++++----
 tests/mir-opt/gvn.repeated_index.GVN.panic-unwind.diff | 10 ++++++----
 3 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index 70d0f22a7b90..66442849863e 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -499,11 +499,17 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                 ProjectionElem::Field(f, ty)
             }
             ProjectionElem::Index(idx) => {
+                if let Value::Repeat(inner, _) = self.get(value) {
+                    return Some(*inner);
+                }
                 let idx = self.locals[idx]?;
                 ProjectionElem::Index(idx)
             }
             ProjectionElem::ConstantIndex { offset, min_length, from_end } => {
                 match self.get(value) {
+                    Value::Repeat(inner, _) => {
+                        return Some(*inner);
+                    }
                     Value::Aggregate(ty, _, operands) if ty.is_array() => {
                         let offset = if from_end {
                             operands.len() - offset as usize
diff --git a/tests/mir-opt/gvn.repeated_index.GVN.panic-abort.diff b/tests/mir-opt/gvn.repeated_index.GVN.panic-abort.diff
index 91af1ccc33a9..d937902e8913 100644
--- a/tests/mir-opt/gvn.repeated_index.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.repeated_index.GVN.panic-abort.diff
@@ -41,8 +41,9 @@
   
       bb1: {
 -         _6 = _3[_7];
-+         _6 = _3[0 of 1];
-          _5 = opaque::<T>(move _6) -> [return: bb2, unwind unreachable];
+-         _5 = opaque::<T>(move _6) -> [return: bb2, unwind unreachable];
++         _6 = _1;
++         _5 = opaque::<T>(_1) -> [return: bb2, unwind unreachable];
       }
   
       bb2: {
@@ -63,8 +64,9 @@
   
       bb3: {
 -         _11 = _3[_12];
-+         _11 = _3[_2];
-          _10 = opaque::<T>(move _11) -> [return: bb4, unwind unreachable];
+-         _10 = opaque::<T>(move _11) -> [return: bb4, unwind unreachable];
++         _11 = _1;
++         _10 = opaque::<T>(_1) -> [return: bb4, unwind unreachable];
       }
   
       bb4: {
diff --git a/tests/mir-opt/gvn.repeated_index.GVN.panic-unwind.diff b/tests/mir-opt/gvn.repeated_index.GVN.panic-unwind.diff
index f887094b509a..dd4d24b12ea6 100644
--- a/tests/mir-opt/gvn.repeated_index.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.repeated_index.GVN.panic-unwind.diff
@@ -41,8 +41,9 @@
   
       bb1: {
 -         _6 = _3[_7];
-+         _6 = _3[0 of 1];
-          _5 = opaque::<T>(move _6) -> [return: bb2, unwind continue];
+-         _5 = opaque::<T>(move _6) -> [return: bb2, unwind continue];
++         _6 = _1;
++         _5 = opaque::<T>(_1) -> [return: bb2, unwind continue];
       }
   
       bb2: {
@@ -63,8 +64,9 @@
   
       bb3: {
 -         _11 = _3[_12];
-+         _11 = _3[_2];
-          _10 = opaque::<T>(move _11) -> [return: bb4, unwind continue];
+-         _10 = opaque::<T>(move _11) -> [return: bb4, unwind continue];
++         _11 = _1;
++         _10 = opaque::<T>(_1) -> [return: bb4, unwind continue];
       }
   
       bb4: {

From 23d4857080a3968447adbb1d55b2720dba46d666 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Wed, 3 May 2023 18:36:53 +0000
Subject: [PATCH 064/297] Do not compute actual aggregate type.

---
 compiler/rustc_mir_transform/src/gvn.rs | 58 ++++++++++++++++++++-----
 1 file changed, 47 insertions(+), 11 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index 66442849863e..9880e239957a 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -56,6 +56,7 @@
 use rustc_const_eval::interpret::{ImmTy, InterpCx, MemPlaceMeta, OpTy, Projectable, Scalar};
 use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
 use rustc_data_structures::graph::dominators::Dominators;
+use rustc_hir::def::DefKind;
 use rustc_index::bit_set::BitSet;
 use rustc_index::IndexVec;
 use rustc_macros::newtype_index;
@@ -64,6 +65,7 @@ use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
 use rustc_middle::ty::layout::LayoutOf;
 use rustc_middle::ty::{self, Ty, TyCtxt, TypeAndMut};
+use rustc_span::def_id::DefId;
 use rustc_span::DUMMY_SP;
 use rustc_target::abi::{self, Abi, Size, VariantIdx, FIRST_VARIANT};
 use std::borrow::Cow;
@@ -136,6 +138,16 @@ newtype_index! {
     struct VnIndex {}
 }
 
+/// Computing the aggregate's type can be quite slow, so we only keep the minimal amount of
+/// information to reconstruct it when needed.
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+enum AggregateTy<'tcx> {
+    /// Invariant: this must not be used for an empty array.
+    Array,
+    Tuple,
+    Def(DefId, ty::GenericArgsRef<'tcx>),
+}
+
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
 enum AddressKind {
     Ref(BorrowKind),
@@ -152,7 +164,7 @@ enum Value<'tcx> {
     Constant(Const<'tcx>),
     /// An aggregate value, either tuple/closure/struct/enum.
     /// This does not contain unions, as we cannot reason with the value.
-    Aggregate(Ty<'tcx>, VariantIdx, Vec<VnIndex>),
+    Aggregate(AggregateTy<'tcx>, VariantIdx, Vec<VnIndex>),
     /// This corresponds to a `[value; count]` expression.
     Repeat(VnIndex, ty::Const<'tcx>),
     /// The address of a place.
@@ -289,11 +301,23 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             Repeat(..) => return None,
 
             Constant(ref constant) => self.ecx.eval_mir_constant(constant, None, None).ok()?,
-            Aggregate(ty, variant, ref fields) => {
+            Aggregate(kind, variant, ref fields) => {
                 let fields = fields
                     .iter()
                     .map(|&f| self.evaluated[f].as_ref())
                     .collect::<Option<Vec<_>>>()?;
+                let ty = match kind {
+                    AggregateTy::Array => {
+                        assert!(fields.len() > 0);
+                        Ty::new_array(self.tcx, fields[0].layout.ty, fields.len() as u64)
+                    }
+                    AggregateTy::Tuple => {
+                        Ty::new_tup_from_iter(self.tcx, fields.iter().map(|f| f.layout.ty))
+                    }
+                    AggregateTy::Def(def_id, args) => {
+                        self.tcx.type_of(def_id).instantiate(self.tcx, args)
+                    }
+                };
                 let variant = if ty.is_enum() { Some(variant) } else { None };
                 let ty = self.ecx.layout_of(ty).ok()?;
                 if ty.is_zst() {
@@ -510,7 +534,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                     Value::Repeat(inner, _) => {
                         return Some(*inner);
                     }
-                    Value::Aggregate(ty, _, operands) if ty.is_array() => {
+                    Value::Aggregate(AggregateTy::Array, _, operands) => {
                         let offset = if from_end {
                             operands.len() - offset as usize
                         } else {
@@ -659,12 +683,23 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             }
             Rvalue::NullaryOp(op, ty) => Value::NullaryOp(op, ty),
             Rvalue::Aggregate(box ref kind, ref mut fields) => {
-                let variant_index = match *kind {
-                    AggregateKind::Array(..)
-                    | AggregateKind::Tuple
-                    | AggregateKind::Closure(..)
-                    | AggregateKind::Coroutine(..) => FIRST_VARIANT,
-                    AggregateKind::Adt(_, variant_index, _, _, None) => variant_index,
+                let (ty, variant_index) = match *kind {
+                    // For empty arrays, we have not mean to recover the type. They are ZSTs
+                    // anyway, so return them as such.
+                    AggregateKind::Array(..) | AggregateKind::Tuple if fields.is_empty() => {
+                        return Some(self.insert(Value::Constant(Const::zero_sized(
+                            rvalue.ty(self.local_decls, self.tcx),
+                        ))));
+                    }
+                    AggregateKind::Array(..) => (AggregateTy::Array, FIRST_VARIANT),
+                    AggregateKind::Tuple => (AggregateTy::Tuple, FIRST_VARIANT),
+                    AggregateKind::Closure(did, substs)
+                    | AggregateKind::Coroutine(did, substs, _) => {
+                        (AggregateTy::Def(did, substs), FIRST_VARIANT)
+                    }
+                    AggregateKind::Adt(did, variant_index, substs, _, None) => {
+                        (AggregateTy::Def(did, substs), variant_index)
+                    }
                     // Do not track unions.
                     AggregateKind::Adt(_, _, _, _, Some(_)) => return None,
                 };
@@ -672,7 +707,6 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                     .iter_mut()
                     .map(|op| self.simplify_operand(op, location).or_else(|| self.new_opaque()))
                     .collect();
-                let ty = rvalue.ty(self.local_decls, self.tcx);
                 Value::Aggregate(ty, variant_index, fields?)
             }
             Rvalue::Ref(_, borrow_kind, ref mut place) => {
@@ -725,8 +759,10 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
 
     fn simplify_discriminant(&mut self, place: VnIndex) -> Option<VnIndex> {
         if let Value::Aggregate(enum_ty, variant, _) = *self.get(place)
-            && enum_ty.is_enum()
+            && let AggregateTy::Def(enum_did, enum_substs) = enum_ty
+            && let DefKind::Enum = self.tcx.def_kind(enum_did)
         {
+            let enum_ty = self.tcx.type_of(enum_did).instantiate(self.tcx, enum_substs);
             let discr = self.ecx.discriminant_for_variant(enum_ty, variant).ok()?;
             return Some(self.insert_scalar(discr.to_scalar(), discr.layout.ty));
         }

From 80a5e8522de648d70a9ef1dfba7621f730832fb3 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sun, 21 May 2023 12:59:38 +0000
Subject: [PATCH 065/297] Extract simplify_aggregate.

---
 compiler/rustc_mir_transform/src/gvn.rs       | 83 +++++++++++++------
 .../gvn.aggregates.GVN.panic-abort.diff       | 66 +++++++++++++++
 .../gvn.aggregates.GVN.panic-unwind.diff      | 66 +++++++++++++++
 tests/mir-opt/gvn.rs                          | 10 +++
 4 files changed, 198 insertions(+), 27 deletions(-)
 create mode 100644 tests/mir-opt/gvn.aggregates.GVN.panic-abort.diff
 create mode 100644 tests/mir-opt/gvn.aggregates.GVN.panic-unwind.diff

diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index 9880e239957a..bbcb39226b7f 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -682,33 +682,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                 Value::Repeat(op, amount)
             }
             Rvalue::NullaryOp(op, ty) => Value::NullaryOp(op, ty),
-            Rvalue::Aggregate(box ref kind, ref mut fields) => {
-                let (ty, variant_index) = match *kind {
-                    // For empty arrays, we have not mean to recover the type. They are ZSTs
-                    // anyway, so return them as such.
-                    AggregateKind::Array(..) | AggregateKind::Tuple if fields.is_empty() => {
-                        return Some(self.insert(Value::Constant(Const::zero_sized(
-                            rvalue.ty(self.local_decls, self.tcx),
-                        ))));
-                    }
-                    AggregateKind::Array(..) => (AggregateTy::Array, FIRST_VARIANT),
-                    AggregateKind::Tuple => (AggregateTy::Tuple, FIRST_VARIANT),
-                    AggregateKind::Closure(did, substs)
-                    | AggregateKind::Coroutine(did, substs, _) => {
-                        (AggregateTy::Def(did, substs), FIRST_VARIANT)
-                    }
-                    AggregateKind::Adt(did, variant_index, substs, _, None) => {
-                        (AggregateTy::Def(did, substs), variant_index)
-                    }
-                    // Do not track unions.
-                    AggregateKind::Adt(_, _, _, _, Some(_)) => return None,
-                };
-                let fields: Option<Vec<_>> = fields
-                    .iter_mut()
-                    .map(|op| self.simplify_operand(op, location).or_else(|| self.new_opaque()))
-                    .collect();
-                Value::Aggregate(ty, variant_index, fields?)
-            }
+            Rvalue::Aggregate(..) => self.simplify_aggregate(rvalue, location)?,
             Rvalue::Ref(_, borrow_kind, ref mut place) => {
                 self.simplify_place_projection(place, location);
                 return self.new_pointer(*place, AddressKind::Ref(borrow_kind));
@@ -769,6 +743,61 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
 
         None
     }
+
+    fn simplify_aggregate(
+        &mut self,
+        rvalue: &mut Rvalue<'tcx>,
+        location: Location,
+    ) -> Option<Value<'tcx>> {
+        let Rvalue::Aggregate(box ref kind, ref mut fields) = *rvalue else { bug!() };
+
+        let tcx = self.tcx;
+        if fields.is_empty() {
+            let is_zst = match *kind {
+                AggregateKind::Array(..) | AggregateKind::Tuple | AggregateKind::Closure(..) => {
+                    true
+                }
+                // Only enums can be non-ZST.
+                AggregateKind::Adt(did, ..) => tcx.def_kind(did) != DefKind::Enum,
+                // Coroutines are never ZST, as they at least contain the implicit states.
+                AggregateKind::Coroutine(..) => false,
+            };
+
+            if is_zst {
+                let ty = rvalue.ty(self.local_decls, tcx);
+                let value = Value::Constant(Const::zero_sized(ty));
+                return Some(value);
+            }
+        }
+
+        let (ty, variant_index) = match *kind {
+            AggregateKind::Array(..) => {
+                assert!(!fields.is_empty());
+                (AggregateTy::Array, FIRST_VARIANT)
+            }
+            AggregateKind::Tuple => {
+                assert!(!fields.is_empty());
+                (AggregateTy::Tuple, FIRST_VARIANT)
+            }
+            AggregateKind::Closure(did, substs) | AggregateKind::Coroutine(did, substs, _) => {
+                (AggregateTy::Def(did, substs), FIRST_VARIANT)
+            }
+            AggregateKind::Adt(did, variant_index, substs, _, None) => {
+                (AggregateTy::Def(did, substs), variant_index)
+            }
+            // Do not track unions.
+            AggregateKind::Adt(_, _, _, _, Some(_)) => return None,
+        };
+
+        let fields: Option<Vec<_>> = fields
+            .iter_mut()
+            .map(|op| self.simplify_operand(op, location).or_else(|| self.new_opaque()))
+            .collect();
+        let fields = fields?;
+
+        let value = Value::Aggregate(ty, variant_index, fields);
+        Some(value)
+    }
 }
 
 fn op_to_prop_const<'tcx>(
diff --git a/tests/mir-opt/gvn.aggregates.GVN.panic-abort.diff b/tests/mir-opt/gvn.aggregates.GVN.panic-abort.diff
new file mode 100644
index 000000000000..c442835bfa99
--- /dev/null
+++ b/tests/mir-opt/gvn.aggregates.GVN.panic-abort.diff
@@ -0,0 +1,66 @@
+- // MIR for `aggregates` before GVN
++ // MIR for `aggregates` after GVN
+  
+  fn aggregates() -> () {
+      let mut _0: ();
+      let _1: S<[u8; 0]>;
+      let mut _2: [u8; 0];
+      let mut _4: [u16; 0];
+      let mut _6: ();
+      let mut _8: ();
+      scope 1 {
+          debug a_array => _1;
+          let _3: S<[u16; 0]>;
+          scope 2 {
+              debug b_array => _3;
+              let _5: S<()>;
+              scope 3 {
+                  debug a_tuple => _5;
+                  let _7: S<()>;
+                  scope 4 {
+                      debug b_tuple => _7;
+                  }
+              }
+          }
+      }
+  
+      bb0: {
+          StorageLive(_1);
+          StorageLive(_2);
+-         _2 = [];
+-         _1 = S::<[u8; 0]>(move _2);
++         _2 = const [];
++         _1 = const S::<[u8; 0]>([]);
+          StorageDead(_2);
+          StorageLive(_3);
+          StorageLive(_4);
+-         _4 = [];
+-         _3 = S::<[u16; 0]>(move _4);
++         _4 = const [];
++         _3 = const S::<[u16; 0]>([]);
+          StorageDead(_4);
+          StorageLive(_5);
+-         StorageLive(_6);
+-         _6 = ();
+-         _5 = S::<()>(move _6);
+-         StorageDead(_6);
++         nop;
++         _6 = const ();
++         _5 = const S::<()>(());
++         nop;
+          StorageLive(_7);
+          StorageLive(_8);
+-         _8 = ();
+-         _7 = S::<()>(move _8);
++         _8 = const ();
++         _7 = const S::<()>(());
+          StorageDead(_8);
+          _0 = const ();
+          StorageDead(_7);
+          StorageDead(_5);
+          StorageDead(_3);
+          StorageDead(_1);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/gvn.aggregates.GVN.panic-unwind.diff b/tests/mir-opt/gvn.aggregates.GVN.panic-unwind.diff
new file mode 100644
index 000000000000..c442835bfa99
--- /dev/null
+++ b/tests/mir-opt/gvn.aggregates.GVN.panic-unwind.diff
@@ -0,0 +1,66 @@
+- // MIR for `aggregates` before GVN
++ // MIR for `aggregates` after GVN
+  
+  fn aggregates() -> () {
+      let mut _0: ();
+      let _1: S<[u8; 0]>;
+      let mut _2: [u8; 0];
+      let mut _4: [u16; 0];
+      let mut _6: ();
+      let mut _8: ();
+      scope 1 {
+          debug a_array => _1;
+          let _3: S<[u16; 0]>;
+          scope 2 {
+              debug b_array => _3;
+              let _5: S<()>;
+              scope 3 {
+                  debug a_tuple => _5;
+                  let _7: S<()>;
+                  scope 4 {
+                      debug b_tuple => _7;
+                  }
+              }
+          }
+      }
+  
+      bb0: {
+          StorageLive(_1);
+          StorageLive(_2);
+-         _2 = [];
+-         _1 = S::<[u8; 0]>(move _2);
++         _2 = const [];
++         _1 = const S::<[u8; 0]>([]);
+          StorageDead(_2);
+          StorageLive(_3);
+          StorageLive(_4);
+-         _4 = [];
+-         _3 = S::<[u16; 0]>(move _4);
++         _4 = const [];
++         _3 = const S::<[u16; 0]>([]);
+          StorageDead(_4);
+          StorageLive(_5);
+-         StorageLive(_6);
+-         _6 = ();
+-         _5 = S::<()>(move _6);
+-         StorageDead(_6);
++         nop;
++         _6 = const ();
++         _5 = const S::<()>(());
++         nop;
+          StorageLive(_7);
+          StorageLive(_8);
+-         _8 = ();
+-         _7 = S::<()>(move _8);
++         _8 = const ();
++         _7 = const S::<()>(());
+          StorageDead(_8);
+          _0 = const ();
+          StorageDead(_7);
+          StorageDead(_5);
+          StorageDead(_3);
+          StorageDead(_1);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/gvn.rs b/tests/mir-opt/gvn.rs
index 3633f9c23cd5..ff176a6597b4 100644
--- a/tests/mir-opt/gvn.rs
+++ b/tests/mir-opt/gvn.rs
@@ -231,6 +231,14 @@ fn slices() {
     assert_eq!(s.as_ptr(), u.as_ptr());
 }
 
+fn aggregates() {
+    let a_array: S<[u8; 0]> = S([]);
+    let b_array: S<[u16; 0]> = S([]); // This must not be merged with `a_array`.
+
+    let a_tuple: S<()> = S(());
+    let b_tuple: S<()> = S(()); // But this can be with `a_tuple`.
+}
+
 fn main() {
     subexpression_elimination(2, 4, 5);
     wrap_unwrap(5);
@@ -243,6 +251,7 @@ fn main() {
     references(5);
     dereferences(&mut 5, &6, &S(7));
     slices();
+    aggregates();
 }
 
 #[inline(never)]
@@ -259,3 +268,4 @@ fn opaque(_: impl Sized) {}
 // EMIT_MIR gvn.references.GVN.diff
 // EMIT_MIR gvn.dereferences.GVN.diff
 // EMIT_MIR gvn.slices.GVN.diff
+// EMIT_MIR gvn.aggregates.GVN.diff

From dbf9ea30dd18db67cdb3f1271c7297876b65e675 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 23 Sep 2023 16:23:37 +0000
Subject: [PATCH 066/297] Transform large arrays into Repeat expressions when
 possible.

---
 compiler/rustc_mir_transform/src/gvn.rs       | 14 ++++
 .../gvn.aggregates.GVN.panic-abort.diff       | 67 +++++++++++++++++++
 .../gvn.aggregates.GVN.panic-unwind.diff      | 67 +++++++++++++++++++
 tests/mir-opt/gvn.rs                          |  4 +-
 4 files changed, 151 insertions(+), 1 deletion(-)

diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index bbcb39226b7f..f71e9de4123d 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -795,6 +795,20 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             .collect();
         let fields = fields?;
 
+        if let AggregateTy::Array = ty && fields.len() > 4 {
+            let first = fields[0];
+            if fields.iter().all(|&v| v == first) {
+                let len = ty::Const::from_target_usize(self.tcx, fields.len().try_into().unwrap());
+                if let Some(const_) = self.try_as_constant(first) {
+                    *rvalue = Rvalue::Repeat(Operand::Constant(Box::new(const_)), len);
+                } else if let Some(local) = self.try_as_local(first, location) {
+                    *rvalue = Rvalue::Repeat(Operand::Copy(local.into()), len);
+                    self.reused_locals.insert(local);
+                }
+                return Some(Value::Repeat(first, len));
+            }
+        }
+
         let value = Value::Aggregate(ty, variant_index, fields);
         Some(value)
     }
diff --git a/tests/mir-opt/gvn.aggregates.GVN.panic-abort.diff b/tests/mir-opt/gvn.aggregates.GVN.panic-abort.diff
index c442835bfa99..b1dae36ab6c3 100644
--- a/tests/mir-opt/gvn.aggregates.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.aggregates.GVN.panic-abort.diff
@@ -8,6 +8,16 @@
       let mut _4: [u16; 0];
       let mut _6: ();
       let mut _8: ();
+      let mut _11: i32;
+      let mut _12: i32;
+      let mut _13: i32;
+      let mut _14: i32;
+      let mut _15: i32;
+      let mut _16: i32;
+      let mut _17: i32;
+      let mut _18: i32;
+      let mut _19: i32;
+      let mut _20: i32;
       scope 1 {
           debug a_array => _1;
           let _3: S<[u16; 0]>;
@@ -19,6 +29,14 @@
                   let _7: S<()>;
                   scope 4 {
                       debug b_tuple => _7;
+                      let _9: i32;
+                      scope 5 {
+                          debug val => _9;
+                          let _10: [i32; 10];
+                          scope 6 {
+                              debug array => _10;
+                          }
+                      }
                   }
               }
           }
@@ -55,7 +73,56 @@
 +         _8 = const ();
 +         _7 = const S::<()>(());
           StorageDead(_8);
+-         StorageLive(_9);
++         nop;
+          _9 = const 5_i32;
+          StorageLive(_10);
+          StorageLive(_11);
+-         _11 = _9;
++         _11 = const 5_i32;
+          StorageLive(_12);
+-         _12 = _9;
++         _12 = const 5_i32;
+          StorageLive(_13);
+-         _13 = _9;
++         _13 = const 5_i32;
+          StorageLive(_14);
+-         _14 = _9;
++         _14 = const 5_i32;
+          StorageLive(_15);
+-         _15 = _9;
++         _15 = const 5_i32;
+          StorageLive(_16);
+-         _16 = _9;
++         _16 = const 5_i32;
+          StorageLive(_17);
+-         _17 = _9;
++         _17 = const 5_i32;
+          StorageLive(_18);
+-         _18 = _9;
++         _18 = const 5_i32;
+          StorageLive(_19);
+-         _19 = _9;
++         _19 = const 5_i32;
+          StorageLive(_20);
+-         _20 = _9;
+-         _10 = [move _11, move _12, move _13, move _14, move _15, move _16, move _17, move _18, move _19, move _20];
++         _20 = const 5_i32;
++         _10 = [const 5_i32; 10];
+          StorageDead(_20);
+          StorageDead(_19);
+          StorageDead(_18);
+          StorageDead(_17);
+          StorageDead(_16);
+          StorageDead(_15);
+          StorageDead(_14);
+          StorageDead(_13);
+          StorageDead(_12);
+          StorageDead(_11);
           _0 = const ();
+          StorageDead(_10);
+-         StorageDead(_9);
++         nop;
           StorageDead(_7);
           StorageDead(_5);
           StorageDead(_3);
diff --git a/tests/mir-opt/gvn.aggregates.GVN.panic-unwind.diff b/tests/mir-opt/gvn.aggregates.GVN.panic-unwind.diff
index c442835bfa99..b1dae36ab6c3 100644
--- a/tests/mir-opt/gvn.aggregates.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.aggregates.GVN.panic-unwind.diff
@@ -8,6 +8,16 @@
       let mut _4: [u16; 0];
       let mut _6: ();
       let mut _8: ();
+      let mut _11: i32;
+      let mut _12: i32;
+      let mut _13: i32;
+      let mut _14: i32;
+      let mut _15: i32;
+      let mut _16: i32;
+      let mut _17: i32;
+      let mut _18: i32;
+      let mut _19: i32;
+      let mut _20: i32;
       scope 1 {
           debug a_array => _1;
           let _3: S<[u16; 0]>;
@@ -19,6 +29,14 @@
                   let _7: S<()>;
                   scope 4 {
                       debug b_tuple => _7;
+                      let _9: i32;
+                      scope 5 {
+                          debug val => _9;
+                          let _10: [i32; 10];
+                          scope 6 {
+                              debug array => _10;
+                          }
+                      }
                   }
               }
           }
@@ -55,7 +73,56 @@
 +         _8 = const ();
 +         _7 = const S::<()>(());
           StorageDead(_8);
+-         StorageLive(_9);
++         nop;
+          _9 = const 5_i32;
+          StorageLive(_10);
+          StorageLive(_11);
+-         _11 = _9;
++         _11 = const 5_i32;
+          StorageLive(_12);
+-         _12 = _9;
++         _12 = const 5_i32;
+          StorageLive(_13);
+-         _13 = _9;
++         _13 = const 5_i32;
+          StorageLive(_14);
+-         _14 = _9;
++         _14 = const 5_i32;
+          StorageLive(_15);
+-         _15 = _9;
++         _15 = const 5_i32;
+          StorageLive(_16);
+-         _16 = _9;
++         _16 = const 5_i32;
+          StorageLive(_17);
+-         _17 = _9;
++         _17 = const 5_i32;
+          StorageLive(_18);
+-         _18 = _9;
++         _18 = const 5_i32;
+          StorageLive(_19);
+-         _19 = _9;
++         _19 = const 5_i32;
+          StorageLive(_20);
+-         _20 = _9;
+-         _10 = [move _11, move _12, move _13, move _14, move _15, move _16, move _17, move _18, move _19, move _20];
++         _20 = const 5_i32;
++         _10 = [const 5_i32; 10];
+          StorageDead(_20);
+          StorageDead(_19);
+          StorageDead(_18);
+          StorageDead(_17);
+          StorageDead(_16);
+          StorageDead(_15);
+          StorageDead(_14);
+          StorageDead(_13);
+          StorageDead(_12);
+          StorageDead(_11);
           _0 = const ();
+          StorageDead(_10);
+-         StorageDead(_9);
++         nop;
           StorageDead(_7);
           StorageDead(_5);
           StorageDead(_3);
diff --git a/tests/mir-opt/gvn.rs b/tests/mir-opt/gvn.rs
index ff176a6597b4..406fe106add5 100644
--- a/tests/mir-opt/gvn.rs
+++ b/tests/mir-opt/gvn.rs
@@ -61,7 +61,6 @@ fn subexpression_elimination(x: u64, y: u64, mut z: u64) {
     let e = &z;
     opaque(*e + x);
     opaque(*e + x);
-
 }
 
 fn wrap_unwrap<T: Copy>(x: T) -> T {
@@ -237,6 +236,9 @@ fn aggregates() {
 
     let a_tuple: S<()> = S(());
     let b_tuple: S<()> = S(()); // But this can be with `a_tuple`.
+
+    let val = 5;
+    let array = [val, val, val, val, val, val, val, val, val, val];
 }
 
 fn main() {

From 8162dc2433dac09966ae23bb4422d966a7aeab53 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Tue, 3 Oct 2023 18:06:47 +0000
Subject: [PATCH 067/297] Do not intern GVN temps.

---
 compiler/rustc_mir_transform/src/gvn.rs | 38 +++++++++++--------------
 1 file changed, 16 insertions(+), 22 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index f71e9de4123d..7457f2e4e6f7 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -53,6 +53,7 @@
 //! _c = *_b // replaced by _c = _a
 //! ```
 
+use rustc_const_eval::interpret::MemoryKind;
 use rustc_const_eval::interpret::{ImmTy, InterpCx, MemPlaceMeta, OpTy, Projectable, Scalar};
 use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
 use rustc_data_structures::graph::dominators::Dominators;
@@ -323,24 +324,19 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                 if ty.is_zst() {
                     ImmTy::uninit(ty).into()
                 } else if matches!(ty.abi, Abi::Scalar(..) | Abi::ScalarPair(..)) {
-                    let alloc_id = self
-                        .ecx
-                        .intern_with_temp_alloc(ty, |ecx, dest| {
-                            let variant_dest = if let Some(variant) = variant {
-                                ecx.project_downcast(dest, variant)?
-                            } else {
-                                dest.clone()
-                            };
-                            for (field_index, op) in fields.into_iter().enumerate() {
-                                let field_dest = ecx.project_field(&variant_dest, field_index)?;
-                                ecx.copy_op(op, &field_dest, /*allow_transmute*/ false)?;
-                            }
-                            ecx.write_discriminant(variant.unwrap_or(FIRST_VARIANT), dest)
-                        })
-                        .ok()?;
-                    let mplace =
-                        self.ecx.raw_const_to_mplace(ConstAlloc { alloc_id, ty: ty.ty }).ok()?;
-                    mplace.into()
+                    let dest = self.ecx.allocate(ty, MemoryKind::Stack).ok()?;
+                    let variant_dest = if let Some(variant) = variant {
+                        self.ecx.project_downcast(&dest, variant).ok()?
+                    } else {
+                        dest.clone()
+                    };
+                    for (field_index, op) in fields.into_iter().enumerate() {
+                        let field_dest = self.ecx.project_field(&variant_dest, field_index).ok()?;
+                        self.ecx.copy_op(op, &field_dest, /*allow_transmute*/ false).ok()?;
+                    }
+                    self.ecx.write_discriminant(variant.unwrap_or(FIRST_VARIANT), &dest).ok()?;
+                    self.ecx.alloc_mark_immutable(dest.ptr().provenance.unwrap()).ok()?;
+                    dest.into()
                 } else {
                     return None;
                 }
@@ -846,10 +842,8 @@ fn op_to_prop_const<'tcx>(
     {
         let pointer = mplace.ptr().into_pointer_or_addr().ok()?;
         let (alloc_id, offset) = pointer.into_parts();
-        return if matches!(ecx.tcx.global_alloc(alloc_id), GlobalAlloc::Memory(_)) {
-            Some(ConstValue::Indirect { alloc_id, offset })
-        } else {
-            None
+        if matches!(ecx.tcx.try_get_global_alloc(alloc_id), Some(GlobalAlloc::Memory(_))) {
+            return Some(ConstValue::Indirect { alloc_id, offset })
         }
     }
 

From ebc87bf5672fd7f7a899599d2ee5a219e1d07d07 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 7 Oct 2023 09:19:37 +0000
Subject: [PATCH 068/297] Directly intern values instead of copying them.

---
 .../rustc_const_eval/src/interpret/intern.rs  | 44 +++++++++++++++++++
 .../rustc_const_eval/src/interpret/mod.rs     |  4 +-
 compiler/rustc_mir_transform/src/gvn.rs       |  9 ++--
 3 files changed, 53 insertions(+), 4 deletions(-)

diff --git a/compiler/rustc_const_eval/src/interpret/intern.rs b/compiler/rustc_const_eval/src/interpret/intern.rs
index fd89e34204f0..4cac00c1be26 100644
--- a/compiler/rustc_const_eval/src/interpret/intern.rs
+++ b/compiler/rustc_const_eval/src/interpret/intern.rs
@@ -450,6 +450,50 @@ pub fn intern_const_alloc_recursive<
     Ok(())
 }
 
+/// Intern `ret`, checking it references no other allocation.
+#[instrument(level = "debug", skip(ecx))]
+pub fn intern_const_alloc_for_constprop<
+    'mir,
+    'tcx: 'mir,
+    T,
+    M: CompileTimeMachine<'mir, 'tcx, T>,
+>(
+    ecx: &mut InterpCx<'mir, 'tcx, M>,
+    ret: &MPlaceTy<'tcx>,
+) -> InterpResult<'tcx, ()> {
+    let Some((size, _align)) = ecx.size_and_align_of_mplace(ret)? else {
+        throw_inval!(ConstPropNonsense)
+    };
+
+    let alloc_ref = ecx.get_ptr_alloc(ret.ptr(), size)?.unwrap();
+    // Do not try interning a value that contains provenance.
+    if alloc_ref.has_provenance() {
+        throw_inval!(ConstPropNonsense)
+    }
+
+    // remove allocation
+    let alloc_id = ret.ptr().provenance.unwrap();
+    let Some((_, mut alloc)) = ecx.memory.alloc_map.remove(&alloc_id) else {
+        // Pointer not found in local memory map. It is either a pointer to the global
+        // map, or dangling.
+        if ecx.tcx.try_get_global_alloc(alloc_id).is_none() {
+            throw_ub!(DeadLocal)
+        }
+        // The constant is already in global memory. Do nothing.
+        return Ok(());
+    };
+
+    alloc.mutability = Mutability::Not;
+
+    // link the alloc id to the actual allocation
+    assert!(alloc.provenance().ptrs().is_empty());
+
+    let alloc = ecx.tcx.mk_const_alloc(alloc);
+    ecx.tcx.set_alloc_id_memory(alloc_id, alloc);
+
+    Ok(())
+}
+
 impl<'mir, 'tcx: 'mir, M: super::intern::CompileTimeMachine<'mir, 'tcx, !>>
     InterpCx<'mir, 'tcx, M>
 {
diff --git a/compiler/rustc_const_eval/src/interpret/mod.rs b/compiler/rustc_const_eval/src/interpret/mod.rs
index 13664456987e..7d286d103adf 100644
--- a/compiler/rustc_const_eval/src/interpret/mod.rs
+++ b/compiler/rustc_const_eval/src/interpret/mod.rs
@@ -21,7 +21,9 @@ mod visitor;
 pub use rustc_middle::mir::interpret::*; // have all the `interpret` symbols in one place: here
 
 pub use self::eval_context::{Frame, FrameInfo, InterpCx, StackPopCleanup};
-pub use self::intern::{intern_const_alloc_recursive, InternKind};
+pub use self::intern::{
+    intern_const_alloc_for_constprop, intern_const_alloc_recursive, InternKind,
+};
 pub use self::machine::{compile_time_machine, AllocMap, Machine, MayLeak, StackPopJump};
 pub use self::memory::{AllocKind, AllocRef, AllocRefMut, FnVal, Memory, MemoryKind};
 pub use self::operand::{ImmTy, Immediate, OpTy, Readable};
diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index 7457f2e4e6f7..ba50ca0b7910 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -53,7 +53,7 @@
 //! _c = *_b // replaced by _c = _a
 //! ```
 
-use rustc_const_eval::interpret::MemoryKind;
+use rustc_const_eval::interpret::{intern_const_alloc_for_constprop, MemoryKind};
 use rustc_const_eval::interpret::{ImmTy, InterpCx, MemPlaceMeta, OpTy, Projectable, Scalar};
 use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
 use rustc_data_structures::graph::dominators::Dominators;
@@ -840,10 +840,13 @@ fn op_to_prop_const<'tcx>(
     if let Either::Left(mplace) = op.as_mplace_or_imm()
         && let MemPlaceMeta::None = mplace.meta()
     {
+        intern_const_alloc_for_constprop(ecx, &mplace).ok()?;
         let pointer = mplace.ptr().into_pointer_or_addr().ok()?;
         let (alloc_id, offset) = pointer.into_parts();
-        if matches!(ecx.tcx.try_get_global_alloc(alloc_id), Some(GlobalAlloc::Memory(_))) {
-            return Some(ConstValue::Indirect { alloc_id, offset })
+        match ecx.tcx.global_alloc(alloc_id) {
+            GlobalAlloc::Memory(_) => return Some(ConstValue::Indirect { alloc_id, offset }),
+            // Fallthrough to copying the data.
+            _ => {}
         }
     }
 

From ff6812cd20da84f84b2cb8540110d08bd54e8060 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Tue, 10 Oct 2023 15:42:23 +0000
Subject: [PATCH 069/297] Move provenance checks out of interning method.

---
 .../rustc_const_eval/src/interpret/intern.rs  | 14 ++-----------
 .../rustc_const_eval/src/interpret/memory.rs  |  2 +-
 compiler/rustc_mir_transform/src/gvn.rs       | 21 +++++++++++++++----
 3 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/compiler/rustc_const_eval/src/interpret/intern.rs b/compiler/rustc_const_eval/src/interpret/intern.rs
index 4cac00c1be26..0d21ef7f3274 100644
--- a/compiler/rustc_const_eval/src/interpret/intern.rs
+++ b/compiler/rustc_const_eval/src/interpret/intern.rs
@@ -450,7 +450,7 @@ pub fn intern_const_alloc_recursive<
     Ok(())
 }
 
-/// Intern `ret`, checking it references no other allocation.
+/// Intern `ret`. This function assumes that `ret` references no other allocation.
 #[instrument(level = "debug", skip(ecx))]
 pub fn intern_const_alloc_for_constprop<
     'mir,
@@ -461,17 +461,7 @@ pub fn intern_const_alloc_for_constprop<
     ecx: &mut InterpCx<'mir, 'tcx, M>,
     ret: &MPlaceTy<'tcx>,
 ) -> InterpResult<'tcx, ()> {
-    let Some((size, _align)) = ecx.size_and_align_of_mplace(ret)? else {
-        throw_inval!(ConstPropNonsense)
-    };
-
-    let alloc_ref = ecx.get_ptr_alloc(ret.ptr(), size)?.unwrap();
-    // Do not try interning a value that contains provenance.
-    if alloc_ref.has_provenance() {
-        throw_inval!(ConstPropNonsense)
-    }
-
-    // remove allocation
+    // Move allocation to `tcx`.
     let alloc_id = ret.ptr().provenance.unwrap();
     let Some((_, mut alloc)) = ecx.memory.alloc_map.remove(&alloc_id) else {
         // Pointer not found in local memory map. It is either a pointer to the global
diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs
index d7c7e2798496..16905e93bf1f 100644
--- a/compiler/rustc_const_eval/src/interpret/memory.rs
+++ b/compiler/rustc_const_eval/src/interpret/memory.rs
@@ -1011,7 +1011,7 @@ impl<'tcx, 'a, Prov: Provenance, Extra, Bytes: AllocBytes> AllocRef<'a, 'tcx, Pr
     }
 
     /// Returns whether the allocation has provenance anywhere in the range of the `AllocRef`.
-    pub(crate) fn has_provenance(&self) -> bool {
+    pub fn has_provenance(&self) -> bool {
         !self.alloc.provenance().range_empty(self.range, &self.tcx)
     }
 }
diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index ba50ca0b7910..cced62ecf373 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -832,6 +832,7 @@ fn op_to_prop_const<'tcx>(
     // If this constant has scalar ABI, return it as a `ConstValue::Scalar`.
     if let Abi::Scalar(abi::Scalar::Initialized { .. }) = op.layout.abi
         && let Ok(scalar) = ecx.read_scalar(op)
+        && scalar.try_to_int().is_ok()
     {
         return Some(ConstValue::Scalar(scalar));
     }
@@ -840,6 +841,14 @@ fn op_to_prop_const<'tcx>(
     if let Either::Left(mplace) = op.as_mplace_or_imm()
         && let MemPlaceMeta::None = mplace.meta()
     {
+        let (size, _align) = ecx.size_and_align_of_mplace(&mplace).ok()??;
+
+        // Do not try interning a value that contains provenance.
+        let alloc_ref = ecx.get_ptr_alloc(mplace.ptr(), size).ok()??;
+        if alloc_ref.has_provenance() {
+            return None;
+        }
+
         intern_const_alloc_for_constprop(ecx, &mplace).ok()?;
         let pointer = mplace.ptr().into_pointer_or_addr().ok()?;
         let (alloc_id, offset) = pointer.into_parts();
@@ -853,7 +862,13 @@ fn op_to_prop_const<'tcx>(
     // Everything failed: create a new allocation to hold the data.
     let alloc_id =
         ecx.intern_with_temp_alloc(op.layout, |ecx, dest| ecx.copy_op(op, dest, false)).ok()?;
-    Some(ConstValue::Indirect { alloc_id, offset: Size::ZERO })
+    let value = ConstValue::Indirect { alloc_id, offset: Size::ZERO };
+
+    if !value.has_provenance(*ecx.tcx, op.layout.size) {
+        return Some(value);
+    }
+
+    None
 }
 
 impl<'tcx> VnState<'_, 'tcx> {
@@ -905,9 +920,7 @@ impl<'tcx> VnState<'_, 'tcx> {
 
         // Check that we do not leak a pointer.
         // Those pointers may lose part of their identity in codegen.
-        if value.has_provenance(self.tcx, op.layout.size) {
-            return None;
-        }
+        assert!(!value.has_provenance(self.tcx, op.layout.size));
 
         let const_ = Const::Val(value, op.layout.ty);
         Some(ConstOperand { span: rustc_span::DUMMY_SP, user_ty: None, const_ })

From fbf0a0c5ff0177afda43e404b2e03a48012cfe8f Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Wed, 11 Oct 2023 16:11:55 +0000
Subject: [PATCH 070/297] Explain why we check variant equality.

---
 compiler/rustc_mir_transform/src/gvn.rs | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index cced62ecf373..b9e75f5c93f0 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -512,6 +512,21 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                     return Some(fields[f.as_usize()]);
                 } else if let Value::Projection(outer_value, ProjectionElem::Downcast(_, read_variant)) = self.get(value)
                     && let Value::Aggregate(_, written_variant, fields) = self.get(*outer_value)
+                    // This pass is not aware of control-flow, so we do not know whether the
+                    // replacement we are doing is actually reachable. We could be in any arm of
+                    // ```
+                    // match Some(x) {
+                    //     Some(y) => /* stuff */,
+                    //     None => /* other */,
+                    // }
+                    // ```
+                    //
+                    // In surface rust, the current statement would be unreachable.
+                    //
+                    // However, from the reference chapter on enums and RFC 2195,
+                    // accessing the wrong variant is not UB if the enum has repr.
+                    // So it's not impossible for a series of MIR opts to generate
+                    // a downcast to an inactive variant.
                     && written_variant == read_variant
                 {
                     return Some(fields[f.as_usize()]);

From 59235a79072b393155e86beb96f9618ea0a914f6 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Thu, 12 Oct 2023 06:35:54 +0000
Subject: [PATCH 071/297] Fortify transmute check.

---
 compiler/rustc_mir_transform/src/gvn.rs | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index b9e75f5c93f0..483a1b3f6dfb 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -471,10 +471,9 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                     // `offset` for immediates only supports scalar/scalar-pair ABIs,
                     // so bail out if the target is not one.
                     if value.as_mplace_or_imm().is_right() {
-                        match to.abi {
-                            Abi::Scalar(..) | Abi::ScalarPair(..) => {}
-                            _ if to.is_zst() => {}
-                            Abi::Aggregate { .. } if to.fields.count() == 0 => {}
+                        match (value.layout.abi, to.abi) {
+                            (Abi::Scalar(..), Abi::Scalar(..)) => {}
+                            (Abi::ScalarPair(..), Abi::ScalarPair(..)) => {}
                             _ => return None,
                         }
                     }

From e3538d11f1ca94737e21676a54f8de65523d614c Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Thu, 12 Oct 2023 16:15:52 +0000
Subject: [PATCH 072/297] Do not require absence of metadata.

---
 compiler/rustc_mir_transform/src/gvn.rs | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index 483a1b3f6dfb..c575da611c74 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -54,7 +54,7 @@
 //! ```
 
 use rustc_const_eval::interpret::{intern_const_alloc_for_constprop, MemoryKind};
-use rustc_const_eval::interpret::{ImmTy, InterpCx, MemPlaceMeta, OpTy, Projectable, Scalar};
+use rustc_const_eval::interpret::{ImmTy, InterpCx, OpTy, Projectable, Scalar};
 use rustc_data_structures::fx::{FxHashMap, FxIndexSet};
 use rustc_data_structures::graph::dominators::Dominators;
 use rustc_hir::def::DefKind;
@@ -852,9 +852,7 @@ fn op_to_prop_const<'tcx>(
     }
 
     // If this constant is a projection of another, we can return it directly.
-    if let Either::Left(mplace) = op.as_mplace_or_imm()
-        && let MemPlaceMeta::None = mplace.meta()
-    {
+    if let Either::Left(mplace) = op.as_mplace_or_imm() {
         let (size, _align) = ecx.size_and_align_of_mplace(&mplace).ok()??;
 
         // Do not try interning a value that contains provenance.

From f08dc9be17aff80e248d37a768faa6cc8738d371 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Thu, 12 Oct 2023 16:17:06 +0000
Subject: [PATCH 073/297] Take an AllocId in intern_const_alloc_for_constprop.

---
 compiler/rustc_const_eval/src/interpret/intern.rs | 3 +--
 compiler/rustc_mir_transform/src/gvn.rs           | 9 ++-------
 2 files changed, 3 insertions(+), 9 deletions(-)

diff --git a/compiler/rustc_const_eval/src/interpret/intern.rs b/compiler/rustc_const_eval/src/interpret/intern.rs
index 0d21ef7f3274..23d6d3219471 100644
--- a/compiler/rustc_const_eval/src/interpret/intern.rs
+++ b/compiler/rustc_const_eval/src/interpret/intern.rs
@@ -459,10 +459,9 @@ pub fn intern_const_alloc_for_constprop<
     M: CompileTimeMachine<'mir, 'tcx, T>,
 >(
     ecx: &mut InterpCx<'mir, 'tcx, M>,
-    ret: &MPlaceTy<'tcx>,
+    alloc_id: AllocId,
 ) -> InterpResult<'tcx, ()> {
     // Move allocation to `tcx`.
-    let alloc_id = ret.ptr().provenance.unwrap();
     let Some((_, mut alloc)) = ecx.memory.alloc_map.remove(&alloc_id) else {
         // Pointer not found in local memory map. It is either a pointer to the global
         // map, or dangling.
diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index c575da611c74..58844aed7839 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -61,7 +61,6 @@ use rustc_hir::def::DefKind;
 use rustc_index::bit_set::BitSet;
 use rustc_index::IndexVec;
 use rustc_macros::newtype_index;
-use rustc_middle::mir::interpret::GlobalAlloc;
 use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
 use rustc_middle::ty::layout::LayoutOf;
@@ -861,14 +860,10 @@ fn op_to_prop_const<'tcx>(
             return None;
         }
 
-        intern_const_alloc_for_constprop(ecx, &mplace).ok()?;
         let pointer = mplace.ptr().into_pointer_or_addr().ok()?;
         let (alloc_id, offset) = pointer.into_parts();
-        match ecx.tcx.global_alloc(alloc_id) {
-            GlobalAlloc::Memory(_) => return Some(ConstValue::Indirect { alloc_id, offset }),
-            // Fallthrough to copying the data.
-            _ => {}
-        }
+        intern_const_alloc_for_constprop(ecx, alloc_id).ok()?;
+        return Some(ConstValue::Indirect { alloc_id, offset });
     }
 
     // Everything failed: create a new allocation to hold the data.

From 5e78b9cdb39f2cd47a0518ba568e7bfd0d1f8d95 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 14 Oct 2023 12:12:54 +0000
Subject: [PATCH 074/297] Disambiguate non-deterministic constants.

---
 compiler/rustc_middle/src/mir/consts.rs       |  34 ++++++
 compiler/rustc_mir_transform/src/gvn.rs       | 111 +++++++++++-------
 .../gvn.duplicate_slice.GVN.panic-abort.diff  |  38 ++++++
 .../gvn.duplicate_slice.GVN.panic-unwind.diff |  38 ++++++
 tests/mir-opt/gvn.rs                          |  54 ++++++++-
 tests/mir-opt/gvn.slices.GVN.panic-abort.diff |   8 +-
 .../mir-opt/gvn.slices.GVN.panic-unwind.diff  |   8 +-
 7 files changed, 238 insertions(+), 53 deletions(-)
 create mode 100644 tests/mir-opt/gvn.duplicate_slice.GVN.panic-abort.diff
 create mode 100644 tests/mir-opt/gvn.duplicate_slice.GVN.panic-unwind.diff

diff --git a/compiler/rustc_middle/src/mir/consts.rs b/compiler/rustc_middle/src/mir/consts.rs
index 9e3ee3f2bd34..375870a4fb5c 100644
--- a/compiler/rustc_middle/src/mir/consts.rs
+++ b/compiler/rustc_middle/src/mir/consts.rs
@@ -497,6 +497,40 @@ impl<'tcx> Const<'tcx> {
             _ => Self::Ty(c),
         }
     }
+
+    /// Return true if any evaluation of this constant returns the same value.
+    pub fn is_deterministic(&self) -> bool {
+        // Some constants may contain pointers. We need to preserve the provenance of these
+        // pointers, but not all constants guarantee this:
+        // - valtrees purposefully do not;
+        // - ConstValue::Slice does not either.
+        match self {
+            Const::Ty(c) => match c.kind() {
+                ty::ConstKind::Value(valtree) => match valtree {
+                    // This is just an integer, keep it.
+                    ty::ValTree::Leaf(_) => true,
+                    // This branch may be a reference. Valtree references correspond to a
+                    // different allocation each time they are evaluated.
+                    ty::ValTree::Branch(_) => false,
+                },
+                ty::ConstKind::Param(..) => true,
+                ty::ConstKind::Unevaluated(..) | ty::ConstKind::Expr(..) => false,
+                // Should not appear in runtime MIR.
+                ty::ConstKind::Infer(..)
+                | ty::ConstKind::Bound(..)
+                | ty::ConstKind::Placeholder(..)
+                | ty::ConstKind::Error(..) => bug!(),
+            },
+            Const::Unevaluated(..) => false,
+            // If the same slice appears twice in the MIR, we cannot guarantee that we will
+            // give the same `AllocId` to the data.
+            Const::Val(ConstValue::Slice { .. }, _) => false,
+            Const::Val(
+                ConstValue::ZeroSized | ConstValue::Scalar(_) | ConstValue::Indirect { .. },
+                _,
+            ) => true,
+        }
+    }
 }
 
 /// An unevaluated (potentially generic) constant used in MIR.
diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index 58844aed7839..de61571fdfcb 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -52,6 +52,35 @@
 //! _a = *_b // _b is &Freeze
 //! _c = *_b // replaced by _c = _a
 //! ```
+//!
+//! # Determinism of constant propagation
+//!
+//! When registering a new `Value`, we attempt to opportunistically evaluate it as a constant.
+//! The evaluated form is inserted in `evaluated` as an `OpTy` or `None` if evaluation failed.
+//!
+//! The difficulty is non-deterministic evaluation of MIR constants. Some `Const` can have
+//! different runtime values each time they are evaluated. This is the case with
+//! `Const::Slice` which have a new pointer each time they are evaluated, and constants that
+//! contain a fn pointer (`AllocId` pointing to a `GlobalAlloc::Function`) pointing to a different
+//! symbol in each codegen unit.
+//!
+//! Meanwhile, we want to be able to read indirect constants. For instance:
+//! ```
+//! static A: &'static &'static u8 = &&63;
+//! fn foo() -> u8 {
+//!     **A // We want to replace by 63.
+//! }
+//! fn bar() -> u8 {
+//!     b"abc"[1] // We want to replace by 'b'.
+//! }
+//! ```
+//!
+//! The `Value::Constant` variant stores a possibly unevaluated constant. Evaluating that constant
+//! may be non-deterministic. When that happens, we assign a disambiguator to ensure that we do not
+//! merge the constants. See `duplicate_slice` test in `gvn.rs`.
+//!
+//! Second, when writing constants in MIR, we do not write `Const::Slice` or `Const`
+//! that contain `AllocId`s.
 
 use rustc_const_eval::interpret::{intern_const_alloc_for_constprop, MemoryKind};
 use rustc_const_eval::interpret::{ImmTy, InterpCx, OpTy, Projectable, Scalar};
@@ -161,7 +190,12 @@ enum Value<'tcx> {
     /// The `usize` is a counter incremented by `new_opaque`.
     Opaque(usize),
     /// Evaluated or unevaluated constant value.
-    Constant(Const<'tcx>),
+    Constant {
+        value: Const<'tcx>,
+        /// Some constants do not have a deterministic value. To avoid merging two instances of the
+        /// same `Const`, we assign them an additional integer index.
+        disambiguator: usize,
+    },
     /// An aggregate value, either tuple/closure/struct/enum.
     /// This does not contain unions, as we cannot reason with the value.
     Aggregate(AggregateTy<'tcx>, VariantIdx, Vec<VnIndex>),
@@ -288,8 +322,24 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         }
     }
 
+    fn insert_constant(&mut self, value: Const<'tcx>) -> Option<VnIndex> {
+        let disambiguator = if value.is_deterministic() {
+            // The constant is deterministic, no need to disambiguate.
+            0
+        } else {
+            // Multiple mentions of this constant will yield different values,
+            // so assign a different `disambiguator` to ensure they do not get the same `VnIndex`.
+            let next_opaque = self.next_opaque.as_mut()?;
+            let disambiguator = *next_opaque;
+            *next_opaque += 1;
+            disambiguator
+        };
+        Some(self.insert(Value::Constant { value, disambiguator }))
+    }
+
     fn insert_scalar(&mut self, scalar: Scalar, ty: Ty<'tcx>) -> VnIndex {
-        self.insert(Value::Constant(Const::from_scalar(self.tcx, scalar, ty)))
+        self.insert_constant(Const::from_scalar(self.tcx, scalar, ty))
+            .expect("scalars are deterministic")
     }
 
     #[instrument(level = "trace", skip(self), ret)]
@@ -300,7 +350,9 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             // Do not bother evaluating repeat expressions. This would uselessly consume memory.
             Repeat(..) => return None,
 
-            Constant(ref constant) => self.ecx.eval_mir_constant(constant, None, None).ok()?,
+            Constant { ref value, disambiguator: _ } => {
+                self.ecx.eval_mir_constant(value, None, None).ok()?
+            }
             Aggregate(kind, variant, ref fields) => {
                 let fields = fields
                     .iter()
@@ -657,7 +709,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         match *operand {
             Operand::Constant(ref mut constant) => {
                 let const_ = constant.const_.normalize(self.tcx, self.param_env);
-                Some(self.insert(Value::Constant(const_)))
+                self.insert_constant(const_)
             }
             Operand::Copy(ref mut place) | Operand::Move(ref mut place) => {
                 let value = self.simplify_place_value(place, location)?;
@@ -691,7 +743,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                 Value::Repeat(op, amount)
             }
             Rvalue::NullaryOp(op, ty) => Value::NullaryOp(op, ty),
-            Rvalue::Aggregate(..) => self.simplify_aggregate(rvalue, location)?,
+            Rvalue::Aggregate(..) => return self.simplify_aggregate(rvalue, location),
             Rvalue::Ref(_, borrow_kind, ref mut place) => {
                 self.simplify_place_projection(place, location);
                 return self.new_pointer(*place, AddressKind::Ref(borrow_kind));
@@ -757,7 +809,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
         &mut self,
         rvalue: &mut Rvalue<'tcx>,
         location: Location,
-    ) -> Option<Value<'tcx>> {
+    ) -> Option<VnIndex> {
         let Rvalue::Aggregate(box ref kind, ref mut fields) = *rvalue else { bug!() };
 
         let tcx = self.tcx;
@@ -774,8 +826,7 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
 
             if is_zst {
                 let ty = rvalue.ty(self.local_decls, tcx);
-                let value = Value::Constant(Const::zero_sized(ty));
-                return Some(value);
+                return self.insert_constant(Const::zero_sized(ty));
             }
         }
 
@@ -814,12 +865,11 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                     *rvalue = Rvalue::Repeat(Operand::Copy(local.into()), len);
                     self.reused_locals.insert(local);
                 }
-                return Some(Value::Repeat(first, len));
+                return Some(self.insert(Value::Repeat(first, len)));
             }
         }
 
-        let value = Value::Aggregate(ty, variant_index, fields);
-        Some(value)
+        Some(self.insert(Value::Aggregate(ty, variant_index, fields)))
     }
 }
 
@@ -882,39 +932,12 @@ impl<'tcx> VnState<'_, 'tcx> {
     /// If `index` is a `Value::Constant`, return the `Constant` to be put in the MIR.
     fn try_as_constant(&mut self, index: VnIndex) -> Option<ConstOperand<'tcx>> {
         // This was already constant in MIR, do not change it.
-        if let Value::Constant(const_) = *self.get(index) {
-            // Some constants may contain pointers. We need to preserve the provenance of these
-            // pointers, but not all constants guarantee this:
-            // - valtrees purposefully do not;
-            // - ConstValue::Slice does not either.
-            let const_ok = match const_ {
-                Const::Ty(c) => match c.kind() {
-                    ty::ConstKind::Value(valtree) => match valtree {
-                        // This is just an integer, keep it.
-                        ty::ValTree::Leaf(_) => true,
-                        ty::ValTree::Branch(_) => false,
-                    },
-                    ty::ConstKind::Param(..)
-                    | ty::ConstKind::Unevaluated(..)
-                    | ty::ConstKind::Expr(..) => true,
-                    // Should not appear in runtime MIR.
-                    ty::ConstKind::Infer(..)
-                    | ty::ConstKind::Bound(..)
-                    | ty::ConstKind::Placeholder(..)
-                    | ty::ConstKind::Error(..) => bug!(),
-                },
-                Const::Unevaluated(..) => true,
-                // If the same slice appears twice in the MIR, we cannot guarantee that we will
-                // give the same `AllocId` to the data.
-                Const::Val(ConstValue::Slice { .. }, _) => false,
-                Const::Val(
-                    ConstValue::ZeroSized | ConstValue::Scalar(_) | ConstValue::Indirect { .. },
-                    _,
-                ) => true,
-            };
-            if const_ok {
-                return Some(ConstOperand { span: rustc_span::DUMMY_SP, user_ty: None, const_ });
-            }
+        if let Value::Constant { value, disambiguator: _ } = *self.get(index)
+            // If the constant is not deterministic, adding an additional mention of it in MIR will
+            // not give the same value as the former mention.
+            && value.is_deterministic()
+        {
+            return Some(ConstOperand { span: rustc_span::DUMMY_SP, user_ty: None, const_: value });
         }
 
         let op = self.evaluated[index].as_ref()?;
diff --git a/tests/mir-opt/gvn.duplicate_slice.GVN.panic-abort.diff b/tests/mir-opt/gvn.duplicate_slice.GVN.panic-abort.diff
new file mode 100644
index 000000000000..7ae1fae68e88
--- /dev/null
+++ b/tests/mir-opt/gvn.duplicate_slice.GVN.panic-abort.diff
@@ -0,0 +1,38 @@
+- // MIR for `duplicate_slice` before GVN
++ // MIR for `duplicate_slice` after GVN
+  
+  fn duplicate_slice() -> (bool, bool) {
+      let mut _0: (bool, bool);
+      let mut _1: u128;
+      let mut _2: u128;
+      let mut _3: u128;
+      let mut _4: u128;
+      let mut _5: &str;
+      let mut _6: &str;
+      let mut _7: (&str,);
+      let mut _8: &str;
+      let mut _9: bool;
+      let mut _10: bool;
+  
+      bb0: {
+          _7 = (const "a",);
+          _1 = (_7.0: &str) as u128 (Transmute);
+          _5 = identity::<&str>((_7.0: &str)) -> [return: bb1, unwind unreachable];
+      }
+  
+      bb1: {
+          _3 = _5 as u128 (Transmute);
+          _8 = const "a";
+          _2 = _8 as u128 (Transmute);
+          _6 = identity::<&str>(_8) -> [return: bb2, unwind unreachable];
+      }
+  
+      bb2: {
+          _4 = _6 as u128 (Transmute);
+          _9 = Eq(_1, _2);
+          _10 = Eq(_3, _4);
+          _0 = (_9, _10);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/gvn.duplicate_slice.GVN.panic-unwind.diff b/tests/mir-opt/gvn.duplicate_slice.GVN.panic-unwind.diff
new file mode 100644
index 000000000000..8c96edaa280e
--- /dev/null
+++ b/tests/mir-opt/gvn.duplicate_slice.GVN.panic-unwind.diff
@@ -0,0 +1,38 @@
+- // MIR for `duplicate_slice` before GVN
++ // MIR for `duplicate_slice` after GVN
+  
+  fn duplicate_slice() -> (bool, bool) {
+      let mut _0: (bool, bool);
+      let mut _1: u128;
+      let mut _2: u128;
+      let mut _3: u128;
+      let mut _4: u128;
+      let mut _5: &str;
+      let mut _6: &str;
+      let mut _7: (&str,);
+      let mut _8: &str;
+      let mut _9: bool;
+      let mut _10: bool;
+  
+      bb0: {
+          _7 = (const "a",);
+          _1 = (_7.0: &str) as u128 (Transmute);
+          _5 = identity::<&str>((_7.0: &str)) -> [return: bb1, unwind continue];
+      }
+  
+      bb1: {
+          _3 = _5 as u128 (Transmute);
+          _8 = const "a";
+          _2 = _8 as u128 (Transmute);
+          _6 = identity::<&str>(_8) -> [return: bb2, unwind continue];
+      }
+  
+      bb2: {
+          _4 = _6 as u128 (Transmute);
+          _9 = Eq(_1, _2);
+          _10 = Eq(_3, _4);
+          _0 = (_9, _10);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/gvn.rs b/tests/mir-opt/gvn.rs
index 406fe106add5..55de186edf16 100644
--- a/tests/mir-opt/gvn.rs
+++ b/tests/mir-opt/gvn.rs
@@ -1,11 +1,17 @@
 // skip-filecheck
 // unit-test: GVN
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+// only-64bit
 
 #![feature(raw_ref_op)]
 #![feature(rustc_attrs)]
+#![feature(custom_mir)]
+#![feature(core_intrinsics)]
 #![allow(unconditional_panic)]
 
+use std::intrinsics::mir::*;
+use std::mem::transmute;
+
 struct S<T>(T);
 
 fn subexpression_elimination(x: u64, y: u64, mut z: u64) {
@@ -225,11 +231,49 @@ fn slices() {
     let t = s; // This should be the same pointer, so cannot be a `Const::Slice`.
     opaque(t);
     assert_eq!(s.as_ptr(), t.as_ptr());
-    let u = unsafe { std::mem::transmute::<&str, &[u8]>(s) };
+    let u = unsafe { transmute::<&str, &[u8]>(s) };
     opaque(u);
     assert_eq!(s.as_ptr(), u.as_ptr());
 }
 
+#[custom_mir(dialect = "analysis")]
+fn duplicate_slice() -> (bool, bool) {
+    mir!(
+        let au: u128;
+        let bu: u128;
+        let cu: u128;
+        let du: u128;
+        let c: &str;
+        let d: &str;
+        {
+            let a = ("a",);
+            Call(au = transmute::<_, u128>(a.0), bb1)
+        }
+        bb1 = {
+            Call(c = identity(a.0), bb2)
+        }
+        bb2 = {
+            Call(cu = transmute::<_, u128>(c), bb3)
+        }
+        bb3 = {
+            let b = "a"; // This slice is different from `a.0`.
+            Call(bu = transmute::<_, u128>(b), bb4) // Hence `bu` is not `au`.
+        }
+        bb4 = {
+            Call(d = identity(b), bb5) // This returns a copy of `b`, which is not `a`.
+        }
+        bb5 = {
+            Call(du = transmute::<_, u128>(d), bb6)
+        }
+        bb6 = {
+            let direct = au == bu; // Must not fold to `true`...
+            let indirect = cu == du; // ...as this will not.
+            RET = (direct, indirect);
+            Return()
+        }
+    )
+}
+
 fn aggregates() {
     let a_array: S<[u8; 0]> = S([]);
     let b_array: S<[u16; 0]> = S([]); // This must not be merged with `a_array`.
@@ -253,12 +297,19 @@ fn main() {
     references(5);
     dereferences(&mut 5, &6, &S(7));
     slices();
+    let (direct, indirect) = duplicate_slice();
+    assert_eq!(direct, indirect);
     aggregates();
 }
 
 #[inline(never)]
 fn opaque(_: impl Sized) {}
 
+#[inline(never)]
+fn identity<T>(x: T) -> T {
+    x
+}
+
 // EMIT_MIR gvn.subexpression_elimination.GVN.diff
 // EMIT_MIR gvn.wrap_unwrap.GVN.diff
 // EMIT_MIR gvn.repeated_index.GVN.diff
@@ -270,4 +321,5 @@ fn opaque(_: impl Sized) {}
 // EMIT_MIR gvn.references.GVN.diff
 // EMIT_MIR gvn.dereferences.GVN.diff
 // EMIT_MIR gvn.slices.GVN.diff
+// EMIT_MIR gvn.duplicate_slice.GVN.diff
 // EMIT_MIR gvn.aggregates.GVN.diff
diff --git a/tests/mir-opt/gvn.slices.GVN.panic-abort.diff b/tests/mir-opt/gvn.slices.GVN.panic-abort.diff
index 9db6e068fa7d..ec4499803124 100644
--- a/tests/mir-opt/gvn.slices.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.slices.GVN.panic-abort.diff
@@ -207,8 +207,8 @@
           _26 = &(*_27);
           StorageLive(_28);
           _28 = Option::<Arguments<'_>>::None;
--         _22 = core::panicking::assert_failed::<*const u8, *const u8>(move _23, move _24, move _26, move _28) -> unwind unreachable;
-+         _22 = core::panicking::assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _24, move _26, move _28) -> unwind unreachable;
+-         _22 = assert_failed::<*const u8, *const u8>(move _23, move _24, move _26, move _28) -> unwind unreachable;
++         _22 = assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _24, move _26, move _28) -> unwind unreachable;
       }
   
       bb7: {
@@ -306,8 +306,8 @@
           _52 = &(*_53);
           StorageLive(_54);
           _54 = Option::<Arguments<'_>>::None;
--         _48 = core::panicking::assert_failed::<*const u8, *const u8>(move _49, move _50, move _52, move _54) -> unwind unreachable;
-+         _48 = core::panicking::assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _50, move _52, move _54) -> unwind unreachable;
+-         _48 = assert_failed::<*const u8, *const u8>(move _49, move _50, move _52, move _54) -> unwind unreachable;
++         _48 = assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _50, move _52, move _54) -> unwind unreachable;
       }
   }
   
diff --git a/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff b/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff
index ac7a3e746888..56a78ca86946 100644
--- a/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.slices.GVN.panic-unwind.diff
@@ -207,8 +207,8 @@
           _26 = &(*_27);
           StorageLive(_28);
           _28 = Option::<Arguments<'_>>::None;
--         _22 = core::panicking::assert_failed::<*const u8, *const u8>(move _23, move _24, move _26, move _28) -> unwind continue;
-+         _22 = core::panicking::assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _24, move _26, move _28) -> unwind continue;
+-         _22 = assert_failed::<*const u8, *const u8>(move _23, move _24, move _26, move _28) -> unwind continue;
++         _22 = assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _24, move _26, move _28) -> unwind continue;
       }
   
       bb7: {
@@ -306,8 +306,8 @@
           _52 = &(*_53);
           StorageLive(_54);
           _54 = Option::<Arguments<'_>>::None;
--         _48 = core::panicking::assert_failed::<*const u8, *const u8>(move _49, move _50, move _52, move _54) -> unwind continue;
-+         _48 = core::panicking::assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _50, move _52, move _54) -> unwind continue;
+-         _48 = assert_failed::<*const u8, *const u8>(move _49, move _50, move _52, move _54) -> unwind continue;
++         _48 = assert_failed::<*const u8, *const u8>(const core::panicking::AssertKind::Eq, move _50, move _52, move _54) -> unwind continue;
       }
   }
   

From f6aa3ee7e8a6c28199ebf09810b330b46adf8610 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 14 Oct 2023 12:53:18 +0000
Subject: [PATCH 075/297] Complete comments.

---
 compiler/rustc_middle/src/mir/consts.rs | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/compiler/rustc_middle/src/mir/consts.rs b/compiler/rustc_middle/src/mir/consts.rs
index 375870a4fb5c..d427b4ec3f4d 100644
--- a/compiler/rustc_middle/src/mir/consts.rs
+++ b/compiler/rustc_middle/src/mir/consts.rs
@@ -498,7 +498,8 @@ impl<'tcx> Const<'tcx> {
         }
     }
 
-    /// Return true if any evaluation of this constant returns the same value.
+    /// Return true if any evaluation of this constant always returns the same value,
+    /// taking into account even pointer identity tests.
     pub fn is_deterministic(&self) -> bool {
         // Some constants may contain pointers. We need to preserve the provenance of these
         // pointers, but not all constants guarantee this:
@@ -506,14 +507,10 @@ impl<'tcx> Const<'tcx> {
         // - ConstValue::Slice does not either.
         match self {
             Const::Ty(c) => match c.kind() {
-                ty::ConstKind::Value(valtree) => match valtree {
-                    // This is just an integer, keep it.
-                    ty::ValTree::Leaf(_) => true,
-                    // This branch may be a reference. Valtree references correspond to a
-                    // different allocation each time they are evaluated.
-                    ty::ValTree::Branch(_) => false,
-                },
                 ty::ConstKind::Param(..) => true,
+                // A valtree may be a reference. Valtree references correspond to a
+                // different allocation each time they are evaluated.
+                ty::ConstKind::Value(_) => false,
                 ty::ConstKind::Unevaluated(..) | ty::ConstKind::Expr(..) => false,
                 // Should not appear in runtime MIR.
                 ty::ConstKind::Infer(..)

From 50559ceec491a40ac079aa6cd7943f63387060fd Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 14 Oct 2023 13:31:32 +0000
Subject: [PATCH 076/297] Valtrees for primitive types are fine.

---
 compiler/rustc_middle/src/mir/consts.rs | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/compiler/rustc_middle/src/mir/consts.rs b/compiler/rustc_middle/src/mir/consts.rs
index d427b4ec3f4d..88d4a37f2109 100644
--- a/compiler/rustc_middle/src/mir/consts.rs
+++ b/compiler/rustc_middle/src/mir/consts.rs
@@ -509,8 +509,9 @@ impl<'tcx> Const<'tcx> {
             Const::Ty(c) => match c.kind() {
                 ty::ConstKind::Param(..) => true,
                 // A valtree may be a reference. Valtree references correspond to a
-                // different allocation each time they are evaluated.
-                ty::ConstKind::Value(_) => false,
+                // different allocation each time they are evaluated. Valtrees for primitive
+                // types are fine though.
+                ty::ConstKind::Value(_) => c.ty().is_primitive(),
                 ty::ConstKind::Unevaluated(..) | ty::ConstKind::Expr(..) => false,
                 // Should not appear in runtime MIR.
                 ty::ConstKind::Infer(..)

From ac0228da5950cb4bde01cb74da5ee6a85f75d304 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Mon, 16 Oct 2023 21:44:36 +0000
Subject: [PATCH 077/297] FileCheck gvn.

---
 .../gvn.aggregates.GVN.panic-abort.diff       | 133 ---
 .../gvn.aggregates.GVN.panic-unwind.diff      | 133 ---
 ...vn.arithmetic_checked.GVN.panic-abort.diff | 294 ------
 ...n.arithmetic_checked.GVN.panic-unwind.diff | 294 ------
 .../gvn.comparison.GVN.panic-abort.diff       |  94 ++
 .../gvn.comparison.GVN.panic-unwind.diff      |  94 ++
 tests/mir-opt/gvn.repeat.GVN.panic-abort.diff |  79 ++
 .../mir-opt/gvn.repeat.GVN.panic-unwind.diff  |  79 ++
 tests/mir-opt/gvn.rs                          | 413 ++++++--
 ...xpression_elimination.GVN.panic-abort.diff | 918 +++++++++---------
 ...pression_elimination.GVN.panic-unwind.diff | 918 +++++++++---------
 11 files changed, 1638 insertions(+), 1811 deletions(-)
 delete mode 100644 tests/mir-opt/gvn.aggregates.GVN.panic-abort.diff
 delete mode 100644 tests/mir-opt/gvn.aggregates.GVN.panic-unwind.diff
 create mode 100644 tests/mir-opt/gvn.comparison.GVN.panic-abort.diff
 create mode 100644 tests/mir-opt/gvn.comparison.GVN.panic-unwind.diff
 create mode 100644 tests/mir-opt/gvn.repeat.GVN.panic-abort.diff
 create mode 100644 tests/mir-opt/gvn.repeat.GVN.panic-unwind.diff

diff --git a/tests/mir-opt/gvn.aggregates.GVN.panic-abort.diff b/tests/mir-opt/gvn.aggregates.GVN.panic-abort.diff
deleted file mode 100644
index b1dae36ab6c3..000000000000
--- a/tests/mir-opt/gvn.aggregates.GVN.panic-abort.diff
+++ /dev/null
@@ -1,133 +0,0 @@
-- // MIR for `aggregates` before GVN
-+ // MIR for `aggregates` after GVN
-  
-  fn aggregates() -> () {
-      let mut _0: ();
-      let _1: S<[u8; 0]>;
-      let mut _2: [u8; 0];
-      let mut _4: [u16; 0];
-      let mut _6: ();
-      let mut _8: ();
-      let mut _11: i32;
-      let mut _12: i32;
-      let mut _13: i32;
-      let mut _14: i32;
-      let mut _15: i32;
-      let mut _16: i32;
-      let mut _17: i32;
-      let mut _18: i32;
-      let mut _19: i32;
-      let mut _20: i32;
-      scope 1 {
-          debug a_array => _1;
-          let _3: S<[u16; 0]>;
-          scope 2 {
-              debug b_array => _3;
-              let _5: S<()>;
-              scope 3 {
-                  debug a_tuple => _5;
-                  let _7: S<()>;
-                  scope 4 {
-                      debug b_tuple => _7;
-                      let _9: i32;
-                      scope 5 {
-                          debug val => _9;
-                          let _10: [i32; 10];
-                          scope 6 {
-                              debug array => _10;
-                          }
-                      }
-                  }
-              }
-          }
-      }
-  
-      bb0: {
-          StorageLive(_1);
-          StorageLive(_2);
--         _2 = [];
--         _1 = S::<[u8; 0]>(move _2);
-+         _2 = const [];
-+         _1 = const S::<[u8; 0]>([]);
-          StorageDead(_2);
-          StorageLive(_3);
-          StorageLive(_4);
--         _4 = [];
--         _3 = S::<[u16; 0]>(move _4);
-+         _4 = const [];
-+         _3 = const S::<[u16; 0]>([]);
-          StorageDead(_4);
-          StorageLive(_5);
--         StorageLive(_6);
--         _6 = ();
--         _5 = S::<()>(move _6);
--         StorageDead(_6);
-+         nop;
-+         _6 = const ();
-+         _5 = const S::<()>(());
-+         nop;
-          StorageLive(_7);
-          StorageLive(_8);
--         _8 = ();
--         _7 = S::<()>(move _8);
-+         _8 = const ();
-+         _7 = const S::<()>(());
-          StorageDead(_8);
--         StorageLive(_9);
-+         nop;
-          _9 = const 5_i32;
-          StorageLive(_10);
-          StorageLive(_11);
--         _11 = _9;
-+         _11 = const 5_i32;
-          StorageLive(_12);
--         _12 = _9;
-+         _12 = const 5_i32;
-          StorageLive(_13);
--         _13 = _9;
-+         _13 = const 5_i32;
-          StorageLive(_14);
--         _14 = _9;
-+         _14 = const 5_i32;
-          StorageLive(_15);
--         _15 = _9;
-+         _15 = const 5_i32;
-          StorageLive(_16);
--         _16 = _9;
-+         _16 = const 5_i32;
-          StorageLive(_17);
--         _17 = _9;
-+         _17 = const 5_i32;
-          StorageLive(_18);
--         _18 = _9;
-+         _18 = const 5_i32;
-          StorageLive(_19);
--         _19 = _9;
-+         _19 = const 5_i32;
-          StorageLive(_20);
--         _20 = _9;
--         _10 = [move _11, move _12, move _13, move _14, move _15, move _16, move _17, move _18, move _19, move _20];
-+         _20 = const 5_i32;
-+         _10 = [const 5_i32; 10];
-          StorageDead(_20);
-          StorageDead(_19);
-          StorageDead(_18);
-          StorageDead(_17);
-          StorageDead(_16);
-          StorageDead(_15);
-          StorageDead(_14);
-          StorageDead(_13);
-          StorageDead(_12);
-          StorageDead(_11);
-          _0 = const ();
-          StorageDead(_10);
--         StorageDead(_9);
-+         nop;
-          StorageDead(_7);
-          StorageDead(_5);
-          StorageDead(_3);
-          StorageDead(_1);
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/gvn.aggregates.GVN.panic-unwind.diff b/tests/mir-opt/gvn.aggregates.GVN.panic-unwind.diff
deleted file mode 100644
index b1dae36ab6c3..000000000000
--- a/tests/mir-opt/gvn.aggregates.GVN.panic-unwind.diff
+++ /dev/null
@@ -1,133 +0,0 @@
-- // MIR for `aggregates` before GVN
-+ // MIR for `aggregates` after GVN
-  
-  fn aggregates() -> () {
-      let mut _0: ();
-      let _1: S<[u8; 0]>;
-      let mut _2: [u8; 0];
-      let mut _4: [u16; 0];
-      let mut _6: ();
-      let mut _8: ();
-      let mut _11: i32;
-      let mut _12: i32;
-      let mut _13: i32;
-      let mut _14: i32;
-      let mut _15: i32;
-      let mut _16: i32;
-      let mut _17: i32;
-      let mut _18: i32;
-      let mut _19: i32;
-      let mut _20: i32;
-      scope 1 {
-          debug a_array => _1;
-          let _3: S<[u16; 0]>;
-          scope 2 {
-              debug b_array => _3;
-              let _5: S<()>;
-              scope 3 {
-                  debug a_tuple => _5;
-                  let _7: S<()>;
-                  scope 4 {
-                      debug b_tuple => _7;
-                      let _9: i32;
-                      scope 5 {
-                          debug val => _9;
-                          let _10: [i32; 10];
-                          scope 6 {
-                              debug array => _10;
-                          }
-                      }
-                  }
-              }
-          }
-      }
-  
-      bb0: {
-          StorageLive(_1);
-          StorageLive(_2);
--         _2 = [];
--         _1 = S::<[u8; 0]>(move _2);
-+         _2 = const [];
-+         _1 = const S::<[u8; 0]>([]);
-          StorageDead(_2);
-          StorageLive(_3);
-          StorageLive(_4);
--         _4 = [];
--         _3 = S::<[u16; 0]>(move _4);
-+         _4 = const [];
-+         _3 = const S::<[u16; 0]>([]);
-          StorageDead(_4);
-          StorageLive(_5);
--         StorageLive(_6);
--         _6 = ();
--         _5 = S::<()>(move _6);
--         StorageDead(_6);
-+         nop;
-+         _6 = const ();
-+         _5 = const S::<()>(());
-+         nop;
-          StorageLive(_7);
-          StorageLive(_8);
--         _8 = ();
--         _7 = S::<()>(move _8);
-+         _8 = const ();
-+         _7 = const S::<()>(());
-          StorageDead(_8);
--         StorageLive(_9);
-+         nop;
-          _9 = const 5_i32;
-          StorageLive(_10);
-          StorageLive(_11);
--         _11 = _9;
-+         _11 = const 5_i32;
-          StorageLive(_12);
--         _12 = _9;
-+         _12 = const 5_i32;
-          StorageLive(_13);
--         _13 = _9;
-+         _13 = const 5_i32;
-          StorageLive(_14);
--         _14 = _9;
-+         _14 = const 5_i32;
-          StorageLive(_15);
--         _15 = _9;
-+         _15 = const 5_i32;
-          StorageLive(_16);
--         _16 = _9;
-+         _16 = const 5_i32;
-          StorageLive(_17);
--         _17 = _9;
-+         _17 = const 5_i32;
-          StorageLive(_18);
--         _18 = _9;
-+         _18 = const 5_i32;
-          StorageLive(_19);
--         _19 = _9;
-+         _19 = const 5_i32;
-          StorageLive(_20);
--         _20 = _9;
--         _10 = [move _11, move _12, move _13, move _14, move _15, move _16, move _17, move _18, move _19, move _20];
-+         _20 = const 5_i32;
-+         _10 = [const 5_i32; 10];
-          StorageDead(_20);
-          StorageDead(_19);
-          StorageDead(_18);
-          StorageDead(_17);
-          StorageDead(_16);
-          StorageDead(_15);
-          StorageDead(_14);
-          StorageDead(_13);
-          StorageDead(_12);
-          StorageDead(_11);
-          _0 = const ();
-          StorageDead(_10);
--         StorageDead(_9);
-+         nop;
-          StorageDead(_7);
-          StorageDead(_5);
-          StorageDead(_3);
-          StorageDead(_1);
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-abort.diff b/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-abort.diff
index 3efe7abc976f..6633df3ae702 100644
--- a/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-abort.diff
@@ -20,57 +20,6 @@
       let mut _15: u64;
       let mut _16: u64;
       let mut _17: (u64, bool);
-      let _18: ();
-      let mut _19: u64;
-      let mut _20: u64;
-      let mut _21: bool;
-      let _22: ();
-      let mut _23: u64;
-      let mut _24: u64;
-      let mut _25: bool;
-      let _26: ();
-      let mut _27: u64;
-      let mut _28: u64;
-      let mut _29: bool;
-      let _30: ();
-      let mut _31: u64;
-      let mut _32: u64;
-      let mut _33: bool;
-      let _34: ();
-      let mut _35: u64;
-      let mut _36: u64;
-      let mut _37: bool;
-      let _38: ();
-      let mut _39: u64;
-      let mut _40: u64;
-      let mut _41: bool;
-      let _42: ();
-      let mut _43: u64;
-      let mut _44: u64;
-      let mut _45: bool;
-      let _46: ();
-      let mut _47: u64;
-      let mut _48: u64;
-      let mut _49: bool;
-      let _50: ();
-      let mut _51: u64;
-      let mut _52: u64;
-      let _53: ();
-      let mut _54: u64;
-      let mut _55: u64;
-      let _56: ();
-      let mut _57: u64;
-      let mut _58: u64;
-      let _59: ();
-      let mut _60: u64;
-      let mut _61: u64;
-      let mut _62: u32;
-      let mut _63: bool;
-      let _64: ();
-      let mut _65: u64;
-      let mut _66: u64;
-      let mut _67: u32;
-      let mut _68: bool;
   
       bb0: {
           StorageLive(_2);
@@ -149,249 +98,6 @@
       bb8: {
           StorageDead(_15);
           StorageDead(_14);
-          StorageLive(_18);
-          StorageLive(_19);
-          StorageLive(_20);
-          _20 = _1;
--         _21 = Eq(const 0_u64, const 0_u64);
--         assert(!move _21, "attempt to divide `{}` by zero", _20) -> [success: bb9, unwind unreachable];
-+         _21 = const true;
-+         assert(!const true, "attempt to divide `{}` by zero", _1) -> [success: bb9, unwind unreachable];
-      }
-  
-      bb9: {
--         _19 = Div(move _20, const 0_u64);
-+         _19 = Div(_1, const 0_u64);
-          StorageDead(_20);
-          _18 = opaque::<u64>(move _19) -> [return: bb10, unwind unreachable];
-      }
-  
-      bb10: {
-          StorageDead(_19);
-          StorageDead(_18);
-          StorageLive(_22);
-          StorageLive(_23);
-          StorageLive(_24);
-          _24 = _1;
--         _25 = Eq(const 1_u64, const 0_u64);
--         assert(!move _25, "attempt to divide `{}` by zero", _24) -> [success: bb11, unwind unreachable];
-+         _25 = const false;
-+         assert(!const false, "attempt to divide `{}` by zero", _1) -> [success: bb11, unwind unreachable];
-      }
-  
-      bb11: {
--         _23 = Div(move _24, const 1_u64);
-+         _23 = Div(_1, const 1_u64);
-          StorageDead(_24);
-          _22 = opaque::<u64>(move _23) -> [return: bb12, unwind unreachable];
-      }
-  
-      bb12: {
-          StorageDead(_23);
-          StorageDead(_22);
-          StorageLive(_26);
-          StorageLive(_27);
-          StorageLive(_28);
-          _28 = _1;
--         _29 = Eq(_28, const 0_u64);
--         assert(!move _29, "attempt to divide `{}` by zero", const 0_u64) -> [success: bb13, unwind unreachable];
-+         _29 = Eq(_1, const 0_u64);
-+         assert(!_29, "attempt to divide `{}` by zero", const 0_u64) -> [success: bb13, unwind unreachable];
-      }
-  
-      bb13: {
--         _27 = Div(const 0_u64, move _28);
-+         _27 = Div(const 0_u64, _1);
-          StorageDead(_28);
-          _26 = opaque::<u64>(move _27) -> [return: bb14, unwind unreachable];
-      }
-  
-      bb14: {
-          StorageDead(_27);
-          StorageDead(_26);
-          StorageLive(_30);
-          StorageLive(_31);
-          StorageLive(_32);
-          _32 = _1;
--         _33 = Eq(_32, const 0_u64);
--         assert(!move _33, "attempt to divide `{}` by zero", const 1_u64) -> [success: bb15, unwind unreachable];
-+         _33 = _29;
-+         assert(!_29, "attempt to divide `{}` by zero", const 1_u64) -> [success: bb15, unwind unreachable];
-      }
-  
-      bb15: {
--         _31 = Div(const 1_u64, move _32);
-+         _31 = Div(const 1_u64, _1);
-          StorageDead(_32);
-          _30 = opaque::<u64>(move _31) -> [return: bb16, unwind unreachable];
-      }
-  
-      bb16: {
-          StorageDead(_31);
-          StorageDead(_30);
-          StorageLive(_34);
-          StorageLive(_35);
-          StorageLive(_36);
-          _36 = _1;
--         _37 = Eq(const 0_u64, const 0_u64);
--         assert(!move _37, "attempt to calculate the remainder of `{}` with a divisor of zero", _36) -> [success: bb17, unwind unreachable];
-+         _37 = const true;
-+         assert(!const true, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb17, unwind unreachable];
-      }
-  
-      bb17: {
--         _35 = Rem(move _36, const 0_u64);
-+         _35 = Rem(_1, const 0_u64);
-          StorageDead(_36);
-          _34 = opaque::<u64>(move _35) -> [return: bb18, unwind unreachable];
-      }
-  
-      bb18: {
-          StorageDead(_35);
-          StorageDead(_34);
-          StorageLive(_38);
-          StorageLive(_39);
-          StorageLive(_40);
-          _40 = _1;
--         _41 = Eq(const 1_u64, const 0_u64);
--         assert(!move _41, "attempt to calculate the remainder of `{}` with a divisor of zero", _40) -> [success: bb19, unwind unreachable];
-+         _41 = const false;
-+         assert(!const false, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb19, unwind unreachable];
-      }
-  
-      bb19: {
--         _39 = Rem(move _40, const 1_u64);
-+         _39 = Rem(_1, const 1_u64);
-          StorageDead(_40);
-          _38 = opaque::<u64>(move _39) -> [return: bb20, unwind unreachable];
-      }
-  
-      bb20: {
-          StorageDead(_39);
-          StorageDead(_38);
-          StorageLive(_42);
-          StorageLive(_43);
-          StorageLive(_44);
-          _44 = _1;
--         _45 = Eq(_44, const 0_u64);
--         assert(!move _45, "attempt to calculate the remainder of `{}` with a divisor of zero", const 0_u64) -> [success: bb21, unwind unreachable];
-+         _45 = _29;
-+         assert(!_29, "attempt to calculate the remainder of `{}` with a divisor of zero", const 0_u64) -> [success: bb21, unwind unreachable];
-      }
-  
-      bb21: {
--         _43 = Rem(const 0_u64, move _44);
-+         _43 = Rem(const 0_u64, _1);
-          StorageDead(_44);
-          _42 = opaque::<u64>(move _43) -> [return: bb22, unwind unreachable];
-      }
-  
-      bb22: {
-          StorageDead(_43);
-          StorageDead(_42);
-          StorageLive(_46);
-          StorageLive(_47);
-          StorageLive(_48);
-          _48 = _1;
--         _49 = Eq(_48, const 0_u64);
--         assert(!move _49, "attempt to calculate the remainder of `{}` with a divisor of zero", const 1_u64) -> [success: bb23, unwind unreachable];
-+         _49 = _29;
-+         assert(!_29, "attempt to calculate the remainder of `{}` with a divisor of zero", const 1_u64) -> [success: bb23, unwind unreachable];
-      }
-  
-      bb23: {
--         _47 = Rem(const 1_u64, move _48);
-+         _47 = Rem(const 1_u64, _1);
-          StorageDead(_48);
-          _46 = opaque::<u64>(move _47) -> [return: bb24, unwind unreachable];
-      }
-  
-      bb24: {
-          StorageDead(_47);
-          StorageDead(_46);
-          StorageLive(_50);
-          StorageLive(_51);
-          StorageLive(_52);
-          _52 = _1;
--         _51 = BitAnd(move _52, const 0_u64);
-+         _51 = BitAnd(_1, const 0_u64);
-          StorageDead(_52);
-          _50 = opaque::<u64>(move _51) -> [return: bb25, unwind unreachable];
-      }
-  
-      bb25: {
-          StorageDead(_51);
-          StorageDead(_50);
-          StorageLive(_53);
-          StorageLive(_54);
-          StorageLive(_55);
-          _55 = _1;
--         _54 = BitOr(move _55, const 0_u64);
-+         _54 = BitOr(_1, const 0_u64);
-          StorageDead(_55);
-          _53 = opaque::<u64>(move _54) -> [return: bb26, unwind unreachable];
-      }
-  
-      bb26: {
-          StorageDead(_54);
-          StorageDead(_53);
-          StorageLive(_56);
-          StorageLive(_57);
-          StorageLive(_58);
-          _58 = _1;
--         _57 = BitXor(move _58, const 0_u64);
-+         _57 = BitXor(_1, const 0_u64);
-          StorageDead(_58);
-          _56 = opaque::<u64>(move _57) -> [return: bb27, unwind unreachable];
-      }
-  
-      bb27: {
-          StorageDead(_57);
-          StorageDead(_56);
-          StorageLive(_59);
-          StorageLive(_60);
-          StorageLive(_61);
-          _61 = _1;
--         _62 = const 0_i32 as u32 (IntToInt);
--         _63 = Lt(move _62, const 64_u32);
--         assert(move _63, "attempt to shift right by `{}`, which would overflow", const 0_i32) -> [success: bb28, unwind unreachable];
-+         _62 = const 0_u32;
-+         _63 = const true;
-+         assert(const true, "attempt to shift right by `{}`, which would overflow", const 0_i32) -> [success: bb28, unwind unreachable];
-      }
-  
-      bb28: {
--         _60 = Shr(move _61, const 0_i32);
-+         _60 = Shr(_1, const 0_i32);
-          StorageDead(_61);
-          _59 = opaque::<u64>(move _60) -> [return: bb29, unwind unreachable];
-      }
-  
-      bb29: {
-          StorageDead(_60);
-          StorageDead(_59);
-          StorageLive(_64);
-          StorageLive(_65);
-          StorageLive(_66);
-          _66 = _1;
--         _67 = const 0_i32 as u32 (IntToInt);
--         _68 = Lt(move _67, const 64_u32);
--         assert(move _68, "attempt to shift left by `{}`, which would overflow", const 0_i32) -> [success: bb30, unwind unreachable];
-+         _67 = const 0_u32;
-+         _68 = const true;
-+         assert(const true, "attempt to shift left by `{}`, which would overflow", const 0_i32) -> [success: bb30, unwind unreachable];
-      }
-  
-      bb30: {
--         _65 = Shl(move _66, const 0_i32);
-+         _65 = Shl(_1, const 0_i32);
-          StorageDead(_66);
-          _64 = opaque::<u64>(move _65) -> [return: bb31, unwind unreachable];
-      }
-  
-      bb31: {
-          StorageDead(_65);
-          StorageDead(_64);
           _0 = const ();
           return;
       }
diff --git a/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-unwind.diff b/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-unwind.diff
index 9fbb8df79a1d..d100a77fee5f 100644
--- a/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.arithmetic_checked.GVN.panic-unwind.diff
@@ -20,57 +20,6 @@
       let mut _15: u64;
       let mut _16: u64;
       let mut _17: (u64, bool);
-      let _18: ();
-      let mut _19: u64;
-      let mut _20: u64;
-      let mut _21: bool;
-      let _22: ();
-      let mut _23: u64;
-      let mut _24: u64;
-      let mut _25: bool;
-      let _26: ();
-      let mut _27: u64;
-      let mut _28: u64;
-      let mut _29: bool;
-      let _30: ();
-      let mut _31: u64;
-      let mut _32: u64;
-      let mut _33: bool;
-      let _34: ();
-      let mut _35: u64;
-      let mut _36: u64;
-      let mut _37: bool;
-      let _38: ();
-      let mut _39: u64;
-      let mut _40: u64;
-      let mut _41: bool;
-      let _42: ();
-      let mut _43: u64;
-      let mut _44: u64;
-      let mut _45: bool;
-      let _46: ();
-      let mut _47: u64;
-      let mut _48: u64;
-      let mut _49: bool;
-      let _50: ();
-      let mut _51: u64;
-      let mut _52: u64;
-      let _53: ();
-      let mut _54: u64;
-      let mut _55: u64;
-      let _56: ();
-      let mut _57: u64;
-      let mut _58: u64;
-      let _59: ();
-      let mut _60: u64;
-      let mut _61: u64;
-      let mut _62: u32;
-      let mut _63: bool;
-      let _64: ();
-      let mut _65: u64;
-      let mut _66: u64;
-      let mut _67: u32;
-      let mut _68: bool;
   
       bb0: {
           StorageLive(_2);
@@ -149,249 +98,6 @@
       bb8: {
           StorageDead(_15);
           StorageDead(_14);
-          StorageLive(_18);
-          StorageLive(_19);
-          StorageLive(_20);
-          _20 = _1;
--         _21 = Eq(const 0_u64, const 0_u64);
--         assert(!move _21, "attempt to divide `{}` by zero", _20) -> [success: bb9, unwind continue];
-+         _21 = const true;
-+         assert(!const true, "attempt to divide `{}` by zero", _1) -> [success: bb9, unwind continue];
-      }
-  
-      bb9: {
--         _19 = Div(move _20, const 0_u64);
-+         _19 = Div(_1, const 0_u64);
-          StorageDead(_20);
-          _18 = opaque::<u64>(move _19) -> [return: bb10, unwind continue];
-      }
-  
-      bb10: {
-          StorageDead(_19);
-          StorageDead(_18);
-          StorageLive(_22);
-          StorageLive(_23);
-          StorageLive(_24);
-          _24 = _1;
--         _25 = Eq(const 1_u64, const 0_u64);
--         assert(!move _25, "attempt to divide `{}` by zero", _24) -> [success: bb11, unwind continue];
-+         _25 = const false;
-+         assert(!const false, "attempt to divide `{}` by zero", _1) -> [success: bb11, unwind continue];
-      }
-  
-      bb11: {
--         _23 = Div(move _24, const 1_u64);
-+         _23 = Div(_1, const 1_u64);
-          StorageDead(_24);
-          _22 = opaque::<u64>(move _23) -> [return: bb12, unwind continue];
-      }
-  
-      bb12: {
-          StorageDead(_23);
-          StorageDead(_22);
-          StorageLive(_26);
-          StorageLive(_27);
-          StorageLive(_28);
-          _28 = _1;
--         _29 = Eq(_28, const 0_u64);
--         assert(!move _29, "attempt to divide `{}` by zero", const 0_u64) -> [success: bb13, unwind continue];
-+         _29 = Eq(_1, const 0_u64);
-+         assert(!_29, "attempt to divide `{}` by zero", const 0_u64) -> [success: bb13, unwind continue];
-      }
-  
-      bb13: {
--         _27 = Div(const 0_u64, move _28);
-+         _27 = Div(const 0_u64, _1);
-          StorageDead(_28);
-          _26 = opaque::<u64>(move _27) -> [return: bb14, unwind continue];
-      }
-  
-      bb14: {
-          StorageDead(_27);
-          StorageDead(_26);
-          StorageLive(_30);
-          StorageLive(_31);
-          StorageLive(_32);
-          _32 = _1;
--         _33 = Eq(_32, const 0_u64);
--         assert(!move _33, "attempt to divide `{}` by zero", const 1_u64) -> [success: bb15, unwind continue];
-+         _33 = _29;
-+         assert(!_29, "attempt to divide `{}` by zero", const 1_u64) -> [success: bb15, unwind continue];
-      }
-  
-      bb15: {
--         _31 = Div(const 1_u64, move _32);
-+         _31 = Div(const 1_u64, _1);
-          StorageDead(_32);
-          _30 = opaque::<u64>(move _31) -> [return: bb16, unwind continue];
-      }
-  
-      bb16: {
-          StorageDead(_31);
-          StorageDead(_30);
-          StorageLive(_34);
-          StorageLive(_35);
-          StorageLive(_36);
-          _36 = _1;
--         _37 = Eq(const 0_u64, const 0_u64);
--         assert(!move _37, "attempt to calculate the remainder of `{}` with a divisor of zero", _36) -> [success: bb17, unwind continue];
-+         _37 = const true;
-+         assert(!const true, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb17, unwind continue];
-      }
-  
-      bb17: {
--         _35 = Rem(move _36, const 0_u64);
-+         _35 = Rem(_1, const 0_u64);
-          StorageDead(_36);
-          _34 = opaque::<u64>(move _35) -> [return: bb18, unwind continue];
-      }
-  
-      bb18: {
-          StorageDead(_35);
-          StorageDead(_34);
-          StorageLive(_38);
-          StorageLive(_39);
-          StorageLive(_40);
-          _40 = _1;
--         _41 = Eq(const 1_u64, const 0_u64);
--         assert(!move _41, "attempt to calculate the remainder of `{}` with a divisor of zero", _40) -> [success: bb19, unwind continue];
-+         _41 = const false;
-+         assert(!const false, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb19, unwind continue];
-      }
-  
-      bb19: {
--         _39 = Rem(move _40, const 1_u64);
-+         _39 = Rem(_1, const 1_u64);
-          StorageDead(_40);
-          _38 = opaque::<u64>(move _39) -> [return: bb20, unwind continue];
-      }
-  
-      bb20: {
-          StorageDead(_39);
-          StorageDead(_38);
-          StorageLive(_42);
-          StorageLive(_43);
-          StorageLive(_44);
-          _44 = _1;
--         _45 = Eq(_44, const 0_u64);
--         assert(!move _45, "attempt to calculate the remainder of `{}` with a divisor of zero", const 0_u64) -> [success: bb21, unwind continue];
-+         _45 = _29;
-+         assert(!_29, "attempt to calculate the remainder of `{}` with a divisor of zero", const 0_u64) -> [success: bb21, unwind continue];
-      }
-  
-      bb21: {
--         _43 = Rem(const 0_u64, move _44);
-+         _43 = Rem(const 0_u64, _1);
-          StorageDead(_44);
-          _42 = opaque::<u64>(move _43) -> [return: bb22, unwind continue];
-      }
-  
-      bb22: {
-          StorageDead(_43);
-          StorageDead(_42);
-          StorageLive(_46);
-          StorageLive(_47);
-          StorageLive(_48);
-          _48 = _1;
--         _49 = Eq(_48, const 0_u64);
--         assert(!move _49, "attempt to calculate the remainder of `{}` with a divisor of zero", const 1_u64) -> [success: bb23, unwind continue];
-+         _49 = _29;
-+         assert(!_29, "attempt to calculate the remainder of `{}` with a divisor of zero", const 1_u64) -> [success: bb23, unwind continue];
-      }
-  
-      bb23: {
--         _47 = Rem(const 1_u64, move _48);
-+         _47 = Rem(const 1_u64, _1);
-          StorageDead(_48);
-          _46 = opaque::<u64>(move _47) -> [return: bb24, unwind continue];
-      }
-  
-      bb24: {
-          StorageDead(_47);
-          StorageDead(_46);
-          StorageLive(_50);
-          StorageLive(_51);
-          StorageLive(_52);
-          _52 = _1;
--         _51 = BitAnd(move _52, const 0_u64);
-+         _51 = BitAnd(_1, const 0_u64);
-          StorageDead(_52);
-          _50 = opaque::<u64>(move _51) -> [return: bb25, unwind continue];
-      }
-  
-      bb25: {
-          StorageDead(_51);
-          StorageDead(_50);
-          StorageLive(_53);
-          StorageLive(_54);
-          StorageLive(_55);
-          _55 = _1;
--         _54 = BitOr(move _55, const 0_u64);
-+         _54 = BitOr(_1, const 0_u64);
-          StorageDead(_55);
-          _53 = opaque::<u64>(move _54) -> [return: bb26, unwind continue];
-      }
-  
-      bb26: {
-          StorageDead(_54);
-          StorageDead(_53);
-          StorageLive(_56);
-          StorageLive(_57);
-          StorageLive(_58);
-          _58 = _1;
--         _57 = BitXor(move _58, const 0_u64);
-+         _57 = BitXor(_1, const 0_u64);
-          StorageDead(_58);
-          _56 = opaque::<u64>(move _57) -> [return: bb27, unwind continue];
-      }
-  
-      bb27: {
-          StorageDead(_57);
-          StorageDead(_56);
-          StorageLive(_59);
-          StorageLive(_60);
-          StorageLive(_61);
-          _61 = _1;
--         _62 = const 0_i32 as u32 (IntToInt);
--         _63 = Lt(move _62, const 64_u32);
--         assert(move _63, "attempt to shift right by `{}`, which would overflow", const 0_i32) -> [success: bb28, unwind continue];
-+         _62 = const 0_u32;
-+         _63 = const true;
-+         assert(const true, "attempt to shift right by `{}`, which would overflow", const 0_i32) -> [success: bb28, unwind continue];
-      }
-  
-      bb28: {
--         _60 = Shr(move _61, const 0_i32);
-+         _60 = Shr(_1, const 0_i32);
-          StorageDead(_61);
-          _59 = opaque::<u64>(move _60) -> [return: bb29, unwind continue];
-      }
-  
-      bb29: {
-          StorageDead(_60);
-          StorageDead(_59);
-          StorageLive(_64);
-          StorageLive(_65);
-          StorageLive(_66);
-          _66 = _1;
--         _67 = const 0_i32 as u32 (IntToInt);
--         _68 = Lt(move _67, const 64_u32);
--         assert(move _68, "attempt to shift left by `{}`, which would overflow", const 0_i32) -> [success: bb30, unwind continue];
-+         _67 = const 0_u32;
-+         _68 = const true;
-+         assert(const true, "attempt to shift left by `{}`, which would overflow", const 0_i32) -> [success: bb30, unwind continue];
-      }
-  
-      bb30: {
--         _65 = Shl(move _66, const 0_i32);
-+         _65 = Shl(_1, const 0_i32);
-          StorageDead(_66);
-          _64 = opaque::<u64>(move _65) -> [return: bb31, unwind continue];
-      }
-  
-      bb31: {
-          StorageDead(_65);
-          StorageDead(_64);
           _0 = const ();
           return;
       }
diff --git a/tests/mir-opt/gvn.comparison.GVN.panic-abort.diff b/tests/mir-opt/gvn.comparison.GVN.panic-abort.diff
new file mode 100644
index 000000000000..ee3b9da2122b
--- /dev/null
+++ b/tests/mir-opt/gvn.comparison.GVN.panic-abort.diff
@@ -0,0 +1,94 @@
+- // MIR for `comparison` before GVN
++ // MIR for `comparison` after GVN
+  
+  fn comparison(_1: u64, _2: u64) -> () {
+      debug x => _1;
+      debug y => _2;
+      let mut _0: ();
+      let _3: ();
+      let mut _4: bool;
+      let mut _5: u64;
+      let mut _6: u64;
+      let _7: ();
+      let mut _8: bool;
+      let mut _9: u64;
+      let mut _10: u64;
+      let _11: ();
+      let mut _12: bool;
+      let mut _13: u64;
+      let mut _14: u64;
+      let _15: ();
+      let mut _16: bool;
+      let mut _17: u64;
+      let mut _18: u64;
+  
+      bb0: {
+          StorageLive(_3);
+          StorageLive(_4);
+          StorageLive(_5);
+          _5 = _1;
+          StorageLive(_6);
+          _6 = _1;
+-         _4 = Eq(move _5, move _6);
++         _4 = Eq(_1, _1);
+          StorageDead(_6);
+          StorageDead(_5);
+          _3 = opaque::<bool>(move _4) -> [return: bb1, unwind unreachable];
+      }
+  
+      bb1: {
+          StorageDead(_4);
+          StorageDead(_3);
+          StorageLive(_7);
+          StorageLive(_8);
+          StorageLive(_9);
+          _9 = _1;
+          StorageLive(_10);
+          _10 = _1;
+-         _8 = Ne(move _9, move _10);
++         _8 = Ne(_1, _1);
+          StorageDead(_10);
+          StorageDead(_9);
+          _7 = opaque::<bool>(move _8) -> [return: bb2, unwind unreachable];
+      }
+  
+      bb2: {
+          StorageDead(_8);
+          StorageDead(_7);
+          StorageLive(_11);
+          StorageLive(_12);
+          StorageLive(_13);
+          _13 = _1;
+          StorageLive(_14);
+          _14 = _2;
+-         _12 = Eq(move _13, move _14);
++         _12 = Eq(_1, _2);
+          StorageDead(_14);
+          StorageDead(_13);
+          _11 = opaque::<bool>(move _12) -> [return: bb3, unwind unreachable];
+      }
+  
+      bb3: {
+          StorageDead(_12);
+          StorageDead(_11);
+          StorageLive(_15);
+          StorageLive(_16);
+          StorageLive(_17);
+          _17 = _1;
+          StorageLive(_18);
+          _18 = _2;
+-         _16 = Ne(move _17, move _18);
++         _16 = Ne(_1, _2);
+          StorageDead(_18);
+          StorageDead(_17);
+          _15 = opaque::<bool>(move _16) -> [return: bb4, unwind unreachable];
+      }
+  
+      bb4: {
+          StorageDead(_16);
+          StorageDead(_15);
+          _0 = const ();
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/gvn.comparison.GVN.panic-unwind.diff b/tests/mir-opt/gvn.comparison.GVN.panic-unwind.diff
new file mode 100644
index 000000000000..a1408fe3434c
--- /dev/null
+++ b/tests/mir-opt/gvn.comparison.GVN.panic-unwind.diff
@@ -0,0 +1,94 @@
+- // MIR for `comparison` before GVN
++ // MIR for `comparison` after GVN
+  
+  fn comparison(_1: u64, _2: u64) -> () {
+      debug x => _1;
+      debug y => _2;
+      let mut _0: ();
+      let _3: ();
+      let mut _4: bool;
+      let mut _5: u64;
+      let mut _6: u64;
+      let _7: ();
+      let mut _8: bool;
+      let mut _9: u64;
+      let mut _10: u64;
+      let _11: ();
+      let mut _12: bool;
+      let mut _13: u64;
+      let mut _14: u64;
+      let _15: ();
+      let mut _16: bool;
+      let mut _17: u64;
+      let mut _18: u64;
+  
+      bb0: {
+          StorageLive(_3);
+          StorageLive(_4);
+          StorageLive(_5);
+          _5 = _1;
+          StorageLive(_6);
+          _6 = _1;
+-         _4 = Eq(move _5, move _6);
++         _4 = Eq(_1, _1);
+          StorageDead(_6);
+          StorageDead(_5);
+          _3 = opaque::<bool>(move _4) -> [return: bb1, unwind continue];
+      }
+  
+      bb1: {
+          StorageDead(_4);
+          StorageDead(_3);
+          StorageLive(_7);
+          StorageLive(_8);
+          StorageLive(_9);
+          _9 = _1;
+          StorageLive(_10);
+          _10 = _1;
+-         _8 = Ne(move _9, move _10);
++         _8 = Ne(_1, _1);
+          StorageDead(_10);
+          StorageDead(_9);
+          _7 = opaque::<bool>(move _8) -> [return: bb2, unwind continue];
+      }
+  
+      bb2: {
+          StorageDead(_8);
+          StorageDead(_7);
+          StorageLive(_11);
+          StorageLive(_12);
+          StorageLive(_13);
+          _13 = _1;
+          StorageLive(_14);
+          _14 = _2;
+-         _12 = Eq(move _13, move _14);
++         _12 = Eq(_1, _2);
+          StorageDead(_14);
+          StorageDead(_13);
+          _11 = opaque::<bool>(move _12) -> [return: bb3, unwind continue];
+      }
+  
+      bb3: {
+          StorageDead(_12);
+          StorageDead(_11);
+          StorageLive(_15);
+          StorageLive(_16);
+          StorageLive(_17);
+          _17 = _1;
+          StorageLive(_18);
+          _18 = _2;
+-         _16 = Ne(move _17, move _18);
++         _16 = Ne(_1, _2);
+          StorageDead(_18);
+          StorageDead(_17);
+          _15 = opaque::<bool>(move _16) -> [return: bb4, unwind continue];
+      }
+  
+      bb4: {
+          StorageDead(_16);
+          StorageDead(_15);
+          _0 = const ();
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/gvn.repeat.GVN.panic-abort.diff b/tests/mir-opt/gvn.repeat.GVN.panic-abort.diff
new file mode 100644
index 000000000000..37915f8578d2
--- /dev/null
+++ b/tests/mir-opt/gvn.repeat.GVN.panic-abort.diff
@@ -0,0 +1,79 @@
+- // MIR for `repeat` before GVN
++ // MIR for `repeat` after GVN
+  
+  fn repeat() -> () {
+      let mut _0: ();
+      let _1: i32;
+      let mut _3: i32;
+      let mut _4: i32;
+      let mut _5: i32;
+      let mut _6: i32;
+      let mut _7: i32;
+      let mut _8: i32;
+      let mut _9: i32;
+      let mut _10: i32;
+      let mut _11: i32;
+      let mut _12: i32;
+      scope 1 {
+          debug val => _1;
+          let _2: [i32; 10];
+          scope 2 {
+              debug array => _2;
+          }
+      }
+  
+      bb0: {
+-         StorageLive(_1);
++         nop;
+          _1 = const 5_i32;
+          StorageLive(_2);
+          StorageLive(_3);
+-         _3 = _1;
++         _3 = const 5_i32;
+          StorageLive(_4);
+-         _4 = _1;
++         _4 = const 5_i32;
+          StorageLive(_5);
+-         _5 = _1;
++         _5 = const 5_i32;
+          StorageLive(_6);
+-         _6 = _1;
++         _6 = const 5_i32;
+          StorageLive(_7);
+-         _7 = _1;
++         _7 = const 5_i32;
+          StorageLive(_8);
+-         _8 = _1;
++         _8 = const 5_i32;
+          StorageLive(_9);
+-         _9 = _1;
++         _9 = const 5_i32;
+          StorageLive(_10);
+-         _10 = _1;
++         _10 = const 5_i32;
+          StorageLive(_11);
+-         _11 = _1;
++         _11 = const 5_i32;
+          StorageLive(_12);
+-         _12 = _1;
+-         _2 = [move _3, move _4, move _5, move _6, move _7, move _8, move _9, move _10, move _11, move _12];
++         _12 = const 5_i32;
++         _2 = [const 5_i32; 10];
+          StorageDead(_12);
+          StorageDead(_11);
+          StorageDead(_10);
+          StorageDead(_9);
+          StorageDead(_8);
+          StorageDead(_7);
+          StorageDead(_6);
+          StorageDead(_5);
+          StorageDead(_4);
+          StorageDead(_3);
+          _0 = const ();
+          StorageDead(_2);
+-         StorageDead(_1);
++         nop;
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/gvn.repeat.GVN.panic-unwind.diff b/tests/mir-opt/gvn.repeat.GVN.panic-unwind.diff
new file mode 100644
index 000000000000..37915f8578d2
--- /dev/null
+++ b/tests/mir-opt/gvn.repeat.GVN.panic-unwind.diff
@@ -0,0 +1,79 @@
+- // MIR for `repeat` before GVN
++ // MIR for `repeat` after GVN
+  
+  fn repeat() -> () {
+      let mut _0: ();
+      let _1: i32;
+      let mut _3: i32;
+      let mut _4: i32;
+      let mut _5: i32;
+      let mut _6: i32;
+      let mut _7: i32;
+      let mut _8: i32;
+      let mut _9: i32;
+      let mut _10: i32;
+      let mut _11: i32;
+      let mut _12: i32;
+      scope 1 {
+          debug val => _1;
+          let _2: [i32; 10];
+          scope 2 {
+              debug array => _2;
+          }
+      }
+  
+      bb0: {
+-         StorageLive(_1);
++         nop;
+          _1 = const 5_i32;
+          StorageLive(_2);
+          StorageLive(_3);
+-         _3 = _1;
++         _3 = const 5_i32;
+          StorageLive(_4);
+-         _4 = _1;
++         _4 = const 5_i32;
+          StorageLive(_5);
+-         _5 = _1;
++         _5 = const 5_i32;
+          StorageLive(_6);
+-         _6 = _1;
++         _6 = const 5_i32;
+          StorageLive(_7);
+-         _7 = _1;
++         _7 = const 5_i32;
+          StorageLive(_8);
+-         _8 = _1;
++         _8 = const 5_i32;
+          StorageLive(_9);
+-         _9 = _1;
++         _9 = const 5_i32;
+          StorageLive(_10);
+-         _10 = _1;
++         _10 = const 5_i32;
+          StorageLive(_11);
+-         _11 = _1;
++         _11 = const 5_i32;
+          StorageLive(_12);
+-         _12 = _1;
+-         _2 = [move _3, move _4, move _5, move _6, move _7, move _8, move _9, move _10, move _11, move _12];
++         _12 = const 5_i32;
++         _2 = [const 5_i32; 10];
+          StorageDead(_12);
+          StorageDead(_11);
+          StorageDead(_10);
+          StorageDead(_9);
+          StorageDead(_8);
+          StorageDead(_7);
+          StorageDead(_6);
+          StorageDead(_5);
+          StorageDead(_4);
+          StorageDead(_3);
+          _0 = const ();
+          StorageDead(_2);
+-         StorageDead(_1);
++         nop;
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/gvn.rs b/tests/mir-opt/gvn.rs
index 55de186edf16..1c14f818044f 100644
--- a/tests/mir-opt/gvn.rs
+++ b/tests/mir-opt/gvn.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // unit-test: GVN
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // only-64bit
@@ -15,48 +14,120 @@ use std::mem::transmute;
 struct S<T>(T);
 
 fn subexpression_elimination(x: u64, y: u64, mut z: u64) {
+    // CHECK-LABEL: fn subexpression_elimination(
+
+    // CHECK: [[add:_.*]] = Add(_1, _2);
+    // CHECK: opaque::<u64>([[add]])
     opaque(x + y);
+    // CHECK: [[mul:_.*]] = Mul(_1, _2);
+    // CHECK: opaque::<u64>([[mul]])
     opaque(x * y);
+    // CHECK: [[sub:_.*]] = Sub(_1, _2);
+    // CHECK: opaque::<u64>([[sub]])
     opaque(x - y);
+    // CHECK: [[div:_.*]] = Div(_1, _2);
+    // CHECK: opaque::<u64>([[div]])
     opaque(x / y);
+    // CHECK: [[rem:_.*]] = Rem(_1, _2);
+    // CHECK: opaque::<u64>([[rem]])
     opaque(x % y);
+    // CHECK: [[and:_.*]] = BitAnd(_1, _2);
+    // CHECK: opaque::<u64>([[and]])
     opaque(x & y);
+    // CHECK: [[or:_.*]] = BitOr(_1, _2);
+    // CHECK: opaque::<u64>([[or]])
     opaque(x | y);
+    // CHECK: [[xor:_.*]] = BitXor(_1, _2);
+    // CHECK: opaque::<u64>([[xor]])
     opaque(x ^ y);
+    // CHECK: [[shl:_.*]] = Shl(_1, _2);
+    // CHECK: opaque::<u64>([[shl]])
     opaque(x << y);
+    // CHECK: [[shr:_.*]] = Shr(_1, _2);
+    // CHECK: opaque::<u64>([[shr]])
     opaque(x >> y);
+    // CHECK: [[int:_.*]] = _1 as u32 (IntToInt);
+    // CHECK: opaque::<u32>([[int]])
     opaque(x as u32);
+    // CHECK: [[float:_.*]] = _1 as f32 (IntToFloat);
+    // CHECK: opaque::<f32>([[float]])
     opaque(x as f32);
+    // CHECK: [[wrap:_.*]] = S::<u64>(_1);
+    // CHECK: opaque::<S<u64>>([[wrap]])
     opaque(S(x));
+    // CHECK: opaque::<u64>(_1)
     opaque(S(x).0);
 
     // Those are duplicates to substitute somehow.
-    opaque((x + y) + z);
-    opaque((x * y) + z);
-    opaque((x - y) + z);
-    opaque((x / y) + z);
-    opaque((x % y) + z);
-    opaque((x & y) + z);
-    opaque((x | y) + z);
-    opaque((x ^ y) + z);
-    opaque((x << y) + z);
-    opaque((x >> y) + z);
+    // CHECK: opaque::<u64>([[add]])
+    opaque(x + y);
+    // CHECK: opaque::<u64>([[mul]])
+    opaque(x * y);
+    // CHECK: opaque::<u64>([[sub]])
+    opaque(x - y);
+    // CHECK: opaque::<u64>([[div]])
+    opaque(x / y);
+    // CHECK: opaque::<u64>([[rem]])
+    opaque(x % y);
+    // CHECK: opaque::<u64>([[and]])
+    opaque(x & y);
+    // CHECK: opaque::<u64>([[or]])
+    opaque(x | y);
+    // CHECK: opaque::<u64>([[xor]])
+    opaque(x ^ y);
+    // CHECK: opaque::<u64>([[shl]])
+    opaque(x << y);
+    // CHECK: opaque::<u64>([[shr]])
+    opaque(x >> y);
+    // CHECK: opaque::<u32>([[int]])
+    opaque(x as u32);
+    // CHECK: opaque::<f32>([[float]])
+    opaque(x as f32);
+    // CHECK: opaque::<S<u64>>([[wrap]])
     opaque(S(x));
+    // CHECK: opaque::<u64>(_1)
     opaque(S(x).0);
 
+    // We can substitute through a complex expression.
+    // CHECK: [[compound:_.*]] = Sub([[mul]], _2);
+    // CHECK: opaque::<u64>([[compound]])
+    // CHECK: opaque::<u64>([[compound]])
+    opaque((x * y) - y);
+    opaque((x * y) - y);
+
     // We can substitute through an immutable reference too.
+    // CHECK: [[ref:_.*]] = &_3;
+    // CHECK: [[deref:_.*]] = (*[[ref]]);
+    // CHECK: [[addref:_.*]] = Add([[deref]], _1);
+    // CHECK: opaque::<u64>([[addref]])
+    // CHECK: opaque::<u64>([[addref]])
     let a = &z;
     opaque(*a + x);
     opaque(*a + x);
 
     // But not through a mutable reference or a pointer.
+    // CHECK: [[mut:_.*]] = &mut _3;
+    // CHECK: [[addmut:_.*]] = Add(
+    // CHECK: opaque::<u64>(move [[addmut]])
+    // CHECK: [[addmut2:_.*]] = Add(
+    // CHECK: opaque::<u64>(move [[addmut2]])
     let b = &mut z;
     opaque(*b + x);
     opaque(*b + x);
     unsafe {
+        // CHECK: [[raw:_.*]] = &raw const _3;
+        // CHECK: [[addraw:_.*]] = Add(
+        // CHECK: opaque::<u64>(move [[addraw]])
+        // CHECK: [[addraw2:_.*]] = Add(
+        // CHECK: opaque::<u64>(move [[addraw2]])
         let c = &raw const z;
         opaque(*c + x);
         opaque(*c + x);
+        // CHECK: [[ptr:_.*]] = &raw mut _3;
+        // CHECK: [[addptr:_.*]] = Add(
+        // CHECK: opaque::<u64>(move [[addptr]])
+        // CHECK: [[addptr2:_.*]] = Add(
+        // CHECK: opaque::<u64>(move [[addptr2]])
         let d = &raw mut z;
         opaque(*d + x);
         opaque(*d + x);
@@ -64,12 +135,21 @@ fn subexpression_elimination(x: u64, y: u64, mut z: u64) {
 
     // We can substitute again, but not with the earlier computations.
     // Important: `e` is not `a`!
+    // CHECK: [[ref2:_.*]] = &_3;
+    // CHECK: [[deref2:_.*]] = (*[[ref2]]);
+    // CHECK: [[addref2:_.*]] = Add([[deref2]], _1);
+    // CHECK: opaque::<u64>([[addref2]])
+    // CHECK: opaque::<u64>([[addref2]])
     let e = &z;
     opaque(*e + x);
     opaque(*e + x);
 }
 
 fn wrap_unwrap<T: Copy>(x: T) -> T {
+    // CHECK-LABEL: fn wrap_unwrap(
+    // CHECK: [[some:_.*]] = Option::<T>::Some(_1);
+    // CHECK: switchInt(const 1_isize)
+    // CHECK: _0 = _1;
     match Some(x) {
         Some(y) => y,
         None => panic!(),
@@ -77,155 +157,354 @@ fn wrap_unwrap<T: Copy>(x: T) -> T {
 }
 
 fn repeated_index<T: Copy, const N: usize>(x: T, idx: usize) {
+    // CHECK-LABEL: fn repeated_index(
+    // CHECK: [[a:_.*]] = [_1; N];
     let a = [x; N];
+    // CHECK: opaque::<T>(_1)
     opaque(a[0]);
+    // CHECK: opaque::<T>(_1)
     opaque(a[idx]);
 }
 
+/// Verify symbolic integer arithmetic simplifications.
 fn arithmetic(x: u64) {
+    // CHECK-LABEL: fn arithmetic(
+    // CHECK: [[add:_.*]] = Add(_1, const 0_u64);
+    // CHECK: opaque::<u64>(move [[add]])
     opaque(x + 0);
+    // CHECK: [[sub:_.*]] = Sub(_1, const 0_u64);
+    // CHECK: opaque::<u64>(move [[sub]])
     opaque(x - 0);
+    // CHECK: [[mul0:_.*]] = Mul(_1, const 0_u64);
+    // CHECK: opaque::<u64>(move [[mul0]])
     opaque(x * 0);
+    // CHECK: [[mul1:_.*]] = Mul(_1, const 1_u64);
+    // CHECK: opaque::<u64>(move [[mul1]])
     opaque(x * 1);
+    // CHECK: [[div0:_.*]] = Div(_1, const 0_u64);
+    // CHECK: opaque::<u64>(move [[div0]])
     opaque(x / 0);
+    // CHECK: [[div1:_.*]] = Div(_1, const 1_u64);
+    // CHECK: opaque::<u64>(move [[div1]])
     opaque(x / 1);
+    // CHECK: [[zdiv:_.*]] = Div(const 0_u64, _1);
+    // CHECK: opaque::<u64>(move [[zdiv]])
     opaque(0 / x);
+    // CHECK: [[odiv:_.*]] = Div(const 1_u64, _1);
+    // CHECK: opaque::<u64>(move [[odiv]])
     opaque(1 / x);
+    // CHECK: [[rem0:_.*]] = Rem(_1, const 0_u64);
+    // CHECK: opaque::<u64>(move [[rem0]])
     opaque(x % 0);
+    // CHECK: [[rem1:_.*]] = Rem(_1, const 1_u64);
+    // CHECK: opaque::<u64>(move [[rem1]])
     opaque(x % 1);
+    // CHECK: [[zrem:_.*]] = Rem(const 0_u64, _1);
+    // CHECK: opaque::<u64>(move [[zrem]])
     opaque(0 % x);
+    // CHECK: [[orem:_.*]] = Rem(const 1_u64, _1);
+    // CHECK: opaque::<u64>(move [[orem]])
     opaque(1 % x);
+    // CHECK: [[and:_.*]] = BitAnd(_1, const 0_u64);
+    // CHECK: opaque::<u64>(move [[and]])
     opaque(x & 0);
+    // CHECK: [[or:_.*]] = BitOr(_1, const 0_u64);
+    // CHECK: opaque::<u64>(move [[or]])
     opaque(x | 0);
+    // CHECK: [[xor:_.*]] = BitXor(_1, const 0_u64);
+    // CHECK: opaque::<u64>(move [[xor]])
     opaque(x ^ 0);
+    // CHECK: [[shr:_.*]] = Shr(_1, const 0_i32);
+    // CHECK: opaque::<u64>(move [[shr]])
     opaque(x >> 0);
+    // CHECK: [[shl:_.*]] = Shl(_1, const 0_i32);
+    // CHECK: opaque::<u64>(move [[shl]])
     opaque(x << 0);
 }
 
+fn comparison(x: u64, y: u64) {
+    // CHECK-LABEL: fn comparison(
+    // CHECK: [[eqxx:_.*]] = Eq(_1, _1);
+    // CHECK: opaque::<bool>(move [[eqxx]])
+    opaque(x == x);
+    // CHECK: [[nexx:_.*]] = Ne(_1, _1);
+    // CHECK: opaque::<bool>(move [[nexx]])
+    opaque(x != x);
+    // CHECK: [[eqxy:_.*]] = Eq(_1, _2);
+    // CHECK: opaque::<bool>(move [[eqxy]])
+    opaque(x == y);
+    // CHECK: [[nexy:_.*]] = Ne(_1, _2);
+    // CHECK: opaque::<bool>(move [[nexy]])
+    opaque(x != y);
+}
+
+/// Verify symbolic integer arithmetic simplifications on checked ops.
 #[rustc_inherit_overflow_checks]
 fn arithmetic_checked(x: u64) {
+    // CHECK-LABEL: fn arithmetic_checked(
+    // CHECK: [[cadd:_.*]] = CheckedAdd(_1, const 0_u64);
+    // CHECK: [[add:_.*]] = move ([[cadd]].0: u64);
+    // CHECK: opaque::<u64>(move [[add]])
     opaque(x + 0);
+    // CHECK: [[csub:_.*]] = CheckedSub(_1, const 0_u64);
+    // CHECK: [[sub:_.*]] = move ([[csub]].0: u64);
+    // CHECK: opaque::<u64>(move [[sub]])
     opaque(x - 0);
+    // CHECK: [[cmul0:_.*]] = CheckedMul(_1, const 0_u64);
+    // CHECK: [[mul0:_.*]] = move ([[cmul0]].0: u64);
+    // CHECK: opaque::<u64>(move [[mul0]])
     opaque(x * 0);
+    // CHECK: [[cmul1:_.*]] = CheckedMul(_1, const 1_u64);
+    // CHECK: [[mul1:_.*]] = move ([[cmul1]].0: u64);
+    // CHECK: opaque::<u64>(move [[mul1]])
     opaque(x * 1);
-    opaque(x / 0);
-    opaque(x / 1);
-    opaque(0 / x);
-    opaque(1 / x);
-    opaque(x % 0);
-    opaque(x % 1);
-    opaque(0 % x);
-    opaque(1 % x);
-    opaque(x & 0);
-    opaque(x | 0);
-    opaque(x ^ 0);
-    opaque(x >> 0);
-    opaque(x << 0);
 }
 
+/// Verify that we do not apply arithmetic simplifications on floats.
 fn arithmetic_float(x: f64) {
+    // CHECK-LABEL: fn arithmetic_float(
+    // CHECK: [[add:_.*]] = Add(_1, const 0f64);
+    // CHECK: opaque::<f64>(move [[add]])
     opaque(x + 0.);
+    // CHECK: [[sub:_.*]] = Sub(_1, const 0f64);
+    // CHECK: opaque::<f64>(move [[sub]])
     opaque(x - 0.);
+    // CHECK: [[mul:_.*]] = Mul(_1, const 0f64);
+    // CHECK: opaque::<f64>(move [[mul]])
     opaque(x * 0.);
+    // CHECK: [[div0:_.*]] = Div(_1, const 0f64);
+    // CHECK: opaque::<f64>(move [[div0]])
     opaque(x / 0.);
+    // CHECK: [[zdiv:_.*]] = Div(const 0f64, _1);
+    // CHECK: opaque::<f64>(move [[zdiv]])
     opaque(0. / x);
+    // CHECK: [[rem0:_.*]] = Rem(_1, const 0f64);
+    // CHECK: opaque::<f64>(move [[rem0]])
     opaque(x % 0.);
+    // CHECK: [[zrem:_.*]] = Rem(const 0f64, _1);
+    // CHECK: opaque::<f64>(move [[zrem]])
     opaque(0. % x);
     // Those are not simplifiable to `true`/`false`, thanks to NaNs.
+    // CHECK: [[eq:_.*]] = Eq(_1, _1);
+    // CHECK: opaque::<bool>(move [[eq]])
     opaque(x == x);
+    // CHECK: [[ne:_.*]] = Ne(_1, _1);
+    // CHECK: opaque::<bool>(move [[ne]])
     opaque(x != x);
 }
 
 fn cast() {
+    // CHECK-LABEL: fn cast(
     let i = 1_i64;
     let u = 1_u64;
     let f = 1_f64;
+    // CHECK: opaque::<u8>(const 1_u8)
     opaque(i as u8);
+    // CHECK: opaque::<u16>(const 1_u16)
     opaque(i as u16);
+    // CHECK: opaque::<u32>(const 1_u32)
     opaque(i as u32);
+    // CHECK: opaque::<u64>(const 1_u64)
     opaque(i as u64);
+    // CHECK: opaque::<i8>(const 1_i8)
     opaque(i as i8);
+    // CHECK: opaque::<i16>(const 1_i16)
     opaque(i as i16);
+    // CHECK: opaque::<i32>(const 1_i32)
     opaque(i as i32);
+    // CHECK: opaque::<i64>(const 1_i64)
     opaque(i as i64);
+    // CHECK: opaque::<f32>(const 1f32)
     opaque(i as f32);
+    // CHECK: opaque::<f64>(const 1f64)
     opaque(i as f64);
+    // CHECK: opaque::<u8>(const 1_u8)
     opaque(u as u8);
+    // CHECK: opaque::<u16>(const 1_u16)
     opaque(u as u16);
+    // CHECK: opaque::<u32>(const 1_u32)
     opaque(u as u32);
+    // CHECK: opaque::<u64>(const 1_u64)
     opaque(u as u64);
+    // CHECK: opaque::<i8>(const 1_i8)
     opaque(u as i8);
+    // CHECK: opaque::<i16>(const 1_i16)
     opaque(u as i16);
+    // CHECK: opaque::<i32>(const 1_i32)
     opaque(u as i32);
+    // CHECK: opaque::<i64>(const 1_i64)
     opaque(u as i64);
+    // CHECK: opaque::<f32>(const 1f32)
     opaque(u as f32);
+    // CHECK: opaque::<f64>(const 1f64)
     opaque(u as f64);
+    // CHECK: opaque::<u8>(const 1_u8)
     opaque(f as u8);
+    // CHECK: opaque::<u16>(const 1_u16)
     opaque(f as u16);
+    // CHECK: opaque::<u32>(const 1_u32)
     opaque(f as u32);
+    // CHECK: opaque::<u64>(const 1_u64)
     opaque(f as u64);
+    // CHECK: opaque::<i8>(const 1_i8)
     opaque(f as i8);
+    // CHECK: opaque::<i16>(const 1_i16)
     opaque(f as i16);
+    // CHECK: opaque::<i32>(const 1_i32)
     opaque(f as i32);
+    // CHECK: opaque::<i64>(const 1_i64)
     opaque(f as i64);
+    // CHECK: opaque::<f32>(const 1f32)
     opaque(f as f32);
+    // CHECK: opaque::<f64>(const 1f64)
     opaque(f as f64);
 }
 
 fn multiple_branches(t: bool, x: u8, y: u8) {
+    // CHECK-LABEL: fn multiple_branches(
+    // CHECK: switchInt(_1) -> [0: [[bbf:bb.*]], otherwise: [[bbt:bb.*]]];
     if t {
-        opaque(x + y); // a
-        opaque(x + y); // should reuse a
+        // CHECK: [[bbt]]: {
+        // CHECK: [[a:_.*]] = Add(_2, _3);
+        // CHECK: opaque::<u8>([[a]])
+        // CHECK: opaque::<u8>([[a]])
+        // CHECK: goto -> [[bbc:bb.*]];
+        opaque(x + y);
+        opaque(x + y);
     } else {
-        opaque(x + y); // b
-        opaque(x + y); // shoud reuse b
+        // CHECK: [[bbf]]: {
+        // CHECK: [[b:_.*]] = Add(_2, _3);
+        // CHECK: opaque::<u8>([[b]])
+        // CHECK: opaque::<u8>([[b]])
+        // CHECK: goto -> [[bbc:bb.*]];
+        opaque(x + y);
+        opaque(x + y);
     }
-    opaque(x + y); // c
+    // Neither `a` nor `b` dominate `c`, so we cannot reuse any of them.
+    // CHECK: [[bbc]]: {
+    // CHECK: [[c:_.*]] = Add(_2, _3);
+    // CHECK: opaque::<u8>([[c]])
+    opaque(x + y);
+
+    // `c` dominates both calls, so we can reuse it.
     if t {
-        opaque(x + y); // should reuse c
+        // CHECK: opaque::<u8>([[c]])
+        opaque(x + y);
     } else {
-        opaque(x + y); // should reuse c
+        // CHECK: opaque::<u8>([[c]])
+        opaque(x + y);
     }
 }
 
+/// Verify that we do not reuse a `&raw? mut?` rvalue.
 fn references(mut x: impl Sized) {
+    // CHECK-LABEL: fn references(
+    // CHECK: [[ref1:_.*]] = &_1;
+    // CHECK: opaque::<&impl Sized>(move [[ref1]])
     opaque(&x);
-    opaque(&x); // should not reuse a
+    // CHECK: [[ref2:_.*]] = &_1;
+    // CHECK: opaque::<&impl Sized>(move [[ref2]])
+    opaque(&x);
+    // CHECK: [[ref3:_.*]] = &mut _1;
+    // CHECK: opaque::<&mut impl Sized>(move [[ref3]])
     opaque(&mut x);
-    opaque(&mut x); // should not reuse a
+    // CHECK: [[ref4:_.*]] = &mut _1;
+    // CHECK: opaque::<&mut impl Sized>(move [[ref4]])
+    opaque(&mut x);
+    // CHECK: [[ref5:_.*]] = &raw const _1;
+    // CHECK: opaque::<*const impl Sized>(move [[ref5]])
     opaque(&raw const x);
-    opaque(&raw const x); // should not reuse a
+    // CHECK: [[ref6:_.*]] = &raw const _1;
+    // CHECK: opaque::<*const impl Sized>(move [[ref6]])
+    opaque(&raw const x);
+    // CHECK: [[ref7:_.*]] = &raw mut _1;
+    // CHECK: opaque::<*mut impl Sized>(move [[ref7]])
+    opaque(&raw mut x);
+    // CHECK: [[ref8:_.*]] = &raw mut _1;
+    // CHECK: opaque::<*mut impl Sized>(move [[ref8]])
     opaque(&raw mut x);
-    opaque(&raw mut x); // should not reuse a
 
     let r = &mut x;
-    let s = S(r).0; // Obfuscate `r`.
-    opaque(&*s); // This is `&*r`.
-    opaque(&mut *s); // This is `&*r`.
-    opaque(&raw const *s); // This is `&*r`.
-    opaque(&raw mut *s); // This is `&*r`.
+    let s = S(r).0; // Obfuscate `r`. Following lines should still reborrow `r`.
+    // CHECK: [[ref9:_.*]] = &mut _1;
+    // CHECK: [[ref10:_.*]] = &(*[[ref9]]);
+    // CHECK: opaque::<&impl Sized>(move [[ref10]])
+    opaque(&*s);
+    // CHECK: [[ref11:_.*]] = &mut (*[[ref9]]);
+    // CHECK: opaque::<&mut impl Sized>(move [[ref11]])
+    opaque(&mut *s);
+    // CHECK: [[ref12:_.*]] = &raw const (*[[ref9]]);
+    // CHECK: opaque::<*const impl Sized>(move [[ref12]])
+    opaque(&raw const *s);
+    // CHECK: [[ref12:_.*]] = &raw mut (*[[ref9]]);
+    // CHECK: opaque::<*mut impl Sized>(move [[ref12]])
+    opaque(&raw mut *s);
 }
 
 fn dereferences(t: &mut u32, u: &impl Copy, s: &S<u32>) {
+    // CHECK-LABEL: fn dereferences(
+
+    // Do not reuse dereferences of `&mut`.
+    // CHECK: [[st1:_.*]] = (*_1);
+    // CHECK: opaque::<u32>(move [[st1]])
+    // CHECK: [[st2:_.*]] = (*_1);
+    // CHECK: opaque::<u32>(move [[st2]])
     opaque(*t);
-    opaque(*t); // this cannot reuse a, as x is &mut.
+    opaque(*t);
+
+    // Do not reuse dereferences of `*const`.
+    // CHECK: [[raw:_.*]] = &raw const (*_1);
+    // CHECK: [[st3:_.*]] = (*[[raw]]);
+    // CHECK: opaque::<u32>(move [[st3]])
+    // CHECK: [[st4:_.*]] = (*[[raw]]);
+    // CHECK: opaque::<u32>(move [[st4]])
     let z = &raw const *t;
     unsafe { opaque(*z) };
-    unsafe { opaque(*z) }; // this cannot reuse a, as x is *const.
+    unsafe { opaque(*z) };
+
+    // Do not reuse dereferences of `*mut`.
+    // CHECK: [[ptr:_.*]] = &raw mut (*_1);
+    // CHECK: [[st5:_.*]] = (*[[ptr]]);
+    // CHECK: opaque::<u32>(move [[st5]])
+    // CHECK: [[st6:_.*]] = (*[[ptr]]);
+    // CHECK: opaque::<u32>(move [[st6]])
     let z = &raw mut *t;
     unsafe { opaque(*z) };
-    unsafe { opaque(*z) }; // this cannot reuse a, as x is *mut.
+    unsafe { opaque(*z) };
+
+    // We can reuse dereferences of `&Freeze`.
+    // CHECK: [[ref:_.*]] = &(*_1);
+    // CHECK: [[st7:_.*]] = (*[[ref]]);
+    // CHECK: opaque::<u32>([[st7]])
+    // CHECK: opaque::<u32>([[st7]])
     let z = &*t;
     opaque(*z);
-    opaque(*z); // this can reuse, as `z` is immutable ref, Freeze and Copy.
-    opaque(&*z); // but not for a reborrow.
+    opaque(*z);
+    // But not in reborrows.
+    // CHECK: [[reborrow:_.*]] = &(*[[ref]]);
+    // CHECK: opaque::<&u32>(move [[reborrow]])
+    opaque(&*z);
+
+    // `*u` is not Freeze, so we cannot reuse.
+    // CHECK: [[st8:_.*]] = (*_2);
+    // CHECK: opaque::<impl Copy>(move [[st8]])
+    // CHECK: [[st9:_.*]] = (*_2);
+    // CHECK: opaque::<impl Copy>(move [[st9]])
     opaque(*u);
-    opaque(*u); // this cannot reuse, as `z` is not Freeze.
+    opaque(*u);
+
+    // `*s` is not Copy, by `(*s).0` is, so we can reuse.
+    // CHECK: [[st10:_.*]] = ((*_3).0: u32);
+    // CHECK: opaque::<u32>([[st10]])
+    // CHECK: opaque::<u32>([[st10]])
+    opaque(s.0);
     opaque(s.0);
-    opaque(s.0); // *s is not Copy, by (*s).0 is, so we can reuse.
 }
 
 fn slices() {
+    // CHECK-LABEL: fn slices(
+    // CHECK: {{_.*}} = const "
+    // CHECK-NOT: {{_.*}} = const "
     let s = "my favourite slice"; // This is a `Const::Slice` in MIR.
     opaque(s);
     let t = s; // This should be the same pointer, so cannot be a `Const::Slice`.
@@ -238,6 +517,7 @@ fn slices() {
 
 #[custom_mir(dialect = "analysis")]
 fn duplicate_slice() -> (bool, bool) {
+    // CHECK-LABEL: fn duplicate_slice(
     mir!(
         let au: u128;
         let bu: u128;
@@ -246,41 +526,50 @@ fn duplicate_slice() -> (bool, bool) {
         let c: &str;
         let d: &str;
         {
+            // CHECK: [[a:_.*]] = (const "a",);
+            // CHECK: [[au:_.*]] = ([[a]].0: &str) as u128 (Transmute);
             let a = ("a",);
             Call(au = transmute::<_, u128>(a.0), bb1)
         }
         bb1 = {
+            // CHECK: [[c:_.*]] = identity::<&str>(([[a]].0: &str))
             Call(c = identity(a.0), bb2)
         }
         bb2 = {
+            // CHECK: [[cu:_.*]] = [[c]] as u128 (Transmute);
             Call(cu = transmute::<_, u128>(c), bb3)
         }
         bb3 = {
-            let b = "a"; // This slice is different from `a.0`.
-            Call(bu = transmute::<_, u128>(b), bb4) // Hence `bu` is not `au`.
+            // This slice is different from `a.0`. Hence `bu` is not `au`.
+            // CHECK: [[b:_.*]] = const "a";
+            // CHECK: [[bu:_.*]] = [[b]] as u128 (Transmute);
+            let b = "a";
+            Call(bu = transmute::<_, u128>(b), bb4)
         }
         bb4 = {
-            Call(d = identity(b), bb5) // This returns a copy of `b`, which is not `a`.
+            // This returns a copy of `b`, which is not `a`.
+            // CHECK: [[d:_.*]] = identity::<&str>([[b]])
+            Call(d = identity(b), bb5)
         }
         bb5 = {
+            // CHECK: [[du:_.*]] = [[d]] as u128 (Transmute);
             Call(du = transmute::<_, u128>(d), bb6)
         }
         bb6 = {
-            let direct = au == bu; // Must not fold to `true`...
-            let indirect = cu == du; // ...as this will not.
+            // `direct` must not fold to `true`, as `indirect` will not.
+            // CHECK: = Eq([[au]], [[bu]]);
+            // CHECK: = Eq([[cu]], [[du]]);
+            let direct = au == bu;
+            let indirect = cu == du;
             RET = (direct, indirect);
             Return()
         }
     )
 }
 
-fn aggregates() {
-    let a_array: S<[u8; 0]> = S([]);
-    let b_array: S<[u16; 0]> = S([]); // This must not be merged with `a_array`.
-
-    let a_tuple: S<()> = S(());
-    let b_tuple: S<()> = S(()); // But this can be with `a_tuple`.
-
+fn repeat() {
+    // CHECK-LABEL: fn repeat(
+    // CHECK: = [const 5_i32; 10];
     let val = 5;
     let array = [val, val, val, val, val, val, val, val, val, val];
 }
@@ -290,6 +579,7 @@ fn main() {
     wrap_unwrap(5);
     repeated_index::<u32, 7>(5, 3);
     arithmetic(5);
+    comparison(5, 6);
     arithmetic_checked(5);
     arithmetic_float(5.);
     cast();
@@ -299,7 +589,7 @@ fn main() {
     slices();
     let (direct, indirect) = duplicate_slice();
     assert_eq!(direct, indirect);
-    aggregates();
+    repeat();
 }
 
 #[inline(never)]
@@ -314,6 +604,7 @@ fn identity<T>(x: T) -> T {
 // EMIT_MIR gvn.wrap_unwrap.GVN.diff
 // EMIT_MIR gvn.repeated_index.GVN.diff
 // EMIT_MIR gvn.arithmetic.GVN.diff
+// EMIT_MIR gvn.comparison.GVN.diff
 // EMIT_MIR gvn.arithmetic_checked.GVN.diff
 // EMIT_MIR gvn.arithmetic_float.GVN.diff
 // EMIT_MIR gvn.cast.GVN.diff
@@ -322,4 +613,4 @@ fn identity<T>(x: T) -> T {
 // EMIT_MIR gvn.dereferences.GVN.diff
 // EMIT_MIR gvn.slices.GVN.diff
 // EMIT_MIR gvn.duplicate_slice.GVN.diff
-// EMIT_MIR gvn.aggregates.GVN.diff
+// EMIT_MIR gvn.repeat.GVN.diff
diff --git a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff
index 23ba2559448b..0a747d3aef0f 100644
--- a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff
+++ b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-abort.diff
@@ -65,11 +65,11 @@
       let mut _60: u64;
       let mut _61: u64;
       let mut _62: u64;
-      let mut _63: u64;
+      let _63: ();
       let mut _64: u64;
-      let _65: ();
+      let mut _65: u64;
       let mut _66: u64;
-      let mut _67: u64;
+      let _67: ();
       let mut _68: u64;
       let mut _69: u64;
       let mut _70: u64;
@@ -77,25 +77,25 @@
       let mut _72: u64;
       let mut _73: u64;
       let mut _74: u64;
-      let mut _75: u64;
-      let mut _76: u64;
-      let _77: ();
+      let mut _75: bool;
+      let _76: ();
+      let mut _77: u64;
       let mut _78: u64;
       let mut _79: u64;
-      let mut _80: u64;
-      let mut _81: u64;
-      let mut _82: bool;
+      let mut _80: bool;
+      let _81: ();
+      let mut _82: u64;
       let mut _83: u64;
-      let _84: ();
-      let mut _85: u64;
+      let mut _84: u64;
+      let _85: ();
       let mut _86: u64;
       let mut _87: u64;
       let mut _88: u64;
-      let mut _89: bool;
+      let _89: ();
       let mut _90: u64;
-      let _91: ();
+      let mut _91: u64;
       let mut _92: u64;
-      let mut _93: u64;
+      let _93: ();
       let mut _94: u64;
       let mut _95: u64;
       let mut _96: u64;
@@ -103,93 +103,91 @@
       let mut _98: u64;
       let mut _99: u64;
       let mut _100: u64;
-      let mut _101: u64;
-      let mut _102: u64;
-      let _103: ();
-      let mut _104: u64;
-      let mut _105: u64;
+      let _101: ();
+      let mut _102: u32;
+      let mut _103: u64;
+      let _104: ();
+      let mut _105: f32;
       let mut _106: u64;
-      let mut _107: u64;
-      let mut _108: u64;
-      let _109: ();
-      let mut _110: u64;
+      let _107: ();
+      let mut _108: S<u64>;
+      let mut _109: u64;
+      let _110: ();
       let mut _111: u64;
-      let mut _112: u64;
+      let mut _112: S<u64>;
       let mut _113: u64;
-      let mut _114: u64;
-      let _115: ();
+      let _114: ();
+      let mut _115: u64;
       let mut _116: u64;
       let mut _117: u64;
       let mut _118: u64;
       let mut _119: u64;
-      let mut _120: u64;
-      let _121: ();
-      let mut _122: S<u64>;
+      let _120: ();
+      let mut _121: u64;
+      let mut _122: u64;
       let mut _123: u64;
-      let _124: ();
+      let mut _124: u64;
       let mut _125: u64;
-      let mut _126: S<u64>;
-      let mut _127: u64;
-      let _128: &u64;
-      let _129: ();
+      let _126: &u64;
+      let _127: ();
+      let mut _128: u64;
+      let mut _129: u64;
       let mut _130: u64;
-      let mut _131: u64;
+      let _131: ();
       let mut _132: u64;
-      let _133: ();
+      let mut _133: u64;
       let mut _134: u64;
-      let mut _135: u64;
-      let mut _136: u64;
-      let _138: ();
+      let _136: ();
+      let mut _137: u64;
+      let mut _138: u64;
       let mut _139: u64;
-      let mut _140: u64;
+      let _140: ();
       let mut _141: u64;
-      let _142: ();
+      let mut _142: u64;
       let mut _143: u64;
-      let mut _144: u64;
-      let mut _145: u64;
+      let _144: ();
       let _146: ();
-      let _148: ();
+      let mut _147: u64;
+      let mut _148: u64;
       let mut _149: u64;
-      let mut _150: u64;
+      let _150: ();
       let mut _151: u64;
-      let _152: ();
+      let mut _152: u64;
       let mut _153: u64;
-      let mut _154: u64;
-      let mut _155: u64;
-      let _157: ();
+      let _155: ();
+      let mut _156: u64;
+      let mut _157: u64;
       let mut _158: u64;
-      let mut _159: u64;
+      let _159: ();
       let mut _160: u64;
-      let _161: ();
+      let mut _161: u64;
       let mut _162: u64;
-      let mut _163: u64;
-      let mut _164: u64;
-      let _166: ();
+      let _164: ();
+      let mut _165: u64;
+      let mut _166: u64;
       let mut _167: u64;
-      let mut _168: u64;
+      let _168: ();
       let mut _169: u64;
-      let _170: ();
+      let mut _170: u64;
       let mut _171: u64;
-      let mut _172: u64;
-      let mut _173: u64;
       scope 1 {
-          debug a => _128;
-          let _137: &mut u64;
+          debug a => _126;
+          let _135: &mut u64;
           scope 2 {
-              debug b => _137;
-              let _165: &u64;
+              debug b => _135;
+              let _163: &u64;
               scope 3 {
-                  let _147: *const u64;
+                  let _145: *const u64;
                   scope 4 {
-                      debug c => _147;
-                      let _156: *mut u64;
+                      debug c => _145;
+                      let _154: *mut u64;
                       scope 5 {
-                          debug d => _156;
+                          debug d => _154;
                       }
                   }
               }
               scope 6 {
-                  debug e => _165;
+                  debug e => _163;
               }
           }
       }
@@ -400,30 +398,36 @@
 +         nop;
           StorageDead(_42);
           StorageLive(_46);
-          StorageLive(_47);
+-         StorageLive(_47);
++         nop;
           StorageLive(_48);
           _48 = _1;
 -         _47 = move _48 as u32 (IntToInt);
 +         _47 = _1 as u32 (IntToInt);
           StorageDead(_48);
-          _46 = opaque::<u32>(move _47) -> [return: bb13, unwind unreachable];
+-         _46 = opaque::<u32>(move _47) -> [return: bb13, unwind unreachable];
++         _46 = opaque::<u32>(_47) -> [return: bb13, unwind unreachable];
       }
   
       bb13: {
-          StorageDead(_47);
+-         StorageDead(_47);
++         nop;
           StorageDead(_46);
           StorageLive(_49);
-          StorageLive(_50);
+-         StorageLive(_50);
++         nop;
           StorageLive(_51);
           _51 = _1;
 -         _50 = move _51 as f32 (IntToFloat);
 +         _50 = _1 as f32 (IntToFloat);
           StorageDead(_51);
-          _49 = opaque::<f32>(move _50) -> [return: bb14, unwind unreachable];
+-         _49 = opaque::<f32>(move _50) -> [return: bb14, unwind unreachable];
++         _49 = opaque::<f32>(_50) -> [return: bb14, unwind unreachable];
       }
   
       bb14: {
-          StorageDead(_50);
+-         StorageDead(_50);
++         nop;
           StorageDead(_49);
           StorageLive(_52);
 -         StorageLive(_53);
@@ -462,479 +466,497 @@
           StorageLive(_59);
           StorageLive(_60);
           StorageLive(_61);
+          _61 = _1;
           StorageLive(_62);
-          _62 = _1;
-          StorageLive(_63);
-          _63 = _2;
--         _61 = Add(move _62, move _63);
-+         _61 = _5;
-          StorageDead(_63);
+          _62 = _2;
+-         _60 = Add(move _61, move _62);
++         _60 = _5;
           StorageDead(_62);
-          StorageLive(_64);
-          _64 = _3;
--         _60 = Add(move _61, move _64);
-+         _60 = Add(_5, move _64);
-          StorageDead(_64);
           StorageDead(_61);
-          _59 = opaque::<u64>(move _60) -> [return: bb17, unwind unreachable];
+-         _59 = opaque::<u64>(move _60) -> [return: bb17, unwind unreachable];
++         _59 = opaque::<u64>(_5) -> [return: bb17, unwind unreachable];
       }
   
       bb17: {
           StorageDead(_60);
           StorageDead(_59);
+          StorageLive(_63);
+          StorageLive(_64);
           StorageLive(_65);
+          _65 = _1;
           StorageLive(_66);
-          StorageLive(_67);
-          StorageLive(_68);
-          _68 = _1;
-          StorageLive(_69);
-          _69 = _2;
--         _67 = Mul(move _68, move _69);
-+         _67 = _9;
-          StorageDead(_69);
-          StorageDead(_68);
-          StorageLive(_70);
-          _70 = _3;
--         _66 = Add(move _67, move _70);
-+         _66 = Add(_9, move _70);
-          StorageDead(_70);
-          StorageDead(_67);
-          _65 = opaque::<u64>(move _66) -> [return: bb18, unwind unreachable];
+          _66 = _2;
+-         _64 = Mul(move _65, move _66);
++         _64 = _9;
+          StorageDead(_66);
+          StorageDead(_65);
+-         _63 = opaque::<u64>(move _64) -> [return: bb18, unwind unreachable];
++         _63 = opaque::<u64>(_9) -> [return: bb18, unwind unreachable];
       }
   
       bb18: {
-          StorageDead(_66);
-          StorageDead(_65);
-          StorageLive(_71);
-          StorageLive(_72);
-          StorageLive(_73);
-          StorageLive(_74);
-          _74 = _1;
-          StorageLive(_75);
-          _75 = _2;
--         _73 = Sub(move _74, move _75);
-+         _73 = _13;
-          StorageDead(_75);
-          StorageDead(_74);
-          StorageLive(_76);
-          _76 = _3;
--         _72 = Add(move _73, move _76);
-+         _72 = Add(_13, move _76);
-          StorageDead(_76);
-          StorageDead(_73);
-          _71 = opaque::<u64>(move _72) -> [return: bb19, unwind unreachable];
+          StorageDead(_64);
+          StorageDead(_63);
+          StorageLive(_67);
+          StorageLive(_68);
+          StorageLive(_69);
+          _69 = _1;
+          StorageLive(_70);
+          _70 = _2;
+-         _68 = Sub(move _69, move _70);
++         _68 = _13;
+          StorageDead(_70);
+          StorageDead(_69);
+-         _67 = opaque::<u64>(move _68) -> [return: bb19, unwind unreachable];
++         _67 = opaque::<u64>(_13) -> [return: bb19, unwind unreachable];
       }
   
       bb19: {
-          StorageDead(_72);
-          StorageDead(_71);
-          StorageLive(_77);
-          StorageLive(_78);
-          StorageLive(_79);
-          StorageLive(_80);
-          _80 = _1;
-          StorageLive(_81);
-          _81 = _2;
--         _82 = Eq(_81, const 0_u64);
--         assert(!move _82, "attempt to divide `{}` by zero", _80) -> [success: bb20, unwind unreachable];
-+         _82 = _20;
+          StorageDead(_68);
+          StorageDead(_67);
+          StorageLive(_71);
+          StorageLive(_72);
+          StorageLive(_73);
+          _73 = _1;
+          StorageLive(_74);
+          _74 = _2;
+-         _75 = Eq(_74, const 0_u64);
+-         assert(!move _75, "attempt to divide `{}` by zero", _73) -> [success: bb20, unwind unreachable];
++         _75 = _20;
 +         assert(!_20, "attempt to divide `{}` by zero", _1) -> [success: bb20, unwind unreachable];
       }
   
       bb20: {
--         _79 = Div(move _80, move _81);
-+         _79 = _17;
-          StorageDead(_81);
-          StorageDead(_80);
-          StorageLive(_83);
-          _83 = _3;
--         _78 = Add(move _79, move _83);
-+         _78 = Add(_17, move _83);
-          StorageDead(_83);
-          StorageDead(_79);
-          _77 = opaque::<u64>(move _78) -> [return: bb21, unwind unreachable];
+-         _72 = Div(move _73, move _74);
++         _72 = _17;
+          StorageDead(_74);
+          StorageDead(_73);
+-         _71 = opaque::<u64>(move _72) -> [return: bb21, unwind unreachable];
++         _71 = opaque::<u64>(_17) -> [return: bb21, unwind unreachable];
       }
   
       bb21: {
+          StorageDead(_72);
+          StorageDead(_71);
+          StorageLive(_76);
+          StorageLive(_77);
+          StorageLive(_78);
+          _78 = _1;
+          StorageLive(_79);
+          _79 = _2;
+-         _80 = Eq(_79, const 0_u64);
+-         assert(!move _80, "attempt to calculate the remainder of `{}` with a divisor of zero", _78) -> [success: bb22, unwind unreachable];
++         _80 = _20;
++         assert(!_20, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb22, unwind unreachable];
+      }
+  
+      bb22: {
+-         _77 = Rem(move _78, move _79);
++         _77 = _22;
+          StorageDead(_79);
           StorageDead(_78);
+-         _76 = opaque::<u64>(move _77) -> [return: bb23, unwind unreachable];
++         _76 = opaque::<u64>(_22) -> [return: bb23, unwind unreachable];
+      }
+  
+      bb23: {
           StorageDead(_77);
+          StorageDead(_76);
+          StorageLive(_81);
+          StorageLive(_82);
+          StorageLive(_83);
+          _83 = _1;
           StorageLive(_84);
+          _84 = _2;
+-         _82 = BitAnd(move _83, move _84);
++         _82 = _27;
+          StorageDead(_84);
+          StorageDead(_83);
+-         _81 = opaque::<u64>(move _82) -> [return: bb24, unwind unreachable];
++         _81 = opaque::<u64>(_27) -> [return: bb24, unwind unreachable];
+      }
+  
+      bb24: {
+          StorageDead(_82);
+          StorageDead(_81);
           StorageLive(_85);
           StorageLive(_86);
           StorageLive(_87);
           _87 = _1;
           StorageLive(_88);
           _88 = _2;
--         _89 = Eq(_88, const 0_u64);
--         assert(!move _89, "attempt to calculate the remainder of `{}` with a divisor of zero", _87) -> [success: bb22, unwind unreachable];
-+         _89 = _20;
-+         assert(!_20, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb22, unwind unreachable];
-      }
-  
-      bb22: {
--         _86 = Rem(move _87, move _88);
-+         _86 = _22;
+-         _86 = BitOr(move _87, move _88);
++         _86 = _31;
           StorageDead(_88);
           StorageDead(_87);
-          StorageLive(_90);
-          _90 = _3;
--         _85 = Add(move _86, move _90);
-+         _85 = Add(_22, move _90);
-          StorageDead(_90);
-          StorageDead(_86);
-          _84 = opaque::<u64>(move _85) -> [return: bb23, unwind unreachable];
-      }
-  
-      bb23: {
-          StorageDead(_85);
-          StorageDead(_84);
-          StorageLive(_91);
-          StorageLive(_92);
-          StorageLive(_93);
-          StorageLive(_94);
-          _94 = _1;
-          StorageLive(_95);
-          _95 = _2;
--         _93 = BitAnd(move _94, move _95);
-+         _93 = _27;
-          StorageDead(_95);
-          StorageDead(_94);
-          StorageLive(_96);
-          _96 = _3;
--         _92 = Add(move _93, move _96);
-+         _92 = Add(_27, move _96);
-          StorageDead(_96);
-          StorageDead(_93);
-          _91 = opaque::<u64>(move _92) -> [return: bb24, unwind unreachable];
-      }
-  
-      bb24: {
-          StorageDead(_92);
-          StorageDead(_91);
-          StorageLive(_97);
-          StorageLive(_98);
-          StorageLive(_99);
-          StorageLive(_100);
-          _100 = _1;
-          StorageLive(_101);
-          _101 = _2;
--         _99 = BitOr(move _100, move _101);
-+         _99 = _31;
-          StorageDead(_101);
-          StorageDead(_100);
-          StorageLive(_102);
-          _102 = _3;
--         _98 = Add(move _99, move _102);
-+         _98 = Add(_31, move _102);
-          StorageDead(_102);
-          StorageDead(_99);
-          _97 = opaque::<u64>(move _98) -> [return: bb25, unwind unreachable];
+-         _85 = opaque::<u64>(move _86) -> [return: bb25, unwind unreachable];
++         _85 = opaque::<u64>(_31) -> [return: bb25, unwind unreachable];
       }
   
       bb25: {
+          StorageDead(_86);
+          StorageDead(_85);
+          StorageLive(_89);
+          StorageLive(_90);
+          StorageLive(_91);
+          _91 = _1;
+          StorageLive(_92);
+          _92 = _2;
+-         _90 = BitXor(move _91, move _92);
++         _90 = _35;
+          StorageDead(_92);
+          StorageDead(_91);
+-         _89 = opaque::<u64>(move _90) -> [return: bb26, unwind unreachable];
++         _89 = opaque::<u64>(_35) -> [return: bb26, unwind unreachable];
+      }
+  
+      bb26: {
+          StorageDead(_90);
+          StorageDead(_89);
+          StorageLive(_93);
+          StorageLive(_94);
+          StorageLive(_95);
+          _95 = _1;
+          StorageLive(_96);
+          _96 = _2;
+-         _94 = Shl(move _95, move _96);
++         _94 = _39;
+          StorageDead(_96);
+          StorageDead(_95);
+-         _93 = opaque::<u64>(move _94) -> [return: bb27, unwind unreachable];
++         _93 = opaque::<u64>(_39) -> [return: bb27, unwind unreachable];
+      }
+  
+      bb27: {
+          StorageDead(_94);
+          StorageDead(_93);
+          StorageLive(_97);
+          StorageLive(_98);
+          StorageLive(_99);
+          _99 = _1;
+          StorageLive(_100);
+          _100 = _2;
+-         _98 = Shr(move _99, move _100);
++         _98 = _43;
+          StorageDead(_100);
+          StorageDead(_99);
+-         _97 = opaque::<u64>(move _98) -> [return: bb28, unwind unreachable];
++         _97 = opaque::<u64>(_43) -> [return: bb28, unwind unreachable];
+      }
+  
+      bb28: {
           StorageDead(_98);
           StorageDead(_97);
+          StorageLive(_101);
+          StorageLive(_102);
           StorageLive(_103);
+          _103 = _1;
+-         _102 = move _103 as u32 (IntToInt);
++         _102 = _47;
+          StorageDead(_103);
+-         _101 = opaque::<u32>(move _102) -> [return: bb29, unwind unreachable];
++         _101 = opaque::<u32>(_47) -> [return: bb29, unwind unreachable];
+      }
+  
+      bb29: {
+          StorageDead(_102);
+          StorageDead(_101);
           StorageLive(_104);
           StorageLive(_105);
           StorageLive(_106);
           _106 = _1;
-          StorageLive(_107);
-          _107 = _2;
--         _105 = BitXor(move _106, move _107);
-+         _105 = _35;
-          StorageDead(_107);
+-         _105 = move _106 as f32 (IntToFloat);
++         _105 = _50;
           StorageDead(_106);
-          StorageLive(_108);
-          _108 = _3;
--         _104 = Add(move _105, move _108);
-+         _104 = Add(_35, move _108);
-          StorageDead(_108);
-          StorageDead(_105);
-          _103 = opaque::<u64>(move _104) -> [return: bb26, unwind unreachable];
+-         _104 = opaque::<f32>(move _105) -> [return: bb30, unwind unreachable];
++         _104 = opaque::<f32>(_50) -> [return: bb30, unwind unreachable];
       }
   
-      bb26: {
+      bb30: {
+          StorageDead(_105);
           StorageDead(_104);
-          StorageDead(_103);
+          StorageLive(_107);
+          StorageLive(_108);
           StorageLive(_109);
+          _109 = _1;
+-         _108 = S::<u64>(move _109);
++         _108 = _53;
+          StorageDead(_109);
+-         _107 = opaque::<S<u64>>(move _108) -> [return: bb31, unwind unreachable];
++         _107 = opaque::<S<u64>>(_53) -> [return: bb31, unwind unreachable];
+      }
+  
+      bb31: {
+          StorageDead(_108);
+          StorageDead(_107);
           StorageLive(_110);
           StorageLive(_111);
           StorageLive(_112);
-          _112 = _1;
           StorageLive(_113);
-          _113 = _2;
--         _111 = Shl(move _112, move _113);
-+         _111 = _39;
+          _113 = _1;
+-         _112 = S::<u64>(move _113);
++         _112 = _53;
           StorageDead(_113);
-          StorageDead(_112);
-          StorageLive(_114);
-          _114 = _3;
--         _110 = Add(move _111, move _114);
-+         _110 = Add(_39, move _114);
-          StorageDead(_114);
-          StorageDead(_111);
-          _109 = opaque::<u64>(move _110) -> [return: bb27, unwind unreachable];
+-         _111 = (_112.0: u64);
+-         _110 = opaque::<u64>(move _111) -> [return: bb32, unwind unreachable];
++         _111 = _1;
++         _110 = opaque::<u64>(_1) -> [return: bb32, unwind unreachable];
       }
   
-      bb27: {
+      bb32: {
+          StorageDead(_111);
+          StorageDead(_112);
           StorageDead(_110);
-          StorageDead(_109);
-          StorageLive(_115);
+          StorageLive(_114);
+-         StorageLive(_115);
++         nop;
           StorageLive(_116);
           StorageLive(_117);
+          _117 = _1;
           StorageLive(_118);
-          _118 = _1;
+          _118 = _2;
+-         _116 = Mul(move _117, move _118);
++         _116 = _9;
+          StorageDead(_118);
+          StorageDead(_117);
           StorageLive(_119);
           _119 = _2;
--         _117 = Shr(move _118, move _119);
-+         _117 = _43;
+-         _115 = Sub(move _116, move _119);
++         _115 = Sub(_9, _2);
           StorageDead(_119);
-          StorageDead(_118);
-          StorageLive(_120);
-          _120 = _3;
--         _116 = Add(move _117, move _120);
-+         _116 = Add(_43, move _120);
-          StorageDead(_120);
-          StorageDead(_117);
-          _115 = opaque::<u64>(move _116) -> [return: bb28, unwind unreachable];
+          StorageDead(_116);
+-         _114 = opaque::<u64>(move _115) -> [return: bb33, unwind unreachable];
++         _114 = opaque::<u64>(_115) -> [return: bb33, unwind unreachable];
       }
   
-      bb28: {
-          StorageDead(_116);
-          StorageDead(_115);
+      bb33: {
+-         StorageDead(_115);
++         nop;
+          StorageDead(_114);
+          StorageLive(_120);
           StorageLive(_121);
           StorageLive(_122);
           StorageLive(_123);
           _123 = _1;
--         _122 = S::<u64>(move _123);
-+         _122 = _53;
-          StorageDead(_123);
--         _121 = opaque::<S<u64>>(move _122) -> [return: bb29, unwind unreachable];
-+         _121 = opaque::<S<u64>>(_53) -> [return: bb29, unwind unreachable];
-      }
-  
-      bb29: {
-          StorageDead(_122);
-          StorageDead(_121);
           StorageLive(_124);
-          StorageLive(_125);
-          StorageLive(_126);
-          StorageLive(_127);
-          _127 = _1;
--         _126 = S::<u64>(move _127);
-+         _126 = _53;
-          StorageDead(_127);
--         _125 = (_126.0: u64);
--         _124 = opaque::<u64>(move _125) -> [return: bb30, unwind unreachable];
-+         _125 = _1;
-+         _124 = opaque::<u64>(_1) -> [return: bb30, unwind unreachable];
-      }
-  
-      bb30: {
-          StorageDead(_125);
-          StorageDead(_126);
+          _124 = _2;
+-         _122 = Mul(move _123, move _124);
++         _122 = _9;
           StorageDead(_124);
--         StorageLive(_128);
-+         nop;
-          _128 = &_3;
-          StorageLive(_129);
--         StorageLive(_130);
--         StorageLive(_131);
-+         nop;
-+         nop;
-          _131 = (*_128);
-          StorageLive(_132);
-          _132 = _1;
--         _130 = Add(move _131, move _132);
-+         _130 = Add(_131, _1);
-          StorageDead(_132);
--         StorageDead(_131);
--         _129 = opaque::<u64>(move _130) -> [return: bb31, unwind unreachable];
-+         nop;
-+         _129 = opaque::<u64>(_130) -> [return: bb31, unwind unreachable];
-      }
-  
-      bb31: {
--         StorageDead(_130);
-+         nop;
-          StorageDead(_129);
-          StorageLive(_133);
-          StorageLive(_134);
-          StorageLive(_135);
--         _135 = (*_128);
-+         _135 = _131;
-          StorageLive(_136);
-          _136 = _1;
--         _134 = Add(move _135, move _136);
-+         _134 = _130;
-          StorageDead(_136);
-          StorageDead(_135);
--         _133 = opaque::<u64>(move _134) -> [return: bb32, unwind unreachable];
-+         _133 = opaque::<u64>(_130) -> [return: bb32, unwind unreachable];
-      }
-  
-      bb32: {
-          StorageDead(_134);
-          StorageDead(_133);
--         StorageLive(_137);
-+         nop;
-          _137 = &mut _3;
-          StorageLive(_138);
-          StorageLive(_139);
-          StorageLive(_140);
-          _140 = (*_137);
-          StorageLive(_141);
-          _141 = _1;
--         _139 = Add(move _140, move _141);
-+         _139 = Add(move _140, _1);
-          StorageDead(_141);
-          StorageDead(_140);
-          _138 = opaque::<u64>(move _139) -> [return: bb33, unwind unreachable];
-      }
-  
-      bb33: {
-          StorageDead(_139);
-          StorageDead(_138);
-          StorageLive(_142);
-          StorageLive(_143);
-          StorageLive(_144);
-          _144 = (*_137);
-          StorageLive(_145);
-          _145 = _1;
--         _143 = Add(move _144, move _145);
-+         _143 = Add(move _144, _1);
-          StorageDead(_145);
-          StorageDead(_144);
-          _142 = opaque::<u64>(move _143) -> [return: bb34, unwind unreachable];
+          StorageDead(_123);
+          StorageLive(_125);
+          _125 = _2;
+-         _121 = Sub(move _122, move _125);
++         _121 = _115;
+          StorageDead(_125);
+          StorageDead(_122);
+-         _120 = opaque::<u64>(move _121) -> [return: bb34, unwind unreachable];
++         _120 = opaque::<u64>(_115) -> [return: bb34, unwind unreachable];
       }
   
       bb34: {
-          StorageDead(_143);
-          StorageDead(_142);
-          StorageLive(_146);
--         StorageLive(_147);
+          StorageDead(_121);
+          StorageDead(_120);
+-         StorageLive(_126);
 +         nop;
-          _147 = &raw const _3;
-          StorageLive(_148);
-          StorageLive(_149);
-          StorageLive(_150);
-          _150 = (*_147);
-          StorageLive(_151);
-          _151 = _1;
--         _149 = Add(move _150, move _151);
-+         _149 = Add(move _150, _1);
-          StorageDead(_151);
-          StorageDead(_150);
-          _148 = opaque::<u64>(move _149) -> [return: bb35, unwind unreachable];
+          _126 = &_3;
+          StorageLive(_127);
+-         StorageLive(_128);
+-         StorageLive(_129);
++         nop;
++         nop;
+          _129 = (*_126);
+          StorageLive(_130);
+          _130 = _1;
+-         _128 = Add(move _129, move _130);
++         _128 = Add(_129, _1);
+          StorageDead(_130);
+-         StorageDead(_129);
+-         _127 = opaque::<u64>(move _128) -> [return: bb35, unwind unreachable];
++         nop;
++         _127 = opaque::<u64>(_128) -> [return: bb35, unwind unreachable];
       }
   
       bb35: {
-          StorageDead(_149);
-          StorageDead(_148);
-          StorageLive(_152);
-          StorageLive(_153);
-          StorageLive(_154);
-          _154 = (*_147);
-          StorageLive(_155);
-          _155 = _1;
--         _153 = Add(move _154, move _155);
-+         _153 = Add(move _154, _1);
-          StorageDead(_155);
-          StorageDead(_154);
-          _152 = opaque::<u64>(move _153) -> [return: bb36, unwind unreachable];
+-         StorageDead(_128);
++         nop;
+          StorageDead(_127);
+          StorageLive(_131);
+          StorageLive(_132);
+          StorageLive(_133);
+-         _133 = (*_126);
++         _133 = _129;
+          StorageLive(_134);
+          _134 = _1;
+-         _132 = Add(move _133, move _134);
++         _132 = _128;
+          StorageDead(_134);
+          StorageDead(_133);
+-         _131 = opaque::<u64>(move _132) -> [return: bb36, unwind unreachable];
++         _131 = opaque::<u64>(_128) -> [return: bb36, unwind unreachable];
       }
   
       bb36: {
-          StorageDead(_153);
-          StorageDead(_152);
--         StorageLive(_156);
+          StorageDead(_132);
+          StorageDead(_131);
+-         StorageLive(_135);
 +         nop;
-          _156 = &raw mut _3;
-          StorageLive(_157);
-          StorageLive(_158);
-          StorageLive(_159);
-          _159 = (*_156);
-          StorageLive(_160);
-          _160 = _1;
--         _158 = Add(move _159, move _160);
-+         _158 = Add(move _159, _1);
-          StorageDead(_160);
-          StorageDead(_159);
-          _157 = opaque::<u64>(move _158) -> [return: bb37, unwind unreachable];
+          _135 = &mut _3;
+          StorageLive(_136);
+          StorageLive(_137);
+          StorageLive(_138);
+          _138 = (*_135);
+          StorageLive(_139);
+          _139 = _1;
+-         _137 = Add(move _138, move _139);
++         _137 = Add(move _138, _1);
+          StorageDead(_139);
+          StorageDead(_138);
+          _136 = opaque::<u64>(move _137) -> [return: bb37, unwind unreachable];
       }
   
       bb37: {
-          StorageDead(_158);
-          StorageDead(_157);
-          StorageLive(_161);
-          StorageLive(_162);
-          StorageLive(_163);
-          _163 = (*_156);
-          StorageLive(_164);
-          _164 = _1;
--         _162 = Add(move _163, move _164);
-+         _162 = Add(move _163, _1);
-          StorageDead(_164);
-          StorageDead(_163);
-          _161 = opaque::<u64>(move _162) -> [return: bb38, unwind unreachable];
+          StorageDead(_137);
+          StorageDead(_136);
+          StorageLive(_140);
+          StorageLive(_141);
+          StorageLive(_142);
+          _142 = (*_135);
+          StorageLive(_143);
+          _143 = _1;
+-         _141 = Add(move _142, move _143);
++         _141 = Add(move _142, _1);
+          StorageDead(_143);
+          StorageDead(_142);
+          _140 = opaque::<u64>(move _141) -> [return: bb38, unwind unreachable];
       }
   
       bb38: {
-          StorageDead(_162);
-          StorageDead(_161);
-          _146 = const ();
--         StorageDead(_156);
--         StorageDead(_147);
+          StorageDead(_141);
+          StorageDead(_140);
+          StorageLive(_144);
+-         StorageLive(_145);
 +         nop;
-+         nop;
-          StorageDead(_146);
--         StorageLive(_165);
-+         nop;
-          _165 = &_3;
-          StorageLive(_166);
--         StorageLive(_167);
--         StorageLive(_168);
-+         nop;
-+         nop;
-          _168 = (*_165);
-          StorageLive(_169);
-          _169 = _1;
--         _167 = Add(move _168, move _169);
-+         _167 = Add(_168, _1);
-          StorageDead(_169);
--         StorageDead(_168);
--         _166 = opaque::<u64>(move _167) -> [return: bb39, unwind unreachable];
-+         nop;
-+         _166 = opaque::<u64>(_167) -> [return: bb39, unwind unreachable];
+          _145 = &raw const _3;
+          StorageLive(_146);
+          StorageLive(_147);
+          StorageLive(_148);
+          _148 = (*_145);
+          StorageLive(_149);
+          _149 = _1;
+-         _147 = Add(move _148, move _149);
++         _147 = Add(move _148, _1);
+          StorageDead(_149);
+          StorageDead(_148);
+          _146 = opaque::<u64>(move _147) -> [return: bb39, unwind unreachable];
       }
   
       bb39: {
--         StorageDead(_167);
-+         nop;
-          StorageDead(_166);
-          StorageLive(_170);
-          StorageLive(_171);
-          StorageLive(_172);
--         _172 = (*_165);
-+         _172 = _168;
-          StorageLive(_173);
-          _173 = _1;
--         _171 = Add(move _172, move _173);
-+         _171 = _167;
-          StorageDead(_173);
-          StorageDead(_172);
--         _170 = opaque::<u64>(move _171) -> [return: bb40, unwind unreachable];
-+         _170 = opaque::<u64>(_167) -> [return: bb40, unwind unreachable];
+          StorageDead(_147);
+          StorageDead(_146);
+          StorageLive(_150);
+          StorageLive(_151);
+          StorageLive(_152);
+          _152 = (*_145);
+          StorageLive(_153);
+          _153 = _1;
+-         _151 = Add(move _152, move _153);
++         _151 = Add(move _152, _1);
+          StorageDead(_153);
+          StorageDead(_152);
+          _150 = opaque::<u64>(move _151) -> [return: bb40, unwind unreachable];
       }
   
       bb40: {
+          StorageDead(_151);
+          StorageDead(_150);
+-         StorageLive(_154);
++         nop;
+          _154 = &raw mut _3;
+          StorageLive(_155);
+          StorageLive(_156);
+          StorageLive(_157);
+          _157 = (*_154);
+          StorageLive(_158);
+          _158 = _1;
+-         _156 = Add(move _157, move _158);
++         _156 = Add(move _157, _1);
+          StorageDead(_158);
+          StorageDead(_157);
+          _155 = opaque::<u64>(move _156) -> [return: bb41, unwind unreachable];
+      }
+  
+      bb41: {
+          StorageDead(_156);
+          StorageDead(_155);
+          StorageLive(_159);
+          StorageLive(_160);
+          StorageLive(_161);
+          _161 = (*_154);
+          StorageLive(_162);
+          _162 = _1;
+-         _160 = Add(move _161, move _162);
++         _160 = Add(move _161, _1);
+          StorageDead(_162);
+          StorageDead(_161);
+          _159 = opaque::<u64>(move _160) -> [return: bb42, unwind unreachable];
+      }
+  
+      bb42: {
+          StorageDead(_160);
+          StorageDead(_159);
+          _144 = const ();
+-         StorageDead(_154);
+-         StorageDead(_145);
++         nop;
++         nop;
+          StorageDead(_144);
+-         StorageLive(_163);
++         nop;
+          _163 = &_3;
+          StorageLive(_164);
+-         StorageLive(_165);
+-         StorageLive(_166);
++         nop;
++         nop;
+          _166 = (*_163);
+          StorageLive(_167);
+          _167 = _1;
+-         _165 = Add(move _166, move _167);
++         _165 = Add(_166, _1);
+          StorageDead(_167);
+-         StorageDead(_166);
+-         _164 = opaque::<u64>(move _165) -> [return: bb43, unwind unreachable];
++         nop;
++         _164 = opaque::<u64>(_165) -> [return: bb43, unwind unreachable];
+      }
+  
+      bb43: {
+-         StorageDead(_165);
++         nop;
+          StorageDead(_164);
+          StorageLive(_168);
+          StorageLive(_169);
+          StorageLive(_170);
+-         _170 = (*_163);
++         _170 = _166;
+          StorageLive(_171);
+          _171 = _1;
+-         _169 = Add(move _170, move _171);
++         _169 = _165;
           StorageDead(_171);
           StorageDead(_170);
+-         _168 = opaque::<u64>(move _169) -> [return: bb44, unwind unreachable];
++         _168 = opaque::<u64>(_165) -> [return: bb44, unwind unreachable];
+      }
+  
+      bb44: {
+          StorageDead(_169);
+          StorageDead(_168);
           _0 = const ();
--         StorageDead(_165);
--         StorageDead(_137);
--         StorageDead(_128);
+-         StorageDead(_163);
+-         StorageDead(_135);
+-         StorageDead(_126);
 +         nop;
 +         nop;
 +         nop;
diff --git a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff
index 062dc6ff5619..119a4d9bbe90 100644
--- a/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff
+++ b/tests/mir-opt/gvn.subexpression_elimination.GVN.panic-unwind.diff
@@ -65,11 +65,11 @@
       let mut _60: u64;
       let mut _61: u64;
       let mut _62: u64;
-      let mut _63: u64;
+      let _63: ();
       let mut _64: u64;
-      let _65: ();
+      let mut _65: u64;
       let mut _66: u64;
-      let mut _67: u64;
+      let _67: ();
       let mut _68: u64;
       let mut _69: u64;
       let mut _70: u64;
@@ -77,25 +77,25 @@
       let mut _72: u64;
       let mut _73: u64;
       let mut _74: u64;
-      let mut _75: u64;
-      let mut _76: u64;
-      let _77: ();
+      let mut _75: bool;
+      let _76: ();
+      let mut _77: u64;
       let mut _78: u64;
       let mut _79: u64;
-      let mut _80: u64;
-      let mut _81: u64;
-      let mut _82: bool;
+      let mut _80: bool;
+      let _81: ();
+      let mut _82: u64;
       let mut _83: u64;
-      let _84: ();
-      let mut _85: u64;
+      let mut _84: u64;
+      let _85: ();
       let mut _86: u64;
       let mut _87: u64;
       let mut _88: u64;
-      let mut _89: bool;
+      let _89: ();
       let mut _90: u64;
-      let _91: ();
+      let mut _91: u64;
       let mut _92: u64;
-      let mut _93: u64;
+      let _93: ();
       let mut _94: u64;
       let mut _95: u64;
       let mut _96: u64;
@@ -103,93 +103,91 @@
       let mut _98: u64;
       let mut _99: u64;
       let mut _100: u64;
-      let mut _101: u64;
-      let mut _102: u64;
-      let _103: ();
-      let mut _104: u64;
-      let mut _105: u64;
+      let _101: ();
+      let mut _102: u32;
+      let mut _103: u64;
+      let _104: ();
+      let mut _105: f32;
       let mut _106: u64;
-      let mut _107: u64;
-      let mut _108: u64;
-      let _109: ();
-      let mut _110: u64;
+      let _107: ();
+      let mut _108: S<u64>;
+      let mut _109: u64;
+      let _110: ();
       let mut _111: u64;
-      let mut _112: u64;
+      let mut _112: S<u64>;
       let mut _113: u64;
-      let mut _114: u64;
-      let _115: ();
+      let _114: ();
+      let mut _115: u64;
       let mut _116: u64;
       let mut _117: u64;
       let mut _118: u64;
       let mut _119: u64;
-      let mut _120: u64;
-      let _121: ();
-      let mut _122: S<u64>;
+      let _120: ();
+      let mut _121: u64;
+      let mut _122: u64;
       let mut _123: u64;
-      let _124: ();
+      let mut _124: u64;
       let mut _125: u64;
-      let mut _126: S<u64>;
-      let mut _127: u64;
-      let _128: &u64;
-      let _129: ();
+      let _126: &u64;
+      let _127: ();
+      let mut _128: u64;
+      let mut _129: u64;
       let mut _130: u64;
-      let mut _131: u64;
+      let _131: ();
       let mut _132: u64;
-      let _133: ();
+      let mut _133: u64;
       let mut _134: u64;
-      let mut _135: u64;
-      let mut _136: u64;
-      let _138: ();
+      let _136: ();
+      let mut _137: u64;
+      let mut _138: u64;
       let mut _139: u64;
-      let mut _140: u64;
+      let _140: ();
       let mut _141: u64;
-      let _142: ();
+      let mut _142: u64;
       let mut _143: u64;
-      let mut _144: u64;
-      let mut _145: u64;
+      let _144: ();
       let _146: ();
-      let _148: ();
+      let mut _147: u64;
+      let mut _148: u64;
       let mut _149: u64;
-      let mut _150: u64;
+      let _150: ();
       let mut _151: u64;
-      let _152: ();
+      let mut _152: u64;
       let mut _153: u64;
-      let mut _154: u64;
-      let mut _155: u64;
-      let _157: ();
+      let _155: ();
+      let mut _156: u64;
+      let mut _157: u64;
       let mut _158: u64;
-      let mut _159: u64;
+      let _159: ();
       let mut _160: u64;
-      let _161: ();
+      let mut _161: u64;
       let mut _162: u64;
-      let mut _163: u64;
-      let mut _164: u64;
-      let _166: ();
+      let _164: ();
+      let mut _165: u64;
+      let mut _166: u64;
       let mut _167: u64;
-      let mut _168: u64;
+      let _168: ();
       let mut _169: u64;
-      let _170: ();
+      let mut _170: u64;
       let mut _171: u64;
-      let mut _172: u64;
-      let mut _173: u64;
       scope 1 {
-          debug a => _128;
-          let _137: &mut u64;
+          debug a => _126;
+          let _135: &mut u64;
           scope 2 {
-              debug b => _137;
-              let _165: &u64;
+              debug b => _135;
+              let _163: &u64;
               scope 3 {
-                  let _147: *const u64;
+                  let _145: *const u64;
                   scope 4 {
-                      debug c => _147;
-                      let _156: *mut u64;
+                      debug c => _145;
+                      let _154: *mut u64;
                       scope 5 {
-                          debug d => _156;
+                          debug d => _154;
                       }
                   }
               }
               scope 6 {
-                  debug e => _165;
+                  debug e => _163;
               }
           }
       }
@@ -400,30 +398,36 @@
 +         nop;
           StorageDead(_42);
           StorageLive(_46);
-          StorageLive(_47);
+-         StorageLive(_47);
++         nop;
           StorageLive(_48);
           _48 = _1;
 -         _47 = move _48 as u32 (IntToInt);
 +         _47 = _1 as u32 (IntToInt);
           StorageDead(_48);
-          _46 = opaque::<u32>(move _47) -> [return: bb13, unwind continue];
+-         _46 = opaque::<u32>(move _47) -> [return: bb13, unwind continue];
++         _46 = opaque::<u32>(_47) -> [return: bb13, unwind continue];
       }
   
       bb13: {
-          StorageDead(_47);
+-         StorageDead(_47);
++         nop;
           StorageDead(_46);
           StorageLive(_49);
-          StorageLive(_50);
+-         StorageLive(_50);
++         nop;
           StorageLive(_51);
           _51 = _1;
 -         _50 = move _51 as f32 (IntToFloat);
 +         _50 = _1 as f32 (IntToFloat);
           StorageDead(_51);
-          _49 = opaque::<f32>(move _50) -> [return: bb14, unwind continue];
+-         _49 = opaque::<f32>(move _50) -> [return: bb14, unwind continue];
++         _49 = opaque::<f32>(_50) -> [return: bb14, unwind continue];
       }
   
       bb14: {
-          StorageDead(_50);
+-         StorageDead(_50);
++         nop;
           StorageDead(_49);
           StorageLive(_52);
 -         StorageLive(_53);
@@ -462,479 +466,497 @@
           StorageLive(_59);
           StorageLive(_60);
           StorageLive(_61);
+          _61 = _1;
           StorageLive(_62);
-          _62 = _1;
-          StorageLive(_63);
-          _63 = _2;
--         _61 = Add(move _62, move _63);
-+         _61 = _5;
-          StorageDead(_63);
+          _62 = _2;
+-         _60 = Add(move _61, move _62);
++         _60 = _5;
           StorageDead(_62);
-          StorageLive(_64);
-          _64 = _3;
--         _60 = Add(move _61, move _64);
-+         _60 = Add(_5, move _64);
-          StorageDead(_64);
           StorageDead(_61);
-          _59 = opaque::<u64>(move _60) -> [return: bb17, unwind continue];
+-         _59 = opaque::<u64>(move _60) -> [return: bb17, unwind continue];
++         _59 = opaque::<u64>(_5) -> [return: bb17, unwind continue];
       }
   
       bb17: {
           StorageDead(_60);
           StorageDead(_59);
+          StorageLive(_63);
+          StorageLive(_64);
           StorageLive(_65);
+          _65 = _1;
           StorageLive(_66);
-          StorageLive(_67);
-          StorageLive(_68);
-          _68 = _1;
-          StorageLive(_69);
-          _69 = _2;
--         _67 = Mul(move _68, move _69);
-+         _67 = _9;
-          StorageDead(_69);
-          StorageDead(_68);
-          StorageLive(_70);
-          _70 = _3;
--         _66 = Add(move _67, move _70);
-+         _66 = Add(_9, move _70);
-          StorageDead(_70);
-          StorageDead(_67);
-          _65 = opaque::<u64>(move _66) -> [return: bb18, unwind continue];
+          _66 = _2;
+-         _64 = Mul(move _65, move _66);
++         _64 = _9;
+          StorageDead(_66);
+          StorageDead(_65);
+-         _63 = opaque::<u64>(move _64) -> [return: bb18, unwind continue];
++         _63 = opaque::<u64>(_9) -> [return: bb18, unwind continue];
       }
   
       bb18: {
-          StorageDead(_66);
-          StorageDead(_65);
-          StorageLive(_71);
-          StorageLive(_72);
-          StorageLive(_73);
-          StorageLive(_74);
-          _74 = _1;
-          StorageLive(_75);
-          _75 = _2;
--         _73 = Sub(move _74, move _75);
-+         _73 = _13;
-          StorageDead(_75);
-          StorageDead(_74);
-          StorageLive(_76);
-          _76 = _3;
--         _72 = Add(move _73, move _76);
-+         _72 = Add(_13, move _76);
-          StorageDead(_76);
-          StorageDead(_73);
-          _71 = opaque::<u64>(move _72) -> [return: bb19, unwind continue];
+          StorageDead(_64);
+          StorageDead(_63);
+          StorageLive(_67);
+          StorageLive(_68);
+          StorageLive(_69);
+          _69 = _1;
+          StorageLive(_70);
+          _70 = _2;
+-         _68 = Sub(move _69, move _70);
++         _68 = _13;
+          StorageDead(_70);
+          StorageDead(_69);
+-         _67 = opaque::<u64>(move _68) -> [return: bb19, unwind continue];
++         _67 = opaque::<u64>(_13) -> [return: bb19, unwind continue];
       }
   
       bb19: {
-          StorageDead(_72);
-          StorageDead(_71);
-          StorageLive(_77);
-          StorageLive(_78);
-          StorageLive(_79);
-          StorageLive(_80);
-          _80 = _1;
-          StorageLive(_81);
-          _81 = _2;
--         _82 = Eq(_81, const 0_u64);
--         assert(!move _82, "attempt to divide `{}` by zero", _80) -> [success: bb20, unwind continue];
-+         _82 = _20;
+          StorageDead(_68);
+          StorageDead(_67);
+          StorageLive(_71);
+          StorageLive(_72);
+          StorageLive(_73);
+          _73 = _1;
+          StorageLive(_74);
+          _74 = _2;
+-         _75 = Eq(_74, const 0_u64);
+-         assert(!move _75, "attempt to divide `{}` by zero", _73) -> [success: bb20, unwind continue];
++         _75 = _20;
 +         assert(!_20, "attempt to divide `{}` by zero", _1) -> [success: bb20, unwind continue];
       }
   
       bb20: {
--         _79 = Div(move _80, move _81);
-+         _79 = _17;
-          StorageDead(_81);
-          StorageDead(_80);
-          StorageLive(_83);
-          _83 = _3;
--         _78 = Add(move _79, move _83);
-+         _78 = Add(_17, move _83);
-          StorageDead(_83);
-          StorageDead(_79);
-          _77 = opaque::<u64>(move _78) -> [return: bb21, unwind continue];
+-         _72 = Div(move _73, move _74);
++         _72 = _17;
+          StorageDead(_74);
+          StorageDead(_73);
+-         _71 = opaque::<u64>(move _72) -> [return: bb21, unwind continue];
++         _71 = opaque::<u64>(_17) -> [return: bb21, unwind continue];
       }
   
       bb21: {
+          StorageDead(_72);
+          StorageDead(_71);
+          StorageLive(_76);
+          StorageLive(_77);
+          StorageLive(_78);
+          _78 = _1;
+          StorageLive(_79);
+          _79 = _2;
+-         _80 = Eq(_79, const 0_u64);
+-         assert(!move _80, "attempt to calculate the remainder of `{}` with a divisor of zero", _78) -> [success: bb22, unwind continue];
++         _80 = _20;
++         assert(!_20, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb22, unwind continue];
+      }
+  
+      bb22: {
+-         _77 = Rem(move _78, move _79);
++         _77 = _22;
+          StorageDead(_79);
           StorageDead(_78);
+-         _76 = opaque::<u64>(move _77) -> [return: bb23, unwind continue];
++         _76 = opaque::<u64>(_22) -> [return: bb23, unwind continue];
+      }
+  
+      bb23: {
           StorageDead(_77);
+          StorageDead(_76);
+          StorageLive(_81);
+          StorageLive(_82);
+          StorageLive(_83);
+          _83 = _1;
           StorageLive(_84);
+          _84 = _2;
+-         _82 = BitAnd(move _83, move _84);
++         _82 = _27;
+          StorageDead(_84);
+          StorageDead(_83);
+-         _81 = opaque::<u64>(move _82) -> [return: bb24, unwind continue];
++         _81 = opaque::<u64>(_27) -> [return: bb24, unwind continue];
+      }
+  
+      bb24: {
+          StorageDead(_82);
+          StorageDead(_81);
           StorageLive(_85);
           StorageLive(_86);
           StorageLive(_87);
           _87 = _1;
           StorageLive(_88);
           _88 = _2;
--         _89 = Eq(_88, const 0_u64);
--         assert(!move _89, "attempt to calculate the remainder of `{}` with a divisor of zero", _87) -> [success: bb22, unwind continue];
-+         _89 = _20;
-+         assert(!_20, "attempt to calculate the remainder of `{}` with a divisor of zero", _1) -> [success: bb22, unwind continue];
-      }
-  
-      bb22: {
--         _86 = Rem(move _87, move _88);
-+         _86 = _22;
+-         _86 = BitOr(move _87, move _88);
++         _86 = _31;
           StorageDead(_88);
           StorageDead(_87);
-          StorageLive(_90);
-          _90 = _3;
--         _85 = Add(move _86, move _90);
-+         _85 = Add(_22, move _90);
-          StorageDead(_90);
-          StorageDead(_86);
-          _84 = opaque::<u64>(move _85) -> [return: bb23, unwind continue];
-      }
-  
-      bb23: {
-          StorageDead(_85);
-          StorageDead(_84);
-          StorageLive(_91);
-          StorageLive(_92);
-          StorageLive(_93);
-          StorageLive(_94);
-          _94 = _1;
-          StorageLive(_95);
-          _95 = _2;
--         _93 = BitAnd(move _94, move _95);
-+         _93 = _27;
-          StorageDead(_95);
-          StorageDead(_94);
-          StorageLive(_96);
-          _96 = _3;
--         _92 = Add(move _93, move _96);
-+         _92 = Add(_27, move _96);
-          StorageDead(_96);
-          StorageDead(_93);
-          _91 = opaque::<u64>(move _92) -> [return: bb24, unwind continue];
-      }
-  
-      bb24: {
-          StorageDead(_92);
-          StorageDead(_91);
-          StorageLive(_97);
-          StorageLive(_98);
-          StorageLive(_99);
-          StorageLive(_100);
-          _100 = _1;
-          StorageLive(_101);
-          _101 = _2;
--         _99 = BitOr(move _100, move _101);
-+         _99 = _31;
-          StorageDead(_101);
-          StorageDead(_100);
-          StorageLive(_102);
-          _102 = _3;
--         _98 = Add(move _99, move _102);
-+         _98 = Add(_31, move _102);
-          StorageDead(_102);
-          StorageDead(_99);
-          _97 = opaque::<u64>(move _98) -> [return: bb25, unwind continue];
+-         _85 = opaque::<u64>(move _86) -> [return: bb25, unwind continue];
++         _85 = opaque::<u64>(_31) -> [return: bb25, unwind continue];
       }
   
       bb25: {
+          StorageDead(_86);
+          StorageDead(_85);
+          StorageLive(_89);
+          StorageLive(_90);
+          StorageLive(_91);
+          _91 = _1;
+          StorageLive(_92);
+          _92 = _2;
+-         _90 = BitXor(move _91, move _92);
++         _90 = _35;
+          StorageDead(_92);
+          StorageDead(_91);
+-         _89 = opaque::<u64>(move _90) -> [return: bb26, unwind continue];
++         _89 = opaque::<u64>(_35) -> [return: bb26, unwind continue];
+      }
+  
+      bb26: {
+          StorageDead(_90);
+          StorageDead(_89);
+          StorageLive(_93);
+          StorageLive(_94);
+          StorageLive(_95);
+          _95 = _1;
+          StorageLive(_96);
+          _96 = _2;
+-         _94 = Shl(move _95, move _96);
++         _94 = _39;
+          StorageDead(_96);
+          StorageDead(_95);
+-         _93 = opaque::<u64>(move _94) -> [return: bb27, unwind continue];
++         _93 = opaque::<u64>(_39) -> [return: bb27, unwind continue];
+      }
+  
+      bb27: {
+          StorageDead(_94);
+          StorageDead(_93);
+          StorageLive(_97);
+          StorageLive(_98);
+          StorageLive(_99);
+          _99 = _1;
+          StorageLive(_100);
+          _100 = _2;
+-         _98 = Shr(move _99, move _100);
++         _98 = _43;
+          StorageDead(_100);
+          StorageDead(_99);
+-         _97 = opaque::<u64>(move _98) -> [return: bb28, unwind continue];
++         _97 = opaque::<u64>(_43) -> [return: bb28, unwind continue];
+      }
+  
+      bb28: {
           StorageDead(_98);
           StorageDead(_97);
+          StorageLive(_101);
+          StorageLive(_102);
           StorageLive(_103);
+          _103 = _1;
+-         _102 = move _103 as u32 (IntToInt);
++         _102 = _47;
+          StorageDead(_103);
+-         _101 = opaque::<u32>(move _102) -> [return: bb29, unwind continue];
++         _101 = opaque::<u32>(_47) -> [return: bb29, unwind continue];
+      }
+  
+      bb29: {
+          StorageDead(_102);
+          StorageDead(_101);
           StorageLive(_104);
           StorageLive(_105);
           StorageLive(_106);
           _106 = _1;
-          StorageLive(_107);
-          _107 = _2;
--         _105 = BitXor(move _106, move _107);
-+         _105 = _35;
-          StorageDead(_107);
+-         _105 = move _106 as f32 (IntToFloat);
++         _105 = _50;
           StorageDead(_106);
-          StorageLive(_108);
-          _108 = _3;
--         _104 = Add(move _105, move _108);
-+         _104 = Add(_35, move _108);
-          StorageDead(_108);
-          StorageDead(_105);
-          _103 = opaque::<u64>(move _104) -> [return: bb26, unwind continue];
+-         _104 = opaque::<f32>(move _105) -> [return: bb30, unwind continue];
++         _104 = opaque::<f32>(_50) -> [return: bb30, unwind continue];
       }
   
-      bb26: {
+      bb30: {
+          StorageDead(_105);
           StorageDead(_104);
-          StorageDead(_103);
+          StorageLive(_107);
+          StorageLive(_108);
           StorageLive(_109);
+          _109 = _1;
+-         _108 = S::<u64>(move _109);
++         _108 = _53;
+          StorageDead(_109);
+-         _107 = opaque::<S<u64>>(move _108) -> [return: bb31, unwind continue];
++         _107 = opaque::<S<u64>>(_53) -> [return: bb31, unwind continue];
+      }
+  
+      bb31: {
+          StorageDead(_108);
+          StorageDead(_107);
           StorageLive(_110);
           StorageLive(_111);
           StorageLive(_112);
-          _112 = _1;
           StorageLive(_113);
-          _113 = _2;
--         _111 = Shl(move _112, move _113);
-+         _111 = _39;
+          _113 = _1;
+-         _112 = S::<u64>(move _113);
++         _112 = _53;
           StorageDead(_113);
-          StorageDead(_112);
-          StorageLive(_114);
-          _114 = _3;
--         _110 = Add(move _111, move _114);
-+         _110 = Add(_39, move _114);
-          StorageDead(_114);
-          StorageDead(_111);
-          _109 = opaque::<u64>(move _110) -> [return: bb27, unwind continue];
+-         _111 = (_112.0: u64);
+-         _110 = opaque::<u64>(move _111) -> [return: bb32, unwind continue];
++         _111 = _1;
++         _110 = opaque::<u64>(_1) -> [return: bb32, unwind continue];
       }
   
-      bb27: {
+      bb32: {
+          StorageDead(_111);
+          StorageDead(_112);
           StorageDead(_110);
-          StorageDead(_109);
-          StorageLive(_115);
+          StorageLive(_114);
+-         StorageLive(_115);
++         nop;
           StorageLive(_116);
           StorageLive(_117);
+          _117 = _1;
           StorageLive(_118);
-          _118 = _1;
+          _118 = _2;
+-         _116 = Mul(move _117, move _118);
++         _116 = _9;
+          StorageDead(_118);
+          StorageDead(_117);
           StorageLive(_119);
           _119 = _2;
--         _117 = Shr(move _118, move _119);
-+         _117 = _43;
+-         _115 = Sub(move _116, move _119);
++         _115 = Sub(_9, _2);
           StorageDead(_119);
-          StorageDead(_118);
-          StorageLive(_120);
-          _120 = _3;
--         _116 = Add(move _117, move _120);
-+         _116 = Add(_43, move _120);
-          StorageDead(_120);
-          StorageDead(_117);
-          _115 = opaque::<u64>(move _116) -> [return: bb28, unwind continue];
+          StorageDead(_116);
+-         _114 = opaque::<u64>(move _115) -> [return: bb33, unwind continue];
++         _114 = opaque::<u64>(_115) -> [return: bb33, unwind continue];
       }
   
-      bb28: {
-          StorageDead(_116);
-          StorageDead(_115);
+      bb33: {
+-         StorageDead(_115);
++         nop;
+          StorageDead(_114);
+          StorageLive(_120);
           StorageLive(_121);
           StorageLive(_122);
           StorageLive(_123);
           _123 = _1;
--         _122 = S::<u64>(move _123);
-+         _122 = _53;
-          StorageDead(_123);
--         _121 = opaque::<S<u64>>(move _122) -> [return: bb29, unwind continue];
-+         _121 = opaque::<S<u64>>(_53) -> [return: bb29, unwind continue];
-      }
-  
-      bb29: {
-          StorageDead(_122);
-          StorageDead(_121);
           StorageLive(_124);
-          StorageLive(_125);
-          StorageLive(_126);
-          StorageLive(_127);
-          _127 = _1;
--         _126 = S::<u64>(move _127);
-+         _126 = _53;
-          StorageDead(_127);
--         _125 = (_126.0: u64);
--         _124 = opaque::<u64>(move _125) -> [return: bb30, unwind continue];
-+         _125 = _1;
-+         _124 = opaque::<u64>(_1) -> [return: bb30, unwind continue];
-      }
-  
-      bb30: {
-          StorageDead(_125);
-          StorageDead(_126);
+          _124 = _2;
+-         _122 = Mul(move _123, move _124);
++         _122 = _9;
           StorageDead(_124);
--         StorageLive(_128);
-+         nop;
-          _128 = &_3;
-          StorageLive(_129);
--         StorageLive(_130);
--         StorageLive(_131);
-+         nop;
-+         nop;
-          _131 = (*_128);
-          StorageLive(_132);
-          _132 = _1;
--         _130 = Add(move _131, move _132);
-+         _130 = Add(_131, _1);
-          StorageDead(_132);
--         StorageDead(_131);
--         _129 = opaque::<u64>(move _130) -> [return: bb31, unwind continue];
-+         nop;
-+         _129 = opaque::<u64>(_130) -> [return: bb31, unwind continue];
-      }
-  
-      bb31: {
--         StorageDead(_130);
-+         nop;
-          StorageDead(_129);
-          StorageLive(_133);
-          StorageLive(_134);
-          StorageLive(_135);
--         _135 = (*_128);
-+         _135 = _131;
-          StorageLive(_136);
-          _136 = _1;
--         _134 = Add(move _135, move _136);
-+         _134 = _130;
-          StorageDead(_136);
-          StorageDead(_135);
--         _133 = opaque::<u64>(move _134) -> [return: bb32, unwind continue];
-+         _133 = opaque::<u64>(_130) -> [return: bb32, unwind continue];
-      }
-  
-      bb32: {
-          StorageDead(_134);
-          StorageDead(_133);
--         StorageLive(_137);
-+         nop;
-          _137 = &mut _3;
-          StorageLive(_138);
-          StorageLive(_139);
-          StorageLive(_140);
-          _140 = (*_137);
-          StorageLive(_141);
-          _141 = _1;
--         _139 = Add(move _140, move _141);
-+         _139 = Add(move _140, _1);
-          StorageDead(_141);
-          StorageDead(_140);
-          _138 = opaque::<u64>(move _139) -> [return: bb33, unwind continue];
-      }
-  
-      bb33: {
-          StorageDead(_139);
-          StorageDead(_138);
-          StorageLive(_142);
-          StorageLive(_143);
-          StorageLive(_144);
-          _144 = (*_137);
-          StorageLive(_145);
-          _145 = _1;
--         _143 = Add(move _144, move _145);
-+         _143 = Add(move _144, _1);
-          StorageDead(_145);
-          StorageDead(_144);
-          _142 = opaque::<u64>(move _143) -> [return: bb34, unwind continue];
+          StorageDead(_123);
+          StorageLive(_125);
+          _125 = _2;
+-         _121 = Sub(move _122, move _125);
++         _121 = _115;
+          StorageDead(_125);
+          StorageDead(_122);
+-         _120 = opaque::<u64>(move _121) -> [return: bb34, unwind continue];
++         _120 = opaque::<u64>(_115) -> [return: bb34, unwind continue];
       }
   
       bb34: {
-          StorageDead(_143);
-          StorageDead(_142);
-          StorageLive(_146);
--         StorageLive(_147);
+          StorageDead(_121);
+          StorageDead(_120);
+-         StorageLive(_126);
 +         nop;
-          _147 = &raw const _3;
-          StorageLive(_148);
-          StorageLive(_149);
-          StorageLive(_150);
-          _150 = (*_147);
-          StorageLive(_151);
-          _151 = _1;
--         _149 = Add(move _150, move _151);
-+         _149 = Add(move _150, _1);
-          StorageDead(_151);
-          StorageDead(_150);
-          _148 = opaque::<u64>(move _149) -> [return: bb35, unwind continue];
+          _126 = &_3;
+          StorageLive(_127);
+-         StorageLive(_128);
+-         StorageLive(_129);
++         nop;
++         nop;
+          _129 = (*_126);
+          StorageLive(_130);
+          _130 = _1;
+-         _128 = Add(move _129, move _130);
++         _128 = Add(_129, _1);
+          StorageDead(_130);
+-         StorageDead(_129);
+-         _127 = opaque::<u64>(move _128) -> [return: bb35, unwind continue];
++         nop;
++         _127 = opaque::<u64>(_128) -> [return: bb35, unwind continue];
       }
   
       bb35: {
-          StorageDead(_149);
-          StorageDead(_148);
-          StorageLive(_152);
-          StorageLive(_153);
-          StorageLive(_154);
-          _154 = (*_147);
-          StorageLive(_155);
-          _155 = _1;
--         _153 = Add(move _154, move _155);
-+         _153 = Add(move _154, _1);
-          StorageDead(_155);
-          StorageDead(_154);
-          _152 = opaque::<u64>(move _153) -> [return: bb36, unwind continue];
+-         StorageDead(_128);
++         nop;
+          StorageDead(_127);
+          StorageLive(_131);
+          StorageLive(_132);
+          StorageLive(_133);
+-         _133 = (*_126);
++         _133 = _129;
+          StorageLive(_134);
+          _134 = _1;
+-         _132 = Add(move _133, move _134);
++         _132 = _128;
+          StorageDead(_134);
+          StorageDead(_133);
+-         _131 = opaque::<u64>(move _132) -> [return: bb36, unwind continue];
++         _131 = opaque::<u64>(_128) -> [return: bb36, unwind continue];
       }
   
       bb36: {
-          StorageDead(_153);
-          StorageDead(_152);
--         StorageLive(_156);
+          StorageDead(_132);
+          StorageDead(_131);
+-         StorageLive(_135);
 +         nop;
-          _156 = &raw mut _3;
-          StorageLive(_157);
-          StorageLive(_158);
-          StorageLive(_159);
-          _159 = (*_156);
-          StorageLive(_160);
-          _160 = _1;
--         _158 = Add(move _159, move _160);
-+         _158 = Add(move _159, _1);
-          StorageDead(_160);
-          StorageDead(_159);
-          _157 = opaque::<u64>(move _158) -> [return: bb37, unwind continue];
+          _135 = &mut _3;
+          StorageLive(_136);
+          StorageLive(_137);
+          StorageLive(_138);
+          _138 = (*_135);
+          StorageLive(_139);
+          _139 = _1;
+-         _137 = Add(move _138, move _139);
++         _137 = Add(move _138, _1);
+          StorageDead(_139);
+          StorageDead(_138);
+          _136 = opaque::<u64>(move _137) -> [return: bb37, unwind continue];
       }
   
       bb37: {
-          StorageDead(_158);
-          StorageDead(_157);
-          StorageLive(_161);
-          StorageLive(_162);
-          StorageLive(_163);
-          _163 = (*_156);
-          StorageLive(_164);
-          _164 = _1;
--         _162 = Add(move _163, move _164);
-+         _162 = Add(move _163, _1);
-          StorageDead(_164);
-          StorageDead(_163);
-          _161 = opaque::<u64>(move _162) -> [return: bb38, unwind continue];
+          StorageDead(_137);
+          StorageDead(_136);
+          StorageLive(_140);
+          StorageLive(_141);
+          StorageLive(_142);
+          _142 = (*_135);
+          StorageLive(_143);
+          _143 = _1;
+-         _141 = Add(move _142, move _143);
++         _141 = Add(move _142, _1);
+          StorageDead(_143);
+          StorageDead(_142);
+          _140 = opaque::<u64>(move _141) -> [return: bb38, unwind continue];
       }
   
       bb38: {
-          StorageDead(_162);
-          StorageDead(_161);
-          _146 = const ();
--         StorageDead(_156);
--         StorageDead(_147);
+          StorageDead(_141);
+          StorageDead(_140);
+          StorageLive(_144);
+-         StorageLive(_145);
 +         nop;
-+         nop;
-          StorageDead(_146);
--         StorageLive(_165);
-+         nop;
-          _165 = &_3;
-          StorageLive(_166);
--         StorageLive(_167);
--         StorageLive(_168);
-+         nop;
-+         nop;
-          _168 = (*_165);
-          StorageLive(_169);
-          _169 = _1;
--         _167 = Add(move _168, move _169);
-+         _167 = Add(_168, _1);
-          StorageDead(_169);
--         StorageDead(_168);
--         _166 = opaque::<u64>(move _167) -> [return: bb39, unwind continue];
-+         nop;
-+         _166 = opaque::<u64>(_167) -> [return: bb39, unwind continue];
+          _145 = &raw const _3;
+          StorageLive(_146);
+          StorageLive(_147);
+          StorageLive(_148);
+          _148 = (*_145);
+          StorageLive(_149);
+          _149 = _1;
+-         _147 = Add(move _148, move _149);
++         _147 = Add(move _148, _1);
+          StorageDead(_149);
+          StorageDead(_148);
+          _146 = opaque::<u64>(move _147) -> [return: bb39, unwind continue];
       }
   
       bb39: {
--         StorageDead(_167);
-+         nop;
-          StorageDead(_166);
-          StorageLive(_170);
-          StorageLive(_171);
-          StorageLive(_172);
--         _172 = (*_165);
-+         _172 = _168;
-          StorageLive(_173);
-          _173 = _1;
--         _171 = Add(move _172, move _173);
-+         _171 = _167;
-          StorageDead(_173);
-          StorageDead(_172);
--         _170 = opaque::<u64>(move _171) -> [return: bb40, unwind continue];
-+         _170 = opaque::<u64>(_167) -> [return: bb40, unwind continue];
+          StorageDead(_147);
+          StorageDead(_146);
+          StorageLive(_150);
+          StorageLive(_151);
+          StorageLive(_152);
+          _152 = (*_145);
+          StorageLive(_153);
+          _153 = _1;
+-         _151 = Add(move _152, move _153);
++         _151 = Add(move _152, _1);
+          StorageDead(_153);
+          StorageDead(_152);
+          _150 = opaque::<u64>(move _151) -> [return: bb40, unwind continue];
       }
   
       bb40: {
+          StorageDead(_151);
+          StorageDead(_150);
+-         StorageLive(_154);
++         nop;
+          _154 = &raw mut _3;
+          StorageLive(_155);
+          StorageLive(_156);
+          StorageLive(_157);
+          _157 = (*_154);
+          StorageLive(_158);
+          _158 = _1;
+-         _156 = Add(move _157, move _158);
++         _156 = Add(move _157, _1);
+          StorageDead(_158);
+          StorageDead(_157);
+          _155 = opaque::<u64>(move _156) -> [return: bb41, unwind continue];
+      }
+  
+      bb41: {
+          StorageDead(_156);
+          StorageDead(_155);
+          StorageLive(_159);
+          StorageLive(_160);
+          StorageLive(_161);
+          _161 = (*_154);
+          StorageLive(_162);
+          _162 = _1;
+-         _160 = Add(move _161, move _162);
++         _160 = Add(move _161, _1);
+          StorageDead(_162);
+          StorageDead(_161);
+          _159 = opaque::<u64>(move _160) -> [return: bb42, unwind continue];
+      }
+  
+      bb42: {
+          StorageDead(_160);
+          StorageDead(_159);
+          _144 = const ();
+-         StorageDead(_154);
+-         StorageDead(_145);
++         nop;
++         nop;
+          StorageDead(_144);
+-         StorageLive(_163);
++         nop;
+          _163 = &_3;
+          StorageLive(_164);
+-         StorageLive(_165);
+-         StorageLive(_166);
++         nop;
++         nop;
+          _166 = (*_163);
+          StorageLive(_167);
+          _167 = _1;
+-         _165 = Add(move _166, move _167);
++         _165 = Add(_166, _1);
+          StorageDead(_167);
+-         StorageDead(_166);
+-         _164 = opaque::<u64>(move _165) -> [return: bb43, unwind continue];
++         nop;
++         _164 = opaque::<u64>(_165) -> [return: bb43, unwind continue];
+      }
+  
+      bb43: {
+-         StorageDead(_165);
++         nop;
+          StorageDead(_164);
+          StorageLive(_168);
+          StorageLive(_169);
+          StorageLive(_170);
+-         _170 = (*_163);
++         _170 = _166;
+          StorageLive(_171);
+          _171 = _1;
+-         _169 = Add(move _170, move _171);
++         _169 = _165;
           StorageDead(_171);
           StorageDead(_170);
+-         _168 = opaque::<u64>(move _169) -> [return: bb44, unwind continue];
++         _168 = opaque::<u64>(_165) -> [return: bb44, unwind continue];
+      }
+  
+      bb44: {
+          StorageDead(_169);
+          StorageDead(_168);
           _0 = const ();
--         StorageDead(_165);
--         StorageDead(_137);
--         StorageDead(_128);
+-         StorageDead(_163);
+-         StorageDead(_135);
+-         StorageDead(_126);
 +         nop;
 +         nop;
 +         nop;

From c4cc9ca0603cfdf5921a82d929d680db94d1d072 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 21 Oct 2023 09:59:03 +0000
Subject: [PATCH 078/297] Do not merge fn pointer casts.

---
 compiler/rustc_mir_transform/src/gvn.rs       |   9 ++
 .../miri/tests/pass/function_pointers.rs      |   5 +-
 .../gvn.fn_pointers.GVN.panic-abort.diff      | 118 ++++++++++++++++++
 .../gvn.fn_pointers.GVN.panic-unwind.diff     | 118 ++++++++++++++++++
 tests/mir-opt/gvn.rs                          |  25 ++++
 5 files changed, 272 insertions(+), 3 deletions(-)
 create mode 100644 tests/mir-opt/gvn.fn_pointers.GVN.panic-abort.diff
 create mode 100644 tests/mir-opt/gvn.fn_pointers.GVN.panic-unwind.diff

diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index de61571fdfcb..cf8a0fdb97e2 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -92,6 +92,7 @@ use rustc_index::IndexVec;
 use rustc_macros::newtype_index;
 use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
+use rustc_middle::ty::adjustment::PointerCoercion;
 use rustc_middle::ty::layout::LayoutOf;
 use rustc_middle::ty::{self, Ty, TyCtxt, TypeAndMut};
 use rustc_span::def_id::DefId;
@@ -761,6 +762,14 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
             Rvalue::Cast(kind, ref mut value, to) => {
                 let from = value.ty(self.local_decls, self.tcx);
                 let value = self.simplify_operand(value, location)?;
+                if let CastKind::PointerCoercion(
+                    PointerCoercion::ReifyFnPointer | PointerCoercion::ClosureFnPointer(_),
+                ) = kind
+                {
+                    // Each reification of a generic fn may get a different pointer.
+                    // Do not try to merge them.
+                    return self.new_opaque();
+                }
                 Value::Cast { kind, value, from, to }
             }
             Rvalue::BinaryOp(op, box (ref mut lhs, ref mut rhs)) => {
diff --git a/src/tools/miri/tests/pass/function_pointers.rs b/src/tools/miri/tests/pass/function_pointers.rs
index 1c99a96feda9..8e58692a0c74 100644
--- a/src/tools/miri/tests/pass/function_pointers.rs
+++ b/src/tools/miri/tests/pass/function_pointers.rs
@@ -80,9 +80,8 @@ fn main() {
     // but Miri currently uses a fixed address for monomorphic functions.
     assert!(return_fn_ptr(i) == i);
     assert!(return_fn_ptr(i) as unsafe fn() -> i32 == i as fn() -> i32 as unsafe fn() -> i32);
-    // We don't check anything for `f`. Miri gives it many different addresses
-    // but mir-opts can turn them into the same address.
-    let _val = return_fn_ptr(f) != f;
+    // Miri gives it many different addresses to different reifications of a generic function.
+    assert!(return_fn_ptr(f) != f);
     // However, if we only turn `f` into a function pointer and use that pointer,
     // it is equal to itself.
     let f2 = f as fn() -> i32;
diff --git a/tests/mir-opt/gvn.fn_pointers.GVN.panic-abort.diff b/tests/mir-opt/gvn.fn_pointers.GVN.panic-abort.diff
new file mode 100644
index 000000000000..d8248d22d38b
--- /dev/null
+++ b/tests/mir-opt/gvn.fn_pointers.GVN.panic-abort.diff
@@ -0,0 +1,118 @@
+- // MIR for `fn_pointers` before GVN
++ // MIR for `fn_pointers` after GVN
+  
+  fn fn_pointers() -> () {
+      let mut _0: ();
+      let _1: fn(u8) -> u8;
+      let _2: ();
+      let mut _3: fn(u8) -> u8;
+      let _5: ();
+      let mut _6: fn(u8) -> u8;
+      let mut _9: {closure@$DIR/gvn.rs:591:19: 591:21};
+      let _10: ();
+      let mut _11: fn();
+      let mut _13: {closure@$DIR/gvn.rs:591:19: 591:21};
+      let _14: ();
+      let mut _15: fn();
+      scope 1 {
+          debug f => _1;
+          let _4: fn(u8) -> u8;
+          scope 2 {
+              debug g => _4;
+              let _7: {closure@$DIR/gvn.rs:591:19: 591:21};
+              scope 3 {
+                  debug closure => _7;
+                  let _8: fn();
+                  scope 4 {
+                      debug cf => _8;
+                      let _12: fn();
+                      scope 5 {
+                          debug cg => _12;
+                      }
+                  }
+              }
+          }
+      }
+  
+      bb0: {
+-         StorageLive(_1);
++         nop;
+          _1 = identity::<u8> as fn(u8) -> u8 (PointerCoercion(ReifyFnPointer));
+          StorageLive(_2);
+          StorageLive(_3);
+          _3 = _1;
+-         _2 = opaque::<fn(u8) -> u8>(move _3) -> [return: bb1, unwind unreachable];
++         _2 = opaque::<fn(u8) -> u8>(_1) -> [return: bb1, unwind unreachable];
+      }
+  
+      bb1: {
+          StorageDead(_3);
+          StorageDead(_2);
+-         StorageLive(_4);
++         nop;
+          _4 = identity::<u8> as fn(u8) -> u8 (PointerCoercion(ReifyFnPointer));
+          StorageLive(_5);
+          StorageLive(_6);
+          _6 = _4;
+-         _5 = opaque::<fn(u8) -> u8>(move _6) -> [return: bb2, unwind unreachable];
++         _5 = opaque::<fn(u8) -> u8>(_4) -> [return: bb2, unwind unreachable];
+      }
+  
+      bb2: {
+          StorageDead(_6);
+          StorageDead(_5);
+-         StorageLive(_7);
+-         _7 = {closure@$DIR/gvn.rs:591:19: 591:21};
+-         StorageLive(_8);
++         nop;
++         _7 = const ZeroSized: {closure@$DIR/gvn.rs:591:19: 591:21};
++         nop;
+          StorageLive(_9);
+-         _9 = _7;
+-         _8 = move _9 as fn() (PointerCoercion(ClosureFnPointer(Normal)));
++         _9 = const ZeroSized: {closure@$DIR/gvn.rs:591:19: 591:21};
++         _8 = const ZeroSized: {closure@$DIR/gvn.rs:591:19: 591:21} as fn() (PointerCoercion(ClosureFnPointer(Normal)));
+          StorageDead(_9);
+          StorageLive(_10);
+          StorageLive(_11);
+          _11 = _8;
+-         _10 = opaque::<fn()>(move _11) -> [return: bb3, unwind unreachable];
++         _10 = opaque::<fn()>(_8) -> [return: bb3, unwind unreachable];
+      }
+  
+      bb3: {
+          StorageDead(_11);
+          StorageDead(_10);
+-         StorageLive(_12);
++         nop;
+          StorageLive(_13);
+-         _13 = _7;
+-         _12 = move _13 as fn() (PointerCoercion(ClosureFnPointer(Normal)));
++         _13 = const ZeroSized: {closure@$DIR/gvn.rs:591:19: 591:21};
++         _12 = const ZeroSized: {closure@$DIR/gvn.rs:591:19: 591:21} as fn() (PointerCoercion(ClosureFnPointer(Normal)));
+          StorageDead(_13);
+          StorageLive(_14);
+          StorageLive(_15);
+          _15 = _12;
+-         _14 = opaque::<fn()>(move _15) -> [return: bb4, unwind unreachable];
++         _14 = opaque::<fn()>(_12) -> [return: bb4, unwind unreachable];
+      }
+  
+      bb4: {
+          StorageDead(_15);
+          StorageDead(_14);
+          _0 = const ();
+-         StorageDead(_12);
+-         StorageDead(_8);
+-         StorageDead(_7);
+-         StorageDead(_4);
+-         StorageDead(_1);
++         nop;
++         nop;
++         nop;
++         nop;
++         nop;
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/gvn.fn_pointers.GVN.panic-unwind.diff b/tests/mir-opt/gvn.fn_pointers.GVN.panic-unwind.diff
new file mode 100644
index 000000000000..e38a3d852092
--- /dev/null
+++ b/tests/mir-opt/gvn.fn_pointers.GVN.panic-unwind.diff
@@ -0,0 +1,118 @@
+- // MIR for `fn_pointers` before GVN
++ // MIR for `fn_pointers` after GVN
+  
+  fn fn_pointers() -> () {
+      let mut _0: ();
+      let _1: fn(u8) -> u8;
+      let _2: ();
+      let mut _3: fn(u8) -> u8;
+      let _5: ();
+      let mut _6: fn(u8) -> u8;
+      let mut _9: {closure@$DIR/gvn.rs:591:19: 591:21};
+      let _10: ();
+      let mut _11: fn();
+      let mut _13: {closure@$DIR/gvn.rs:591:19: 591:21};
+      let _14: ();
+      let mut _15: fn();
+      scope 1 {
+          debug f => _1;
+          let _4: fn(u8) -> u8;
+          scope 2 {
+              debug g => _4;
+              let _7: {closure@$DIR/gvn.rs:591:19: 591:21};
+              scope 3 {
+                  debug closure => _7;
+                  let _8: fn();
+                  scope 4 {
+                      debug cf => _8;
+                      let _12: fn();
+                      scope 5 {
+                          debug cg => _12;
+                      }
+                  }
+              }
+          }
+      }
+  
+      bb0: {
+-         StorageLive(_1);
++         nop;
+          _1 = identity::<u8> as fn(u8) -> u8 (PointerCoercion(ReifyFnPointer));
+          StorageLive(_2);
+          StorageLive(_3);
+          _3 = _1;
+-         _2 = opaque::<fn(u8) -> u8>(move _3) -> [return: bb1, unwind continue];
++         _2 = opaque::<fn(u8) -> u8>(_1) -> [return: bb1, unwind continue];
+      }
+  
+      bb1: {
+          StorageDead(_3);
+          StorageDead(_2);
+-         StorageLive(_4);
++         nop;
+          _4 = identity::<u8> as fn(u8) -> u8 (PointerCoercion(ReifyFnPointer));
+          StorageLive(_5);
+          StorageLive(_6);
+          _6 = _4;
+-         _5 = opaque::<fn(u8) -> u8>(move _6) -> [return: bb2, unwind continue];
++         _5 = opaque::<fn(u8) -> u8>(_4) -> [return: bb2, unwind continue];
+      }
+  
+      bb2: {
+          StorageDead(_6);
+          StorageDead(_5);
+-         StorageLive(_7);
+-         _7 = {closure@$DIR/gvn.rs:591:19: 591:21};
+-         StorageLive(_8);
++         nop;
++         _7 = const ZeroSized: {closure@$DIR/gvn.rs:591:19: 591:21};
++         nop;
+          StorageLive(_9);
+-         _9 = _7;
+-         _8 = move _9 as fn() (PointerCoercion(ClosureFnPointer(Normal)));
++         _9 = const ZeroSized: {closure@$DIR/gvn.rs:591:19: 591:21};
++         _8 = const ZeroSized: {closure@$DIR/gvn.rs:591:19: 591:21} as fn() (PointerCoercion(ClosureFnPointer(Normal)));
+          StorageDead(_9);
+          StorageLive(_10);
+          StorageLive(_11);
+          _11 = _8;
+-         _10 = opaque::<fn()>(move _11) -> [return: bb3, unwind continue];
++         _10 = opaque::<fn()>(_8) -> [return: bb3, unwind continue];
+      }
+  
+      bb3: {
+          StorageDead(_11);
+          StorageDead(_10);
+-         StorageLive(_12);
++         nop;
+          StorageLive(_13);
+-         _13 = _7;
+-         _12 = move _13 as fn() (PointerCoercion(ClosureFnPointer(Normal)));
++         _13 = const ZeroSized: {closure@$DIR/gvn.rs:591:19: 591:21};
++         _12 = const ZeroSized: {closure@$DIR/gvn.rs:591:19: 591:21} as fn() (PointerCoercion(ClosureFnPointer(Normal)));
+          StorageDead(_13);
+          StorageLive(_14);
+          StorageLive(_15);
+          _15 = _12;
+-         _14 = opaque::<fn()>(move _15) -> [return: bb4, unwind continue];
++         _14 = opaque::<fn()>(_12) -> [return: bb4, unwind continue];
+      }
+  
+      bb4: {
+          StorageDead(_15);
+          StorageDead(_14);
+          _0 = const ();
+-         StorageDead(_12);
+-         StorageDead(_8);
+-         StorageDead(_7);
+-         StorageDead(_4);
+-         StorageDead(_1);
++         nop;
++         nop;
++         nop;
++         nop;
++         nop;
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/gvn.rs b/tests/mir-opt/gvn.rs
index 1c14f818044f..7ce851905e06 100644
--- a/tests/mir-opt/gvn.rs
+++ b/tests/mir-opt/gvn.rs
@@ -574,6 +574,29 @@ fn repeat() {
     let array = [val, val, val, val, val, val, val, val, val, val];
 }
 
+/// Verify that we do not merge fn pointers created by casts.
+fn fn_pointers() {
+    // CHECK-LABEL: fn fn_pointers(
+    // CHECK: [[f:_.*]] = identity::<u8> as fn(u8) -> u8 (PointerCoercion(ReifyFnPointer
+    // CHECK: opaque::<fn(u8) -> u8>([[f]])
+    let f = identity as fn(u8) -> u8;
+    opaque(f);
+    // CHECK: [[g:_.*]] = identity::<u8> as fn(u8) -> u8 (PointerCoercion(ReifyFnPointer
+    // CHECK: opaque::<fn(u8) -> u8>([[g]])
+    let g = identity as fn(u8) -> u8;
+    opaque(g);
+
+    // CHECK: [[cf:_.*]] = const {{.*}} as fn() (PointerCoercion(ClosureFnPointer
+    // CHECK: opaque::<fn()>([[cf]])
+    let closure = || {};
+    let cf = closure as fn();
+    opaque(cf);
+    // CHECK: [[cg:_.*]] = const {{.*}} as fn() (PointerCoercion(ClosureFnPointer
+    // CHECK: opaque::<fn()>([[cg]])
+    let cg = closure as fn();
+    opaque(cg);
+}
+
 fn main() {
     subexpression_elimination(2, 4, 5);
     wrap_unwrap(5);
@@ -590,6 +613,7 @@ fn main() {
     let (direct, indirect) = duplicate_slice();
     assert_eq!(direct, indirect);
     repeat();
+    fn_pointers();
 }
 
 #[inline(never)]
@@ -614,3 +638,4 @@ fn identity<T>(x: T) -> T {
 // EMIT_MIR gvn.slices.GVN.diff
 // EMIT_MIR gvn.duplicate_slice.GVN.diff
 // EMIT_MIR gvn.repeat.GVN.diff
+// EMIT_MIR gvn.fn_pointers.GVN.diff

From 0b96e479ca11daf3bdc5a1a6f002e8bbe516f487 Mon Sep 17 00:00:00 2001
From: Caio <c410.f3r@gmail.com>
Date: Wed, 25 Oct 2023 09:42:56 -0300
Subject: [PATCH 079/297] Remove `cfg_match` from the prelude

---
 library/core/src/lib.rs        |   3 +
 library/core/src/macros/mod.rs | 177 ++++++++++++++++-----------------
 2 files changed, 91 insertions(+), 89 deletions(-)

diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs
index 03243e31348e..8d3abe987765 100644
--- a/library/core/src/lib.rs
+++ b/library/core/src/lib.rs
@@ -291,6 +291,9 @@ pub mod assert_matches {
     pub use crate::macros::{assert_matches, debug_assert_matches};
 }
 
+#[unstable(feature = "cfg_match", issue = "115585")]
+pub use crate::macros::cfg_match;
+
 #[macro_use]
 mod internal_macros;
 
diff --git a/library/core/src/macros/mod.rs b/library/core/src/macros/mod.rs
index c367b53b7202..125a6f57bfba 100644
--- a/library/core/src/macros/mod.rs
+++ b/library/core/src/macros/mod.rs
@@ -168,6 +168,94 @@ pub macro assert_matches {
     },
 }
 
+/// A macro for defining `#[cfg]` match-like statements.
+///
+/// It is similar to the `if/elif` C preprocessor macro by allowing definition of a cascade of
+/// `#[cfg]` cases, emitting the implementation which matches first.
+///
+/// This allows you to conveniently provide a long list `#[cfg]`'d blocks of code
+/// without having to rewrite each clause multiple times.
+///
+/// Trailing `_` wildcard match arms are **optional** and they indicate a fallback branch when
+/// all previous declarations do not evaluate to true.
+///
+/// # Example
+///
+/// ```
+/// #![feature(cfg_match)]
+///
+/// cfg_match! {
+///     cfg(unix) => {
+///         fn foo() { /* unix specific functionality */ }
+///     }
+///     cfg(target_pointer_width = "32") => {
+///         fn foo() { /* non-unix, 32-bit functionality */ }
+///     }
+///     _ => {
+///         fn foo() { /* fallback implementation */ }
+///     }
+/// }
+/// ```
+#[unstable(feature = "cfg_match", issue = "115585")]
+#[rustc_diagnostic_item = "cfg_match"]
+pub macro cfg_match {
+    // with a final wildcard
+    (
+        $(cfg($initial_meta:meta) => { $($initial_tokens:item)* })+
+        _ => { $($extra_tokens:item)* }
+    ) => {
+        cfg_match! {
+            @__items ();
+            $((($initial_meta) ($($initial_tokens)*)),)+
+            (() ($($extra_tokens)*)),
+        }
+    },
+
+    // without a final wildcard
+    (
+        $(cfg($extra_meta:meta) => { $($extra_tokens:item)* })*
+    ) => {
+        cfg_match! {
+            @__items ();
+            $((($extra_meta) ($($extra_tokens)*)),)*
+        }
+    },
+
+    // Internal and recursive macro to emit all the items
+    //
+    // Collects all the previous cfgs in a list at the beginning, so they can be
+    // negated. After the semicolon is all the remaining items.
+    (@__items ($($_:meta,)*);) => {},
+    (
+        @__items ($($no:meta,)*);
+        (($($yes:meta)?) ($($tokens:item)*)),
+        $($rest:tt,)*
+    ) => {
+        // Emit all items within one block, applying an appropriate #[cfg]. The
+        // #[cfg] will require all `$yes` matchers specified and must also negate
+        // all previous matchers.
+        #[cfg(all(
+            $($yes,)?
+            not(any($($no),*))
+        ))]
+        cfg_match! { @__identity $($tokens)* }
+
+        // Recurse to emit all other items in `$rest`, and when we do so add all
+        // our `$yes` matchers to the list of `$no` matchers as future emissions
+        // will have to negate everything we just matched as well.
+        cfg_match! {
+            @__items ($($no,)* $($yes,)?);
+            $($rest,)*
+        }
+    },
+
+    // Internal macro to make __apply work out right for different match types,
+    // because of how macros match/expand stuff.
+    (@__identity $($tokens:item)*) => {
+        $($tokens)*
+    }
+}
+
 /// Asserts that a boolean expression is `true` at runtime.
 ///
 /// This will invoke the [`panic!`] macro if the provided expression cannot be
@@ -321,95 +409,6 @@ pub macro debug_assert_matches($($arg:tt)*) {
     }
 }
 
-/// A macro for defining `#[cfg]` match-like statements.
-///
-/// It is similar to the `if/elif` C preprocessor macro by allowing definition of a cascade of
-/// `#[cfg]` cases, emitting the implementation which matches first.
-///
-/// This allows you to conveniently provide a long list `#[cfg]`'d blocks of code
-/// without having to rewrite each clause multiple times.
-///
-/// Trailing `_` wildcard match arms are **optional** and they indicate a fallback branch when
-/// all previous declarations do not evaluate to true.
-///
-/// # Example
-///
-/// ```
-/// #![feature(cfg_match)]
-///
-/// cfg_match! {
-///     cfg(unix) => {
-///         fn foo() { /* unix specific functionality */ }
-///     }
-///     cfg(target_pointer_width = "32") => {
-///         fn foo() { /* non-unix, 32-bit functionality */ }
-///     }
-///     _ => {
-///         fn foo() { /* fallback implementation */ }
-///     }
-/// }
-/// ```
-#[macro_export]
-#[unstable(feature = "cfg_match", issue = "115585")]
-#[rustc_diagnostic_item = "cfg_match"]
-macro_rules! cfg_match {
-    // with a final wildcard
-    (
-        $(cfg($initial_meta:meta) => { $($initial_tokens:item)* })+
-        _ => { $($extra_tokens:item)* }
-    ) => {
-        cfg_match! {
-            @__items ();
-            $((($initial_meta) ($($initial_tokens)*)),)+
-            (() ($($extra_tokens)*)),
-        }
-    };
-
-    // without a final wildcard
-    (
-        $(cfg($extra_meta:meta) => { $($extra_tokens:item)* })*
-    ) => {
-        cfg_match! {
-            @__items ();
-            $((($extra_meta) ($($extra_tokens)*)),)*
-        }
-    };
-
-    // Internal and recursive macro to emit all the items
-    //
-    // Collects all the previous cfgs in a list at the beginning, so they can be
-    // negated. After the semicolon is all the remaining items.
-    (@__items ($($_:meta,)*);) => {};
-    (
-        @__items ($($no:meta,)*);
-        (($($yes:meta)?) ($($tokens:item)*)),
-        $($rest:tt,)*
-    ) => {
-        // Emit all items within one block, applying an appropriate #[cfg]. The
-        // #[cfg] will require all `$yes` matchers specified and must also negate
-        // all previous matchers.
-        #[cfg(all(
-            $($yes,)?
-            not(any($($no),*))
-        ))]
-        cfg_match! { @__identity $($tokens)* }
-
-        // Recurse to emit all other items in `$rest`, and when we do so add all
-        // our `$yes` matchers to the list of `$no` matchers as future emissions
-        // will have to negate everything we just matched as well.
-        cfg_match! {
-            @__items ($($no,)* $($yes,)?);
-            $($rest,)*
-        }
-    };
-
-    // Internal macro to make __apply work out right for different match types,
-    // because of how macros match/expand stuff.
-    (@__identity $($tokens:item)*) => {
-        $($tokens)*
-    };
-}
-
 /// Returns whether the given expression matches any of the given patterns.
 ///
 /// Like in a `match` expression, the pattern can be optionally followed by `if`

From c15ad9e7a50ef618ee9c8cb51aa37e51ab65f9f2 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez <guillaume1.gomez@gmail.com>
Date: Wed, 25 Oct 2023 15:53:31 +0200
Subject: [PATCH 080/297] Regenerate intrinsics mapping

---
 src/intrinsic/archs.rs | 1455 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 1455 insertions(+)

diff --git a/src/intrinsic/archs.rs b/src/intrinsic/archs.rs
index e01299d32fda..15d67385c3e9 100644
--- a/src/intrinsic/archs.rs
+++ b/src/intrinsic/archs.rs
@@ -2285,8 +2285,1460 @@ match name {
     "llvm.loongarch.iocsrwr.d" => "__builtin_loongarch_iocsrwr_d",
     "llvm.loongarch.iocsrwr.h" => "__builtin_loongarch_iocsrwr_h",
     "llvm.loongarch.iocsrwr.w" => "__builtin_loongarch_iocsrwr_w",
+    "llvm.loongarch.lasx.vext2xv.d.b" => "__builtin_lasx_vext2xv_d_b",
+    "llvm.loongarch.lasx.vext2xv.d.h" => "__builtin_lasx_vext2xv_d_h",
+    "llvm.loongarch.lasx.vext2xv.d.w" => "__builtin_lasx_vext2xv_d_w",
+    "llvm.loongarch.lasx.vext2xv.du.bu" => "__builtin_lasx_vext2xv_du_bu",
+    "llvm.loongarch.lasx.vext2xv.du.hu" => "__builtin_lasx_vext2xv_du_hu",
+    "llvm.loongarch.lasx.vext2xv.du.wu" => "__builtin_lasx_vext2xv_du_wu",
+    "llvm.loongarch.lasx.vext2xv.h.b" => "__builtin_lasx_vext2xv_h_b",
+    "llvm.loongarch.lasx.vext2xv.hu.bu" => "__builtin_lasx_vext2xv_hu_bu",
+    "llvm.loongarch.lasx.vext2xv.w.b" => "__builtin_lasx_vext2xv_w_b",
+    "llvm.loongarch.lasx.vext2xv.w.h" => "__builtin_lasx_vext2xv_w_h",
+    "llvm.loongarch.lasx.vext2xv.wu.bu" => "__builtin_lasx_vext2xv_wu_bu",
+    "llvm.loongarch.lasx.vext2xv.wu.hu" => "__builtin_lasx_vext2xv_wu_hu",
+    "llvm.loongarch.lasx.xbnz.b" => "__builtin_lasx_xbnz_b",
+    "llvm.loongarch.lasx.xbnz.d" => "__builtin_lasx_xbnz_d",
+    "llvm.loongarch.lasx.xbnz.h" => "__builtin_lasx_xbnz_h",
+    "llvm.loongarch.lasx.xbnz.v" => "__builtin_lasx_xbnz_v",
+    "llvm.loongarch.lasx.xbnz.w" => "__builtin_lasx_xbnz_w",
+    "llvm.loongarch.lasx.xbz.b" => "__builtin_lasx_xbz_b",
+    "llvm.loongarch.lasx.xbz.d" => "__builtin_lasx_xbz_d",
+    "llvm.loongarch.lasx.xbz.h" => "__builtin_lasx_xbz_h",
+    "llvm.loongarch.lasx.xbz.v" => "__builtin_lasx_xbz_v",
+    "llvm.loongarch.lasx.xbz.w" => "__builtin_lasx_xbz_w",
+    "llvm.loongarch.lasx.xvabsd.b" => "__builtin_lasx_xvabsd_b",
+    "llvm.loongarch.lasx.xvabsd.bu" => "__builtin_lasx_xvabsd_bu",
+    "llvm.loongarch.lasx.xvabsd.d" => "__builtin_lasx_xvabsd_d",
+    "llvm.loongarch.lasx.xvabsd.du" => "__builtin_lasx_xvabsd_du",
+    "llvm.loongarch.lasx.xvabsd.h" => "__builtin_lasx_xvabsd_h",
+    "llvm.loongarch.lasx.xvabsd.hu" => "__builtin_lasx_xvabsd_hu",
+    "llvm.loongarch.lasx.xvabsd.w" => "__builtin_lasx_xvabsd_w",
+    "llvm.loongarch.lasx.xvabsd.wu" => "__builtin_lasx_xvabsd_wu",
+    "llvm.loongarch.lasx.xvadd.b" => "__builtin_lasx_xvadd_b",
+    "llvm.loongarch.lasx.xvadd.d" => "__builtin_lasx_xvadd_d",
+    "llvm.loongarch.lasx.xvadd.h" => "__builtin_lasx_xvadd_h",
+    "llvm.loongarch.lasx.xvadd.q" => "__builtin_lasx_xvadd_q",
+    "llvm.loongarch.lasx.xvadd.w" => "__builtin_lasx_xvadd_w",
+    "llvm.loongarch.lasx.xvadda.b" => "__builtin_lasx_xvadda_b",
+    "llvm.loongarch.lasx.xvadda.d" => "__builtin_lasx_xvadda_d",
+    "llvm.loongarch.lasx.xvadda.h" => "__builtin_lasx_xvadda_h",
+    "llvm.loongarch.lasx.xvadda.w" => "__builtin_lasx_xvadda_w",
+    "llvm.loongarch.lasx.xvaddi.bu" => "__builtin_lasx_xvaddi_bu",
+    "llvm.loongarch.lasx.xvaddi.du" => "__builtin_lasx_xvaddi_du",
+    "llvm.loongarch.lasx.xvaddi.hu" => "__builtin_lasx_xvaddi_hu",
+    "llvm.loongarch.lasx.xvaddi.wu" => "__builtin_lasx_xvaddi_wu",
+    "llvm.loongarch.lasx.xvaddwev.d.w" => "__builtin_lasx_xvaddwev_d_w",
+    "llvm.loongarch.lasx.xvaddwev.d.wu" => "__builtin_lasx_xvaddwev_d_wu",
+    "llvm.loongarch.lasx.xvaddwev.d.wu.w" => "__builtin_lasx_xvaddwev_d_wu_w",
+    "llvm.loongarch.lasx.xvaddwev.h.b" => "__builtin_lasx_xvaddwev_h_b",
+    "llvm.loongarch.lasx.xvaddwev.h.bu" => "__builtin_lasx_xvaddwev_h_bu",
+    "llvm.loongarch.lasx.xvaddwev.h.bu.b" => "__builtin_lasx_xvaddwev_h_bu_b",
+    "llvm.loongarch.lasx.xvaddwev.q.d" => "__builtin_lasx_xvaddwev_q_d",
+    "llvm.loongarch.lasx.xvaddwev.q.du" => "__builtin_lasx_xvaddwev_q_du",
+    "llvm.loongarch.lasx.xvaddwev.q.du.d" => "__builtin_lasx_xvaddwev_q_du_d",
+    "llvm.loongarch.lasx.xvaddwev.w.h" => "__builtin_lasx_xvaddwev_w_h",
+    "llvm.loongarch.lasx.xvaddwev.w.hu" => "__builtin_lasx_xvaddwev_w_hu",
+    "llvm.loongarch.lasx.xvaddwev.w.hu.h" => "__builtin_lasx_xvaddwev_w_hu_h",
+    "llvm.loongarch.lasx.xvaddwod.d.w" => "__builtin_lasx_xvaddwod_d_w",
+    "llvm.loongarch.lasx.xvaddwod.d.wu" => "__builtin_lasx_xvaddwod_d_wu",
+    "llvm.loongarch.lasx.xvaddwod.d.wu.w" => "__builtin_lasx_xvaddwod_d_wu_w",
+    "llvm.loongarch.lasx.xvaddwod.h.b" => "__builtin_lasx_xvaddwod_h_b",
+    "llvm.loongarch.lasx.xvaddwod.h.bu" => "__builtin_lasx_xvaddwod_h_bu",
+    "llvm.loongarch.lasx.xvaddwod.h.bu.b" => "__builtin_lasx_xvaddwod_h_bu_b",
+    "llvm.loongarch.lasx.xvaddwod.q.d" => "__builtin_lasx_xvaddwod_q_d",
+    "llvm.loongarch.lasx.xvaddwod.q.du" => "__builtin_lasx_xvaddwod_q_du",
+    "llvm.loongarch.lasx.xvaddwod.q.du.d" => "__builtin_lasx_xvaddwod_q_du_d",
+    "llvm.loongarch.lasx.xvaddwod.w.h" => "__builtin_lasx_xvaddwod_w_h",
+    "llvm.loongarch.lasx.xvaddwod.w.hu" => "__builtin_lasx_xvaddwod_w_hu",
+    "llvm.loongarch.lasx.xvaddwod.w.hu.h" => "__builtin_lasx_xvaddwod_w_hu_h",
+    "llvm.loongarch.lasx.xvand.v" => "__builtin_lasx_xvand_v",
+    "llvm.loongarch.lasx.xvandi.b" => "__builtin_lasx_xvandi_b",
+    "llvm.loongarch.lasx.xvandn.v" => "__builtin_lasx_xvandn_v",
+    "llvm.loongarch.lasx.xvavg.b" => "__builtin_lasx_xvavg_b",
+    "llvm.loongarch.lasx.xvavg.bu" => "__builtin_lasx_xvavg_bu",
+    "llvm.loongarch.lasx.xvavg.d" => "__builtin_lasx_xvavg_d",
+    "llvm.loongarch.lasx.xvavg.du" => "__builtin_lasx_xvavg_du",
+    "llvm.loongarch.lasx.xvavg.h" => "__builtin_lasx_xvavg_h",
+    "llvm.loongarch.lasx.xvavg.hu" => "__builtin_lasx_xvavg_hu",
+    "llvm.loongarch.lasx.xvavg.w" => "__builtin_lasx_xvavg_w",
+    "llvm.loongarch.lasx.xvavg.wu" => "__builtin_lasx_xvavg_wu",
+    "llvm.loongarch.lasx.xvavgr.b" => "__builtin_lasx_xvavgr_b",
+    "llvm.loongarch.lasx.xvavgr.bu" => "__builtin_lasx_xvavgr_bu",
+    "llvm.loongarch.lasx.xvavgr.d" => "__builtin_lasx_xvavgr_d",
+    "llvm.loongarch.lasx.xvavgr.du" => "__builtin_lasx_xvavgr_du",
+    "llvm.loongarch.lasx.xvavgr.h" => "__builtin_lasx_xvavgr_h",
+    "llvm.loongarch.lasx.xvavgr.hu" => "__builtin_lasx_xvavgr_hu",
+    "llvm.loongarch.lasx.xvavgr.w" => "__builtin_lasx_xvavgr_w",
+    "llvm.loongarch.lasx.xvavgr.wu" => "__builtin_lasx_xvavgr_wu",
+    "llvm.loongarch.lasx.xvbitclr.b" => "__builtin_lasx_xvbitclr_b",
+    "llvm.loongarch.lasx.xvbitclr.d" => "__builtin_lasx_xvbitclr_d",
+    "llvm.loongarch.lasx.xvbitclr.h" => "__builtin_lasx_xvbitclr_h",
+    "llvm.loongarch.lasx.xvbitclr.w" => "__builtin_lasx_xvbitclr_w",
+    "llvm.loongarch.lasx.xvbitclri.b" => "__builtin_lasx_xvbitclri_b",
+    "llvm.loongarch.lasx.xvbitclri.d" => "__builtin_lasx_xvbitclri_d",
+    "llvm.loongarch.lasx.xvbitclri.h" => "__builtin_lasx_xvbitclri_h",
+    "llvm.loongarch.lasx.xvbitclri.w" => "__builtin_lasx_xvbitclri_w",
+    "llvm.loongarch.lasx.xvbitrev.b" => "__builtin_lasx_xvbitrev_b",
+    "llvm.loongarch.lasx.xvbitrev.d" => "__builtin_lasx_xvbitrev_d",
+    "llvm.loongarch.lasx.xvbitrev.h" => "__builtin_lasx_xvbitrev_h",
+    "llvm.loongarch.lasx.xvbitrev.w" => "__builtin_lasx_xvbitrev_w",
+    "llvm.loongarch.lasx.xvbitrevi.b" => "__builtin_lasx_xvbitrevi_b",
+    "llvm.loongarch.lasx.xvbitrevi.d" => "__builtin_lasx_xvbitrevi_d",
+    "llvm.loongarch.lasx.xvbitrevi.h" => "__builtin_lasx_xvbitrevi_h",
+    "llvm.loongarch.lasx.xvbitrevi.w" => "__builtin_lasx_xvbitrevi_w",
+    "llvm.loongarch.lasx.xvbitsel.v" => "__builtin_lasx_xvbitsel_v",
+    "llvm.loongarch.lasx.xvbitseli.b" => "__builtin_lasx_xvbitseli_b",
+    "llvm.loongarch.lasx.xvbitset.b" => "__builtin_lasx_xvbitset_b",
+    "llvm.loongarch.lasx.xvbitset.d" => "__builtin_lasx_xvbitset_d",
+    "llvm.loongarch.lasx.xvbitset.h" => "__builtin_lasx_xvbitset_h",
+    "llvm.loongarch.lasx.xvbitset.w" => "__builtin_lasx_xvbitset_w",
+    "llvm.loongarch.lasx.xvbitseti.b" => "__builtin_lasx_xvbitseti_b",
+    "llvm.loongarch.lasx.xvbitseti.d" => "__builtin_lasx_xvbitseti_d",
+    "llvm.loongarch.lasx.xvbitseti.h" => "__builtin_lasx_xvbitseti_h",
+    "llvm.loongarch.lasx.xvbitseti.w" => "__builtin_lasx_xvbitseti_w",
+    "llvm.loongarch.lasx.xvbsll.v" => "__builtin_lasx_xvbsll_v",
+    "llvm.loongarch.lasx.xvbsrl.v" => "__builtin_lasx_xvbsrl_v",
+    "llvm.loongarch.lasx.xvclo.b" => "__builtin_lasx_xvclo_b",
+    "llvm.loongarch.lasx.xvclo.d" => "__builtin_lasx_xvclo_d",
+    "llvm.loongarch.lasx.xvclo.h" => "__builtin_lasx_xvclo_h",
+    "llvm.loongarch.lasx.xvclo.w" => "__builtin_lasx_xvclo_w",
+    "llvm.loongarch.lasx.xvclz.b" => "__builtin_lasx_xvclz_b",
+    "llvm.loongarch.lasx.xvclz.d" => "__builtin_lasx_xvclz_d",
+    "llvm.loongarch.lasx.xvclz.h" => "__builtin_lasx_xvclz_h",
+    "llvm.loongarch.lasx.xvclz.w" => "__builtin_lasx_xvclz_w",
+    "llvm.loongarch.lasx.xvdiv.b" => "__builtin_lasx_xvdiv_b",
+    "llvm.loongarch.lasx.xvdiv.bu" => "__builtin_lasx_xvdiv_bu",
+    "llvm.loongarch.lasx.xvdiv.d" => "__builtin_lasx_xvdiv_d",
+    "llvm.loongarch.lasx.xvdiv.du" => "__builtin_lasx_xvdiv_du",
+    "llvm.loongarch.lasx.xvdiv.h" => "__builtin_lasx_xvdiv_h",
+    "llvm.loongarch.lasx.xvdiv.hu" => "__builtin_lasx_xvdiv_hu",
+    "llvm.loongarch.lasx.xvdiv.w" => "__builtin_lasx_xvdiv_w",
+    "llvm.loongarch.lasx.xvdiv.wu" => "__builtin_lasx_xvdiv_wu",
+    "llvm.loongarch.lasx.xvexth.d.w" => "__builtin_lasx_xvexth_d_w",
+    "llvm.loongarch.lasx.xvexth.du.wu" => "__builtin_lasx_xvexth_du_wu",
+    "llvm.loongarch.lasx.xvexth.h.b" => "__builtin_lasx_xvexth_h_b",
+    "llvm.loongarch.lasx.xvexth.hu.bu" => "__builtin_lasx_xvexth_hu_bu",
+    "llvm.loongarch.lasx.xvexth.q.d" => "__builtin_lasx_xvexth_q_d",
+    "llvm.loongarch.lasx.xvexth.qu.du" => "__builtin_lasx_xvexth_qu_du",
+    "llvm.loongarch.lasx.xvexth.w.h" => "__builtin_lasx_xvexth_w_h",
+    "llvm.loongarch.lasx.xvexth.wu.hu" => "__builtin_lasx_xvexth_wu_hu",
+    "llvm.loongarch.lasx.xvextl.q.d" => "__builtin_lasx_xvextl_q_d",
+    "llvm.loongarch.lasx.xvextl.qu.du" => "__builtin_lasx_xvextl_qu_du",
+    "llvm.loongarch.lasx.xvextrins.b" => "__builtin_lasx_xvextrins_b",
+    "llvm.loongarch.lasx.xvextrins.d" => "__builtin_lasx_xvextrins_d",
+    "llvm.loongarch.lasx.xvextrins.h" => "__builtin_lasx_xvextrins_h",
+    "llvm.loongarch.lasx.xvextrins.w" => "__builtin_lasx_xvextrins_w",
+    "llvm.loongarch.lasx.xvfadd.d" => "__builtin_lasx_xvfadd_d",
+    "llvm.loongarch.lasx.xvfadd.s" => "__builtin_lasx_xvfadd_s",
+    "llvm.loongarch.lasx.xvfclass.d" => "__builtin_lasx_xvfclass_d",
+    "llvm.loongarch.lasx.xvfclass.s" => "__builtin_lasx_xvfclass_s",
+    "llvm.loongarch.lasx.xvfcmp.caf.d" => "__builtin_lasx_xvfcmp_caf_d",
+    "llvm.loongarch.lasx.xvfcmp.caf.s" => "__builtin_lasx_xvfcmp_caf_s",
+    "llvm.loongarch.lasx.xvfcmp.ceq.d" => "__builtin_lasx_xvfcmp_ceq_d",
+    "llvm.loongarch.lasx.xvfcmp.ceq.s" => "__builtin_lasx_xvfcmp_ceq_s",
+    "llvm.loongarch.lasx.xvfcmp.cle.d" => "__builtin_lasx_xvfcmp_cle_d",
+    "llvm.loongarch.lasx.xvfcmp.cle.s" => "__builtin_lasx_xvfcmp_cle_s",
+    "llvm.loongarch.lasx.xvfcmp.clt.d" => "__builtin_lasx_xvfcmp_clt_d",
+    "llvm.loongarch.lasx.xvfcmp.clt.s" => "__builtin_lasx_xvfcmp_clt_s",
+    "llvm.loongarch.lasx.xvfcmp.cne.d" => "__builtin_lasx_xvfcmp_cne_d",
+    "llvm.loongarch.lasx.xvfcmp.cne.s" => "__builtin_lasx_xvfcmp_cne_s",
+    "llvm.loongarch.lasx.xvfcmp.cor.d" => "__builtin_lasx_xvfcmp_cor_d",
+    "llvm.loongarch.lasx.xvfcmp.cor.s" => "__builtin_lasx_xvfcmp_cor_s",
+    "llvm.loongarch.lasx.xvfcmp.cueq.d" => "__builtin_lasx_xvfcmp_cueq_d",
+    "llvm.loongarch.lasx.xvfcmp.cueq.s" => "__builtin_lasx_xvfcmp_cueq_s",
+    "llvm.loongarch.lasx.xvfcmp.cule.d" => "__builtin_lasx_xvfcmp_cule_d",
+    "llvm.loongarch.lasx.xvfcmp.cule.s" => "__builtin_lasx_xvfcmp_cule_s",
+    "llvm.loongarch.lasx.xvfcmp.cult.d" => "__builtin_lasx_xvfcmp_cult_d",
+    "llvm.loongarch.lasx.xvfcmp.cult.s" => "__builtin_lasx_xvfcmp_cult_s",
+    "llvm.loongarch.lasx.xvfcmp.cun.d" => "__builtin_lasx_xvfcmp_cun_d",
+    "llvm.loongarch.lasx.xvfcmp.cun.s" => "__builtin_lasx_xvfcmp_cun_s",
+    "llvm.loongarch.lasx.xvfcmp.cune.d" => "__builtin_lasx_xvfcmp_cune_d",
+    "llvm.loongarch.lasx.xvfcmp.cune.s" => "__builtin_lasx_xvfcmp_cune_s",
+    "llvm.loongarch.lasx.xvfcmp.saf.d" => "__builtin_lasx_xvfcmp_saf_d",
+    "llvm.loongarch.lasx.xvfcmp.saf.s" => "__builtin_lasx_xvfcmp_saf_s",
+    "llvm.loongarch.lasx.xvfcmp.seq.d" => "__builtin_lasx_xvfcmp_seq_d",
+    "llvm.loongarch.lasx.xvfcmp.seq.s" => "__builtin_lasx_xvfcmp_seq_s",
+    "llvm.loongarch.lasx.xvfcmp.sle.d" => "__builtin_lasx_xvfcmp_sle_d",
+    "llvm.loongarch.lasx.xvfcmp.sle.s" => "__builtin_lasx_xvfcmp_sle_s",
+    "llvm.loongarch.lasx.xvfcmp.slt.d" => "__builtin_lasx_xvfcmp_slt_d",
+    "llvm.loongarch.lasx.xvfcmp.slt.s" => "__builtin_lasx_xvfcmp_slt_s",
+    "llvm.loongarch.lasx.xvfcmp.sne.d" => "__builtin_lasx_xvfcmp_sne_d",
+    "llvm.loongarch.lasx.xvfcmp.sne.s" => "__builtin_lasx_xvfcmp_sne_s",
+    "llvm.loongarch.lasx.xvfcmp.sor.d" => "__builtin_lasx_xvfcmp_sor_d",
+    "llvm.loongarch.lasx.xvfcmp.sor.s" => "__builtin_lasx_xvfcmp_sor_s",
+    "llvm.loongarch.lasx.xvfcmp.sueq.d" => "__builtin_lasx_xvfcmp_sueq_d",
+    "llvm.loongarch.lasx.xvfcmp.sueq.s" => "__builtin_lasx_xvfcmp_sueq_s",
+    "llvm.loongarch.lasx.xvfcmp.sule.d" => "__builtin_lasx_xvfcmp_sule_d",
+    "llvm.loongarch.lasx.xvfcmp.sule.s" => "__builtin_lasx_xvfcmp_sule_s",
+    "llvm.loongarch.lasx.xvfcmp.sult.d" => "__builtin_lasx_xvfcmp_sult_d",
+    "llvm.loongarch.lasx.xvfcmp.sult.s" => "__builtin_lasx_xvfcmp_sult_s",
+    "llvm.loongarch.lasx.xvfcmp.sun.d" => "__builtin_lasx_xvfcmp_sun_d",
+    "llvm.loongarch.lasx.xvfcmp.sun.s" => "__builtin_lasx_xvfcmp_sun_s",
+    "llvm.loongarch.lasx.xvfcmp.sune.d" => "__builtin_lasx_xvfcmp_sune_d",
+    "llvm.loongarch.lasx.xvfcmp.sune.s" => "__builtin_lasx_xvfcmp_sune_s",
+    "llvm.loongarch.lasx.xvfcvt.h.s" => "__builtin_lasx_xvfcvt_h_s",
+    "llvm.loongarch.lasx.xvfcvt.s.d" => "__builtin_lasx_xvfcvt_s_d",
+    "llvm.loongarch.lasx.xvfcvth.d.s" => "__builtin_lasx_xvfcvth_d_s",
+    "llvm.loongarch.lasx.xvfcvth.s.h" => "__builtin_lasx_xvfcvth_s_h",
+    "llvm.loongarch.lasx.xvfcvtl.d.s" => "__builtin_lasx_xvfcvtl_d_s",
+    "llvm.loongarch.lasx.xvfcvtl.s.h" => "__builtin_lasx_xvfcvtl_s_h",
+    "llvm.loongarch.lasx.xvfdiv.d" => "__builtin_lasx_xvfdiv_d",
+    "llvm.loongarch.lasx.xvfdiv.s" => "__builtin_lasx_xvfdiv_s",
+    "llvm.loongarch.lasx.xvffint.d.l" => "__builtin_lasx_xvffint_d_l",
+    "llvm.loongarch.lasx.xvffint.d.lu" => "__builtin_lasx_xvffint_d_lu",
+    "llvm.loongarch.lasx.xvffint.s.l" => "__builtin_lasx_xvffint_s_l",
+    "llvm.loongarch.lasx.xvffint.s.w" => "__builtin_lasx_xvffint_s_w",
+    "llvm.loongarch.lasx.xvffint.s.wu" => "__builtin_lasx_xvffint_s_wu",
+    "llvm.loongarch.lasx.xvffinth.d.w" => "__builtin_lasx_xvffinth_d_w",
+    "llvm.loongarch.lasx.xvffintl.d.w" => "__builtin_lasx_xvffintl_d_w",
+    "llvm.loongarch.lasx.xvflogb.d" => "__builtin_lasx_xvflogb_d",
+    "llvm.loongarch.lasx.xvflogb.s" => "__builtin_lasx_xvflogb_s",
+    "llvm.loongarch.lasx.xvfmadd.d" => "__builtin_lasx_xvfmadd_d",
+    "llvm.loongarch.lasx.xvfmadd.s" => "__builtin_lasx_xvfmadd_s",
+    "llvm.loongarch.lasx.xvfmax.d" => "__builtin_lasx_xvfmax_d",
+    "llvm.loongarch.lasx.xvfmax.s" => "__builtin_lasx_xvfmax_s",
+    "llvm.loongarch.lasx.xvfmaxa.d" => "__builtin_lasx_xvfmaxa_d",
+    "llvm.loongarch.lasx.xvfmaxa.s" => "__builtin_lasx_xvfmaxa_s",
+    "llvm.loongarch.lasx.xvfmin.d" => "__builtin_lasx_xvfmin_d",
+    "llvm.loongarch.lasx.xvfmin.s" => "__builtin_lasx_xvfmin_s",
+    "llvm.loongarch.lasx.xvfmina.d" => "__builtin_lasx_xvfmina_d",
+    "llvm.loongarch.lasx.xvfmina.s" => "__builtin_lasx_xvfmina_s",
+    "llvm.loongarch.lasx.xvfmsub.d" => "__builtin_lasx_xvfmsub_d",
+    "llvm.loongarch.lasx.xvfmsub.s" => "__builtin_lasx_xvfmsub_s",
+    "llvm.loongarch.lasx.xvfmul.d" => "__builtin_lasx_xvfmul_d",
+    "llvm.loongarch.lasx.xvfmul.s" => "__builtin_lasx_xvfmul_s",
+    "llvm.loongarch.lasx.xvfnmadd.d" => "__builtin_lasx_xvfnmadd_d",
+    "llvm.loongarch.lasx.xvfnmadd.s" => "__builtin_lasx_xvfnmadd_s",
+    "llvm.loongarch.lasx.xvfnmsub.d" => "__builtin_lasx_xvfnmsub_d",
+    "llvm.loongarch.lasx.xvfnmsub.s" => "__builtin_lasx_xvfnmsub_s",
+    "llvm.loongarch.lasx.xvfrecip.d" => "__builtin_lasx_xvfrecip_d",
+    "llvm.loongarch.lasx.xvfrecip.s" => "__builtin_lasx_xvfrecip_s",
+    "llvm.loongarch.lasx.xvfrint.d" => "__builtin_lasx_xvfrint_d",
+    "llvm.loongarch.lasx.xvfrint.s" => "__builtin_lasx_xvfrint_s",
+    "llvm.loongarch.lasx.xvfrintrm.d" => "__builtin_lasx_xvfrintrm_d",
+    "llvm.loongarch.lasx.xvfrintrm.s" => "__builtin_lasx_xvfrintrm_s",
+    "llvm.loongarch.lasx.xvfrintrne.d" => "__builtin_lasx_xvfrintrne_d",
+    "llvm.loongarch.lasx.xvfrintrne.s" => "__builtin_lasx_xvfrintrne_s",
+    "llvm.loongarch.lasx.xvfrintrp.d" => "__builtin_lasx_xvfrintrp_d",
+    "llvm.loongarch.lasx.xvfrintrp.s" => "__builtin_lasx_xvfrintrp_s",
+    "llvm.loongarch.lasx.xvfrintrz.d" => "__builtin_lasx_xvfrintrz_d",
+    "llvm.loongarch.lasx.xvfrintrz.s" => "__builtin_lasx_xvfrintrz_s",
+    "llvm.loongarch.lasx.xvfrsqrt.d" => "__builtin_lasx_xvfrsqrt_d",
+    "llvm.loongarch.lasx.xvfrsqrt.s" => "__builtin_lasx_xvfrsqrt_s",
+    "llvm.loongarch.lasx.xvfrstp.b" => "__builtin_lasx_xvfrstp_b",
+    "llvm.loongarch.lasx.xvfrstp.h" => "__builtin_lasx_xvfrstp_h",
+    "llvm.loongarch.lasx.xvfrstpi.b" => "__builtin_lasx_xvfrstpi_b",
+    "llvm.loongarch.lasx.xvfrstpi.h" => "__builtin_lasx_xvfrstpi_h",
+    "llvm.loongarch.lasx.xvfsqrt.d" => "__builtin_lasx_xvfsqrt_d",
+    "llvm.loongarch.lasx.xvfsqrt.s" => "__builtin_lasx_xvfsqrt_s",
+    "llvm.loongarch.lasx.xvfsub.d" => "__builtin_lasx_xvfsub_d",
+    "llvm.loongarch.lasx.xvfsub.s" => "__builtin_lasx_xvfsub_s",
+    "llvm.loongarch.lasx.xvftint.l.d" => "__builtin_lasx_xvftint_l_d",
+    "llvm.loongarch.lasx.xvftint.lu.d" => "__builtin_lasx_xvftint_lu_d",
+    "llvm.loongarch.lasx.xvftint.w.d" => "__builtin_lasx_xvftint_w_d",
+    "llvm.loongarch.lasx.xvftint.w.s" => "__builtin_lasx_xvftint_w_s",
+    "llvm.loongarch.lasx.xvftint.wu.s" => "__builtin_lasx_xvftint_wu_s",
+    "llvm.loongarch.lasx.xvftinth.l.s" => "__builtin_lasx_xvftinth_l_s",
+    "llvm.loongarch.lasx.xvftintl.l.s" => "__builtin_lasx_xvftintl_l_s",
+    "llvm.loongarch.lasx.xvftintrm.l.d" => "__builtin_lasx_xvftintrm_l_d",
+    "llvm.loongarch.lasx.xvftintrm.w.d" => "__builtin_lasx_xvftintrm_w_d",
+    "llvm.loongarch.lasx.xvftintrm.w.s" => "__builtin_lasx_xvftintrm_w_s",
+    "llvm.loongarch.lasx.xvftintrmh.l.s" => "__builtin_lasx_xvftintrmh_l_s",
+    "llvm.loongarch.lasx.xvftintrml.l.s" => "__builtin_lasx_xvftintrml_l_s",
+    "llvm.loongarch.lasx.xvftintrne.l.d" => "__builtin_lasx_xvftintrne_l_d",
+    "llvm.loongarch.lasx.xvftintrne.w.d" => "__builtin_lasx_xvftintrne_w_d",
+    "llvm.loongarch.lasx.xvftintrne.w.s" => "__builtin_lasx_xvftintrne_w_s",
+    "llvm.loongarch.lasx.xvftintrneh.l.s" => "__builtin_lasx_xvftintrneh_l_s",
+    "llvm.loongarch.lasx.xvftintrnel.l.s" => "__builtin_lasx_xvftintrnel_l_s",
+    "llvm.loongarch.lasx.xvftintrp.l.d" => "__builtin_lasx_xvftintrp_l_d",
+    "llvm.loongarch.lasx.xvftintrp.w.d" => "__builtin_lasx_xvftintrp_w_d",
+    "llvm.loongarch.lasx.xvftintrp.w.s" => "__builtin_lasx_xvftintrp_w_s",
+    "llvm.loongarch.lasx.xvftintrph.l.s" => "__builtin_lasx_xvftintrph_l_s",
+    "llvm.loongarch.lasx.xvftintrpl.l.s" => "__builtin_lasx_xvftintrpl_l_s",
+    "llvm.loongarch.lasx.xvftintrz.l.d" => "__builtin_lasx_xvftintrz_l_d",
+    "llvm.loongarch.lasx.xvftintrz.lu.d" => "__builtin_lasx_xvftintrz_lu_d",
+    "llvm.loongarch.lasx.xvftintrz.w.d" => "__builtin_lasx_xvftintrz_w_d",
+    "llvm.loongarch.lasx.xvftintrz.w.s" => "__builtin_lasx_xvftintrz_w_s",
+    "llvm.loongarch.lasx.xvftintrz.wu.s" => "__builtin_lasx_xvftintrz_wu_s",
+    "llvm.loongarch.lasx.xvftintrzh.l.s" => "__builtin_lasx_xvftintrzh_l_s",
+    "llvm.loongarch.lasx.xvftintrzl.l.s" => "__builtin_lasx_xvftintrzl_l_s",
+    "llvm.loongarch.lasx.xvhaddw.d.w" => "__builtin_lasx_xvhaddw_d_w",
+    "llvm.loongarch.lasx.xvhaddw.du.wu" => "__builtin_lasx_xvhaddw_du_wu",
+    "llvm.loongarch.lasx.xvhaddw.h.b" => "__builtin_lasx_xvhaddw_h_b",
+    "llvm.loongarch.lasx.xvhaddw.hu.bu" => "__builtin_lasx_xvhaddw_hu_bu",
+    "llvm.loongarch.lasx.xvhaddw.q.d" => "__builtin_lasx_xvhaddw_q_d",
+    "llvm.loongarch.lasx.xvhaddw.qu.du" => "__builtin_lasx_xvhaddw_qu_du",
+    "llvm.loongarch.lasx.xvhaddw.w.h" => "__builtin_lasx_xvhaddw_w_h",
+    "llvm.loongarch.lasx.xvhaddw.wu.hu" => "__builtin_lasx_xvhaddw_wu_hu",
+    "llvm.loongarch.lasx.xvhsubw.d.w" => "__builtin_lasx_xvhsubw_d_w",
+    "llvm.loongarch.lasx.xvhsubw.du.wu" => "__builtin_lasx_xvhsubw_du_wu",
+    "llvm.loongarch.lasx.xvhsubw.h.b" => "__builtin_lasx_xvhsubw_h_b",
+    "llvm.loongarch.lasx.xvhsubw.hu.bu" => "__builtin_lasx_xvhsubw_hu_bu",
+    "llvm.loongarch.lasx.xvhsubw.q.d" => "__builtin_lasx_xvhsubw_q_d",
+    "llvm.loongarch.lasx.xvhsubw.qu.du" => "__builtin_lasx_xvhsubw_qu_du",
+    "llvm.loongarch.lasx.xvhsubw.w.h" => "__builtin_lasx_xvhsubw_w_h",
+    "llvm.loongarch.lasx.xvhsubw.wu.hu" => "__builtin_lasx_xvhsubw_wu_hu",
+    "llvm.loongarch.lasx.xvilvh.b" => "__builtin_lasx_xvilvh_b",
+    "llvm.loongarch.lasx.xvilvh.d" => "__builtin_lasx_xvilvh_d",
+    "llvm.loongarch.lasx.xvilvh.h" => "__builtin_lasx_xvilvh_h",
+    "llvm.loongarch.lasx.xvilvh.w" => "__builtin_lasx_xvilvh_w",
+    "llvm.loongarch.lasx.xvilvl.b" => "__builtin_lasx_xvilvl_b",
+    "llvm.loongarch.lasx.xvilvl.d" => "__builtin_lasx_xvilvl_d",
+    "llvm.loongarch.lasx.xvilvl.h" => "__builtin_lasx_xvilvl_h",
+    "llvm.loongarch.lasx.xvilvl.w" => "__builtin_lasx_xvilvl_w",
+    "llvm.loongarch.lasx.xvinsgr2vr.d" => "__builtin_lasx_xvinsgr2vr_d",
+    "llvm.loongarch.lasx.xvinsgr2vr.w" => "__builtin_lasx_xvinsgr2vr_w",
+    "llvm.loongarch.lasx.xvinsve0.d" => "__builtin_lasx_xvinsve0_d",
+    "llvm.loongarch.lasx.xvinsve0.w" => "__builtin_lasx_xvinsve0_w",
+    "llvm.loongarch.lasx.xvld" => "__builtin_lasx_xvld",
+    "llvm.loongarch.lasx.xvldi" => "__builtin_lasx_xvldi",
+    "llvm.loongarch.lasx.xvldrepl.b" => "__builtin_lasx_xvldrepl_b",
+    "llvm.loongarch.lasx.xvldrepl.d" => "__builtin_lasx_xvldrepl_d",
+    "llvm.loongarch.lasx.xvldrepl.h" => "__builtin_lasx_xvldrepl_h",
+    "llvm.loongarch.lasx.xvldrepl.w" => "__builtin_lasx_xvldrepl_w",
+    "llvm.loongarch.lasx.xvldx" => "__builtin_lasx_xvldx",
+    "llvm.loongarch.lasx.xvmadd.b" => "__builtin_lasx_xvmadd_b",
+    "llvm.loongarch.lasx.xvmadd.d" => "__builtin_lasx_xvmadd_d",
+    "llvm.loongarch.lasx.xvmadd.h" => "__builtin_lasx_xvmadd_h",
+    "llvm.loongarch.lasx.xvmadd.w" => "__builtin_lasx_xvmadd_w",
+    "llvm.loongarch.lasx.xvmaddwev.d.w" => "__builtin_lasx_xvmaddwev_d_w",
+    "llvm.loongarch.lasx.xvmaddwev.d.wu" => "__builtin_lasx_xvmaddwev_d_wu",
+    "llvm.loongarch.lasx.xvmaddwev.d.wu.w" => "__builtin_lasx_xvmaddwev_d_wu_w",
+    "llvm.loongarch.lasx.xvmaddwev.h.b" => "__builtin_lasx_xvmaddwev_h_b",
+    "llvm.loongarch.lasx.xvmaddwev.h.bu" => "__builtin_lasx_xvmaddwev_h_bu",
+    "llvm.loongarch.lasx.xvmaddwev.h.bu.b" => "__builtin_lasx_xvmaddwev_h_bu_b",
+    "llvm.loongarch.lasx.xvmaddwev.q.d" => "__builtin_lasx_xvmaddwev_q_d",
+    "llvm.loongarch.lasx.xvmaddwev.q.du" => "__builtin_lasx_xvmaddwev_q_du",
+    "llvm.loongarch.lasx.xvmaddwev.q.du.d" => "__builtin_lasx_xvmaddwev_q_du_d",
+    "llvm.loongarch.lasx.xvmaddwev.w.h" => "__builtin_lasx_xvmaddwev_w_h",
+    "llvm.loongarch.lasx.xvmaddwev.w.hu" => "__builtin_lasx_xvmaddwev_w_hu",
+    "llvm.loongarch.lasx.xvmaddwev.w.hu.h" => "__builtin_lasx_xvmaddwev_w_hu_h",
+    "llvm.loongarch.lasx.xvmaddwod.d.w" => "__builtin_lasx_xvmaddwod_d_w",
+    "llvm.loongarch.lasx.xvmaddwod.d.wu" => "__builtin_lasx_xvmaddwod_d_wu",
+    "llvm.loongarch.lasx.xvmaddwod.d.wu.w" => "__builtin_lasx_xvmaddwod_d_wu_w",
+    "llvm.loongarch.lasx.xvmaddwod.h.b" => "__builtin_lasx_xvmaddwod_h_b",
+    "llvm.loongarch.lasx.xvmaddwod.h.bu" => "__builtin_lasx_xvmaddwod_h_bu",
+    "llvm.loongarch.lasx.xvmaddwod.h.bu.b" => "__builtin_lasx_xvmaddwod_h_bu_b",
+    "llvm.loongarch.lasx.xvmaddwod.q.d" => "__builtin_lasx_xvmaddwod_q_d",
+    "llvm.loongarch.lasx.xvmaddwod.q.du" => "__builtin_lasx_xvmaddwod_q_du",
+    "llvm.loongarch.lasx.xvmaddwod.q.du.d" => "__builtin_lasx_xvmaddwod_q_du_d",
+    "llvm.loongarch.lasx.xvmaddwod.w.h" => "__builtin_lasx_xvmaddwod_w_h",
+    "llvm.loongarch.lasx.xvmaddwod.w.hu" => "__builtin_lasx_xvmaddwod_w_hu",
+    "llvm.loongarch.lasx.xvmaddwod.w.hu.h" => "__builtin_lasx_xvmaddwod_w_hu_h",
+    "llvm.loongarch.lasx.xvmax.b" => "__builtin_lasx_xvmax_b",
+    "llvm.loongarch.lasx.xvmax.bu" => "__builtin_lasx_xvmax_bu",
+    "llvm.loongarch.lasx.xvmax.d" => "__builtin_lasx_xvmax_d",
+    "llvm.loongarch.lasx.xvmax.du" => "__builtin_lasx_xvmax_du",
+    "llvm.loongarch.lasx.xvmax.h" => "__builtin_lasx_xvmax_h",
+    "llvm.loongarch.lasx.xvmax.hu" => "__builtin_lasx_xvmax_hu",
+    "llvm.loongarch.lasx.xvmax.w" => "__builtin_lasx_xvmax_w",
+    "llvm.loongarch.lasx.xvmax.wu" => "__builtin_lasx_xvmax_wu",
+    "llvm.loongarch.lasx.xvmaxi.b" => "__builtin_lasx_xvmaxi_b",
+    "llvm.loongarch.lasx.xvmaxi.bu" => "__builtin_lasx_xvmaxi_bu",
+    "llvm.loongarch.lasx.xvmaxi.d" => "__builtin_lasx_xvmaxi_d",
+    "llvm.loongarch.lasx.xvmaxi.du" => "__builtin_lasx_xvmaxi_du",
+    "llvm.loongarch.lasx.xvmaxi.h" => "__builtin_lasx_xvmaxi_h",
+    "llvm.loongarch.lasx.xvmaxi.hu" => "__builtin_lasx_xvmaxi_hu",
+    "llvm.loongarch.lasx.xvmaxi.w" => "__builtin_lasx_xvmaxi_w",
+    "llvm.loongarch.lasx.xvmaxi.wu" => "__builtin_lasx_xvmaxi_wu",
+    "llvm.loongarch.lasx.xvmin.b" => "__builtin_lasx_xvmin_b",
+    "llvm.loongarch.lasx.xvmin.bu" => "__builtin_lasx_xvmin_bu",
+    "llvm.loongarch.lasx.xvmin.d" => "__builtin_lasx_xvmin_d",
+    "llvm.loongarch.lasx.xvmin.du" => "__builtin_lasx_xvmin_du",
+    "llvm.loongarch.lasx.xvmin.h" => "__builtin_lasx_xvmin_h",
+    "llvm.loongarch.lasx.xvmin.hu" => "__builtin_lasx_xvmin_hu",
+    "llvm.loongarch.lasx.xvmin.w" => "__builtin_lasx_xvmin_w",
+    "llvm.loongarch.lasx.xvmin.wu" => "__builtin_lasx_xvmin_wu",
+    "llvm.loongarch.lasx.xvmini.b" => "__builtin_lasx_xvmini_b",
+    "llvm.loongarch.lasx.xvmini.bu" => "__builtin_lasx_xvmini_bu",
+    "llvm.loongarch.lasx.xvmini.d" => "__builtin_lasx_xvmini_d",
+    "llvm.loongarch.lasx.xvmini.du" => "__builtin_lasx_xvmini_du",
+    "llvm.loongarch.lasx.xvmini.h" => "__builtin_lasx_xvmini_h",
+    "llvm.loongarch.lasx.xvmini.hu" => "__builtin_lasx_xvmini_hu",
+    "llvm.loongarch.lasx.xvmini.w" => "__builtin_lasx_xvmini_w",
+    "llvm.loongarch.lasx.xvmini.wu" => "__builtin_lasx_xvmini_wu",
+    "llvm.loongarch.lasx.xvmod.b" => "__builtin_lasx_xvmod_b",
+    "llvm.loongarch.lasx.xvmod.bu" => "__builtin_lasx_xvmod_bu",
+    "llvm.loongarch.lasx.xvmod.d" => "__builtin_lasx_xvmod_d",
+    "llvm.loongarch.lasx.xvmod.du" => "__builtin_lasx_xvmod_du",
+    "llvm.loongarch.lasx.xvmod.h" => "__builtin_lasx_xvmod_h",
+    "llvm.loongarch.lasx.xvmod.hu" => "__builtin_lasx_xvmod_hu",
+    "llvm.loongarch.lasx.xvmod.w" => "__builtin_lasx_xvmod_w",
+    "llvm.loongarch.lasx.xvmod.wu" => "__builtin_lasx_xvmod_wu",
+    "llvm.loongarch.lasx.xvmskgez.b" => "__builtin_lasx_xvmskgez_b",
+    "llvm.loongarch.lasx.xvmskltz.b" => "__builtin_lasx_xvmskltz_b",
+    "llvm.loongarch.lasx.xvmskltz.d" => "__builtin_lasx_xvmskltz_d",
+    "llvm.loongarch.lasx.xvmskltz.h" => "__builtin_lasx_xvmskltz_h",
+    "llvm.loongarch.lasx.xvmskltz.w" => "__builtin_lasx_xvmskltz_w",
+    "llvm.loongarch.lasx.xvmsknz.b" => "__builtin_lasx_xvmsknz_b",
+    "llvm.loongarch.lasx.xvmsub.b" => "__builtin_lasx_xvmsub_b",
+    "llvm.loongarch.lasx.xvmsub.d" => "__builtin_lasx_xvmsub_d",
+    "llvm.loongarch.lasx.xvmsub.h" => "__builtin_lasx_xvmsub_h",
+    "llvm.loongarch.lasx.xvmsub.w" => "__builtin_lasx_xvmsub_w",
+    "llvm.loongarch.lasx.xvmuh.b" => "__builtin_lasx_xvmuh_b",
+    "llvm.loongarch.lasx.xvmuh.bu" => "__builtin_lasx_xvmuh_bu",
+    "llvm.loongarch.lasx.xvmuh.d" => "__builtin_lasx_xvmuh_d",
+    "llvm.loongarch.lasx.xvmuh.du" => "__builtin_lasx_xvmuh_du",
+    "llvm.loongarch.lasx.xvmuh.h" => "__builtin_lasx_xvmuh_h",
+    "llvm.loongarch.lasx.xvmuh.hu" => "__builtin_lasx_xvmuh_hu",
+    "llvm.loongarch.lasx.xvmuh.w" => "__builtin_lasx_xvmuh_w",
+    "llvm.loongarch.lasx.xvmuh.wu" => "__builtin_lasx_xvmuh_wu",
+    "llvm.loongarch.lasx.xvmul.b" => "__builtin_lasx_xvmul_b",
+    "llvm.loongarch.lasx.xvmul.d" => "__builtin_lasx_xvmul_d",
+    "llvm.loongarch.lasx.xvmul.h" => "__builtin_lasx_xvmul_h",
+    "llvm.loongarch.lasx.xvmul.w" => "__builtin_lasx_xvmul_w",
+    "llvm.loongarch.lasx.xvmulwev.d.w" => "__builtin_lasx_xvmulwev_d_w",
+    "llvm.loongarch.lasx.xvmulwev.d.wu" => "__builtin_lasx_xvmulwev_d_wu",
+    "llvm.loongarch.lasx.xvmulwev.d.wu.w" => "__builtin_lasx_xvmulwev_d_wu_w",
+    "llvm.loongarch.lasx.xvmulwev.h.b" => "__builtin_lasx_xvmulwev_h_b",
+    "llvm.loongarch.lasx.xvmulwev.h.bu" => "__builtin_lasx_xvmulwev_h_bu",
+    "llvm.loongarch.lasx.xvmulwev.h.bu.b" => "__builtin_lasx_xvmulwev_h_bu_b",
+    "llvm.loongarch.lasx.xvmulwev.q.d" => "__builtin_lasx_xvmulwev_q_d",
+    "llvm.loongarch.lasx.xvmulwev.q.du" => "__builtin_lasx_xvmulwev_q_du",
+    "llvm.loongarch.lasx.xvmulwev.q.du.d" => "__builtin_lasx_xvmulwev_q_du_d",
+    "llvm.loongarch.lasx.xvmulwev.w.h" => "__builtin_lasx_xvmulwev_w_h",
+    "llvm.loongarch.lasx.xvmulwev.w.hu" => "__builtin_lasx_xvmulwev_w_hu",
+    "llvm.loongarch.lasx.xvmulwev.w.hu.h" => "__builtin_lasx_xvmulwev_w_hu_h",
+    "llvm.loongarch.lasx.xvmulwod.d.w" => "__builtin_lasx_xvmulwod_d_w",
+    "llvm.loongarch.lasx.xvmulwod.d.wu" => "__builtin_lasx_xvmulwod_d_wu",
+    "llvm.loongarch.lasx.xvmulwod.d.wu.w" => "__builtin_lasx_xvmulwod_d_wu_w",
+    "llvm.loongarch.lasx.xvmulwod.h.b" => "__builtin_lasx_xvmulwod_h_b",
+    "llvm.loongarch.lasx.xvmulwod.h.bu" => "__builtin_lasx_xvmulwod_h_bu",
+    "llvm.loongarch.lasx.xvmulwod.h.bu.b" => "__builtin_lasx_xvmulwod_h_bu_b",
+    "llvm.loongarch.lasx.xvmulwod.q.d" => "__builtin_lasx_xvmulwod_q_d",
+    "llvm.loongarch.lasx.xvmulwod.q.du" => "__builtin_lasx_xvmulwod_q_du",
+    "llvm.loongarch.lasx.xvmulwod.q.du.d" => "__builtin_lasx_xvmulwod_q_du_d",
+    "llvm.loongarch.lasx.xvmulwod.w.h" => "__builtin_lasx_xvmulwod_w_h",
+    "llvm.loongarch.lasx.xvmulwod.w.hu" => "__builtin_lasx_xvmulwod_w_hu",
+    "llvm.loongarch.lasx.xvmulwod.w.hu.h" => "__builtin_lasx_xvmulwod_w_hu_h",
+    "llvm.loongarch.lasx.xvneg.b" => "__builtin_lasx_xvneg_b",
+    "llvm.loongarch.lasx.xvneg.d" => "__builtin_lasx_xvneg_d",
+    "llvm.loongarch.lasx.xvneg.h" => "__builtin_lasx_xvneg_h",
+    "llvm.loongarch.lasx.xvneg.w" => "__builtin_lasx_xvneg_w",
+    "llvm.loongarch.lasx.xvnor.v" => "__builtin_lasx_xvnor_v",
+    "llvm.loongarch.lasx.xvnori.b" => "__builtin_lasx_xvnori_b",
+    "llvm.loongarch.lasx.xvor.v" => "__builtin_lasx_xvor_v",
+    "llvm.loongarch.lasx.xvori.b" => "__builtin_lasx_xvori_b",
+    "llvm.loongarch.lasx.xvorn.v" => "__builtin_lasx_xvorn_v",
+    "llvm.loongarch.lasx.xvpackev.b" => "__builtin_lasx_xvpackev_b",
+    "llvm.loongarch.lasx.xvpackev.d" => "__builtin_lasx_xvpackev_d",
+    "llvm.loongarch.lasx.xvpackev.h" => "__builtin_lasx_xvpackev_h",
+    "llvm.loongarch.lasx.xvpackev.w" => "__builtin_lasx_xvpackev_w",
+    "llvm.loongarch.lasx.xvpackod.b" => "__builtin_lasx_xvpackod_b",
+    "llvm.loongarch.lasx.xvpackod.d" => "__builtin_lasx_xvpackod_d",
+    "llvm.loongarch.lasx.xvpackod.h" => "__builtin_lasx_xvpackod_h",
+    "llvm.loongarch.lasx.xvpackod.w" => "__builtin_lasx_xvpackod_w",
+    "llvm.loongarch.lasx.xvpcnt.b" => "__builtin_lasx_xvpcnt_b",
+    "llvm.loongarch.lasx.xvpcnt.d" => "__builtin_lasx_xvpcnt_d",
+    "llvm.loongarch.lasx.xvpcnt.h" => "__builtin_lasx_xvpcnt_h",
+    "llvm.loongarch.lasx.xvpcnt.w" => "__builtin_lasx_xvpcnt_w",
+    "llvm.loongarch.lasx.xvperm.w" => "__builtin_lasx_xvperm_w",
+    "llvm.loongarch.lasx.xvpermi.d" => "__builtin_lasx_xvpermi_d",
+    "llvm.loongarch.lasx.xvpermi.q" => "__builtin_lasx_xvpermi_q",
+    "llvm.loongarch.lasx.xvpermi.w" => "__builtin_lasx_xvpermi_w",
+    "llvm.loongarch.lasx.xvpickev.b" => "__builtin_lasx_xvpickev_b",
+    "llvm.loongarch.lasx.xvpickev.d" => "__builtin_lasx_xvpickev_d",
+    "llvm.loongarch.lasx.xvpickev.h" => "__builtin_lasx_xvpickev_h",
+    "llvm.loongarch.lasx.xvpickev.w" => "__builtin_lasx_xvpickev_w",
+    "llvm.loongarch.lasx.xvpickod.b" => "__builtin_lasx_xvpickod_b",
+    "llvm.loongarch.lasx.xvpickod.d" => "__builtin_lasx_xvpickod_d",
+    "llvm.loongarch.lasx.xvpickod.h" => "__builtin_lasx_xvpickod_h",
+    "llvm.loongarch.lasx.xvpickod.w" => "__builtin_lasx_xvpickod_w",
+    "llvm.loongarch.lasx.xvpickve.d" => "__builtin_lasx_xvpickve_d",
+    "llvm.loongarch.lasx.xvpickve.d.f" => "__builtin_lasx_xvpickve_d_f",
+    "llvm.loongarch.lasx.xvpickve.w" => "__builtin_lasx_xvpickve_w",
+    "llvm.loongarch.lasx.xvpickve.w.f" => "__builtin_lasx_xvpickve_w_f",
+    "llvm.loongarch.lasx.xvpickve2gr.d" => "__builtin_lasx_xvpickve2gr_d",
+    "llvm.loongarch.lasx.xvpickve2gr.du" => "__builtin_lasx_xvpickve2gr_du",
+    "llvm.loongarch.lasx.xvpickve2gr.w" => "__builtin_lasx_xvpickve2gr_w",
+    "llvm.loongarch.lasx.xvpickve2gr.wu" => "__builtin_lasx_xvpickve2gr_wu",
+    "llvm.loongarch.lasx.xvrepl128vei.b" => "__builtin_lasx_xvrepl128vei_b",
+    "llvm.loongarch.lasx.xvrepl128vei.d" => "__builtin_lasx_xvrepl128vei_d",
+    "llvm.loongarch.lasx.xvrepl128vei.h" => "__builtin_lasx_xvrepl128vei_h",
+    "llvm.loongarch.lasx.xvrepl128vei.w" => "__builtin_lasx_xvrepl128vei_w",
+    "llvm.loongarch.lasx.xvreplgr2vr.b" => "__builtin_lasx_xvreplgr2vr_b",
+    "llvm.loongarch.lasx.xvreplgr2vr.d" => "__builtin_lasx_xvreplgr2vr_d",
+    "llvm.loongarch.lasx.xvreplgr2vr.h" => "__builtin_lasx_xvreplgr2vr_h",
+    "llvm.loongarch.lasx.xvreplgr2vr.w" => "__builtin_lasx_xvreplgr2vr_w",
+    "llvm.loongarch.lasx.xvrepli.b" => "__builtin_lasx_xvrepli_b",
+    "llvm.loongarch.lasx.xvrepli.d" => "__builtin_lasx_xvrepli_d",
+    "llvm.loongarch.lasx.xvrepli.h" => "__builtin_lasx_xvrepli_h",
+    "llvm.loongarch.lasx.xvrepli.w" => "__builtin_lasx_xvrepli_w",
+    "llvm.loongarch.lasx.xvreplve.b" => "__builtin_lasx_xvreplve_b",
+    "llvm.loongarch.lasx.xvreplve.d" => "__builtin_lasx_xvreplve_d",
+    "llvm.loongarch.lasx.xvreplve.h" => "__builtin_lasx_xvreplve_h",
+    "llvm.loongarch.lasx.xvreplve.w" => "__builtin_lasx_xvreplve_w",
+    "llvm.loongarch.lasx.xvreplve0.b" => "__builtin_lasx_xvreplve0_b",
+    "llvm.loongarch.lasx.xvreplve0.d" => "__builtin_lasx_xvreplve0_d",
+    "llvm.loongarch.lasx.xvreplve0.h" => "__builtin_lasx_xvreplve0_h",
+    "llvm.loongarch.lasx.xvreplve0.q" => "__builtin_lasx_xvreplve0_q",
+    "llvm.loongarch.lasx.xvreplve0.w" => "__builtin_lasx_xvreplve0_w",
+    "llvm.loongarch.lasx.xvrotr.b" => "__builtin_lasx_xvrotr_b",
+    "llvm.loongarch.lasx.xvrotr.d" => "__builtin_lasx_xvrotr_d",
+    "llvm.loongarch.lasx.xvrotr.h" => "__builtin_lasx_xvrotr_h",
+    "llvm.loongarch.lasx.xvrotr.w" => "__builtin_lasx_xvrotr_w",
+    "llvm.loongarch.lasx.xvrotri.b" => "__builtin_lasx_xvrotri_b",
+    "llvm.loongarch.lasx.xvrotri.d" => "__builtin_lasx_xvrotri_d",
+    "llvm.loongarch.lasx.xvrotri.h" => "__builtin_lasx_xvrotri_h",
+    "llvm.loongarch.lasx.xvrotri.w" => "__builtin_lasx_xvrotri_w",
+    "llvm.loongarch.lasx.xvsadd.b" => "__builtin_lasx_xvsadd_b",
+    "llvm.loongarch.lasx.xvsadd.bu" => "__builtin_lasx_xvsadd_bu",
+    "llvm.loongarch.lasx.xvsadd.d" => "__builtin_lasx_xvsadd_d",
+    "llvm.loongarch.lasx.xvsadd.du" => "__builtin_lasx_xvsadd_du",
+    "llvm.loongarch.lasx.xvsadd.h" => "__builtin_lasx_xvsadd_h",
+    "llvm.loongarch.lasx.xvsadd.hu" => "__builtin_lasx_xvsadd_hu",
+    "llvm.loongarch.lasx.xvsadd.w" => "__builtin_lasx_xvsadd_w",
+    "llvm.loongarch.lasx.xvsadd.wu" => "__builtin_lasx_xvsadd_wu",
+    "llvm.loongarch.lasx.xvsat.b" => "__builtin_lasx_xvsat_b",
+    "llvm.loongarch.lasx.xvsat.bu" => "__builtin_lasx_xvsat_bu",
+    "llvm.loongarch.lasx.xvsat.d" => "__builtin_lasx_xvsat_d",
+    "llvm.loongarch.lasx.xvsat.du" => "__builtin_lasx_xvsat_du",
+    "llvm.loongarch.lasx.xvsat.h" => "__builtin_lasx_xvsat_h",
+    "llvm.loongarch.lasx.xvsat.hu" => "__builtin_lasx_xvsat_hu",
+    "llvm.loongarch.lasx.xvsat.w" => "__builtin_lasx_xvsat_w",
+    "llvm.loongarch.lasx.xvsat.wu" => "__builtin_lasx_xvsat_wu",
+    "llvm.loongarch.lasx.xvseq.b" => "__builtin_lasx_xvseq_b",
+    "llvm.loongarch.lasx.xvseq.d" => "__builtin_lasx_xvseq_d",
+    "llvm.loongarch.lasx.xvseq.h" => "__builtin_lasx_xvseq_h",
+    "llvm.loongarch.lasx.xvseq.w" => "__builtin_lasx_xvseq_w",
+    "llvm.loongarch.lasx.xvseqi.b" => "__builtin_lasx_xvseqi_b",
+    "llvm.loongarch.lasx.xvseqi.d" => "__builtin_lasx_xvseqi_d",
+    "llvm.loongarch.lasx.xvseqi.h" => "__builtin_lasx_xvseqi_h",
+    "llvm.loongarch.lasx.xvseqi.w" => "__builtin_lasx_xvseqi_w",
+    "llvm.loongarch.lasx.xvshuf.b" => "__builtin_lasx_xvshuf_b",
+    "llvm.loongarch.lasx.xvshuf.d" => "__builtin_lasx_xvshuf_d",
+    "llvm.loongarch.lasx.xvshuf.h" => "__builtin_lasx_xvshuf_h",
+    "llvm.loongarch.lasx.xvshuf.w" => "__builtin_lasx_xvshuf_w",
+    "llvm.loongarch.lasx.xvshuf4i.b" => "__builtin_lasx_xvshuf4i_b",
+    "llvm.loongarch.lasx.xvshuf4i.d" => "__builtin_lasx_xvshuf4i_d",
+    "llvm.loongarch.lasx.xvshuf4i.h" => "__builtin_lasx_xvshuf4i_h",
+    "llvm.loongarch.lasx.xvshuf4i.w" => "__builtin_lasx_xvshuf4i_w",
+    "llvm.loongarch.lasx.xvsigncov.b" => "__builtin_lasx_xvsigncov_b",
+    "llvm.loongarch.lasx.xvsigncov.d" => "__builtin_lasx_xvsigncov_d",
+    "llvm.loongarch.lasx.xvsigncov.h" => "__builtin_lasx_xvsigncov_h",
+    "llvm.loongarch.lasx.xvsigncov.w" => "__builtin_lasx_xvsigncov_w",
+    "llvm.loongarch.lasx.xvsle.b" => "__builtin_lasx_xvsle_b",
+    "llvm.loongarch.lasx.xvsle.bu" => "__builtin_lasx_xvsle_bu",
+    "llvm.loongarch.lasx.xvsle.d" => "__builtin_lasx_xvsle_d",
+    "llvm.loongarch.lasx.xvsle.du" => "__builtin_lasx_xvsle_du",
+    "llvm.loongarch.lasx.xvsle.h" => "__builtin_lasx_xvsle_h",
+    "llvm.loongarch.lasx.xvsle.hu" => "__builtin_lasx_xvsle_hu",
+    "llvm.loongarch.lasx.xvsle.w" => "__builtin_lasx_xvsle_w",
+    "llvm.loongarch.lasx.xvsle.wu" => "__builtin_lasx_xvsle_wu",
+    "llvm.loongarch.lasx.xvslei.b" => "__builtin_lasx_xvslei_b",
+    "llvm.loongarch.lasx.xvslei.bu" => "__builtin_lasx_xvslei_bu",
+    "llvm.loongarch.lasx.xvslei.d" => "__builtin_lasx_xvslei_d",
+    "llvm.loongarch.lasx.xvslei.du" => "__builtin_lasx_xvslei_du",
+    "llvm.loongarch.lasx.xvslei.h" => "__builtin_lasx_xvslei_h",
+    "llvm.loongarch.lasx.xvslei.hu" => "__builtin_lasx_xvslei_hu",
+    "llvm.loongarch.lasx.xvslei.w" => "__builtin_lasx_xvslei_w",
+    "llvm.loongarch.lasx.xvslei.wu" => "__builtin_lasx_xvslei_wu",
+    "llvm.loongarch.lasx.xvsll.b" => "__builtin_lasx_xvsll_b",
+    "llvm.loongarch.lasx.xvsll.d" => "__builtin_lasx_xvsll_d",
+    "llvm.loongarch.lasx.xvsll.h" => "__builtin_lasx_xvsll_h",
+    "llvm.loongarch.lasx.xvsll.w" => "__builtin_lasx_xvsll_w",
+    "llvm.loongarch.lasx.xvslli.b" => "__builtin_lasx_xvslli_b",
+    "llvm.loongarch.lasx.xvslli.d" => "__builtin_lasx_xvslli_d",
+    "llvm.loongarch.lasx.xvslli.h" => "__builtin_lasx_xvslli_h",
+    "llvm.loongarch.lasx.xvslli.w" => "__builtin_lasx_xvslli_w",
+    "llvm.loongarch.lasx.xvsllwil.d.w" => "__builtin_lasx_xvsllwil_d_w",
+    "llvm.loongarch.lasx.xvsllwil.du.wu" => "__builtin_lasx_xvsllwil_du_wu",
+    "llvm.loongarch.lasx.xvsllwil.h.b" => "__builtin_lasx_xvsllwil_h_b",
+    "llvm.loongarch.lasx.xvsllwil.hu.bu" => "__builtin_lasx_xvsllwil_hu_bu",
+    "llvm.loongarch.lasx.xvsllwil.w.h" => "__builtin_lasx_xvsllwil_w_h",
+    "llvm.loongarch.lasx.xvsllwil.wu.hu" => "__builtin_lasx_xvsllwil_wu_hu",
+    "llvm.loongarch.lasx.xvslt.b" => "__builtin_lasx_xvslt_b",
+    "llvm.loongarch.lasx.xvslt.bu" => "__builtin_lasx_xvslt_bu",
+    "llvm.loongarch.lasx.xvslt.d" => "__builtin_lasx_xvslt_d",
+    "llvm.loongarch.lasx.xvslt.du" => "__builtin_lasx_xvslt_du",
+    "llvm.loongarch.lasx.xvslt.h" => "__builtin_lasx_xvslt_h",
+    "llvm.loongarch.lasx.xvslt.hu" => "__builtin_lasx_xvslt_hu",
+    "llvm.loongarch.lasx.xvslt.w" => "__builtin_lasx_xvslt_w",
+    "llvm.loongarch.lasx.xvslt.wu" => "__builtin_lasx_xvslt_wu",
+    "llvm.loongarch.lasx.xvslti.b" => "__builtin_lasx_xvslti_b",
+    "llvm.loongarch.lasx.xvslti.bu" => "__builtin_lasx_xvslti_bu",
+    "llvm.loongarch.lasx.xvslti.d" => "__builtin_lasx_xvslti_d",
+    "llvm.loongarch.lasx.xvslti.du" => "__builtin_lasx_xvslti_du",
+    "llvm.loongarch.lasx.xvslti.h" => "__builtin_lasx_xvslti_h",
+    "llvm.loongarch.lasx.xvslti.hu" => "__builtin_lasx_xvslti_hu",
+    "llvm.loongarch.lasx.xvslti.w" => "__builtin_lasx_xvslti_w",
+    "llvm.loongarch.lasx.xvslti.wu" => "__builtin_lasx_xvslti_wu",
+    "llvm.loongarch.lasx.xvsra.b" => "__builtin_lasx_xvsra_b",
+    "llvm.loongarch.lasx.xvsra.d" => "__builtin_lasx_xvsra_d",
+    "llvm.loongarch.lasx.xvsra.h" => "__builtin_lasx_xvsra_h",
+    "llvm.loongarch.lasx.xvsra.w" => "__builtin_lasx_xvsra_w",
+    "llvm.loongarch.lasx.xvsrai.b" => "__builtin_lasx_xvsrai_b",
+    "llvm.loongarch.lasx.xvsrai.d" => "__builtin_lasx_xvsrai_d",
+    "llvm.loongarch.lasx.xvsrai.h" => "__builtin_lasx_xvsrai_h",
+    "llvm.loongarch.lasx.xvsrai.w" => "__builtin_lasx_xvsrai_w",
+    "llvm.loongarch.lasx.xvsran.b.h" => "__builtin_lasx_xvsran_b_h",
+    "llvm.loongarch.lasx.xvsran.h.w" => "__builtin_lasx_xvsran_h_w",
+    "llvm.loongarch.lasx.xvsran.w.d" => "__builtin_lasx_xvsran_w_d",
+    "llvm.loongarch.lasx.xvsrani.b.h" => "__builtin_lasx_xvsrani_b_h",
+    "llvm.loongarch.lasx.xvsrani.d.q" => "__builtin_lasx_xvsrani_d_q",
+    "llvm.loongarch.lasx.xvsrani.h.w" => "__builtin_lasx_xvsrani_h_w",
+    "llvm.loongarch.lasx.xvsrani.w.d" => "__builtin_lasx_xvsrani_w_d",
+    "llvm.loongarch.lasx.xvsrar.b" => "__builtin_lasx_xvsrar_b",
+    "llvm.loongarch.lasx.xvsrar.d" => "__builtin_lasx_xvsrar_d",
+    "llvm.loongarch.lasx.xvsrar.h" => "__builtin_lasx_xvsrar_h",
+    "llvm.loongarch.lasx.xvsrar.w" => "__builtin_lasx_xvsrar_w",
+    "llvm.loongarch.lasx.xvsrari.b" => "__builtin_lasx_xvsrari_b",
+    "llvm.loongarch.lasx.xvsrari.d" => "__builtin_lasx_xvsrari_d",
+    "llvm.loongarch.lasx.xvsrari.h" => "__builtin_lasx_xvsrari_h",
+    "llvm.loongarch.lasx.xvsrari.w" => "__builtin_lasx_xvsrari_w",
+    "llvm.loongarch.lasx.xvsrarn.b.h" => "__builtin_lasx_xvsrarn_b_h",
+    "llvm.loongarch.lasx.xvsrarn.h.w" => "__builtin_lasx_xvsrarn_h_w",
+    "llvm.loongarch.lasx.xvsrarn.w.d" => "__builtin_lasx_xvsrarn_w_d",
+    "llvm.loongarch.lasx.xvsrarni.b.h" => "__builtin_lasx_xvsrarni_b_h",
+    "llvm.loongarch.lasx.xvsrarni.d.q" => "__builtin_lasx_xvsrarni_d_q",
+    "llvm.loongarch.lasx.xvsrarni.h.w" => "__builtin_lasx_xvsrarni_h_w",
+    "llvm.loongarch.lasx.xvsrarni.w.d" => "__builtin_lasx_xvsrarni_w_d",
+    "llvm.loongarch.lasx.xvsrl.b" => "__builtin_lasx_xvsrl_b",
+    "llvm.loongarch.lasx.xvsrl.d" => "__builtin_lasx_xvsrl_d",
+    "llvm.loongarch.lasx.xvsrl.h" => "__builtin_lasx_xvsrl_h",
+    "llvm.loongarch.lasx.xvsrl.w" => "__builtin_lasx_xvsrl_w",
+    "llvm.loongarch.lasx.xvsrli.b" => "__builtin_lasx_xvsrli_b",
+    "llvm.loongarch.lasx.xvsrli.d" => "__builtin_lasx_xvsrli_d",
+    "llvm.loongarch.lasx.xvsrli.h" => "__builtin_lasx_xvsrli_h",
+    "llvm.loongarch.lasx.xvsrli.w" => "__builtin_lasx_xvsrli_w",
+    "llvm.loongarch.lasx.xvsrln.b.h" => "__builtin_lasx_xvsrln_b_h",
+    "llvm.loongarch.lasx.xvsrln.h.w" => "__builtin_lasx_xvsrln_h_w",
+    "llvm.loongarch.lasx.xvsrln.w.d" => "__builtin_lasx_xvsrln_w_d",
+    "llvm.loongarch.lasx.xvsrlni.b.h" => "__builtin_lasx_xvsrlni_b_h",
+    "llvm.loongarch.lasx.xvsrlni.d.q" => "__builtin_lasx_xvsrlni_d_q",
+    "llvm.loongarch.lasx.xvsrlni.h.w" => "__builtin_lasx_xvsrlni_h_w",
+    "llvm.loongarch.lasx.xvsrlni.w.d" => "__builtin_lasx_xvsrlni_w_d",
+    "llvm.loongarch.lasx.xvsrlr.b" => "__builtin_lasx_xvsrlr_b",
+    "llvm.loongarch.lasx.xvsrlr.d" => "__builtin_lasx_xvsrlr_d",
+    "llvm.loongarch.lasx.xvsrlr.h" => "__builtin_lasx_xvsrlr_h",
+    "llvm.loongarch.lasx.xvsrlr.w" => "__builtin_lasx_xvsrlr_w",
+    "llvm.loongarch.lasx.xvsrlri.b" => "__builtin_lasx_xvsrlri_b",
+    "llvm.loongarch.lasx.xvsrlri.d" => "__builtin_lasx_xvsrlri_d",
+    "llvm.loongarch.lasx.xvsrlri.h" => "__builtin_lasx_xvsrlri_h",
+    "llvm.loongarch.lasx.xvsrlri.w" => "__builtin_lasx_xvsrlri_w",
+    "llvm.loongarch.lasx.xvsrlrn.b.h" => "__builtin_lasx_xvsrlrn_b_h",
+    "llvm.loongarch.lasx.xvsrlrn.h.w" => "__builtin_lasx_xvsrlrn_h_w",
+    "llvm.loongarch.lasx.xvsrlrn.w.d" => "__builtin_lasx_xvsrlrn_w_d",
+    "llvm.loongarch.lasx.xvsrlrni.b.h" => "__builtin_lasx_xvsrlrni_b_h",
+    "llvm.loongarch.lasx.xvsrlrni.d.q" => "__builtin_lasx_xvsrlrni_d_q",
+    "llvm.loongarch.lasx.xvsrlrni.h.w" => "__builtin_lasx_xvsrlrni_h_w",
+    "llvm.loongarch.lasx.xvsrlrni.w.d" => "__builtin_lasx_xvsrlrni_w_d",
+    "llvm.loongarch.lasx.xvssran.b.h" => "__builtin_lasx_xvssran_b_h",
+    "llvm.loongarch.lasx.xvssran.bu.h" => "__builtin_lasx_xvssran_bu_h",
+    "llvm.loongarch.lasx.xvssran.h.w" => "__builtin_lasx_xvssran_h_w",
+    "llvm.loongarch.lasx.xvssran.hu.w" => "__builtin_lasx_xvssran_hu_w",
+    "llvm.loongarch.lasx.xvssran.w.d" => "__builtin_lasx_xvssran_w_d",
+    "llvm.loongarch.lasx.xvssran.wu.d" => "__builtin_lasx_xvssran_wu_d",
+    "llvm.loongarch.lasx.xvssrani.b.h" => "__builtin_lasx_xvssrani_b_h",
+    "llvm.loongarch.lasx.xvssrani.bu.h" => "__builtin_lasx_xvssrani_bu_h",
+    "llvm.loongarch.lasx.xvssrani.d.q" => "__builtin_lasx_xvssrani_d_q",
+    "llvm.loongarch.lasx.xvssrani.du.q" => "__builtin_lasx_xvssrani_du_q",
+    "llvm.loongarch.lasx.xvssrani.h.w" => "__builtin_lasx_xvssrani_h_w",
+    "llvm.loongarch.lasx.xvssrani.hu.w" => "__builtin_lasx_xvssrani_hu_w",
+    "llvm.loongarch.lasx.xvssrani.w.d" => "__builtin_lasx_xvssrani_w_d",
+    "llvm.loongarch.lasx.xvssrani.wu.d" => "__builtin_lasx_xvssrani_wu_d",
+    "llvm.loongarch.lasx.xvssrarn.b.h" => "__builtin_lasx_xvssrarn_b_h",
+    "llvm.loongarch.lasx.xvssrarn.bu.h" => "__builtin_lasx_xvssrarn_bu_h",
+    "llvm.loongarch.lasx.xvssrarn.h.w" => "__builtin_lasx_xvssrarn_h_w",
+    "llvm.loongarch.lasx.xvssrarn.hu.w" => "__builtin_lasx_xvssrarn_hu_w",
+    "llvm.loongarch.lasx.xvssrarn.w.d" => "__builtin_lasx_xvssrarn_w_d",
+    "llvm.loongarch.lasx.xvssrarn.wu.d" => "__builtin_lasx_xvssrarn_wu_d",
+    "llvm.loongarch.lasx.xvssrarni.b.h" => "__builtin_lasx_xvssrarni_b_h",
+    "llvm.loongarch.lasx.xvssrarni.bu.h" => "__builtin_lasx_xvssrarni_bu_h",
+    "llvm.loongarch.lasx.xvssrarni.d.q" => "__builtin_lasx_xvssrarni_d_q",
+    "llvm.loongarch.lasx.xvssrarni.du.q" => "__builtin_lasx_xvssrarni_du_q",
+    "llvm.loongarch.lasx.xvssrarni.h.w" => "__builtin_lasx_xvssrarni_h_w",
+    "llvm.loongarch.lasx.xvssrarni.hu.w" => "__builtin_lasx_xvssrarni_hu_w",
+    "llvm.loongarch.lasx.xvssrarni.w.d" => "__builtin_lasx_xvssrarni_w_d",
+    "llvm.loongarch.lasx.xvssrarni.wu.d" => "__builtin_lasx_xvssrarni_wu_d",
+    "llvm.loongarch.lasx.xvssrln.b.h" => "__builtin_lasx_xvssrln_b_h",
+    "llvm.loongarch.lasx.xvssrln.bu.h" => "__builtin_lasx_xvssrln_bu_h",
+    "llvm.loongarch.lasx.xvssrln.h.w" => "__builtin_lasx_xvssrln_h_w",
+    "llvm.loongarch.lasx.xvssrln.hu.w" => "__builtin_lasx_xvssrln_hu_w",
+    "llvm.loongarch.lasx.xvssrln.w.d" => "__builtin_lasx_xvssrln_w_d",
+    "llvm.loongarch.lasx.xvssrln.wu.d" => "__builtin_lasx_xvssrln_wu_d",
+    "llvm.loongarch.lasx.xvssrlni.b.h" => "__builtin_lasx_xvssrlni_b_h",
+    "llvm.loongarch.lasx.xvssrlni.bu.h" => "__builtin_lasx_xvssrlni_bu_h",
+    "llvm.loongarch.lasx.xvssrlni.d.q" => "__builtin_lasx_xvssrlni_d_q",
+    "llvm.loongarch.lasx.xvssrlni.du.q" => "__builtin_lasx_xvssrlni_du_q",
+    "llvm.loongarch.lasx.xvssrlni.h.w" => "__builtin_lasx_xvssrlni_h_w",
+    "llvm.loongarch.lasx.xvssrlni.hu.w" => "__builtin_lasx_xvssrlni_hu_w",
+    "llvm.loongarch.lasx.xvssrlni.w.d" => "__builtin_lasx_xvssrlni_w_d",
+    "llvm.loongarch.lasx.xvssrlni.wu.d" => "__builtin_lasx_xvssrlni_wu_d",
+    "llvm.loongarch.lasx.xvssrlrn.b.h" => "__builtin_lasx_xvssrlrn_b_h",
+    "llvm.loongarch.lasx.xvssrlrn.bu.h" => "__builtin_lasx_xvssrlrn_bu_h",
+    "llvm.loongarch.lasx.xvssrlrn.h.w" => "__builtin_lasx_xvssrlrn_h_w",
+    "llvm.loongarch.lasx.xvssrlrn.hu.w" => "__builtin_lasx_xvssrlrn_hu_w",
+    "llvm.loongarch.lasx.xvssrlrn.w.d" => "__builtin_lasx_xvssrlrn_w_d",
+    "llvm.loongarch.lasx.xvssrlrn.wu.d" => "__builtin_lasx_xvssrlrn_wu_d",
+    "llvm.loongarch.lasx.xvssrlrni.b.h" => "__builtin_lasx_xvssrlrni_b_h",
+    "llvm.loongarch.lasx.xvssrlrni.bu.h" => "__builtin_lasx_xvssrlrni_bu_h",
+    "llvm.loongarch.lasx.xvssrlrni.d.q" => "__builtin_lasx_xvssrlrni_d_q",
+    "llvm.loongarch.lasx.xvssrlrni.du.q" => "__builtin_lasx_xvssrlrni_du_q",
+    "llvm.loongarch.lasx.xvssrlrni.h.w" => "__builtin_lasx_xvssrlrni_h_w",
+    "llvm.loongarch.lasx.xvssrlrni.hu.w" => "__builtin_lasx_xvssrlrni_hu_w",
+    "llvm.loongarch.lasx.xvssrlrni.w.d" => "__builtin_lasx_xvssrlrni_w_d",
+    "llvm.loongarch.lasx.xvssrlrni.wu.d" => "__builtin_lasx_xvssrlrni_wu_d",
+    "llvm.loongarch.lasx.xvssub.b" => "__builtin_lasx_xvssub_b",
+    "llvm.loongarch.lasx.xvssub.bu" => "__builtin_lasx_xvssub_bu",
+    "llvm.loongarch.lasx.xvssub.d" => "__builtin_lasx_xvssub_d",
+    "llvm.loongarch.lasx.xvssub.du" => "__builtin_lasx_xvssub_du",
+    "llvm.loongarch.lasx.xvssub.h" => "__builtin_lasx_xvssub_h",
+    "llvm.loongarch.lasx.xvssub.hu" => "__builtin_lasx_xvssub_hu",
+    "llvm.loongarch.lasx.xvssub.w" => "__builtin_lasx_xvssub_w",
+    "llvm.loongarch.lasx.xvssub.wu" => "__builtin_lasx_xvssub_wu",
+    "llvm.loongarch.lasx.xvst" => "__builtin_lasx_xvst",
+    "llvm.loongarch.lasx.xvstelm.b" => "__builtin_lasx_xvstelm_b",
+    "llvm.loongarch.lasx.xvstelm.d" => "__builtin_lasx_xvstelm_d",
+    "llvm.loongarch.lasx.xvstelm.h" => "__builtin_lasx_xvstelm_h",
+    "llvm.loongarch.lasx.xvstelm.w" => "__builtin_lasx_xvstelm_w",
+    "llvm.loongarch.lasx.xvstx" => "__builtin_lasx_xvstx",
+    "llvm.loongarch.lasx.xvsub.b" => "__builtin_lasx_xvsub_b",
+    "llvm.loongarch.lasx.xvsub.d" => "__builtin_lasx_xvsub_d",
+    "llvm.loongarch.lasx.xvsub.h" => "__builtin_lasx_xvsub_h",
+    "llvm.loongarch.lasx.xvsub.q" => "__builtin_lasx_xvsub_q",
+    "llvm.loongarch.lasx.xvsub.w" => "__builtin_lasx_xvsub_w",
+    "llvm.loongarch.lasx.xvsubi.bu" => "__builtin_lasx_xvsubi_bu",
+    "llvm.loongarch.lasx.xvsubi.du" => "__builtin_lasx_xvsubi_du",
+    "llvm.loongarch.lasx.xvsubi.hu" => "__builtin_lasx_xvsubi_hu",
+    "llvm.loongarch.lasx.xvsubi.wu" => "__builtin_lasx_xvsubi_wu",
+    "llvm.loongarch.lasx.xvsubwev.d.w" => "__builtin_lasx_xvsubwev_d_w",
+    "llvm.loongarch.lasx.xvsubwev.d.wu" => "__builtin_lasx_xvsubwev_d_wu",
+    "llvm.loongarch.lasx.xvsubwev.h.b" => "__builtin_lasx_xvsubwev_h_b",
+    "llvm.loongarch.lasx.xvsubwev.h.bu" => "__builtin_lasx_xvsubwev_h_bu",
+    "llvm.loongarch.lasx.xvsubwev.q.d" => "__builtin_lasx_xvsubwev_q_d",
+    "llvm.loongarch.lasx.xvsubwev.q.du" => "__builtin_lasx_xvsubwev_q_du",
+    "llvm.loongarch.lasx.xvsubwev.w.h" => "__builtin_lasx_xvsubwev_w_h",
+    "llvm.loongarch.lasx.xvsubwev.w.hu" => "__builtin_lasx_xvsubwev_w_hu",
+    "llvm.loongarch.lasx.xvsubwod.d.w" => "__builtin_lasx_xvsubwod_d_w",
+    "llvm.loongarch.lasx.xvsubwod.d.wu" => "__builtin_lasx_xvsubwod_d_wu",
+    "llvm.loongarch.lasx.xvsubwod.h.b" => "__builtin_lasx_xvsubwod_h_b",
+    "llvm.loongarch.lasx.xvsubwod.h.bu" => "__builtin_lasx_xvsubwod_h_bu",
+    "llvm.loongarch.lasx.xvsubwod.q.d" => "__builtin_lasx_xvsubwod_q_d",
+    "llvm.loongarch.lasx.xvsubwod.q.du" => "__builtin_lasx_xvsubwod_q_du",
+    "llvm.loongarch.lasx.xvsubwod.w.h" => "__builtin_lasx_xvsubwod_w_h",
+    "llvm.loongarch.lasx.xvsubwod.w.hu" => "__builtin_lasx_xvsubwod_w_hu",
+    "llvm.loongarch.lasx.xvxor.v" => "__builtin_lasx_xvxor_v",
+    "llvm.loongarch.lasx.xvxori.b" => "__builtin_lasx_xvxori_b",
     "llvm.loongarch.lddir.d" => "__builtin_loongarch_lddir_d",
     "llvm.loongarch.ldpte.d" => "__builtin_loongarch_ldpte_d",
+    "llvm.loongarch.lsx.bnz.b" => "__builtin_lsx_bnz_b",
+    "llvm.loongarch.lsx.bnz.d" => "__builtin_lsx_bnz_d",
+    "llvm.loongarch.lsx.bnz.h" => "__builtin_lsx_bnz_h",
+    "llvm.loongarch.lsx.bnz.v" => "__builtin_lsx_bnz_v",
+    "llvm.loongarch.lsx.bnz.w" => "__builtin_lsx_bnz_w",
+    "llvm.loongarch.lsx.bz.b" => "__builtin_lsx_bz_b",
+    "llvm.loongarch.lsx.bz.d" => "__builtin_lsx_bz_d",
+    "llvm.loongarch.lsx.bz.h" => "__builtin_lsx_bz_h",
+    "llvm.loongarch.lsx.bz.v" => "__builtin_lsx_bz_v",
+    "llvm.loongarch.lsx.bz.w" => "__builtin_lsx_bz_w",
+    "llvm.loongarch.lsx.vabsd.b" => "__builtin_lsx_vabsd_b",
+    "llvm.loongarch.lsx.vabsd.bu" => "__builtin_lsx_vabsd_bu",
+    "llvm.loongarch.lsx.vabsd.d" => "__builtin_lsx_vabsd_d",
+    "llvm.loongarch.lsx.vabsd.du" => "__builtin_lsx_vabsd_du",
+    "llvm.loongarch.lsx.vabsd.h" => "__builtin_lsx_vabsd_h",
+    "llvm.loongarch.lsx.vabsd.hu" => "__builtin_lsx_vabsd_hu",
+    "llvm.loongarch.lsx.vabsd.w" => "__builtin_lsx_vabsd_w",
+    "llvm.loongarch.lsx.vabsd.wu" => "__builtin_lsx_vabsd_wu",
+    "llvm.loongarch.lsx.vadd.b" => "__builtin_lsx_vadd_b",
+    "llvm.loongarch.lsx.vadd.d" => "__builtin_lsx_vadd_d",
+    "llvm.loongarch.lsx.vadd.h" => "__builtin_lsx_vadd_h",
+    "llvm.loongarch.lsx.vadd.q" => "__builtin_lsx_vadd_q",
+    "llvm.loongarch.lsx.vadd.w" => "__builtin_lsx_vadd_w",
+    "llvm.loongarch.lsx.vadda.b" => "__builtin_lsx_vadda_b",
+    "llvm.loongarch.lsx.vadda.d" => "__builtin_lsx_vadda_d",
+    "llvm.loongarch.lsx.vadda.h" => "__builtin_lsx_vadda_h",
+    "llvm.loongarch.lsx.vadda.w" => "__builtin_lsx_vadda_w",
+    "llvm.loongarch.lsx.vaddi.bu" => "__builtin_lsx_vaddi_bu",
+    "llvm.loongarch.lsx.vaddi.du" => "__builtin_lsx_vaddi_du",
+    "llvm.loongarch.lsx.vaddi.hu" => "__builtin_lsx_vaddi_hu",
+    "llvm.loongarch.lsx.vaddi.wu" => "__builtin_lsx_vaddi_wu",
+    "llvm.loongarch.lsx.vaddwev.d.w" => "__builtin_lsx_vaddwev_d_w",
+    "llvm.loongarch.lsx.vaddwev.d.wu" => "__builtin_lsx_vaddwev_d_wu",
+    "llvm.loongarch.lsx.vaddwev.d.wu.w" => "__builtin_lsx_vaddwev_d_wu_w",
+    "llvm.loongarch.lsx.vaddwev.h.b" => "__builtin_lsx_vaddwev_h_b",
+    "llvm.loongarch.lsx.vaddwev.h.bu" => "__builtin_lsx_vaddwev_h_bu",
+    "llvm.loongarch.lsx.vaddwev.h.bu.b" => "__builtin_lsx_vaddwev_h_bu_b",
+    "llvm.loongarch.lsx.vaddwev.q.d" => "__builtin_lsx_vaddwev_q_d",
+    "llvm.loongarch.lsx.vaddwev.q.du" => "__builtin_lsx_vaddwev_q_du",
+    "llvm.loongarch.lsx.vaddwev.q.du.d" => "__builtin_lsx_vaddwev_q_du_d",
+    "llvm.loongarch.lsx.vaddwev.w.h" => "__builtin_lsx_vaddwev_w_h",
+    "llvm.loongarch.lsx.vaddwev.w.hu" => "__builtin_lsx_vaddwev_w_hu",
+    "llvm.loongarch.lsx.vaddwev.w.hu.h" => "__builtin_lsx_vaddwev_w_hu_h",
+    "llvm.loongarch.lsx.vaddwod.d.w" => "__builtin_lsx_vaddwod_d_w",
+    "llvm.loongarch.lsx.vaddwod.d.wu" => "__builtin_lsx_vaddwod_d_wu",
+    "llvm.loongarch.lsx.vaddwod.d.wu.w" => "__builtin_lsx_vaddwod_d_wu_w",
+    "llvm.loongarch.lsx.vaddwod.h.b" => "__builtin_lsx_vaddwod_h_b",
+    "llvm.loongarch.lsx.vaddwod.h.bu" => "__builtin_lsx_vaddwod_h_bu",
+    "llvm.loongarch.lsx.vaddwod.h.bu.b" => "__builtin_lsx_vaddwod_h_bu_b",
+    "llvm.loongarch.lsx.vaddwod.q.d" => "__builtin_lsx_vaddwod_q_d",
+    "llvm.loongarch.lsx.vaddwod.q.du" => "__builtin_lsx_vaddwod_q_du",
+    "llvm.loongarch.lsx.vaddwod.q.du.d" => "__builtin_lsx_vaddwod_q_du_d",
+    "llvm.loongarch.lsx.vaddwod.w.h" => "__builtin_lsx_vaddwod_w_h",
+    "llvm.loongarch.lsx.vaddwod.w.hu" => "__builtin_lsx_vaddwod_w_hu",
+    "llvm.loongarch.lsx.vaddwod.w.hu.h" => "__builtin_lsx_vaddwod_w_hu_h",
+    "llvm.loongarch.lsx.vand.v" => "__builtin_lsx_vand_v",
+    "llvm.loongarch.lsx.vandi.b" => "__builtin_lsx_vandi_b",
+    "llvm.loongarch.lsx.vandn.v" => "__builtin_lsx_vandn_v",
+    "llvm.loongarch.lsx.vavg.b" => "__builtin_lsx_vavg_b",
+    "llvm.loongarch.lsx.vavg.bu" => "__builtin_lsx_vavg_bu",
+    "llvm.loongarch.lsx.vavg.d" => "__builtin_lsx_vavg_d",
+    "llvm.loongarch.lsx.vavg.du" => "__builtin_lsx_vavg_du",
+    "llvm.loongarch.lsx.vavg.h" => "__builtin_lsx_vavg_h",
+    "llvm.loongarch.lsx.vavg.hu" => "__builtin_lsx_vavg_hu",
+    "llvm.loongarch.lsx.vavg.w" => "__builtin_lsx_vavg_w",
+    "llvm.loongarch.lsx.vavg.wu" => "__builtin_lsx_vavg_wu",
+    "llvm.loongarch.lsx.vavgr.b" => "__builtin_lsx_vavgr_b",
+    "llvm.loongarch.lsx.vavgr.bu" => "__builtin_lsx_vavgr_bu",
+    "llvm.loongarch.lsx.vavgr.d" => "__builtin_lsx_vavgr_d",
+    "llvm.loongarch.lsx.vavgr.du" => "__builtin_lsx_vavgr_du",
+    "llvm.loongarch.lsx.vavgr.h" => "__builtin_lsx_vavgr_h",
+    "llvm.loongarch.lsx.vavgr.hu" => "__builtin_lsx_vavgr_hu",
+    "llvm.loongarch.lsx.vavgr.w" => "__builtin_lsx_vavgr_w",
+    "llvm.loongarch.lsx.vavgr.wu" => "__builtin_lsx_vavgr_wu",
+    "llvm.loongarch.lsx.vbitclr.b" => "__builtin_lsx_vbitclr_b",
+    "llvm.loongarch.lsx.vbitclr.d" => "__builtin_lsx_vbitclr_d",
+    "llvm.loongarch.lsx.vbitclr.h" => "__builtin_lsx_vbitclr_h",
+    "llvm.loongarch.lsx.vbitclr.w" => "__builtin_lsx_vbitclr_w",
+    "llvm.loongarch.lsx.vbitclri.b" => "__builtin_lsx_vbitclri_b",
+    "llvm.loongarch.lsx.vbitclri.d" => "__builtin_lsx_vbitclri_d",
+    "llvm.loongarch.lsx.vbitclri.h" => "__builtin_lsx_vbitclri_h",
+    "llvm.loongarch.lsx.vbitclri.w" => "__builtin_lsx_vbitclri_w",
+    "llvm.loongarch.lsx.vbitrev.b" => "__builtin_lsx_vbitrev_b",
+    "llvm.loongarch.lsx.vbitrev.d" => "__builtin_lsx_vbitrev_d",
+    "llvm.loongarch.lsx.vbitrev.h" => "__builtin_lsx_vbitrev_h",
+    "llvm.loongarch.lsx.vbitrev.w" => "__builtin_lsx_vbitrev_w",
+    "llvm.loongarch.lsx.vbitrevi.b" => "__builtin_lsx_vbitrevi_b",
+    "llvm.loongarch.lsx.vbitrevi.d" => "__builtin_lsx_vbitrevi_d",
+    "llvm.loongarch.lsx.vbitrevi.h" => "__builtin_lsx_vbitrevi_h",
+    "llvm.loongarch.lsx.vbitrevi.w" => "__builtin_lsx_vbitrevi_w",
+    "llvm.loongarch.lsx.vbitsel.v" => "__builtin_lsx_vbitsel_v",
+    "llvm.loongarch.lsx.vbitseli.b" => "__builtin_lsx_vbitseli_b",
+    "llvm.loongarch.lsx.vbitset.b" => "__builtin_lsx_vbitset_b",
+    "llvm.loongarch.lsx.vbitset.d" => "__builtin_lsx_vbitset_d",
+    "llvm.loongarch.lsx.vbitset.h" => "__builtin_lsx_vbitset_h",
+    "llvm.loongarch.lsx.vbitset.w" => "__builtin_lsx_vbitset_w",
+    "llvm.loongarch.lsx.vbitseti.b" => "__builtin_lsx_vbitseti_b",
+    "llvm.loongarch.lsx.vbitseti.d" => "__builtin_lsx_vbitseti_d",
+    "llvm.loongarch.lsx.vbitseti.h" => "__builtin_lsx_vbitseti_h",
+    "llvm.loongarch.lsx.vbitseti.w" => "__builtin_lsx_vbitseti_w",
+    "llvm.loongarch.lsx.vbsll.v" => "__builtin_lsx_vbsll_v",
+    "llvm.loongarch.lsx.vbsrl.v" => "__builtin_lsx_vbsrl_v",
+    "llvm.loongarch.lsx.vclo.b" => "__builtin_lsx_vclo_b",
+    "llvm.loongarch.lsx.vclo.d" => "__builtin_lsx_vclo_d",
+    "llvm.loongarch.lsx.vclo.h" => "__builtin_lsx_vclo_h",
+    "llvm.loongarch.lsx.vclo.w" => "__builtin_lsx_vclo_w",
+    "llvm.loongarch.lsx.vclz.b" => "__builtin_lsx_vclz_b",
+    "llvm.loongarch.lsx.vclz.d" => "__builtin_lsx_vclz_d",
+    "llvm.loongarch.lsx.vclz.h" => "__builtin_lsx_vclz_h",
+    "llvm.loongarch.lsx.vclz.w" => "__builtin_lsx_vclz_w",
+    "llvm.loongarch.lsx.vdiv.b" => "__builtin_lsx_vdiv_b",
+    "llvm.loongarch.lsx.vdiv.bu" => "__builtin_lsx_vdiv_bu",
+    "llvm.loongarch.lsx.vdiv.d" => "__builtin_lsx_vdiv_d",
+    "llvm.loongarch.lsx.vdiv.du" => "__builtin_lsx_vdiv_du",
+    "llvm.loongarch.lsx.vdiv.h" => "__builtin_lsx_vdiv_h",
+    "llvm.loongarch.lsx.vdiv.hu" => "__builtin_lsx_vdiv_hu",
+    "llvm.loongarch.lsx.vdiv.w" => "__builtin_lsx_vdiv_w",
+    "llvm.loongarch.lsx.vdiv.wu" => "__builtin_lsx_vdiv_wu",
+    "llvm.loongarch.lsx.vexth.d.w" => "__builtin_lsx_vexth_d_w",
+    "llvm.loongarch.lsx.vexth.du.wu" => "__builtin_lsx_vexth_du_wu",
+    "llvm.loongarch.lsx.vexth.h.b" => "__builtin_lsx_vexth_h_b",
+    "llvm.loongarch.lsx.vexth.hu.bu" => "__builtin_lsx_vexth_hu_bu",
+    "llvm.loongarch.lsx.vexth.q.d" => "__builtin_lsx_vexth_q_d",
+    "llvm.loongarch.lsx.vexth.qu.du" => "__builtin_lsx_vexth_qu_du",
+    "llvm.loongarch.lsx.vexth.w.h" => "__builtin_lsx_vexth_w_h",
+    "llvm.loongarch.lsx.vexth.wu.hu" => "__builtin_lsx_vexth_wu_hu",
+    "llvm.loongarch.lsx.vextl.q.d" => "__builtin_lsx_vextl_q_d",
+    "llvm.loongarch.lsx.vextl.qu.du" => "__builtin_lsx_vextl_qu_du",
+    "llvm.loongarch.lsx.vextrins.b" => "__builtin_lsx_vextrins_b",
+    "llvm.loongarch.lsx.vextrins.d" => "__builtin_lsx_vextrins_d",
+    "llvm.loongarch.lsx.vextrins.h" => "__builtin_lsx_vextrins_h",
+    "llvm.loongarch.lsx.vextrins.w" => "__builtin_lsx_vextrins_w",
+    "llvm.loongarch.lsx.vfadd.d" => "__builtin_lsx_vfadd_d",
+    "llvm.loongarch.lsx.vfadd.s" => "__builtin_lsx_vfadd_s",
+    "llvm.loongarch.lsx.vfclass.d" => "__builtin_lsx_vfclass_d",
+    "llvm.loongarch.lsx.vfclass.s" => "__builtin_lsx_vfclass_s",
+    "llvm.loongarch.lsx.vfcmp.caf.d" => "__builtin_lsx_vfcmp_caf_d",
+    "llvm.loongarch.lsx.vfcmp.caf.s" => "__builtin_lsx_vfcmp_caf_s",
+    "llvm.loongarch.lsx.vfcmp.ceq.d" => "__builtin_lsx_vfcmp_ceq_d",
+    "llvm.loongarch.lsx.vfcmp.ceq.s" => "__builtin_lsx_vfcmp_ceq_s",
+    "llvm.loongarch.lsx.vfcmp.cle.d" => "__builtin_lsx_vfcmp_cle_d",
+    "llvm.loongarch.lsx.vfcmp.cle.s" => "__builtin_lsx_vfcmp_cle_s",
+    "llvm.loongarch.lsx.vfcmp.clt.d" => "__builtin_lsx_vfcmp_clt_d",
+    "llvm.loongarch.lsx.vfcmp.clt.s" => "__builtin_lsx_vfcmp_clt_s",
+    "llvm.loongarch.lsx.vfcmp.cne.d" => "__builtin_lsx_vfcmp_cne_d",
+    "llvm.loongarch.lsx.vfcmp.cne.s" => "__builtin_lsx_vfcmp_cne_s",
+    "llvm.loongarch.lsx.vfcmp.cor.d" => "__builtin_lsx_vfcmp_cor_d",
+    "llvm.loongarch.lsx.vfcmp.cor.s" => "__builtin_lsx_vfcmp_cor_s",
+    "llvm.loongarch.lsx.vfcmp.cueq.d" => "__builtin_lsx_vfcmp_cueq_d",
+    "llvm.loongarch.lsx.vfcmp.cueq.s" => "__builtin_lsx_vfcmp_cueq_s",
+    "llvm.loongarch.lsx.vfcmp.cule.d" => "__builtin_lsx_vfcmp_cule_d",
+    "llvm.loongarch.lsx.vfcmp.cule.s" => "__builtin_lsx_vfcmp_cule_s",
+    "llvm.loongarch.lsx.vfcmp.cult.d" => "__builtin_lsx_vfcmp_cult_d",
+    "llvm.loongarch.lsx.vfcmp.cult.s" => "__builtin_lsx_vfcmp_cult_s",
+    "llvm.loongarch.lsx.vfcmp.cun.d" => "__builtin_lsx_vfcmp_cun_d",
+    "llvm.loongarch.lsx.vfcmp.cun.s" => "__builtin_lsx_vfcmp_cun_s",
+    "llvm.loongarch.lsx.vfcmp.cune.d" => "__builtin_lsx_vfcmp_cune_d",
+    "llvm.loongarch.lsx.vfcmp.cune.s" => "__builtin_lsx_vfcmp_cune_s",
+    "llvm.loongarch.lsx.vfcmp.saf.d" => "__builtin_lsx_vfcmp_saf_d",
+    "llvm.loongarch.lsx.vfcmp.saf.s" => "__builtin_lsx_vfcmp_saf_s",
+    "llvm.loongarch.lsx.vfcmp.seq.d" => "__builtin_lsx_vfcmp_seq_d",
+    "llvm.loongarch.lsx.vfcmp.seq.s" => "__builtin_lsx_vfcmp_seq_s",
+    "llvm.loongarch.lsx.vfcmp.sle.d" => "__builtin_lsx_vfcmp_sle_d",
+    "llvm.loongarch.lsx.vfcmp.sle.s" => "__builtin_lsx_vfcmp_sle_s",
+    "llvm.loongarch.lsx.vfcmp.slt.d" => "__builtin_lsx_vfcmp_slt_d",
+    "llvm.loongarch.lsx.vfcmp.slt.s" => "__builtin_lsx_vfcmp_slt_s",
+    "llvm.loongarch.lsx.vfcmp.sne.d" => "__builtin_lsx_vfcmp_sne_d",
+    "llvm.loongarch.lsx.vfcmp.sne.s" => "__builtin_lsx_vfcmp_sne_s",
+    "llvm.loongarch.lsx.vfcmp.sor.d" => "__builtin_lsx_vfcmp_sor_d",
+    "llvm.loongarch.lsx.vfcmp.sor.s" => "__builtin_lsx_vfcmp_sor_s",
+    "llvm.loongarch.lsx.vfcmp.sueq.d" => "__builtin_lsx_vfcmp_sueq_d",
+    "llvm.loongarch.lsx.vfcmp.sueq.s" => "__builtin_lsx_vfcmp_sueq_s",
+    "llvm.loongarch.lsx.vfcmp.sule.d" => "__builtin_lsx_vfcmp_sule_d",
+    "llvm.loongarch.lsx.vfcmp.sule.s" => "__builtin_lsx_vfcmp_sule_s",
+    "llvm.loongarch.lsx.vfcmp.sult.d" => "__builtin_lsx_vfcmp_sult_d",
+    "llvm.loongarch.lsx.vfcmp.sult.s" => "__builtin_lsx_vfcmp_sult_s",
+    "llvm.loongarch.lsx.vfcmp.sun.d" => "__builtin_lsx_vfcmp_sun_d",
+    "llvm.loongarch.lsx.vfcmp.sun.s" => "__builtin_lsx_vfcmp_sun_s",
+    "llvm.loongarch.lsx.vfcmp.sune.d" => "__builtin_lsx_vfcmp_sune_d",
+    "llvm.loongarch.lsx.vfcmp.sune.s" => "__builtin_lsx_vfcmp_sune_s",
+    "llvm.loongarch.lsx.vfcvt.h.s" => "__builtin_lsx_vfcvt_h_s",
+    "llvm.loongarch.lsx.vfcvt.s.d" => "__builtin_lsx_vfcvt_s_d",
+    "llvm.loongarch.lsx.vfcvth.d.s" => "__builtin_lsx_vfcvth_d_s",
+    "llvm.loongarch.lsx.vfcvth.s.h" => "__builtin_lsx_vfcvth_s_h",
+    "llvm.loongarch.lsx.vfcvtl.d.s" => "__builtin_lsx_vfcvtl_d_s",
+    "llvm.loongarch.lsx.vfcvtl.s.h" => "__builtin_lsx_vfcvtl_s_h",
+    "llvm.loongarch.lsx.vfdiv.d" => "__builtin_lsx_vfdiv_d",
+    "llvm.loongarch.lsx.vfdiv.s" => "__builtin_lsx_vfdiv_s",
+    "llvm.loongarch.lsx.vffint.d.l" => "__builtin_lsx_vffint_d_l",
+    "llvm.loongarch.lsx.vffint.d.lu" => "__builtin_lsx_vffint_d_lu",
+    "llvm.loongarch.lsx.vffint.s.l" => "__builtin_lsx_vffint_s_l",
+    "llvm.loongarch.lsx.vffint.s.w" => "__builtin_lsx_vffint_s_w",
+    "llvm.loongarch.lsx.vffint.s.wu" => "__builtin_lsx_vffint_s_wu",
+    "llvm.loongarch.lsx.vffinth.d.w" => "__builtin_lsx_vffinth_d_w",
+    "llvm.loongarch.lsx.vffintl.d.w" => "__builtin_lsx_vffintl_d_w",
+    "llvm.loongarch.lsx.vflogb.d" => "__builtin_lsx_vflogb_d",
+    "llvm.loongarch.lsx.vflogb.s" => "__builtin_lsx_vflogb_s",
+    "llvm.loongarch.lsx.vfmadd.d" => "__builtin_lsx_vfmadd_d",
+    "llvm.loongarch.lsx.vfmadd.s" => "__builtin_lsx_vfmadd_s",
+    "llvm.loongarch.lsx.vfmax.d" => "__builtin_lsx_vfmax_d",
+    "llvm.loongarch.lsx.vfmax.s" => "__builtin_lsx_vfmax_s",
+    "llvm.loongarch.lsx.vfmaxa.d" => "__builtin_lsx_vfmaxa_d",
+    "llvm.loongarch.lsx.vfmaxa.s" => "__builtin_lsx_vfmaxa_s",
+    "llvm.loongarch.lsx.vfmin.d" => "__builtin_lsx_vfmin_d",
+    "llvm.loongarch.lsx.vfmin.s" => "__builtin_lsx_vfmin_s",
+    "llvm.loongarch.lsx.vfmina.d" => "__builtin_lsx_vfmina_d",
+    "llvm.loongarch.lsx.vfmina.s" => "__builtin_lsx_vfmina_s",
+    "llvm.loongarch.lsx.vfmsub.d" => "__builtin_lsx_vfmsub_d",
+    "llvm.loongarch.lsx.vfmsub.s" => "__builtin_lsx_vfmsub_s",
+    "llvm.loongarch.lsx.vfmul.d" => "__builtin_lsx_vfmul_d",
+    "llvm.loongarch.lsx.vfmul.s" => "__builtin_lsx_vfmul_s",
+    "llvm.loongarch.lsx.vfnmadd.d" => "__builtin_lsx_vfnmadd_d",
+    "llvm.loongarch.lsx.vfnmadd.s" => "__builtin_lsx_vfnmadd_s",
+    "llvm.loongarch.lsx.vfnmsub.d" => "__builtin_lsx_vfnmsub_d",
+    "llvm.loongarch.lsx.vfnmsub.s" => "__builtin_lsx_vfnmsub_s",
+    "llvm.loongarch.lsx.vfrecip.d" => "__builtin_lsx_vfrecip_d",
+    "llvm.loongarch.lsx.vfrecip.s" => "__builtin_lsx_vfrecip_s",
+    "llvm.loongarch.lsx.vfrint.d" => "__builtin_lsx_vfrint_d",
+    "llvm.loongarch.lsx.vfrint.s" => "__builtin_lsx_vfrint_s",
+    "llvm.loongarch.lsx.vfrintrm.d" => "__builtin_lsx_vfrintrm_d",
+    "llvm.loongarch.lsx.vfrintrm.s" => "__builtin_lsx_vfrintrm_s",
+    "llvm.loongarch.lsx.vfrintrne.d" => "__builtin_lsx_vfrintrne_d",
+    "llvm.loongarch.lsx.vfrintrne.s" => "__builtin_lsx_vfrintrne_s",
+    "llvm.loongarch.lsx.vfrintrp.d" => "__builtin_lsx_vfrintrp_d",
+    "llvm.loongarch.lsx.vfrintrp.s" => "__builtin_lsx_vfrintrp_s",
+    "llvm.loongarch.lsx.vfrintrz.d" => "__builtin_lsx_vfrintrz_d",
+    "llvm.loongarch.lsx.vfrintrz.s" => "__builtin_lsx_vfrintrz_s",
+    "llvm.loongarch.lsx.vfrsqrt.d" => "__builtin_lsx_vfrsqrt_d",
+    "llvm.loongarch.lsx.vfrsqrt.s" => "__builtin_lsx_vfrsqrt_s",
+    "llvm.loongarch.lsx.vfrstp.b" => "__builtin_lsx_vfrstp_b",
+    "llvm.loongarch.lsx.vfrstp.h" => "__builtin_lsx_vfrstp_h",
+    "llvm.loongarch.lsx.vfrstpi.b" => "__builtin_lsx_vfrstpi_b",
+    "llvm.loongarch.lsx.vfrstpi.h" => "__builtin_lsx_vfrstpi_h",
+    "llvm.loongarch.lsx.vfsqrt.d" => "__builtin_lsx_vfsqrt_d",
+    "llvm.loongarch.lsx.vfsqrt.s" => "__builtin_lsx_vfsqrt_s",
+    "llvm.loongarch.lsx.vfsub.d" => "__builtin_lsx_vfsub_d",
+    "llvm.loongarch.lsx.vfsub.s" => "__builtin_lsx_vfsub_s",
+    "llvm.loongarch.lsx.vftint.l.d" => "__builtin_lsx_vftint_l_d",
+    "llvm.loongarch.lsx.vftint.lu.d" => "__builtin_lsx_vftint_lu_d",
+    "llvm.loongarch.lsx.vftint.w.d" => "__builtin_lsx_vftint_w_d",
+    "llvm.loongarch.lsx.vftint.w.s" => "__builtin_lsx_vftint_w_s",
+    "llvm.loongarch.lsx.vftint.wu.s" => "__builtin_lsx_vftint_wu_s",
+    "llvm.loongarch.lsx.vftinth.l.s" => "__builtin_lsx_vftinth_l_s",
+    "llvm.loongarch.lsx.vftintl.l.s" => "__builtin_lsx_vftintl_l_s",
+    "llvm.loongarch.lsx.vftintrm.l.d" => "__builtin_lsx_vftintrm_l_d",
+    "llvm.loongarch.lsx.vftintrm.w.d" => "__builtin_lsx_vftintrm_w_d",
+    "llvm.loongarch.lsx.vftintrm.w.s" => "__builtin_lsx_vftintrm_w_s",
+    "llvm.loongarch.lsx.vftintrmh.l.s" => "__builtin_lsx_vftintrmh_l_s",
+    "llvm.loongarch.lsx.vftintrml.l.s" => "__builtin_lsx_vftintrml_l_s",
+    "llvm.loongarch.lsx.vftintrne.l.d" => "__builtin_lsx_vftintrne_l_d",
+    "llvm.loongarch.lsx.vftintrne.w.d" => "__builtin_lsx_vftintrne_w_d",
+    "llvm.loongarch.lsx.vftintrne.w.s" => "__builtin_lsx_vftintrne_w_s",
+    "llvm.loongarch.lsx.vftintrneh.l.s" => "__builtin_lsx_vftintrneh_l_s",
+    "llvm.loongarch.lsx.vftintrnel.l.s" => "__builtin_lsx_vftintrnel_l_s",
+    "llvm.loongarch.lsx.vftintrp.l.d" => "__builtin_lsx_vftintrp_l_d",
+    "llvm.loongarch.lsx.vftintrp.w.d" => "__builtin_lsx_vftintrp_w_d",
+    "llvm.loongarch.lsx.vftintrp.w.s" => "__builtin_lsx_vftintrp_w_s",
+    "llvm.loongarch.lsx.vftintrph.l.s" => "__builtin_lsx_vftintrph_l_s",
+    "llvm.loongarch.lsx.vftintrpl.l.s" => "__builtin_lsx_vftintrpl_l_s",
+    "llvm.loongarch.lsx.vftintrz.l.d" => "__builtin_lsx_vftintrz_l_d",
+    "llvm.loongarch.lsx.vftintrz.lu.d" => "__builtin_lsx_vftintrz_lu_d",
+    "llvm.loongarch.lsx.vftintrz.w.d" => "__builtin_lsx_vftintrz_w_d",
+    "llvm.loongarch.lsx.vftintrz.w.s" => "__builtin_lsx_vftintrz_w_s",
+    "llvm.loongarch.lsx.vftintrz.wu.s" => "__builtin_lsx_vftintrz_wu_s",
+    "llvm.loongarch.lsx.vftintrzh.l.s" => "__builtin_lsx_vftintrzh_l_s",
+    "llvm.loongarch.lsx.vftintrzl.l.s" => "__builtin_lsx_vftintrzl_l_s",
+    "llvm.loongarch.lsx.vhaddw.d.w" => "__builtin_lsx_vhaddw_d_w",
+    "llvm.loongarch.lsx.vhaddw.du.wu" => "__builtin_lsx_vhaddw_du_wu",
+    "llvm.loongarch.lsx.vhaddw.h.b" => "__builtin_lsx_vhaddw_h_b",
+    "llvm.loongarch.lsx.vhaddw.hu.bu" => "__builtin_lsx_vhaddw_hu_bu",
+    "llvm.loongarch.lsx.vhaddw.q.d" => "__builtin_lsx_vhaddw_q_d",
+    "llvm.loongarch.lsx.vhaddw.qu.du" => "__builtin_lsx_vhaddw_qu_du",
+    "llvm.loongarch.lsx.vhaddw.w.h" => "__builtin_lsx_vhaddw_w_h",
+    "llvm.loongarch.lsx.vhaddw.wu.hu" => "__builtin_lsx_vhaddw_wu_hu",
+    "llvm.loongarch.lsx.vhsubw.d.w" => "__builtin_lsx_vhsubw_d_w",
+    "llvm.loongarch.lsx.vhsubw.du.wu" => "__builtin_lsx_vhsubw_du_wu",
+    "llvm.loongarch.lsx.vhsubw.h.b" => "__builtin_lsx_vhsubw_h_b",
+    "llvm.loongarch.lsx.vhsubw.hu.bu" => "__builtin_lsx_vhsubw_hu_bu",
+    "llvm.loongarch.lsx.vhsubw.q.d" => "__builtin_lsx_vhsubw_q_d",
+    "llvm.loongarch.lsx.vhsubw.qu.du" => "__builtin_lsx_vhsubw_qu_du",
+    "llvm.loongarch.lsx.vhsubw.w.h" => "__builtin_lsx_vhsubw_w_h",
+    "llvm.loongarch.lsx.vhsubw.wu.hu" => "__builtin_lsx_vhsubw_wu_hu",
+    "llvm.loongarch.lsx.vilvh.b" => "__builtin_lsx_vilvh_b",
+    "llvm.loongarch.lsx.vilvh.d" => "__builtin_lsx_vilvh_d",
+    "llvm.loongarch.lsx.vilvh.h" => "__builtin_lsx_vilvh_h",
+    "llvm.loongarch.lsx.vilvh.w" => "__builtin_lsx_vilvh_w",
+    "llvm.loongarch.lsx.vilvl.b" => "__builtin_lsx_vilvl_b",
+    "llvm.loongarch.lsx.vilvl.d" => "__builtin_lsx_vilvl_d",
+    "llvm.loongarch.lsx.vilvl.h" => "__builtin_lsx_vilvl_h",
+    "llvm.loongarch.lsx.vilvl.w" => "__builtin_lsx_vilvl_w",
+    "llvm.loongarch.lsx.vinsgr2vr.b" => "__builtin_lsx_vinsgr2vr_b",
+    "llvm.loongarch.lsx.vinsgr2vr.d" => "__builtin_lsx_vinsgr2vr_d",
+    "llvm.loongarch.lsx.vinsgr2vr.h" => "__builtin_lsx_vinsgr2vr_h",
+    "llvm.loongarch.lsx.vinsgr2vr.w" => "__builtin_lsx_vinsgr2vr_w",
+    "llvm.loongarch.lsx.vld" => "__builtin_lsx_vld",
+    "llvm.loongarch.lsx.vldi" => "__builtin_lsx_vldi",
+    "llvm.loongarch.lsx.vldrepl.b" => "__builtin_lsx_vldrepl_b",
+    "llvm.loongarch.lsx.vldrepl.d" => "__builtin_lsx_vldrepl_d",
+    "llvm.loongarch.lsx.vldrepl.h" => "__builtin_lsx_vldrepl_h",
+    "llvm.loongarch.lsx.vldrepl.w" => "__builtin_lsx_vldrepl_w",
+    "llvm.loongarch.lsx.vldx" => "__builtin_lsx_vldx",
+    "llvm.loongarch.lsx.vmadd.b" => "__builtin_lsx_vmadd_b",
+    "llvm.loongarch.lsx.vmadd.d" => "__builtin_lsx_vmadd_d",
+    "llvm.loongarch.lsx.vmadd.h" => "__builtin_lsx_vmadd_h",
+    "llvm.loongarch.lsx.vmadd.w" => "__builtin_lsx_vmadd_w",
+    "llvm.loongarch.lsx.vmaddwev.d.w" => "__builtin_lsx_vmaddwev_d_w",
+    "llvm.loongarch.lsx.vmaddwev.d.wu" => "__builtin_lsx_vmaddwev_d_wu",
+    "llvm.loongarch.lsx.vmaddwev.d.wu.w" => "__builtin_lsx_vmaddwev_d_wu_w",
+    "llvm.loongarch.lsx.vmaddwev.h.b" => "__builtin_lsx_vmaddwev_h_b",
+    "llvm.loongarch.lsx.vmaddwev.h.bu" => "__builtin_lsx_vmaddwev_h_bu",
+    "llvm.loongarch.lsx.vmaddwev.h.bu.b" => "__builtin_lsx_vmaddwev_h_bu_b",
+    "llvm.loongarch.lsx.vmaddwev.q.d" => "__builtin_lsx_vmaddwev_q_d",
+    "llvm.loongarch.lsx.vmaddwev.q.du" => "__builtin_lsx_vmaddwev_q_du",
+    "llvm.loongarch.lsx.vmaddwev.q.du.d" => "__builtin_lsx_vmaddwev_q_du_d",
+    "llvm.loongarch.lsx.vmaddwev.w.h" => "__builtin_lsx_vmaddwev_w_h",
+    "llvm.loongarch.lsx.vmaddwev.w.hu" => "__builtin_lsx_vmaddwev_w_hu",
+    "llvm.loongarch.lsx.vmaddwev.w.hu.h" => "__builtin_lsx_vmaddwev_w_hu_h",
+    "llvm.loongarch.lsx.vmaddwod.d.w" => "__builtin_lsx_vmaddwod_d_w",
+    "llvm.loongarch.lsx.vmaddwod.d.wu" => "__builtin_lsx_vmaddwod_d_wu",
+    "llvm.loongarch.lsx.vmaddwod.d.wu.w" => "__builtin_lsx_vmaddwod_d_wu_w",
+    "llvm.loongarch.lsx.vmaddwod.h.b" => "__builtin_lsx_vmaddwod_h_b",
+    "llvm.loongarch.lsx.vmaddwod.h.bu" => "__builtin_lsx_vmaddwod_h_bu",
+    "llvm.loongarch.lsx.vmaddwod.h.bu.b" => "__builtin_lsx_vmaddwod_h_bu_b",
+    "llvm.loongarch.lsx.vmaddwod.q.d" => "__builtin_lsx_vmaddwod_q_d",
+    "llvm.loongarch.lsx.vmaddwod.q.du" => "__builtin_lsx_vmaddwod_q_du",
+    "llvm.loongarch.lsx.vmaddwod.q.du.d" => "__builtin_lsx_vmaddwod_q_du_d",
+    "llvm.loongarch.lsx.vmaddwod.w.h" => "__builtin_lsx_vmaddwod_w_h",
+    "llvm.loongarch.lsx.vmaddwod.w.hu" => "__builtin_lsx_vmaddwod_w_hu",
+    "llvm.loongarch.lsx.vmaddwod.w.hu.h" => "__builtin_lsx_vmaddwod_w_hu_h",
+    "llvm.loongarch.lsx.vmax.b" => "__builtin_lsx_vmax_b",
+    "llvm.loongarch.lsx.vmax.bu" => "__builtin_lsx_vmax_bu",
+    "llvm.loongarch.lsx.vmax.d" => "__builtin_lsx_vmax_d",
+    "llvm.loongarch.lsx.vmax.du" => "__builtin_lsx_vmax_du",
+    "llvm.loongarch.lsx.vmax.h" => "__builtin_lsx_vmax_h",
+    "llvm.loongarch.lsx.vmax.hu" => "__builtin_lsx_vmax_hu",
+    "llvm.loongarch.lsx.vmax.w" => "__builtin_lsx_vmax_w",
+    "llvm.loongarch.lsx.vmax.wu" => "__builtin_lsx_vmax_wu",
+    "llvm.loongarch.lsx.vmaxi.b" => "__builtin_lsx_vmaxi_b",
+    "llvm.loongarch.lsx.vmaxi.bu" => "__builtin_lsx_vmaxi_bu",
+    "llvm.loongarch.lsx.vmaxi.d" => "__builtin_lsx_vmaxi_d",
+    "llvm.loongarch.lsx.vmaxi.du" => "__builtin_lsx_vmaxi_du",
+    "llvm.loongarch.lsx.vmaxi.h" => "__builtin_lsx_vmaxi_h",
+    "llvm.loongarch.lsx.vmaxi.hu" => "__builtin_lsx_vmaxi_hu",
+    "llvm.loongarch.lsx.vmaxi.w" => "__builtin_lsx_vmaxi_w",
+    "llvm.loongarch.lsx.vmaxi.wu" => "__builtin_lsx_vmaxi_wu",
+    "llvm.loongarch.lsx.vmin.b" => "__builtin_lsx_vmin_b",
+    "llvm.loongarch.lsx.vmin.bu" => "__builtin_lsx_vmin_bu",
+    "llvm.loongarch.lsx.vmin.d" => "__builtin_lsx_vmin_d",
+    "llvm.loongarch.lsx.vmin.du" => "__builtin_lsx_vmin_du",
+    "llvm.loongarch.lsx.vmin.h" => "__builtin_lsx_vmin_h",
+    "llvm.loongarch.lsx.vmin.hu" => "__builtin_lsx_vmin_hu",
+    "llvm.loongarch.lsx.vmin.w" => "__builtin_lsx_vmin_w",
+    "llvm.loongarch.lsx.vmin.wu" => "__builtin_lsx_vmin_wu",
+    "llvm.loongarch.lsx.vmini.b" => "__builtin_lsx_vmini_b",
+    "llvm.loongarch.lsx.vmini.bu" => "__builtin_lsx_vmini_bu",
+    "llvm.loongarch.lsx.vmini.d" => "__builtin_lsx_vmini_d",
+    "llvm.loongarch.lsx.vmini.du" => "__builtin_lsx_vmini_du",
+    "llvm.loongarch.lsx.vmini.h" => "__builtin_lsx_vmini_h",
+    "llvm.loongarch.lsx.vmini.hu" => "__builtin_lsx_vmini_hu",
+    "llvm.loongarch.lsx.vmini.w" => "__builtin_lsx_vmini_w",
+    "llvm.loongarch.lsx.vmini.wu" => "__builtin_lsx_vmini_wu",
+    "llvm.loongarch.lsx.vmod.b" => "__builtin_lsx_vmod_b",
+    "llvm.loongarch.lsx.vmod.bu" => "__builtin_lsx_vmod_bu",
+    "llvm.loongarch.lsx.vmod.d" => "__builtin_lsx_vmod_d",
+    "llvm.loongarch.lsx.vmod.du" => "__builtin_lsx_vmod_du",
+    "llvm.loongarch.lsx.vmod.h" => "__builtin_lsx_vmod_h",
+    "llvm.loongarch.lsx.vmod.hu" => "__builtin_lsx_vmod_hu",
+    "llvm.loongarch.lsx.vmod.w" => "__builtin_lsx_vmod_w",
+    "llvm.loongarch.lsx.vmod.wu" => "__builtin_lsx_vmod_wu",
+    "llvm.loongarch.lsx.vmskgez.b" => "__builtin_lsx_vmskgez_b",
+    "llvm.loongarch.lsx.vmskltz.b" => "__builtin_lsx_vmskltz_b",
+    "llvm.loongarch.lsx.vmskltz.d" => "__builtin_lsx_vmskltz_d",
+    "llvm.loongarch.lsx.vmskltz.h" => "__builtin_lsx_vmskltz_h",
+    "llvm.loongarch.lsx.vmskltz.w" => "__builtin_lsx_vmskltz_w",
+    "llvm.loongarch.lsx.vmsknz.b" => "__builtin_lsx_vmsknz_b",
+    "llvm.loongarch.lsx.vmsub.b" => "__builtin_lsx_vmsub_b",
+    "llvm.loongarch.lsx.vmsub.d" => "__builtin_lsx_vmsub_d",
+    "llvm.loongarch.lsx.vmsub.h" => "__builtin_lsx_vmsub_h",
+    "llvm.loongarch.lsx.vmsub.w" => "__builtin_lsx_vmsub_w",
+    "llvm.loongarch.lsx.vmuh.b" => "__builtin_lsx_vmuh_b",
+    "llvm.loongarch.lsx.vmuh.bu" => "__builtin_lsx_vmuh_bu",
+    "llvm.loongarch.lsx.vmuh.d" => "__builtin_lsx_vmuh_d",
+    "llvm.loongarch.lsx.vmuh.du" => "__builtin_lsx_vmuh_du",
+    "llvm.loongarch.lsx.vmuh.h" => "__builtin_lsx_vmuh_h",
+    "llvm.loongarch.lsx.vmuh.hu" => "__builtin_lsx_vmuh_hu",
+    "llvm.loongarch.lsx.vmuh.w" => "__builtin_lsx_vmuh_w",
+    "llvm.loongarch.lsx.vmuh.wu" => "__builtin_lsx_vmuh_wu",
+    "llvm.loongarch.lsx.vmul.b" => "__builtin_lsx_vmul_b",
+    "llvm.loongarch.lsx.vmul.d" => "__builtin_lsx_vmul_d",
+    "llvm.loongarch.lsx.vmul.h" => "__builtin_lsx_vmul_h",
+    "llvm.loongarch.lsx.vmul.w" => "__builtin_lsx_vmul_w",
+    "llvm.loongarch.lsx.vmulwev.d.w" => "__builtin_lsx_vmulwev_d_w",
+    "llvm.loongarch.lsx.vmulwev.d.wu" => "__builtin_lsx_vmulwev_d_wu",
+    "llvm.loongarch.lsx.vmulwev.d.wu.w" => "__builtin_lsx_vmulwev_d_wu_w",
+    "llvm.loongarch.lsx.vmulwev.h.b" => "__builtin_lsx_vmulwev_h_b",
+    "llvm.loongarch.lsx.vmulwev.h.bu" => "__builtin_lsx_vmulwev_h_bu",
+    "llvm.loongarch.lsx.vmulwev.h.bu.b" => "__builtin_lsx_vmulwev_h_bu_b",
+    "llvm.loongarch.lsx.vmulwev.q.d" => "__builtin_lsx_vmulwev_q_d",
+    "llvm.loongarch.lsx.vmulwev.q.du" => "__builtin_lsx_vmulwev_q_du",
+    "llvm.loongarch.lsx.vmulwev.q.du.d" => "__builtin_lsx_vmulwev_q_du_d",
+    "llvm.loongarch.lsx.vmulwev.w.h" => "__builtin_lsx_vmulwev_w_h",
+    "llvm.loongarch.lsx.vmulwev.w.hu" => "__builtin_lsx_vmulwev_w_hu",
+    "llvm.loongarch.lsx.vmulwev.w.hu.h" => "__builtin_lsx_vmulwev_w_hu_h",
+    "llvm.loongarch.lsx.vmulwod.d.w" => "__builtin_lsx_vmulwod_d_w",
+    "llvm.loongarch.lsx.vmulwod.d.wu" => "__builtin_lsx_vmulwod_d_wu",
+    "llvm.loongarch.lsx.vmulwod.d.wu.w" => "__builtin_lsx_vmulwod_d_wu_w",
+    "llvm.loongarch.lsx.vmulwod.h.b" => "__builtin_lsx_vmulwod_h_b",
+    "llvm.loongarch.lsx.vmulwod.h.bu" => "__builtin_lsx_vmulwod_h_bu",
+    "llvm.loongarch.lsx.vmulwod.h.bu.b" => "__builtin_lsx_vmulwod_h_bu_b",
+    "llvm.loongarch.lsx.vmulwod.q.d" => "__builtin_lsx_vmulwod_q_d",
+    "llvm.loongarch.lsx.vmulwod.q.du" => "__builtin_lsx_vmulwod_q_du",
+    "llvm.loongarch.lsx.vmulwod.q.du.d" => "__builtin_lsx_vmulwod_q_du_d",
+    "llvm.loongarch.lsx.vmulwod.w.h" => "__builtin_lsx_vmulwod_w_h",
+    "llvm.loongarch.lsx.vmulwod.w.hu" => "__builtin_lsx_vmulwod_w_hu",
+    "llvm.loongarch.lsx.vmulwod.w.hu.h" => "__builtin_lsx_vmulwod_w_hu_h",
+    "llvm.loongarch.lsx.vneg.b" => "__builtin_lsx_vneg_b",
+    "llvm.loongarch.lsx.vneg.d" => "__builtin_lsx_vneg_d",
+    "llvm.loongarch.lsx.vneg.h" => "__builtin_lsx_vneg_h",
+    "llvm.loongarch.lsx.vneg.w" => "__builtin_lsx_vneg_w",
+    "llvm.loongarch.lsx.vnor.v" => "__builtin_lsx_vnor_v",
+    "llvm.loongarch.lsx.vnori.b" => "__builtin_lsx_vnori_b",
+    "llvm.loongarch.lsx.vor.v" => "__builtin_lsx_vor_v",
+    "llvm.loongarch.lsx.vori.b" => "__builtin_lsx_vori_b",
+    "llvm.loongarch.lsx.vorn.v" => "__builtin_lsx_vorn_v",
+    "llvm.loongarch.lsx.vpackev.b" => "__builtin_lsx_vpackev_b",
+    "llvm.loongarch.lsx.vpackev.d" => "__builtin_lsx_vpackev_d",
+    "llvm.loongarch.lsx.vpackev.h" => "__builtin_lsx_vpackev_h",
+    "llvm.loongarch.lsx.vpackev.w" => "__builtin_lsx_vpackev_w",
+    "llvm.loongarch.lsx.vpackod.b" => "__builtin_lsx_vpackod_b",
+    "llvm.loongarch.lsx.vpackod.d" => "__builtin_lsx_vpackod_d",
+    "llvm.loongarch.lsx.vpackod.h" => "__builtin_lsx_vpackod_h",
+    "llvm.loongarch.lsx.vpackod.w" => "__builtin_lsx_vpackod_w",
+    "llvm.loongarch.lsx.vpcnt.b" => "__builtin_lsx_vpcnt_b",
+    "llvm.loongarch.lsx.vpcnt.d" => "__builtin_lsx_vpcnt_d",
+    "llvm.loongarch.lsx.vpcnt.h" => "__builtin_lsx_vpcnt_h",
+    "llvm.loongarch.lsx.vpcnt.w" => "__builtin_lsx_vpcnt_w",
+    "llvm.loongarch.lsx.vpermi.w" => "__builtin_lsx_vpermi_w",
+    "llvm.loongarch.lsx.vpickev.b" => "__builtin_lsx_vpickev_b",
+    "llvm.loongarch.lsx.vpickev.d" => "__builtin_lsx_vpickev_d",
+    "llvm.loongarch.lsx.vpickev.h" => "__builtin_lsx_vpickev_h",
+    "llvm.loongarch.lsx.vpickev.w" => "__builtin_lsx_vpickev_w",
+    "llvm.loongarch.lsx.vpickod.b" => "__builtin_lsx_vpickod_b",
+    "llvm.loongarch.lsx.vpickod.d" => "__builtin_lsx_vpickod_d",
+    "llvm.loongarch.lsx.vpickod.h" => "__builtin_lsx_vpickod_h",
+    "llvm.loongarch.lsx.vpickod.w" => "__builtin_lsx_vpickod_w",
+    "llvm.loongarch.lsx.vpickve2gr.b" => "__builtin_lsx_vpickve2gr_b",
+    "llvm.loongarch.lsx.vpickve2gr.bu" => "__builtin_lsx_vpickve2gr_bu",
+    "llvm.loongarch.lsx.vpickve2gr.d" => "__builtin_lsx_vpickve2gr_d",
+    "llvm.loongarch.lsx.vpickve2gr.du" => "__builtin_lsx_vpickve2gr_du",
+    "llvm.loongarch.lsx.vpickve2gr.h" => "__builtin_lsx_vpickve2gr_h",
+    "llvm.loongarch.lsx.vpickve2gr.hu" => "__builtin_lsx_vpickve2gr_hu",
+    "llvm.loongarch.lsx.vpickve2gr.w" => "__builtin_lsx_vpickve2gr_w",
+    "llvm.loongarch.lsx.vpickve2gr.wu" => "__builtin_lsx_vpickve2gr_wu",
+    "llvm.loongarch.lsx.vreplgr2vr.b" => "__builtin_lsx_vreplgr2vr_b",
+    "llvm.loongarch.lsx.vreplgr2vr.d" => "__builtin_lsx_vreplgr2vr_d",
+    "llvm.loongarch.lsx.vreplgr2vr.h" => "__builtin_lsx_vreplgr2vr_h",
+    "llvm.loongarch.lsx.vreplgr2vr.w" => "__builtin_lsx_vreplgr2vr_w",
+    "llvm.loongarch.lsx.vrepli.b" => "__builtin_lsx_vrepli_b",
+    "llvm.loongarch.lsx.vrepli.d" => "__builtin_lsx_vrepli_d",
+    "llvm.loongarch.lsx.vrepli.h" => "__builtin_lsx_vrepli_h",
+    "llvm.loongarch.lsx.vrepli.w" => "__builtin_lsx_vrepli_w",
+    "llvm.loongarch.lsx.vreplve.b" => "__builtin_lsx_vreplve_b",
+    "llvm.loongarch.lsx.vreplve.d" => "__builtin_lsx_vreplve_d",
+    "llvm.loongarch.lsx.vreplve.h" => "__builtin_lsx_vreplve_h",
+    "llvm.loongarch.lsx.vreplve.w" => "__builtin_lsx_vreplve_w",
+    "llvm.loongarch.lsx.vreplvei.b" => "__builtin_lsx_vreplvei_b",
+    "llvm.loongarch.lsx.vreplvei.d" => "__builtin_lsx_vreplvei_d",
+    "llvm.loongarch.lsx.vreplvei.h" => "__builtin_lsx_vreplvei_h",
+    "llvm.loongarch.lsx.vreplvei.w" => "__builtin_lsx_vreplvei_w",
+    "llvm.loongarch.lsx.vrotr.b" => "__builtin_lsx_vrotr_b",
+    "llvm.loongarch.lsx.vrotr.d" => "__builtin_lsx_vrotr_d",
+    "llvm.loongarch.lsx.vrotr.h" => "__builtin_lsx_vrotr_h",
+    "llvm.loongarch.lsx.vrotr.w" => "__builtin_lsx_vrotr_w",
+    "llvm.loongarch.lsx.vrotri.b" => "__builtin_lsx_vrotri_b",
+    "llvm.loongarch.lsx.vrotri.d" => "__builtin_lsx_vrotri_d",
+    "llvm.loongarch.lsx.vrotri.h" => "__builtin_lsx_vrotri_h",
+    "llvm.loongarch.lsx.vrotri.w" => "__builtin_lsx_vrotri_w",
+    "llvm.loongarch.lsx.vsadd.b" => "__builtin_lsx_vsadd_b",
+    "llvm.loongarch.lsx.vsadd.bu" => "__builtin_lsx_vsadd_bu",
+    "llvm.loongarch.lsx.vsadd.d" => "__builtin_lsx_vsadd_d",
+    "llvm.loongarch.lsx.vsadd.du" => "__builtin_lsx_vsadd_du",
+    "llvm.loongarch.lsx.vsadd.h" => "__builtin_lsx_vsadd_h",
+    "llvm.loongarch.lsx.vsadd.hu" => "__builtin_lsx_vsadd_hu",
+    "llvm.loongarch.lsx.vsadd.w" => "__builtin_lsx_vsadd_w",
+    "llvm.loongarch.lsx.vsadd.wu" => "__builtin_lsx_vsadd_wu",
+    "llvm.loongarch.lsx.vsat.b" => "__builtin_lsx_vsat_b",
+    "llvm.loongarch.lsx.vsat.bu" => "__builtin_lsx_vsat_bu",
+    "llvm.loongarch.lsx.vsat.d" => "__builtin_lsx_vsat_d",
+    "llvm.loongarch.lsx.vsat.du" => "__builtin_lsx_vsat_du",
+    "llvm.loongarch.lsx.vsat.h" => "__builtin_lsx_vsat_h",
+    "llvm.loongarch.lsx.vsat.hu" => "__builtin_lsx_vsat_hu",
+    "llvm.loongarch.lsx.vsat.w" => "__builtin_lsx_vsat_w",
+    "llvm.loongarch.lsx.vsat.wu" => "__builtin_lsx_vsat_wu",
+    "llvm.loongarch.lsx.vseq.b" => "__builtin_lsx_vseq_b",
+    "llvm.loongarch.lsx.vseq.d" => "__builtin_lsx_vseq_d",
+    "llvm.loongarch.lsx.vseq.h" => "__builtin_lsx_vseq_h",
+    "llvm.loongarch.lsx.vseq.w" => "__builtin_lsx_vseq_w",
+    "llvm.loongarch.lsx.vseqi.b" => "__builtin_lsx_vseqi_b",
+    "llvm.loongarch.lsx.vseqi.d" => "__builtin_lsx_vseqi_d",
+    "llvm.loongarch.lsx.vseqi.h" => "__builtin_lsx_vseqi_h",
+    "llvm.loongarch.lsx.vseqi.w" => "__builtin_lsx_vseqi_w",
+    "llvm.loongarch.lsx.vshuf.b" => "__builtin_lsx_vshuf_b",
+    "llvm.loongarch.lsx.vshuf.d" => "__builtin_lsx_vshuf_d",
+    "llvm.loongarch.lsx.vshuf.h" => "__builtin_lsx_vshuf_h",
+    "llvm.loongarch.lsx.vshuf.w" => "__builtin_lsx_vshuf_w",
+    "llvm.loongarch.lsx.vshuf4i.b" => "__builtin_lsx_vshuf4i_b",
+    "llvm.loongarch.lsx.vshuf4i.d" => "__builtin_lsx_vshuf4i_d",
+    "llvm.loongarch.lsx.vshuf4i.h" => "__builtin_lsx_vshuf4i_h",
+    "llvm.loongarch.lsx.vshuf4i.w" => "__builtin_lsx_vshuf4i_w",
+    "llvm.loongarch.lsx.vsigncov.b" => "__builtin_lsx_vsigncov_b",
+    "llvm.loongarch.lsx.vsigncov.d" => "__builtin_lsx_vsigncov_d",
+    "llvm.loongarch.lsx.vsigncov.h" => "__builtin_lsx_vsigncov_h",
+    "llvm.loongarch.lsx.vsigncov.w" => "__builtin_lsx_vsigncov_w",
+    "llvm.loongarch.lsx.vsle.b" => "__builtin_lsx_vsle_b",
+    "llvm.loongarch.lsx.vsle.bu" => "__builtin_lsx_vsle_bu",
+    "llvm.loongarch.lsx.vsle.d" => "__builtin_lsx_vsle_d",
+    "llvm.loongarch.lsx.vsle.du" => "__builtin_lsx_vsle_du",
+    "llvm.loongarch.lsx.vsle.h" => "__builtin_lsx_vsle_h",
+    "llvm.loongarch.lsx.vsle.hu" => "__builtin_lsx_vsle_hu",
+    "llvm.loongarch.lsx.vsle.w" => "__builtin_lsx_vsle_w",
+    "llvm.loongarch.lsx.vsle.wu" => "__builtin_lsx_vsle_wu",
+    "llvm.loongarch.lsx.vslei.b" => "__builtin_lsx_vslei_b",
+    "llvm.loongarch.lsx.vslei.bu" => "__builtin_lsx_vslei_bu",
+    "llvm.loongarch.lsx.vslei.d" => "__builtin_lsx_vslei_d",
+    "llvm.loongarch.lsx.vslei.du" => "__builtin_lsx_vslei_du",
+    "llvm.loongarch.lsx.vslei.h" => "__builtin_lsx_vslei_h",
+    "llvm.loongarch.lsx.vslei.hu" => "__builtin_lsx_vslei_hu",
+    "llvm.loongarch.lsx.vslei.w" => "__builtin_lsx_vslei_w",
+    "llvm.loongarch.lsx.vslei.wu" => "__builtin_lsx_vslei_wu",
+    "llvm.loongarch.lsx.vsll.b" => "__builtin_lsx_vsll_b",
+    "llvm.loongarch.lsx.vsll.d" => "__builtin_lsx_vsll_d",
+    "llvm.loongarch.lsx.vsll.h" => "__builtin_lsx_vsll_h",
+    "llvm.loongarch.lsx.vsll.w" => "__builtin_lsx_vsll_w",
+    "llvm.loongarch.lsx.vslli.b" => "__builtin_lsx_vslli_b",
+    "llvm.loongarch.lsx.vslli.d" => "__builtin_lsx_vslli_d",
+    "llvm.loongarch.lsx.vslli.h" => "__builtin_lsx_vslli_h",
+    "llvm.loongarch.lsx.vslli.w" => "__builtin_lsx_vslli_w",
+    "llvm.loongarch.lsx.vsllwil.d.w" => "__builtin_lsx_vsllwil_d_w",
+    "llvm.loongarch.lsx.vsllwil.du.wu" => "__builtin_lsx_vsllwil_du_wu",
+    "llvm.loongarch.lsx.vsllwil.h.b" => "__builtin_lsx_vsllwil_h_b",
+    "llvm.loongarch.lsx.vsllwil.hu.bu" => "__builtin_lsx_vsllwil_hu_bu",
+    "llvm.loongarch.lsx.vsllwil.w.h" => "__builtin_lsx_vsllwil_w_h",
+    "llvm.loongarch.lsx.vsllwil.wu.hu" => "__builtin_lsx_vsllwil_wu_hu",
+    "llvm.loongarch.lsx.vslt.b" => "__builtin_lsx_vslt_b",
+    "llvm.loongarch.lsx.vslt.bu" => "__builtin_lsx_vslt_bu",
+    "llvm.loongarch.lsx.vslt.d" => "__builtin_lsx_vslt_d",
+    "llvm.loongarch.lsx.vslt.du" => "__builtin_lsx_vslt_du",
+    "llvm.loongarch.lsx.vslt.h" => "__builtin_lsx_vslt_h",
+    "llvm.loongarch.lsx.vslt.hu" => "__builtin_lsx_vslt_hu",
+    "llvm.loongarch.lsx.vslt.w" => "__builtin_lsx_vslt_w",
+    "llvm.loongarch.lsx.vslt.wu" => "__builtin_lsx_vslt_wu",
+    "llvm.loongarch.lsx.vslti.b" => "__builtin_lsx_vslti_b",
+    "llvm.loongarch.lsx.vslti.bu" => "__builtin_lsx_vslti_bu",
+    "llvm.loongarch.lsx.vslti.d" => "__builtin_lsx_vslti_d",
+    "llvm.loongarch.lsx.vslti.du" => "__builtin_lsx_vslti_du",
+    "llvm.loongarch.lsx.vslti.h" => "__builtin_lsx_vslti_h",
+    "llvm.loongarch.lsx.vslti.hu" => "__builtin_lsx_vslti_hu",
+    "llvm.loongarch.lsx.vslti.w" => "__builtin_lsx_vslti_w",
+    "llvm.loongarch.lsx.vslti.wu" => "__builtin_lsx_vslti_wu",
+    "llvm.loongarch.lsx.vsra.b" => "__builtin_lsx_vsra_b",
+    "llvm.loongarch.lsx.vsra.d" => "__builtin_lsx_vsra_d",
+    "llvm.loongarch.lsx.vsra.h" => "__builtin_lsx_vsra_h",
+    "llvm.loongarch.lsx.vsra.w" => "__builtin_lsx_vsra_w",
+    "llvm.loongarch.lsx.vsrai.b" => "__builtin_lsx_vsrai_b",
+    "llvm.loongarch.lsx.vsrai.d" => "__builtin_lsx_vsrai_d",
+    "llvm.loongarch.lsx.vsrai.h" => "__builtin_lsx_vsrai_h",
+    "llvm.loongarch.lsx.vsrai.w" => "__builtin_lsx_vsrai_w",
+    "llvm.loongarch.lsx.vsran.b.h" => "__builtin_lsx_vsran_b_h",
+    "llvm.loongarch.lsx.vsran.h.w" => "__builtin_lsx_vsran_h_w",
+    "llvm.loongarch.lsx.vsran.w.d" => "__builtin_lsx_vsran_w_d",
+    "llvm.loongarch.lsx.vsrani.b.h" => "__builtin_lsx_vsrani_b_h",
+    "llvm.loongarch.lsx.vsrani.d.q" => "__builtin_lsx_vsrani_d_q",
+    "llvm.loongarch.lsx.vsrani.h.w" => "__builtin_lsx_vsrani_h_w",
+    "llvm.loongarch.lsx.vsrani.w.d" => "__builtin_lsx_vsrani_w_d",
+    "llvm.loongarch.lsx.vsrar.b" => "__builtin_lsx_vsrar_b",
+    "llvm.loongarch.lsx.vsrar.d" => "__builtin_lsx_vsrar_d",
+    "llvm.loongarch.lsx.vsrar.h" => "__builtin_lsx_vsrar_h",
+    "llvm.loongarch.lsx.vsrar.w" => "__builtin_lsx_vsrar_w",
+    "llvm.loongarch.lsx.vsrari.b" => "__builtin_lsx_vsrari_b",
+    "llvm.loongarch.lsx.vsrari.d" => "__builtin_lsx_vsrari_d",
+    "llvm.loongarch.lsx.vsrari.h" => "__builtin_lsx_vsrari_h",
+    "llvm.loongarch.lsx.vsrari.w" => "__builtin_lsx_vsrari_w",
+    "llvm.loongarch.lsx.vsrarn.b.h" => "__builtin_lsx_vsrarn_b_h",
+    "llvm.loongarch.lsx.vsrarn.h.w" => "__builtin_lsx_vsrarn_h_w",
+    "llvm.loongarch.lsx.vsrarn.w.d" => "__builtin_lsx_vsrarn_w_d",
+    "llvm.loongarch.lsx.vsrarni.b.h" => "__builtin_lsx_vsrarni_b_h",
+    "llvm.loongarch.lsx.vsrarni.d.q" => "__builtin_lsx_vsrarni_d_q",
+    "llvm.loongarch.lsx.vsrarni.h.w" => "__builtin_lsx_vsrarni_h_w",
+    "llvm.loongarch.lsx.vsrarni.w.d" => "__builtin_lsx_vsrarni_w_d",
+    "llvm.loongarch.lsx.vsrl.b" => "__builtin_lsx_vsrl_b",
+    "llvm.loongarch.lsx.vsrl.d" => "__builtin_lsx_vsrl_d",
+    "llvm.loongarch.lsx.vsrl.h" => "__builtin_lsx_vsrl_h",
+    "llvm.loongarch.lsx.vsrl.w" => "__builtin_lsx_vsrl_w",
+    "llvm.loongarch.lsx.vsrli.b" => "__builtin_lsx_vsrli_b",
+    "llvm.loongarch.lsx.vsrli.d" => "__builtin_lsx_vsrli_d",
+    "llvm.loongarch.lsx.vsrli.h" => "__builtin_lsx_vsrli_h",
+    "llvm.loongarch.lsx.vsrli.w" => "__builtin_lsx_vsrli_w",
+    "llvm.loongarch.lsx.vsrln.b.h" => "__builtin_lsx_vsrln_b_h",
+    "llvm.loongarch.lsx.vsrln.h.w" => "__builtin_lsx_vsrln_h_w",
+    "llvm.loongarch.lsx.vsrln.w.d" => "__builtin_lsx_vsrln_w_d",
+    "llvm.loongarch.lsx.vsrlni.b.h" => "__builtin_lsx_vsrlni_b_h",
+    "llvm.loongarch.lsx.vsrlni.d.q" => "__builtin_lsx_vsrlni_d_q",
+    "llvm.loongarch.lsx.vsrlni.h.w" => "__builtin_lsx_vsrlni_h_w",
+    "llvm.loongarch.lsx.vsrlni.w.d" => "__builtin_lsx_vsrlni_w_d",
+    "llvm.loongarch.lsx.vsrlr.b" => "__builtin_lsx_vsrlr_b",
+    "llvm.loongarch.lsx.vsrlr.d" => "__builtin_lsx_vsrlr_d",
+    "llvm.loongarch.lsx.vsrlr.h" => "__builtin_lsx_vsrlr_h",
+    "llvm.loongarch.lsx.vsrlr.w" => "__builtin_lsx_vsrlr_w",
+    "llvm.loongarch.lsx.vsrlri.b" => "__builtin_lsx_vsrlri_b",
+    "llvm.loongarch.lsx.vsrlri.d" => "__builtin_lsx_vsrlri_d",
+    "llvm.loongarch.lsx.vsrlri.h" => "__builtin_lsx_vsrlri_h",
+    "llvm.loongarch.lsx.vsrlri.w" => "__builtin_lsx_vsrlri_w",
+    "llvm.loongarch.lsx.vsrlrn.b.h" => "__builtin_lsx_vsrlrn_b_h",
+    "llvm.loongarch.lsx.vsrlrn.h.w" => "__builtin_lsx_vsrlrn_h_w",
+    "llvm.loongarch.lsx.vsrlrn.w.d" => "__builtin_lsx_vsrlrn_w_d",
+    "llvm.loongarch.lsx.vsrlrni.b.h" => "__builtin_lsx_vsrlrni_b_h",
+    "llvm.loongarch.lsx.vsrlrni.d.q" => "__builtin_lsx_vsrlrni_d_q",
+    "llvm.loongarch.lsx.vsrlrni.h.w" => "__builtin_lsx_vsrlrni_h_w",
+    "llvm.loongarch.lsx.vsrlrni.w.d" => "__builtin_lsx_vsrlrni_w_d",
+    "llvm.loongarch.lsx.vssran.b.h" => "__builtin_lsx_vssran_b_h",
+    "llvm.loongarch.lsx.vssran.bu.h" => "__builtin_lsx_vssran_bu_h",
+    "llvm.loongarch.lsx.vssran.h.w" => "__builtin_lsx_vssran_h_w",
+    "llvm.loongarch.lsx.vssran.hu.w" => "__builtin_lsx_vssran_hu_w",
+    "llvm.loongarch.lsx.vssran.w.d" => "__builtin_lsx_vssran_w_d",
+    "llvm.loongarch.lsx.vssran.wu.d" => "__builtin_lsx_vssran_wu_d",
+    "llvm.loongarch.lsx.vssrani.b.h" => "__builtin_lsx_vssrani_b_h",
+    "llvm.loongarch.lsx.vssrani.bu.h" => "__builtin_lsx_vssrani_bu_h",
+    "llvm.loongarch.lsx.vssrani.d.q" => "__builtin_lsx_vssrani_d_q",
+    "llvm.loongarch.lsx.vssrani.du.q" => "__builtin_lsx_vssrani_du_q",
+    "llvm.loongarch.lsx.vssrani.h.w" => "__builtin_lsx_vssrani_h_w",
+    "llvm.loongarch.lsx.vssrani.hu.w" => "__builtin_lsx_vssrani_hu_w",
+    "llvm.loongarch.lsx.vssrani.w.d" => "__builtin_lsx_vssrani_w_d",
+    "llvm.loongarch.lsx.vssrani.wu.d" => "__builtin_lsx_vssrani_wu_d",
+    "llvm.loongarch.lsx.vssrarn.b.h" => "__builtin_lsx_vssrarn_b_h",
+    "llvm.loongarch.lsx.vssrarn.bu.h" => "__builtin_lsx_vssrarn_bu_h",
+    "llvm.loongarch.lsx.vssrarn.h.w" => "__builtin_lsx_vssrarn_h_w",
+    "llvm.loongarch.lsx.vssrarn.hu.w" => "__builtin_lsx_vssrarn_hu_w",
+    "llvm.loongarch.lsx.vssrarn.w.d" => "__builtin_lsx_vssrarn_w_d",
+    "llvm.loongarch.lsx.vssrarn.wu.d" => "__builtin_lsx_vssrarn_wu_d",
+    "llvm.loongarch.lsx.vssrarni.b.h" => "__builtin_lsx_vssrarni_b_h",
+    "llvm.loongarch.lsx.vssrarni.bu.h" => "__builtin_lsx_vssrarni_bu_h",
+    "llvm.loongarch.lsx.vssrarni.d.q" => "__builtin_lsx_vssrarni_d_q",
+    "llvm.loongarch.lsx.vssrarni.du.q" => "__builtin_lsx_vssrarni_du_q",
+    "llvm.loongarch.lsx.vssrarni.h.w" => "__builtin_lsx_vssrarni_h_w",
+    "llvm.loongarch.lsx.vssrarni.hu.w" => "__builtin_lsx_vssrarni_hu_w",
+    "llvm.loongarch.lsx.vssrarni.w.d" => "__builtin_lsx_vssrarni_w_d",
+    "llvm.loongarch.lsx.vssrarni.wu.d" => "__builtin_lsx_vssrarni_wu_d",
+    "llvm.loongarch.lsx.vssrln.b.h" => "__builtin_lsx_vssrln_b_h",
+    "llvm.loongarch.lsx.vssrln.bu.h" => "__builtin_lsx_vssrln_bu_h",
+    "llvm.loongarch.lsx.vssrln.h.w" => "__builtin_lsx_vssrln_h_w",
+    "llvm.loongarch.lsx.vssrln.hu.w" => "__builtin_lsx_vssrln_hu_w",
+    "llvm.loongarch.lsx.vssrln.w.d" => "__builtin_lsx_vssrln_w_d",
+    "llvm.loongarch.lsx.vssrln.wu.d" => "__builtin_lsx_vssrln_wu_d",
+    "llvm.loongarch.lsx.vssrlni.b.h" => "__builtin_lsx_vssrlni_b_h",
+    "llvm.loongarch.lsx.vssrlni.bu.h" => "__builtin_lsx_vssrlni_bu_h",
+    "llvm.loongarch.lsx.vssrlni.d.q" => "__builtin_lsx_vssrlni_d_q",
+    "llvm.loongarch.lsx.vssrlni.du.q" => "__builtin_lsx_vssrlni_du_q",
+    "llvm.loongarch.lsx.vssrlni.h.w" => "__builtin_lsx_vssrlni_h_w",
+    "llvm.loongarch.lsx.vssrlni.hu.w" => "__builtin_lsx_vssrlni_hu_w",
+    "llvm.loongarch.lsx.vssrlni.w.d" => "__builtin_lsx_vssrlni_w_d",
+    "llvm.loongarch.lsx.vssrlni.wu.d" => "__builtin_lsx_vssrlni_wu_d",
+    "llvm.loongarch.lsx.vssrlrn.b.h" => "__builtin_lsx_vssrlrn_b_h",
+    "llvm.loongarch.lsx.vssrlrn.bu.h" => "__builtin_lsx_vssrlrn_bu_h",
+    "llvm.loongarch.lsx.vssrlrn.h.w" => "__builtin_lsx_vssrlrn_h_w",
+    "llvm.loongarch.lsx.vssrlrn.hu.w" => "__builtin_lsx_vssrlrn_hu_w",
+    "llvm.loongarch.lsx.vssrlrn.w.d" => "__builtin_lsx_vssrlrn_w_d",
+    "llvm.loongarch.lsx.vssrlrn.wu.d" => "__builtin_lsx_vssrlrn_wu_d",
+    "llvm.loongarch.lsx.vssrlrni.b.h" => "__builtin_lsx_vssrlrni_b_h",
+    "llvm.loongarch.lsx.vssrlrni.bu.h" => "__builtin_lsx_vssrlrni_bu_h",
+    "llvm.loongarch.lsx.vssrlrni.d.q" => "__builtin_lsx_vssrlrni_d_q",
+    "llvm.loongarch.lsx.vssrlrni.du.q" => "__builtin_lsx_vssrlrni_du_q",
+    "llvm.loongarch.lsx.vssrlrni.h.w" => "__builtin_lsx_vssrlrni_h_w",
+    "llvm.loongarch.lsx.vssrlrni.hu.w" => "__builtin_lsx_vssrlrni_hu_w",
+    "llvm.loongarch.lsx.vssrlrni.w.d" => "__builtin_lsx_vssrlrni_w_d",
+    "llvm.loongarch.lsx.vssrlrni.wu.d" => "__builtin_lsx_vssrlrni_wu_d",
+    "llvm.loongarch.lsx.vssub.b" => "__builtin_lsx_vssub_b",
+    "llvm.loongarch.lsx.vssub.bu" => "__builtin_lsx_vssub_bu",
+    "llvm.loongarch.lsx.vssub.d" => "__builtin_lsx_vssub_d",
+    "llvm.loongarch.lsx.vssub.du" => "__builtin_lsx_vssub_du",
+    "llvm.loongarch.lsx.vssub.h" => "__builtin_lsx_vssub_h",
+    "llvm.loongarch.lsx.vssub.hu" => "__builtin_lsx_vssub_hu",
+    "llvm.loongarch.lsx.vssub.w" => "__builtin_lsx_vssub_w",
+    "llvm.loongarch.lsx.vssub.wu" => "__builtin_lsx_vssub_wu",
+    "llvm.loongarch.lsx.vst" => "__builtin_lsx_vst",
+    "llvm.loongarch.lsx.vstelm.b" => "__builtin_lsx_vstelm_b",
+    "llvm.loongarch.lsx.vstelm.d" => "__builtin_lsx_vstelm_d",
+    "llvm.loongarch.lsx.vstelm.h" => "__builtin_lsx_vstelm_h",
+    "llvm.loongarch.lsx.vstelm.w" => "__builtin_lsx_vstelm_w",
+    "llvm.loongarch.lsx.vstx" => "__builtin_lsx_vstx",
+    "llvm.loongarch.lsx.vsub.b" => "__builtin_lsx_vsub_b",
+    "llvm.loongarch.lsx.vsub.d" => "__builtin_lsx_vsub_d",
+    "llvm.loongarch.lsx.vsub.h" => "__builtin_lsx_vsub_h",
+    "llvm.loongarch.lsx.vsub.q" => "__builtin_lsx_vsub_q",
+    "llvm.loongarch.lsx.vsub.w" => "__builtin_lsx_vsub_w",
+    "llvm.loongarch.lsx.vsubi.bu" => "__builtin_lsx_vsubi_bu",
+    "llvm.loongarch.lsx.vsubi.du" => "__builtin_lsx_vsubi_du",
+    "llvm.loongarch.lsx.vsubi.hu" => "__builtin_lsx_vsubi_hu",
+    "llvm.loongarch.lsx.vsubi.wu" => "__builtin_lsx_vsubi_wu",
+    "llvm.loongarch.lsx.vsubwev.d.w" => "__builtin_lsx_vsubwev_d_w",
+    "llvm.loongarch.lsx.vsubwev.d.wu" => "__builtin_lsx_vsubwev_d_wu",
+    "llvm.loongarch.lsx.vsubwev.h.b" => "__builtin_lsx_vsubwev_h_b",
+    "llvm.loongarch.lsx.vsubwev.h.bu" => "__builtin_lsx_vsubwev_h_bu",
+    "llvm.loongarch.lsx.vsubwev.q.d" => "__builtin_lsx_vsubwev_q_d",
+    "llvm.loongarch.lsx.vsubwev.q.du" => "__builtin_lsx_vsubwev_q_du",
+    "llvm.loongarch.lsx.vsubwev.w.h" => "__builtin_lsx_vsubwev_w_h",
+    "llvm.loongarch.lsx.vsubwev.w.hu" => "__builtin_lsx_vsubwev_w_hu",
+    "llvm.loongarch.lsx.vsubwod.d.w" => "__builtin_lsx_vsubwod_d_w",
+    "llvm.loongarch.lsx.vsubwod.d.wu" => "__builtin_lsx_vsubwod_d_wu",
+    "llvm.loongarch.lsx.vsubwod.h.b" => "__builtin_lsx_vsubwod_h_b",
+    "llvm.loongarch.lsx.vsubwod.h.bu" => "__builtin_lsx_vsubwod_h_bu",
+    "llvm.loongarch.lsx.vsubwod.q.d" => "__builtin_lsx_vsubwod_q_d",
+    "llvm.loongarch.lsx.vsubwod.q.du" => "__builtin_lsx_vsubwod_q_du",
+    "llvm.loongarch.lsx.vsubwod.w.h" => "__builtin_lsx_vsubwod_w_h",
+    "llvm.loongarch.lsx.vsubwod.w.hu" => "__builtin_lsx_vsubwod_w_hu",
+    "llvm.loongarch.lsx.vxor.v" => "__builtin_lsx_vxor_v",
+    "llvm.loongarch.lsx.vxori.b" => "__builtin_lsx_vxori_b",
     "llvm.loongarch.movfcsr2gr" => "__builtin_loongarch_movfcsr2gr",
     "llvm.loongarch.movgr2fcsr" => "__builtin_loongarch_movgr2fcsr",
     "llvm.loongarch.syscall" => "__builtin_loongarch_syscall",
@@ -4033,6 +5485,7 @@ match name {
     "llvm.ppc.maddhd" => "__builtin_ppc_maddhd",
     "llvm.ppc.maddhdu" => "__builtin_ppc_maddhdu",
     "llvm.ppc.maddld" => "__builtin_ppc_maddld",
+    "llvm.ppc.mffsl" => "__builtin_ppc_mffsl",
     "llvm.ppc.mfmsr" => "__builtin_ppc_mfmsr",
     "llvm.ppc.mftbu" => "__builtin_ppc_mftbu",
     "llvm.ppc.mtfsb0" => "__builtin_ppc_mtfsb0",
@@ -7970,6 +9423,8 @@ match name {
     "llvm.x86.tpause" => "__builtin_ia32_tpause",
     "llvm.x86.umonitor" => "__builtin_ia32_umonitor",
     "llvm.x86.umwait" => "__builtin_ia32_umwait",
+    "llvm.x86.urdmsr" => "__builtin_ia32_urdmsr",
+    "llvm.x86.uwrmsr" => "__builtin_ia32_uwrmsr",
     "llvm.x86.vbcstnebf162ps128" => "__builtin_ia32_vbcstnebf162ps128",
     "llvm.x86.vbcstnebf162ps256" => "__builtin_ia32_vbcstnebf162ps256",
     "llvm.x86.vbcstnesh2ps128" => "__builtin_ia32_vbcstnesh2ps128",

From 783789f8313819a605782d9291050707cc11941a Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Thu, 19 Oct 2023 07:48:42 -0400
Subject: [PATCH 081/297] Build the sysroot and run more tests

---
 .github/workflows/failures.yml                | 44 +++++++++++++------
 .github/workflows/gcc12.yml                   | 37 ++++++++--------
 build_system/src/build.rs                     | 30 ++++++++-----
 build_system/src/prepare.rs                   | 25 ++++++++---
 failing-ui-tests12.txt                        |  8 ++--
 ...0001-core-Disable-portable-simd-test.patch | 32 ++++++++++++++
 src/intrinsic/llvm.rs                         | 24 ++++++----
 src/type_.rs                                  | 10 ++---
 tests/lang_tests_common.rs                    | 26 +++++++++--
 9 files changed, 167 insertions(+), 69 deletions(-)
 create mode 100644 patches/libgccjit12/0001-core-Disable-portable-simd-test.patch

diff --git a/.github/workflows/failures.yml b/.github/workflows/failures.yml
index e6a9716d18cc..27864dcadd0f 100644
--- a/.github/workflows/failures.yml
+++ b/.github/workflows/failures.yml
@@ -19,8 +19,16 @@ jobs:
       fail-fast: false
       matrix:
         libgccjit_version:
-          - { gcc: "libgccjit.so", artifacts_branch: "master" }
-          - { gcc: "libgccjit_without_int128.so", artifacts_branch: "master-without-128bit-integers" }
+          - gcc: "libgccjit.so"
+            artifacts_branch: "master"
+          - gcc: "libgccjit_without_int128.so"
+            artifacts_branch: "master-without-128bit-integers"
+          - gcc: "libgccjit12.so"
+            artifacts_branch: "gcc12"
+            extra: "--no-default-features"
+            # FIXME(antoyo): we need to set GCC_EXEC_PREFIX so that the linker can find the linker plugin.
+            # Not sure why it's not found otherwise.
+            env_extra: "TEST_FLAGS='-Cpanic=abort -Zpanic-abort-tests' GCC_EXEC_PREFIX=/usr/lib/gcc/"
 
     steps:
     - uses: actions/checkout@v3
@@ -28,7 +36,16 @@ jobs:
     - name: Install packages
       run: sudo apt-get install ninja-build ripgrep
 
+    - name: Install libgccjit12
+      if: matrix.libgccjit_version.gcc == 'libgccjit12.so'
+      run: sudo apt-get install libgccjit-12-dev
+
+    - name: Setup path to libgccjit
+      if: matrix.libgccjit_version.gcc == 'libgccjit12.so'
+      run: echo /usr/lib/gcc/x86_64-linux-gnu/12 > gcc_path
+
     - name: Download artifact
+      if: matrix.libgccjit_version.gcc != 'libgccjit12.so'
       uses: dawidd6/action-download-artifact@v2
       with:
           workflow: main.yml
@@ -40,6 +57,7 @@ jobs:
           search_artifacts: true # Because, instead, the action only check the last job ran and that won't work since we want multiple artifacts.
 
     - name: Setup path to libgccjit
+      if: matrix.libgccjit_version.gcc != 'libgccjit12.so'
       run: |
           sudo dpkg --force-overwrite -i gcc-13/gcc-13.deb
           echo /usr/lib/ > gcc_path
@@ -81,18 +99,18 @@ jobs:
         #path: rust
         #key: ${{ runner.os }}-packages-${{ hashFiles('rust/.git/HEAD') }}
 
-    - name: Build
-      run: |
-        ./y.sh prepare --only-libcore
-        ./y.sh build
-        cargo test
-        ./clean_all.sh
-
-    - name: Prepare dependencies
+    - name: Git config
       run: |
         git config --global user.email "user@example.com"
         git config --global user.name "User"
-        ./y.sh prepare
+
+    - name: Prepare dependencies
+      if: matrix.libgccjit_version.gcc == 'libgccjit12.so'
+      run: ./y.sh prepare --libgccjit12-patches
+
+    - name: Prepare dependencies
+      if: matrix.libgccjit_version.gcc != 'libgccjit12.so'
+      run: ./y.sh prepare
 
     # Compile is a separate step, as the actions-rs/cargo action supports error annotations
     - name: Compile
@@ -107,5 +125,5 @@ jobs:
     - name: Run tests
       id: tests
       run: |
-        ./test.sh --release --clean --build-sysroot --test-failing-rustc | tee output_log
-        rg "test result" output_log >> $GITHUB_STEP_SUMMARY
+        ${{ matrix.libgccjit_version.env_extra }} ./test.sh --release --clean --build-sysroot --test-failing-rustc ${{ matrix.libgccjit_version.extra }} | tee output_log
+        rg --text "test result" output_log >> $GITHUB_STEP_SUMMARY
diff --git a/.github/workflows/gcc12.yml b/.github/workflows/gcc12.yml
index 295f43acb385..a0d363cf1fbd 100644
--- a/.github/workflows/gcc12.yml
+++ b/.github/workflows/gcc12.yml
@@ -11,6 +11,9 @@ env:
   # Enable backtraces for easier debugging
   RUST_BACKTRACE: 1
   TEST_FLAGS: "-Cpanic=abort -Zpanic-abort-tests"
+  # FIXME(antoyo): we need to set GCC_EXEC_PREFIX so that the linker can find the linker plugin.
+  # Not sure why it's not found otherwise.
+  GCC_EXEC_PREFIX: /usr/lib/gcc/
 
 jobs:
   build:
@@ -21,17 +24,15 @@ jobs:
       matrix:
         commands: [
           "--mini-tests",
-          # TODO(antoyo): re-enable those commands when the build with libgccjit 12 is fixed.
-          #"--std-tests",
+          "--std-tests",
           # FIXME: re-enable asm tests when GCC can emit in the right syntax.
           # "--asm-tests",
-          #"--test-libcore",
-          #"--extended-rand-tests",
-          #"--extended-regex-example-tests",
-          #"--extended-regex-tests",
-          #"--test-successful-rustc --nb-parts 2 --current-part 0",
-          #"--test-successful-rustc --nb-parts 2 --current-part 1",
-          #"--test-failing-rustc",
+          "--test-libcore",
+          "--extended-rand-tests",
+          "--extended-regex-example-tests",
+          "--extended-regex-tests",
+          "--test-successful-rustc --nb-parts 2 --current-part 0",
+          "--test-successful-rustc --nb-parts 2 --current-part 1",
         ]
 
     steps:
@@ -85,18 +86,16 @@ jobs:
 
     - name: Build
       run: |
-        ./y.sh prepare --only-libcore
-        # TODO(antoyo): build the sysroot when the build with libgccjit 12 is fixed.
-        #./y.sh build --no-default-features
-        # TODO(antoyo): run the tests when we can build the sysroot with libgccjit 12.
-        #cargo test --no-default-features
+        ./y.sh prepare --only-libcore --libgccjit12-patches
+        ./y.sh build --no-default-features --sysroot-panic-abort
+        cargo test --no-default-features
         ./clean_all.sh
 
     - name: Prepare dependencies
       run: |
         git config --global user.email "user@example.com"
         git config --global user.name "User"
-        ./y.sh prepare
+        ./y.sh prepare --libgccjit12-patches
 
     # Compile is a separate step, as the actions-rs/cargo action supports error annotations
     - name: Compile
@@ -108,9 +107,9 @@ jobs:
     - name: Add more failing tests for GCC 12
       run: cat failing-ui-tests12.txt >> failing-ui-tests.txt
 
+    - name: Add more failing tests because the sysroot is not compiled with LTO
+      run: cat failing-non-lto-tests.txt >> failing-ui-tests.txt
+
     - name: Run tests
       run: |
-        # TODO(antoyo): add --build-sysroot when the build with libgccjit 12 is fixed.
-        # FIXME(antoyo): we need to set GCC_EXEC_PREFIX so that the linker can find the linker plugin.
-        # Not sure why it's not found otherwise.
-        GCC_EXEC_PREFIX=/usr/lib/gcc/ ./test.sh --release --clean ${{ matrix.commands }} --no-default-features
+        ./test.sh --release --clean --build-sysroot ${{ matrix.commands }} --no-default-features
diff --git a/build_system/src/build.rs b/build_system/src/build.rs
index c71954e4d113..eaca7a987d6b 100644
--- a/build_system/src/build.rs
+++ b/build_system/src/build.rs
@@ -11,6 +11,7 @@ use std::path::Path;
 struct BuildArg {
     codegen_release_channel: bool,
     sysroot_release_channel: bool,
+    sysroot_panic_abort: bool,
     flags: Vec<String>,
     gcc_path: String,
 }
@@ -32,6 +33,9 @@ impl BuildArg {
                 "--no-default-features" => {
                     build_arg.flags.push("--no-default-features".to_string());
                 }
+                "--sysroot-panic-abort" => {
+                    build_arg.sysroot_panic_abort = true;
+                },
                 "--features" => {
                     if let Some(arg) = args.next() {
                         build_arg.flags.push("--features".to_string());
@@ -77,6 +81,7 @@ impl BuildArg {
 
     --release              : Build codegen in release mode
     --release-sysroot      : Build sysroot in release mode
+    --sysroot-panic-abort  : Build the sysroot without unwinding support.
     --no-default-features  : Add `--no-default-features` flag
     --features [arg]       : Add a new feature [arg]
     --target-triple [arg]  : Set the target triple to [arg]
@@ -88,7 +93,7 @@ impl BuildArg {
 
 fn build_sysroot(
     env: &mut HashMap<String, String>,
-    release_mode: bool,
+    args: &BuildArg,
     config: &ConfigInfo,
 ) -> Result<(), String> {
     std::env::set_current_dir("build_sysroot")
@@ -138,15 +143,18 @@ fn build_sysroot(
     let _ = fs::remove_dir_all("sysroot");
 
     // Builds libs
-    let channel = if release_mode {
-        let rustflags = env
-            .get("RUSTFLAGS")
-            .cloned()
-            .unwrap_or_default();
-        env.insert(
-            "RUSTFLAGS".to_string(),
-            format!("{} -Zmir-opt-level=3", rustflags),
-        );
+    let mut rustflags = env
+        .get("RUSTFLAGS")
+        .cloned()
+        .unwrap_or_default();
+    if args.sysroot_panic_abort {
+        rustflags.push_str(" -Cpanic=abort -Zpanic-abort-tests");
+    }
+    env.insert(
+        "RUSTFLAGS".to_string(),
+        format!("{} -Zmir-opt-level=3", rustflags),
+    );
+    let channel = if args.sysroot_release_channel {
         run_command_with_output_and_env(
             &[
                 &"cargo",
@@ -224,7 +232,7 @@ fn build_codegen(args: &BuildArg) -> Result<(), String> {
     println!("[BUILD] sysroot");
     build_sysroot(
         &mut env,
-        args.sysroot_release_channel,
+        args,
         &config,
     )?;
     Ok(())
diff --git a/build_system/src/prepare.rs b/build_system/src/prepare.rs
index d5d034c419c6..6c7c85868345 100644
--- a/build_system/src/prepare.rs
+++ b/build_system/src/prepare.rs
@@ -4,7 +4,7 @@ use crate::utils::{cargo_install, git_clone, run_command, run_command_with_outpu
 use std::fs;
 use std::path::Path;
 
-fn prepare_libcore(sysroot_path: &Path, cross_compile: bool) -> Result<(), String> {
+fn prepare_libcore(sysroot_path: &Path, libgccjit12_patches: bool, cross_compile: bool) -> Result<(), String> {
     let rustc_path = match get_rustc_path() {
         Some(path) => path,
         None => return Err("`rustc` path not found".to_string()),
@@ -93,6 +93,16 @@ fn prepare_libcore(sysroot_path: &Path, cross_compile: bool) -> Result<(), Strin
             Ok(())
         })?;
     }
+    if libgccjit12_patches {
+        walk_dir(
+            "patches/libgccjit12",
+            |_| Ok(()),
+            |file_path: &Path| {
+                patches.push(file_path.to_path_buf());
+                Ok(())
+            },
+        )?;
+    }
     patches.sort();
     for file_path in patches {
         println!("[GIT] apply `{}`", file_path.display());
@@ -164,17 +174,20 @@ where
 struct PrepareArg {
     cross_compile: bool,
     only_libcore: bool,
+    libgccjit12_patches: bool,
 }
 
 impl PrepareArg {
     fn new() -> Result<Option<Self>, String> {
         let mut only_libcore = false;
         let mut cross_compile = false;
+        let mut libgccjit12_patches = false;
 
         for arg in std::env::args().skip(2) {
             match arg.as_str() {
                 "--only-libcore" => only_libcore = true,
                 "--cross" => cross_compile = true,
+                "--libgccjit12-patches" => libgccjit12_patches = true,
                 "--help" => {
                     Self::usage();
                     return Ok(None);
@@ -185,6 +198,7 @@ impl PrepareArg {
         Ok(Some(Self {
             cross_compile,
             only_libcore,
+            libgccjit12_patches,
         }))
     }
 
@@ -193,9 +207,10 @@ impl PrepareArg {
             r#"
 `prepare` command help:
 
-    --only-libcore  : Only setup libcore and don't clone other repositories
-    --cross         : Apply the patches needed to do cross-compilation
-    --help          : Show this help
+    --only-libcore           : Only setup libcore and don't clone other repositories
+    --cross                  : Apply the patches needed to do cross-compilation
+    --libgccjit12-patches    : Apply patches needed for libgccjit12
+    --help                   : Show this help
 "#
         )
     }
@@ -207,7 +222,7 @@ pub fn run() -> Result<(), String> {
         None => return Ok(()),
     };
     let sysroot_path = Path::new("build_sysroot");
-    prepare_libcore(sysroot_path, args.cross_compile)?;
+    prepare_libcore(sysroot_path, args.libgccjit12_patches, args.cross_compile)?;
 
     if !args.only_libcore {
         cargo_install("hyperfine")?;
diff --git a/failing-ui-tests12.txt b/failing-ui-tests12.txt
index 0ac0a034af40..f027afa78a38 100644
--- a/failing-ui-tests12.txt
+++ b/failing-ui-tests12.txt
@@ -19,15 +19,12 @@ tests/ui/simd/intrinsic/generic-reduction-pass.rs
 tests/ui/simd/intrinsic/generic-select-pass.rs
 tests/ui/simd/intrinsic/inlining-issue67557-ice.rs
 tests/ui/simd/intrinsic/inlining-issue67557.rs
-tests/ui/simd/monomorphize-shuffle-index.rs
 tests/ui/simd/shuffle.rs
 tests/ui/simd/simd-bitmask.rs
-tests/ui/generator/resume-after-return.rs
 tests/ui/iterators/iter-step-overflow-debug.rs
-tests/ui/macros/rfc-2011-nicer-assert-messages/all-expr-kinds.rs
 tests/ui/numbers-arithmetic/next-power-of-two-overflow-debug.rs
 tests/ui/privacy/reachable-unnameable-items.rs
-tests/ui/rfc-1937-termination-trait/termination-trait-in-test.rs
+tests/ui/rfcs/rfc-1937-termination-trait/termination-trait-in-test.rs
 tests/ui/async-await/async-fn-size-moved-locals.rs
 tests/ui/async-await/async-fn-size-uninit-locals.rs
 tests/ui/cfg/cfg-panic.rs
@@ -38,3 +35,6 @@ tests/ui/simd/issue-85915-simd-ptrs.rs
 tests/ui/issues/issue-68010-large-zst-consts.rs
 tests/ui/rust-2018/proc-macro-crate-in-paths.rs
 tests/ui/target-feature/missing-plusminus.rs
+tests/ui/sse2.rs
+tests/ui/codegen/issue-79865-llvm-miscompile.rs
+tests/ui/intrinsics/intrinsics-integer.rs
diff --git a/patches/libgccjit12/0001-core-Disable-portable-simd-test.patch b/patches/libgccjit12/0001-core-Disable-portable-simd-test.patch
new file mode 100644
index 000000000000..9520a5a39edc
--- /dev/null
+++ b/patches/libgccjit12/0001-core-Disable-portable-simd-test.patch
@@ -0,0 +1,32 @@
+From 7bcd24ec6d4a96121874cb1ae5a23ea274aeff34 Mon Sep 17 00:00:00 2001
+From: None <none@example.com>
+Date: Thu, 19 Oct 2023 13:12:51 -0400
+Subject: [PATCH] [core] Disable portable-simd test
+
+---
+ library/core/tests/lib.rs | 2 --
+ 1 file changed, 2 deletions(-)
+
+diff --git a/library/core/tests/lib.rs b/library/core/tests/lib.rs
+index 5814ed4..194ad4c 100644
+--- a/library/core/tests/lib.rs
++++ b/library/core/tests/lib.rs
+@@ -90,7 +90,6 @@
+ #![feature(unwrap_infallible)]
+ #![feature(pointer_byte_offsets)]
+ #![feature(pointer_is_aligned)]
+-#![feature(portable_simd)]
+ #![feature(ptr_metadata)]
+ #![feature(lazy_cell)]
+ #![feature(unsized_tuple_coercion)]
+@@ -157,7 +156,6 @@ mod pin;
+ mod pin_macro;
+ mod ptr;
+ mod result;
+-mod simd;
+ mod slice;
+ mod str;
+ mod str_lossy;
+-- 
+2.42.0
+
diff --git a/src/intrinsic/llvm.rs b/src/intrinsic/llvm.rs
index 5996623bdc58..35eb4a11005b 100644
--- a/src/intrinsic/llvm.rs
+++ b/src/intrinsic/llvm.rs
@@ -432,15 +432,21 @@ pub fn ignore_arg_cast(func_name: &str, index: usize, args_len: usize) -> bool {
 
 #[cfg(not(feature="master"))]
 pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function<'gcc> {
-    match name {
-        "llvm.x86.xgetbv" | "llvm.x86.sse2.pause" => {
-            let gcc_name = "__builtin_trap";
-            let func = cx.context.get_builtin_function(gcc_name);
-            cx.functions.borrow_mut().insert(gcc_name.to_string(), func);
-            return func;
-        },
-        _ => unimplemented!("unsupported LLVM intrinsic {}", name),
-    }
+    let gcc_name =
+        match name {
+            "llvm.x86.sse2.pause" => {
+                // NOTE: pause is only a hint, so we use a dummy built-in because target built-ins
+                // are not supported in libgccjit 12.
+                "__builtin_inff"
+            },
+            "llvm.x86.xgetbv" => {
+                "__builtin_trap"
+            },
+            _ => unimplemented!("unsupported LLVM intrinsic {}", name),
+        };
+    let func = cx.context.get_builtin_function(gcc_name);
+    cx.functions.borrow_mut().insert(gcc_name.to_string(), func);
+    return func;
 }
 
 #[cfg(feature="master")]
diff --git a/src/type_.rs b/src/type_.rs
index 4914792c7b12..7a89fe81d384 100644
--- a/src/type_.rs
+++ b/src/type_.rs
@@ -216,17 +216,17 @@ impl<'gcc, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
         value.get_type()
     }
 
-    fn type_array(&self, ty: Type<'gcc>, len: u64) -> Type<'gcc> {
-        // TODO: remove this as well?
-        /*if let Some(struct_type) = ty.is_struct() {
+    #[cfg_attr(feature="master", allow(unused_mut))]
+    fn type_array(&self, ty: Type<'gcc>, mut len: u64) -> Type<'gcc> {
+        #[cfg(not(feature="master"))]
+        if let Some(struct_type) = ty.is_struct() {
             if struct_type.get_field_count() == 0 {
                 // NOTE: since gccjit only supports i32 for the array size and libcore's tests uses a
                 // size of usize::MAX in test_binary_search, we workaround this by setting the size to
                 // zero for ZSTs.
-                // FIXME(antoyo): fix gccjit API.
                 len = 0;
             }
-        }*/
+        }
 
         self.context.new_array_type(None, ty, len)
     }
diff --git a/tests/lang_tests_common.rs b/tests/lang_tests_common.rs
index 940c7cfd2662..af0133aad461 100644
--- a/tests/lang_tests_common.rs
+++ b/tests/lang_tests_common.rs
@@ -1,8 +1,8 @@
 //! The common code for `tests/lang_tests_*.rs`
 use std::{
     env::{self, current_dir},
-    path::PathBuf,
-    process::{self, Command},
+    path::{Path, PathBuf},
+    process::Command,
 };
 
 use lang_tester::LangTester;
@@ -23,9 +23,29 @@ pub fn main_inner(profile: Profile) {
     let gcc_path = include_str!("../gcc_path");
     let gcc_path = gcc_path.trim();
     env::set_var("LD_LIBRARY_PATH", gcc_path);
+
+    fn rust_filter(filename: &Path) -> bool {
+        filename.extension().expect("extension").to_str().expect("to_str") == "rs"
+    }
+
+    #[cfg(feature="master")]
+    fn filter(filename: &Path) -> bool {
+        rust_filter(filename)
+    }
+
+    #[cfg(not(feature="master"))]
+    fn filter(filename: &Path) -> bool {
+        if let Some(filename) = filename.to_str() {
+            if filename.ends_with("gep.rs") {
+                return false;
+            }
+        }
+        rust_filter(filename)
+    }
+
     LangTester::new()
         .test_dir("tests/run")
-        .test_file_filter(|path| path.extension().expect("extension").to_str().expect("to_str") == "rs")
+        .test_file_filter(filter)
         .test_extract(|source| {
             let lines =
                 source.lines()

From 6642b4b1e2f2edec71cdf3fabef8fcdc8b8517a7 Mon Sep 17 00:00:00 2001
From: Havard Eidnes <he@NetBSD.org>
Date: Wed, 25 Oct 2023 15:23:34 +0000
Subject: [PATCH 082/297] Add support for i586-unknown-netbsd as target.

This restricts instructions to those offered by Pentium,
to support e.g. AMD Geode.

There is already an entry for this target in the NetBSD
platform support page at

  src/doc/rustc/src/platform-support/netbsd.md

...so this should forestall its removal.

Additional fixes are needed for some vendored modules, this
is the changes in the rust compiler core itself.
---
 compiler/rustc_llvm/build.rs                  |  6 +++++
 .../src/spec/i586_unknown_netbsd.rs           | 22 +++++++++++++++++++
 compiler/rustc_target/src/spec/mod.rs         |  1 +
 3 files changed, 29 insertions(+)
 create mode 100644 compiler/rustc_target/src/spec/i586_unknown_netbsd.rs

diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs
index f606fa483caf..acb1d9607a4f 100644
--- a/compiler/rustc_llvm/build.rs
+++ b/compiler/rustc_llvm/build.rs
@@ -258,6 +258,12 @@ fn main() {
     {
         println!("cargo:rustc-link-lib=z");
     } else if target.contains("netbsd") {
+        // Building for i586 or i686, we need -latomic for 64-bit atomics
+        if target.starts_with("i586")
+           || target.starts_with("i686")
+        {
+            println!("cargo:rustc-link-lib=atomic");
+        }
         println!("cargo:rustc-link-lib=z");
         println!("cargo:rustc-link-lib=execinfo");
     }
diff --git a/compiler/rustc_target/src/spec/i586_unknown_netbsd.rs b/compiler/rustc_target/src/spec/i586_unknown_netbsd.rs
new file mode 100644
index 000000000000..9b36a3c28d35
--- /dev/null
+++ b/compiler/rustc_target/src/spec/i586_unknown_netbsd.rs
@@ -0,0 +1,22 @@
+use crate::spec::{Cc, Lld, LinkerFlavor, StackProbeType, Target, TargetOptions};
+
+pub fn target() -> Target {
+    let mut base = super::netbsd_base::opts();
+    base.cpu = "pentium".into();
+    base.max_atomic_width = Some(64);
+    base.pre_link_args
+        .entry(LinkerFlavor::Gnu(Cc::Yes, Lld::No))
+        .or_default()
+        .push("-m32".into());
+    base.stack_probes = StackProbeType::Call;
+
+    Target {
+        llvm_target: "i586-unknown-netbsdelf".into(),
+        pointer_width: 32,
+        data_layout: "e-m:e-p:32:32-p270:32:32-p271:32:32-p272:64:64-\
+            f64:32:64-f80:32-n8:16:32-S128"
+            .into(),
+        arch: "x86".into(),
+        options: TargetOptions { mcount: "__mcount".into(), ..base },
+    }
+}
diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index f1c7513d8856..73e5c6c858b9 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -1426,6 +1426,7 @@ supported_targets! {
     ("aarch64_be-unknown-netbsd", aarch64_be_unknown_netbsd),
     ("armv6-unknown-netbsd-eabihf", armv6_unknown_netbsd_eabihf),
     ("armv7-unknown-netbsd-eabihf", armv7_unknown_netbsd_eabihf),
+    ("i586-unknown-netbsd", i586_unknown_netbsd),
     ("i686-unknown-netbsd", i686_unknown_netbsd),
     ("powerpc-unknown-netbsd", powerpc_unknown_netbsd),
     ("riscv64gc-unknown-netbsd", riscv64gc_unknown_netbsd),

From 2c1dbed4aa049d5bb5b7d83d3c856519ee27a1dc Mon Sep 17 00:00:00 2001
From: DaniPopes <57450786+DaniPopes@users.noreply.github.com>
Date: Wed, 25 Oct 2023 01:05:26 +0200
Subject: [PATCH 083/297] Print variadic argument pattern in HIR pretty printer

---
 compiler/rustc_hir_pretty/src/lib.rs | 19 ++++++++++++-------
 tests/pretty/hir-fn-variadic.pp      | 15 +++++++++++++++
 tests/pretty/hir-fn-variadic.rs      | 13 +++++++++++++
 3 files changed, 40 insertions(+), 7 deletions(-)
 create mode 100644 tests/pretty/hir-fn-variadic.pp
 create mode 100644 tests/pretty/hir-fn-variadic.rs

diff --git a/compiler/rustc_hir_pretty/src/lib.rs b/compiler/rustc_hir_pretty/src/lib.rs
index ab6ab391484f..1ce6bb6ca15f 100644
--- a/compiler/rustc_hir_pretty/src/lib.rs
+++ b/compiler/rustc_hir_pretty/src/lib.rs
@@ -502,13 +502,13 @@ impl<'a> State<'a> {
                 self.word(";");
                 self.end(); // end the outer cbox
             }
-            hir::ItemKind::Fn(ref sig, param_names, body) => {
+            hir::ItemKind::Fn(ref sig, generics, body) => {
                 self.head("");
                 self.print_fn(
                     sig.decl,
                     sig.header,
                     Some(item.ident.name),
-                    param_names,
+                    generics,
                     &[],
                     Some(body),
                 );
@@ -1948,11 +1948,10 @@ impl<'a> State<'a> {
         self.print_generic_params(generics.params);
 
         self.popen();
-        let mut i = 0;
         // Make sure we aren't supplied *both* `arg_names` and `body_id`.
         assert!(arg_names.is_empty() || body_id.is_none());
-        self.commasep(Inconsistent, decl.inputs, |s, ty| {
-            s.ibox(INDENT_UNIT);
+        let mut i = 0;
+        let mut print_arg = |s: &mut Self| {
             if let Some(arg_name) = arg_names.get(i) {
                 s.word(arg_name.to_string());
                 s.word(":");
@@ -1963,11 +1962,17 @@ impl<'a> State<'a> {
                 s.space();
             }
             i += 1;
+        };
+        self.commasep(Inconsistent, decl.inputs, |s, ty| {
+            s.ibox(INDENT_UNIT);
+            print_arg(s);
             s.print_type(ty);
-            s.end()
+            s.end();
         });
         if decl.c_variadic {
-            self.word(", ...");
+            self.word(", ");
+            print_arg(self);
+            self.word("...");
         }
         self.pclose();
 
diff --git a/tests/pretty/hir-fn-variadic.pp b/tests/pretty/hir-fn-variadic.pp
new file mode 100644
index 000000000000..d4fadb254a6c
--- /dev/null
+++ b/tests/pretty/hir-fn-variadic.pp
@@ -0,0 +1,15 @@
+// pretty-compare-only
+// pretty-mode:hir
+// pp-exact:hir-fn-variadic.pp
+
+#![feature(c_variadic)]
+#[prelude_import]
+use ::std::prelude::rust_2015::*;
+#[macro_use]
+extern crate std;
+
+extern "C" {
+    fn foo(x: i32, ...);
+}
+
+unsafe extern "C" fn bar(_: i32, mut va2: ...) -> usize { va2.arg::<usize>() }
diff --git a/tests/pretty/hir-fn-variadic.rs b/tests/pretty/hir-fn-variadic.rs
new file mode 100644
index 000000000000..efb2754df623
--- /dev/null
+++ b/tests/pretty/hir-fn-variadic.rs
@@ -0,0 +1,13 @@
+// pretty-compare-only
+// pretty-mode:hir
+// pp-exact:hir-fn-variadic.pp
+
+#![feature(c_variadic)]
+
+extern "C" {
+    pub fn foo(x: i32, va1: ...);
+}
+
+pub unsafe extern "C" fn bar(_: i32, mut va2: ...) -> usize {
+    va2.arg::<usize>()
+}

From 6aead74ff276d249c2461962ebc3bad3ad93c955 Mon Sep 17 00:00:00 2001
From: DaniPopes <57450786+DaniPopes@users.noreply.github.com>
Date: Wed, 25 Oct 2023 17:44:17 +0200
Subject: [PATCH 084/297] Remove unnecessary CVarArgs name skipping logic

---
 compiler/rustc_ast_lowering/src/lib.rs | 9 +--------
 tests/pretty/hir-fn-variadic.pp        | 2 +-
 2 files changed, 2 insertions(+), 9 deletions(-)

diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs
index 1e51449e70cc..f48c5fc1af5d 100644
--- a/compiler/rustc_ast_lowering/src/lib.rs
+++ b/compiler/rustc_ast_lowering/src/lib.rs
@@ -1737,14 +1737,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
     }
 
     fn lower_fn_params_to_names(&mut self, decl: &FnDecl) -> &'hir [Ident] {
-        // Skip the `...` (`CVarArgs`) trailing arguments from the AST,
-        // as they are not explicit in HIR/Ty function signatures.
-        // (instead, the `c_variadic` flag is set to `true`)
-        let mut inputs = &decl.inputs[..];
-        if decl.c_variadic() {
-            inputs = &inputs[..inputs.len() - 1];
-        }
-        self.arena.alloc_from_iter(inputs.iter().map(|param| match param.pat.kind {
+        self.arena.alloc_from_iter(decl.inputs.iter().map(|param| match param.pat.kind {
             PatKind::Ident(_, ident, _) => self.lower_ident(ident),
             _ => Ident::new(kw::Empty, self.lower_span(param.pat.span)),
         }))
diff --git a/tests/pretty/hir-fn-variadic.pp b/tests/pretty/hir-fn-variadic.pp
index d4fadb254a6c..577d9400ad4e 100644
--- a/tests/pretty/hir-fn-variadic.pp
+++ b/tests/pretty/hir-fn-variadic.pp
@@ -9,7 +9,7 @@ use ::std::prelude::rust_2015::*;
 extern crate std;
 
 extern "C" {
-    fn foo(x: i32, ...);
+    fn foo(x: i32, va1: ...);
 }
 
 unsafe extern "C" fn bar(_: i32, mut va2: ...) -> usize { va2.arg::<usize>() }

From d80eb3a498fe6dea75ea37142aae81ba26783efc Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sun, 22 Oct 2023 14:49:00 +0000
Subject: [PATCH 085/297] Verify that the alloc_id is Memory.

---
 compiler/rustc_mir_transform/src/gvn.rs       |  7 ++++++-
 .../gvn.indirect_static.GVN.panic-abort.diff  | 19 +++++++++++++++++++
 .../gvn.indirect_static.GVN.panic-unwind.diff | 19 +++++++++++++++++++
 tests/mir-opt/gvn.rs                          | 14 ++++++++++++++
 4 files changed, 58 insertions(+), 1 deletion(-)
 create mode 100644 tests/mir-opt/gvn.indirect_static.GVN.panic-abort.diff
 create mode 100644 tests/mir-opt/gvn.indirect_static.GVN.panic-unwind.diff

diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index cf8a0fdb97e2..cfa315fe61ab 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -90,6 +90,7 @@ use rustc_hir::def::DefKind;
 use rustc_index::bit_set::BitSet;
 use rustc_index::IndexVec;
 use rustc_macros::newtype_index;
+use rustc_middle::mir::interpret::GlobalAlloc;
 use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
 use rustc_middle::ty::adjustment::PointerCoercion;
@@ -922,7 +923,11 @@ fn op_to_prop_const<'tcx>(
         let pointer = mplace.ptr().into_pointer_or_addr().ok()?;
         let (alloc_id, offset) = pointer.into_parts();
         intern_const_alloc_for_constprop(ecx, alloc_id).ok()?;
-        return Some(ConstValue::Indirect { alloc_id, offset });
+        if matches!(ecx.tcx.global_alloc(alloc_id), GlobalAlloc::Memory(_)) {
+            // `alloc_id` may point to a static. Codegen will choke on an `Indirect` with anything
+            // by `GlobalAlloc::Memory`, so do fall through to copying if needed.
+            return Some(ConstValue::Indirect { alloc_id, offset });
+        }
     }
 
     // Everything failed: create a new allocation to hold the data.
diff --git a/tests/mir-opt/gvn.indirect_static.GVN.panic-abort.diff b/tests/mir-opt/gvn.indirect_static.GVN.panic-abort.diff
new file mode 100644
index 000000000000..f853942bbb66
--- /dev/null
+++ b/tests/mir-opt/gvn.indirect_static.GVN.panic-abort.diff
@@ -0,0 +1,19 @@
+- // MIR for `indirect_static` before GVN
++ // MIR for `indirect_static` after GVN
+  
+  fn indirect_static() -> () {
+      let mut _0: ();
+      let mut _1: &std::option::Option<u8>;
+      let mut _2: u8;
+  
+      bb0: {
+          _1 = const {ALLOC0: &Option<u8>};
+          _2 = (((*_1) as variant#1).0: u8);
+          return;
+      }
+  }
+  
+  ALLOC0 (static: A, size: 2, align: 1) {
+      00 __                                           │ .░
+  }
+  
diff --git a/tests/mir-opt/gvn.indirect_static.GVN.panic-unwind.diff b/tests/mir-opt/gvn.indirect_static.GVN.panic-unwind.diff
new file mode 100644
index 000000000000..f853942bbb66
--- /dev/null
+++ b/tests/mir-opt/gvn.indirect_static.GVN.panic-unwind.diff
@@ -0,0 +1,19 @@
+- // MIR for `indirect_static` before GVN
++ // MIR for `indirect_static` after GVN
+  
+  fn indirect_static() -> () {
+      let mut _0: ();
+      let mut _1: &std::option::Option<u8>;
+      let mut _2: u8;
+  
+      bb0: {
+          _1 = const {ALLOC0: &Option<u8>};
+          _2 = (((*_1) as variant#1).0: u8);
+          return;
+      }
+  }
+  
+  ALLOC0 (static: A, size: 2, align: 1) {
+      00 __                                           │ .░
+  }
+  
diff --git a/tests/mir-opt/gvn.rs b/tests/mir-opt/gvn.rs
index 7ce851905e06..10a66ced026e 100644
--- a/tests/mir-opt/gvn.rs
+++ b/tests/mir-opt/gvn.rs
@@ -597,6 +597,18 @@ fn fn_pointers() {
     opaque(cg);
 }
 
+/// Verify that we do not create a `ConstValue::Indirect` backed by a static's AllocId.
+#[custom_mir(dialect = "analysis")]
+fn indirect_static() {
+    static A: Option<u8> = None;
+
+    mir!({
+        let ptr = Static(A);
+        let out = Field::<u8>(Variant(*ptr, 1), 0);
+        Return()
+    })
+}
+
 fn main() {
     subexpression_elimination(2, 4, 5);
     wrap_unwrap(5);
@@ -614,6 +626,7 @@ fn main() {
     assert_eq!(direct, indirect);
     repeat();
     fn_pointers();
+    indirect_static();
 }
 
 #[inline(never)]
@@ -639,3 +652,4 @@ fn identity<T>(x: T) -> T {
 // EMIT_MIR gvn.duplicate_slice.GVN.diff
 // EMIT_MIR gvn.repeat.GVN.diff
 // EMIT_MIR gvn.fn_pointers.GVN.diff
+// EMIT_MIR gvn.indirect_static.GVN.diff

From eda1928baa868495cd3481358390f8a4e286a659 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Mon, 23 Oct 2023 18:17:32 +0000
Subject: [PATCH 086/297] Typo.

---
 src/tools/miri/tests/pass/function_pointers.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tools/miri/tests/pass/function_pointers.rs b/src/tools/miri/tests/pass/function_pointers.rs
index 8e58692a0c74..36679b7180a6 100644
--- a/src/tools/miri/tests/pass/function_pointers.rs
+++ b/src/tools/miri/tests/pass/function_pointers.rs
@@ -80,7 +80,7 @@ fn main() {
     // but Miri currently uses a fixed address for monomorphic functions.
     assert!(return_fn_ptr(i) == i);
     assert!(return_fn_ptr(i) as unsafe fn() -> i32 == i as fn() -> i32 as unsafe fn() -> i32);
-    // Miri gives it many different addresses to different reifications of a generic function.
+    // Miri gives different addresses to different reifications of a generic function.
     assert!(return_fn_ptr(f) != f);
     // However, if we only turn `f` into a function pointer and use that pointer,
     // it is equal to itself.

From 72f0e0e79535246c31b0b1e992f9b0ef8a072dc4 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Mon, 23 Oct 2023 18:27:54 +0000
Subject: [PATCH 087/297] Rename has_provance and tweaks comments.

---
 .../rustc_const_eval/src/interpret/intern.rs  |  5 ++++-
 compiler/rustc_middle/src/mir/consts.rs       | 19 +++++++++++--------
 compiler/rustc_mir_transform/src/gvn.rs       | 12 +++++++++---
 3 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/compiler/rustc_const_eval/src/interpret/intern.rs b/compiler/rustc_const_eval/src/interpret/intern.rs
index 23d6d3219471..3d90e95c09c7 100644
--- a/compiler/rustc_const_eval/src/interpret/intern.rs
+++ b/compiler/rustc_const_eval/src/interpret/intern.rs
@@ -474,9 +474,12 @@ pub fn intern_const_alloc_for_constprop<
 
     alloc.mutability = Mutability::Not;
 
-    // link the alloc id to the actual allocation
+    // We are not doing recursive interning, so we don't currently support provenance.
+    // (If this assertion ever triggers, we should just implement a
+    // proper recursive interning loop.)
     assert!(alloc.provenance().ptrs().is_empty());
 
+    // Link the alloc id to the actual allocation
     let alloc = ecx.tcx.mk_const_alloc(alloc);
     ecx.tcx.set_alloc_id_memory(alloc_id, alloc);
 
diff --git a/compiler/rustc_middle/src/mir/consts.rs b/compiler/rustc_middle/src/mir/consts.rs
index 88d4a37f2109..9ddf3903d04c 100644
--- a/compiler/rustc_middle/src/mir/consts.rs
+++ b/compiler/rustc_middle/src/mir/consts.rs
@@ -173,16 +173,19 @@ impl<'tcx> ConstValue<'tcx> {
         Some(data.inner().inspect_with_uninit_and_ptr_outside_interpreter(start..end))
     }
 
-    pub fn has_provenance(&self, tcx: TyCtxt<'tcx>, size: Size) -> bool {
-        let (alloc, start, end) = match *self {
+    /// Check if a constant may contain provenance information. This is used by MIR opts.
+    pub fn may_have_provenance(&self, tcx: TyCtxt<'tcx>, size: Size) -> bool {
+        match *self {
             ConstValue::ZeroSized | ConstValue::Scalar(Scalar::Int(_)) => return false,
             ConstValue::Scalar(Scalar::Ptr(..)) => return true,
-            ConstValue::Slice { data, meta } => (data, Size::ZERO, Size::from_bytes(meta)),
-            ConstValue::Indirect { alloc_id, offset } => {
-                (tcx.global_alloc(alloc_id).unwrap_memory(), offset, offset + size)
-            }
-        };
-        !alloc.inner().provenance().range_empty(super::AllocRange::from(start..end), &tcx)
+            ConstValue::Slice { data, meta: _ } => !data.inner().provenance().ptrs().is_empty(),
+            ConstValue::Indirect { alloc_id, offset } => !tcx
+                .global_alloc(alloc_id)
+                .unwrap_memory()
+                .inner()
+                .provenance()
+                .range_empty(super::AllocRange::from(offset..offset + size), &tcx),
+        }
     }
 }
 
diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index cfa315fe61ab..3f30f399588f 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -910,11 +910,13 @@ fn op_to_prop_const<'tcx>(
         return Some(ConstValue::Scalar(scalar));
     }
 
-    // If this constant is a projection of another, we can return it directly.
+    // If this constant is already represented as an `Allocation`,
+    // try putting it into global memory to return it.
     if let Either::Left(mplace) = op.as_mplace_or_imm() {
         let (size, _align) = ecx.size_and_align_of_mplace(&mplace).ok()??;
 
         // Do not try interning a value that contains provenance.
+        // Due to https://github.com/rust-lang/rust/issues/79738, doing so could lead to bugs.
         let alloc_ref = ecx.get_ptr_alloc(mplace.ptr(), size).ok()??;
         if alloc_ref.has_provenance() {
             return None;
@@ -935,7 +937,10 @@ fn op_to_prop_const<'tcx>(
         ecx.intern_with_temp_alloc(op.layout, |ecx, dest| ecx.copy_op(op, dest, false)).ok()?;
     let value = ConstValue::Indirect { alloc_id, offset: Size::ZERO };
 
-    if !value.has_provenance(*ecx.tcx, op.layout.size) {
+    // Check that we do not leak a pointer.
+    // Those pointers may lose part of their identity in codegen.
+    // See https://github.com/rust-lang/rust/issues/79738.
+    if !value.may_have_provenance(*ecx.tcx, op.layout.size) {
         return Some(value);
     }
 
@@ -964,7 +969,8 @@ impl<'tcx> VnState<'_, 'tcx> {
 
         // Check that we do not leak a pointer.
         // Those pointers may lose part of their identity in codegen.
-        assert!(!value.has_provenance(self.tcx, op.layout.size));
+        // See https://github.com/rust-lang/rust/issues/79738.
+        assert!(!value.may_have_provenance(self.tcx, op.layout.size));
 
         let const_ = Const::Val(value, op.layout.ty);
         Some(ConstOperand { span: rustc_span::DUMMY_SP, user_ty: None, const_ })

From 856161886a9b2bea4621441c4f6c2ea358cddf56 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Mon, 23 Oct 2023 19:37:01 +0000
Subject: [PATCH 088/297] Directly check provenance from the AllocId.

---
 compiler/rustc_mir_transform/src/gvn.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index 3f30f399588f..39065962c695 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -940,7 +940,7 @@ fn op_to_prop_const<'tcx>(
     // Check that we do not leak a pointer.
     // Those pointers may lose part of their identity in codegen.
     // See https://github.com/rust-lang/rust/issues/79738.
-    if !value.may_have_provenance(*ecx.tcx, op.layout.size) {
+    if ecx.tcx.global_alloc(alloc_id).unwrap_memory().inner().provenance().ptrs().is_empty() {
         return Some(value);
     }
 

From 991bf752f1bbd0863864a978fee1236a93499464 Mon Sep 17 00:00:00 2001
From: Milo <50248166+Milo123459@users.noreply.github.com>
Date: Sat, 14 Oct 2023 18:49:51 +0000
Subject: [PATCH 089/297] make E0277 use short paths

add note

change wording

short_ty_string on t
---
 .../traits/error_reporting/type_err_ctxt_ext.rs  | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
index 8adfb27a3f44..f83ee576c294 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
@@ -429,14 +429,18 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             return;
                         }
                         let trait_ref = trait_predicate.to_poly_trait_ref();
-
-                        let (post_message, pre_message, type_def) = self
+                        let (post_message, pre_message, type_def, file_note) = self
                             .get_parent_trait_ref(obligation.cause.code())
                             .map(|(t, s)| {
+                                let (t, file) = self.tcx.short_ty_string(t);
                                 (
                                     format!(" in `{t}`"),
                                     format!("within `{t}`, "),
                                     s.map(|s| (format!("within this `{t}`"), s)),
+                                    file.and_then(|file| Some(format!(
+                                        "the full trait has been written to '{}'",
+                                        file.display(),
+                                    )))
                                 )
                             })
                             .unwrap_or_default();
@@ -544,6 +548,8 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                             err.emit();
                             return;
                         }
+
+                        file_note.map(|note| err.note(note));
                         if let Some(s) = label {
                             // If it has a custom `#[rustc_on_unimplemented]`
                             // error message, let's display it as the label!
@@ -1077,7 +1083,7 @@ pub(super) trait InferCtxtPrivExt<'tcx> {
     fn get_parent_trait_ref(
         &self,
         code: &ObligationCauseCode<'tcx>,
-    ) -> Option<(String, Option<Span>)>;
+    ) -> Option<(Ty<'tcx>, Option<Span>)>;
 
     /// If the `Self` type of the unsatisfied trait `trait_ref` implements a trait
     /// with the same path as `trait_ref`, a help message about
@@ -1927,7 +1933,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
     fn get_parent_trait_ref(
         &self,
         code: &ObligationCauseCode<'tcx>,
-    ) -> Option<(String, Option<Span>)> {
+    ) -> Option<(Ty<'tcx>, Option<Span>)> {
         match code {
             ObligationCauseCode::BuiltinDerivedObligation(data) => {
                 let parent_trait_ref = self.resolve_vars_if_possible(data.parent_trait_pred);
@@ -1937,7 +1943,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         let ty = parent_trait_ref.skip_binder().self_ty();
                         let span = TyCategory::from_ty(self.tcx, ty)
                             .map(|(_, def_id)| self.tcx.def_span(def_id));
-                        Some((ty.to_string(), span))
+                        Some((ty, span))
                     }
                 }
             }

From 62fe807e3c6cec9ae31a0a3e7e1f5bc694c2e30c Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 11:10:30 -0700
Subject: [PATCH 090/297] FileCheck asm_unwind

---
 tests/mir-opt/inline/asm_unwind.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/asm_unwind.rs b/tests/mir-opt/inline/asm_unwind.rs
index 0cf21fda72fa..596a4592f25f 100644
--- a/tests/mir-opt/inline/asm_unwind.rs
+++ b/tests/mir-opt/inline/asm_unwind.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // Tests inlining of `may_unwind` inline assembly.
 //
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
@@ -20,5 +19,7 @@ fn foo() {
 
 // EMIT_MIR asm_unwind.main.Inline.diff
 pub fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: (inlined foo)
     foo();
 }

From 2d0a34bb78f5546a0a7d7b447ffe4fa1c1ea13d9 Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 11:11:06 -0700
Subject: [PATCH 091/297] FileCheck caller_with_trivial_bound

---
 tests/mir-opt/inline/caller_with_trivial_bound.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/caller_with_trivial_bound.rs b/tests/mir-opt/inline/caller_with_trivial_bound.rs
index 3829cbdd3024..722c7f6724ef 100644
--- a/tests/mir-opt/inline/caller_with_trivial_bound.rs
+++ b/tests/mir-opt/inline/caller_with_trivial_bound.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // needs-unwind
 
@@ -18,6 +17,8 @@ pub fn foo<T>()
 where
     IntFactory: Factory<T>,
 {
+    // CHECK-LABEL: fn foo(
+    // CHECK-NOT: (inlined bar::<T>)
     let mut x: <IntFactory as Factory<T>>::Item = bar::<T>();
 }
 

From 2f9aa7da9fefdd028d09e4c7450994b3323b2cfa Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 11:11:22 -0700
Subject: [PATCH 092/297] FileCheck cycle

---
 tests/mir-opt/inline/cycle.rs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/cycle.rs b/tests/mir-opt/inline/cycle.rs
index 3e4f06834358..8716b44b3dc0 100644
--- a/tests/mir-opt/inline/cycle.rs
+++ b/tests/mir-opt/inline/cycle.rs
@@ -1,20 +1,27 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // compile-flags: -Zinline-mir-hint-threshold=1000
 
 // EMIT_MIR cycle.f.Inline.diff
 #[inline(always)]
 fn f(g: impl Fn()) {
+    // CHECK-LABEL: fn f(
+    // CHECK-NOT: inlined
     g();
 }
 
 // EMIT_MIR cycle.g.Inline.diff
 #[inline(always)]
 fn g() {
+    // CHECK-LABEL: fn g(
+    // CHECK: (inlined f::<fn() {main}>)
+    // CHECK-NOT: inlined
     f(main);
 }
 
 // EMIT_MIR cycle.main.Inline.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: (inlined f::<fn() {g}>)
+    // CHECK-NOT: inlined
     f(g);
 }

From 9b3f5e15277a2089140db89704277a682374efba Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 11:11:37 -0700
Subject: [PATCH 093/297] FileCheck dont_ice_on_generic_rust_call

---
 tests/mir-opt/inline/dont_ice_on_generic_rust_call.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/dont_ice_on_generic_rust_call.rs b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.rs
index 4147325ec44b..3307a1408fb5 100644
--- a/tests/mir-opt/inline/dont_ice_on_generic_rust_call.rs
+++ b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // compile-flags: -Zmir-enable-passes=+Inline --crate-type=lib
 
@@ -8,5 +7,7 @@ use std::marker::Tuple;
 
 // EMIT_MIR dont_ice_on_generic_rust_call.call.Inline.diff
 pub fn call<I: Tuple>(mut mock: Box<dyn FnMut<I, Output = ()>>, input: I) {
+    // CHECK-LABEL: fn call(
+    // CHECK-NOT: (inlined <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut)
     mock.call_mut(input)
 }

From 76faae9cdcf0d37884809ab0d21689fbc7598b98 Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 11:11:50 -0700
Subject: [PATCH 094/297] FileCheck dyn_trait

---
 tests/mir-opt/inline/dyn_trait.rs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/dyn_trait.rs b/tests/mir-opt/inline/dyn_trait.rs
index 7b41b1e1171e..005cf155fdb9 100644
--- a/tests/mir-opt/inline/dyn_trait.rs
+++ b/tests/mir-opt/inline/dyn_trait.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 #![crate_type = "lib"]
 
@@ -20,18 +19,26 @@ pub trait Query {
 // EMIT_MIR dyn_trait.mk_cycle.Inline.diff
 #[inline(always)]
 pub fn mk_cycle<V: Debug>(c: &dyn Cache<V = V>) {
+    // CHECK-LABEL: fn mk_cycle(
+    // CHECK-NOT: (inlined <dyn Cache<V = V> as Cache>::store_nocache)
     c.store_nocache()
 }
 
 // EMIT_MIR dyn_trait.try_execute_query.Inline.diff
 #[inline(always)]
 pub fn try_execute_query<C: Cache>(c: &C) {
+    // CHECK-LABEL: fn try_execute_query(
+    // CHECK: (inlined mk_cycle::<<C as Cache>::V>)
     mk_cycle(c)
 }
 
 // EMIT_MIR dyn_trait.get_query.Inline.diff
 #[inline(always)]
 pub fn get_query<Q: Query, T>(t: &T) {
+    // CHECK-LABEL: fn get_query(
+    // CHECK-NOT: (inlined <Q as Query>::cache::<T>)
     let c = Q::cache(t);
+    // CHECK: (inlined try_execute_query::<<Q as Query>::C>)
+    // CHECK: (inlined mk_cycle::<<Q as Query>::V>)
     try_execute_query(c)
 }

From f005d2325a5b838a4015b0233ee022c455962801 Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 11:12:05 -0700
Subject: [PATCH 095/297] FileCheck exponential_runtime

---
 tests/mir-opt/inline/exponential_runtime.rs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/exponential_runtime.rs b/tests/mir-opt/inline/exponential_runtime.rs
index 6d3af8b9c572..499bec33ffc9 100644
--- a/tests/mir-opt/inline/exponential_runtime.rs
+++ b/tests/mir-opt/inline/exponential_runtime.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // Checks that code with exponential runtime does not have exponential behavior in inlining.
 
@@ -85,5 +84,13 @@ impl A for () {
 
 // EMIT_MIR exponential_runtime.main.Inline.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: (inlined <() as G>::call)
+    // CHECK: (inlined <() as F>::call)
+    // CHECK: (inlined <() as E>::call)
+    // CHECK: (inlined <() as D>::call)
+    // CHECK: (inlined <() as C>::call)
+    // CHECK: (inlined <() as B>::call)
+    // CHECK-NOT: inlined
     <() as G>::call();
 }

From 22679cd36df7ad05eaea153b64712e97ffe69ecf Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 11:12:17 -0700
Subject: [PATCH 096/297] FileCheck inline_any_operand

---
 tests/mir-opt/inline/inline_any_operand.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/inline_any_operand.rs b/tests/mir-opt/inline/inline_any_operand.rs
index e131cd6ef7ee..659b7c3a0a1f 100644
--- a/tests/mir-opt/inline/inline_any_operand.rs
+++ b/tests/mir-opt/inline/inline_any_operand.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // compile-flags: -Z span_free_formats
 
 // Tests that MIR inliner works for any operand
@@ -9,6 +8,8 @@ fn main() {
 
 // EMIT_MIR inline_any_operand.bar.Inline.after.mir
 fn bar() -> bool {
+    // CHECK-LABEL: fn bar(
+    // CHECK: (inlined foo)
     let f = foo;
     f(1, -1)
 }

From de56d2d9b2c501985b5f30eb6506475bf312c0c0 Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 11:24:38 -0700
Subject: [PATCH 097/297] FileCheck inline_box_fn

---
 tests/mir-opt/inline/inline_box_fn.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/inline_box_fn.rs b/tests/mir-opt/inline/inline_box_fn.rs
index f6a90b92c911..7686477c291a 100644
--- a/tests/mir-opt/inline/inline_box_fn.rs
+++ b/tests/mir-opt/inline/inline_box_fn.rs
@@ -1,9 +1,10 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // unit-test: Inline
 // compile-flags: --crate-type=lib
 
 // EMIT_MIR inline_box_fn.call.Inline.diff
 fn call(x: Box<dyn Fn(i32)>) {
+    // CHECK-LABEL: fn call(
+    // CHECK-NOT: (inlined <Box<dyn Fn(i32)> as Fn<(i32,)>>::call)
     x(1);
 }

From 9d61e6a4e46899a00da5908cf6db34a7d2bebb28 Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 11:28:38 -0700
Subject: [PATCH 098/297] FileCheck inline_closure_borrows_arg

---
 tests/mir-opt/inline/inline_closure_borrows_arg.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/inline_closure_borrows_arg.rs b/tests/mir-opt/inline/inline_closure_borrows_arg.rs
index a5cc7d103653..1570ab057c72 100644
--- a/tests/mir-opt/inline/inline_closure_borrows_arg.rs
+++ b/tests/mir-opt/inline/inline_closure_borrows_arg.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // compile-flags: -Z span_free_formats -Zunsound-mir-opts
 
 // Tests that MIR inliner can handle closure arguments,
@@ -14,5 +13,8 @@ fn foo<T: Copy>(_t: T, q: &i32) -> i32 {
         let variable = &*r;
         *variable
     };
+
+    // CHECK-LABEL: fn foo(
+    // CHECK: (inlined foo::<T>::{closure#0})
     x(q, q)
 }

From 5caee416a5c53c997c35d2358cb61e2499a1e263 Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 11:30:04 -0700
Subject: [PATCH 099/297] FileCheck inline_closure_captures

---
 tests/mir-opt/inline/inline_closure_captures.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/inline_closure_captures.rs b/tests/mir-opt/inline/inline_closure_captures.rs
index 0d95564e5dde..2b08b1068872 100644
--- a/tests/mir-opt/inline/inline_closure_captures.rs
+++ b/tests/mir-opt/inline/inline_closure_captures.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // compile-flags: -Z span_free_formats
 
 // Tests that MIR inliner can handle closure captures.
@@ -10,5 +9,8 @@ fn main() {
 // EMIT_MIR inline_closure_captures.foo.Inline.after.mir
 fn foo<T: Copy>(t: T, q: i32) -> (i32, T) {
     let x = |_q| (q, t);
+
+    // CHECK-LABEL: fn foo(
+    // CHECK: (inlined foo::<T>::{closure#0})
     x(q)
 }

From 7ee05d24b5103df74d93c267ab4e1d8060aec444 Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 11:31:05 -0700
Subject: [PATCH 100/297] FileCheck inline_closure

---
 tests/mir-opt/inline/inline_closure.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/inline_closure.rs b/tests/mir-opt/inline/inline_closure.rs
index bd4c84ff0ca2..65f55d49a806 100644
--- a/tests/mir-opt/inline/inline_closure.rs
+++ b/tests/mir-opt/inline/inline_closure.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // compile-flags: -Z span_free_formats
 
 // Tests that MIR inliner can handle closure arguments. (#45894)
@@ -10,5 +9,8 @@ fn main() {
 // EMIT_MIR inline_closure.foo.Inline.after.mir
 fn foo<T: Copy>(_t: T, q: i32) -> i32 {
     let x = |_t, _q| _t;
+
+    // CHECK-LABEL: fn foo(
+    // CHECK: (inlined foo::<T>::{closure#0})
     x(q, q)
 }

From d8f33ef93d6dbff4267fd7d779d73a76f1e46e28 Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 18:03:13 -0700
Subject: [PATCH 101/297] FileCheck inline_diverging

---
 tests/mir-opt/inline/inline_diverging.rs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/inline_diverging.rs b/tests/mir-opt/inline/inline_diverging.rs
index 9e50b1ead2bc..25a5b9c5c5e7 100644
--- a/tests/mir-opt/inline/inline_diverging.rs
+++ b/tests/mir-opt/inline/inline_diverging.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // Tests inlining of diverging calls.
 //
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
@@ -7,6 +6,8 @@
 
 // EMIT_MIR inline_diverging.f.Inline.diff
 pub fn f() {
+    // CHECK-LABEL: fn f(
+    // CHECK: (inlined sleep)
     sleep();
 }
 
@@ -15,12 +16,17 @@ pub fn g(i: i32) -> u32 {
     if i > 0 {
         i as u32
     } else {
+        // CHECK-LABEL: fn g(
+        // CHECK: (inlined panic)
         panic();
     }
 }
 
 // EMIT_MIR inline_diverging.h.Inline.diff
 pub fn h() {
+    // CHECK-LABEL: fn h(
+    // CHECK: (inlined call_twice::<!, fn() -> ! {sleep}>)
+    // CHECK-NOT: inlined
     call_twice(sleep);
 }
 

From 20e7caa737b1ee4ae34a89201ebc972a4d495982 Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 18:15:32 -0700
Subject: [PATCH 102/297] FileCheck inline_coroutine

---
 ...ne_coroutine.main.Inline.panic-unwind.diff | 22 +++++++++----------
 tests/mir-opt/inline/inline_coroutine.rs      |  4 +++-
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/tests/mir-opt/inline/inline_coroutine.main.Inline.panic-unwind.diff b/tests/mir-opt/inline/inline_coroutine.main.Inline.panic-unwind.diff
index e7d020968389..fdb42bf3d8a0 100644
--- a/tests/mir-opt/inline/inline_coroutine.main.Inline.panic-unwind.diff
+++ b/tests/mir-opt/inline/inline_coroutine.main.Inline.panic-unwind.diff
@@ -4,26 +4,26 @@
   fn main() -> () {
       let mut _0: ();
       let _1: std::ops::CoroutineState<i32, bool>;
-      let mut _2: std::pin::Pin<&mut {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8}>;
-      let mut _3: &mut {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8};
-      let mut _4: {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8};
+      let mut _2: std::pin::Pin<&mut {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8}>;
+      let mut _3: &mut {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8};
+      let mut _4: {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8};
 +     let mut _5: bool;
       scope 1 {
           debug _r => _1;
       }
 +     scope 2 (inlined g) {
 +     }
-+     scope 3 (inlined Pin::<&mut {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8}>::new) {
++     scope 3 (inlined Pin::<&mut {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8}>::new) {
 +         debug pointer => _3;
 +         scope 4 {
-+             scope 5 (inlined Pin::<&mut {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8}>::new_unchecked) {
++             scope 5 (inlined Pin::<&mut {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8}>::new_unchecked) {
 +                 debug pointer => _3;
 +             }
 +         }
 +     }
 +     scope 6 (inlined g::{closure#0}) {
 +         debug a => _5;
-+         let mut _6: &mut {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8};
++         let mut _6: &mut {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8};
 +         let mut _7: u32;
 +         let mut _8: i32;
 +     }
@@ -37,20 +37,20 @@
 -     }
 - 
 -     bb1: {
-+         _4 = {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8 (#0)};
++         _4 = {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8 (#0)};
           _3 = &mut _4;
--         _2 = Pin::<&mut {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8}>::new(move _3) -> [return: bb2, unwind: bb5];
+-         _2 = Pin::<&mut {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8}>::new(move _3) -> [return: bb2, unwind: bb5];
 -     }
 - 
 -     bb2: {
-+         _2 = Pin::<&mut {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8}> { pointer: move _3 };
++         _2 = Pin::<&mut {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8}> { pointer: move _3 };
           StorageDead(_3);
--         _1 = <{coroutine@$DIR/inline_coroutine.rs:17:5: 17:8} as Coroutine<bool>>::resume(move _2, const false) -> [return: bb3, unwind: bb5];
+-         _1 = <{coroutine@$DIR/inline_coroutine.rs:19:5: 19:8} as Coroutine<bool>>::resume(move _2, const false) -> [return: bb3, unwind: bb5];
 +         StorageLive(_5);
 +         _5 = const false;
 +         StorageLive(_6);
 +         StorageLive(_7);
-+         _6 = (_2.0: &mut {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8});
++         _6 = (_2.0: &mut {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8});
 +         _7 = discriminant((*_6));
 +         switchInt(move _7) -> [0: bb5, 1: bb9, 3: bb10, otherwise: bb11];
       }
diff --git a/tests/mir-opt/inline/inline_coroutine.rs b/tests/mir-opt/inline/inline_coroutine.rs
index d021cdac28ef..a82586bf2bfa 100644
--- a/tests/mir-opt/inline/inline_coroutine.rs
+++ b/tests/mir-opt/inline/inline_coroutine.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // compile-flags: -Zinline-mir-hint-threshold=1000
 #![feature(coroutines, coroutine_trait)]
@@ -8,6 +7,9 @@ use std::pin::Pin;
 
 // EMIT_MIR inline_coroutine.main.Inline.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: (inlined g)
+    // CHECK: (inlined g::{closure#0})
     let _r = Pin::new(&mut g()).resume(false);
 }
 

From 19c36a96dfce27de7ad60f084e4bbc9abfdf4768 Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 18:21:41 -0700
Subject: [PATCH 103/297] FileCheck inline_instruction_set

---
 tests/mir-opt/inline/inline_instruction_set.rs | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/inline_instruction_set.rs b/tests/mir-opt/inline/inline_instruction_set.rs
index 4ac4d462f82d..7cb59645587b 100644
--- a/tests/mir-opt/inline/inline_instruction_set.rs
+++ b/tests/mir-opt/inline/inline_instruction_set.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // Checks that only functions with the compatible instruction_set attributes are inlined.
 //
 // A function is "compatible" when the *callee* has the same attribute or no attribute.
@@ -47,16 +46,26 @@ fn inline_always_and_using_inline_asm() {
 // EMIT_MIR inline_instruction_set.t32.Inline.diff
 #[instruction_set(arm::t32)]
 pub fn t32() {
+    // CHECK-LABEL: fn t32(
+    // CHECK-NOT: (inlined instruction_set_a32)
     instruction_set_a32();
+    // CHECK: (inlined instruction_set_t32)
     instruction_set_t32();
+    // CHECK: (inlined instruction_set_default)
     instruction_set_default();
+    // CHECK-NOT: (inlined inline_always_and_using_inline_asm)
     inline_always_and_using_inline_asm();
 }
 
 // EMIT_MIR inline_instruction_set.default.Inline.diff
 pub fn default() {
+    // CHECK-LABEL: fn default(
+    // CHECK-NOT: (inlined instruction_set_a32)
     instruction_set_a32();
+    // CHECK-NOT: (inlined instruction_set_t32)
     instruction_set_t32();
+    // CHECK: (inlined instruction_set_default)
     instruction_set_default();
+    // CHECK: (inlined inline_always_and_using_inline_asm)
     inline_always_and_using_inline_asm();
 }

From de8255194a292d59c14b4d4a4424e576c7820373 Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 20:33:22 -0700
Subject: [PATCH 104/297] FileCheck inline_into_box_place

---
 tests/mir-opt/inline/inline_into_box_place.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/inline_into_box_place.rs b/tests/mir-opt/inline/inline_into_box_place.rs
index b755692afc21..65f8e2916b63 100644
--- a/tests/mir-opt/inline/inline_into_box_place.rs
+++ b/tests/mir-opt/inline/inline_into_box_place.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // ignore-endian-big
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // ignore-debug MIR alignment checks in std alter the diff, breaking the test
@@ -6,5 +5,7 @@
 
 // EMIT_MIR inline_into_box_place.main.Inline.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: (inlined Box::<Vec<u32>>::new)
     let _x: Box<Vec<u32>> = Box::new(Vec::new());
 }

From 3202d4e35761cc73b70465fa55f414840728e83a Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 20:34:41 -0700
Subject: [PATCH 105/297] FileCheck inline_options

---
 tests/mir-opt/inline/inline_options.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/inline_options.rs b/tests/mir-opt/inline/inline_options.rs
index 394a8c4945cd..b940c64f0b88 100644
--- a/tests/mir-opt/inline/inline_options.rs
+++ b/tests/mir-opt/inline/inline_options.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // Checks that inlining threshold can be controlled with
 // inline-mir-threshold and inline-hint-threshold options.
@@ -8,7 +7,10 @@
 
 // EMIT_MIR inline_options.main.Inline.after.mir
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK-NOT: (inlined not_inlined)
     not_inlined();
+    // CHECK: (inlined inlined::<u32>)
     inlined::<u32>();
 }
 

From 21a4c39cb849d32c1209329a740e2c3013ebe063 Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 20:38:15 -0700
Subject: [PATCH 106/297] FileCheck inline_retag

---
 tests/mir-opt/inline/inline_retag.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/inline_retag.rs b/tests/mir-opt/inline/inline_retag.rs
index f695b9f22e61..c90c0b819b7b 100644
--- a/tests/mir-opt/inline/inline_retag.rs
+++ b/tests/mir-opt/inline/inline_retag.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // compile-flags: -Z span_free_formats -Z mir-emit-retag
 
 // Tests that MIR inliner fixes up `Retag`'s `fn_entry` flag
@@ -9,6 +8,8 @@ fn main() {
 
 // EMIT_MIR inline_retag.bar.Inline.after.mir
 fn bar() -> bool {
+    // CHECK-LABEL: fn bar(
+    // CHECK: (inlined foo)
     let f = foo;
     f(&1, &-1)
 }

From 25325667f2be173d0d8968f053e20d36ed24545b Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 20:40:20 -0700
Subject: [PATCH 107/297] FileCheck inline_specialization

---
 tests/mir-opt/inline/inline_specialization.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/inline_specialization.rs b/tests/mir-opt/inline/inline_specialization.rs
index eb0cf891dad0..6453abc00812 100644
--- a/tests/mir-opt/inline/inline_specialization.rs
+++ b/tests/mir-opt/inline/inline_specialization.rs
@@ -1,9 +1,10 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 #![feature(specialization)]
 
 // EMIT_MIR inline_specialization.main.Inline.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: (inlined <Vec<()> as Foo>::bar)
     let x = <Vec::<()> as Foo>::bar();
 }
 

From f7acf17945b6a5739fc498330c698579982fbaf8 Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 20:41:43 -0700
Subject: [PATCH 108/297] FileCheck inline_trait_method_2

---
 tests/mir-opt/inline/inline_trait_method_2.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/inline_trait_method_2.rs b/tests/mir-opt/inline/inline_trait_method_2.rs
index e87609a8c7e0..b0b6a7b9b01d 100644
--- a/tests/mir-opt/inline/inline_trait_method_2.rs
+++ b/tests/mir-opt/inline/inline_trait_method_2.rs
@@ -1,9 +1,11 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // compile-flags: -Z span_free_formats -Z mir-opt-level=4
 
 // EMIT_MIR inline_trait_method_2.test2.Inline.after.mir
 fn test2(x: &dyn X) -> bool {
+    // CHECK-LABEL: fn test2(
+    // CHECK: (inlined test)
+    // CHECK-NOT: (inlined <dyn X as X>::y)
     test(x)
 }
 

From 1b9cb5d59bb704c8a7920abcdbdef83f3273afba Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 21:32:56 -0700
Subject: [PATCH 109/297] FileCheck inline_trait_method

---
 tests/mir-opt/inline/inline_trait_method.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/inline_trait_method.rs b/tests/mir-opt/inline/inline_trait_method.rs
index 8a95adf3713e..ea09966a2aa9 100644
--- a/tests/mir-opt/inline/inline_trait_method.rs
+++ b/tests/mir-opt/inline/inline_trait_method.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // compile-flags: -Z span_free_formats
 
@@ -8,6 +7,8 @@ fn main() {
 
 // EMIT_MIR inline_trait_method.test.Inline.after.mir
 fn test(x: &dyn X) -> u32 {
+    // CHECK-LABEL: fn test(
+    // CHECK-NOT: (inlined <dyn X as X>::y)
     x.y()
 }
 

From 773dc6275666afc56d2ef442f07e2b3908c692d3 Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 21:41:07 -0700
Subject: [PATCH 110/297] FileCheck inline_as_ref_as_mut

---
 tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.rs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.rs b/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.rs
index da779fed76fb..4517c88d7134 100644
--- a/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.rs
+++ b/tests/mir-opt/inline/issue_58867_inline_as_ref_as_mut.rs
@@ -1,21 +1,28 @@
-// skip-filecheck
 // EMIT_MIR issue_58867_inline_as_ref_as_mut.a.Inline.after.mir
 pub fn a<T>(x: &mut [T]) -> &mut [T] {
+    // CHECK-LABEL: fn a(
+    // CHECK: (inlined <[T] as AsMut<[T]>>::as_mut)
     x.as_mut()
 }
 
 // EMIT_MIR issue_58867_inline_as_ref_as_mut.b.Inline.after.mir
 pub fn b<T>(x: &mut Box<T>) -> &mut T {
+    // CHECK-LABEL: fn b(
+    // CHECK: (inlined <Box<T> as AsMut<T>>::as_mut)
     x.as_mut()
 }
 
 // EMIT_MIR issue_58867_inline_as_ref_as_mut.c.Inline.after.mir
 pub fn c<T>(x: &[T]) -> &[T] {
+    // CHECK-LABEL: fn c(
+    // CHECK: (inlined <[T] as AsRef<[T]>>::as_ref)
     x.as_ref()
 }
 
 // EMIT_MIR issue_58867_inline_as_ref_as_mut.d.Inline.after.mir
 pub fn d<T>(x: &Box<T>) -> &T {
+    // CHECK-LABEL: fn d(
+    // CHECK: (inlined <Box<T> as AsRef<T>>::as_ref)
     x.as_ref()
 }
 

From bb695977de920f47eb87ccf7580bb04a6bd36350 Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 21:57:08 -0700
Subject: [PATCH 111/297] FileCheck inline_scopes_parenting

---
 ...ssue_76997_inline_scopes_parenting.main.Inline.after.mir | 6 +++---
 tests/mir-opt/inline/issue_76997_inline_scopes_parenting.rs | 6 +++++-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.main.Inline.after.mir b/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.main.Inline.after.mir
index d28c0441f63c..ba5eb757cf97 100644
--- a/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.main.Inline.after.mir
+++ b/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.main.Inline.after.mir
@@ -2,8 +2,8 @@
 
 fn main() -> () {
     let mut _0: ();
-    let _1: {closure@$DIR/issue_76997_inline_scopes_parenting.rs:6:13: 6:16};
-    let mut _2: &{closure@$DIR/issue_76997_inline_scopes_parenting.rs:6:13: 6:16};
+    let _1: {closure@$DIR/issue_76997_inline_scopes_parenting.rs:10:13: 10:16};
+    let mut _2: &{closure@$DIR/issue_76997_inline_scopes_parenting.rs:10:13: 10:16};
     let mut _3: ((),);
     let mut _4: ();
     let mut _5: ();
@@ -19,7 +19,7 @@ fn main() -> () {
 
     bb0: {
         StorageLive(_1);
-        _1 = {closure@$DIR/issue_76997_inline_scopes_parenting.rs:6:13: 6:16};
+        _1 = {closure@$DIR/issue_76997_inline_scopes_parenting.rs:10:13: 10:16};
         StorageLive(_2);
         _2 = &_1;
         StorageLive(_3);
diff --git a/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.rs b/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.rs
index c7147d42df99..2a7c931a659e 100644
--- a/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.rs
+++ b/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.rs
@@ -1,8 +1,12 @@
-// skip-filecheck
 // Tests that MIR inliner can handle `SourceScopeData` parenting correctly. (#76997)
 
 // EMIT_MIR issue_76997_inline_scopes_parenting.main.Inline.after.mir
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: scope 2
+    // CHECK-NEXT: debug x
+    // CHECK-NEXT: scope 3
+    // CHECK-NEXT: debug y
     let f = |x| { let y = x; y };
     f(())
 }

From 5cf65eb16a57469212a682d469829c082ec25fb2 Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 22:06:38 -0700
Subject: [PATCH 112/297] FileCheck issue_78442

---
 tests/mir-opt/inline/issue_78442.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/issue_78442.rs b/tests/mir-opt/inline/issue_78442.rs
index f83ed70d0db3..f9a5234283a0 100644
--- a/tests/mir-opt/inline/issue_78442.rs
+++ b/tests/mir-opt/inline/issue_78442.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // compile-flags: -Z mir-opt-level=3 -Z inline-mir
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 #![crate_type = "lib"]
@@ -9,6 +8,11 @@ pub fn bar<P>(
     // Error won't happen if "bar" is not generic
     _baz: P,
 ) {
+    // CHECK-LABEL: fn bar(
+    // CHECK: let mut {{.*}}: &fn() {foo};
+    // CHECK: let {{.*}}: fn() {foo};
+    // CHECK: (inlined hide_foo)
+    // CHECK-NOT: inlined
     hide_foo()();
 }
 

From 6e047c07a66ed5a69e38e38191c911b7cb579e9e Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 22:18:02 -0700
Subject: [PATCH 113/297] FileCheck unchecked_shifts

---
 tests/mir-opt/inline/unchecked_shifts.rs | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/unchecked_shifts.rs b/tests/mir-opt/inline/unchecked_shifts.rs
index 67666f2f713f..805195dd8b37 100644
--- a/tests/mir-opt/inline/unchecked_shifts.rs
+++ b/tests/mir-opt/inline/unchecked_shifts.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 #![crate_type = "lib"]
 #![feature(unchecked_math)]
@@ -9,23 +8,31 @@
 // EMIT_MIR unchecked_shifts.unchecked_shl_unsigned_smaller.Inline.diff
 // EMIT_MIR unchecked_shifts.unchecked_shl_unsigned_smaller.PreCodegen.after.mir
 pub unsafe fn unchecked_shl_unsigned_smaller(a: u16, b: u32) -> u16 {
+    // CHECK-LABEL: fn unchecked_shl_unsigned_smaller(
+    // CHECK: (inlined core::num::<impl u16>::unchecked_shl)
     a.unchecked_shl(b)
 }
 
 // EMIT_MIR unchecked_shifts.unchecked_shr_signed_smaller.Inline.diff
 // EMIT_MIR unchecked_shifts.unchecked_shr_signed_smaller.PreCodegen.after.mir
 pub unsafe fn unchecked_shr_signed_smaller(a: i16, b: u32) -> i16 {
+    // CHECK-LABEL: fn unchecked_shr_signed_smaller(
+    // CHECK: (inlined core::num::<impl i16>::unchecked_shr)
     a.unchecked_shr(b)
 }
 
 // EMIT_MIR unchecked_shifts.unchecked_shl_unsigned_bigger.Inline.diff
 // EMIT_MIR unchecked_shifts.unchecked_shl_unsigned_bigger.PreCodegen.after.mir
 pub unsafe fn unchecked_shl_unsigned_bigger(a: u64, b: u32) -> u64 {
+    // CHECK-LABEL: fn unchecked_shl_unsigned_bigger(
+    // CHECK: (inlined core::num::<impl u64>::unchecked_shl)
     a.unchecked_shl(b)
 }
 
 // EMIT_MIR unchecked_shifts.unchecked_shr_signed_bigger.Inline.diff
 // EMIT_MIR unchecked_shifts.unchecked_shr_signed_bigger.PreCodegen.after.mir
 pub unsafe fn unchecked_shr_signed_bigger(a: i64, b: u32) -> i64 {
+    // CHECK-LABEL: fn unchecked_shr_signed_bigger(
+    // CHECK: (inlined core::num::<impl i64>::unchecked_shr)
     a.unchecked_shr(b)
 }

From 3faf05b6e7bfc88f2415cd4c09968ffe454b295f Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 22:21:03 -0700
Subject: [PATCH 114/297] FileCheck unsized_argument

---
 tests/mir-opt/inline/unsized_argument.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/unsized_argument.rs b/tests/mir-opt/inline/unsized_argument.rs
index 22c88b83f9b1..e8c2bc10be22 100644
--- a/tests/mir-opt/inline/unsized_argument.rs
+++ b/tests/mir-opt/inline/unsized_argument.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 // needs-unwind
 #![feature(unsized_fn_params)]
 
@@ -7,6 +6,8 @@ fn callee(y: [i32]) {}
 
 // EMIT_MIR unsized_argument.caller.Inline.diff
 fn caller(x: Box<[i32]>) {
+    // CHECK-LABEL: fn caller(
+    // CHECK-NOT: (inlined callee)
     callee(*x);
 }
 

From 6ab66c3f3724e71d228d498a52f02cb358f9247e Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Fri, 20 Oct 2023 22:22:32 -0700
Subject: [PATCH 115/297] FileCheck unwrap_unchecked

---
 tests/mir-opt/inline/unwrap_unchecked.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/mir-opt/inline/unwrap_unchecked.rs b/tests/mir-opt/inline/unwrap_unchecked.rs
index f8964eba227a..be133706e5c2 100644
--- a/tests/mir-opt/inline/unwrap_unchecked.rs
+++ b/tests/mir-opt/inline/unwrap_unchecked.rs
@@ -1,4 +1,3 @@
-// skip-filecheck
 #![crate_type = "lib"]
 
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
@@ -8,5 +7,7 @@
 // EMIT_MIR unwrap_unchecked.unwrap_unchecked.Inline.diff
 // EMIT_MIR unwrap_unchecked.unwrap_unchecked.PreCodegen.after.mir
 pub unsafe fn unwrap_unchecked<T>(slf: Option<T>) -> T {
+    // CHECK-LABEL: fn unwrap_unchecked(
+    // CHECK: (inlined #[track_caller] Option::<T>::unwrap_unchecked)
     slf.unwrap_unchecked()
 }

From 1ec10ec77f6fa64de7c03150bfbb831dec376357 Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Sun, 22 Oct 2023 09:27:43 -0700
Subject: [PATCH 116/297] address review comments

---
 tests/mir-opt/inline/asm_unwind.rs            |  3 +
 .../inline/caller_with_trivial_bound.rs       |  3 +
 tests/mir-opt/inline/cycle.rs                 |  2 +
 .../inline/dont_ice_on_generic_rust_call.rs   |  2 +-
 tests/mir-opt/inline/dyn_trait.rs             |  4 +-
 tests/mir-opt/inline/exponential_runtime.rs   |  1 +
 tests/mir-opt/inline/inline_box_fn.rs         |  2 +-
 .../inline/inline_retag.bar.Inline.after.mir  | 59 -----------------
 .../inline/inline_retag.bar.Inline.diff       | 65 +++++++++++++++++++
 tests/mir-opt/inline/inline_retag.rs          | 11 +++-
 tests/mir-opt/inline/inline_trait_method.rs   |  3 +-
 ...ine_scopes_parenting.main.Inline.after.mir |  6 +-
 .../issue_76997_inline_scopes_parenting.rs    |  9 ++-
 13 files changed, 100 insertions(+), 70 deletions(-)
 delete mode 100644 tests/mir-opt/inline/inline_retag.bar.Inline.after.mir
 create mode 100644 tests/mir-opt/inline/inline_retag.bar.Inline.diff

diff --git a/tests/mir-opt/inline/asm_unwind.rs b/tests/mir-opt/inline/asm_unwind.rs
index 596a4592f25f..0ae20e522111 100644
--- a/tests/mir-opt/inline/asm_unwind.rs
+++ b/tests/mir-opt/inline/asm_unwind.rs
@@ -2,6 +2,7 @@
 //
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // needs-asm-support
+// needs-unwind
 // compile-flags: -Zinline-mir-hint-threshold=1000
 #![feature(asm_unwind)]
 
@@ -21,5 +22,7 @@ fn foo() {
 pub fn main() {
     // CHECK-LABEL: fn main(
     // CHECK: (inlined foo)
+    // CHECK: asm!("", options(MAY_UNWIND)) -> [return: {{bb.*}}, unwind: [[unwind:bb.*]]];
+    // CHECK: [[unwind]] (cleanup)
     foo();
 }
diff --git a/tests/mir-opt/inline/caller_with_trivial_bound.rs b/tests/mir-opt/inline/caller_with_trivial_bound.rs
index 722c7f6724ef..40f7f4bbab2f 100644
--- a/tests/mir-opt/inline/caller_with_trivial_bound.rs
+++ b/tests/mir-opt/inline/caller_with_trivial_bound.rs
@@ -15,6 +15,9 @@ impl<T> Factory<T> for IntFactory {
 // EMIT_MIR caller_with_trivial_bound.foo.Inline.diff
 pub fn foo<T>()
 where
+    // Because of this trivial bound, the inliner fails to normalize
+    // `<IntFactory as Factory<T>>::Item`.
+    // Verify that we do not inline anything, which would cause validation ICEs.
     IntFactory: Factory<T>,
 {
     // CHECK-LABEL: fn foo(
diff --git a/tests/mir-opt/inline/cycle.rs b/tests/mir-opt/inline/cycle.rs
index 8716b44b3dc0..350724235ba0 100644
--- a/tests/mir-opt/inline/cycle.rs
+++ b/tests/mir-opt/inline/cycle.rs
@@ -13,6 +13,7 @@ fn f(g: impl Fn()) {
 #[inline(always)]
 fn g() {
     // CHECK-LABEL: fn g(
+    // CHECK-NOT: inlined
     // CHECK: (inlined f::<fn() {main}>)
     // CHECK-NOT: inlined
     f(main);
@@ -21,6 +22,7 @@ fn g() {
 // EMIT_MIR cycle.main.Inline.diff
 fn main() {
     // CHECK-LABEL: fn main(
+    // CHECK-NOT: inlined
     // CHECK: (inlined f::<fn() {g}>)
     // CHECK-NOT: inlined
     f(g);
diff --git a/tests/mir-opt/inline/dont_ice_on_generic_rust_call.rs b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.rs
index 3307a1408fb5..ce5e1855a716 100644
--- a/tests/mir-opt/inline/dont_ice_on_generic_rust_call.rs
+++ b/tests/mir-opt/inline/dont_ice_on_generic_rust_call.rs
@@ -8,6 +8,6 @@ use std::marker::Tuple;
 // EMIT_MIR dont_ice_on_generic_rust_call.call.Inline.diff
 pub fn call<I: Tuple>(mut mock: Box<dyn FnMut<I, Output = ()>>, input: I) {
     // CHECK-LABEL: fn call(
-    // CHECK-NOT: (inlined <Box<dyn FnMut<I, Output = ()>> as FnMut<I>>::call_mut)
+    // CHECK-NOT: inlined
     mock.call_mut(input)
 }
diff --git a/tests/mir-opt/inline/dyn_trait.rs b/tests/mir-opt/inline/dyn_trait.rs
index 005cf155fdb9..ecf220a85e6a 100644
--- a/tests/mir-opt/inline/dyn_trait.rs
+++ b/tests/mir-opt/inline/dyn_trait.rs
@@ -20,7 +20,7 @@ pub trait Query {
 #[inline(always)]
 pub fn mk_cycle<V: Debug>(c: &dyn Cache<V = V>) {
     // CHECK-LABEL: fn mk_cycle(
-    // CHECK-NOT: (inlined <dyn Cache<V = V> as Cache>::store_nocache)
+    // CHECK-NOT: inlined
     c.store_nocache()
 }
 
@@ -36,7 +36,7 @@ pub fn try_execute_query<C: Cache>(c: &C) {
 #[inline(always)]
 pub fn get_query<Q: Query, T>(t: &T) {
     // CHECK-LABEL: fn get_query(
-    // CHECK-NOT: (inlined <Q as Query>::cache::<T>)
+    // CHECK-NOT: inlined
     let c = Q::cache(t);
     // CHECK: (inlined try_execute_query::<<Q as Query>::C>)
     // CHECK: (inlined mk_cycle::<<Q as Query>::V>)
diff --git a/tests/mir-opt/inline/exponential_runtime.rs b/tests/mir-opt/inline/exponential_runtime.rs
index 499bec33ffc9..1199ce4e5588 100644
--- a/tests/mir-opt/inline/exponential_runtime.rs
+++ b/tests/mir-opt/inline/exponential_runtime.rs
@@ -85,6 +85,7 @@ impl A for () {
 // EMIT_MIR exponential_runtime.main.Inline.diff
 fn main() {
     // CHECK-LABEL: fn main(
+    // CHECK-NOT: inlined
     // CHECK: (inlined <() as G>::call)
     // CHECK: (inlined <() as F>::call)
     // CHECK: (inlined <() as E>::call)
diff --git a/tests/mir-opt/inline/inline_box_fn.rs b/tests/mir-opt/inline/inline_box_fn.rs
index 7686477c291a..d2da23939923 100644
--- a/tests/mir-opt/inline/inline_box_fn.rs
+++ b/tests/mir-opt/inline/inline_box_fn.rs
@@ -5,6 +5,6 @@
 // EMIT_MIR inline_box_fn.call.Inline.diff
 fn call(x: Box<dyn Fn(i32)>) {
     // CHECK-LABEL: fn call(
-    // CHECK-NOT: (inlined <Box<dyn Fn(i32)> as Fn<(i32,)>>::call)
+    // CHECK-NOT: inlined
     x(1);
 }
diff --git a/tests/mir-opt/inline/inline_retag.bar.Inline.after.mir b/tests/mir-opt/inline/inline_retag.bar.Inline.after.mir
deleted file mode 100644
index 8c3f3a4589e6..000000000000
--- a/tests/mir-opt/inline/inline_retag.bar.Inline.after.mir
+++ /dev/null
@@ -1,59 +0,0 @@
-// MIR for `bar` after Inline
-
-fn bar() -> bool {
-    let mut _0: bool;
-    let _1: for<'a, 'b> fn(&'a i32, &'b i32) -> bool {foo};
-    let mut _2: for<'a, 'b> fn(&'a i32, &'b i32) -> bool {foo};
-    let mut _3: &i32;
-    let _4: &i32;
-    let _5: i32;
-    let mut _6: &i32;
-    let _7: &i32;
-    let _8: i32;
-    scope 1 {
-        debug f => _1;
-        let mut _9: &i32;
-        let mut _10: &i32;
-        scope 2 (inlined foo) {
-            debug x => _3;
-            debug y => _6;
-            let mut _11: i32;
-            let mut _12: i32;
-        }
-    }
-
-    bb0: {
-        StorageLive(_1);
-        _1 = foo;
-        StorageLive(_2);
-        _2 = _1;
-        StorageLive(_3);
-        StorageLive(_4);
-        _10 = const _;
-        Retag(_10);
-        _4 = &(*_10);
-        _3 = &(*_4);
-        StorageLive(_6);
-        StorageLive(_7);
-        _9 = const _;
-        Retag(_9);
-        _7 = &(*_9);
-        _6 = &(*_7);
-        Retag(_3);
-        Retag(_6);
-        StorageLive(_11);
-        _11 = (*_3);
-        StorageLive(_12);
-        _12 = (*_6);
-        _0 = Eq(move _11, move _12);
-        StorageDead(_12);
-        StorageDead(_11);
-        StorageDead(_6);
-        StorageDead(_3);
-        StorageDead(_2);
-        StorageDead(_1);
-        StorageDead(_7);
-        StorageDead(_4);
-        return;
-    }
-}
diff --git a/tests/mir-opt/inline/inline_retag.bar.Inline.diff b/tests/mir-opt/inline/inline_retag.bar.Inline.diff
new file mode 100644
index 000000000000..8f53f6342ec6
--- /dev/null
+++ b/tests/mir-opt/inline/inline_retag.bar.Inline.diff
@@ -0,0 +1,65 @@
+- // MIR for `bar` before Inline
++ // MIR for `bar` after Inline
+  
+  fn bar() -> bool {
+      let mut _0: bool;
+      let _1: for<'a, 'b> fn(&'a i32, &'b i32) -> bool {foo};
+      let mut _2: for<'a, 'b> fn(&'a i32, &'b i32) -> bool {foo};
+      let mut _3: &i32;
+      let _4: &i32;
+      let _5: i32;
+      let mut _6: &i32;
+      let _7: &i32;
+      let _8: i32;
+      scope 1 {
+          debug f => _1;
+          let mut _9: &i32;
+          let mut _10: &i32;
++         scope 2 (inlined foo) {
++             debug x => _3;
++             debug y => _6;
++             let mut _11: i32;
++             let mut _12: i32;
++         }
+      }
+  
+      bb0: {
+          StorageLive(_1);
+          _1 = foo;
+          StorageLive(_2);
+          _2 = _1;
+          StorageLive(_3);
+          StorageLive(_4);
+          _10 = const _;
+          Retag(_10);
+          _4 = &(*_10);
+          _3 = &(*_4);
+          StorageLive(_6);
+          StorageLive(_7);
+          _9 = const _;
+          Retag(_9);
+          _7 = &(*_9);
+          _6 = &(*_7);
+-         _0 = move _2(move _3, move _6) -> [return: bb1, unwind continue];
+-     }
+- 
+-     bb1: {
++         Retag(_3);
++         Retag(_6);
++         StorageLive(_11);
++         _11 = (*_3);
++         StorageLive(_12);
++         _12 = (*_6);
++         _0 = Eq(move _11, move _12);
++         StorageDead(_12);
++         StorageDead(_11);
+          StorageDead(_6);
+          StorageDead(_3);
+          StorageDead(_2);
+          StorageDead(_1);
+          StorageDead(_7);
+          StorageDead(_4);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/inline_retag.rs b/tests/mir-opt/inline/inline_retag.rs
index c90c0b819b7b..b9058905892b 100644
--- a/tests/mir-opt/inline/inline_retag.rs
+++ b/tests/mir-opt/inline/inline_retag.rs
@@ -6,10 +6,19 @@ fn main() {
     println!("{}", bar());
 }
 
-// EMIT_MIR inline_retag.bar.Inline.after.mir
+// EMIT_MIR inline_retag.bar.Inline.diff
 fn bar() -> bool {
     // CHECK-LABEL: fn bar(
     // CHECK: (inlined foo)
+    // CHECK: debug x => [[x:_.*]];
+    // CHECK: debug y => [[y:_.*]];
+    // CHECK: bb0: {
+    // CHECK: Retag
+    // CHECK: Retag
+    // CHECK: Retag([[x]]);
+    // CHECK: Retag([[y]]);
+    // CHECK: return;
+    // CHECK-NEXT: }
     let f = foo;
     f(&1, &-1)
 }
diff --git a/tests/mir-opt/inline/inline_trait_method.rs b/tests/mir-opt/inline/inline_trait_method.rs
index ea09966a2aa9..b39355637a1c 100644
--- a/tests/mir-opt/inline/inline_trait_method.rs
+++ b/tests/mir-opt/inline/inline_trait_method.rs
@@ -1,3 +1,4 @@
+// Verify that we do not inline the default impl in a trait object.
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
 // compile-flags: -Z span_free_formats
 
@@ -8,7 +9,7 @@ fn main() {
 // EMIT_MIR inline_trait_method.test.Inline.after.mir
 fn test(x: &dyn X) -> u32 {
     // CHECK-LABEL: fn test(
-    // CHECK-NOT: (inlined <dyn X as X>::y)
+    // CHECK-NOT: inlined
     x.y()
 }
 
diff --git a/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.main.Inline.after.mir b/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.main.Inline.after.mir
index ba5eb757cf97..ba4f91b28d53 100644
--- a/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.main.Inline.after.mir
+++ b/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.main.Inline.after.mir
@@ -2,8 +2,8 @@
 
 fn main() -> () {
     let mut _0: ();
-    let _1: {closure@$DIR/issue_76997_inline_scopes_parenting.rs:10:13: 10:16};
-    let mut _2: &{closure@$DIR/issue_76997_inline_scopes_parenting.rs:10:13: 10:16};
+    let _1: {closure@$DIR/issue_76997_inline_scopes_parenting.rs:15:13: 15:16};
+    let mut _2: &{closure@$DIR/issue_76997_inline_scopes_parenting.rs:15:13: 15:16};
     let mut _3: ((),);
     let mut _4: ();
     let mut _5: ();
@@ -19,7 +19,7 @@ fn main() -> () {
 
     bb0: {
         StorageLive(_1);
-        _1 = {closure@$DIR/issue_76997_inline_scopes_parenting.rs:10:13: 10:16};
+        _1 = {closure@$DIR/issue_76997_inline_scopes_parenting.rs:15:13: 15:16};
         StorageLive(_2);
         _2 = &_1;
         StorageLive(_3);
diff --git a/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.rs b/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.rs
index 2a7c931a659e..2fb363c19044 100644
--- a/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.rs
+++ b/tests/mir-opt/inline/issue_76997_inline_scopes_parenting.rs
@@ -3,10 +3,15 @@
 // EMIT_MIR issue_76997_inline_scopes_parenting.main.Inline.after.mir
 fn main() {
     // CHECK-LABEL: fn main(
-    // CHECK: scope 2
+    // CHECK: scope 1 {
+    // CHECK-NEXT: debug f
+    // CHECK-NEXT: scope 2 (inlined main::{closure#0}) {
     // CHECK-NEXT: debug x
-    // CHECK-NEXT: scope 3
+    // CHECK-NEXT: scope 3 {
     // CHECK-NEXT: debug y
+    // CHECK-NEXT: }
+    // CHECK-NEXT: }
+    // CHECK-NEXT: }
     let f = |x| { let y = x; y };
     f(())
 }

From 42e37059a3bf0576dac84f4666bce3d09840f19b Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Wed, 25 Oct 2023 20:41:39 -0400
Subject: [PATCH 117/297] Fix rebase

---
 src/abi.rs | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/src/abi.rs b/src/abi.rs
index 562e1e9a0911..f601cd95f2a6 100644
--- a/src/abi.rs
+++ b/src/abi.rs
@@ -153,21 +153,6 @@ impl<'gcc, 'tcx> FnAbiGccExt<'gcc, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             ty
         };
 
-        #[cfg(feature = "master")]
-        let apply_attrs = |ty: Type<'gcc>, attrs: &ArgAttributes| {
-            if cx.sess().opts.optimize != config::OptLevel::No
-                && attrs.regular.contains(rustc_target::abi::call::ArgAttribute::NoAlias)
-            {
-                ty.make_restrict()
-            } else {
-                ty
-            }
-        };
-        #[cfg(not(feature = "master"))]
-        let apply_attrs = |ty: Type<'gcc>, _attrs: &ArgAttributes| {
-            ty
-        };
-
         for arg in self.args.iter() {
             let arg_ty = match arg.mode {
                 PassMode::Ignore => continue,

From 9efb4ce8badae357b02818b167624b9e14f9fd3b Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Wed, 25 Oct 2023 20:41:45 -0400
Subject: [PATCH 118/297] Update to nightly-2023-10-25

---
 rust-toolchain | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rust-toolchain b/rust-toolchain
index 25a1cea98cce..1dff813db590 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2023-10-08"
+channel = "nightly-2023-10-25"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]

From c12ac7ea76d4051261010932d98a1b7f394a9eaa Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Wed, 25 Oct 2023 20:42:47 -0400
Subject: [PATCH 119/297] Fix warning

---
 src/lib.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/lib.rs b/src/lib.rs
index 66e3475b527a..a530fc994a22 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -12,6 +12,7 @@
  * TODO(antoyo): remove the patches.
  */
 
+#![cfg_attr(not(bootstrap), allow(internal_features))]
 #![cfg_attr(not(bootstrap), doc(rust_logo))]
 #![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
 #![feature(

From 2a2b3ea48bd458d30fdfa044111932b77f690099 Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Wed, 25 Oct 2023 20:43:51 -0400
Subject: [PATCH 120/297] Remove duplication in CI

---
 .github/workflows/ci.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c3bb1bc088e6..65e7a697ab0d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -119,9 +119,6 @@ jobs:
     - name: Add more failing tests because the sysroot is not compiled with LTO
       run: cat failing-non-lto-tests.txt >> failing-ui-tests.txt
 
-    - name: Add more failing tests because the sysroot is not compiled with LTO
-      run: cat failing-non-lto-tests.txt >> failing-ui-tests.txt
-
     - name: Run tests
       run: |
         ./test.sh --release --clean --build-sysroot ${{ matrix.commands }}

From a61cf673cd10ed24394c3403e03d8f7cf1f50170 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Thu, 5 Oct 2023 11:30:55 +0000
Subject: [PATCH 121/297] Reserve `gen` keyword for `gen {}` blocks and `gen
 fn` in 2024 edition

---
 compiler/rustc_ast/src/ast.rs             |  6 +++++
 compiler/rustc_ast/src/token.rs           |  1 +
 compiler/rustc_parse/messages.ftl         |  3 +++
 compiler/rustc_parse/src/errors.rs        |  8 +++++++
 compiler/rustc_parse/src/parser/expr.rs   | 18 ++++++++++++++
 compiler/rustc_parse/src/parser/item.rs   | 11 +++++++--
 compiler/rustc_parse/src/parser/mod.rs    | 11 +++++++++
 compiler/rustc_span/src/symbol.rs         |  6 +++--
 tests/ui/coroutine/gen_block.e2024.stderr | 10 ++++++++
 tests/ui/coroutine/gen_block.none.stderr  | 29 +++++++++++++++++++++++
 tests/ui/coroutine/gen_block.rs           | 13 ++++++++++
 tests/ui/coroutine/gen_fn.e2024.stderr    | 10 ++++++++
 tests/ui/coroutine/gen_fn.none.stderr     |  8 +++++++
 tests/ui/coroutine/gen_fn.rs              |  8 +++++++
 14 files changed, 138 insertions(+), 4 deletions(-)
 create mode 100644 tests/ui/coroutine/gen_block.e2024.stderr
 create mode 100644 tests/ui/coroutine/gen_block.none.stderr
 create mode 100644 tests/ui/coroutine/gen_block.rs
 create mode 100644 tests/ui/coroutine/gen_fn.e2024.stderr
 create mode 100644 tests/ui/coroutine/gen_fn.none.stderr
 create mode 100644 tests/ui/coroutine/gen_fn.rs

diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index e94e26a63bc6..543c528db448 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -2357,6 +2357,12 @@ pub enum Async {
     No,
 }
 
+#[derive(Copy, Clone, Encodable, Decodable, Debug)]
+pub enum Gen {
+    Yes { span: Span, closure_id: NodeId, return_impl_trait_id: NodeId },
+    No,
+}
+
 impl Async {
     pub fn is_async(self) -> bool {
         matches!(self, Async::Yes { .. })
diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs
index dd879a14567c..914c97a14ac0 100644
--- a/compiler/rustc_ast/src/token.rs
+++ b/compiler/rustc_ast/src/token.rs
@@ -197,6 +197,7 @@ pub fn ident_can_begin_expr(name: Symbol, span: Span, is_raw: bool) -> bool {
             kw::Continue,
             kw::False,
             kw::For,
+            kw::Gen,
             kw::If,
             kw::Let,
             kw::Loop,
diff --git a/compiler/rustc_parse/messages.ftl b/compiler/rustc_parse/messages.ftl
index e0778f72bfe6..9bedd7b8a338 100644
--- a/compiler/rustc_parse/messages.ftl
+++ b/compiler/rustc_parse/messages.ftl
@@ -278,6 +278,9 @@ parse_found_expr_would_be_stmt = expected expression, found `{$token}`
 parse_function_body_equals_expr = function body cannot be `= expression;`
     .suggestion = surround the expression with `{"{"}` and `{"}"}` instead of `=` and `;`
 
+parse_gen_block = `gen` blocks are not yet implemented
+    .help = only the keyword is reserved for now
+
 parse_generic_args_in_pat_require_turbofish_syntax = generic args in patterns require the turbofish syntax
 
 parse_generic_parameters_without_angle_brackets = generic parameters without surrounding angle brackets
diff --git a/compiler/rustc_parse/src/errors.rs b/compiler/rustc_parse/src/errors.rs
index aeb4fd0a304a..c0e94d15da0b 100644
--- a/compiler/rustc_parse/src/errors.rs
+++ b/compiler/rustc_parse/src/errors.rs
@@ -520,6 +520,14 @@ pub(crate) struct CatchAfterTry {
     pub span: Span,
 }
 
+#[derive(Diagnostic)]
+#[diag(parse_gen_block)]
+#[help]
+pub(crate) struct GenBlock {
+    #[primary_span]
+    pub span: Span,
+}
+
 #[derive(Diagnostic)]
 #[diag(parse_comma_after_base_struct)]
 #[note]
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 5157106f4e26..e59076acf4c1 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -1422,6 +1422,9 @@ impl<'a> Parser<'a> {
             } else if this.is_try_block() {
                 this.expect_keyword(kw::Try)?;
                 this.parse_try_block(lo)
+            } else if this.is_gen_block() {
+                this.expect_keyword(kw::Gen)?;
+                this.parse_gen_block(lo)
             } else if this.eat_keyword(kw::Return) {
                 this.parse_expr_return()
             } else if this.eat_keyword(kw::Continue) {
@@ -3040,6 +3043,14 @@ impl<'a> Parser<'a> {
         }
     }
 
+    /// Parses a `gen {...}` expression (`gen` token already eaten).
+    fn parse_gen_block(&mut self, _span_lo: Span) -> PResult<'a, P<Expr>> {
+        let (_attrs, _body) = self.parse_inner_attrs_and_block()?;
+
+        Err(errors::GenBlock { span: self.prev_token.span }
+            .into_diagnostic(&self.sess.span_diagnostic))
+    }
+
     fn is_do_catch_block(&self) -> bool {
         self.token.is_keyword(kw::Do)
             && self.is_keyword_ahead(1, &[kw::Catch])
@@ -3059,6 +3070,13 @@ impl<'a> Parser<'a> {
             && self.token.uninterpolated_span().at_least_rust_2018()
     }
 
+    fn is_gen_block(&self) -> bool {
+        self.token.is_keyword(kw::Gen)
+            && self
+                .look_ahead(1, |t| *t == token::OpenDelim(Delimiter::Brace) || t.is_whole_block())
+            && self.token.uninterpolated_span().at_least_rust_2024()
+    }
+
     /// Parses an `async move? {...}` expression.
     fn parse_async_block(&mut self) -> PResult<'a, P<Expr>> {
         let lo = self.token.span;
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
index 982f601c0d5a..4c76ade01393 100644
--- a/compiler/rustc_parse/src/parser/item.rs
+++ b/compiler/rustc_parse/src/parser/item.rs
@@ -2297,9 +2297,9 @@ impl<'a> Parser<'a> {
         // `pub` is added in case users got confused with the ordering like `async pub fn`,
         // only if it wasn't preceded by `default` as `default pub` is invalid.
         let quals: &[Symbol] = if check_pub {
-            &[kw::Pub, kw::Const, kw::Async, kw::Unsafe, kw::Extern]
+            &[kw::Pub, kw::Gen, kw::Const, kw::Async, kw::Unsafe, kw::Extern]
         } else {
-            &[kw::Const, kw::Async, kw::Unsafe, kw::Extern]
+            &[kw::Gen, kw::Const, kw::Async, kw::Unsafe, kw::Extern]
         };
         self.check_keyword_case(kw::Fn, case) // Definitely an `fn`.
             // `$qual fn` or `$qual $qual`:
@@ -2353,6 +2353,9 @@ impl<'a> Parser<'a> {
         let async_start_sp = self.token.span;
         let asyncness = self.parse_asyncness(case);
 
+        let _gen_start_sp = self.token.span;
+        let genness = self.parse_genness(case);
+
         let unsafe_start_sp = self.token.span;
         let unsafety = self.parse_unsafety(case);
 
@@ -2368,6 +2371,10 @@ impl<'a> Parser<'a> {
             }
         }
 
+        if let Gen::Yes { span, .. } = genness {
+            self.sess.emit_err(errors::GenBlock { span });
+        }
+
         if !self.eat_keyword_case(kw::Fn, case) {
             // It is possible for `expect_one_of` to recover given the contents of
             // `self.expected_tokens`, therefore, do not use `self.unexpected()` which doesn't
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index 597303cae73e..53c6d8acff15 100644
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -11,6 +11,7 @@ mod stmt;
 mod ty;
 
 use crate::lexer::UnmatchedDelim;
+use ast::Gen;
 pub use attr_wrapper::AttrWrapper;
 pub use diagnostics::AttemptLocalParseRecovery;
 pub(crate) use expr::ForbiddenLetReason;
@@ -1126,6 +1127,16 @@ impl<'a> Parser<'a> {
         }
     }
 
+    /// Parses genness: `gen` or nothing.
+    fn parse_genness(&mut self, case: Case) -> Gen {
+        if self.token.span.at_least_rust_2024() && self.eat_keyword_case(kw::Gen, case) {
+            let span = self.prev_token.uninterpolated_span();
+            Gen::Yes { span, closure_id: DUMMY_NODE_ID, return_impl_trait_id: DUMMY_NODE_ID }
+        } else {
+            Gen::No
+        }
+    }
+
     /// Parses unsafety: `unsafe` or nothing.
     fn parse_unsafety(&mut self, case: Case) -> Unsafe {
         if self.eat_keyword_case(kw::Unsafe, case) {
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index ff61143a12ba..8ef16ed0dbf7 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -98,6 +98,7 @@ symbols! {
         Builtin:            "builtin",
         Catch:              "catch",
         Default:            "default",
+        Gen:                "gen",
         MacroRules:         "macro_rules",
         Raw:                "raw",
         Union:              "union",
@@ -2188,8 +2189,9 @@ impl Symbol {
         self >= kw::Abstract && self <= kw::Yield
     }
 
-    fn is_unused_keyword_conditional(self, edition: impl FnOnce() -> Edition) -> bool {
-        self == kw::Try && edition() >= Edition::Edition2018
+    fn is_unused_keyword_conditional(self, edition: impl Copy + FnOnce() -> Edition) -> bool {
+        self == kw::Try && edition().at_least_rust_2018()
+            || self == kw::Gen && edition().at_least_rust_2024()
     }
 
     pub fn is_reserved(self, edition: impl Copy + FnOnce() -> Edition) -> bool {
diff --git a/tests/ui/coroutine/gen_block.e2024.stderr b/tests/ui/coroutine/gen_block.e2024.stderr
new file mode 100644
index 000000000000..099a9d13ceef
--- /dev/null
+++ b/tests/ui/coroutine/gen_block.e2024.stderr
@@ -0,0 +1,10 @@
+error: `gen` blocks are not yet implemented
+  --> $DIR/gen_block.rs:5:18
+   |
+LL |     let x = gen {};
+   |                  ^
+   |
+   = help: only the keyword is reserved for now
+
+error: aborting due to previous error
+
diff --git a/tests/ui/coroutine/gen_block.none.stderr b/tests/ui/coroutine/gen_block.none.stderr
new file mode 100644
index 000000000000..1cdd5d070ee5
--- /dev/null
+++ b/tests/ui/coroutine/gen_block.none.stderr
@@ -0,0 +1,29 @@
+error: expected identifier, found reserved keyword `yield`
+  --> $DIR/gen_block.rs:8:19
+   |
+LL |     let y = gen { yield 42 };
+   |             ---   ^^^^^ expected identifier, found reserved keyword
+   |             |
+   |             while parsing this struct
+
+error[E0422]: cannot find struct, variant or union type `gen` in this scope
+  --> $DIR/gen_block.rs:5:13
+   |
+LL |     let x = gen {};
+   |             ^^^ not found in this scope
+
+error[E0422]: cannot find struct, variant or union type `gen` in this scope
+  --> $DIR/gen_block.rs:8:13
+   |
+LL |     let y = gen { yield 42 };
+   |             ^^^ not found in this scope
+
+error[E0422]: cannot find struct, variant or union type `gen` in this scope
+  --> $DIR/gen_block.rs:11:5
+   |
+LL |     gen {};
+   |     ^^^ not found in this scope
+
+error: aborting due to 4 previous errors
+
+For more information about this error, try `rustc --explain E0422`.
diff --git a/tests/ui/coroutine/gen_block.rs b/tests/ui/coroutine/gen_block.rs
new file mode 100644
index 000000000000..5c0db5b3d38d
--- /dev/null
+++ b/tests/ui/coroutine/gen_block.rs
@@ -0,0 +1,13 @@
+// revisions: e2024 none
+//[e2024] compile-flags: --edition 2024 -Zunstable-options
+
+fn main() {
+    let x = gen {};
+    //[none]~^ ERROR: cannot find
+    //[e2024]~^^ ERROR: `gen` blocks are not yet implemented
+    let y = gen { yield 42 };
+    //[none]~^ ERROR: found reserved keyword `yield`
+    //[none]~| ERROR: cannot find
+    gen {};
+    //[none]~^ ERROR: cannot find
+}
diff --git a/tests/ui/coroutine/gen_fn.e2024.stderr b/tests/ui/coroutine/gen_fn.e2024.stderr
new file mode 100644
index 000000000000..41bd04d97691
--- /dev/null
+++ b/tests/ui/coroutine/gen_fn.e2024.stderr
@@ -0,0 +1,10 @@
+error: `gen` blocks are not yet implemented
+  --> $DIR/gen_fn.rs:4:1
+   |
+LL | gen fn foo() {}
+   | ^^^
+   |
+   = help: only the keyword is reserved for now
+
+error: aborting due to previous error
+
diff --git a/tests/ui/coroutine/gen_fn.none.stderr b/tests/ui/coroutine/gen_fn.none.stderr
new file mode 100644
index 000000000000..5e7bd9d8bbfb
--- /dev/null
+++ b/tests/ui/coroutine/gen_fn.none.stderr
@@ -0,0 +1,8 @@
+error: expected one of `#`, `async`, `const`, `default`, `extern`, `fn`, `pub`, `unsafe`, or `use`, found `gen`
+  --> $DIR/gen_fn.rs:4:1
+   |
+LL | gen fn foo() {}
+   | ^^^ expected one of 9 possible tokens
+
+error: aborting due to previous error
+
diff --git a/tests/ui/coroutine/gen_fn.rs b/tests/ui/coroutine/gen_fn.rs
new file mode 100644
index 000000000000..9566660dfb56
--- /dev/null
+++ b/tests/ui/coroutine/gen_fn.rs
@@ -0,0 +1,8 @@
+// revisions: e2024 none
+//[e2024] compile-flags: --edition 2024 -Zunstable-options
+
+gen fn foo() {}
+//[none]~^ ERROR: expected one of `#`, `async`, `const`, `default`, `extern`, `fn`, `pub`, `unsafe`, or `use`, found `gen`
+//[e2024]~^^ ERROR: `gen` blocks are not yet implemented
+
+fn main() {}

From 14423080f1f9e9729d372f9b5da06c0ab4aef6e3 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Fri, 20 Oct 2023 19:21:24 +0000
Subject: [PATCH 122/297] Add hir::GeneratorKind::Gen

---
 compiler/rustc_ast_lowering/src/expr.rs       |  7 +++--
 .../src/diagnostics/conflict_errors.rs        |  5 ++++
 .../src/diagnostics/region_name.rs            | 14 ++++++++++
 .../src/debuginfo/type_names.rs               |  3 ++
 compiler/rustc_hir/src/hir.rs                 | 12 ++++++++
 compiler/rustc_hir_typeck/src/check.rs        | 19 +++++++------
 compiler/rustc_middle/src/mir/terminator.rs   | 10 +++++++
 compiler/rustc_middle/src/ty/util.rs          |  2 ++
 compiler/rustc_smir/src/rustc_smir/mod.rs     | 28 +++++++++++++------
 .../src/traits/error_reporting/suggestions.rs | 17 ++++++++++-
 .../error_reporting/type_err_ctxt_ext.rs      |  3 ++
 compiler/stable_mir/src/mir/body.rs           |  1 +
 12 files changed, 99 insertions(+), 22 deletions(-)

diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index b82f878ea87e..ab82bbd46a6e 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -712,7 +712,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let full_span = expr.span.to(await_kw_span);
         match self.coroutine_kind {
             Some(hir::CoroutineKind::Async(_)) => {}
-            Some(hir::CoroutineKind::Coroutine) | None => {
+            Some(hir::CoroutineKind::Coroutine) | Some(hir::CoroutineKind::Gen(_)) | None => {
                 self.tcx.sess.emit_err(AwaitOnlyInAsyncFnAndBlocks {
                     await_kw_span,
                     item_span: self.current_item,
@@ -936,8 +936,8 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 }
                 Some(movability)
             }
-            Some(hir::CoroutineKind::Async(_)) => {
-                panic!("non-`async` closure body turned `async` during lowering");
+            Some(hir::CoroutineKind::Gen(_)) | Some(hir::CoroutineKind::Async(_)) => {
+                panic!("non-`async`/`gen` closure body turned `async`/`gen` during lowering");
             }
             None => {
                 if movability == Movability::Static {
@@ -1446,6 +1446,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
     fn lower_expr_yield(&mut self, span: Span, opt_expr: Option<&Expr>) -> hir::ExprKind<'hir> {
         match self.coroutine_kind {
             Some(hir::CoroutineKind::Coroutine) => {}
+            Some(hir::CoroutineKind::Gen(_)) => {}
             Some(hir::CoroutineKind::Async(_)) => {
                 self.tcx.sess.emit_err(AsyncCoroutinesNotSupported { span });
             }
diff --git a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
index 928c25d10617..9db7dec1cf8e 100644
--- a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
@@ -2505,6 +2505,11 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         };
         let kind = match use_span.coroutine_kind() {
             Some(coroutine_kind) => match coroutine_kind {
+                CoroutineKind::Gen(kind) => match kind {
+                    CoroutineSource::Block => "gen block",
+                    CoroutineSource::Closure => "gen closure",
+                    _ => bug!("gen block/closure expected, but gen function found."),
+                },
                 CoroutineKind::Async(async_kind) => match async_kind {
                     CoroutineSource::Block => "async block",
                     CoroutineSource::Closure => "async closure",
diff --git a/compiler/rustc_borrowck/src/diagnostics/region_name.rs b/compiler/rustc_borrowck/src/diagnostics/region_name.rs
index e630ac7c4fae..d38cfbc54d7a 100644
--- a/compiler/rustc_borrowck/src/diagnostics/region_name.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/region_name.rs
@@ -698,6 +698,20 @@ impl<'tcx> MirBorrowckCtxt<'_, 'tcx> {
                             " of async function"
                         }
                     },
+                    Some(hir::CoroutineKind::Gen(gen)) => match gen {
+                        hir::CoroutineSource::Block => " of gen block",
+                        hir::CoroutineSource::Closure => " of gen closure",
+                        hir::CoroutineSource::Fn => {
+                            let parent_item =
+                                hir.get_by_def_id(hir.get_parent_item(mir_hir_id).def_id);
+                            let output = &parent_item
+                                .fn_decl()
+                                .expect("coroutine lowered from gen fn should be in fn")
+                                .output;
+                            span = output.span();
+                            " of gen function"
+                        }
+                    },
                     Some(hir::CoroutineKind::Coroutine) => " of coroutine",
                     None => " of closure",
                 };
diff --git a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
index 5900c764073d..1a85eb8dd797 100644
--- a/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
+++ b/compiler/rustc_codegen_ssa/src/debuginfo/type_names.rs
@@ -560,6 +560,9 @@ pub fn push_item_name(tcx: TyCtxt<'_>, def_id: DefId, qualified: bool, output: &
 
 fn coroutine_kind_label(coroutine_kind: Option<CoroutineKind>) -> &'static str {
     match coroutine_kind {
+        Some(CoroutineKind::Gen(CoroutineSource::Block)) => "gen_block",
+        Some(CoroutineKind::Gen(CoroutineSource::Closure)) => "gen_closure",
+        Some(CoroutineKind::Gen(CoroutineSource::Fn)) => "gen_fn",
         Some(CoroutineKind::Async(CoroutineSource::Block)) => "async_block",
         Some(CoroutineKind::Async(CoroutineSource::Closure)) => "async_closure",
         Some(CoroutineKind::Async(CoroutineSource::Fn)) => "async_fn",
diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs
index 259af4f565bd..22ee39634a56 100644
--- a/compiler/rustc_hir/src/hir.rs
+++ b/compiler/rustc_hir/src/hir.rs
@@ -1513,6 +1513,9 @@ pub enum CoroutineKind {
     /// An explicit `async` block or the body of an async function.
     Async(CoroutineSource),
 
+    /// An explicit `gen` block or the body of a `gen` function.
+    Gen(CoroutineSource),
+
     /// A coroutine literal created via a `yield` inside a closure.
     Coroutine,
 }
@@ -1529,6 +1532,14 @@ impl fmt::Display for CoroutineKind {
                 k.fmt(f)
             }
             CoroutineKind::Coroutine => f.write_str("coroutine"),
+            CoroutineKind::Gen(k) => {
+                if f.alternate() {
+                    f.write_str("`gen` ")?;
+                } else {
+                    f.write_str("gen ")?
+                }
+                k.fmt(f)
+            }
         }
     }
 }
@@ -2242,6 +2253,7 @@ impl From<CoroutineKind> for YieldSource {
             // Guess based on the kind of the current coroutine.
             CoroutineKind::Coroutine => Self::Yield,
             CoroutineKind::Async(_) => Self::Await { expr: None },
+            CoroutineKind::Gen(_) => Self::Yield,
         }
     }
 }
diff --git a/compiler/rustc_hir_typeck/src/check.rs b/compiler/rustc_hir_typeck/src/check.rs
index e7060dac844b..14cd3881a068 100644
--- a/compiler/rustc_hir_typeck/src/check.rs
+++ b/compiler/rustc_hir_typeck/src/check.rs
@@ -58,15 +58,16 @@ pub(super) fn check_fn<'a, 'tcx>(
     if let Some(kind) = body.coroutine_kind
         && can_be_coroutine.is_some()
     {
-        let yield_ty = if kind == hir::CoroutineKind::Coroutine {
-            let yield_ty = fcx.next_ty_var(TypeVariableOrigin {
-                kind: TypeVariableOriginKind::TypeInference,
-                span,
-            });
-            fcx.require_type_is_sized(yield_ty, span, traits::SizedYieldType);
-            yield_ty
-        } else {
-            Ty::new_unit(tcx)
+        let yield_ty = match kind {
+            hir::CoroutineKind::Gen(..) | hir::CoroutineKind::Coroutine => {
+                let yield_ty = fcx.next_ty_var(TypeVariableOrigin {
+                    kind: TypeVariableOriginKind::TypeInference,
+                    span,
+                });
+                fcx.require_type_is_sized(yield_ty, span, traits::SizedYieldType);
+                yield_ty
+            }
+            hir::CoroutineKind::Async(..) => Ty::new_unit(tcx),
         };
 
         // Resume type defaults to `()` if the coroutine has no argument.
diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs
index affa83fa3485..ef1d0c867ea1 100644
--- a/compiler/rustc_middle/src/mir/terminator.rs
+++ b/compiler/rustc_middle/src/mir/terminator.rs
@@ -148,8 +148,14 @@ impl<O> AssertKind<O> {
             RemainderByZero(_) => "attempt to calculate the remainder with a divisor of zero",
             ResumedAfterReturn(CoroutineKind::Coroutine) => "coroutine resumed after completion",
             ResumedAfterReturn(CoroutineKind::Async(_)) => "`async fn` resumed after completion",
+            ResumedAfterReturn(CoroutineKind::Gen(_)) => {
+                bug!("`gen fn` should just keep returning `None` after the first time")
+            }
             ResumedAfterPanic(CoroutineKind::Coroutine) => "coroutine resumed after panicking",
             ResumedAfterPanic(CoroutineKind::Async(_)) => "`async fn` resumed after panicking",
+            ResumedAfterPanic(CoroutineKind::Gen(_)) => {
+                bug!("`gen fn` should just keep returning `None` after panicking")
+            }
             BoundsCheck { .. } | MisalignedPointerDereference { .. } => {
                 bug!("Unexpected AssertKind")
             }
@@ -236,10 +242,14 @@ impl<O> AssertKind<O> {
             DivisionByZero(_) => middle_assert_divide_by_zero,
             RemainderByZero(_) => middle_assert_remainder_by_zero,
             ResumedAfterReturn(CoroutineKind::Async(_)) => middle_assert_async_resume_after_return,
+            // FIXME(gen_blocks): custom error message for `gen` blocks
+            ResumedAfterReturn(CoroutineKind::Gen(_)) => middle_assert_async_resume_after_return,
             ResumedAfterReturn(CoroutineKind::Coroutine) => {
                 middle_assert_coroutine_resume_after_return
             }
             ResumedAfterPanic(CoroutineKind::Async(_)) => middle_assert_async_resume_after_panic,
+            // FIXME(gen_blocks): custom error message for `gen` blocks
+            ResumedAfterPanic(CoroutineKind::Gen(_)) => middle_assert_async_resume_after_panic,
             ResumedAfterPanic(CoroutineKind::Coroutine) => {
                 middle_assert_coroutine_resume_after_panic
             }
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index be48c0e8926b..d29d1902404f 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -749,6 +749,7 @@ impl<'tcx> TyCtxt<'tcx> {
             DefKind::Coroutine => match self.coroutine_kind(def_id).unwrap() {
                 rustc_hir::CoroutineKind::Async(..) => "async closure",
                 rustc_hir::CoroutineKind::Coroutine => "coroutine",
+                rustc_hir::CoroutineKind::Gen(..) => "gen closure",
             },
             _ => def_kind.descr(def_id),
         }
@@ -766,6 +767,7 @@ impl<'tcx> TyCtxt<'tcx> {
             DefKind::Coroutine => match self.coroutine_kind(def_id).unwrap() {
                 rustc_hir::CoroutineKind::Async(..) => "an",
                 rustc_hir::CoroutineKind::Coroutine => "a",
+                rustc_hir::CoroutineKind::Gen(..) => "a",
             },
             _ => def_kind.article(),
         }
diff --git a/compiler/rustc_smir/src/rustc_smir/mod.rs b/compiler/rustc_smir/src/rustc_smir/mod.rs
index 928536721c95..219f6d284dd8 100644
--- a/compiler/rustc_smir/src/rustc_smir/mod.rs
+++ b/compiler/rustc_smir/src/rustc_smir/mod.rs
@@ -880,18 +880,28 @@ impl<'tcx> Stable<'tcx> for mir::AggregateKind<'tcx> {
     }
 }
 
+impl<'tcx> Stable<'tcx> for rustc_hir::CoroutineSource {
+    type T = stable_mir::mir::CoroutineSource;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        use rustc_hir::CoroutineSource;
+        match self {
+            CoroutineSource::Block => stable_mir::mir::CoroutineSource::Block,
+            CoroutineSource::Closure => stable_mir::mir::CoroutineSource::Closure,
+            CoroutineSource::Fn => stable_mir::mir::CoroutineSource::Fn,
+        }
+    }
+}
+
 impl<'tcx> Stable<'tcx> for rustc_hir::CoroutineKind {
     type T = stable_mir::mir::CoroutineKind;
-    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        use rustc_hir::{CoroutineKind, CoroutineSource};
+    fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
+        use rustc_hir::CoroutineKind;
         match self {
-            CoroutineKind::Async(async_gen) => {
-                let async_gen = match async_gen {
-                    CoroutineSource::Block => stable_mir::mir::CoroutineSource::Block,
-                    CoroutineSource::Closure => stable_mir::mir::CoroutineSource::Closure,
-                    CoroutineSource::Fn => stable_mir::mir::CoroutineSource::Fn,
-                };
-                stable_mir::mir::CoroutineKind::Async(async_gen)
+            CoroutineKind::Async(source) => {
+                stable_mir::mir::CoroutineKind::Async(source.stable(tables))
+            }
+            CoroutineKind::Gen(source) => {
+                stable_mir::mir::CoroutineKind::Gen(source.stable(tables))
             }
             CoroutineKind::Coroutine => stable_mir::mir::CoroutineKind::Coroutine,
         }
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
index b9a08056ad16..3087b734cf27 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
@@ -2425,6 +2425,21 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         CoroutineKind::Async(CoroutineSource::Closure) => {
                             format!("future created by async closure is not {trait_name}")
                         }
+                        CoroutineKind::Gen(CoroutineSource::Fn) => self
+                            .tcx
+                            .parent(coroutine_did)
+                            .as_local()
+                            .map(|parent_did| hir.local_def_id_to_hir_id(parent_did))
+                            .and_then(|parent_hir_id| hir.opt_name(parent_hir_id))
+                            .map(|name| {
+                                format!("iterator returned by `{name}` is not {trait_name}")
+                            })?,
+                        CoroutineKind::Gen(CoroutineSource::Block) => {
+                            format!("iterator created by gen block is not {trait_name}")
+                        }
+                        CoroutineKind::Gen(CoroutineSource::Closure) => {
+                            format!("iterator created by gen closure is not {trait_name}")
+                        }
                     })
                 })
                 .unwrap_or_else(|| format!("{future_or_coroutine} is not {trait_name}"));
@@ -2905,7 +2920,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             }
             ObligationCauseCode::SizedCoroutineInterior(coroutine_def_id) => {
                 let what = match self.tcx.coroutine_kind(coroutine_def_id) {
-                    None | Some(hir::CoroutineKind::Coroutine) => "yield",
+                    None | Some(hir::CoroutineKind::Coroutine) | Some(hir::CoroutineKind::Gen(_)) => "yield",
                     Some(hir::CoroutineKind::Async(..)) => "await",
                 };
                 err.note(format!(
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
index 353a69ef6d32..030813a4ac37 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
@@ -1614,6 +1614,9 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
             hir::CoroutineKind::Async(hir::CoroutineSource::Block) => "an async block",
             hir::CoroutineKind::Async(hir::CoroutineSource::Fn) => "an async function",
             hir::CoroutineKind::Async(hir::CoroutineSource::Closure) => "an async closure",
+            hir::CoroutineKind::Gen(hir::CoroutineSource::Block) => "a gen block",
+            hir::CoroutineKind::Gen(hir::CoroutineSource::Fn) => "a gen function",
+            hir::CoroutineKind::Gen(hir::CoroutineSource::Closure) => "a gen closure",
         })
     }
 
diff --git a/compiler/stable_mir/src/mir/body.rs b/compiler/stable_mir/src/mir/body.rs
index c2e79eaaf3ff..b686004b2fef 100644
--- a/compiler/stable_mir/src/mir/body.rs
+++ b/compiler/stable_mir/src/mir/body.rs
@@ -137,6 +137,7 @@ pub enum UnOp {
 pub enum CoroutineKind {
     Async(CoroutineSource),
     Coroutine,
+    Gen(CoroutineSource),
 }
 
 #[derive(Clone, Debug)]

From a6984f5961c6a211b64797ff0db595228d00abf5 Mon Sep 17 00:00:00 2001
From: Antoni Boucher <bouanto@zoho.com>
Date: Wed, 25 Oct 2023 20:56:39 -0400
Subject: [PATCH 123/297] Fix tests

---
 Readme.md                 | 2 +-
 failing-non-lto-tests.txt | 2 +-
 failing-ui-tests.txt      | 8 ++++----
 failing-ui-tests12.txt    | 2 +-
 rust-toolchain            | 2 +-
 test.sh                   | 4 ++--
 6 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/Readme.md b/Readme.md
index 1bad1e71137c..95fc6374c09a 100644
--- a/Readme.md
+++ b/Readme.md
@@ -292,7 +292,7 @@ To send the changes to the rust repo:
 ```bash
 cd ../rust
 git pull origin master
-git checkbout -b subtree-update_cg_gcc_YYYY-MM-DD
+git checkout -b subtree-update_cg_gcc_YYYY-MM-DD
 PATH="$HOME/bin:$PATH" ~/bin/git-subtree pull --prefix=compiler/rustc_codegen_gcc/ https://github.com/rust-lang/rustc_codegen_gcc.git master
 git push
 ```
diff --git a/failing-non-lto-tests.txt b/failing-non-lto-tests.txt
index 2f338f7d3c87..4fd60f2b8e4f 100644
--- a/failing-non-lto-tests.txt
+++ b/failing-non-lto-tests.txt
@@ -5,7 +5,7 @@ tests/ui/lto/lto-many-codegen-units.rs
 tests/ui/lto/issue-100772.rs
 tests/ui/lto/lto-rustc-loads-linker-plugin.rs
 tests/ui/panic-runtime/lto-unwind.rs
-tests/ui/sanitize/issue-111184-generator-witness.rs
+tests/ui/sanitize/issue-111184-coroutine-witness.rs
 tests/ui/sepcomp/sepcomp-lib-lto.rs
 tests/ui/lto/lto-opt-level-s.rs
 tests/ui/lto/lto-opt-level-z.rs
diff --git a/failing-ui-tests.txt b/failing-ui-tests.txt
index 771da5812957..22044eabe969 100644
--- a/failing-ui-tests.txt
+++ b/failing-ui-tests.txt
@@ -21,8 +21,8 @@ tests/ui/cfg/cfg-panic-abort.rs
 tests/ui/drop/dynamic-drop-async.rs
 tests/ui/drop/repeat-drop.rs
 tests/ui/fmt/format-args-capture.rs
-tests/ui/generator/panic-drops-resume.rs
-tests/ui/generator/panic-drops.rs
+tests/ui/coroutine/panic-drops-resume.rs
+tests/ui/coroutine/panic-drops.rs
 tests/ui/intrinsics/panic-uninitialized-zeroed.rs
 tests/ui/iterators/iter-sum-overflow-debug.rs
 tests/ui/iterators/iter-sum-overflow-overflow-checks.rs
@@ -53,7 +53,7 @@ tests/ui/simd/issue-89193.rs
 tests/ui/statics/issue-91050-1.rs
 tests/ui/statics/issue-91050-2.rs
 tests/ui/alloc-error/default-alloc-error-hook.rs
-tests/ui/generator/panic-safe.rs
+tests/ui/coroutine/panic-safe.rs
 tests/ui/issues/issue-14875.rs
 tests/ui/issues/issue-29948.rs
 tests/ui/panics/nested_panic_caught.rs
@@ -70,5 +70,5 @@ tests/ui/lto/lto-thin-rustc-loads-linker-plugin.rs
 tests/ui/lto/all-crates.rs
 tests/ui/async-await/deep-futures-are-freeze.rs
 tests/ui/closures/capture-unsized-by-ref.rs
-tests/ui/generator/resume-after-return.rs
+tests/ui/coroutine/resume-after-return.rs
 tests/ui/macros/rfc-2011-nicer-assert-messages/all-expr-kinds.rs
diff --git a/failing-ui-tests12.txt b/failing-ui-tests12.txt
index f027afa78a38..f91aa9253184 100644
--- a/failing-ui-tests12.txt
+++ b/failing-ui-tests12.txt
@@ -28,7 +28,7 @@ tests/ui/rfcs/rfc-1937-termination-trait/termination-trait-in-test.rs
 tests/ui/async-await/async-fn-size-moved-locals.rs
 tests/ui/async-await/async-fn-size-uninit-locals.rs
 tests/ui/cfg/cfg-panic.rs
-tests/ui/generator/size-moved-locals.rs
+tests/ui/coroutine/size-moved-locals.rs
 tests/ui/macros/rfc-2011-nicer-assert-messages/all-not-available-cases.rs
 tests/ui/simd/intrinsic/generic-gather-pass.rs
 tests/ui/simd/issue-85915-simd-ptrs.rs
diff --git a/rust-toolchain b/rust-toolchain
index 1dff813db590..205ec53b425b 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2023-10-25"
+channel = "nightly-2023-10-21"
 components = ["rust-src", "rustc-dev", "llvm-tools-preview"]
diff --git a/test.sh b/test.sh
index 2eceee7c1e9d..fa34c31dbaa7 100755
--- a/test.sh
+++ b/test.sh
@@ -371,10 +371,10 @@ function test_rustc() {
 
     git checkout -- tests/ui/issues/auxiliary/issue-3136-a.rs # contains //~ERROR, but shouldn't be removed
 
-    rm -r tests/ui/{abi*,extern/,unsized-locals/,proc-macro/,threads-sendsync/,thinlto/,borrowck/,chalkify/bugs/,test*,consts/const-float-bits-reject-conv.rs,consts/issue-miri-1910.rs} || true
+    rm -r tests/ui/{abi*,extern/,unsized-locals/,proc-macro/,threads-sendsync/,borrowck/,test*,consts/issue-miri-1910.rs} || true
     rm tests/ui/mir/mir_heavy_promoted.rs # this test is oom-killed in the CI.
     # Tests generating errors.
-    rm tests/ui/consts/const-eval/nonnull_as_ref_ub.rs tests/ui/consts/issue-94675.rs
+    rm tests/ui/consts/issue-94675.rs
     for test in $(rg --files-with-matches "thread" tests/ui); do
       rm $test
     done

From 391b472a370a5d35c43bd0a26182076cf0c17ca9 Mon Sep 17 00:00:00 2001
From: Havard Eidnes <he@NetBSD.org>
Date: Thu, 26 Oct 2023 17:10:16 +0000
Subject: [PATCH 124/297] rustc_llvm/build.rs: improve comment for NetBSD/i386
 targets

...explaining why we need -latomic (gcc & g++ built for i486,
and LLVM insisting on use of 64-bit atomics).
---
 compiler/rustc_llvm/build.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs
index acb1d9607a4f..e4412132136b 100644
--- a/compiler/rustc_llvm/build.rs
+++ b/compiler/rustc_llvm/build.rs
@@ -258,7 +258,9 @@ fn main() {
     {
         println!("cargo:rustc-link-lib=z");
     } else if target.contains("netbsd") {
-        // Building for i586 or i686, we need -latomic for 64-bit atomics
+        // On NetBSD/i386, gcc and g++ is built for i486 (to maximize backward compat)
+        // However, LLVM insists on using 64-bit atomics.
+        // This gives rise to a need to link rust itself with -latomic for these targets
         if target.starts_with("i586")
            || target.starts_with("i686")
         {

From 828f069c12b356631ef2f7b707d51708861a788d Mon Sep 17 00:00:00 2001
From: Urgau <urgau@numericable.fr>
Date: Fri, 20 Oct 2023 19:33:22 +0200
Subject: [PATCH 125/297] Remove most indentation in check-cfg impl

---
 compiler/rustc_interface/src/interface.rs | 397 ++++++++++------------
 1 file changed, 189 insertions(+), 208 deletions(-)

diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index d2ce77ad5351..221889253821 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -139,7 +139,7 @@ pub fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -> Check
             let filename = FileName::cfg_spec_source_code(&s);
 
             macro_rules! error {
-                ($reason: expr) => {
+                ($reason:expr) => {
                     handler.early_error(format!(
                         concat!("invalid `--check-cfg` argument: `{}` (", $reason, ")"),
                         s
@@ -147,217 +147,198 @@ pub fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -> Check
                 };
             }
 
-            let expected_error =
-                || error!("expected `cfg(name, values(\"value1\", \"value2\", ... \"valueN\"))`");
+            let expected_error = || -> ! {
+                error!("expected `cfg(name, values(\"value1\", \"value2\", ... \"valueN\"))`")
+            };
 
-            match maybe_new_parser_from_source_str(&sess, filename, s.to_string()) {
-                Ok(mut parser) => match parser.parse_meta_item() {
-                    Ok(meta_item) if parser.token == token::Eof => {
-                        if let Some(args) = meta_item.meta_item_list() {
-                            if meta_item.has_name(sym::names) {
-                                // defaults are flipped for the old syntax
-                                if old_syntax == None {
-                                    check_cfg.exhaustive_names = false;
-                                    check_cfg.exhaustive_values = false;
-                                }
-                                old_syntax = Some(true);
+            let Ok(mut parser) = maybe_new_parser_from_source_str(&sess, filename, s.to_string())
+            else {
+                expected_error();
+            };
 
-                                check_cfg.exhaustive_names = true;
-                                for arg in args {
-                                    if arg.is_word() && arg.ident().is_some() {
-                                        let ident = arg.ident().expect("multi-segment cfg key");
-                                        check_cfg
-                                            .expecteds
-                                            .entry(ident.name.to_string())
-                                            .or_insert(ExpectedValues::Any);
-                                    } else {
-                                        error!("`names()` arguments must be simple identifiers");
-                                    }
-                                }
-                            } else if meta_item.has_name(sym::values) {
-                                // defaults are flipped for the old syntax
-                                if old_syntax == None {
-                                    check_cfg.exhaustive_names = false;
-                                    check_cfg.exhaustive_values = false;
-                                }
-                                old_syntax = Some(true);
-
-                                if let Some((name, values)) = args.split_first() {
-                                    if name.is_word() && name.ident().is_some() {
-                                        let ident = name.ident().expect("multi-segment cfg key");
-                                        let expected_values = check_cfg
-                                            .expecteds
-                                            .entry(ident.name.to_string())
-                                            .and_modify(|expected_values| match expected_values {
-                                                ExpectedValues::Some(_) => {}
-                                                ExpectedValues::Any => {
-                                                    // handle the case where names(...) was done
-                                                    // before values by changing to a list
-                                                    *expected_values =
-                                                        ExpectedValues::Some(FxHashSet::default());
-                                                }
-                                            })
-                                            .or_insert_with(|| {
-                                                ExpectedValues::Some(FxHashSet::default())
-                                            });
-
-                                        let ExpectedValues::Some(expected_values) = expected_values
-                                        else {
-                                            bug!("`expected_values` should be a list a values")
-                                        };
-
-                                        for val in values {
-                                            if let Some(LitKind::Str(s, _)) =
-                                                val.lit().map(|lit| &lit.kind)
-                                            {
-                                                expected_values.insert(Some(s.to_string()));
-                                            } else {
-                                                error!(
-                                                    "`values()` arguments must be string literals"
-                                                );
-                                            }
-                                        }
-
-                                        if values.is_empty() {
-                                            expected_values.insert(None);
-                                        }
-                                    } else {
-                                        error!(
-                                            "`values()` first argument must be a simple identifier"
-                                        );
-                                    }
-                                } else if args.is_empty() {
-                                    check_cfg.exhaustive_values = true;
-                                } else {
-                                    expected_error();
-                                }
-                            } else if meta_item.has_name(sym::cfg) {
-                                old_syntax = Some(false);
-
-                                let mut names = Vec::new();
-                                let mut values: FxHashSet<_> = Default::default();
-
-                                let mut any_specified = false;
-                                let mut values_specified = false;
-                                let mut values_any_specified = false;
-
-                                for arg in args {
-                                    if arg.is_word() && let Some(ident) = arg.ident() {
-                                        if values_specified {
-                                            error!("`cfg()` names cannot be after values");
-                                        }
-                                        names.push(ident);
-                                    } else if arg.has_name(sym::any)
-                                        && let Some(args) = arg.meta_item_list()
-                                    {
-                                        if any_specified {
-                                            error!("`any()` cannot be specified multiple times");
-                                        }
-                                        any_specified = true;
-                                        if !args.is_empty() {
-                                            error!("`any()` must be empty");
-                                        }
-                                    } else if arg.has_name(sym::values)
-                                        && let Some(args) = arg.meta_item_list()
-                                    {
-                                        if names.is_empty() {
-                                            error!(
-                                                "`values()` cannot be specified before the names"
-                                            );
-                                        } else if values_specified {
-                                            error!(
-                                                "`values()` cannot be specified multiple times"
-                                            );
-                                        }
-                                        values_specified = true;
-
-                                        for arg in args {
-                                            if let Some(LitKind::Str(s, _)) =
-                                                arg.lit().map(|lit| &lit.kind)
-                                            {
-                                                values.insert(Some(s.to_string()));
-                                            } else if arg.has_name(sym::any)
-                                                && let Some(args) = arg.meta_item_list()
-                                            {
-                                                if values_any_specified {
-                                                    error!(
-                                                        "`any()` in `values()` cannot be specified multiple times"
-                                                    );
-                                                }
-                                                values_any_specified = true;
-                                                if !args.is_empty() {
-                                                    error!("`any()` must be empty");
-                                                }
-                                            } else {
-                                                error!(
-                                                    "`values()` arguments must be string literals or `any()`"
-                                                );
-                                            }
-                                        }
-                                    } else {
-                                        error!(
-                                            "`cfg()` arguments must be simple identifiers, `any()` or `values(...)`"
-                                        );
-                                    }
-                                }
-
-                                if values.is_empty() && !values_any_specified && !any_specified {
-                                    values.insert(None);
-                                } else if !values.is_empty() && values_any_specified {
-                                    error!(
-                                        "`values()` arguments cannot specify string literals and `any()` at the same time"
-                                    );
-                                }
-
-                                if any_specified {
-                                    if !names.is_empty()
-                                        || !values.is_empty()
-                                        || values_any_specified
-                                    {
-                                        error!("`cfg(any())` can only be provided in isolation");
-                                    }
-
-                                    check_cfg.exhaustive_names = false;
-                                } else {
-                                    for name in names {
-                                        check_cfg
-                                            .expecteds
-                                            .entry(name.to_string())
-                                            .and_modify(|v| match v {
-                                                ExpectedValues::Some(v)
-                                                    if !values_any_specified =>
-                                                {
-                                                    v.extend(values.clone())
-                                                }
-                                                ExpectedValues::Some(_) => *v = ExpectedValues::Any,
-                                                ExpectedValues::Any => {}
-                                            })
-                                            .or_insert_with(|| {
-                                                if values_any_specified {
-                                                    ExpectedValues::Any
-                                                } else {
-                                                    ExpectedValues::Some(values.clone())
-                                                }
-                                            });
-                                    }
-                                }
-                            } else {
-                                expected_error();
-                            }
-                        } else {
-                            expected_error();
-                        }
-                    }
-                    Ok(..) => expected_error(),
-                    Err(err) => {
-                        err.cancel();
-                        expected_error();
-                    }
-                },
-                Err(errs) => {
-                    drop(errs);
+            let meta_item = match parser.parse_meta_item() {
+                Ok(meta_item) if parser.token == token::Eof => meta_item,
+                Ok(..) => expected_error(),
+                Err(err) => {
+                    err.cancel();
                     expected_error();
                 }
+            };
+
+            let Some(args) = meta_item.meta_item_list() else {
+                expected_error();
+            };
+
+            if meta_item.has_name(sym::names) {
+                // defaults are flipped for the old syntax
+                if old_syntax == None {
+                    check_cfg.exhaustive_names = false;
+                    check_cfg.exhaustive_values = false;
+                }
+                old_syntax = Some(true);
+
+                check_cfg.exhaustive_names = true;
+                for arg in args {
+                    if arg.is_word() && arg.ident().is_some() {
+                        let ident = arg.ident().expect("multi-segment cfg key");
+                        check_cfg
+                            .expecteds
+                            .entry(ident.name.to_string())
+                            .or_insert(ExpectedValues::Any);
+                    } else {
+                        error!("`names()` arguments must be simple identifiers");
+                    }
+                }
+            } else if meta_item.has_name(sym::values) {
+                // defaults are flipped for the old syntax
+                if old_syntax == None {
+                    check_cfg.exhaustive_names = false;
+                    check_cfg.exhaustive_values = false;
+                }
+                old_syntax = Some(true);
+
+                if let Some((name, values)) = args.split_first() {
+                    if name.is_word() && name.ident().is_some() {
+                        let ident = name.ident().expect("multi-segment cfg key");
+                        let expected_values = check_cfg
+                            .expecteds
+                            .entry(ident.name.to_string())
+                            .and_modify(|expected_values| match expected_values {
+                                ExpectedValues::Some(_) => {}
+                                ExpectedValues::Any => {
+                                    // handle the case where names(...) was done
+                                    // before values by changing to a list
+                                    *expected_values = ExpectedValues::Some(FxHashSet::default());
+                                }
+                            })
+                            .or_insert_with(|| ExpectedValues::Some(FxHashSet::default()));
+
+                        let ExpectedValues::Some(expected_values) = expected_values else {
+                            bug!("`expected_values` should be a list a values")
+                        };
+
+                        for val in values {
+                            if let Some(LitKind::Str(s, _)) = val.lit().map(|lit| &lit.kind) {
+                                expected_values.insert(Some(s.to_string()));
+                            } else {
+                                error!("`values()` arguments must be string literals");
+                            }
+                        }
+
+                        if values.is_empty() {
+                            expected_values.insert(None);
+                        }
+                    } else {
+                        error!("`values()` first argument must be a simple identifier");
+                    }
+                } else if args.is_empty() {
+                    check_cfg.exhaustive_values = true;
+                } else {
+                    expected_error();
+                }
+            } else if meta_item.has_name(sym::cfg) {
+                old_syntax = Some(false);
+
+                let mut names = Vec::new();
+                let mut values: FxHashSet<_> = Default::default();
+
+                let mut any_specified = false;
+                let mut values_specified = false;
+                let mut values_any_specified = false;
+
+                for arg in args {
+                    if arg.is_word() && let Some(ident) = arg.ident() {
+                        if values_specified {
+                            error!("`cfg()` names cannot be after values");
+                        }
+                        names.push(ident);
+                    } else if arg.has_name(sym::any)
+                        && let Some(args) = arg.meta_item_list()
+                    {
+                        if any_specified {
+                            error!("`any()` cannot be specified multiple times");
+                        }
+                        any_specified = true;
+                        if !args.is_empty() {
+                            error!("`any()` must be empty");
+                        }
+                    } else if arg.has_name(sym::values)
+                        && let Some(args) = arg.meta_item_list()
+                    {
+                        if names.is_empty() {
+                            error!("`values()` cannot be specified before the names");
+                        } else if values_specified {
+                            error!("`values()` cannot be specified multiple times");
+                        }
+                        values_specified = true;
+
+                        for arg in args {
+                            if let Some(LitKind::Str(s, _)) =
+                                arg.lit().map(|lit| &lit.kind)
+                            {
+                                values.insert(Some(s.to_string()));
+                            } else if arg.has_name(sym::any)
+                                && let Some(args) = arg.meta_item_list()
+                            {
+                                if values_any_specified {
+                                    error!(
+                                        "`any()` in `values()` cannot be specified multiple times"
+                                    );
+                                }
+                                values_any_specified = true;
+                                if !args.is_empty() {
+                                    error!("`any()` must be empty");
+                                }
+                            } else {
+                                error!(
+                                    "`values()` arguments must be string literals or `any()`"
+                                );
+                            }
+                        }
+                    } else {
+                        error!(
+                            "`cfg()` arguments must be simple identifiers, `any()` or `values(...)`"
+                        );
+                    }
+                }
+
+                if values.is_empty() && !values_any_specified && !any_specified {
+                    values.insert(None);
+                } else if !values.is_empty() && values_any_specified {
+                    error!(
+                        "`values()` arguments cannot specify string literals and `any()` at the same time"
+                    );
+                }
+
+                if any_specified {
+                    if !names.is_empty() || !values.is_empty() || values_any_specified {
+                        error!("`cfg(any())` can only be provided in isolation");
+                    }
+
+                    check_cfg.exhaustive_names = false;
+                } else {
+                    for name in names {
+                        check_cfg
+                            .expecteds
+                            .entry(name.to_string())
+                            .and_modify(|v| match v {
+                                ExpectedValues::Some(v) if !values_any_specified => {
+                                    v.extend(values.clone())
+                                }
+                                ExpectedValues::Some(_) => *v = ExpectedValues::Any,
+                                ExpectedValues::Any => {}
+                            })
+                            .or_insert_with(|| {
+                                if values_any_specified {
+                                    ExpectedValues::Any
+                                } else {
+                                    ExpectedValues::Some(values.clone())
+                                }
+                            });
+                    }
+                }
+            } else {
+                expected_error();
             }
         }
 

From 1ef96a9e06124ca3ce95f9ceb3be6f24d0e7154a Mon Sep 17 00:00:00 2001
From: Urgau <urgau@numericable.fr>
Date: Fri, 20 Oct 2023 19:34:45 +0200
Subject: [PATCH 126/297] Fix residual (never merged) check-cfg syntax in doc

---
 src/doc/unstable-book/src/compiler-flags/check-cfg.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/doc/unstable-book/src/compiler-flags/check-cfg.md b/src/doc/unstable-book/src/compiler-flags/check-cfg.md
index 7a3ef5e9e2be..0e15c79076fa 100644
--- a/src/doc/unstable-book/src/compiler-flags/check-cfg.md
+++ b/src/doc/unstable-book/src/compiler-flags/check-cfg.md
@@ -139,7 +139,7 @@ fn do_mumble_frotz() {}
 
 ```bash
 # This turns on checking for feature values, but not for condition names.
-rustc --check-cfg 'configure(feature, values("zapping", "lasers"))' \
+rustc --check-cfg 'cfg(feature, values("zapping", "lasers"))' \
       --check-cfg 'cfg(any())' \
       --cfg 'feature="zapping"' -Z unstable-options
 ```

From b7debe34e6fb2d329e21739c97ce3d7161303af5 Mon Sep 17 00:00:00 2001
From: David Tolnay <dtolnay@gmail.com>
Date: Thu, 26 Oct 2023 17:18:21 -0700
Subject: [PATCH 127/297] Parse rustc version at compile time

---
 compiler/rustc_attr/src/builtin.rs            | 41 ++++---------
 compiler/rustc_attr/src/lib.rs                |  2 +-
 compiler/rustc_macros/src/current_version.rs  | 59 +++++++++++++++++++
 compiler/rustc_macros/src/lib.rs              |  6 ++
 compiler/rustc_middle/src/middle/stability.rs | 27 ++++-----
 compiler/rustc_session/src/lib.rs             |  3 +
 compiler/rustc_session/src/version.rs         | 19 ++++++
 src/librustdoc/html/render/mod.rs             |  5 +-
 .../clippy_utils/src/qualify_min_const_fn.rs  | 23 +++-----
 9 files changed, 124 insertions(+), 61 deletions(-)
 create mode 100644 compiler/rustc_macros/src/current_version.rs
 create mode 100644 compiler/rustc_session/src/version.rs

diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs
index 363ba0d4a815..ff23e3da4380 100644
--- a/compiler/rustc_attr/src/builtin.rs
+++ b/compiler/rustc_attr/src/builtin.rs
@@ -10,10 +10,9 @@ use rustc_session::config::ExpectedValues;
 use rustc_session::lint::builtin::UNEXPECTED_CFGS;
 use rustc_session::lint::BuiltinLintDiagnostics;
 use rustc_session::parse::{feature_err, ParseSess};
-use rustc_session::Session;
+use rustc_session::{RustcVersion, Session};
 use rustc_span::hygiene::Transparency;
 use rustc_span::{symbol::sym, symbol::Symbol, Span};
-use std::fmt::{self, Display};
 use std::num::NonZeroU32;
 
 use crate::session_diagnostics::{self, IncorrectReprFormatGenericCause};
@@ -24,8 +23,6 @@ use crate::session_diagnostics::{self, IncorrectReprFormatGenericCause};
 /// For more, see [this pull request](https://github.com/rust-lang/rust/pull/100591).
 pub const VERSION_PLACEHOLDER: &str = "CURRENT_RUSTC_VERSION";
 
-pub const CURRENT_RUSTC_VERSION: &str = env!("CFG_RELEASE");
-
 pub fn is_builtin_attr(attr: &Attribute) -> bool {
     attr.is_doc_comment() || attr.ident().is_some_and(|ident| is_builtin_attr_name(ident.name))
 }
@@ -153,7 +150,7 @@ pub enum StabilityLevel {
 #[derive(Encodable, Decodable, PartialEq, Copy, Clone, Debug, Eq, Hash)]
 #[derive(HashStable_Generic)]
 pub enum Since {
-    Version(Version),
+    Version(RustcVersion),
     /// Stabilized in the upcoming version, whatever number that is.
     Current,
     /// Failed to parse a stabilization version.
@@ -382,7 +379,7 @@ fn parse_stability(sess: &Session, attr: &Attribute) -> Option<(Symbol, Stabilit
     let since = if let Some(since) = since {
         if since.as_str() == VERSION_PLACEHOLDER {
             Since::Current
-        } else if let Some(version) = parse_version(since.as_str(), false) {
+        } else if let Some(version) = parse_version(since) {
             Since::Version(version)
         } else {
             sess.emit_err(session_diagnostics::InvalidSince { span: attr.span });
@@ -567,31 +564,20 @@ fn gate_cfg(gated_cfg: &GatedCfg, cfg_span: Span, sess: &ParseSess, features: &F
     }
 }
 
-#[derive(Encodable, Decodable, Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
-#[derive(HashStable_Generic)]
-pub struct Version {
-    pub major: u16,
-    pub minor: u16,
-    pub patch: u16,
-}
-
-fn parse_version(s: &str, allow_appendix: bool) -> Option<Version> {
-    let mut components = s.split('-');
+/// Parse a rustc version number written inside string literal in an attribute,
+/// like appears in `since = "1.0.0"`. Suffixes like "-dev" and "-nightly" are
+/// not accepted in this position, unlike when parsing CFG_RELEASE.
+fn parse_version(s: Symbol) -> Option<RustcVersion> {
+    let mut components = s.as_str().split('-');
     let d = components.next()?;
-    if !allow_appendix && components.next().is_some() {
+    if components.next().is_some() {
         return None;
     }
     let mut digits = d.splitn(3, '.');
     let major = digits.next()?.parse().ok()?;
     let minor = digits.next()?.parse().ok()?;
     let patch = digits.next().unwrap_or("0").parse().ok()?;
-    Some(Version { major, minor, patch })
-}
-
-impl Display for Version {
-    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(formatter, "{}.{}.{}", self.major, self.minor, self.patch)
-    }
+    Some(RustcVersion { major, minor, patch })
 }
 
 /// Evaluate a cfg-like condition (with `any` and `all`), using `eval` to
@@ -623,17 +609,16 @@ pub fn eval_condition(
                     return false;
                 }
             };
-            let Some(min_version) = parse_version(min_version.as_str(), false) else {
+            let Some(min_version) = parse_version(*min_version) else {
                 sess.emit_warning(session_diagnostics::UnknownVersionLiteral { span: *span });
                 return false;
             };
-            let rustc_version = parse_version(CURRENT_RUSTC_VERSION, true).unwrap();
 
             // See https://github.com/rust-lang/rust/issues/64796#issuecomment-640851454 for details
             if sess.assume_incomplete_release {
-                rustc_version > min_version
+                RustcVersion::CURRENT > min_version
             } else {
-                rustc_version >= min_version
+                RustcVersion::CURRENT >= min_version
             }
         }
         ast::MetaItemKind::List(mis) => {
diff --git a/compiler/rustc_attr/src/lib.rs b/compiler/rustc_attr/src/lib.rs
index 53e3eaaab376..868c04122558 100644
--- a/compiler/rustc_attr/src/lib.rs
+++ b/compiler/rustc_attr/src/lib.rs
@@ -27,6 +27,6 @@ pub use StabilityLevel::*;
 
 pub use rustc_ast::attr::*;
 
-pub(crate) use rustc_ast::HashStableContext;
+pub(crate) use rustc_session::HashStableContext;
 
 fluent_messages! { "../messages.ftl" }
diff --git a/compiler/rustc_macros/src/current_version.rs b/compiler/rustc_macros/src/current_version.rs
new file mode 100644
index 000000000000..5e3b91c17bf6
--- /dev/null
+++ b/compiler/rustc_macros/src/current_version.rs
@@ -0,0 +1,59 @@
+use proc_macro::TokenStream;
+use proc_macro2::Span;
+use quote::quote;
+use syn::parse::{Parse, ParseStream};
+use syn::{parenthesized, parse_macro_input, LitStr, Token};
+
+pub struct Input {
+    variable: LitStr,
+}
+
+mod kw {
+    syn::custom_keyword!(env);
+}
+
+impl Parse for Input {
+    // Input syntax is `env!("CFG_RELEASE")` to facilitate grepping.
+    fn parse(input: ParseStream<'_>) -> syn::Result<Self> {
+        let paren;
+        input.parse::<kw::env>()?;
+        input.parse::<Token![!]>()?;
+        parenthesized!(paren in input);
+        let variable: LitStr = paren.parse()?;
+        Ok(Input { variable })
+    }
+}
+
+pub(crate) fn current_version(input: TokenStream) -> TokenStream {
+    let input = parse_macro_input!(input as Input);
+
+    TokenStream::from(match RustcVersion::parse_env_var(&input.variable) {
+        Ok(RustcVersion { major, minor, patch }) => quote!(
+            Self { major: #major, minor: #minor, patch: #patch }
+        ),
+        Err(err) => syn::Error::new(Span::call_site(), err).into_compile_error(),
+    })
+}
+
+struct RustcVersion {
+    major: u16,
+    minor: u16,
+    patch: u16,
+}
+
+impl RustcVersion {
+    fn parse_env_var(env_var: &LitStr) -> Result<Self, Box<dyn std::error::Error>> {
+        let value = proc_macro::tracked_env::var(env_var.value())?;
+        Self::parse_str(&value)
+            .ok_or_else(|| format!("failed to parse rustc version: {:?}", value).into())
+    }
+
+    fn parse_str(value: &str) -> Option<Self> {
+        // Ignore any suffixes such as "-dev" or "-nightly".
+        let mut components = value.split('-').next().unwrap().splitn(3, '.');
+        let major = components.next()?.parse().ok()?;
+        let minor = components.next()?.parse().ok()?;
+        let patch = components.next().unwrap_or("0").parse().ok()?;
+        Some(RustcVersion { major, minor, patch })
+    }
+}
diff --git a/compiler/rustc_macros/src/lib.rs b/compiler/rustc_macros/src/lib.rs
index 776ba8f9ca11..193dbd75fbd5 100644
--- a/compiler/rustc_macros/src/lib.rs
+++ b/compiler/rustc_macros/src/lib.rs
@@ -15,6 +15,7 @@ use synstructure::decl_derive;
 
 use proc_macro::TokenStream;
 
+mod current_version;
 mod diagnostics;
 mod hash_stable;
 mod lift;
@@ -25,6 +26,11 @@ mod symbols;
 mod type_foldable;
 mod type_visitable;
 
+#[proc_macro]
+pub fn current_rustc_version(input: TokenStream) -> TokenStream {
+    current_version::current_version(input)
+}
+
 #[proc_macro]
 pub fn rustc_queries(input: TokenStream) -> TokenStream {
     query::rustc_queries(input)
diff --git a/compiler/rustc_middle/src/middle/stability.rs b/compiler/rustc_middle/src/middle/stability.rs
index 84893b8e627a..20547696d5a7 100644
--- a/compiler/rustc_middle/src/middle/stability.rs
+++ b/compiler/rustc_middle/src/middle/stability.rs
@@ -16,7 +16,7 @@ use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_session::lint::builtin::{DEPRECATED, DEPRECATED_IN_FUTURE, SOFT_UNSTABLE};
 use rustc_session::lint::{BuiltinLintDiagnostics, Level, Lint, LintBuffer};
 use rustc_session::parse::feature_err_issue;
-use rustc_session::Session;
+use rustc_session::{RustcVersion, Session};
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::Span;
 use std::num::NonZeroU32;
@@ -129,11 +129,6 @@ pub fn deprecation_in_effect(depr: &Deprecation) -> bool {
     let is_since_rustc_version = depr.is_since_rustc_version;
     let since = depr.since.as_ref().map(Symbol::as_str);
 
-    fn parse_version(ver: &str) -> Vec<u32> {
-        // We ignore non-integer components of the version (e.g., "nightly").
-        ver.split(|c| c == '.' || c == '-').flat_map(|s| s.parse()).collect()
-    }
-
     if !is_since_rustc_version {
         // The `since` field doesn't have semantic purpose without `#![staged_api]`.
         return true;
@@ -144,16 +139,18 @@ pub fn deprecation_in_effect(depr: &Deprecation) -> bool {
             return false;
         }
 
-        if let Some(rustc) = option_env!("CFG_RELEASE") {
-            let since: Vec<u32> = parse_version(&since);
-            let rustc: Vec<u32> = parse_version(rustc);
-            // We simply treat invalid `since` attributes as relating to a previous
-            // Rust version, thus always displaying the warning.
-            if since.len() != 3 {
-                return true;
-            }
-            return since <= rustc;
+        // We ignore non-integer components of the version (e.g., "nightly").
+        let since: Vec<u16> =
+            since.split(|c| c == '.' || c == '-').flat_map(|s| s.parse()).collect();
+
+        // We simply treat invalid `since` attributes as relating to a previous
+        // Rust version, thus always displaying the warning.
+        if since.len() != 3 {
+            return true;
         }
+
+        let rustc = RustcVersion::CURRENT;
+        return since.as_slice() <= &[rustc.major, rustc.minor, rustc.patch];
     };
 
     // Assume deprecation is in effect if "since" field is missing
diff --git a/compiler/rustc_session/src/lib.rs b/compiler/rustc_session/src/lib.rs
index 7da0bcf01bfb..17ac3e991c58 100644
--- a/compiler/rustc_session/src/lib.rs
+++ b/compiler/rustc_session/src/lib.rs
@@ -43,6 +43,9 @@ pub mod output;
 
 pub use getopts;
 
+mod version;
+pub use version::RustcVersion;
+
 fluent_messages! { "../messages.ftl" }
 
 /// Requirements for a `StableHashingContext` to be used in this crate.
diff --git a/compiler/rustc_session/src/version.rs b/compiler/rustc_session/src/version.rs
new file mode 100644
index 000000000000..1ad8620bfba5
--- /dev/null
+++ b/compiler/rustc_session/src/version.rs
@@ -0,0 +1,19 @@
+use std::fmt::{self, Display};
+
+#[derive(Encodable, Decodable, Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(HashStable_Generic)]
+pub struct RustcVersion {
+    pub major: u16,
+    pub minor: u16,
+    pub patch: u16,
+}
+
+impl RustcVersion {
+    pub const CURRENT: Self = current_rustc_version!(env!("CFG_RELEASE"));
+}
+
+impl Display for RustcVersion {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(formatter, "{}.{}.{}", self.major, self.minor, self.patch)
+    }
+}
diff --git a/src/librustdoc/html/render/mod.rs b/src/librustdoc/html/render/mod.rs
index e01341acf438..88d01b7a5e57 100644
--- a/src/librustdoc/html/render/mod.rs
+++ b/src/librustdoc/html/render/mod.rs
@@ -48,13 +48,14 @@ use std::str;
 use std::string::ToString;
 
 use askama::Template;
-use rustc_attr::{ConstStability, Deprecation, Since, StabilityLevel, CURRENT_RUSTC_VERSION};
+use rustc_attr::{ConstStability, Deprecation, Since, StabilityLevel};
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_hir::def_id::{DefId, DefIdSet};
 use rustc_hir::Mutability;
 use rustc_middle::middle::stability;
 use rustc_middle::ty::{self, TyCtxt};
+use rustc_session::RustcVersion;
 use rustc_span::{
     symbol::{sym, Symbol},
     BytePos, FileName, RealFileName,
@@ -979,7 +980,7 @@ fn render_stability_since_raw_with_extra(
 fn since_to_string(since: &Since) -> Option<String> {
     match since {
         Since::Version(since) => Some(since.to_string()),
-        Since::Current => Some(CURRENT_RUSTC_VERSION.to_owned()),
+        Since::Current => Some(RustcVersion::CURRENT.to_string()),
         Since::Err => None,
     }
 }
diff --git a/src/tools/clippy/clippy_utils/src/qualify_min_const_fn.rs b/src/tools/clippy/clippy_utils/src/qualify_min_const_fn.rs
index 1e465ac91b76..31f7b87de635 100644
--- a/src/tools/clippy/clippy_utils/src/qualify_min_const_fn.rs
+++ b/src/tools/clippy/clippy_utils/src/qualify_min_const_fn.rs
@@ -5,7 +5,7 @@
 
 use crate::msrvs::Msrv;
 use hir::LangItem;
-use rustc_attr::{Since, CURRENT_RUSTC_VERSION};
+use rustc_attr::Since;
 use rustc_const_eval::transform::check_consts::ConstCx;
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
@@ -372,23 +372,16 @@ fn is_const_fn(tcx: TyCtxt<'_>, def_id: DefId, msrv: &Msrv) -> bool {
                 // as a part of an unimplemented MSRV check https://github.com/rust-lang/rust/issues/65262.
 
                 let const_stab_rust_version = match since {
-                    Since::Version(version) => RustcVersion::new(
-                        u32::from(version.major),
-                        u32::from(version.minor),
-                        u32::from(version.patch),
-                    ),
-                    Since::Current => {
-                        // HACK(nilstrieb): CURRENT_RUSTC_VERSION can return versions like 1.66.0-dev.
-                        // `rustc-semver` doesn't accept the `-dev` version number so we have to strip it off.
-                        let short_version = CURRENT_RUSTC_VERSION.split('-').next().unwrap();
-                        RustcVersion::parse(short_version).unwrap_or_else(|err| {
-                            panic!("`rustc_attr::StabilityLevel::Stable::since` is ill-formatted: `{CURRENT_RUSTC_VERSION}`, {err:?}")
-                        })
-                    },
+                    Since::Version(version) => version,
+                    Since::Current => rustc_session::RustcVersion::CURRENT,
                     Since::Err => return false,
                 };
 
-                msrv.meets(const_stab_rust_version)
+                msrv.meets(RustcVersion::new(
+                    u32::from(const_stab_rust_version.major),
+                    u32::from(const_stab_rust_version.minor),
+                    u32::from(const_stab_rust_version.patch),
+                ))
             } else {
                 // Unstable const fn with the feature enabled.
                 msrv.current().is_none()

From 0a82920b568d05f230ccdd0b84dc73229bfe650e Mon Sep 17 00:00:00 2001
From: David Tolnay <dtolnay@gmail.com>
Date: Thu, 26 Oct 2023 19:06:16 -0700
Subject: [PATCH 128/297] Declare rustc_target dependency on object/macho

---
 compiler/rustc_target/Cargo.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler/rustc_target/Cargo.toml b/compiler/rustc_target/Cargo.toml
index a91eb41b18ae..779d03cd094d 100644
--- a/compiler/rustc_target/Cargo.toml
+++ b/compiler/rustc_target/Cargo.toml
@@ -19,4 +19,4 @@ rustc_index = { path = "../rustc_index" }
 [dependencies.object]
 version = "0.32.0"
 default-features = false
-features = ["elf"]
+features = ["elf", "macho"]

From 893e7266376f8c7629a5b5130e95bc3063128f77 Mon Sep 17 00:00:00 2001
From: Havard Eidnes <he@NetBSD.org>
Date: Fri, 27 Oct 2023 07:24:31 +0000
Subject: [PATCH 129/297] i586_unknown_netbsd.rs: fix formatting.

This hopefully fixes the CI run after integration of this
target.
---
 compiler/rustc_target/src/spec/i586_unknown_netbsd.rs | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/compiler/rustc_target/src/spec/i586_unknown_netbsd.rs b/compiler/rustc_target/src/spec/i586_unknown_netbsd.rs
index 9b36a3c28d35..957cb38b7933 100644
--- a/compiler/rustc_target/src/spec/i586_unknown_netbsd.rs
+++ b/compiler/rustc_target/src/spec/i586_unknown_netbsd.rs
@@ -1,13 +1,10 @@
-use crate::spec::{Cc, Lld, LinkerFlavor, StackProbeType, Target, TargetOptions};
+use crate::spec::{Cc, LinkerFlavor, Lld, StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::netbsd_base::opts();
     base.cpu = "pentium".into();
     base.max_atomic_width = Some(64);
-    base.pre_link_args
-        .entry(LinkerFlavor::Gnu(Cc::Yes, Lld::No))
-        .or_default()
-        .push("-m32".into());
+    base.pre_link_args.entry(LinkerFlavor::Gnu(Cc::Yes, Lld::No)).or_default().push("-m32".into());
     base.stack_probes = StackProbeType::Call;
 
     Target {

From 0f04e2dd8f08951b3b9d0641febdcf3003f876aa Mon Sep 17 00:00:00 2001
From: Havard Eidnes <he@NetBSD.org>
Date: Fri, 27 Oct 2023 09:37:25 +0000
Subject: [PATCH 130/297] For i586/NetBSD: fix another formatting insistence.

---
 compiler/rustc_llvm/build.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs
index e4412132136b..fe13162cd4a3 100644
--- a/compiler/rustc_llvm/build.rs
+++ b/compiler/rustc_llvm/build.rs
@@ -261,9 +261,7 @@ fn main() {
         // On NetBSD/i386, gcc and g++ is built for i486 (to maximize backward compat)
         // However, LLVM insists on using 64-bit atomics.
         // This gives rise to a need to link rust itself with -latomic for these targets
-        if target.starts_with("i586")
-           || target.starts_with("i686")
-        {
+        if target.starts_with("i586") || target.starts_with("i686") {
             println!("cargo:rustc-link-lib=atomic");
         }
         println!("cargo:rustc-link-lib=z");

From 84a1a689ccc8eb6b8be48f3385bc9b0aeb5c71a6 Mon Sep 17 00:00:00 2001
From: Urgau <urgau@numericable.fr>
Date: Sat, 21 Oct 2023 19:11:24 +0200
Subject: [PATCH 131/297] Better guard against wrong input with check-cfg any()

---
 compiler/rustc_interface/src/interface.rs              | 10 +++++++---
 tests/ui/check-cfg/invalid-arguments.any_values.stderr |  2 ++
 tests/ui/check-cfg/invalid-arguments.rs                |  4 +++-
 .../ui/check-cfg/invalid-arguments.unterminated.stderr |  2 ++
 4 files changed, 14 insertions(+), 4 deletions(-)
 create mode 100644 tests/ui/check-cfg/invalid-arguments.any_values.stderr
 create mode 100644 tests/ui/check-cfg/invalid-arguments.unterminated.stderr

diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index 221889253821..422c2fc65ad6 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -311,11 +311,15 @@ pub fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -> Check
                 }
 
                 if any_specified {
-                    if !names.is_empty() || !values.is_empty() || values_any_specified {
+                    if names.is_empty()
+                        && values.is_empty()
+                        && !values_specified
+                        && !values_any_specified
+                    {
+                        check_cfg.exhaustive_names = false;
+                    } else {
                         error!("`cfg(any())` can only be provided in isolation");
                     }
-
-                    check_cfg.exhaustive_names = false;
                 } else {
                     for name in names {
                         check_cfg
diff --git a/tests/ui/check-cfg/invalid-arguments.any_values.stderr b/tests/ui/check-cfg/invalid-arguments.any_values.stderr
new file mode 100644
index 000000000000..f9a9c4a6e132
--- /dev/null
+++ b/tests/ui/check-cfg/invalid-arguments.any_values.stderr
@@ -0,0 +1,2 @@
+error: invalid `--check-cfg` argument: `cfg(any(),values())` (`values()` cannot be specified before the names)
+
diff --git a/tests/ui/check-cfg/invalid-arguments.rs b/tests/ui/check-cfg/invalid-arguments.rs
index 79bef89c9574..a56f48e0af93 100644
--- a/tests/ui/check-cfg/invalid-arguments.rs
+++ b/tests/ui/check-cfg/invalid-arguments.rs
@@ -6,7 +6,7 @@
 // revisions: multiple_values_any not_empty_any not_empty_values_any
 // revisions: values_any_missing_values values_any_before_ident ident_in_values_1
 // revisions: ident_in_values_2 unknown_meta_item_1 unknown_meta_item_2 unknown_meta_item_3
-// revisions: mixed_values_any mixed_any giberich
+// revisions: mixed_values_any mixed_any any_values giberich unterminated
 //
 // compile-flags: -Z unstable-options
 // [anything_else]compile-flags: --check-cfg=anything_else(...)
@@ -29,6 +29,8 @@
 // [unknown_meta_item_3]compile-flags: --check-cfg=cfg(foo,values(test()))
 // [mixed_values_any]compile-flags: --check-cfg=cfg(foo,values("bar",any()))
 // [mixed_any]compile-flags: --check-cfg=cfg(any(),values(any()))
+// [any_values]compile-flags: --check-cfg=cfg(any(),values())
 // [giberich]compile-flags: --check-cfg=cfg(...)
+// [unterminated]compile-flags: --check-cfg=cfg(
 
 fn main() {}
diff --git a/tests/ui/check-cfg/invalid-arguments.unterminated.stderr b/tests/ui/check-cfg/invalid-arguments.unterminated.stderr
new file mode 100644
index 000000000000..80161a6aa0fc
--- /dev/null
+++ b/tests/ui/check-cfg/invalid-arguments.unterminated.stderr
@@ -0,0 +1,2 @@
+error: invalid `--check-cfg` argument: `cfg(` (expected `cfg(name, values("value1", "value2", ... "valueN"))`)
+

From 160b1793b2e29cfb10d7c7a5932ccb7f991f02bd Mon Sep 17 00:00:00 2001
From: Georg Semmler <github@weiznich.de>
Date: Thu, 26 Oct 2023 08:20:23 +0200
Subject: [PATCH 132/297] Allows `#[diagnostic::on_unimplemented]` attributes
 to have multiple notes

This commit extends the `#[diagnostic::on_unimplemented]` (and
`#[rustc_on_unimplemented]`) attributes to allow multiple `note`
options. This enables emitting multiple notes for custom error messages.
For now I've opted to not change any of the existing usages of
`#[rustc_on_unimplemented]` and just updated the relevant compile tests.
---
 .../error_reporting/on_unimplemented.rs       | 32 +++++++------
 .../error_reporting/type_err_ctxt_ext.rs      | 14 +++---
 library/core/src/marker.rs                    | 15 +++++-
 ...ptions_and_continue_to_use_fallback.stderr |  1 +
 .../on_unimplemented/multiple_notes.rs        | 18 +++++++
 .../on_unimplemented/multiple_notes.stderr    | 47 +++++++++++++++++++
 .../fn-traits.stderr                          |  3 ++
 tests/ui/suggestions/path-display.stderr      |  2 +
 tests/ui/traits/new-solver/fn-trait.stderr    |  2 +
 9 files changed, 111 insertions(+), 23 deletions(-)
 create mode 100644 tests/ui/diagnostic_namespace/on_unimplemented/multiple_notes.rs
 create mode 100644 tests/ui/diagnostic_namespace/on_unimplemented/multiple_notes.stderr

diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs
index c96e41b88bd7..a5508c741346 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/on_unimplemented.rs
@@ -319,7 +319,7 @@ pub struct OnUnimplementedDirective {
     pub subcommands: Vec<OnUnimplementedDirective>,
     pub message: Option<OnUnimplementedFormatString>,
     pub label: Option<OnUnimplementedFormatString>,
-    pub note: Option<OnUnimplementedFormatString>,
+    pub notes: Vec<OnUnimplementedFormatString>,
     pub parent_label: Option<OnUnimplementedFormatString>,
     pub append_const_msg: Option<AppendConstMessage>,
 }
@@ -329,7 +329,7 @@ pub struct OnUnimplementedDirective {
 pub struct OnUnimplementedNote {
     pub message: Option<String>,
     pub label: Option<String>,
-    pub note: Option<String>,
+    pub notes: Vec<String>,
     pub parent_label: Option<String>,
     // If none, should fall back to a generic message
     pub append_const_msg: Option<AppendConstMessage>,
@@ -399,7 +399,7 @@ impl<'tcx> OnUnimplementedDirective {
 
         let mut message = None;
         let mut label = None;
-        let mut note = None;
+        let mut notes = Vec::new();
         let mut parent_label = None;
         let mut subcommands = vec![];
         let mut append_const_msg = None;
@@ -415,10 +415,12 @@ impl<'tcx> OnUnimplementedDirective {
                     label = parse_value(label_)?;
                     continue;
                 }
-            } else if item.has_name(sym::note) && note.is_none() {
+            } else if item.has_name(sym::note) {
                 if let Some(note_) = item.value_str() {
-                    note = parse_value(note_)?;
-                    continue;
+                    if let Some(note) = parse_value(note_)? {
+                        notes.push(note);
+                        continue;
+                    }
                 }
             } else if item.has_name(sym::parent_label)
                 && parent_label.is_none()
@@ -432,7 +434,7 @@ impl<'tcx> OnUnimplementedDirective {
                 && is_root
                 && message.is_none()
                 && label.is_none()
-                && note.is_none()
+                && notes.is_empty()
                 && !is_diagnostic_namespace_variant
             // FIXME(diagnostic_namespace): disallow filters for now
             {
@@ -487,7 +489,7 @@ impl<'tcx> OnUnimplementedDirective {
                 subcommands,
                 message,
                 label,
-                note,
+                notes,
                 parent_label,
                 append_const_msg,
             }))
@@ -505,12 +507,14 @@ impl<'tcx> OnUnimplementedDirective {
                     if let Some(aggr) = aggr {
                         let mut subcommands = aggr.subcommands;
                         subcommands.extend(directive.subcommands);
+                        let mut notes = aggr.notes;
+                        notes.extend(directive.notes);
                         Ok(Some(Self {
                             condition: aggr.condition.or(directive.condition),
                             subcommands,
                             message: aggr.message.or(directive.message),
                             label: aggr.label.or(directive.label),
-                            note: aggr.note.or(directive.note),
+                            notes,
                             parent_label: aggr.parent_label.or(directive.parent_label),
                             append_const_msg: aggr.append_const_msg.or(directive.append_const_msg),
                         }))
@@ -543,7 +547,7 @@ impl<'tcx> OnUnimplementedDirective {
                         value,
                         attr.span,
                     )?),
-                    note: None,
+                    notes: Vec::new(),
                     parent_label: None,
                     append_const_msg: None,
                 }))
@@ -600,7 +604,7 @@ impl<'tcx> OnUnimplementedDirective {
     ) -> OnUnimplementedNote {
         let mut message = None;
         let mut label = None;
-        let mut note = None;
+        let mut notes = Vec::new();
         let mut parent_label = None;
         let mut append_const_msg = None;
         info!("evaluate({:?}, trait_ref={:?}, options={:?})", self, trait_ref, options);
@@ -637,9 +641,7 @@ impl<'tcx> OnUnimplementedDirective {
                 label = Some(label_.clone());
             }
 
-            if let Some(ref note_) = command.note {
-                note = Some(note_.clone());
-            }
+            notes.extend(command.notes.clone());
 
             if let Some(ref parent_label_) = command.parent_label {
                 parent_label = Some(parent_label_.clone());
@@ -651,7 +653,7 @@ impl<'tcx> OnUnimplementedDirective {
         OnUnimplementedNote {
             label: label.map(|l| l.format(tcx, trait_ref, &options_map)),
             message: message.map(|m| m.format(tcx, trait_ref, &options_map)),
-            note: note.map(|n| n.format(tcx, trait_ref, &options_map)),
+            notes: notes.into_iter().map(|n| n.format(tcx, trait_ref, &options_map)).collect(),
             parent_label: parent_label.map(|e_s| e_s.format(tcx, trait_ref, &options_map)),
             append_const_msg,
         }
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
index 353a69ef6d32..ce7510cccb4f 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
@@ -428,7 +428,7 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         let OnUnimplementedNote {
                             message,
                             label,
-                            note,
+                            notes,
                             parent_label,
                             append_const_msg,
                         } = self.on_unimplemented_note(trait_ref, &obligation);
@@ -436,21 +436,21 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         let is_try_conversion = self.is_try_conversion(span, trait_ref.def_id());
                         let is_unsize =
                             Some(trait_ref.def_id()) == self.tcx.lang_items().unsize_trait();
-                        let (message, note, append_const_msg) = if is_try_conversion {
+                        let (message, notes, append_const_msg) = if is_try_conversion {
                             (
                                 Some(format!(
                                     "`?` couldn't convert the error to `{}`",
                                     trait_ref.skip_binder().self_ty(),
                                 )),
-                                Some(
+                                vec![
                                     "the question mark operation (`?`) implicitly performs a \
                                      conversion on the error value using the `From` trait"
                                         .to_owned(),
-                                ),
+                                ],
                                 Some(AppendConstMessage::Default),
                             )
                         } else {
-                            (message, note, append_const_msg)
+                            (message, notes, append_const_msg)
                         };
 
                         let err_msg = self.get_standard_error_message(
@@ -569,9 +569,9 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         if let Some((msg, span)) = type_def {
                             err.span_label(span, msg);
                         }
-                        if let Some(s) = note {
+                        for note in notes {
                             // If it has a custom `#[rustc_on_unimplemented]` note, let's display it
-                            err.note(s);
+                            err.note(note);
                         }
                         if let Some(s) = parent_label {
                             let body = obligation.cause.body_id;
diff --git a/library/core/src/marker.rs b/library/core/src/marker.rs
index f1594501d408..d396a707b55d 100644
--- a/library/core/src/marker.rs
+++ b/library/core/src/marker.rs
@@ -621,7 +621,20 @@ impl<T: ?Sized> Copy for &T {}
         note = "if you want to do aliasing and mutation between multiple threads, use `std::sync::RwLock` or `std::sync::atomic::AtomicBool` instead",
     ),
     on(
-        _Self = "core::cell::Cell<T>",
+        all(
+            _Self = "core::cell::Cell<T>",
+            not(_Self = "core::cell::Cell<u8>"),
+            not(_Self = "core::cell::Cell<u16>"),
+            not(_Self = "core::cell::Cell<u32>"),
+            not(_Self = "core::cell::Cell<u64>"),
+            not(_Self = "core::cell::Cell<usize>"),
+            not(_Self = "core::cell::Cell<i8>"),
+            not(_Self = "core::cell::Cell<i16>"),
+            not(_Self = "core::cell::Cell<i32>"),
+            not(_Self = "core::cell::Cell<i64>"),
+            not(_Self = "core::cell::Cell<isize>"),
+            not(_Self = "core::cell::Cell<bool>")
+        ),
         note = "if you want to do aliasing and mutation between multiple threads, use `std::sync::RwLock`",
     ),
     on(
diff --git a/tests/ui/diagnostic_namespace/on_unimplemented/ignore_unsupported_options_and_continue_to_use_fallback.stderr b/tests/ui/diagnostic_namespace/on_unimplemented/ignore_unsupported_options_and_continue_to_use_fallback.stderr
index 7860e540589d..906472beb49d 100644
--- a/tests/ui/diagnostic_namespace/on_unimplemented/ignore_unsupported_options_and_continue_to_use_fallback.stderr
+++ b/tests/ui/diagnostic_namespace/on_unimplemented/ignore_unsupported_options_and_continue_to_use_fallback.stderr
@@ -26,6 +26,7 @@ LL |     takes_foo(());
    |
    = help: the trait `Foo` is not implemented for `()`
    = note: custom note
+   = note: fallback note
 help: this trait has no implementations, consider adding one
   --> $DIR/ignore_unsupported_options_and_continue_to_use_fallback.rs:13:1
    |
diff --git a/tests/ui/diagnostic_namespace/on_unimplemented/multiple_notes.rs b/tests/ui/diagnostic_namespace/on_unimplemented/multiple_notes.rs
new file mode 100644
index 000000000000..34cdb99c754b
--- /dev/null
+++ b/tests/ui/diagnostic_namespace/on_unimplemented/multiple_notes.rs
@@ -0,0 +1,18 @@
+#![feature(diagnostic_namespace)]
+
+#[diagnostic::on_unimplemented(message = "Foo", label = "Bar", note = "Baz", note = "Boom")]
+trait Foo {}
+
+#[diagnostic::on_unimplemented(message = "Bar", label = "Foo", note = "Baz")]
+#[diagnostic::on_unimplemented(note = "Baz2")]
+trait Bar {}
+
+fn takes_foo(_: impl Foo) {}
+fn takes_bar(_: impl Bar) {}
+
+fn main() {
+    takes_foo(());
+    //~^ERROR Foo
+    takes_bar(());
+    //~^ERROR Bar
+}
diff --git a/tests/ui/diagnostic_namespace/on_unimplemented/multiple_notes.stderr b/tests/ui/diagnostic_namespace/on_unimplemented/multiple_notes.stderr
new file mode 100644
index 000000000000..c72321d4617f
--- /dev/null
+++ b/tests/ui/diagnostic_namespace/on_unimplemented/multiple_notes.stderr
@@ -0,0 +1,47 @@
+error[E0277]: Foo
+  --> $DIR/multiple_notes.rs:14:15
+   |
+LL |     takes_foo(());
+   |     --------- ^^ Bar
+   |     |
+   |     required by a bound introduced by this call
+   |
+   = help: the trait `Foo` is not implemented for `()`
+   = note: Baz
+   = note: Boom
+help: this trait has no implementations, consider adding one
+  --> $DIR/multiple_notes.rs:4:1
+   |
+LL | trait Foo {}
+   | ^^^^^^^^^
+note: required by a bound in `takes_foo`
+  --> $DIR/multiple_notes.rs:10:22
+   |
+LL | fn takes_foo(_: impl Foo) {}
+   |                      ^^^ required by this bound in `takes_foo`
+
+error[E0277]: Bar
+  --> $DIR/multiple_notes.rs:16:15
+   |
+LL |     takes_bar(());
+   |     --------- ^^ Foo
+   |     |
+   |     required by a bound introduced by this call
+   |
+   = help: the trait `Bar` is not implemented for `()`
+   = note: Baz
+   = note: Baz2
+help: this trait has no implementations, consider adding one
+  --> $DIR/multiple_notes.rs:8:1
+   |
+LL | trait Bar {}
+   | ^^^^^^^^^
+note: required by a bound in `takes_bar`
+  --> $DIR/multiple_notes.rs:11:22
+   |
+LL | fn takes_bar(_: impl Bar) {}
+   |                      ^^^ required by this bound in `takes_bar`
+
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0277`.
diff --git a/tests/ui/rfcs/rfc-2396-target_feature-11/fn-traits.stderr b/tests/ui/rfcs/rfc-2396-target_feature-11/fn-traits.stderr
index fc7bf22775dd..4fb0d43d1b79 100644
--- a/tests/ui/rfcs/rfc-2396-target_feature-11/fn-traits.stderr
+++ b/tests/ui/rfcs/rfc-2396-target_feature-11/fn-traits.stderr
@@ -58,6 +58,7 @@ LL |     call(foo_unsafe);
    |     required by a bound introduced by this call
    |
    = help: the trait `Fn<()>` is not implemented for fn item `unsafe fn() {foo_unsafe}`
+   = note: unsafe function cannot be called generically without an unsafe block
    = note: wrap the `unsafe fn() {foo_unsafe}` in a closure with no arguments: `|| { /* code */ }`
    = note: `#[target_feature]` functions do not implement the `Fn` traits
 note: required by a bound in `call`
@@ -75,6 +76,7 @@ LL |     call_mut(foo_unsafe);
    |     required by a bound introduced by this call
    |
    = help: the trait `FnMut<()>` is not implemented for fn item `unsafe fn() {foo_unsafe}`
+   = note: unsafe function cannot be called generically without an unsafe block
    = note: wrap the `unsafe fn() {foo_unsafe}` in a closure with no arguments: `|| { /* code */ }`
    = note: `#[target_feature]` functions do not implement the `Fn` traits
 note: required by a bound in `call_mut`
@@ -92,6 +94,7 @@ LL |     call_once(foo_unsafe);
    |     required by a bound introduced by this call
    |
    = help: the trait `FnOnce<()>` is not implemented for fn item `unsafe fn() {foo_unsafe}`
+   = note: unsafe function cannot be called generically without an unsafe block
    = note: wrap the `unsafe fn() {foo_unsafe}` in a closure with no arguments: `|| { /* code */ }`
    = note: `#[target_feature]` functions do not implement the `Fn` traits
 note: required by a bound in `call_once`
diff --git a/tests/ui/suggestions/path-display.stderr b/tests/ui/suggestions/path-display.stderr
index 8359b36588e6..46d0b35825bc 100644
--- a/tests/ui/suggestions/path-display.stderr
+++ b/tests/ui/suggestions/path-display.stderr
@@ -5,6 +5,7 @@ LL |     println!("{}", path);
    |                    ^^^^ `Path` cannot be formatted with the default formatter; call `.display()` on it
    |
    = help: the trait `std::fmt::Display` is not implemented for `Path`
+   = note: in format strings you may be able to use `{:?}` (or {:#?} for pretty-print) instead
    = note: call `.display()` or `.to_string_lossy()` to safely print paths, as they may contain non-Unicode data
    = note: this error originates in the macro `$crate::format_args_nl` which comes from the expansion of the macro `println` (in Nightly builds, run with -Z macro-backtrace for more info)
 
@@ -15,6 +16,7 @@ LL |     println!("{}", path);
    |                    ^^^^ `PathBuf` cannot be formatted with the default formatter; call `.display()` on it
    |
    = help: the trait `std::fmt::Display` is not implemented for `PathBuf`
+   = note: in format strings you may be able to use `{:?}` (or {:#?} for pretty-print) instead
    = note: call `.display()` or `.to_string_lossy()` to safely print paths, as they may contain non-Unicode data
    = note: this error originates in the macro `$crate::format_args_nl` which comes from the expansion of the macro `println` (in Nightly builds, run with -Z macro-backtrace for more info)
 
diff --git a/tests/ui/traits/new-solver/fn-trait.stderr b/tests/ui/traits/new-solver/fn-trait.stderr
index d52bcaf25b87..ff6903c5dbf2 100644
--- a/tests/ui/traits/new-solver/fn-trait.stderr
+++ b/tests/ui/traits/new-solver/fn-trait.stderr
@@ -7,6 +7,7 @@ LL |     require_fn(f as unsafe fn() -> i32);
    |     required by a bound introduced by this call
    |
    = help: the trait `Fn<()>` is not implemented for `unsafe fn() -> i32`
+   = note: unsafe function cannot be called generically without an unsafe block
    = note: wrap the `unsafe fn() -> i32` in a closure with no arguments: `|| { /* code */ }`
 note: required by a bound in `require_fn`
   --> $DIR/fn-trait.rs:3:23
@@ -97,6 +98,7 @@ LL |     require_fn(h);
    |     required by a bound introduced by this call
    |
    = help: the trait `Fn<()>` is not implemented for fn item `unsafe fn() -> i32 {h}`
+   = note: unsafe function cannot be called generically without an unsafe block
    = note: wrap the `unsafe fn() -> i32 {h}` in a closure with no arguments: `|| { /* code */ }`
 note: required by a bound in `require_fn`
   --> $DIR/fn-trait.rs:3:23

From a58327dc014dabac8d69b9602011fee75d201815 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Tue, 2 Feb 2021 19:45:17 +0100
Subject: [PATCH 133/297] Distribute cg_clif as a rustup component

---
 src/bootstrap/src/core/build_steps/dist.rs    | 109 +++++++++++++++---
 src/bootstrap/src/core/build_steps/install.rs |  14 +++
 src/bootstrap/src/core/build_steps/test.rs    |  16 +--
 src/bootstrap/src/core/builder.rs             |   1 +
 src/bootstrap/src/utils/helpers.rs            |  13 +++
 src/bootstrap/src/utils/tarball.rs            |   7 ++
 src/ci/run.sh                                 |   3 +-
 src/tools/build-manifest/src/main.rs          |  16 ++-
 src/tools/build-manifest/src/versions.rs      |   4 +
 9 files changed, 152 insertions(+), 31 deletions(-)

diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs
index 47f41ab288d5..acab3cffe077 100644
--- a/src/bootstrap/src/core/build_steps/dist.rs
+++ b/src/bootstrap/src/core/build_steps/dist.rs
@@ -27,7 +27,7 @@ use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step};
 use crate::core::config::TargetSelection;
 use crate::utils::cache::{Interned, INTERNER};
 use crate::utils::channel;
-use crate::utils::helpers::{exe, is_dylib, output, t, timeit};
+use crate::utils::helpers::{exe, is_dylib, output, t, target_supports_cranelift_backend, timeit};
 use crate::utils::tarball::{GeneratedTarball, OverlayKind, Tarball};
 use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS};
 
@@ -443,19 +443,6 @@ impl Step for Rustc {
                 }
             }
 
-            // Copy over the codegen backends
-            let backends_src = builder.sysroot_codegen_backends(compiler);
-            let backends_rel = backends_src
-                .strip_prefix(&src)
-                .unwrap()
-                .strip_prefix(builder.sysroot_libdir_relative(compiler))
-                .unwrap();
-            // Don't use custom libdir here because ^lib/ will be resolved again with installer
-            let backends_dst = image.join("lib").join(&backends_rel);
-
-            t!(fs::create_dir_all(&backends_dst));
-            builder.cp_r(&backends_src, &backends_dst);
-
             // Copy libLLVM.so to the lib dir as well, if needed. While not
             // technically needed by rustc itself it's needed by lots of other
             // components like the llvm tools and LLD. LLD is included below and
@@ -1282,6 +1269,91 @@ impl Step for Miri {
     }
 }
 
+#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct CodegenBackend {
+    pub compiler: Compiler,
+    pub backend: Interned<String>,
+}
+
+impl Step for CodegenBackend {
+    type Output = Option<GeneratedTarball>;
+    const DEFAULT: bool = true;
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("compiler/rustc_codegen_cranelift")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        for &backend in &run.builder.config.rust_codegen_backends {
+            if backend == "llvm" {
+                continue; // Already built as part of rustc
+            }
+
+            run.builder.ensure(CodegenBackend {
+                compiler: run.builder.compiler(run.builder.top_stage, run.target),
+                backend,
+            });
+        }
+    }
+
+    fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+        // This prevents rustc_codegen_cranelift from being built for "dist"
+        // or "install" on the stable/beta channels. It is not yet stable and
+        // should not be included.
+        if !builder.build.unstable_features() {
+            return None;
+        }
+
+        if self.backend == "cranelift" {
+            if !target_supports_cranelift_backend(self.compiler.host) {
+                builder.info("target not supported by rustc_codegen_cranelift. skipping");
+                return None;
+            }
+
+            if self.compiler.host.contains("windows") {
+                builder.info(
+                    "dist currently disabled for windows by rustc_codegen_cranelift. skipping",
+                );
+                return None;
+            }
+        }
+
+        let compiler = self.compiler;
+        let backend = self.backend;
+
+        let mut tarball =
+            Tarball::new(builder, &format!("rustc-codegen-{}", backend), &compiler.host.triple);
+        if backend == "cranelift" {
+            tarball.set_overlay(OverlayKind::RustcCodegenCranelift);
+        } else {
+            panic!("Unknown backend rustc_codegen_{}", backend);
+        }
+        tarball.is_preview(true);
+        tarball.add_legal_and_readme_to(format!("share/doc/rustc_codegen_{}", backend));
+
+        let src = builder.sysroot(compiler);
+        let backends_src = builder.sysroot_codegen_backends(compiler);
+        let backends_rel = backends_src
+            .strip_prefix(&src)
+            .unwrap()
+            .strip_prefix(builder.sysroot_libdir_relative(compiler))
+            .unwrap();
+        // Don't use custom libdir here because ^lib/ will be resolved again with installer
+        let backends_dst = PathBuf::from("lib").join(&backends_rel);
+
+        let backend_name = format!("rustc_codegen_{}", backend);
+        for backend in fs::read_dir(&backends_src).unwrap() {
+            let file_name = backend.unwrap().file_name();
+            if file_name.to_str().unwrap().contains(&backend_name) {
+                tarball.add_file(backends_src.join(file_name), &backends_dst, 0o644);
+            }
+        }
+
+        Some(tarball.generate())
+    }
+}
+
 #[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)]
 pub struct Rustfmt {
     pub compiler: Compiler,
@@ -1452,6 +1524,10 @@ impl Step for Extended {
         add_component!("clippy" => Clippy { compiler, target });
         add_component!("miri" => Miri { compiler, target });
         add_component!("analysis" => Analysis { compiler, target });
+        add_component!("rustc-codegen-cranelift" => CodegenBackend {
+            compiler: builder.compiler(stage, target),
+            backend: INTERNER.intern_str("cranelift"),
+        });
 
         let etc = builder.src.join("src/etc/installer");
 
@@ -1548,6 +1624,7 @@ impl Step for Extended {
                     prepare(tool);
                 }
             }
+            prepare("rustc-codegen-cranelift");
             // create an 'uninstall' package
             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
             pkgbuild("uninstall");
@@ -1587,6 +1664,10 @@ impl Step for Extended {
                     "rust-demangler-preview".to_string()
                 } else if name == "miri" {
                     "miri-preview".to_string()
+                } else if name == "rustc-codegen-cranelift" {
+                    // FIXME add installer support for cg_clif once it is ready to be distributed on
+                    // windows.
+                    unreachable!("cg_clif shouldn't be built for windows");
                 } else {
                     name.to_string()
                 };
diff --git a/src/bootstrap/src/core/build_steps/install.rs b/src/bootstrap/src/core/build_steps/install.rs
index 391995b7c3b8..75d263d7794f 100644
--- a/src/bootstrap/src/core/build_steps/install.rs
+++ b/src/bootstrap/src/core/build_steps/install.rs
@@ -13,6 +13,7 @@ use crate::core::builder::{Builder, RunConfig, ShouldRun, Step};
 use crate::core::config::{Config, TargetSelection};
 use crate::utils::helpers::t;
 use crate::utils::tarball::GeneratedTarball;
+use crate::INTERNER;
 use crate::{Compiler, Kind};
 
 #[cfg(target_os = "illumos")]
@@ -281,6 +282,19 @@ install!((self, builder, _config),
         });
         install_sh(builder, "rustc", self.compiler.stage, Some(self.target), &tarball);
     };
+    RustcCodegenCranelift, alias = "rustc-codegen-cranelift", Self::should_build(_config), only_hosts: true, {
+        if let Some(tarball) = builder.ensure(dist::CodegenBackend {
+            compiler: self.compiler,
+            backend: INTERNER.intern_str("cranelift"),
+        }) {
+            install_sh(builder, "rustc-codegen-cranelift", self.compiler.stage, Some(self.target), &tarball);
+        } else {
+            builder.info(
+                &format!("skipping Install CodegenBackend(\"cranelift\") stage{} ({})",
+                         self.compiler.stage, self.target),
+            );
+        }
+    };
 );
 
 #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs
index 27eac212b143..9d2eeaee03e5 100644
--- a/src/bootstrap/src/core/build_steps/test.rs
+++ b/src/bootstrap/src/core/build_steps/test.rs
@@ -29,7 +29,8 @@ use crate::utils;
 use crate::utils::cache::{Interned, INTERNER};
 use crate::utils::exec::BootstrapCommand;
 use crate::utils::helpers::{
-    self, add_link_lib_path, dylib_path, dylib_path_var, output, t, up_to_date,
+    self, add_link_lib_path, dylib_path, dylib_path_var, output, t,
+    target_supports_cranelift_backend, up_to_date,
 };
 use crate::utils::render_tests::{add_flags_and_try_run_tests, try_run_tests};
 use crate::{envify, CLang, DocTests, GitRepo, Mode};
@@ -2998,18 +2999,7 @@ impl Step for CodegenCranelift {
             return;
         }
 
-        let triple = run.target.triple;
-        let target_supported = if triple.contains("linux") {
-            triple.contains("x86_64")
-                || triple.contains("aarch64")
-                || triple.contains("s390x")
-                || triple.contains("riscv64gc")
-        } else if triple.contains("darwin") || triple.contains("windows") {
-            triple.contains("x86_64")
-        } else {
-            false
-        };
-        if !target_supported {
+        if !target_supports_cranelift_backend(run.target) {
             builder.info("target not supported by rustc_codegen_cranelift. skipping");
             return;
         }
diff --git a/src/bootstrap/src/core/builder.rs b/src/bootstrap/src/core/builder.rs
index 36653dcfb206..90e09d12a9d5 100644
--- a/src/bootstrap/src/core/builder.rs
+++ b/src/bootstrap/src/core/builder.rs
@@ -815,6 +815,7 @@ impl<'a> Builder<'a> {
                 dist::JsonDocs,
                 dist::Mingw,
                 dist::Rustc,
+                dist::CodegenBackend,
                 dist::Std,
                 dist::RustcDev,
                 dist::Analysis,
diff --git a/src/bootstrap/src/utils/helpers.rs b/src/bootstrap/src/utils/helpers.rs
index b58a1c258425..82a5607d9037 100644
--- a/src/bootstrap/src/utils/helpers.rs
+++ b/src/bootstrap/src/utils/helpers.rs
@@ -183,6 +183,19 @@ pub fn use_host_linker(target: TargetSelection) -> bool {
         || target.contains("switch"))
 }
 
+pub fn target_supports_cranelift_backend(target: TargetSelection) -> bool {
+    if target.contains("linux") {
+        target.contains("x86_64")
+            || target.contains("aarch64")
+            || target.contains("s390x")
+            || target.contains("riscv64gc")
+    } else if target.contains("darwin") || target.contains("windows") {
+        target.contains("x86_64")
+    } else {
+        false
+    }
+}
+
 pub fn is_valid_test_suite_arg<'a, P: AsRef<Path>>(
     path: &'a Path,
     suite_path: P,
diff --git a/src/bootstrap/src/utils/tarball.rs b/src/bootstrap/src/utils/tarball.rs
index b437456f8a10..a8393f88f8a1 100644
--- a/src/bootstrap/src/utils/tarball.rs
+++ b/src/bootstrap/src/utils/tarball.rs
@@ -19,6 +19,7 @@ pub(crate) enum OverlayKind {
     RustDemangler,
     RLS,
     RustAnalyzer,
+    RustcCodegenCranelift,
 }
 
 impl OverlayKind {
@@ -58,6 +59,11 @@ impl OverlayKind {
                 "src/tools/rust-analyzer/LICENSE-APACHE",
                 "src/tools/rust-analyzer/LICENSE-MIT",
             ],
+            OverlayKind::RustcCodegenCranelift => &[
+                "compiler/rustc_codegen_cranelift/Readme.md",
+                "compiler/rustc_codegen_cranelift/LICENSE-APACHE",
+                "compiler/rustc_codegen_cranelift/LICENSE-MIT",
+            ],
         }
     }
 
@@ -80,6 +86,7 @@ impl OverlayKind {
             OverlayKind::RustAnalyzer => builder
                 .rust_analyzer_info
                 .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")),
+            OverlayKind::RustcCodegenCranelift => builder.rust_version(),
         }
     }
 }
diff --git a/src/ci/run.sh b/src/ci/run.sh
index abeff7b837dc..17ce7a807b63 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -104,6 +104,8 @@ if [ "$DEPLOY$DEPLOY_ALT" = "1" ]; then
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-assertions"
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
   fi
+
+  RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift"
 else
   # We almost always want debug assertions enabled, but sometimes this takes too
   # long for too little benefit, so we just turn them off.
@@ -124,7 +126,6 @@ else
 
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
 
-  # Test the Cranelift backend in on CI, but don't ship it.
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift"
 
   # We enable this for non-dist builders, since those aren't trying to produce
diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs
index b768722acf89..2c795ebb2148 100644
--- a/src/tools/build-manifest/src/main.rs
+++ b/src/tools/build-manifest/src/main.rs
@@ -192,7 +192,8 @@ static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin"
 
 static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"];
 
-static NIGHTLY_ONLY_COMPONENTS: &[PkgType] = &[PkgType::Miri, PkgType::JsonDocs];
+static NIGHTLY_ONLY_COMPONENTS: &[PkgType] =
+    &[PkgType::Miri, PkgType::JsonDocs, PkgType::RustcCodegenCranelift];
 
 macro_rules! t {
     ($e:expr) => {
@@ -336,7 +337,15 @@ impl Builder {
 
         // NOTE: this profile is effectively deprecated; do not add new components to it.
         let mut complete = default;
-        complete.extend([Rls, RustAnalyzer, RustSrc, LlvmTools, RustAnalysis, Miri]);
+        complete.extend([
+            Rls,
+            RustAnalyzer,
+            RustSrc,
+            LlvmTools,
+            RustAnalysis,
+            Miri,
+            RustcCodegenCranelift,
+        ]);
         profile("complete", &complete);
 
         // The compiler libraries are not stable for end users, and they're also huge, so we only
@@ -423,7 +432,8 @@ impl Builder {
                 | PkgType::Rustfmt
                 | PkgType::LlvmTools
                 | PkgType::RustAnalysis
-                | PkgType::JsonDocs => {
+                | PkgType::JsonDocs
+                | PkgType::RustcCodegenCranelift => {
                     extensions.push(host_component(pkg));
                 }
                 PkgType::RustcDev | PkgType::RustcDocs => {
diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs
index 7a4c15d01ead..57f94bdd90a1 100644
--- a/src/tools/build-manifest/src/versions.rs
+++ b/src/tools/build-manifest/src/versions.rs
@@ -57,6 +57,7 @@ pkg_type! {
     LlvmTools = "llvm-tools"; preview = true,
     Miri = "miri"; preview = true,
     JsonDocs = "rust-docs-json"; preview = true,
+    RustcCodegenCranelift = "rustc-codegen-cranelift"; preview = true,
 }
 
 impl PkgType {
@@ -80,6 +81,7 @@ impl PkgType {
             PkgType::Rustfmt => false,
             PkgType::LlvmTools => false,
             PkgType::Miri => false,
+            PkgType::RustcCodegenCranelift => false,
 
             PkgType::Rust => true,
             PkgType::RustStd => true,
@@ -106,6 +108,8 @@ impl PkgType {
             ReproducibleArtifacts => HOSTS,
             RustcDocs => HOSTS,
             Cargo => HOSTS,
+            // FIXME should this use the exact list of targets for which we build cg_clif?
+            RustcCodegenCranelift => HOSTS,
             RustMingw => MINGW,
             RustStd => TARGETS,
             HtmlDocs => HOSTS,

From add99438c81bb6cba8b7052357e0bffffc7a0a35 Mon Sep 17 00:00:00 2001
From: bjorn3 <17426603+bjorn3@users.noreply.github.com>
Date: Sun, 22 Oct 2023 19:25:16 +0000
Subject: [PATCH 134/297] Fix review comments

---
 src/ci/run.sh                            | 5 ++---
 src/tools/build-manifest/src/versions.rs | 1 -
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/ci/run.sh b/src/ci/run.sh
index 17ce7a807b63..bcfe1fc04104 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -84,6 +84,8 @@ if [ "$DIST_SRC" = "" ]; then
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-dist-src"
 fi
 
+RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift"
+
 # Always set the release channel for bootstrap; this is normally not important (i.e., only dist
 # builds would seem to matter) but in practice bootstrap wants to know whether we're targeting
 # master, beta, or stable with a build to determine whether to run some checks (notably toolstate).
@@ -105,7 +107,6 @@ if [ "$DEPLOY$DEPLOY_ALT" = "1" ]; then
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
   fi
 
-  RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift"
 else
   # We almost always want debug assertions enabled, but sometimes this takes too
   # long for too little benefit, so we just turn them off.
@@ -126,8 +127,6 @@ else
 
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
 
-  RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift"
-
   # We enable this for non-dist builders, since those aren't trying to produce
   # fresh binaries. We currently don't entirely support distributing a fresh
   # copy of the compiler (including llvm tools, etc.) if we haven't actually
diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs
index 57f94bdd90a1..e4cdf965eb60 100644
--- a/src/tools/build-manifest/src/versions.rs
+++ b/src/tools/build-manifest/src/versions.rs
@@ -108,7 +108,6 @@ impl PkgType {
             ReproducibleArtifacts => HOSTS,
             RustcDocs => HOSTS,
             Cargo => HOSTS,
-            // FIXME should this use the exact list of targets for which we build cg_clif?
             RustcCodegenCranelift => HOSTS,
             RustMingw => MINGW,
             RustStd => TARGETS,

From 344752ab53c311b5fc8505dba4acff8c91beb4cc Mon Sep 17 00:00:00 2001
From: bjorn3 <17426603+bjorn3@users.noreply.github.com>
Date: Wed, 25 Oct 2023 17:19:49 +0000
Subject: [PATCH 135/297] Update Cranelift to 0.101.2 and disable host-arch
 feature of cranelift-codegen

This ensures that cg_clif can be built for targets that aren't natively
supported by Cranelift. It will not be possible to compile for the host
in this case, but cross-compilation will still be possible.

We won't distribute cg_clif as rustup component for any targets that
aren't natively supported by Cranelift, but will still build it if
codegen-backends lists "cranelift".
---
 compiler/rustc_codegen_cranelift/Cargo.lock | 52 ++++++++++-----------
 compiler/rustc_codegen_cranelift/Cargo.toml | 12 ++---
 2 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/compiler/rustc_codegen_cranelift/Cargo.lock b/compiler/rustc_codegen_cranelift/Cargo.lock
index c716d5011731..e8558198b94f 100644
--- a/compiler/rustc_codegen_cranelift/Cargo.lock
+++ b/compiler/rustc_codegen_cranelift/Cargo.lock
@@ -45,18 +45,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1512c3bb6b13018e7109fc3ac964bc87b329eaf3a77825d337558d0c7f6f1be"
+checksum = "f773437307980ac0f424bf9b9a5d0cd21a0f17248c6860c9a65bec8b5975f3fe"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16cb8fb9220a6ea7a226705a273ab905309ee546267bdf34948d57932d7f0396"
+checksum = "443c2ac50e97fb7de1a0f862753fce3f27215558811a6fcee508eb0c3747fa79"
 dependencies = [
  "bumpalo",
  "cranelift-bforest",
@@ -75,39 +75,39 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab3a8d3b0d4745b183da5ea0792b13d79f5c23d6e69ac04761728e2532b56649"
+checksum = "c5b174c411480c79ce0793c55042fa51bec27e486381d103a53cab3b480cb2db"
 dependencies = [
  "cranelift-codegen-shared",
 ]
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "524141c8e68f2abc2043de4c2b31f6d9dd42432738c246431d0572a1422a4a84"
+checksum = "73fa0151a528066a369de6debeea4d4b23a32aba68b5add8c46d3dc8091ff434"
 
 [[package]]
 name = "cranelift-control"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97513b57c961c713789a03886a57b43e14ebcd204cbaa8ae50ca6c70a8e716b3"
+checksum = "b8adf1e6398493c9bea1190e37d28a0eb0eca5fddbc80e01e506cda34db92b1f"
 dependencies = [
  "arbitrary",
 ]
 
 [[package]]
 name = "cranelift-entity"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3f23d3cf3afa7e45f239702612c76d87964f652a55e28d13ed6d7e20f3479dd"
+checksum = "4917e2ed3bb5fe87d0ed88395ca6d644018d119a034faedd1f3e1f2c33cd52b2"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "554cd4947ec9209b58bf9ae5bf83581b5ddf9128bd967208e334b504a57db54e"
+checksum = "9aaadf1e7cf28886bbf046eaf7ef538997bc8a7e020e578ea4957b39da87d5a1"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -117,15 +117,15 @@ dependencies = [
 
 [[package]]
 name = "cranelift-isle"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c1892a439696b6413cb54083806f5fd9fc431768b8de74864b3d9e8b93b124f"
+checksum = "a67fda31b9d69eaa1c49a2081939454c45857596a9d45af6744680541c628b4c"
 
 [[package]]
 name = "cranelift-jit"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32209252fb38acaf1662ccd0397907bbe0e92bdb13b6ddbfd2f74e437f83e685"
+checksum = "d6bf32710628e7ff298739f1ed80a0bfdafc0c6a3e284c4540b23f18e8889d4b"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -143,9 +143,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf42656f5f6df7bfafc4dd7b63a1888b0627c07b43b2cb9aa54e13843fed39eb"
+checksum = "4d693e93a0fbf56b4bc93cffe6b107c2e52f070e1111950505fc8c83ac440b9d"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -154,9 +154,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0c2d3badd4b9690865f5bb68a71fa94de592fa2df3f3d11a5a062c60c0a107a"
+checksum = "76fb52ba71be98312f35e798d9e98e45ab2586f27584231bf7c644fa9501e8af"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -165,9 +165,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88eca54bbecea3170035168357306e9c779d4a63d8bf036c9e16bd21fdaa69b5"
+checksum = "2551b2e185022b89e9efa5e04c0f17f679b86ef73d9f7feabc48b608ff23120d"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -374,9 +374,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
 [[package]]
 name = "wasmtime-jit-icache-coherence"
-version = "14.0.1"
+version = "14.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9aaf2fa8fd2d6b65abae9b92edfe69254cc5d6b166e342364036c3e347de8da9"
+checksum = "0980a96b16abbdaf829858d2389697b1d6cfc6a903873fd74b7e47a6b1045584"
 dependencies = [
  "cfg-if",
  "libc",
diff --git a/compiler/rustc_codegen_cranelift/Cargo.toml b/compiler/rustc_codegen_cranelift/Cargo.toml
index f2ce714e8ff1..b5e6f431123e 100644
--- a/compiler/rustc_codegen_cranelift/Cargo.toml
+++ b/compiler/rustc_codegen_cranelift/Cargo.toml
@@ -8,12 +8,12 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { version = "0.101.1", features = ["unwind", "all-arch"] }
-cranelift-frontend = { version = "0.101.1" }
-cranelift-module = { version = "0.101.1" }
-cranelift-native = { version = "0.101.1" }
-cranelift-jit = { version = "0.101.1", optional = true }
-cranelift-object = { version = "0.101.1" }
+cranelift-codegen = { version = "0.101.2", default-features = false, features = ["std", "unwind", "all-arch"] }
+cranelift-frontend = { version = "0.101.2" }
+cranelift-module = { version = "0.101.2" }
+cranelift-native = { version = "0.101.2" }
+cranelift-jit = { version = "0.101.2", optional = true }
+cranelift-object = { version = "0.101.2" }
 target-lexicon = "0.12.0"
 gimli = { version = "0.28", default-features = false, features = ["write"]}
 object = { version = "0.32", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }

From 1e39bbf74c69f64f570d378d7cd68b314b1bae93 Mon Sep 17 00:00:00 2001
From: bjorn3 <17426603+bjorn3@users.noreply.github.com>
Date: Wed, 25 Oct 2023 17:19:49 +0000
Subject: [PATCH 136/297] Update Cranelift to 0.101.2 and disable host-arch
 feature of cranelift-codegen

This ensures that cg_clif can be built for targets that aren't natively
supported by Cranelift. It will not be possible to compile for the host
in this case, but cross-compilation will still be possible.

We won't distribute cg_clif as rustup component for any targets that
aren't natively supported by Cranelift, but will still build it if
codegen-backends lists "cranelift".
---
 Cargo.lock | 52 ++++++++++++++++++++++++++--------------------------
 Cargo.toml | 12 ++++++------
 2 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index c716d5011731..e8558198b94f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -45,18 +45,18 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cranelift-bforest"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1512c3bb6b13018e7109fc3ac964bc87b329eaf3a77825d337558d0c7f6f1be"
+checksum = "f773437307980ac0f424bf9b9a5d0cd21a0f17248c6860c9a65bec8b5975f3fe"
 dependencies = [
  "cranelift-entity",
 ]
 
 [[package]]
 name = "cranelift-codegen"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16cb8fb9220a6ea7a226705a273ab905309ee546267bdf34948d57932d7f0396"
+checksum = "443c2ac50e97fb7de1a0f862753fce3f27215558811a6fcee508eb0c3747fa79"
 dependencies = [
  "bumpalo",
  "cranelift-bforest",
@@ -75,39 +75,39 @@ dependencies = [
 
 [[package]]
 name = "cranelift-codegen-meta"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab3a8d3b0d4745b183da5ea0792b13d79f5c23d6e69ac04761728e2532b56649"
+checksum = "c5b174c411480c79ce0793c55042fa51bec27e486381d103a53cab3b480cb2db"
 dependencies = [
  "cranelift-codegen-shared",
 ]
 
 [[package]]
 name = "cranelift-codegen-shared"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "524141c8e68f2abc2043de4c2b31f6d9dd42432738c246431d0572a1422a4a84"
+checksum = "73fa0151a528066a369de6debeea4d4b23a32aba68b5add8c46d3dc8091ff434"
 
 [[package]]
 name = "cranelift-control"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97513b57c961c713789a03886a57b43e14ebcd204cbaa8ae50ca6c70a8e716b3"
+checksum = "b8adf1e6398493c9bea1190e37d28a0eb0eca5fddbc80e01e506cda34db92b1f"
 dependencies = [
  "arbitrary",
 ]
 
 [[package]]
 name = "cranelift-entity"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3f23d3cf3afa7e45f239702612c76d87964f652a55e28d13ed6d7e20f3479dd"
+checksum = "4917e2ed3bb5fe87d0ed88395ca6d644018d119a034faedd1f3e1f2c33cd52b2"
 
 [[package]]
 name = "cranelift-frontend"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "554cd4947ec9209b58bf9ae5bf83581b5ddf9128bd967208e334b504a57db54e"
+checksum = "9aaadf1e7cf28886bbf046eaf7ef538997bc8a7e020e578ea4957b39da87d5a1"
 dependencies = [
  "cranelift-codegen",
  "log",
@@ -117,15 +117,15 @@ dependencies = [
 
 [[package]]
 name = "cranelift-isle"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c1892a439696b6413cb54083806f5fd9fc431768b8de74864b3d9e8b93b124f"
+checksum = "a67fda31b9d69eaa1c49a2081939454c45857596a9d45af6744680541c628b4c"
 
 [[package]]
 name = "cranelift-jit"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32209252fb38acaf1662ccd0397907bbe0e92bdb13b6ddbfd2f74e437f83e685"
+checksum = "d6bf32710628e7ff298739f1ed80a0bfdafc0c6a3e284c4540b23f18e8889d4b"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -143,9 +143,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-module"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf42656f5f6df7bfafc4dd7b63a1888b0627c07b43b2cb9aa54e13843fed39eb"
+checksum = "4d693e93a0fbf56b4bc93cffe6b107c2e52f070e1111950505fc8c83ac440b9d"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -154,9 +154,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-native"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0c2d3badd4b9690865f5bb68a71fa94de592fa2df3f3d11a5a062c60c0a107a"
+checksum = "76fb52ba71be98312f35e798d9e98e45ab2586f27584231bf7c644fa9501e8af"
 dependencies = [
  "cranelift-codegen",
  "libc",
@@ -165,9 +165,9 @@ dependencies = [
 
 [[package]]
 name = "cranelift-object"
-version = "0.101.1"
+version = "0.101.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88eca54bbecea3170035168357306e9c779d4a63d8bf036c9e16bd21fdaa69b5"
+checksum = "2551b2e185022b89e9efa5e04c0f17f679b86ef73d9f7feabc48b608ff23120d"
 dependencies = [
  "anyhow",
  "cranelift-codegen",
@@ -374,9 +374,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
 [[package]]
 name = "wasmtime-jit-icache-coherence"
-version = "14.0.1"
+version = "14.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9aaf2fa8fd2d6b65abae9b92edfe69254cc5d6b166e342364036c3e347de8da9"
+checksum = "0980a96b16abbdaf829858d2389697b1d6cfc6a903873fd74b7e47a6b1045584"
 dependencies = [
  "cfg-if",
  "libc",
diff --git a/Cargo.toml b/Cargo.toml
index f2ce714e8ff1..b5e6f431123e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -8,12 +8,12 @@ crate-type = ["dylib"]
 
 [dependencies]
 # These have to be in sync with each other
-cranelift-codegen = { version = "0.101.1", features = ["unwind", "all-arch"] }
-cranelift-frontend = { version = "0.101.1" }
-cranelift-module = { version = "0.101.1" }
-cranelift-native = { version = "0.101.1" }
-cranelift-jit = { version = "0.101.1", optional = true }
-cranelift-object = { version = "0.101.1" }
+cranelift-codegen = { version = "0.101.2", default-features = false, features = ["std", "unwind", "all-arch"] }
+cranelift-frontend = { version = "0.101.2" }
+cranelift-module = { version = "0.101.2" }
+cranelift-native = { version = "0.101.2" }
+cranelift-jit = { version = "0.101.2", optional = true }
+cranelift-object = { version = "0.101.2" }
 target-lexicon = "0.12.0"
 gimli = { version = "0.28", default-features = false, features = ["write"]}
 object = { version = "0.32", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }

From d89582c8e5171f9d9a5b3c0feedb9f7be114c90e Mon Sep 17 00:00:00 2001
From: bjorn3 <17426603+bjorn3@users.noreply.github.com>
Date: Thu, 26 Oct 2023 17:38:52 +0000
Subject: [PATCH 137/297] Update target-lexicon to 0.12.12

This adds support for loongarch and a bunch of other targets
---
 compiler/rustc_codegen_cranelift/Cargo.lock | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler/rustc_codegen_cranelift/Cargo.lock b/compiler/rustc_codegen_cranelift/Cargo.lock
index e8558198b94f..dcb6cc57584c 100644
--- a/compiler/rustc_codegen_cranelift/Cargo.lock
+++ b/compiler/rustc_codegen_cranelift/Cargo.lock
@@ -362,9 +362,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
 
 [[package]]
 name = "target-lexicon"
-version = "0.12.5"
+version = "0.12.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9410d0f6853b1d94f0e519fb95df60f29d2c1eff2d921ffdf01a4c8a3b54f12d"
+checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a"
 
 [[package]]
 name = "version_check"

From 1cb7bdb757d35d51a39219351d688e4631b86ff9 Mon Sep 17 00:00:00 2001
From: bjorn3 <17426603+bjorn3@users.noreply.github.com>
Date: Thu, 26 Oct 2023 17:38:52 +0000
Subject: [PATCH 138/297] Update target-lexicon to 0.12.12

This adds support for loongarch and a bunch of other targets
---
 Cargo.lock | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index e8558198b94f..dcb6cc57584c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -362,9 +362,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
 
 [[package]]
 name = "target-lexicon"
-version = "0.12.5"
+version = "0.12.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9410d0f6853b1d94f0e519fb95df60f29d2c1eff2d921ffdf01a4c8a3b54f12d"
+checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a"
 
 [[package]]
 name = "version_check"

From 621494382da59e3a899a968e7d445170dd37ba1d Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Fri, 20 Oct 2023 21:26:57 +0000
Subject: [PATCH 139/297] Add gen blocks to ast and do some broken ast lowering

---
 compiler/rustc_ast/src/ast.rs                 | 32 ++++++++--
 compiler/rustc_ast/src/mut_visit.rs           |  2 +-
 compiler/rustc_ast/src/util/classify.rs       |  2 +-
 compiler/rustc_ast/src/util/parser.rs         |  4 +-
 compiler/rustc_ast/src/visit.rs               |  2 +-
 compiler/rustc_ast_lowering/src/expr.rs       | 61 ++++++++++++++++++-
 .../rustc_ast_pretty/src/pprust/state/expr.rs |  4 +-
 .../src/assert/context.rs                     |  2 +-
 compiler/rustc_parse/src/parser/expr.rs       | 32 +++++-----
 compiler/rustc_passes/src/hir_stats.rs        |  4 +-
 compiler/rustc_resolve/src/def_collector.rs   |  2 +-
 compiler/rustc_resolve/src/late.rs            |  2 +-
 .../src/suspicious_operation_groupings.rs     |  2 +-
 .../clippy/clippy_utils/src/ast_utils.rs      |  2 +-
 src/tools/clippy/clippy_utils/src/sugg.rs     |  2 +-
 src/tools/rustfmt/src/closures.rs             |  2 +-
 src/tools/rustfmt/src/expr.rs                 |  8 +--
 src/tools/rustfmt/src/utils.rs                |  2 +-
 tests/ui/coroutine/gen_block.e2024.stderr     |  9 ++-
 tests/ui/coroutine/gen_block.none.stderr      |  8 +--
 tests/ui/coroutine/gen_block.rs               |  3 +-
 21 files changed, 131 insertions(+), 56 deletions(-)

diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs
index 543c528db448..aceee07a7341 100644
--- a/compiler/rustc_ast/src/ast.rs
+++ b/compiler/rustc_ast/src/ast.rs
@@ -1232,7 +1232,7 @@ impl Expr {
             ExprKind::Closure(..) => ExprPrecedence::Closure,
             ExprKind::Block(..) => ExprPrecedence::Block,
             ExprKind::TryBlock(..) => ExprPrecedence::TryBlock,
-            ExprKind::Async(..) => ExprPrecedence::Async,
+            ExprKind::Gen(..) => ExprPrecedence::Gen,
             ExprKind::Await(..) => ExprPrecedence::Await,
             ExprKind::Assign(..) => ExprPrecedence::Assign,
             ExprKind::AssignOp(..) => ExprPrecedence::AssignOp,
@@ -1401,11 +1401,9 @@ pub enum ExprKind {
     Closure(Box<Closure>),
     /// A block (`'label: { ... }`).
     Block(P<Block>, Option<Label>),
-    /// An async block (`async move { ... }`).
-    ///
-    /// The async block used to have a `NodeId`, which was removed in favor of
-    /// using the parent `NodeId` of the parent `Expr`.
-    Async(CaptureBy, P<Block>),
+    /// An `async` block (`async move { ... }`),
+    /// or a `gen` block (`gen move { ... }`)
+    Gen(CaptureBy, P<Block>, GenBlockKind),
     /// An await expression (`my_future.await`). Span is of await keyword.
     Await(P<Expr>, Span),
 
@@ -1495,6 +1493,28 @@ pub enum ExprKind {
     Err,
 }
 
+/// Used to differentiate between `async {}` blocks and `gen {}` blocks.
+#[derive(Clone, Encodable, Decodable, Debug, PartialEq, Eq)]
+pub enum GenBlockKind {
+    Async,
+    Gen,
+}
+
+impl fmt::Display for GenBlockKind {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.modifier().fmt(f)
+    }
+}
+
+impl GenBlockKind {
+    pub fn modifier(&self) -> &'static str {
+        match self {
+            GenBlockKind::Async => "async",
+            GenBlockKind::Gen => "gen",
+        }
+    }
+}
+
 /// The explicit `Self` type in a "qualified path". The actual
 /// path, including the trait and the associated item, is stored
 /// separately. `position` represents the index of the associated
diff --git a/compiler/rustc_ast/src/mut_visit.rs b/compiler/rustc_ast/src/mut_visit.rs
index ba2887146cfe..0634ee970ec5 100644
--- a/compiler/rustc_ast/src/mut_visit.rs
+++ b/compiler/rustc_ast/src/mut_visit.rs
@@ -1418,7 +1418,7 @@ pub fn noop_visit_expr<T: MutVisitor>(
             vis.visit_block(blk);
             visit_opt(label, |label| vis.visit_label(label));
         }
-        ExprKind::Async(_capture_by, body) => {
+        ExprKind::Gen(_capture_by, body, _) => {
             vis.visit_block(body);
         }
         ExprKind::Await(expr, await_kw_span) => {
diff --git a/compiler/rustc_ast/src/util/classify.rs b/compiler/rustc_ast/src/util/classify.rs
index f9f1c0cf956b..821fca6656cd 100644
--- a/compiler/rustc_ast/src/util/classify.rs
+++ b/compiler/rustc_ast/src/util/classify.rs
@@ -46,7 +46,7 @@ pub fn expr_trailing_brace(mut expr: &ast::Expr) -> Option<&ast::Expr> {
             Closure(closure) => {
                 expr = &closure.body;
             }
-            Async(..) | Block(..) | ForLoop(..) | If(..) | Loop(..) | Match(..) | Struct(..)
+            Gen(..) | Block(..) | ForLoop(..) | If(..) | Loop(..) | Match(..) | Struct(..)
             | TryBlock(..) | While(..) => break Some(expr),
             _ => break None,
         }
diff --git a/compiler/rustc_ast/src/util/parser.rs b/compiler/rustc_ast/src/util/parser.rs
index d3e43e202354..13768c120179 100644
--- a/compiler/rustc_ast/src/util/parser.rs
+++ b/compiler/rustc_ast/src/util/parser.rs
@@ -285,7 +285,7 @@ pub enum ExprPrecedence {
     Block,
     TryBlock,
     Struct,
-    Async,
+    Gen,
     Await,
     Err,
 }
@@ -351,7 +351,7 @@ impl ExprPrecedence {
             | ExprPrecedence::ConstBlock
             | ExprPrecedence::Block
             | ExprPrecedence::TryBlock
-            | ExprPrecedence::Async
+            | ExprPrecedence::Gen
             | ExprPrecedence::Struct
             | ExprPrecedence::Err => PREC_PAREN,
         }
diff --git a/compiler/rustc_ast/src/visit.rs b/compiler/rustc_ast/src/visit.rs
index e66c4a9ee267..e091961a1443 100644
--- a/compiler/rustc_ast/src/visit.rs
+++ b/compiler/rustc_ast/src/visit.rs
@@ -872,7 +872,7 @@ pub fn walk_expr<'a, V: Visitor<'a>>(visitor: &mut V, expression: &'a Expr) {
             walk_list!(visitor, visit_label, opt_label);
             visitor.visit_block(block);
         }
-        ExprKind::Async(_, body) => {
+        ExprKind::Gen(_, body, _) => {
             visitor.visit_block(body);
         }
         ExprKind::Await(expr, _) => visitor.visit_expr(expr),
diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index ab82bbd46a6e..cce0b44fd98e 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -183,7 +183,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     self.arena.alloc_from_iter(arms.iter().map(|x| self.lower_arm(x))),
                     hir::MatchSource::Normal,
                 ),
-                ExprKind::Async(capture_clause, block) => self.make_async_expr(
+                ExprKind::Gen(capture_clause, block, GenBlockKind::Async) => self.make_async_expr(
                     *capture_clause,
                     e.id,
                     None,
@@ -317,6 +317,14 @@ impl<'hir> LoweringContext<'_, 'hir> {
                         rest,
                     )
                 }
+                ExprKind::Gen(capture_clause, block, GenBlockKind::Gen) => self.make_gen_expr(
+                    *capture_clause,
+                    e.id,
+                    None,
+                    e.span,
+                    hir::CoroutineSource::Block,
+                    |this| this.with_new_scopes(|this| this.lower_block_expr(block)),
+                ),
                 ExprKind::Yield(opt_expr) => self.lower_expr_yield(e.span, opt_expr.as_deref()),
                 ExprKind::Err => hir::ExprKind::Err(
                     self.tcx.sess.delay_span_bug(e.span, "lowered ExprKind::Err"),
@@ -661,6 +669,57 @@ impl<'hir> LoweringContext<'_, 'hir> {
         }))
     }
 
+    /// Lower a `gen` construct to a generator that implements `Iterator`.
+    ///
+    /// This results in:
+    ///
+    /// ```text
+    /// static move? |()| -> () {
+    ///     <body>
+    /// }
+    /// ```
+    pub(super) fn make_gen_expr(
+        &mut self,
+        capture_clause: CaptureBy,
+        closure_node_id: NodeId,
+        _yield_ty: Option<hir::FnRetTy<'hir>>,
+        span: Span,
+        gen_kind: hir::CoroutineSource,
+        body: impl FnOnce(&mut Self) -> hir::Expr<'hir>,
+    ) -> hir::ExprKind<'hir> {
+        let output = hir::FnRetTy::DefaultReturn(self.lower_span(span));
+
+        // The closure/generator `FnDecl` takes a single (resume) argument of type `input_ty`.
+        let fn_decl = self.arena.alloc(hir::FnDecl {
+            inputs: &[],
+            output,
+            c_variadic: false,
+            implicit_self: hir::ImplicitSelfKind::None,
+            lifetime_elision_allowed: false,
+        });
+
+        let body = self.lower_body(move |this| {
+            this.coroutine_kind = Some(hir::CoroutineKind::Gen(gen_kind));
+
+            let res = body(this);
+            (&[], res)
+        });
+
+        // `static |()| -> () { body }`:
+        hir::ExprKind::Closure(self.arena.alloc(hir::Closure {
+            def_id: self.local_def_id(closure_node_id),
+            binder: hir::ClosureBinder::Default,
+            capture_clause,
+            bound_generic_params: &[],
+            fn_decl,
+            body,
+            fn_decl_span: self.lower_span(span),
+            fn_arg_span: None,
+            movability: Some(hir::Movability::Static),
+            constness: hir::Constness::NotConst,
+        }))
+    }
+
     /// Forwards a possible `#[track_caller]` annotation from `outer_hir_id` to
     /// `inner_hir_id` in case the `async_fn_track_caller` feature is enabled.
     pub(super) fn maybe_forward_track_caller(
diff --git a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
index 269cb8f23802..e84af12d3f9c 100644
--- a/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
+++ b/compiler/rustc_ast_pretty/src/pprust/state/expr.rs
@@ -445,8 +445,8 @@ impl<'a> State<'a> {
                 self.ibox(0);
                 self.print_block_with_attrs(blk, attrs);
             }
-            ast::ExprKind::Async(capture_clause, blk) => {
-                self.word_nbsp("async");
+            ast::ExprKind::Gen(capture_clause, blk, kind) => {
+                self.word_nbsp(kind.modifier());
                 self.print_capture_clause(*capture_clause);
                 // cbox/ibox in analogy to the `ExprKind::Block` arm above
                 self.cbox(0);
diff --git a/compiler/rustc_builtin_macros/src/assert/context.rs b/compiler/rustc_builtin_macros/src/assert/context.rs
index 6733b9e56add..2a4bfe9e200f 100644
--- a/compiler/rustc_builtin_macros/src/assert/context.rs
+++ b/compiler/rustc_builtin_macros/src/assert/context.rs
@@ -294,7 +294,7 @@ impl<'cx, 'a> Context<'cx, 'a> {
             // sync with the `rfc-2011-nicer-assert-messages/all-expr-kinds.rs` test.
             ExprKind::Assign(_, _, _)
             | ExprKind::AssignOp(_, _, _)
-            | ExprKind::Async(_, _)
+            | ExprKind::Gen(_, _, _)
             | ExprKind::Await(_, _)
             | ExprKind::Block(_, _)
             | ExprKind::Break(_, _)
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index e59076acf4c1..a1b04105a4cb 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -9,7 +9,7 @@ use super::{
 use crate::errors;
 use crate::maybe_recover_from_interpolated_ty_qpath;
 use ast::mut_visit::{noop_visit_expr, MutVisitor};
-use ast::{Path, PathSegment};
+use ast::{GenBlockKind, Path, PathSegment};
 use core::mem;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter, Token, TokenKind};
@@ -1422,9 +1422,6 @@ impl<'a> Parser<'a> {
             } else if this.is_try_block() {
                 this.expect_keyword(kw::Try)?;
                 this.parse_try_block(lo)
-            } else if this.is_gen_block() {
-                this.expect_keyword(kw::Gen)?;
-                this.parse_gen_block(lo)
             } else if this.eat_keyword(kw::Return) {
                 this.parse_expr_return()
             } else if this.eat_keyword(kw::Continue) {
@@ -1446,12 +1443,14 @@ impl<'a> Parser<'a> {
                 if this.check_keyword(kw::Async) {
                     if this.is_async_block() {
                         // Check for `async {` and `async move {`.
-                        this.parse_async_block()
+                        this.parse_gen_block()
                     } else {
                         this.parse_expr_closure()
                     }
                 } else if this.eat_keyword(kw::Await) {
                     this.recover_incorrect_await_syntax(lo, this.prev_token.span)
+                } else if this.token.uninterpolated_span().at_least_rust_2024() {
+                    if this.is_gen_block() { this.parse_gen_block() } else { this.parse_expr_lit() }
                 } else {
                     this.parse_expr_lit()
                 }
@@ -3043,14 +3042,6 @@ impl<'a> Parser<'a> {
         }
     }
 
-    /// Parses a `gen {...}` expression (`gen` token already eaten).
-    fn parse_gen_block(&mut self, _span_lo: Span) -> PResult<'a, P<Expr>> {
-        let (_attrs, _body) = self.parse_inner_attrs_and_block()?;
-
-        Err(errors::GenBlock { span: self.prev_token.span }
-            .into_diagnostic(&self.sess.span_diagnostic))
-    }
-
     fn is_do_catch_block(&self) -> bool {
         self.token.is_keyword(kw::Do)
             && self.is_keyword_ahead(1, &[kw::Catch])
@@ -3077,13 +3068,18 @@ impl<'a> Parser<'a> {
             && self.token.uninterpolated_span().at_least_rust_2024()
     }
 
-    /// Parses an `async move? {...}` expression.
-    fn parse_async_block(&mut self) -> PResult<'a, P<Expr>> {
+    /// Parses an `async move? {...}` or `gen move? {...}` expression.
+    fn parse_gen_block(&mut self) -> PResult<'a, P<Expr>> {
         let lo = self.token.span;
-        self.expect_keyword(kw::Async)?;
+        let kind = if self.eat_keyword(kw::Async) {
+            GenBlockKind::Async
+        } else {
+            assert!(self.eat_keyword(kw::Gen));
+            GenBlockKind::Gen
+        };
         let capture_clause = self.parse_capture_clause()?;
         let (attrs, body) = self.parse_inner_attrs_and_block()?;
-        let kind = ExprKind::Async(capture_clause, body);
+        let kind = ExprKind::Gen(capture_clause, body, kind);
         Ok(self.mk_expr_with_attrs(lo.to(self.prev_token.span), kind, attrs))
     }
 
@@ -3614,7 +3610,7 @@ impl MutVisitor for CondChecker<'_> {
             | ExprKind::Match(_, _)
             | ExprKind::Closure(_)
             | ExprKind::Block(_, _)
-            | ExprKind::Async(_, _)
+            | ExprKind::Gen(_, _, _)
             | ExprKind::TryBlock(_)
             | ExprKind::Underscore
             | ExprKind::Path(_, _)
diff --git a/compiler/rustc_passes/src/hir_stats.rs b/compiler/rustc_passes/src/hir_stats.rs
index 24087a4eabbc..f915c1057d6c 100644
--- a/compiler/rustc_passes/src/hir_stats.rs
+++ b/compiler/rustc_passes/src/hir_stats.rs
@@ -567,10 +567,10 @@ impl<'v> ast_visit::Visitor<'v> for StatCollector<'v> {
             (self, e, e.kind, Id::None, ast, Expr, ExprKind),
             [
                 Array, ConstBlock, Call, MethodCall, Tup, Binary, Unary, Lit, Cast, Type, Let,
-                If, While, ForLoop, Loop, Match, Closure, Block, Async, Await, TryBlock, Assign,
+                If, While, ForLoop, Loop, Match, Closure, Block, Await, TryBlock, Assign,
                 AssignOp, Field, Index, Range, Underscore, Path, AddrOf, Break, Continue, Ret,
                 InlineAsm, FormatArgs, OffsetOf, MacCall, Struct, Repeat, Paren, Try, Yield, Yeet,
-                Become, IncludedBytes, Err
+                Become, IncludedBytes, Gen, Err
             ]
         );
         ast_visit::walk_expr(self, e)
diff --git a/compiler/rustc_resolve/src/def_collector.rs b/compiler/rustc_resolve/src/def_collector.rs
index 356d7f365fe7..13df7efe636d 100644
--- a/compiler/rustc_resolve/src/def_collector.rs
+++ b/compiler/rustc_resolve/src/def_collector.rs
@@ -260,7 +260,7 @@ impl<'a, 'b, 'tcx> visit::Visitor<'a> for DefCollector<'a, 'b, 'tcx> {
                     Async::No => closure_def,
                 }
             }
-            ExprKind::Async(_, _) => self.create_def(expr.id, DefPathData::ClosureExpr, expr.span),
+            ExprKind::Gen(_, _, _) => self.create_def(expr.id, DefPathData::ClosureExpr, expr.span),
             _ => self.parent_def,
         };
 
diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs
index 42ca407cddf6..83f9f3a4210b 100644
--- a/compiler/rustc_resolve/src/late.rs
+++ b/compiler/rustc_resolve/src/late.rs
@@ -4321,7 +4321,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
                 );
             }
             ExprKind::Closure(..) => visit::walk_expr(self, expr),
-            ExprKind::Async(..) => {
+            ExprKind::Gen(..) => {
                 self.with_label_rib(RibKind::ClosureOrAsync, |this| visit::walk_expr(this, expr));
             }
             ExprKind::Repeat(ref elem, ref ct) => {
diff --git a/src/tools/clippy/clippy_lints/src/suspicious_operation_groupings.rs b/src/tools/clippy/clippy_lints/src/suspicious_operation_groupings.rs
index d10f10ef87e8..7dff37a2b8ff 100644
--- a/src/tools/clippy/clippy_lints/src/suspicious_operation_groupings.rs
+++ b/src/tools/clippy/clippy_lints/src/suspicious_operation_groupings.rs
@@ -578,7 +578,7 @@ fn ident_difference_expr_with_base_location(
         | (Assign(_, _, _), Assign(_, _, _))
         | (TryBlock(_), TryBlock(_))
         | (Await(_, _), Await(_, _))
-        | (Async(_, _), Async(_, _))
+        | (Gen(_, _, _), Gen(_, _, _))
         | (Block(_, _), Block(_, _))
         | (Closure(_), Closure(_))
         | (Match(_, _), Match(_, _))
diff --git a/src/tools/clippy/clippy_utils/src/ast_utils.rs b/src/tools/clippy/clippy_utils/src/ast_utils.rs
index a78ff02021f2..a2c61e07b70a 100644
--- a/src/tools/clippy/clippy_utils/src/ast_utils.rs
+++ b/src/tools/clippy/clippy_utils/src/ast_utils.rs
@@ -211,7 +211,7 @@ pub fn eq_expr(l: &Expr, r: &Expr) -> bool {
                 && eq_fn_decl(lf, rf)
                 && eq_expr(le, re)
         },
-        (Async(lc, lb), Async(rc, rb)) => lc == rc && eq_block(lb, rb),
+        (Gen(lc, lb, lk), Gen(rc, rb, rk)) => lc == rc && eq_block(lb, rb) && lk == rk,
         (Range(lf, lt, ll), Range(rf, rt, rl)) => ll == rl && eq_expr_opt(lf, rf) && eq_expr_opt(lt, rt),
         (AddrOf(lbk, lm, le), AddrOf(rbk, rm, re)) => lbk == rbk && lm == rm && eq_expr(le, re),
         (Path(lq, lp), Path(rq, rp)) => both(lq, rq, eq_qself) && eq_path(lp, rp),
diff --git a/src/tools/clippy/clippy_utils/src/sugg.rs b/src/tools/clippy/clippy_utils/src/sugg.rs
index ae8ee371ffae..836f8cc19168 100644
--- a/src/tools/clippy/clippy_utils/src/sugg.rs
+++ b/src/tools/clippy/clippy_utils/src/sugg.rs
@@ -190,7 +190,7 @@ impl<'a> Sugg<'a> {
                 (snip, false) => Sugg::MaybeParen(snip),
                 (snip, true) => Sugg::NonParen(snip),
             },
-            ast::ExprKind::Async(..)
+            ast::ExprKind::Gen(..)
             | ast::ExprKind::Block(..)
             | ast::ExprKind::Break(..)
             | ast::ExprKind::Call(..)
diff --git a/src/tools/rustfmt/src/closures.rs b/src/tools/rustfmt/src/closures.rs
index a09146e95924..16b8ce7a916a 100644
--- a/src/tools/rustfmt/src/closures.rs
+++ b/src/tools/rustfmt/src/closures.rs
@@ -188,7 +188,7 @@ fn rewrite_closure_expr(
     fn allow_multi_line(expr: &ast::Expr) -> bool {
         match expr.kind {
             ast::ExprKind::Match(..)
-            | ast::ExprKind::Async(..)
+            | ast::ExprKind::Gen(..)
             | ast::ExprKind::Block(..)
             | ast::ExprKind::TryBlock(..)
             | ast::ExprKind::Loop(..)
diff --git a/src/tools/rustfmt/src/expr.rs b/src/tools/rustfmt/src/expr.rs
index acde88093296..8c2262fde811 100644
--- a/src/tools/rustfmt/src/expr.rs
+++ b/src/tools/rustfmt/src/expr.rs
@@ -367,7 +367,7 @@ pub(crate) fn format_expr(
                 ))
             }
         }
-        ast::ExprKind::Async(capture_by, ref block) => {
+        ast::ExprKind::Gen(capture_by, ref block, ref kind) => {
             let mover = if capture_by == ast::CaptureBy::Value {
                 "move "
             } else {
@@ -375,7 +375,7 @@ pub(crate) fn format_expr(
             };
             if let rw @ Some(_) = rewrite_single_line_block(
                 context,
-                format!("async {mover}").as_str(),
+                format!("{kind} {mover}").as_str(),
                 block,
                 Some(&expr.attrs),
                 None,
@@ -386,7 +386,7 @@ pub(crate) fn format_expr(
                 // 6 = `async `
                 let budget = shape.width.saturating_sub(6);
                 Some(format!(
-                    "async {mover}{}",
+                    "{kind} {mover}{}",
                     rewrite_block(
                         block,
                         Some(&expr.attrs),
@@ -1371,7 +1371,7 @@ pub(crate) fn can_be_overflowed_expr(
         }
 
         // Handle always block-like expressions
-        ast::ExprKind::Async(..) | ast::ExprKind::Block(..) | ast::ExprKind::Closure(..) => true,
+        ast::ExprKind::Gen(..) | ast::ExprKind::Block(..) | ast::ExprKind::Closure(..) => true,
 
         // Handle `[]` and `{}`-like expressions
         ast::ExprKind::Array(..) | ast::ExprKind::Struct(..) => {
diff --git a/src/tools/rustfmt/src/utils.rs b/src/tools/rustfmt/src/utils.rs
index 79a759d68ce8..fd49030bf1b4 100644
--- a/src/tools/rustfmt/src/utils.rs
+++ b/src/tools/rustfmt/src/utils.rs
@@ -473,7 +473,7 @@ pub(crate) fn is_block_expr(context: &RewriteContext<'_>, expr: &ast::Expr, repr
         | ast::ExprKind::If(..)
         | ast::ExprKind::Block(..)
         | ast::ExprKind::ConstBlock(..)
-        | ast::ExprKind::Async(..)
+        | ast::ExprKind::Gen(..)
         | ast::ExprKind::Loop(..)
         | ast::ExprKind::ForLoop(..)
         | ast::ExprKind::TryBlock(..)
diff --git a/tests/ui/coroutine/gen_block.e2024.stderr b/tests/ui/coroutine/gen_block.e2024.stderr
index 099a9d13ceef..28cf036b7e8c 100644
--- a/tests/ui/coroutine/gen_block.e2024.stderr
+++ b/tests/ui/coroutine/gen_block.e2024.stderr
@@ -1,10 +1,9 @@
-error: `gen` blocks are not yet implemented
-  --> $DIR/gen_block.rs:5:18
+error[E0282]: type annotations needed
+  --> $DIR/gen_block.rs:6:17
    |
 LL |     let x = gen {};
-   |                  ^
-   |
-   = help: only the keyword is reserved for now
+   |                 ^^ cannot infer type
 
 error: aborting due to previous error
 
+For more information about this error, try `rustc --explain E0282`.
diff --git a/tests/ui/coroutine/gen_block.none.stderr b/tests/ui/coroutine/gen_block.none.stderr
index 1cdd5d070ee5..d5ed28c5317b 100644
--- a/tests/ui/coroutine/gen_block.none.stderr
+++ b/tests/ui/coroutine/gen_block.none.stderr
@@ -1,5 +1,5 @@
 error: expected identifier, found reserved keyword `yield`
-  --> $DIR/gen_block.rs:8:19
+  --> $DIR/gen_block.rs:9:19
    |
 LL |     let y = gen { yield 42 };
    |             ---   ^^^^^ expected identifier, found reserved keyword
@@ -7,19 +7,19 @@ LL |     let y = gen { yield 42 };
    |             while parsing this struct
 
 error[E0422]: cannot find struct, variant or union type `gen` in this scope
-  --> $DIR/gen_block.rs:5:13
+  --> $DIR/gen_block.rs:6:13
    |
 LL |     let x = gen {};
    |             ^^^ not found in this scope
 
 error[E0422]: cannot find struct, variant or union type `gen` in this scope
-  --> $DIR/gen_block.rs:8:13
+  --> $DIR/gen_block.rs:9:13
    |
 LL |     let y = gen { yield 42 };
    |             ^^^ not found in this scope
 
 error[E0422]: cannot find struct, variant or union type `gen` in this scope
-  --> $DIR/gen_block.rs:11:5
+  --> $DIR/gen_block.rs:12:5
    |
 LL |     gen {};
    |     ^^^ not found in this scope
diff --git a/tests/ui/coroutine/gen_block.rs b/tests/ui/coroutine/gen_block.rs
index 5c0db5b3d38d..fa5027682362 100644
--- a/tests/ui/coroutine/gen_block.rs
+++ b/tests/ui/coroutine/gen_block.rs
@@ -1,10 +1,11 @@
 // revisions: e2024 none
 //[e2024] compile-flags: --edition 2024 -Zunstable-options
+#![cfg_attr(e2024, feature(coroutines))]
 
 fn main() {
     let x = gen {};
     //[none]~^ ERROR: cannot find
-    //[e2024]~^^ ERROR: `gen` blocks are not yet implemented
+    //[e2024]~^^ ERROR: type annotations needed
     let y = gen { yield 42 };
     //[none]~^ ERROR: found reserved keyword `yield`
     //[none]~| ERROR: cannot find

From 998a816106be58324598d8c4ea9ef9da6f3e38fd Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Fri, 20 Oct 2023 22:48:37 +0000
Subject: [PATCH 140/297] Make `gen` blocks implement the `Iterator` trait

---
 compiler/rustc_hir_typeck/src/closure.rs      |  3 ++
 compiler/rustc_middle/src/traits/select.rs    |  4 ++
 compiler/rustc_middle/src/ty/context.rs       | 11 +++++
 compiler/rustc_span/src/symbol.rs             |  1 +
 .../src/solve/assembly/mod.rs                 | 12 ++++-
 .../src/solve/project_goals/mod.rs            | 35 +++++++++++++-
 .../src/solve/trait_goals.rs                  | 26 +++++++++-
 .../src/traits/project.rs                     | 48 ++++++++++++++++++-
 .../src/traits/select/candidate_assembly.rs   | 23 ++++++++-
 .../src/traits/select/confirmation.rs         | 35 ++++++++++++++
 .../src/traits/select/mod.rs                  |  6 +++
 .../rustc_trait_selection/src/traits/util.rs  | 11 +++++
 tests/ui/coroutine/gen_block_is_coro.rs       | 18 +++++++
 tests/ui/coroutine/gen_block_is_coro.stderr   | 21 ++++++++
 tests/ui/coroutine/gen_block_is_iter.rs       | 19 ++++++++
 tests/ui/coroutine/gen_block_is_no_future.rs  |  8 ++++
 .../coroutine/gen_block_is_no_future.stderr   | 12 +++++
 17 files changed, 286 insertions(+), 7 deletions(-)
 create mode 100644 tests/ui/coroutine/gen_block_is_coro.rs
 create mode 100644 tests/ui/coroutine/gen_block_is_coro.stderr
 create mode 100644 tests/ui/coroutine/gen_block_is_iter.rs
 create mode 100644 tests/ui/coroutine/gen_block_is_no_future.rs
 create mode 100644 tests/ui/coroutine/gen_block_is_no_future.stderr

diff --git a/compiler/rustc_hir_typeck/src/closure.rs b/compiler/rustc_hir_typeck/src/closure.rs
index 47bde21ceb2e..b102ddd8d866 100644
--- a/compiler/rustc_hir_typeck/src/closure.rs
+++ b/compiler/rustc_hir_typeck/src/closure.rs
@@ -652,6 +652,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         },
                     )
                 }
+                Some(hir::CoroutineKind::Gen(hir::CoroutineSource::Fn)) => {
+                    todo!("gen closures do not exist yet")
+                }
 
                 _ => astconv.ty_infer(None, decl.output.span()),
             },
diff --git a/compiler/rustc_middle/src/traits/select.rs b/compiler/rustc_middle/src/traits/select.rs
index bc6856bc9009..f33421bbaa6c 100644
--- a/compiler/rustc_middle/src/traits/select.rs
+++ b/compiler/rustc_middle/src/traits/select.rs
@@ -144,6 +144,10 @@ pub enum SelectionCandidate<'tcx> {
     /// generated for an async construct.
     FutureCandidate,
 
+    /// Implementation of an `Iterator` trait by one of the generator types
+    /// generated for a gen construct.
+    IteratorCandidate,
+
     /// Implementation of a `Fn`-family trait by one of the anonymous
     /// types generated for a fn pointer type (e.g., `fn(int) -> int`)
     FnPointerCandidate {
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index a669ff8d9611..0b3b6700cec1 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -782,6 +782,17 @@ impl<'tcx> TyCtxt<'tcx> {
         matches!(self.coroutine_kind(def_id), Some(hir::CoroutineKind::Async(_)))
     }
 
+    /// Returns `true` if the node pointed to by `def_id` is a general coroutine that implements `Coroutine`.
+    /// This means it is neither an `async` or `gen` construct.
+    pub fn is_general_coroutine(self, def_id: DefId) -> bool {
+        matches!(self.coroutine_kind(def_id), Some(hir::CoroutineKind::Coroutine))
+    }
+
+    /// Returns `true` if the node pointed to by `def_id` is a coroutine for a gen construct.
+    pub fn coroutine_is_gen(self, def_id: DefId) -> bool {
+        matches!(self.coroutine_kind(def_id), Some(hir::CoroutineKind::Gen(_)))
+    }
+
     pub fn stability(self) -> &'tcx stability::Index {
         self.stability_index(())
     }
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 8ef16ed0dbf7..0d21c19e5198 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -226,6 +226,7 @@ symbols! {
         IpAddr,
         IrTyKind,
         Is,
+        Item,
         ItemContext,
         IterEmpty,
         IterOnce,
diff --git a/compiler/rustc_trait_selection/src/solve/assembly/mod.rs b/compiler/rustc_trait_selection/src/solve/assembly/mod.rs
index 0c214ca87538..e6bfa62acc88 100644
--- a/compiler/rustc_trait_selection/src/solve/assembly/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/assembly/mod.rs
@@ -199,7 +199,15 @@ pub(super) trait GoalKind<'tcx>:
         goal: Goal<'tcx, Self>,
     ) -> QueryResult<'tcx>;
 
-    /// A coroutine (that doesn't come from an `async` desugaring) is known to
+    /// A coroutine (that comes from a `gen` desugaring) is known to implement
+    /// `Iterator<Item = O>`, where `O` is given by the generator's yield type
+    /// that was computed during type-checking.
+    fn consider_builtin_iterator_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx>;
+
+    /// A coroutine (that doesn't come from an `async` or `gen` desugaring) is known to
     /// implement `Coroutine<R, Yield = Y, Return = O>`, given the resume, yield,
     /// and return types of the coroutine computed during type-checking.
     fn consider_builtin_coroutine_candidate(
@@ -552,6 +560,8 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             G::consider_builtin_pointee_candidate(self, goal)
         } else if lang_items.future_trait() == Some(trait_def_id) {
             G::consider_builtin_future_candidate(self, goal)
+        } else if lang_items.iterator_trait() == Some(trait_def_id) {
+            G::consider_builtin_iterator_candidate(self, goal)
         } else if lang_items.gen_trait() == Some(trait_def_id) {
             G::consider_builtin_coroutine_candidate(self, goal)
         } else if lang_items.discriminant_kind_trait() == Some(trait_def_id) {
diff --git a/compiler/rustc_trait_selection/src/solve/project_goals/mod.rs b/compiler/rustc_trait_selection/src/solve/project_goals/mod.rs
index 4950a444ffb5..b31b916f5ef7 100644
--- a/compiler/rustc_trait_selection/src/solve/project_goals/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/project_goals/mod.rs
@@ -485,6 +485,37 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
         )
     }
 
+    fn consider_builtin_iterator_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        let self_ty = goal.predicate.self_ty();
+        let ty::Coroutine(def_id, args, _) = *self_ty.kind() else {
+            return Err(NoSolution);
+        };
+
+        // Coroutines are not Iterators unless they come from `gen` desugaring
+        let tcx = ecx.tcx();
+        if !tcx.coroutine_is_gen(def_id) {
+            return Err(NoSolution);
+        }
+
+        let term = args.as_coroutine().yield_ty().into();
+
+        Self::consider_implied_clause(
+            ecx,
+            goal,
+            ty::ProjectionPredicate {
+                projection_ty: ty::AliasTy::new(ecx.tcx(), goal.predicate.def_id(), [self_ty]),
+                term,
+            }
+            .to_predicate(tcx),
+            // Technically, we need to check that the iterator type is Sized,
+            // but that's already proven by the generator being WF.
+            [],
+        )
+    }
+
     fn consider_builtin_coroutine_candidate(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
@@ -496,7 +527,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
 
         // `async`-desugared coroutines do not implement the coroutine trait
         let tcx = ecx.tcx();
-        if tcx.coroutine_is_async(def_id) {
+        if !tcx.is_general_coroutine(def_id) {
             return Err(NoSolution);
         }
 
@@ -523,7 +554,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for ProjectionPredicate<'tcx> {
                 term,
             }
             .to_predicate(tcx),
-            // Technically, we need to check that the future type is Sized,
+            // Technically, we need to check that the coroutine type is Sized,
             // but that's already proven by the coroutine being WF.
             [],
         )
diff --git a/compiler/rustc_trait_selection/src/solve/trait_goals.rs b/compiler/rustc_trait_selection/src/solve/trait_goals.rs
index fa91a125b6ac..0d5d492afec0 100644
--- a/compiler/rustc_trait_selection/src/solve/trait_goals.rs
+++ b/compiler/rustc_trait_selection/src/solve/trait_goals.rs
@@ -335,6 +335,30 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
         ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
     }
 
+    fn consider_builtin_iterator_candidate(
+        ecx: &mut EvalCtxt<'_, 'tcx>,
+        goal: Goal<'tcx, Self>,
+    ) -> QueryResult<'tcx> {
+        if goal.predicate.polarity != ty::ImplPolarity::Positive {
+            return Err(NoSolution);
+        }
+
+        let ty::Coroutine(def_id, _, _) = *goal.predicate.self_ty().kind() else {
+            return Err(NoSolution);
+        };
+
+        // Coroutines are not iterators unless they come from `gen` desugaring
+        let tcx = ecx.tcx();
+        if !tcx.coroutine_is_gen(def_id) {
+            return Err(NoSolution);
+        }
+
+        // Gen coroutines unconditionally implement `Iterator`
+        // Technically, we need to check that the iterator output type is Sized,
+        // but that's already proven by the coroutines being WF.
+        ecx.evaluate_added_goals_and_make_canonical_response(Certainty::Yes)
+    }
+
     fn consider_builtin_coroutine_candidate(
         ecx: &mut EvalCtxt<'_, 'tcx>,
         goal: Goal<'tcx, Self>,
@@ -350,7 +374,7 @@ impl<'tcx> assembly::GoalKind<'tcx> for TraitPredicate<'tcx> {
 
         // `async`-desugared coroutines do not implement the coroutine trait
         let tcx = ecx.tcx();
-        if tcx.coroutine_is_async(def_id) {
+        if !tcx.is_general_coroutine(def_id) {
             return Err(NoSolution);
         }
 
diff --git a/compiler/rustc_trait_selection/src/traits/project.rs b/compiler/rustc_trait_selection/src/traits/project.rs
index af30d9121d1a..20aa3cec8733 100644
--- a/compiler/rustc_trait_selection/src/traits/project.rs
+++ b/compiler/rustc_trait_selection/src/traits/project.rs
@@ -1798,7 +1798,7 @@ fn assemble_candidates_from_impls<'cx, 'tcx>(
                 let self_ty = selcx.infcx.shallow_resolve(obligation.predicate.self_ty());
 
                 let lang_items = selcx.tcx().lang_items();
-                if [lang_items.gen_trait(), lang_items.future_trait()].contains(&Some(trait_ref.def_id))
+                if [lang_items.gen_trait(), lang_items.future_trait(), lang_items.iterator_trait()].contains(&Some(trait_ref.def_id))
                     || selcx.tcx().fn_trait_kind_from_def_id(trait_ref.def_id).is_some()
                 {
                     true
@@ -2015,6 +2015,8 @@ fn confirm_select_candidate<'cx, 'tcx>(
                 confirm_coroutine_candidate(selcx, obligation, data)
             } else if lang_items.future_trait() == Some(trait_def_id) {
                 confirm_future_candidate(selcx, obligation, data)
+            } else if lang_items.iterator_trait() == Some(trait_def_id) {
+                confirm_iterator_candidate(selcx, obligation, data)
             } else if selcx.tcx().fn_trait_kind_from_def_id(trait_def_id).is_some() {
                 if obligation.predicate.self_ty().is_closure() {
                     confirm_closure_candidate(selcx, obligation, data)
@@ -2135,6 +2137,50 @@ fn confirm_future_candidate<'cx, 'tcx>(
         .with_addl_obligations(obligations)
 }
 
+fn confirm_iterator_candidate<'cx, 'tcx>(
+    selcx: &mut SelectionContext<'cx, 'tcx>,
+    obligation: &ProjectionTyObligation<'tcx>,
+    nested: Vec<PredicateObligation<'tcx>>,
+) -> Progress<'tcx> {
+    let ty::Coroutine(_, args, _) =
+        selcx.infcx.shallow_resolve(obligation.predicate.self_ty()).kind()
+    else {
+        unreachable!()
+    };
+    let gen_sig = args.as_coroutine().poly_sig();
+    let Normalized { value: gen_sig, obligations } = normalize_with_depth(
+        selcx,
+        obligation.param_env,
+        obligation.cause.clone(),
+        obligation.recursion_depth + 1,
+        gen_sig,
+    );
+
+    debug!(?obligation, ?gen_sig, ?obligations, "confirm_future_candidate");
+
+    let tcx = selcx.tcx();
+    let iter_def_id = tcx.require_lang_item(LangItem::Iterator, None);
+
+    let predicate = super::util::iterator_trait_ref_and_outputs(
+        tcx,
+        iter_def_id,
+        obligation.predicate.self_ty(),
+        gen_sig,
+    )
+    .map_bound(|(trait_ref, yield_ty)| {
+        debug_assert_eq!(tcx.associated_item(obligation.predicate.def_id).name, sym::Item);
+
+        ty::ProjectionPredicate {
+            projection_ty: ty::AliasTy::new(tcx, obligation.predicate.def_id, trait_ref.args),
+            term: yield_ty.into(),
+        }
+    });
+
+    confirm_param_env_candidate(selcx, obligation, predicate, false)
+        .with_addl_obligations(nested)
+        .with_addl_obligations(obligations)
+}
+
 fn confirm_builtin_candidate<'cx, 'tcx>(
     selcx: &mut SelectionContext<'cx, 'tcx>,
     obligation: &ProjectionTyObligation<'tcx>,
diff --git a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
index ee50a36be55b..722f1d6bbb64 100644
--- a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
@@ -114,6 +114,8 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     self.assemble_coroutine_candidates(obligation, &mut candidates);
                 } else if lang_items.future_trait() == Some(def_id) {
                     self.assemble_future_candidates(obligation, &mut candidates);
+                } else if lang_items.iterator_trait() == Some(def_id) {
+                    self.assemble_iterator_candidates(obligation, &mut candidates);
                 }
 
                 self.assemble_closure_candidates(obligation, &mut candidates);
@@ -211,9 +213,9 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         // type/region parameters.
         let self_ty = obligation.self_ty().skip_binder();
         match self_ty.kind() {
-            // async constructs get lowered to a special kind of coroutine that
+            // `async`/`gen` constructs get lowered to a special kind of coroutine that
             // should *not* `impl Coroutine`.
-            ty::Coroutine(did, ..) if !self.tcx().coroutine_is_async(*did) => {
+            ty::Coroutine(did, ..) if self.tcx().is_general_coroutine(*did) => {
                 debug!(?self_ty, ?obligation, "assemble_coroutine_candidates",);
 
                 candidates.vec.push(CoroutineCandidate);
@@ -243,6 +245,23 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         }
     }
 
+    fn assemble_iterator_candidates(
+        &mut self,
+        obligation: &PolyTraitObligation<'tcx>,
+        candidates: &mut SelectionCandidateSet<'tcx>,
+    ) {
+        let self_ty = obligation.self_ty().skip_binder();
+        if let ty::Coroutine(did, ..) = self_ty.kind() {
+            // gen constructs get lowered to a special kind of coroutine that
+            // should directly `impl Iterator`.
+            if self.tcx().coroutine_is_gen(*did) {
+                debug!(?self_ty, ?obligation, "assemble_iterator_candidates",);
+
+                candidates.vec.push(IteratorCandidate);
+            }
+        }
+    }
+
     /// Checks for the artificial impl that the compiler will create for an obligation like `X :
     /// FnMut<..>` where `X` is a closure type.
     ///
diff --git a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
index fce439f21fa2..952184175f4d 100644
--- a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
@@ -93,6 +93,11 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                 ImplSource::Builtin(BuiltinImplSource::Misc, vtable_future)
             }
 
+            IteratorCandidate => {
+                let vtable_iterator = self.confirm_iterator_candidate(obligation)?;
+                ImplSource::Builtin(BuiltinImplSource::Misc, vtable_iterator)
+            }
+
             FnPointerCandidate { is_const } => {
                 let data = self.confirm_fn_pointer_candidate(obligation, is_const)?;
                 ImplSource::Builtin(BuiltinImplSource::Misc, data)
@@ -780,6 +785,36 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
         Ok(nested)
     }
 
+    fn confirm_iterator_candidate(
+        &mut self,
+        obligation: &PolyTraitObligation<'tcx>,
+    ) -> Result<Vec<PredicateObligation<'tcx>>, SelectionError<'tcx>> {
+        // Okay to skip binder because the args on coroutine types never
+        // touch bound regions, they just capture the in-scope
+        // type/region parameters.
+        let self_ty = self.infcx.shallow_resolve(obligation.self_ty().skip_binder());
+        let ty::Coroutine(coroutine_def_id, args, _) = *self_ty.kind() else {
+            bug!("closure candidate for non-closure {:?}", obligation);
+        };
+
+        debug!(?obligation, ?coroutine_def_id, ?args, "confirm_iterator_candidate");
+
+        let gen_sig = args.as_coroutine().poly_sig();
+
+        let trait_ref = super::util::iterator_trait_ref_and_outputs(
+            self.tcx(),
+            obligation.predicate.def_id(),
+            obligation.predicate.no_bound_vars().expect("iterator has no bound vars").self_ty(),
+            gen_sig,
+        )
+        .map_bound(|(trait_ref, ..)| trait_ref);
+
+        let nested = self.confirm_poly_trait_refs(obligation, trait_ref)?;
+        debug!(?trait_ref, ?nested, "iterator candidate obligations");
+
+        Ok(nested)
+    }
+
     #[instrument(skip(self), level = "debug")]
     fn confirm_closure_candidate(
         &mut self,
diff --git a/compiler/rustc_trait_selection/src/traits/select/mod.rs b/compiler/rustc_trait_selection/src/traits/select/mod.rs
index 67cb39bc0045..cf52e6726a17 100644
--- a/compiler/rustc_trait_selection/src/traits/select/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/mod.rs
@@ -1888,6 +1888,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 | ClosureCandidate { .. }
                 | CoroutineCandidate
                 | FutureCandidate
+                | IteratorCandidate
                 | FnPointerCandidate { .. }
                 | BuiltinObjectCandidate
                 | BuiltinUnsizeCandidate
@@ -1916,6 +1917,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 | ClosureCandidate { .. }
                 | CoroutineCandidate
                 | FutureCandidate
+                | IteratorCandidate
                 | FnPointerCandidate { .. }
                 | BuiltinObjectCandidate
                 | BuiltinUnsizeCandidate
@@ -1950,6 +1952,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 | ClosureCandidate { .. }
                 | CoroutineCandidate
                 | FutureCandidate
+                | IteratorCandidate
                 | FnPointerCandidate { .. }
                 | BuiltinObjectCandidate
                 | BuiltinUnsizeCandidate
@@ -1964,6 +1967,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 | ClosureCandidate { .. }
                 | CoroutineCandidate
                 | FutureCandidate
+                | IteratorCandidate
                 | FnPointerCandidate { .. }
                 | BuiltinObjectCandidate
                 | BuiltinUnsizeCandidate
@@ -2070,6 +2074,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 | ClosureCandidate { .. }
                 | CoroutineCandidate
                 | FutureCandidate
+                | IteratorCandidate
                 | FnPointerCandidate { .. }
                 | BuiltinObjectCandidate
                 | BuiltinUnsizeCandidate
@@ -2080,6 +2085,7 @@ impl<'tcx> SelectionContext<'_, 'tcx> {
                 | ClosureCandidate { .. }
                 | CoroutineCandidate
                 | FutureCandidate
+                | IteratorCandidate
                 | FnPointerCandidate { .. }
                 | BuiltinObjectCandidate
                 | BuiltinUnsizeCandidate
diff --git a/compiler/rustc_trait_selection/src/traits/util.rs b/compiler/rustc_trait_selection/src/traits/util.rs
index 67681fb6ec11..bbde0c82743b 100644
--- a/compiler/rustc_trait_selection/src/traits/util.rs
+++ b/compiler/rustc_trait_selection/src/traits/util.rs
@@ -297,6 +297,17 @@ pub fn future_trait_ref_and_outputs<'tcx>(
     sig.map_bound(|sig| (trait_ref, sig.return_ty))
 }
 
+pub fn iterator_trait_ref_and_outputs<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    iterator_def_id: DefId,
+    self_ty: Ty<'tcx>,
+    sig: ty::PolyGenSig<'tcx>,
+) -> ty::Binder<'tcx, (ty::TraitRef<'tcx>, Ty<'tcx>)> {
+    assert!(!self_ty.has_escaping_bound_vars());
+    let trait_ref = ty::TraitRef::new(tcx, iterator_def_id, [self_ty]);
+    sig.map_bound(|sig| (trait_ref, sig.yield_ty))
+}
+
 pub fn impl_item_is_final(tcx: TyCtxt<'_>, assoc_item: &ty::AssocItem) -> bool {
     assoc_item.defaultness(tcx).is_final()
         && tcx.defaultness(assoc_item.container_id(tcx)).is_final()
diff --git a/tests/ui/coroutine/gen_block_is_coro.rs b/tests/ui/coroutine/gen_block_is_coro.rs
new file mode 100644
index 000000000000..814d5843a1b6
--- /dev/null
+++ b/tests/ui/coroutine/gen_block_is_coro.rs
@@ -0,0 +1,18 @@
+//compile-flags: --edition 2024 -Zunstable-options
+#![feature(coroutines, coroutine_trait)]
+
+use std::ops::Coroutine;
+
+fn foo() -> impl Coroutine<Yield = u32, Return = ()> { //~ ERROR: Coroutine` is not satisfied
+    gen { yield 42 }
+}
+
+fn bar() -> impl Coroutine<Yield = i64, Return = ()> { //~ ERROR: Coroutine` is not satisfied
+    gen { yield 42 }
+}
+
+fn baz() -> impl Coroutine<Yield = i32, Return = ()> { //~ ERROR: Coroutine` is not satisfied
+    gen { yield 42 }
+}
+
+fn main() {}
diff --git a/tests/ui/coroutine/gen_block_is_coro.stderr b/tests/ui/coroutine/gen_block_is_coro.stderr
new file mode 100644
index 000000000000..83a674fa53ce
--- /dev/null
+++ b/tests/ui/coroutine/gen_block_is_coro.stderr
@@ -0,0 +1,21 @@
+error[E0277]: the trait bound `{gen block@$DIR/gen_block_is_coro.rs:7:5: 7:21}: Coroutine` is not satisfied
+  --> $DIR/gen_block_is_coro.rs:6:13
+   |
+LL | fn foo() -> impl Coroutine<Yield = u32, Return = ()> {
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the trait `Coroutine` is not implemented for `{gen block@$DIR/gen_block_is_coro.rs:7:5: 7:21}`
+
+error[E0277]: the trait bound `{gen block@$DIR/gen_block_is_coro.rs:11:5: 11:21}: Coroutine` is not satisfied
+  --> $DIR/gen_block_is_coro.rs:10:13
+   |
+LL | fn bar() -> impl Coroutine<Yield = i64, Return = ()> {
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the trait `Coroutine` is not implemented for `{gen block@$DIR/gen_block_is_coro.rs:11:5: 11:21}`
+
+error[E0277]: the trait bound `{gen block@$DIR/gen_block_is_coro.rs:15:5: 15:21}: Coroutine` is not satisfied
+  --> $DIR/gen_block_is_coro.rs:14:13
+   |
+LL | fn baz() -> impl Coroutine<Yield = i32, Return = ()> {
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the trait `Coroutine` is not implemented for `{gen block@$DIR/gen_block_is_coro.rs:15:5: 15:21}`
+
+error: aborting due to 3 previous errors
+
+For more information about this error, try `rustc --explain E0277`.
diff --git a/tests/ui/coroutine/gen_block_is_iter.rs b/tests/ui/coroutine/gen_block_is_iter.rs
new file mode 100644
index 000000000000..16d5813c69dc
--- /dev/null
+++ b/tests/ui/coroutine/gen_block_is_iter.rs
@@ -0,0 +1,19 @@
+// revisions: next old
+//compile-flags: --edition 2024 -Zunstable-options
+//[next] compile-flags: -Ztrait-solver=next
+// check-pass
+#![feature(coroutines)]
+
+fn foo() -> impl Iterator<Item = u32> {
+    gen { yield 42 }
+}
+
+fn bar() -> impl Iterator<Item = i64> {
+    gen { yield 42 }
+}
+
+fn baz() -> impl Iterator<Item = i32> {
+    gen { yield 42 }
+}
+
+fn main() {}
diff --git a/tests/ui/coroutine/gen_block_is_no_future.rs b/tests/ui/coroutine/gen_block_is_no_future.rs
new file mode 100644
index 000000000000..8927544a7b98
--- /dev/null
+++ b/tests/ui/coroutine/gen_block_is_no_future.rs
@@ -0,0 +1,8 @@
+//compile-flags: --edition 2024 -Zunstable-options
+#![feature(coroutines)]
+
+fn foo() -> impl std::future::Future { //~ ERROR is not a future
+    gen { yield 42 }
+}
+
+fn main() {}
diff --git a/tests/ui/coroutine/gen_block_is_no_future.stderr b/tests/ui/coroutine/gen_block_is_no_future.stderr
new file mode 100644
index 000000000000..db0c3c19b581
--- /dev/null
+++ b/tests/ui/coroutine/gen_block_is_no_future.stderr
@@ -0,0 +1,12 @@
+error[E0277]: `{gen block@$DIR/gen_block_is_no_future.rs:5:5: 5:21}` is not a future
+  --> $DIR/gen_block_is_no_future.rs:4:13
+   |
+LL | fn foo() -> impl std::future::Future {
+   |             ^^^^^^^^^^^^^^^^^^^^^^^^ `{gen block@$DIR/gen_block_is_no_future.rs:5:5: 5:21}` is not a future
+   |
+   = help: the trait `Future` is not implemented for `{gen block@$DIR/gen_block_is_no_future.rs:5:5: 5:21}`
+   = note: {gen block@$DIR/gen_block_is_no_future.rs:5:5: 5:21} must be a future or must implement `IntoFuture` to be awaited
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0277`.

From c892b28c021908d58763667453cb26c941bda745 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Fri, 20 Oct 2023 23:19:40 +0000
Subject: [PATCH 141/297] Basic generators work

---
 compiler/rustc_middle/src/mir/terminator.rs   |  5 +-
 compiler/rustc_mir_transform/src/coroutine.rs | 96 ++++++++++++-------
 compiler/rustc_ty_utils/src/instance.rs       | 13 +++
 tests/ui/coroutine/gen_block_iterate.rs       | 18 ++++
 4 files changed, 98 insertions(+), 34 deletions(-)
 create mode 100644 tests/ui/coroutine/gen_block_iterate.rs

diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs
index ef1d0c867ea1..06e63b0f3d99 100644
--- a/compiler/rustc_middle/src/mir/terminator.rs
+++ b/compiler/rustc_middle/src/mir/terminator.rs
@@ -149,13 +149,14 @@ impl<O> AssertKind<O> {
             ResumedAfterReturn(CoroutineKind::Coroutine) => "coroutine resumed after completion",
             ResumedAfterReturn(CoroutineKind::Async(_)) => "`async fn` resumed after completion",
             ResumedAfterReturn(CoroutineKind::Gen(_)) => {
-                bug!("`gen fn` should just keep returning `None` after the first time")
+                "`gen fn` should just keep returning `None` after completion"
             }
             ResumedAfterPanic(CoroutineKind::Coroutine) => "coroutine resumed after panicking",
             ResumedAfterPanic(CoroutineKind::Async(_)) => "`async fn` resumed after panicking",
             ResumedAfterPanic(CoroutineKind::Gen(_)) => {
-                bug!("`gen fn` should just keep returning `None` after panicking")
+                "`gen fn` should just keep returning `None` after panicking"
             }
+
             BoundsCheck { .. } | MisalignedPointerDereference { .. } => {
                 bug!("Unexpected AssertKind")
             }
diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs
index fa56d59dd80b..8fecff16a919 100644
--- a/compiler/rustc_mir_transform/src/coroutine.rs
+++ b/compiler/rustc_mir_transform/src/coroutine.rs
@@ -224,7 +224,7 @@ struct SuspensionPoint<'tcx> {
 
 struct TransformVisitor<'tcx> {
     tcx: TyCtxt<'tcx>,
-    is_async_kind: bool,
+    coroutine_kind: hir::CoroutineKind,
     state_adt_ref: AdtDef<'tcx>,
     state_args: GenericArgsRef<'tcx>,
 
@@ -261,31 +261,53 @@ impl<'tcx> TransformVisitor<'tcx> {
         is_return: bool,
         statements: &mut Vec<Statement<'tcx>>,
     ) {
-        let idx = VariantIdx::new(match (is_return, self.is_async_kind) {
-            (true, false) => 1,  // CoroutineState::Complete
-            (false, false) => 0, // CoroutineState::Yielded
-            (true, true) => 0,   // Poll::Ready
-            (false, true) => 1,  // Poll::Pending
+        let idx = VariantIdx::new(match (is_return, self.coroutine_kind) {
+            (true, hir::CoroutineKind::Coroutine) => 1, // CoroutineState::Complete
+            (false, hir::CoroutineKind::Coroutine) => 0, // CoroutineState::Yielded
+            (true, hir::CoroutineKind::Async(_)) => 0,  // Poll::Ready
+            (false, hir::CoroutineKind::Async(_)) => 1, // Poll::Pending
+            (true, hir::CoroutineKind::Gen(_)) => 0,    // Option::None
+            (false, hir::CoroutineKind::Gen(_)) => 1,   // Option::Some
         });
 
         let kind = AggregateKind::Adt(self.state_adt_ref.did(), idx, self.state_args, None, None);
 
-        // `Poll::Pending`
-        if self.is_async_kind && idx == VariantIdx::new(1) {
-            assert_eq!(self.state_adt_ref.variant(idx).fields.len(), 0);
+        match self.coroutine_kind {
+            // `Poll::Pending`
+            CoroutineKind::Async(_) => {
+                if !is_return {
+                    assert_eq!(self.state_adt_ref.variant(idx).fields.len(), 0);
 
-            // FIXME(swatinem): assert that `val` is indeed unit?
-            statements.push(Statement {
-                kind: StatementKind::Assign(Box::new((
-                    Place::return_place(),
-                    Rvalue::Aggregate(Box::new(kind), IndexVec::new()),
-                ))),
-                source_info,
-            });
-            return;
+                    // FIXME(swatinem): assert that `val` is indeed unit?
+                    statements.push(Statement {
+                        kind: StatementKind::Assign(Box::new((
+                            Place::return_place(),
+                            Rvalue::Aggregate(Box::new(kind), IndexVec::new()),
+                        ))),
+                        source_info,
+                    });
+                    return;
+                }
+            }
+            // `Option::None`
+            CoroutineKind::Gen(_) => {
+                if is_return {
+                    assert_eq!(self.state_adt_ref.variant(idx).fields.len(), 0);
+
+                    statements.push(Statement {
+                        kind: StatementKind::Assign(Box::new((
+                            Place::return_place(),
+                            Rvalue::Aggregate(Box::new(kind), IndexVec::new()),
+                        ))),
+                        source_info,
+                    });
+                    return;
+                }
+            }
+            CoroutineKind::Coroutine => {}
         }
 
-        // else: `Poll::Ready(x)`, `CoroutineState::Yielded(x)` or `CoroutineState::Complete(x)`
+        // else: `Poll::Ready(x)`, `CoroutineState::Yielded(x)`, `CoroutineState::Complete(x)`, or `Option::Some(x)`
         assert_eq!(self.state_adt_ref.variant(idx).fields.len(), 1);
 
         statements.push(Statement {
@@ -1439,18 +1461,28 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
         };
 
         let is_async_kind = matches!(body.coroutine_kind(), Some(CoroutineKind::Async(_)));
-        let (state_adt_ref, state_args) = if is_async_kind {
-            // Compute Poll<return_ty>
-            let poll_did = tcx.require_lang_item(LangItem::Poll, None);
-            let poll_adt_ref = tcx.adt_def(poll_did);
-            let poll_args = tcx.mk_args(&[body.return_ty().into()]);
-            (poll_adt_ref, poll_args)
-        } else {
-            // Compute CoroutineState<yield_ty, return_ty>
-            let state_did = tcx.require_lang_item(LangItem::CoroutineState, None);
-            let state_adt_ref = tcx.adt_def(state_did);
-            let state_args = tcx.mk_args(&[yield_ty.into(), body.return_ty().into()]);
-            (state_adt_ref, state_args)
+        let (state_adt_ref, state_args) = match body.coroutine_kind().unwrap() {
+            CoroutineKind::Async(_) => {
+                // Compute Poll<return_ty>
+                let poll_did = tcx.require_lang_item(LangItem::Poll, None);
+                let poll_adt_ref = tcx.adt_def(poll_did);
+                let poll_args = tcx.mk_args(&[body.return_ty().into()]);
+                (poll_adt_ref, poll_args)
+            }
+            CoroutineKind::Gen(_) => {
+                // Compute Option<yield_ty>
+                let option_did = tcx.require_lang_item(LangItem::Option, None);
+                let option_adt_ref = tcx.adt_def(option_did);
+                let option_args = tcx.mk_args(&[body.yield_ty().unwrap().into()]);
+                (option_adt_ref, option_args)
+            }
+            CoroutineKind::Coroutine => {
+                // Compute CoroutineState<yield_ty, return_ty>
+                let state_did = tcx.require_lang_item(LangItem::CoroutineState, None);
+                let state_adt_ref = tcx.adt_def(state_did);
+                let state_args = tcx.mk_args(&[yield_ty.into(), body.return_ty().into()]);
+                (state_adt_ref, state_args)
+            }
         };
         let ret_ty = Ty::new_adt(tcx, state_adt_ref, state_args);
 
@@ -1518,7 +1550,7 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
         // or Poll::Ready(x) and Poll::Pending respectively depending on `is_async_kind`.
         let mut transform = TransformVisitor {
             tcx,
-            is_async_kind,
+            coroutine_kind: body.coroutine_kind().unwrap(),
             state_adt_ref,
             state_args,
             remap,
diff --git a/compiler/rustc_ty_utils/src/instance.rs b/compiler/rustc_ty_utils/src/instance.rs
index 0e9d79c15c3e..c0fe13ac9965 100644
--- a/compiler/rustc_ty_utils/src/instance.rs
+++ b/compiler/rustc_ty_utils/src/instance.rs
@@ -258,6 +258,19 @@ fn resolve_associated_item<'tcx>(
                     debug_assert!(tcx.defaultness(trait_item_id).has_value());
                     Some(Instance::new(trait_item_id, rcvr_args))
                 }
+            } else if Some(trait_ref.def_id) == lang_items.iterator_trait() {
+                let ty::Coroutine(coroutine_def_id, args, _) = *rcvr_args.type_at(0).kind() else {
+                    bug!()
+                };
+                if Some(trait_item_id) == tcx.lang_items().next_fn() {
+                    // `Iterator::next` is generated by the compiler.
+                    Some(Instance { def: ty::InstanceDef::Item(coroutine_def_id), args })
+                } else {
+                    // All other methods are default methods of the `Iterator` trait.
+                    // (this assumes that `ImplSource::Builtin` is only used for methods on `Iterator`)
+                    debug_assert!(tcx.defaultness(trait_item_id).has_value());
+                    Some(Instance::new(trait_item_id, rcvr_args))
+                }
             } else if Some(trait_ref.def_id) == lang_items.gen_trait() {
                 let ty::Coroutine(coroutine_def_id, args, _) = *rcvr_args.type_at(0).kind() else {
                     bug!()
diff --git a/tests/ui/coroutine/gen_block_iterate.rs b/tests/ui/coroutine/gen_block_iterate.rs
new file mode 100644
index 000000000000..269327237799
--- /dev/null
+++ b/tests/ui/coroutine/gen_block_iterate.rs
@@ -0,0 +1,18 @@
+// revisions: next old
+//compile-flags: --edition 2024 -Zunstable-options
+//[next] compile-flags: -Ztrait-solver=next
+// run-pass
+#![feature(coroutines)]
+
+fn foo() -> impl Iterator<Item = u32> {
+    gen { yield 42; for x in 3..6 { yield x } }
+}
+
+fn main() {
+    let mut iter = foo();
+    assert_eq!(iter.next(), Some(42));
+    assert_eq!(iter.next(), Some(3));
+    assert_eq!(iter.next(), Some(4));
+    assert_eq!(iter.next(), Some(5));
+    assert_eq!(iter.next(), None);
+}

From 638d2d6fc109620a45d15b22e7cb21292de6e45b Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Mon, 23 Oct 2023 11:34:27 +0000
Subject: [PATCH 142/297] Feature gate `gen` blocks, even in 2024 edition

---
 compiler/rustc_ast_passes/src/feature_gate.rs |  7 ++++-
 compiler/rustc_feature/src/unstable.rs        |  2 ++
 compiler/rustc_parse/src/parser/expr.rs       |  3 +-
 compiler/rustc_span/src/symbol.rs             |  2 ++
 tests/ui/coroutine/gen_block.rs               |  2 +-
 tests/ui/coroutine/gen_block_is_coro.rs       |  2 +-
 tests/ui/coroutine/gen_block_is_iter.rs       |  2 +-
 tests/ui/coroutine/gen_block_is_no_future.rs  |  2 +-
 tests/ui/coroutine/gen_block_iterate.rs       |  2 +-
 .../feature-gate-coroutines.e2024.stderr      |  9 ++++++
 ...rr => feature-gate-coroutines.none.stderr} | 19 +++++++++----
 .../feature-gates/feature-gate-coroutines.rs  | 12 ++++++--
 .../feature-gate-gen_blocks.e2024.stderr      | 28 +++++++++++++++++++
 .../feature-gate-gen_blocks.none.stderr       |  9 ++++++
 .../feature-gates/feature-gate-gen_blocks.rs  | 15 ++++++++++
 15 files changed, 101 insertions(+), 15 deletions(-)
 create mode 100644 tests/ui/feature-gates/feature-gate-coroutines.e2024.stderr
 rename tests/ui/feature-gates/{feature-gate-coroutines.stderr => feature-gate-coroutines.none.stderr} (65%)
 create mode 100644 tests/ui/feature-gates/feature-gate-gen_blocks.e2024.stderr
 create mode 100644 tests/ui/feature-gates/feature-gate-gen_blocks.none.stderr
 create mode 100644 tests/ui/feature-gates/feature-gate-gen_blocks.rs

diff --git a/compiler/rustc_ast_passes/src/feature_gate.rs b/compiler/rustc_ast_passes/src/feature_gate.rs
index 34532967d9e2..a1bd2679137d 100644
--- a/compiler/rustc_ast_passes/src/feature_gate.rs
+++ b/compiler/rustc_ast_passes/src/feature_gate.rs
@@ -554,7 +554,12 @@ pub fn check_crate(krate: &ast::Crate, sess: &Session, features: &Features) {
         "consider removing `for<...>`"
     );
     gate_all!(more_qualified_paths, "usage of qualified paths in this context is experimental");
-    gate_all!(coroutines, "yield syntax is experimental");
+    for &span in spans.get(&sym::yield_expr).iter().copied().flatten() {
+        if !span.at_least_rust_2024() {
+            gate_feature_post!(&visitor, coroutines, span, "yield syntax is experimental");
+        }
+    }
+    gate_all!(gen_blocks, "gen blocks are experimental");
     gate_all!(raw_ref_op, "raw address of syntax is experimental");
     gate_all!(const_trait_impl, "const trait impls are experimental");
     gate_all!(
diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs
index 695de54eefa0..72100863bb52 100644
--- a/compiler/rustc_feature/src/unstable.rs
+++ b/compiler/rustc_feature/src/unstable.rs
@@ -456,6 +456,8 @@ declare_features! (
     (unstable, ffi_returns_twice, "1.34.0", Some(58314), None),
     /// Allows using `#[repr(align(...))]` on function items
     (unstable, fn_align, "1.53.0", Some(82232), None),
+    /// Allows defining gen blocks and `gen fn`.
+    (unstable, gen_blocks, "CURRENT_RUSTC_VERSION", Some(117078), None),
     /// Infer generic args for both consts and types.
     (unstable, generic_arg_infer, "1.55.0", Some(85077), None),
     /// An extension to the `generic_associated_types` feature, allowing incomplete features.
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index a1b04105a4cb..c6c7f025418e 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -1850,7 +1850,7 @@ impl<'a> Parser<'a> {
         let lo = self.prev_token.span;
         let kind = ExprKind::Yield(self.parse_expr_opt()?);
         let span = lo.to(self.prev_token.span);
-        self.sess.gated_spans.gate(sym::coroutines, span);
+        self.sess.gated_spans.gate(sym::yield_expr, span);
         let expr = self.mk_expr(span, kind);
         self.maybe_recover_from_bad_qpath(expr)
     }
@@ -3075,6 +3075,7 @@ impl<'a> Parser<'a> {
             GenBlockKind::Async
         } else {
             assert!(self.eat_keyword(kw::Gen));
+            self.sess.gated_spans.gate(sym::gen_blocks, lo.to(self.token.span));
             GenBlockKind::Gen
         };
         let capture_clause = self.parse_capture_clause()?;
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 0d21c19e5198..ba362c059657 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -819,6 +819,7 @@ symbols! {
         future_trait,
         gdb_script_file,
         ge,
+        gen_blocks,
         gen_future,
         gen_kill,
         generator_clone,
@@ -1780,6 +1781,7 @@ symbols! {
         xmm_reg,
         yeet_desugar_details,
         yeet_expr,
+        yield_expr,
         ymm_reg,
         zmm_reg,
     }
diff --git a/tests/ui/coroutine/gen_block.rs b/tests/ui/coroutine/gen_block.rs
index fa5027682362..c09fa2ae9be8 100644
--- a/tests/ui/coroutine/gen_block.rs
+++ b/tests/ui/coroutine/gen_block.rs
@@ -1,6 +1,6 @@
 // revisions: e2024 none
 //[e2024] compile-flags: --edition 2024 -Zunstable-options
-#![cfg_attr(e2024, feature(coroutines))]
+#![cfg_attr(e2024, feature(gen_blocks))]
 
 fn main() {
     let x = gen {};
diff --git a/tests/ui/coroutine/gen_block_is_coro.rs b/tests/ui/coroutine/gen_block_is_coro.rs
index 814d5843a1b6..c66ccefba85e 100644
--- a/tests/ui/coroutine/gen_block_is_coro.rs
+++ b/tests/ui/coroutine/gen_block_is_coro.rs
@@ -1,5 +1,5 @@
 //compile-flags: --edition 2024 -Zunstable-options
-#![feature(coroutines, coroutine_trait)]
+#![feature(coroutines, coroutine_trait, gen_blocks)]
 
 use std::ops::Coroutine;
 
diff --git a/tests/ui/coroutine/gen_block_is_iter.rs b/tests/ui/coroutine/gen_block_is_iter.rs
index 16d5813c69dc..92625cf7c284 100644
--- a/tests/ui/coroutine/gen_block_is_iter.rs
+++ b/tests/ui/coroutine/gen_block_is_iter.rs
@@ -2,7 +2,7 @@
 //compile-flags: --edition 2024 -Zunstable-options
 //[next] compile-flags: -Ztrait-solver=next
 // check-pass
-#![feature(coroutines)]
+#![feature(gen_blocks)]
 
 fn foo() -> impl Iterator<Item = u32> {
     gen { yield 42 }
diff --git a/tests/ui/coroutine/gen_block_is_no_future.rs b/tests/ui/coroutine/gen_block_is_no_future.rs
index 8927544a7b98..947665197387 100644
--- a/tests/ui/coroutine/gen_block_is_no_future.rs
+++ b/tests/ui/coroutine/gen_block_is_no_future.rs
@@ -1,5 +1,5 @@
 //compile-flags: --edition 2024 -Zunstable-options
-#![feature(coroutines)]
+#![feature(gen_blocks)]
 
 fn foo() -> impl std::future::Future { //~ ERROR is not a future
     gen { yield 42 }
diff --git a/tests/ui/coroutine/gen_block_iterate.rs b/tests/ui/coroutine/gen_block_iterate.rs
index 269327237799..04c44d58e524 100644
--- a/tests/ui/coroutine/gen_block_iterate.rs
+++ b/tests/ui/coroutine/gen_block_iterate.rs
@@ -2,7 +2,7 @@
 //compile-flags: --edition 2024 -Zunstable-options
 //[next] compile-flags: -Ztrait-solver=next
 // run-pass
-#![feature(coroutines)]
+#![feature(gen_blocks)]
 
 fn foo() -> impl Iterator<Item = u32> {
     gen { yield 42; for x in 3..6 { yield x } }
diff --git a/tests/ui/feature-gates/feature-gate-coroutines.e2024.stderr b/tests/ui/feature-gates/feature-gate-coroutines.e2024.stderr
new file mode 100644
index 000000000000..595b42176b10
--- /dev/null
+++ b/tests/ui/feature-gates/feature-gate-coroutines.e2024.stderr
@@ -0,0 +1,9 @@
+error[E0627]: yield expression outside of coroutine literal
+  --> $DIR/feature-gate-coroutines.rs:5:5
+   |
+LL |     yield true;
+   |     ^^^^^^^^^^
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0627`.
diff --git a/tests/ui/feature-gates/feature-gate-coroutines.stderr b/tests/ui/feature-gates/feature-gate-coroutines.none.stderr
similarity index 65%
rename from tests/ui/feature-gates/feature-gate-coroutines.stderr
rename to tests/ui/feature-gates/feature-gate-coroutines.none.stderr
index dd5616439011..680bd1f28788 100644
--- a/tests/ui/feature-gates/feature-gate-coroutines.stderr
+++ b/tests/ui/feature-gates/feature-gate-coroutines.none.stderr
@@ -1,5 +1,5 @@
 error[E0658]: yield syntax is experimental
-  --> $DIR/feature-gate-coroutines.rs:2:5
+  --> $DIR/feature-gate-coroutines.rs:5:5
    |
 LL |     yield true;
    |     ^^^^^^^^^^
@@ -8,7 +8,16 @@ LL |     yield true;
    = help: add `#![feature(coroutines)]` to the crate attributes to enable
 
 error[E0658]: yield syntax is experimental
-  --> $DIR/feature-gate-coroutines.rs:8:5
+  --> $DIR/feature-gate-coroutines.rs:8:16
+   |
+LL |     let _ = || yield true;
+   |                ^^^^^^^^^^
+   |
+   = note: see issue #43122 <https://github.com/rust-lang/rust/issues/43122> for more information
+   = help: add `#![feature(coroutines)]` to the crate attributes to enable
+
+error[E0658]: yield syntax is experimental
+  --> $DIR/feature-gate-coroutines.rs:14:5
    |
 LL |     yield;
    |     ^^^^^
@@ -17,7 +26,7 @@ LL |     yield;
    = help: add `#![feature(coroutines)]` to the crate attributes to enable
 
 error[E0658]: yield syntax is experimental
-  --> $DIR/feature-gate-coroutines.rs:9:5
+  --> $DIR/feature-gate-coroutines.rs:15:5
    |
 LL |     yield 0;
    |     ^^^^^^^
@@ -26,12 +35,12 @@ LL |     yield 0;
    = help: add `#![feature(coroutines)]` to the crate attributes to enable
 
 error[E0627]: yield expression outside of coroutine literal
-  --> $DIR/feature-gate-coroutines.rs:2:5
+  --> $DIR/feature-gate-coroutines.rs:5:5
    |
 LL |     yield true;
    |     ^^^^^^^^^^
 
-error: aborting due to 4 previous errors
+error: aborting due to 5 previous errors
 
 Some errors have detailed explanations: E0627, E0658.
 For more information about an error, try `rustc --explain E0627`.
diff --git a/tests/ui/feature-gates/feature-gate-coroutines.rs b/tests/ui/feature-gates/feature-gate-coroutines.rs
index c3c5aec88248..552435491758 100644
--- a/tests/ui/feature-gates/feature-gate-coroutines.rs
+++ b/tests/ui/feature-gates/feature-gate-coroutines.rs
@@ -1,10 +1,16 @@
+// revisions: e2024 none
+//[e2024] compile-flags: --edition 2024 -Zunstable-options
+
 fn main() {
-    yield true; //~ ERROR yield syntax is experimental
+    yield true; //[none]~ ERROR yield syntax is experimental
                 //~^ ERROR yield expression outside of coroutine literal
+
+    let _ = || yield true; //[none]~ ERROR yield syntax is experimental
 }
 
 #[cfg(FALSE)]
 fn foo() {
-    yield; //~ ERROR yield syntax is experimental
-    yield 0; //~ ERROR yield syntax is experimental
+    // Ok in 2024 edition
+    yield; //[none]~ ERROR yield syntax is experimental
+    yield 0; //[none]~ ERROR yield syntax is experimental
 }
diff --git a/tests/ui/feature-gates/feature-gate-gen_blocks.e2024.stderr b/tests/ui/feature-gates/feature-gate-gen_blocks.e2024.stderr
new file mode 100644
index 000000000000..1462c41e957b
--- /dev/null
+++ b/tests/ui/feature-gates/feature-gate-gen_blocks.e2024.stderr
@@ -0,0 +1,28 @@
+error[E0658]: gen blocks are experimental
+  --> $DIR/feature-gate-gen_blocks.rs:5:5
+   |
+LL |     gen {};
+   |     ^^^^^
+   |
+   = note: see issue #117078 <https://github.com/rust-lang/rust/issues/117078> for more information
+   = help: add `#![feature(gen_blocks)]` to the crate attributes to enable
+
+error[E0658]: gen blocks are experimental
+  --> $DIR/feature-gate-gen_blocks.rs:13:5
+   |
+LL |     gen {};
+   |     ^^^^^
+   |
+   = note: see issue #117078 <https://github.com/rust-lang/rust/issues/117078> for more information
+   = help: add `#![feature(gen_blocks)]` to the crate attributes to enable
+
+error[E0282]: type annotations needed
+  --> $DIR/feature-gate-gen_blocks.rs:5:9
+   |
+LL |     gen {};
+   |         ^^ cannot infer type
+
+error: aborting due to 3 previous errors
+
+Some errors have detailed explanations: E0282, E0658.
+For more information about an error, try `rustc --explain E0282`.
diff --git a/tests/ui/feature-gates/feature-gate-gen_blocks.none.stderr b/tests/ui/feature-gates/feature-gate-gen_blocks.none.stderr
new file mode 100644
index 000000000000..b448c35e8462
--- /dev/null
+++ b/tests/ui/feature-gates/feature-gate-gen_blocks.none.stderr
@@ -0,0 +1,9 @@
+error[E0422]: cannot find struct, variant or union type `gen` in this scope
+  --> $DIR/feature-gate-gen_blocks.rs:5:5
+   |
+LL |     gen {};
+   |     ^^^ not found in this scope
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0422`.
diff --git a/tests/ui/feature-gates/feature-gate-gen_blocks.rs b/tests/ui/feature-gates/feature-gate-gen_blocks.rs
new file mode 100644
index 000000000000..e2e1574a36a0
--- /dev/null
+++ b/tests/ui/feature-gates/feature-gate-gen_blocks.rs
@@ -0,0 +1,15 @@
+// revisions: e2024 none
+//[e2024] compile-flags: --edition 2024 -Zunstable-options
+
+fn main() {
+    gen {};
+    //[none]~^ ERROR: cannot find struct, variant or union type `gen`
+    //[e2024]~^^ ERROR: gen blocks are experimental
+    //[e2024]~| ERROR: type annotations needed
+}
+
+#[cfg(FALSE)]
+fn foo() {
+    gen {};
+    //[e2024]~^ ERROR: gen blocks are experimental
+}

From 2e5b36741bdfd0ecccefac0c8b6d4fa2cb5b0770 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Mon, 23 Oct 2023 14:00:34 +0000
Subject: [PATCH 143/297] Demonstrate issue with `yield` checks

---
 tests/ui/coroutine/gen_block.none.stderr | 14 ++++++++++++--
 tests/ui/coroutine/gen_block.rs          |  3 +++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/tests/ui/coroutine/gen_block.none.stderr b/tests/ui/coroutine/gen_block.none.stderr
index d5ed28c5317b..f1dd125efba7 100644
--- a/tests/ui/coroutine/gen_block.none.stderr
+++ b/tests/ui/coroutine/gen_block.none.stderr
@@ -24,6 +24,16 @@ error[E0422]: cannot find struct, variant or union type `gen` in this scope
 LL |     gen {};
    |     ^^^ not found in this scope
 
-error: aborting due to 4 previous errors
+error[E0658]: yield syntax is experimental
+  --> $DIR/gen_block.rs:14:16
+   |
+LL |     let _ = || yield true;
+   |                ^^^^^^^^^^
+   |
+   = note: see issue #43122 <https://github.com/rust-lang/rust/issues/43122> for more information
+   = help: add `#![feature(coroutines)]` to the crate attributes to enable
 
-For more information about this error, try `rustc --explain E0422`.
+error: aborting due to 5 previous errors
+
+Some errors have detailed explanations: E0422, E0658.
+For more information about an error, try `rustc --explain E0422`.
diff --git a/tests/ui/coroutine/gen_block.rs b/tests/ui/coroutine/gen_block.rs
index c09fa2ae9be8..041dcf0c02af 100644
--- a/tests/ui/coroutine/gen_block.rs
+++ b/tests/ui/coroutine/gen_block.rs
@@ -11,4 +11,7 @@ fn main() {
     //[none]~| ERROR: cannot find
     gen {};
     //[none]~^ ERROR: cannot find
+
+    // FIXME(gen_blocks): should also error in 2024 edition
+    let _ = || yield true; //[none]~ ERROR yield syntax is experimental
 }

From cece90c65f280033790a228517dcb5a3c69805da Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Mon, 23 Oct 2023 14:29:00 +0000
Subject: [PATCH 144/297] Feature gate coroutine `yield` usage

---
 compiler/rustc_ast_lowering/src/expr.rs       | 14 ++++++++--
 tests/ui/coroutine/gen_block.e2024.stderr     | 14 ++++++++--
 tests/ui/coroutine/gen_block.none.stderr      | 14 ++++++++--
 tests/ui/coroutine/gen_block.rs               |  2 +-
 .../feature-gate-coroutines.e2024.stderr      | 23 +++++++++++++--
 .../feature-gate-coroutines.none.stderr       | 28 ++++++++++++++++---
 .../feature-gates/feature-gate-coroutines.rs  |  6 ++--
 7 files changed, 86 insertions(+), 15 deletions(-)

diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index cce0b44fd98e..5abb0ba722e2 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -1504,12 +1504,22 @@ impl<'hir> LoweringContext<'_, 'hir> {
 
     fn lower_expr_yield(&mut self, span: Span, opt_expr: Option<&Expr>) -> hir::ExprKind<'hir> {
         match self.coroutine_kind {
-            Some(hir::CoroutineKind::Coroutine) => {}
             Some(hir::CoroutineKind::Gen(_)) => {}
             Some(hir::CoroutineKind::Async(_)) => {
                 self.tcx.sess.emit_err(AsyncCoroutinesNotSupported { span });
             }
-            None => self.coroutine_kind = Some(hir::CoroutineKind::Coroutine),
+            Some(hir::CoroutineKind::Coroutine) | None => {
+                if !self.tcx.features().coroutines {
+                    rustc_session::parse::feature_err(
+                        &self.tcx.sess.parse_sess,
+                        sym::coroutines,
+                        span,
+                        "yield syntax is experimental",
+                    )
+                    .emit();
+                }
+                self.coroutine_kind = Some(hir::CoroutineKind::Coroutine)
+            }
         }
 
         let expr =
diff --git a/tests/ui/coroutine/gen_block.e2024.stderr b/tests/ui/coroutine/gen_block.e2024.stderr
index 28cf036b7e8c..f250e2f79c77 100644
--- a/tests/ui/coroutine/gen_block.e2024.stderr
+++ b/tests/ui/coroutine/gen_block.e2024.stderr
@@ -1,9 +1,19 @@
+error[E0658]: yield syntax is experimental
+  --> $DIR/gen_block.rs:15:16
+   |
+LL |     let _ = || yield true;
+   |                ^^^^^^^^^^
+   |
+   = note: see issue #43122 <https://github.com/rust-lang/rust/issues/43122> for more information
+   = help: add `#![feature(coroutines)]` to the crate attributes to enable
+
 error[E0282]: type annotations needed
   --> $DIR/gen_block.rs:6:17
    |
 LL |     let x = gen {};
    |                 ^^ cannot infer type
 
-error: aborting due to previous error
+error: aborting due to 2 previous errors
 
-For more information about this error, try `rustc --explain E0282`.
+Some errors have detailed explanations: E0282, E0658.
+For more information about an error, try `rustc --explain E0282`.
diff --git a/tests/ui/coroutine/gen_block.none.stderr b/tests/ui/coroutine/gen_block.none.stderr
index f1dd125efba7..012a8308c7f7 100644
--- a/tests/ui/coroutine/gen_block.none.stderr
+++ b/tests/ui/coroutine/gen_block.none.stderr
@@ -25,7 +25,7 @@ LL |     gen {};
    |     ^^^ not found in this scope
 
 error[E0658]: yield syntax is experimental
-  --> $DIR/gen_block.rs:14:16
+  --> $DIR/gen_block.rs:15:16
    |
 LL |     let _ = || yield true;
    |                ^^^^^^^^^^
@@ -33,7 +33,17 @@ LL |     let _ = || yield true;
    = note: see issue #43122 <https://github.com/rust-lang/rust/issues/43122> for more information
    = help: add `#![feature(coroutines)]` to the crate attributes to enable
 
-error: aborting due to 5 previous errors
+error[E0658]: yield syntax is experimental
+  --> $DIR/gen_block.rs:15:16
+   |
+LL |     let _ = || yield true;
+   |                ^^^^^^^^^^
+   |
+   = note: see issue #43122 <https://github.com/rust-lang/rust/issues/43122> for more information
+   = help: add `#![feature(coroutines)]` to the crate attributes to enable
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+
+error: aborting due to 6 previous errors
 
 Some errors have detailed explanations: E0422, E0658.
 For more information about an error, try `rustc --explain E0422`.
diff --git a/tests/ui/coroutine/gen_block.rs b/tests/ui/coroutine/gen_block.rs
index 041dcf0c02af..852c7c455a69 100644
--- a/tests/ui/coroutine/gen_block.rs
+++ b/tests/ui/coroutine/gen_block.rs
@@ -12,6 +12,6 @@ fn main() {
     gen {};
     //[none]~^ ERROR: cannot find
 
-    // FIXME(gen_blocks): should also error in 2024 edition
     let _ = || yield true; //[none]~ ERROR yield syntax is experimental
+    //~^ ERROR yield syntax is experimental
 }
diff --git a/tests/ui/feature-gates/feature-gate-coroutines.e2024.stderr b/tests/ui/feature-gates/feature-gate-coroutines.e2024.stderr
index 595b42176b10..2e529236ad8f 100644
--- a/tests/ui/feature-gates/feature-gate-coroutines.e2024.stderr
+++ b/tests/ui/feature-gates/feature-gate-coroutines.e2024.stderr
@@ -1,9 +1,28 @@
+error[E0658]: yield syntax is experimental
+  --> $DIR/feature-gate-coroutines.rs:5:5
+   |
+LL |     yield true;
+   |     ^^^^^^^^^^
+   |
+   = note: see issue #43122 <https://github.com/rust-lang/rust/issues/43122> for more information
+   = help: add `#![feature(coroutines)]` to the crate attributes to enable
+
+error[E0658]: yield syntax is experimental
+  --> $DIR/feature-gate-coroutines.rs:9:16
+   |
+LL |     let _ = || yield true;
+   |                ^^^^^^^^^^
+   |
+   = note: see issue #43122 <https://github.com/rust-lang/rust/issues/43122> for more information
+   = help: add `#![feature(coroutines)]` to the crate attributes to enable
+
 error[E0627]: yield expression outside of coroutine literal
   --> $DIR/feature-gate-coroutines.rs:5:5
    |
 LL |     yield true;
    |     ^^^^^^^^^^
 
-error: aborting due to previous error
+error: aborting due to 3 previous errors
 
-For more information about this error, try `rustc --explain E0627`.
+Some errors have detailed explanations: E0627, E0658.
+For more information about an error, try `rustc --explain E0627`.
diff --git a/tests/ui/feature-gates/feature-gate-coroutines.none.stderr b/tests/ui/feature-gates/feature-gate-coroutines.none.stderr
index 680bd1f28788..ab24805e4677 100644
--- a/tests/ui/feature-gates/feature-gate-coroutines.none.stderr
+++ b/tests/ui/feature-gates/feature-gate-coroutines.none.stderr
@@ -8,7 +8,7 @@ LL |     yield true;
    = help: add `#![feature(coroutines)]` to the crate attributes to enable
 
 error[E0658]: yield syntax is experimental
-  --> $DIR/feature-gate-coroutines.rs:8:16
+  --> $DIR/feature-gate-coroutines.rs:9:16
    |
 LL |     let _ = || yield true;
    |                ^^^^^^^^^^
@@ -17,7 +17,7 @@ LL |     let _ = || yield true;
    = help: add `#![feature(coroutines)]` to the crate attributes to enable
 
 error[E0658]: yield syntax is experimental
-  --> $DIR/feature-gate-coroutines.rs:14:5
+  --> $DIR/feature-gate-coroutines.rs:16:5
    |
 LL |     yield;
    |     ^^^^^
@@ -26,7 +26,7 @@ LL |     yield;
    = help: add `#![feature(coroutines)]` to the crate attributes to enable
 
 error[E0658]: yield syntax is experimental
-  --> $DIR/feature-gate-coroutines.rs:15:5
+  --> $DIR/feature-gate-coroutines.rs:17:5
    |
 LL |     yield 0;
    |     ^^^^^^^
@@ -34,13 +34,33 @@ LL |     yield 0;
    = note: see issue #43122 <https://github.com/rust-lang/rust/issues/43122> for more information
    = help: add `#![feature(coroutines)]` to the crate attributes to enable
 
+error[E0658]: yield syntax is experimental
+  --> $DIR/feature-gate-coroutines.rs:5:5
+   |
+LL |     yield true;
+   |     ^^^^^^^^^^
+   |
+   = note: see issue #43122 <https://github.com/rust-lang/rust/issues/43122> for more information
+   = help: add `#![feature(coroutines)]` to the crate attributes to enable
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+
+error[E0658]: yield syntax is experimental
+  --> $DIR/feature-gate-coroutines.rs:9:16
+   |
+LL |     let _ = || yield true;
+   |                ^^^^^^^^^^
+   |
+   = note: see issue #43122 <https://github.com/rust-lang/rust/issues/43122> for more information
+   = help: add `#![feature(coroutines)]` to the crate attributes to enable
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+
 error[E0627]: yield expression outside of coroutine literal
   --> $DIR/feature-gate-coroutines.rs:5:5
    |
 LL |     yield true;
    |     ^^^^^^^^^^
 
-error: aborting due to 5 previous errors
+error: aborting due to 7 previous errors
 
 Some errors have detailed explanations: E0627, E0658.
 For more information about an error, try `rustc --explain E0627`.
diff --git a/tests/ui/feature-gates/feature-gate-coroutines.rs b/tests/ui/feature-gates/feature-gate-coroutines.rs
index 552435491758..53b58d486a80 100644
--- a/tests/ui/feature-gates/feature-gate-coroutines.rs
+++ b/tests/ui/feature-gates/feature-gate-coroutines.rs
@@ -2,10 +2,12 @@
 //[e2024] compile-flags: --edition 2024 -Zunstable-options
 
 fn main() {
-    yield true; //[none]~ ERROR yield syntax is experimental
+    yield true; //~ ERROR yield syntax is experimental
                 //~^ ERROR yield expression outside of coroutine literal
+                //[none]~^^ ERROR yield syntax is experimental
 
-    let _ = || yield true; //[none]~ ERROR yield syntax is experimental
+    let _ = || yield true; //~ ERROR yield syntax is experimental
+    //[none]~^ ERROR yield syntax is experimental
 }
 
 #[cfg(FALSE)]

From 4ac25faf9f1d7b2b2aaa76cb4b1da6688bc72dc5 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Tue, 24 Oct 2023 09:50:49 +0000
Subject: [PATCH 145/297] Handle `move` generators

---
 .../src/diagnostics/conflict_errors.rs        | 16 ++++++----
 compiler/rustc_parse/src/parser/expr.rs       | 19 +++++-------
 tests/ui/coroutine/gen_block_iterate.rs       | 15 ++++++++++
 tests/ui/coroutine/gen_block_iterate_fused.rs | 19 ++++++++++++
 tests/ui/coroutine/gen_block_move.fixed       | 17 +++++++++++
 tests/ui/coroutine/gen_block_move.rs          | 17 +++++++++++
 tests/ui/coroutine/gen_block_move.stderr      | 30 +++++++++++++++++++
 7 files changed, 117 insertions(+), 16 deletions(-)
 create mode 100644 tests/ui/coroutine/gen_block_iterate_fused.rs
 create mode 100644 tests/ui/coroutine/gen_block_move.fixed
 create mode 100644 tests/ui/coroutine/gen_block_move.rs
 create mode 100644 tests/ui/coroutine/gen_block_move.stderr

diff --git a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
index 9db7dec1cf8e..247200dcd269 100644
--- a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
+++ b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs
@@ -2491,11 +2491,17 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
 
         let (sugg_span, suggestion) = match tcx.sess.source_map().span_to_snippet(args_span) {
             Ok(string) => {
-                if string.starts_with("async ") {
-                    let pos = args_span.lo() + BytePos(6);
-                    (args_span.with_lo(pos).with_hi(pos), "move ")
-                } else if string.starts_with("async|") {
-                    let pos = args_span.lo() + BytePos(5);
+                let coro_prefix = if string.starts_with("async") {
+                    // `async` is 5 chars long. Not using `.len()` to avoid the cast from `usize` to `u32`
+                    Some(5)
+                } else if string.starts_with("gen") {
+                    // `gen` is 3 chars long
+                    Some(3)
+                } else {
+                    None
+                };
+                if let Some(n) = coro_prefix {
+                    let pos = args_span.lo() + BytePos(n);
                     (args_span.with_lo(pos).with_hi(pos), " move")
                 } else {
                     (args_span.shrink_to_lo(), "move ")
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index c6c7f025418e..0b5ec9b59ea8 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -1441,7 +1441,7 @@ impl<'a> Parser<'a> {
             } else if this.token.uninterpolated_span().at_least_rust_2018() {
                 // `Span:.at_least_rust_2018()` is somewhat expensive; don't get it repeatedly.
                 if this.check_keyword(kw::Async) {
-                    if this.is_async_block() {
+                    if this.is_gen_block(kw::Async) {
                         // Check for `async {` and `async move {`.
                         this.parse_gen_block()
                     } else {
@@ -1450,7 +1450,11 @@ impl<'a> Parser<'a> {
                 } else if this.eat_keyword(kw::Await) {
                     this.recover_incorrect_await_syntax(lo, this.prev_token.span)
                 } else if this.token.uninterpolated_span().at_least_rust_2024() {
-                    if this.is_gen_block() { this.parse_gen_block() } else { this.parse_expr_lit() }
+                    if this.is_gen_block(kw::Gen) {
+                        this.parse_gen_block()
+                    } else {
+                        this.parse_expr_lit()
+                    }
                 } else {
                     this.parse_expr_lit()
                 }
@@ -3061,13 +3065,6 @@ impl<'a> Parser<'a> {
             && self.token.uninterpolated_span().at_least_rust_2018()
     }
 
-    fn is_gen_block(&self) -> bool {
-        self.token.is_keyword(kw::Gen)
-            && self
-                .look_ahead(1, |t| *t == token::OpenDelim(Delimiter::Brace) || t.is_whole_block())
-            && self.token.uninterpolated_span().at_least_rust_2024()
-    }
-
     /// Parses an `async move? {...}` or `gen move? {...}` expression.
     fn parse_gen_block(&mut self) -> PResult<'a, P<Expr>> {
         let lo = self.token.span;
@@ -3084,8 +3081,8 @@ impl<'a> Parser<'a> {
         Ok(self.mk_expr_with_attrs(lo.to(self.prev_token.span), kind, attrs))
     }
 
-    fn is_async_block(&self) -> bool {
-        self.token.is_keyword(kw::Async)
+    fn is_gen_block(&self, kw: Symbol) -> bool {
+        self.token.is_keyword(kw)
             && ((
                 // `async move {`
                 self.is_keyword_ahead(1, &[kw::Move])
diff --git a/tests/ui/coroutine/gen_block_iterate.rs b/tests/ui/coroutine/gen_block_iterate.rs
index 04c44d58e524..131dd6879360 100644
--- a/tests/ui/coroutine/gen_block_iterate.rs
+++ b/tests/ui/coroutine/gen_block_iterate.rs
@@ -8,6 +8,16 @@ fn foo() -> impl Iterator<Item = u32> {
     gen { yield 42; for x in 3..6 { yield x } }
 }
 
+fn moved() -> impl Iterator<Item = u32> {
+    let mut x = "foo".to_string();
+    gen move {
+        yield 42;
+        if x == "foo" { return }
+        x.clear();
+        for x in 3..6 { yield x }
+    }
+}
+
 fn main() {
     let mut iter = foo();
     assert_eq!(iter.next(), Some(42));
@@ -15,4 +25,9 @@ fn main() {
     assert_eq!(iter.next(), Some(4));
     assert_eq!(iter.next(), Some(5));
     assert_eq!(iter.next(), None);
+
+    let mut iter = moved();
+    assert_eq!(iter.next(), Some(42));
+    assert_eq!(iter.next(), None);
+
 }
diff --git a/tests/ui/coroutine/gen_block_iterate_fused.rs b/tests/ui/coroutine/gen_block_iterate_fused.rs
new file mode 100644
index 000000000000..8ee6baf10602
--- /dev/null
+++ b/tests/ui/coroutine/gen_block_iterate_fused.rs
@@ -0,0 +1,19 @@
+// revisions: next old
+//compile-flags: --edition 2024 -Zunstable-options
+//[next] compile-flags: -Ztrait-solver=next
+// run-fail
+#![feature(gen_blocks)]
+
+fn foo() -> impl Iterator<Item = u32> {
+    gen { yield 42; for x in 3..6 { yield x } }
+}
+
+fn main() {
+    let mut iter = foo();
+    assert_eq!(iter.next(), Some(42));
+    assert_eq!(iter.next(), Some(3));
+    assert_eq!(iter.next(), Some(4));
+    assert_eq!(iter.next(), Some(5));
+    assert_eq!(iter.next(), None);
+    assert_eq!(iter.next(), None);
+}
diff --git a/tests/ui/coroutine/gen_block_move.fixed b/tests/ui/coroutine/gen_block_move.fixed
new file mode 100644
index 000000000000..5c6c8062322d
--- /dev/null
+++ b/tests/ui/coroutine/gen_block_move.fixed
@@ -0,0 +1,17 @@
+// compile-flags: --edition 2024 -Zunstable-options
+// run-rustfix
+#![feature(gen_blocks)]
+
+fn moved() -> impl Iterator<Item = u32> {
+    let mut x = "foo".to_string();
+    gen move { //~ ERROR: gen block may outlive the current function
+        yield 42;
+        if x == "foo" { return }
+        x.clear();
+        for x in 3..6 { yield x }
+    }
+}
+
+fn main() {
+    for _ in moved() {}
+}
diff --git a/tests/ui/coroutine/gen_block_move.rs b/tests/ui/coroutine/gen_block_move.rs
new file mode 100644
index 000000000000..abbf8132476c
--- /dev/null
+++ b/tests/ui/coroutine/gen_block_move.rs
@@ -0,0 +1,17 @@
+// compile-flags: --edition 2024 -Zunstable-options
+// run-rustfix
+#![feature(gen_blocks)]
+
+fn moved() -> impl Iterator<Item = u32> {
+    let mut x = "foo".to_string();
+    gen { //~ ERROR: gen block may outlive the current function
+        yield 42;
+        if x == "foo" { return }
+        x.clear();
+        for x in 3..6 { yield x }
+    }
+}
+
+fn main() {
+    for _ in moved() {}
+}
diff --git a/tests/ui/coroutine/gen_block_move.stderr b/tests/ui/coroutine/gen_block_move.stderr
new file mode 100644
index 000000000000..b93ac65f5e7d
--- /dev/null
+++ b/tests/ui/coroutine/gen_block_move.stderr
@@ -0,0 +1,30 @@
+error[E0373]: gen block may outlive the current function, but it borrows `x`, which is owned by the current function
+  --> $DIR/gen_block_move.rs:7:5
+   |
+LL | /     gen {
+LL | |         yield 42;
+LL | |         if x == "foo" { return }
+LL | |         x.clear();
+   | |         - `x` is borrowed here
+LL | |         for x in 3..6 { yield x }
+LL | |     }
+   | |_____^ may outlive borrowed value `x`
+   |
+note: gen block is returned here
+  --> $DIR/gen_block_move.rs:7:5
+   |
+LL | /     gen {
+LL | |         yield 42;
+LL | |         if x == "foo" { return }
+LL | |         x.clear();
+LL | |         for x in 3..6 { yield x }
+LL | |     }
+   | |_____^
+help: to force the gen block to take ownership of `x` (and any other referenced variables), use the `move` keyword
+   |
+LL |     gen move {
+   |         ++++
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0373`.

From 62237440782574b47f2a325c804e4b8cf77569e7 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Tue, 24 Oct 2023 10:03:40 +0000
Subject: [PATCH 146/297] Prevent generators from being movable

---
 compiler/rustc_ast_lowering/src/expr.rs         |  2 +-
 .../ui/coroutine/self_referential_gen_block.rs  | 17 +++++++++++++++++
 .../coroutine/self_referential_gen_block.stderr | 11 +++++++++++
 3 files changed, 29 insertions(+), 1 deletion(-)
 create mode 100644 tests/ui/coroutine/self_referential_gen_block.rs
 create mode 100644 tests/ui/coroutine/self_referential_gen_block.stderr

diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index 5abb0ba722e2..d6fa74dc3e6f 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -715,7 +715,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             body,
             fn_decl_span: self.lower_span(span),
             fn_arg_span: None,
-            movability: Some(hir::Movability::Static),
+            movability: Some(Movability::Movable),
             constness: hir::Constness::NotConst,
         }))
     }
diff --git a/tests/ui/coroutine/self_referential_gen_block.rs b/tests/ui/coroutine/self_referential_gen_block.rs
new file mode 100644
index 000000000000..538d436ed144
--- /dev/null
+++ b/tests/ui/coroutine/self_referential_gen_block.rs
@@ -0,0 +1,17 @@
+// compile-flags: --edition 2024 -Zunstable-options
+#![feature(gen_blocks)]
+//! This test checks that we don't allow self-referential generators
+
+fn main() {
+    let mut x = {
+        let mut x = gen {
+            let y = 42;
+            let z = &y; //~ ERROR: borrow may still be in use when coroutine yields
+            yield 43;
+            panic!("{z}");
+        };
+        x.next();
+        Box::new(x)
+    };
+    x.next();
+}
diff --git a/tests/ui/coroutine/self_referential_gen_block.stderr b/tests/ui/coroutine/self_referential_gen_block.stderr
new file mode 100644
index 000000000000..f9cadf8eb04c
--- /dev/null
+++ b/tests/ui/coroutine/self_referential_gen_block.stderr
@@ -0,0 +1,11 @@
+error[E0626]: borrow may still be in use when coroutine yields
+  --> $DIR/self_referential_gen_block.rs:9:21
+   |
+LL |             let z = &y;
+   |                     ^^
+LL |             yield 43;
+   |             -------- possible yield occurs here
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0626`.

From bb90c4bf3593d5edf61f16c37ca9c285af0b47f4 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Tue, 24 Oct 2023 10:12:49 +0000
Subject: [PATCH 147/297] Use targetted diagnostic for borrow across yield
 error

---
 compiler/rustc_borrowck/src/borrowck_errors.rs       | 3 ++-
 tests/ui/coroutine/self_referential_gen_block.rs     | 2 +-
 tests/ui/coroutine/self_referential_gen_block.stderr | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_borrowck/src/borrowck_errors.rs b/compiler/rustc_borrowck/src/borrowck_errors.rs
index 2ceec1b5658d..6731ef123066 100644
--- a/compiler/rustc_borrowck/src/borrowck_errors.rs
+++ b/compiler/rustc_borrowck/src/borrowck_errors.rs
@@ -373,11 +373,12 @@ impl<'cx, 'tcx> crate::MirBorrowckCtxt<'cx, 'tcx> {
         span: Span,
         yield_span: Span,
     ) -> DiagnosticBuilder<'tcx, ErrorGuaranteed> {
+        let coroutine_kind = self.body.coroutine.as_ref().unwrap().coroutine_kind;
         let mut err = struct_span_err!(
             self,
             span,
             E0626,
-            "borrow may still be in use when coroutine yields",
+            "borrow may still be in use when {coroutine_kind:#} yields",
         );
         err.span_label(yield_span, "possible yield occurs here");
         err
diff --git a/tests/ui/coroutine/self_referential_gen_block.rs b/tests/ui/coroutine/self_referential_gen_block.rs
index 538d436ed144..14daa2e9c357 100644
--- a/tests/ui/coroutine/self_referential_gen_block.rs
+++ b/tests/ui/coroutine/self_referential_gen_block.rs
@@ -6,7 +6,7 @@ fn main() {
     let mut x = {
         let mut x = gen {
             let y = 42;
-            let z = &y; //~ ERROR: borrow may still be in use when coroutine yields
+            let z = &y; //~ ERROR: borrow may still be in use when `gen` block yields
             yield 43;
             panic!("{z}");
         };
diff --git a/tests/ui/coroutine/self_referential_gen_block.stderr b/tests/ui/coroutine/self_referential_gen_block.stderr
index f9cadf8eb04c..586f53df8f26 100644
--- a/tests/ui/coroutine/self_referential_gen_block.stderr
+++ b/tests/ui/coroutine/self_referential_gen_block.stderr
@@ -1,4 +1,4 @@
-error[E0626]: borrow may still be in use when coroutine yields
+error[E0626]: borrow may still be in use when `gen` block yields
   --> $DIR/self_referential_gen_block.rs:9:21
    |
 LL |             let z = &y;

From b8bfd08999babd4b70822c445eb7b016e1f52fab Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Fri, 27 Oct 2023 12:12:25 +0000
Subject: [PATCH 148/297] Rename `RibKind::ClosureOrAsync` to reflect how it is
 actually used

---
 compiler/rustc_resolve/src/ident.rs |  6 +++---
 compiler/rustc_resolve/src/late.rs  | 18 +++++++++---------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/compiler/rustc_resolve/src/ident.rs b/compiler/rustc_resolve/src/ident.rs
index f2fd4b0bf60c..1a50bd5ec986 100644
--- a/compiler/rustc_resolve/src/ident.rs
+++ b/compiler/rustc_resolve/src/ident.rs
@@ -1083,7 +1083,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                 for rib in ribs {
                     match rib.kind {
                         RibKind::Normal
-                        | RibKind::ClosureOrAsync
+                        | RibKind::FnOrCoroutine
                         | RibKind::Module(..)
                         | RibKind::MacroDefinition(..)
                         | RibKind::ForwardGenericParamBan => {
@@ -1156,7 +1156,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                 for rib in ribs {
                     let has_generic_params: HasGenericParams = match rib.kind {
                         RibKind::Normal
-                        | RibKind::ClosureOrAsync
+                        | RibKind::FnOrCoroutine
                         | RibKind::Module(..)
                         | RibKind::MacroDefinition(..)
                         | RibKind::InlineAsmSym
@@ -1240,7 +1240,7 @@ impl<'a, 'tcx> Resolver<'a, 'tcx> {
                 for rib in ribs {
                     let has_generic_params = match rib.kind {
                         RibKind::Normal
-                        | RibKind::ClosureOrAsync
+                        | RibKind::FnOrCoroutine
                         | RibKind::Module(..)
                         | RibKind::MacroDefinition(..)
                         | RibKind::InlineAsmSym
diff --git a/compiler/rustc_resolve/src/late.rs b/compiler/rustc_resolve/src/late.rs
index 83f9f3a4210b..3be962dab90b 100644
--- a/compiler/rustc_resolve/src/late.rs
+++ b/compiler/rustc_resolve/src/late.rs
@@ -177,8 +177,8 @@ pub(crate) enum RibKind<'a> {
     /// upvars).
     AssocItem,
 
-    /// We passed through a closure. Disallow labels.
-    ClosureOrAsync,
+    /// We passed through a function, closure or coroutine signature. Disallow labels.
+    FnOrCoroutine,
 
     /// We passed through an item scope. Disallow upvars.
     Item(HasGenericParams),
@@ -215,7 +215,7 @@ impl RibKind<'_> {
     pub(crate) fn contains_params(&self) -> bool {
         match self {
             RibKind::Normal
-            | RibKind::ClosureOrAsync
+            | RibKind::FnOrCoroutine
             | RibKind::ConstantItem(..)
             | RibKind::Module(_)
             | RibKind::MacroDefinition(_)
@@ -231,7 +231,7 @@ impl RibKind<'_> {
             RibKind::Normal | RibKind::MacroDefinition(..) => false,
 
             RibKind::AssocItem
-            | RibKind::ClosureOrAsync
+            | RibKind::FnOrCoroutine
             | RibKind::Item(..)
             | RibKind::ConstantItem(..)
             | RibKind::Module(..)
@@ -924,9 +924,9 @@ impl<'a: 'ast, 'ast, 'tcx> Visitor<'ast> for LateResolutionVisitor<'a, '_, 'ast,
         debug!("(resolving function) entering function");
 
         // Create a value rib for the function.
-        self.with_rib(ValueNS, RibKind::ClosureOrAsync, |this| {
+        self.with_rib(ValueNS, RibKind::FnOrCoroutine, |this| {
             // Create a label rib for the function.
-            this.with_label_rib(RibKind::ClosureOrAsync, |this| {
+            this.with_label_rib(RibKind::FnOrCoroutine, |this| {
                 match fn_kind {
                     FnKind::Fn(_, _, sig, _, generics, body) => {
                         this.visit_generics(generics);
@@ -4287,7 +4287,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
                 ..
             }) => {
                 self.with_rib(ValueNS, RibKind::Normal, |this| {
-                    this.with_label_rib(RibKind::ClosureOrAsync, |this| {
+                    this.with_label_rib(RibKind::FnOrCoroutine, |this| {
                         // Resolve arguments:
                         this.resolve_params(&fn_decl.inputs);
                         // No need to resolve return type --
@@ -4304,7 +4304,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
                     })
                 });
             }
-            // For closures, ClosureOrAsyncRibKind is added in visit_fn
+            // For closures, RibKind::FnOrCoroutine is added in visit_fn
             ExprKind::Closure(box ast::Closure {
                 binder: ClosureBinder::For { ref generic_params, span },
                 ..
@@ -4322,7 +4322,7 @@ impl<'a: 'ast, 'b, 'ast, 'tcx> LateResolutionVisitor<'a, 'b, 'ast, 'tcx> {
             }
             ExprKind::Closure(..) => visit::walk_expr(self, expr),
             ExprKind::Gen(..) => {
-                self.with_label_rib(RibKind::ClosureOrAsync, |this| visit::walk_expr(this, expr));
+                self.with_label_rib(RibKind::FnOrCoroutine, |this| visit::walk_expr(this, expr));
             }
             ExprKind::Repeat(ref elem, ref ct) => {
                 self.visit_expr(elem);

From eb66d10cc3a6947cad6b4b169ed86b8c07f464d3 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Fri, 27 Oct 2023 15:20:44 +0000
Subject: [PATCH 149/297] Fuse `gen` blocks

---
 compiler/rustc_middle/src/mir/terminator.rs   |  5 +-
 compiler/rustc_mir_transform/src/coroutine.rs | 63 ++++++++++++++-----
 tests/ui/coroutine/gen_block_iterate.rs       |  2 +
 tests/ui/coroutine/gen_block_iterate_fused.rs | 19 ------
 4 files changed, 54 insertions(+), 35 deletions(-)
 delete mode 100644 tests/ui/coroutine/gen_block_iterate_fused.rs

diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs
index 06e63b0f3d99..c2aa015f4b7e 100644
--- a/compiler/rustc_middle/src/mir/terminator.rs
+++ b/compiler/rustc_middle/src/mir/terminator.rs
@@ -243,8 +243,9 @@ impl<O> AssertKind<O> {
             DivisionByZero(_) => middle_assert_divide_by_zero,
             RemainderByZero(_) => middle_assert_remainder_by_zero,
             ResumedAfterReturn(CoroutineKind::Async(_)) => middle_assert_async_resume_after_return,
-            // FIXME(gen_blocks): custom error message for `gen` blocks
-            ResumedAfterReturn(CoroutineKind::Gen(_)) => middle_assert_async_resume_after_return,
+            ResumedAfterReturn(CoroutineKind::Gen(_)) => {
+                bug!("gen blocks can be resumed after they return and will keep returning `None`")
+            }
             ResumedAfterReturn(CoroutineKind::Coroutine) => {
                 middle_assert_coroutine_resume_after_return
             }
diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs
index 8fecff16a919..50d244d2831d 100644
--- a/compiler/rustc_mir_transform/src/coroutine.rs
+++ b/compiler/rustc_mir_transform/src/coroutine.rs
@@ -249,6 +249,47 @@ struct TransformVisitor<'tcx> {
 }
 
 impl<'tcx> TransformVisitor<'tcx> {
+    fn insert_none_ret_block(&self, body: &mut Body<'tcx>) -> BasicBlock {
+        let block = BasicBlock::new(body.basic_blocks.len());
+
+        let source_info = SourceInfo::outermost(body.span);
+
+        let (kind, idx) = self.coroutine_state_adt_and_variant_idx(true);
+        assert_eq!(self.state_adt_ref.variant(idx).fields.len(), 0);
+        let statements = vec![Statement {
+            kind: StatementKind::Assign(Box::new((
+                Place::return_place(),
+                Rvalue::Aggregate(Box::new(kind), IndexVec::new()),
+            ))),
+            source_info,
+        }];
+
+        body.basic_blocks_mut().push(BasicBlockData {
+            statements,
+            terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }),
+            is_cleanup: false,
+        });
+
+        block
+    }
+
+    fn coroutine_state_adt_and_variant_idx(
+        &self,
+        is_return: bool,
+    ) -> (AggregateKind<'tcx>, VariantIdx) {
+        let idx = VariantIdx::new(match (is_return, self.coroutine_kind) {
+            (true, hir::CoroutineKind::Coroutine) => 1, // CoroutineState::Complete
+            (false, hir::CoroutineKind::Coroutine) => 0, // CoroutineState::Yielded
+            (true, hir::CoroutineKind::Async(_)) => 0,  // Poll::Ready
+            (false, hir::CoroutineKind::Async(_)) => 1, // Poll::Pending
+            (true, hir::CoroutineKind::Gen(_)) => 0,    // Option::None
+            (false, hir::CoroutineKind::Gen(_)) => 1,   // Option::Some
+        });
+
+        let kind = AggregateKind::Adt(self.state_adt_ref.did(), idx, self.state_args, None, None);
+        (kind, idx)
+    }
+
     // Make a `CoroutineState` or `Poll` variant assignment.
     //
     // `core::ops::CoroutineState` only has single element tuple variants,
@@ -261,16 +302,7 @@ impl<'tcx> TransformVisitor<'tcx> {
         is_return: bool,
         statements: &mut Vec<Statement<'tcx>>,
     ) {
-        let idx = VariantIdx::new(match (is_return, self.coroutine_kind) {
-            (true, hir::CoroutineKind::Coroutine) => 1, // CoroutineState::Complete
-            (false, hir::CoroutineKind::Coroutine) => 0, // CoroutineState::Yielded
-            (true, hir::CoroutineKind::Async(_)) => 0,  // Poll::Ready
-            (false, hir::CoroutineKind::Async(_)) => 1, // Poll::Pending
-            (true, hir::CoroutineKind::Gen(_)) => 0,    // Option::None
-            (false, hir::CoroutineKind::Gen(_)) => 1,   // Option::Some
-        });
-
-        let kind = AggregateKind::Adt(self.state_adt_ref.did(), idx, self.state_args, None, None);
+        let (kind, idx) = self.coroutine_state_adt_and_variant_idx(is_return);
 
         match self.coroutine_kind {
             // `Poll::Pending`
@@ -1285,10 +1317,13 @@ fn create_coroutine_resume_function<'tcx>(
     }
 
     if can_return {
-        cases.insert(
-            1,
-            (RETURNED, insert_panic_block(tcx, body, ResumedAfterReturn(coroutine_kind))),
-        );
+        let block = match coroutine_kind {
+            CoroutineKind::Async(_) | CoroutineKind::Coroutine => {
+                insert_panic_block(tcx, body, ResumedAfterReturn(coroutine_kind))
+            }
+            CoroutineKind::Gen(_) => transform.insert_none_ret_block(body),
+        };
+        cases.insert(1, (RETURNED, block));
     }
 
     insert_switch(body, cases, &transform, TerminatorKind::Unreachable);
diff --git a/tests/ui/coroutine/gen_block_iterate.rs b/tests/ui/coroutine/gen_block_iterate.rs
index 131dd6879360..18e1bb887723 100644
--- a/tests/ui/coroutine/gen_block_iterate.rs
+++ b/tests/ui/coroutine/gen_block_iterate.rs
@@ -25,6 +25,8 @@ fn main() {
     assert_eq!(iter.next(), Some(4));
     assert_eq!(iter.next(), Some(5));
     assert_eq!(iter.next(), None);
+    // `gen` blocks are fused
+    assert_eq!(iter.next(), None);
 
     let mut iter = moved();
     assert_eq!(iter.next(), Some(42));
diff --git a/tests/ui/coroutine/gen_block_iterate_fused.rs b/tests/ui/coroutine/gen_block_iterate_fused.rs
deleted file mode 100644
index 8ee6baf10602..000000000000
--- a/tests/ui/coroutine/gen_block_iterate_fused.rs
+++ /dev/null
@@ -1,19 +0,0 @@
-// revisions: next old
-//compile-flags: --edition 2024 -Zunstable-options
-//[next] compile-flags: -Ztrait-solver=next
-// run-fail
-#![feature(gen_blocks)]
-
-fn foo() -> impl Iterator<Item = u32> {
-    gen { yield 42; for x in 3..6 { yield x } }
-}
-
-fn main() {
-    let mut iter = foo();
-    assert_eq!(iter.next(), Some(42));
-    assert_eq!(iter.next(), Some(3));
-    assert_eq!(iter.next(), Some(4));
-    assert_eq!(iter.next(), Some(5));
-    assert_eq!(iter.next(), None);
-    assert_eq!(iter.next(), None);
-}

From eb10fa345e3d2d80c450aae42fb868fbc5449915 Mon Sep 17 00:00:00 2001
From: Guillaume Gomez <guillaume1.gomez@gmail.com>
Date: Fri, 27 Oct 2023 16:55:40 +0200
Subject: [PATCH 150/297] Fix build scripts

---
 cargo.sh  |   2 +-
 config.sh |  35 +++++++++++++----
 test.sh   | 114 +++++++++++++++++++++++++++++++++---------------------
 3 files changed, 98 insertions(+), 53 deletions(-)

diff --git a/cargo.sh b/cargo.sh
index 16e49b20423c..b68a08ee88f8 100755
--- a/cargo.sh
+++ b/cargo.sh
@@ -12,7 +12,7 @@ TOOLCHAIN=$(cat rust-toolchain | grep channel | sed 's/channel = "\(.*\)"/\1/')
 
 popd >/dev/null
 
-if [[ $(rustc -V) != $(rustc +${TOOLCHAIN} -V) ]]; then
+if [[ $(${RUSTC} -V) != $(${RUSTC} +${TOOLCHAIN} -V) ]]; then
     echo "rustc_codegen_gcc is build for $(rustc +${TOOLCHAIN} -V) but the default rustc version is $(rustc -V)."
     echo "Using $(rustc +${TOOLCHAIN} -V)."
 fi
diff --git a/config.sh b/config.sh
index 99ee9b054c6f..006758e19e19 100644
--- a/config.sh
+++ b/config.sh
@@ -4,19 +4,25 @@ export CARGO_INCREMENTAL=0
 
 if [ -f ./gcc_path ]; then
     export GCC_PATH=$(cat gcc_path)
+elif (( $use_system_gcc == 1 )); then
+    echo 'Using system GCC'
 else
     echo 'Please put the path to your custom build of libgccjit in the file `gcc_path`, see Readme.md for details'
     exit 1
 fi
 
+if [[ -z "$RUSTC" ]]; then
+    export RUSTC="rustc"
+fi
+
 unamestr=`uname`
 if [[ "$unamestr" == 'Linux' ]]; then
-   dylib_ext='so'
+    dylib_ext='so'
 elif [[ "$unamestr" == 'Darwin' ]]; then
-   dylib_ext='dylib'
+    dylib_ext='dylib'
 else
-   echo "Unsupported os"
-   exit 1
+    echo "Unsupported os"
+    exit 1
 fi
 
 HOST_TRIPLE=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ")
@@ -44,17 +50,30 @@ if [[ ! -v FAT_LTO ]]; then
     disable_lto_flags='-Clto=off'
 fi
 
-export RUSTFLAGS="$CG_RUSTFLAGS $linker -Csymbol-mangling-version=v0 -Cdebuginfo=2 $disable_lto_flags -Zcodegen-backend=$(pwd)/target/${CHANNEL:-debug}/librustc_codegen_gcc.$dylib_ext --sysroot $(pwd)/build_sysroot/sysroot $TEST_FLAGS"
+if [[ -z "$BUILTIN_BACKEND" ]]; then
+    export RUSTFLAGS="$CG_RUSTFLAGS $linker -Csymbol-mangling-version=v0 -Cdebuginfo=2 $disable_lto_flags -Zcodegen-backend=$(pwd)/target/${CHANNEL:-debug}/librustc_codegen_gcc.$dylib_ext --sysroot $(pwd)/build_sysroot/sysroot $TEST_FLAGS"
+else
+    export RUSTFLAGS="$CG_RUSTFLAGS $linker -Csymbol-mangling-version=v0 -Cdebuginfo=2 $disable_lto_flags -Zcodegen-backend=gcc $TEST_FLAGS -Cpanic=abort"
+fi
 
 # FIXME(antoyo): remove once the atomic shim is gone
 if [[ unamestr == 'Darwin' ]]; then
-   export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
+    export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup"
 fi
 
-RUSTC="rustc $RUSTFLAGS -L crate=target/out --out-dir target/out"
+if [[ -z "$cargo_target_dir" ]]; then
+    RUST_CMD="$RUSTC $RUSTFLAGS -L crate=target/out --out-dir target/out"
+    cargo_target_dir="target/out"
+else
+    RUST_CMD="$RUSTC $RUSTFLAGS -L crate=$cargo_target_dir --out-dir $cargo_target_dir"
+fi
 export RUSTC_LOG=warn # display metadata load errors
 
-export LD_LIBRARY_PATH="$(pwd)/target/out:$(pwd)/build_sysroot/sysroot/lib/rustlib/$TARGET_TRIPLE/lib:$GCC_PATH"
+export LD_LIBRARY_PATH="$(pwd)/target/out:$(pwd)/build_sysroot/sysroot/lib/rustlib/$TARGET_TRIPLE/lib"
+if [[ ! -z "$:$GCC_PATH" ]]; then
+    export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GCC_PATH"
+fi
+
 export DYLD_LIBRARY_PATH=$LD_LIBRARY_PATH
 # NOTE: To avoid the -fno-inline errors, use /opt/gcc/bin/gcc instead of cc.
 # To do so, add a symlink for cc to /opt/gcc/bin/gcc in our PATH.
diff --git a/test.sh b/test.sh
index fa34c31dbaa7..e896237a1ea4 100755
--- a/test.sh
+++ b/test.sh
@@ -5,16 +5,6 @@
 set -e
 #set -x
 
-if [ -f ./gcc_path ]; then
-    export GCC_PATH=$(cat gcc_path)
-else
-    echo 'Please put the path to your custom build of libgccjit in the file `gcc_path`, see Readme.md for details'
-    exit 1
-fi
-
-export LD_LIBRARY_PATH="$GCC_PATH"
-export LIBRARY_PATH="$GCC_PATH"
-
 flags=
 gcc_master_branch=1
 channel="debug"
@@ -22,12 +12,18 @@ funcs=()
 build_only=0
 nb_parts=0
 current_part=0
+use_system_gcc=0
+use_backend=0
+cargo_target_dir=""
+
+export CHANNEL='debug'
 
 while [[ $# -gt 0 ]]; do
     case $1 in
         --release)
             codegen_channel=release
             channel="release"
+            export CHANNEL='release'
             shift
             ;;
         --release-sysroot)
@@ -111,6 +107,22 @@ while [[ $# -gt 0 ]]; do
             build_only=1
             shift
             ;;
+        "--use-system-gcc")
+            use_system_gcc=1
+            shift
+            ;;
+        "--use-backend")
+            use_backend=1
+            shift
+            export BUILTIN_BACKEND=$1
+            shift
+            ;;
+        "--out-dir")
+            shift
+            export CARGO_TARGET_DIR=$1
+            cargo_target_dir=$1
+            shift
+            ;;
         "--nb-parts")
             shift
             nb_parts=$1
@@ -128,13 +140,25 @@ while [[ $# -gt 0 ]]; do
     esac
 done
 
-if [[ $channel == "release" ]]; then
-    export CHANNEL='release'
-    CARGO_INCREMENTAL=1 cargo rustc --release $flags
+if [ -f ./gcc_path ]; then
+    export GCC_PATH=$(cat gcc_path)
+elif (( $use_system_gcc == 1 )); then
+    echo 'Using system GCC'
 else
-    echo $LD_LIBRARY_PATH
-    export CHANNEL='debug'
-    cargo rustc $flags
+    echo 'Please put the path to your custom build of libgccjit in the file `gcc_path`, see Readme.md for details'
+    exit 1
+fi
+
+export LD_LIBRARY_PATH="$GCC_PATH"
+export LIBRARY_PATH="$GCC_PATH"
+
+if [[ $use_backend == 0 ]]; then
+    if [[ $channel == "release" ]]; then
+        CARGO_INCREMENTAL=1 cargo rustc --release $flags
+    else
+        echo $LD_LIBRARY_PATH
+        cargo rustc $flags
+    fi
 fi
 
 if (( $build_only == 1 )); then
@@ -145,24 +169,26 @@ fi
 source config.sh
 
 function clean() {
-    rm -r target/out || true
-    mkdir -p target/out/gccjit
+    rm -r $cargo_target_dir || true
+    mkdir -p $cargo_target_dir/gccjit
 }
 
 function mini_tests() {
     echo "[BUILD] mini_core"
     crate_types="lib,dylib"
+
     if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then
         crate_types="lib"
     fi
-    $RUSTC example/mini_core.rs --crate-name mini_core --crate-type $crate_types --target $TARGET_TRIPLE
+
+    $RUST_CMD example/mini_core.rs --crate-name mini_core --crate-type $crate_types --target $TARGET_TRIPLE
 
     echo "[BUILD] example"
-    $RUSTC example/example.rs --crate-type lib --target $TARGET_TRIPLE
+    $RUST_CMD example/example.rs --crate-type lib --target $TARGET_TRIPLE
 
     echo "[AOT] mini_core_hello_world"
-    $RUSTC example/mini_core_hello_world.rs --crate-name mini_core_hello_world --crate-type bin -g --target $TARGET_TRIPLE
-    $RUN_WRAPPER ./target/out/mini_core_hello_world abc bcd
+    $RUST_CMD example/mini_core_hello_world.rs --crate-name mini_core_hello_world --crate-type bin -g --target $TARGET_TRIPLE
+    $RUN_WRAPPER $cargo_target_dir/mini_core_hello_world abc bcd
 }
 
 function build_sysroot() {
@@ -189,42 +215,42 @@ function run_in_vm() {
 
 function std_tests() {
     echo "[AOT] arbitrary_self_types_pointers_and_wrappers"
-    $RUSTC example/arbitrary_self_types_pointers_and_wrappers.rs --crate-name arbitrary_self_types_pointers_and_wrappers --crate-type bin --target $TARGET_TRIPLE
-    $RUN_WRAPPER ./target/out/arbitrary_self_types_pointers_and_wrappers
+    $RUST_CMD example/arbitrary_self_types_pointers_and_wrappers.rs --crate-name arbitrary_self_types_pointers_and_wrappers --crate-type bin --target $TARGET_TRIPLE
+    $RUN_WRAPPER $cargo_target_dir/arbitrary_self_types_pointers_and_wrappers
 
     echo "[AOT] alloc_system"
-    $RUSTC example/alloc_system.rs --crate-type lib --target "$TARGET_TRIPLE"
+    $RUST_CMD example/alloc_system.rs --crate-type lib --target "$TARGET_TRIPLE"
 
     # FIXME: doesn't work on m68k.
     if [[ "$HOST_TRIPLE" == "$TARGET_TRIPLE" ]]; then
         echo "[AOT] alloc_example"
-        $RUSTC example/alloc_example.rs --crate-type bin --target $TARGET_TRIPLE
-        $RUN_WRAPPER ./target/out/alloc_example
+        $RUST_CMD example/alloc_example.rs --crate-type bin --target $TARGET_TRIPLE
+        $RUN_WRAPPER $cargo_target_dir/alloc_example
     fi
 
     echo "[AOT] dst_field_align"
     # FIXME(antoyo): Re-add -Zmir-opt-level=2 once rust-lang/rust#67529 is fixed.
-    $RUSTC example/dst-field-align.rs --crate-name dst_field_align --crate-type bin --target $TARGET_TRIPLE
-    $RUN_WRAPPER ./target/out/dst_field_align || (echo $?; false)
+    $RUST_CMD example/dst-field-align.rs --crate-name dst_field_align --crate-type bin --target $TARGET_TRIPLE
+    $RUN_WRAPPER $cargo_target_dir/dst_field_align || (echo $?; false)
 
     echo "[AOT] std_example"
     std_flags="--cfg feature=\"master\""
     if (( $gcc_master_branch == 0 )); then
         std_flags=""
     fi
-    $RUSTC example/std_example.rs --crate-type bin --target $TARGET_TRIPLE $std_flags
-    $RUN_WRAPPER ./target/out/std_example --target $TARGET_TRIPLE
+    $RUST_CMD example/std_example.rs --crate-type bin --target $TARGET_TRIPLE $std_flags
+    $RUN_WRAPPER $cargo_target_dir/std_example --target $TARGET_TRIPLE
 
     echo "[AOT] subslice-patterns-const-eval"
-    $RUSTC example/subslice-patterns-const-eval.rs --crate-type bin $TEST_FLAGS --target $TARGET_TRIPLE
-    $RUN_WRAPPER ./target/out/subslice-patterns-const-eval
+    $RUST_CMD example/subslice-patterns-const-eval.rs --crate-type bin $TEST_FLAGS --target $TARGET_TRIPLE
+    $RUN_WRAPPER $cargo_target_dir/subslice-patterns-const-eval
 
     echo "[AOT] track-caller-attribute"
-    $RUSTC example/track-caller-attribute.rs --crate-type bin $TEST_FLAGS --target $TARGET_TRIPLE
-    $RUN_WRAPPER ./target/out/track-caller-attribute
+    $RUST_CMD example/track-caller-attribute.rs --crate-type bin $TEST_FLAGS --target $TARGET_TRIPLE
+    $RUN_WRAPPER $cargo_target_dir/track-caller-attribute
 
     echo "[BUILD] mod_bench"
-    $RUSTC example/mod_bench.rs --crate-type bin --target $TARGET_TRIPLE
+    $RUST_CMD example/mod_bench.rs --crate-type bin --target $TARGET_TRIPLE
 }
 
 function setup_rustc() {
@@ -233,7 +259,7 @@ function setup_rustc() {
     git clone https://github.com/rust-lang/rust.git || true
     cd rust
     git fetch
-    git checkout $(rustc -V | cut -d' ' -f3 | tr -d '(')
+    git checkout $($RUSTC -V | cut -d' ' -f3 | tr -d '(')
     export RUSTFLAGS=
 
     rm config.toml || true
@@ -258,8 +284,8 @@ llvm-filecheck = "`which FileCheck-10 || which FileCheck-11 || which FileCheck-1
 download-ci-llvm = false
 EOF
 
-    rustc -V | cut -d' ' -f3 | tr -d '('
-    git checkout $(rustc -V | cut -d' ' -f3 | tr -d '(') tests
+    $RUSTC -V | cut -d' ' -f3 | tr -d '('
+    git checkout $($RUSTC -V | cut -d' ' -f3 | tr -d '(') tests
 }
 
 function asm_tests() {
@@ -286,17 +312,17 @@ function test_libcore() {
 #echo "[BENCH COMPILE] mod_bench"
 
 #COMPILE_MOD_BENCH_INLINE="$RUSTC example/mod_bench.rs --crate-type bin -Zmir-opt-level=3 -O --crate-name mod_bench_inline"
-#COMPILE_MOD_BENCH_LLVM_0="rustc example/mod_bench.rs --crate-type bin -Copt-level=0 -o target/out/mod_bench_llvm_0 -Cpanic=abort"
-#COMPILE_MOD_BENCH_LLVM_1="rustc example/mod_bench.rs --crate-type bin -Copt-level=1 -o target/out/mod_bench_llvm_1 -Cpanic=abort"
-#COMPILE_MOD_BENCH_LLVM_2="rustc example/mod_bench.rs --crate-type bin -Copt-level=2 -o target/out/mod_bench_llvm_2 -Cpanic=abort"
-#COMPILE_MOD_BENCH_LLVM_3="rustc example/mod_bench.rs --crate-type bin -Copt-level=3 -o target/out/mod_bench_llvm_3 -Cpanic=abort"
+#COMPILE_MOD_BENCH_LLVM_0="rustc example/mod_bench.rs --crate-type bin -Copt-level=0 -o $cargo_target_dir/mod_bench_llvm_0 -Cpanic=abort"
+#COMPILE_MOD_BENCH_LLVM_1="rustc example/mod_bench.rs --crate-type bin -Copt-level=1 -o $cargo_target_dir/mod_bench_llvm_1 -Cpanic=abort"
+#COMPILE_MOD_BENCH_LLVM_2="rustc example/mod_bench.rs --crate-type bin -Copt-level=2 -o $cargo_target_dir/mod_bench_llvm_2 -Cpanic=abort"
+#COMPILE_MOD_BENCH_LLVM_3="rustc example/mod_bench.rs --crate-type bin -Copt-level=3 -o $cargo_target_dir/mod_bench_llvm_3 -Cpanic=abort"
 
 ## Use 100 runs, because a single compilations doesn't take more than ~150ms, so it isn't very slow
 #hyperfine --runs ${COMPILE_RUNS:-100} "$COMPILE_MOD_BENCH_INLINE" "$COMPILE_MOD_BENCH_LLVM_0" "$COMPILE_MOD_BENCH_LLVM_1" "$COMPILE_MOD_BENCH_LLVM_2" "$COMPILE_MOD_BENCH_LLVM_3"
 
 #echo
 #echo "[BENCH RUN] mod_bench"
-#hyperfine --runs ${RUN_RUNS:-10} ./target/out/mod_bench{,_inline} ./target/out/mod_bench_llvm_*
+#hyperfine --runs ${RUN_RUNS:-10} $cargo_target_dir/mod_bench{,_inline} $cargo_target_dir/mod_bench_llvm_*
 
 function extended_rand_tests() {
     if (( $gcc_master_branch == 0 )); then

From 2ef5897a89f9bbd9d38b9eb1222b5bedef98814b Mon Sep 17 00:00:00 2001
From: Ralf Jung <post@ralfj.de>
Date: Fri, 27 Oct 2023 13:19:24 +0200
Subject: [PATCH 151/297] fix failure to detect a too-big-type after adding
 padding

---
 compiler/rustc_abi/src/layout.rs               |  5 +++++
 .../rustc_ty_utils/src/layout_sanity_check.rs  |  3 +++
 tests/ui/layout/too-big-with-padding.rs        | 18 ++++++++++++++++++
 tests/ui/layout/too-big-with-padding.stderr    |  8 ++++++++
 4 files changed, 34 insertions(+)
 create mode 100644 tests/ui/layout/too-big-with-padding.rs
 create mode 100644 tests/ui/layout/too-big-with-padding.stderr

diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs
index 00d862ca27b7..9127e1d06e88 100644
--- a/compiler/rustc_abi/src/layout.rs
+++ b/compiler/rustc_abi/src/layout.rs
@@ -539,6 +539,7 @@ pub trait LayoutCalculator {
         // Align the maximum variant size to the largest alignment.
         size = size.align_to(align.abi);
 
+        // FIXME(oli-obk): deduplicate and harden these checks
         if size.bytes() >= dl.obj_size_bound() {
             return None;
         }
@@ -1103,6 +1104,10 @@ fn univariant<
         inverse_memory_index.into_iter().map(|it| it.index() as u32).collect()
     };
     let size = min_size.align_to(align.abi);
+    // FIXME(oli-obk): deduplicate and harden these checks
+    if size.bytes() >= dl.obj_size_bound() {
+        return None;
+    }
     let mut layout_of_single_non_zst_field = None;
     let mut abi = Abi::Aggregate { sized };
     // Try to make this a Scalar/ScalarPair.
diff --git a/compiler/rustc_ty_utils/src/layout_sanity_check.rs b/compiler/rustc_ty_utils/src/layout_sanity_check.rs
index 8633334381ad..6332c614a90b 100644
--- a/compiler/rustc_ty_utils/src/layout_sanity_check.rs
+++ b/compiler/rustc_ty_utils/src/layout_sanity_check.rs
@@ -19,6 +19,9 @@ pub(super) fn sanity_check_layout<'tcx>(
     if layout.size.bytes() % layout.align.abi.bytes() != 0 {
         bug!("size is not a multiple of align, in the following layout:\n{layout:#?}");
     }
+    if layout.size.bytes() >= cx.tcx.data_layout.obj_size_bound() {
+        bug!("size is too large, in the following layout:\n{layout:#?}");
+    }
 
     if !cfg!(debug_assertions) {
         // Stop here, the rest is kind of expensive.
diff --git a/tests/ui/layout/too-big-with-padding.rs b/tests/ui/layout/too-big-with-padding.rs
new file mode 100644
index 000000000000..cf41ac872c21
--- /dev/null
+++ b/tests/ui/layout/too-big-with-padding.rs
@@ -0,0 +1,18 @@
+// build-fail
+// compile-flags: --target i686-unknown-linux-gnu --crate-type lib
+// needs-llvm-components: x86
+#![feature(no_core, lang_items)]
+#![allow(internal_features)]
+#![no_std]
+#![no_core]
+
+// 0x7fffffff is fine, but after rounding up it becomes too big
+#[repr(C, align(2))]
+pub struct Example([u8; 0x7fffffff]);
+
+pub fn lib(_x: Example) {} //~ERROR: too big for the current architecture
+
+#[lang = "sized"]
+pub trait Sized {}
+#[lang = "copy"]
+pub trait Copy: Sized {}
diff --git a/tests/ui/layout/too-big-with-padding.stderr b/tests/ui/layout/too-big-with-padding.stderr
new file mode 100644
index 000000000000..5cc854adce0d
--- /dev/null
+++ b/tests/ui/layout/too-big-with-padding.stderr
@@ -0,0 +1,8 @@
+error: values of the type `Example` are too big for the current architecture
+  --> $DIR/too-big-with-padding.rs:13:1
+   |
+LL | pub fn lib(_x: Example) {}
+   | ^^^^^^^^^^^^^^^^^^^^^^^
+
+error: aborting due to previous error
+

From 6dbad236419075603a616e7b19f585a1a308cf56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Esteban=20K=C3=BCber?= <esteban@kuber.com.ar>
Date: Thu, 19 Oct 2023 16:34:41 +0000
Subject: [PATCH 152/297] When encountering sealed traits, point types that
 implement it

```
error[E0277]: the trait bound `S: d::Hidden` is not satisfied
  --> $DIR/sealed-trait-local.rs:53:20
   |
LL | impl c::Sealed for S {}
   |                    ^ the trait `d::Hidden` is not implemented for `S`
   |
note: required by a bound in `c::Sealed`
  --> $DIR/sealed-trait-local.rs:17:23
   |
LL |     pub trait Sealed: self::d::Hidden {
   |                       ^^^^^^^^^^^^^^^ required by this bound in `Sealed`
   = note: `Sealed` is a "sealed trait", because to implement it you also need to implement `c::d::Hidden`, which is not accessible; this is usually done to force you to use one of the provided types that already implement it
   = help: the following types implement the trait:
            - c::X
            - c::Y
```

The last `help` is new.
---
 .../src/traits/error_reporting/suggestions.rs | 30 ++++++++++++-
 .../sealed-traits/sealed-trait-local.rs       | 40 ++++++++++++++++-
 .../sealed-traits/sealed-trait-local.stderr   | 44 ++++++++++++++++---
 3 files changed, 105 insertions(+), 9 deletions(-)

diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
index 947ea3eece3f..aad07d7683a7 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/suggestions.rs
@@ -2691,8 +2691,6 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                         if let DefKind::Trait = tcx.def_kind(item_def_id)
                             && !visible_item
                         {
-                            // FIXME(estebank): extend this to search for all the types that do
-                            // implement this trait and list them.
                             err.note(format!(
                                 "`{short_item_name}` is a \"sealed trait\", because to implement \
                                  it you also need to implement `{}`, which is not accessible; \
@@ -2700,6 +2698,34 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                                  types that already implement it",
                                 with_no_trimmed_paths!(tcx.def_path_str(def_id)),
                             ));
+                            let impls_of = tcx.trait_impls_of(def_id);
+                            let impls = impls_of
+                                .non_blanket_impls()
+                                .values()
+                                .flatten()
+                                .chain(impls_of.blanket_impls().iter())
+                                .collect::<Vec<_>>();
+                            if !impls.is_empty() {
+                                let len = impls.len();
+                                let mut types = impls.iter()
+                                    .map(|t| with_no_trimmed_paths!(format!(
+                                        "  {}",
+                                        tcx.type_of(*t).instantiate_identity(),
+                                    )))
+                                    .collect::<Vec<_>>();
+                                let post = if types.len() > 9 {
+                                    types.truncate(8);
+                                    format!("\nand {} others", len - 8)
+                                } else {
+                                    String::new()
+                                };
+                                err.help(format!(
+                                    "the following type{} implement{} the trait:\n{}{post}",
+                                    pluralize!(len),
+                                    if len == 1 { "s" } else { "" },
+                                    types.join("\n"),
+                                ));
+                            }
                         }
                     }
                 } else {
diff --git a/tests/ui/privacy/sealed-traits/sealed-trait-local.rs b/tests/ui/privacy/sealed-traits/sealed-trait-local.rs
index 778ddf0f8176..9ae01259a781 100644
--- a/tests/ui/privacy/sealed-traits/sealed-trait-local.rs
+++ b/tests/ui/privacy/sealed-traits/sealed-trait-local.rs
@@ -13,7 +13,43 @@ pub mod a {
     }
 }
 
-struct S;
-impl a::Sealed for S {} //~ ERROR the trait bound `S: Hidden` is not satisfied
+pub mod c {
+    pub trait Sealed: self::d::Hidden {
+        fn foo() {}
+    }
 
+    struct X;
+    impl Sealed for X {}
+    impl self::d::Hidden for X {}
+
+    struct Y;
+    impl Sealed for Y {}
+    impl self::d::Hidden for Y {}
+
+    mod d {
+        pub trait Hidden {}
+    }
+}
+
+pub mod e {
+    pub trait Sealed: self::f::Hidden {
+        fn foo() {}
+    }
+
+    struct X;
+    impl self::f::Hidden for X {}
+
+    struct Y;
+    impl self::f::Hidden for Y {}
+    impl<T: self::f::Hidden> Sealed for T {}
+
+    mod f {
+        pub trait Hidden {}
+    }
+}
+
+struct S;
+impl a::Sealed for S {} //~ ERROR the trait bound
+impl c::Sealed for S {} //~ ERROR the trait bound
+impl e::Sealed for S {} //~ ERROR the trait bound
 fn main() {}
diff --git a/tests/ui/privacy/sealed-traits/sealed-trait-local.stderr b/tests/ui/privacy/sealed-traits/sealed-trait-local.stderr
index 5f8076fc84d0..a7f77a1c0c02 100644
--- a/tests/ui/privacy/sealed-traits/sealed-trait-local.stderr
+++ b/tests/ui/privacy/sealed-traits/sealed-trait-local.stderr
@@ -1,16 +1,50 @@
-error[E0277]: the trait bound `S: Hidden` is not satisfied
-  --> $DIR/sealed-trait-local.rs:17:20
+error[E0277]: the trait bound `S: b::Hidden` is not satisfied
+  --> $DIR/sealed-trait-local.rs:52:20
    |
 LL | impl a::Sealed for S {}
-   |                    ^ the trait `Hidden` is not implemented for `S`
+   |                    ^ the trait `b::Hidden` is not implemented for `S`
    |
-note: required by a bound in `Sealed`
+note: required by a bound in `a::Sealed`
   --> $DIR/sealed-trait-local.rs:3:23
    |
 LL |     pub trait Sealed: self::b::Hidden {
    |                       ^^^^^^^^^^^^^^^ required by this bound in `Sealed`
    = note: `Sealed` is a "sealed trait", because to implement it you also need to implement `a::b::Hidden`, which is not accessible; this is usually done to force you to use one of the provided types that already implement it
+   = help: the following type implements the trait:
+             a::X
 
-error: aborting due to previous error
+error[E0277]: the trait bound `S: d::Hidden` is not satisfied
+  --> $DIR/sealed-trait-local.rs:53:20
+   |
+LL | impl c::Sealed for S {}
+   |                    ^ the trait `d::Hidden` is not implemented for `S`
+   |
+note: required by a bound in `c::Sealed`
+  --> $DIR/sealed-trait-local.rs:17:23
+   |
+LL |     pub trait Sealed: self::d::Hidden {
+   |                       ^^^^^^^^^^^^^^^ required by this bound in `Sealed`
+   = note: `Sealed` is a "sealed trait", because to implement it you also need to implement `c::d::Hidden`, which is not accessible; this is usually done to force you to use one of the provided types that already implement it
+   = help: the following types implement the trait:
+             c::X
+             c::Y
+
+error[E0277]: the trait bound `S: f::Hidden` is not satisfied
+  --> $DIR/sealed-trait-local.rs:54:20
+   |
+LL | impl e::Sealed for S {}
+   |                    ^ the trait `f::Hidden` is not implemented for `S`
+   |
+note: required by a bound in `e::Sealed`
+  --> $DIR/sealed-trait-local.rs:35:23
+   |
+LL |     pub trait Sealed: self::f::Hidden {
+   |                       ^^^^^^^^^^^^^^^ required by this bound in `Sealed`
+   = note: `Sealed` is a "sealed trait", because to implement it you also need to implement `e::f::Hidden`, which is not accessible; this is usually done to force you to use one of the provided types that already implement it
+   = help: the following types implement the trait:
+             e::X
+             e::Y
+
+error: aborting due to 3 previous errors
 
 For more information about this error, try `rustc --explain E0277`.

From 11268b48a1dda5e3fad958fbab7f9f829c19de79 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Thu, 12 Oct 2023 22:23:38 +0200
Subject: [PATCH 153/297] Add tests

---
 .../pointer-sized-int.allow.stderr            |   2 +-
 .../pointer-sized-int.deny.stderr             | 106 +++++++++++++-----
 .../integer-ranges/pointer-sized-int.rs       |  10 ++
 3 files changed, 92 insertions(+), 26 deletions(-)

diff --git a/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.allow.stderr b/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.allow.stderr
index 9f277fa1e180..12a928bf0180 100644
--- a/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.allow.stderr
+++ b/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.allow.stderr
@@ -1,5 +1,5 @@
 error[E0004]: non-exhaustive patterns: type `usize` is non-empty
-  --> $DIR/pointer-sized-int.rs:48:11
+  --> $DIR/pointer-sized-int.rs:58:11
    |
 LL |     match 7usize {}
    |           ^^^^^^
diff --git a/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.deny.stderr b/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.deny.stderr
index df330c60b1e8..7fa9eb8eda66 100644
--- a/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.deny.stderr
+++ b/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.deny.stderr
@@ -1,5 +1,5 @@
 error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/pointer-sized-int.rs:12:11
+  --> $DIR/pointer-sized-int.rs:14:11
    |
 LL |     match 0usize {
    |           ^^^^^^ pattern `_` not covered
@@ -14,7 +14,7 @@ LL +         _ => todo!()
    |
 
 error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/pointer-sized-int.rs:17:11
+  --> $DIR/pointer-sized-int.rs:19:11
    |
 LL |     match 0isize {
    |           ^^^^^^ pattern `_` not covered
@@ -28,24 +28,10 @@ LL ~         isize::MIN ..= isize::MAX => {},
 LL +         _ => todo!()
    |
 
-error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/pointer-sized-int.rs:22:8
-   |
-LL |     m!(0usize, 0..=usize::MAX);
-   |        ^^^^^^ pattern `_` not covered
-   |
-   = note: the matched value is of type `usize`
-   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
-   = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
-help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
-   |
-LL |         match $s { $($t)+ => {}, _ => todo!() }
-   |                                ++++++++++++++
-
 error[E0004]: non-exhaustive patterns: `_` not covered
   --> $DIR/pointer-sized-int.rs:24:8
    |
-LL |     m!(0usize, 0..5 | 5..=usize::MAX);
+LL |     m!(0usize, 0..);
    |        ^^^^^^ pattern `_` not covered
    |
    = note: the matched value is of type `usize`
@@ -59,6 +45,34 @@ LL |         match $s { $($t)+ => {}, _ => todo!() }
 error[E0004]: non-exhaustive patterns: `_` not covered
   --> $DIR/pointer-sized-int.rs:26:8
    |
+LL |     m!(0usize, 0..=usize::MAX);
+   |        ^^^^^^ pattern `_` not covered
+   |
+   = note: the matched value is of type `usize`
+   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL |         match $s { $($t)+ => {}, _ => todo!() }
+   |                                ++++++++++++++
+
+error[E0004]: non-exhaustive patterns: `_` not covered
+  --> $DIR/pointer-sized-int.rs:28:8
+   |
+LL |     m!(0usize, 0..5 | 5..=usize::MAX);
+   |        ^^^^^^ pattern `_` not covered
+   |
+   = note: the matched value is of type `usize`
+   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL |         match $s { $($t)+ => {}, _ => todo!() }
+   |                                ++++++++++++++
+
+error[E0004]: non-exhaustive patterns: `_` not covered
+  --> $DIR/pointer-sized-int.rs:30:8
+   |
 LL |     m!(0usize, 0..usize::MAX | usize::MAX);
    |        ^^^^^^ pattern `_` not covered
    |
@@ -71,7 +85,7 @@ LL |         match $s { $($t)+ => {}, _ => todo!() }
    |                                ++++++++++++++
 
 error[E0004]: non-exhaustive patterns: `(_, _)` not covered
-  --> $DIR/pointer-sized-int.rs:28:8
+  --> $DIR/pointer-sized-int.rs:32:8
    |
 LL |     m!((0usize, true), (0..5, true) | (5..=usize::MAX, true) | (0..=usize::MAX, false));
    |        ^^^^^^^^^^^^^^ pattern `(_, _)` not covered
@@ -85,7 +99,35 @@ LL |         match $s { $($t)+ => {}, (_, _) => todo!() }
    |                                +++++++++++++++++++
 
 error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/pointer-sized-int.rs:31:8
+  --> $DIR/pointer-sized-int.rs:34:8
+   |
+LL |     m!(0usize, 0..=usize::MAX | usize::MAX..);
+   |        ^^^^^^ pattern `_` not covered
+   |
+   = note: the matched value is of type `usize`
+   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL |         match $s { $($t)+ => {}, _ => todo!() }
+   |                                ++++++++++++++
+
+error[E0004]: non-exhaustive patterns: `_` not covered
+  --> $DIR/pointer-sized-int.rs:37:8
+   |
+LL |     m!(0isize, ..0 | 0..);
+   |        ^^^^^^ pattern `_` not covered
+   |
+   = note: the matched value is of type `isize`
+   = note: `isize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `isize` matching
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL |         match $s { $($t)+ => {}, _ => todo!() }
+   |                                ++++++++++++++
+
+error[E0004]: non-exhaustive patterns: `_` not covered
+  --> $DIR/pointer-sized-int.rs:39:8
    |
 LL |     m!(0isize, isize::MIN..=isize::MAX);
    |        ^^^^^^ pattern `_` not covered
@@ -99,7 +141,7 @@ LL |         match $s { $($t)+ => {}, _ => todo!() }
    |                                ++++++++++++++
 
 error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/pointer-sized-int.rs:33:8
+  --> $DIR/pointer-sized-int.rs:41:8
    |
 LL |     m!(0isize, isize::MIN..5 | 5..=isize::MAX);
    |        ^^^^^^ pattern `_` not covered
@@ -113,7 +155,7 @@ LL |         match $s { $($t)+ => {}, _ => todo!() }
    |                                ++++++++++++++
 
 error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/pointer-sized-int.rs:35:8
+  --> $DIR/pointer-sized-int.rs:43:8
    |
 LL |     m!(0isize, isize::MIN..isize::MAX | isize::MAX);
    |        ^^^^^^ pattern `_` not covered
@@ -127,7 +169,7 @@ LL |         match $s { $($t)+ => {}, _ => todo!() }
    |                                ++++++++++++++
 
 error[E0004]: non-exhaustive patterns: `(_, _)` not covered
-  --> $DIR/pointer-sized-int.rs:37:8
+  --> $DIR/pointer-sized-int.rs:45:8
    |
 LL |     m!((0isize, true), (isize::MIN..5, true)
    |        ^^^^^^^^^^^^^^ pattern `(_, _)` not covered
@@ -141,7 +183,21 @@ LL |         match $s { $($t)+ => {}, (_, _) => todo!() }
    |                                +++++++++++++++++++
 
 error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/pointer-sized-int.rs:41:11
+  --> $DIR/pointer-sized-int.rs:48:8
+   |
+LL |     m!(0isize, ..=isize::MIN | isize::MIN..=isize::MAX | isize::MAX..);
+   |        ^^^^^^ pattern `_` not covered
+   |
+   = note: the matched value is of type `isize`
+   = note: `isize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `isize` matching
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+   |
+LL |         match $s { $($t)+ => {}, _ => todo!() }
+   |                                ++++++++++++++
+
+error[E0004]: non-exhaustive patterns: `_` not covered
+  --> $DIR/pointer-sized-int.rs:51:11
    |
 LL |     match 0isize {
    |           ^^^^^^ pattern `_` not covered
@@ -156,7 +212,7 @@ LL +         _ => todo!()
    |
 
 error[E0004]: non-exhaustive patterns: type `usize` is non-empty
-  --> $DIR/pointer-sized-int.rs:48:11
+  --> $DIR/pointer-sized-int.rs:58:11
    |
 LL |     match 7usize {}
    |           ^^^^^^
@@ -169,6 +225,6 @@ LL +         _ => todo!(),
 LL +     }
    |
 
-error: aborting due to 12 previous errors
+error: aborting due to 16 previous errors
 
 For more information about this error, try `rustc --explain E0004`.
diff --git a/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.rs b/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.rs
index 1ed18c267635..b55f8f3e5168 100644
--- a/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.rs
+++ b/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.rs
@@ -1,6 +1,7 @@
 // revisions: allow deny
 #![feature(exclusive_range_pattern)]
 #![cfg_attr(allow, feature(precise_pointer_size_matching))]
+#![allow(overlapping_range_endpoints)]
 
 macro_rules! m {
     ($s:expr, $($t:tt)+) => {
@@ -8,6 +9,7 @@ macro_rules! m {
     }
 }
 
+#[rustfmt::skip]
 fn main() {
     match 0usize {
         //[deny]~^ ERROR non-exhaustive patterns
@@ -19,6 +21,8 @@ fn main() {
         isize::MIN ..= isize::MAX => {}
     }
 
+    m!(0usize, 0..);
+    //[deny]~^ ERROR non-exhaustive patterns
     m!(0usize, 0..=usize::MAX);
     //[deny]~^ ERROR non-exhaustive patterns
     m!(0usize, 0..5 | 5..=usize::MAX);
@@ -27,7 +31,11 @@ fn main() {
     //[deny]~^ ERROR non-exhaustive patterns
     m!((0usize, true), (0..5, true) | (5..=usize::MAX, true) | (0..=usize::MAX, false));
     //[deny]~^ ERROR non-exhaustive patterns
+    m!(0usize, 0..=usize::MAX | usize::MAX..);
+    //[deny]~^ ERROR non-exhaustive patterns
 
+    m!(0isize, ..0 | 0..);
+    //[deny]~^ ERROR non-exhaustive patterns
     m!(0isize, isize::MIN..=isize::MAX);
     //[deny]~^ ERROR non-exhaustive patterns
     m!(0isize, isize::MIN..5 | 5..=isize::MAX);
@@ -37,6 +45,8 @@ fn main() {
     m!((0isize, true), (isize::MIN..5, true)
         | (5..=isize::MAX, true) | (isize::MIN..=isize::MAX, false));
     //[deny]~^^ ERROR non-exhaustive patterns
+    m!(0isize, ..=isize::MIN | isize::MIN..=isize::MAX | isize::MAX..);
+    //[deny]~^ ERROR non-exhaustive patterns
 
     match 0isize {
         //[deny]~^ ERROR non-exhaustive patterns

From 8a77b3248f6e147d782ab29915dc1aa093ef7a7c Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Thu, 12 Oct 2023 23:29:16 +0200
Subject: [PATCH 154/297] Abstract over `PatRange` boundary value

---
 compiler/rustc_middle/src/thir.rs             | 223 +++++++++++++++++-
 compiler/rustc_middle/src/ty/util.rs          |  82 ++++---
 .../rustc_mir_build/src/build/matches/mod.rs  |   2 +-
 .../src/build/matches/simplify.rs             |  44 +---
 .../rustc_mir_build/src/build/matches/test.rs |  47 +---
 .../src/thir/pattern/deconstruct_pat.rs       |  24 +-
 .../rustc_mir_build/src/thir/pattern/mod.rs   |  92 ++------
 7 files changed, 299 insertions(+), 215 deletions(-)

diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs
index f1747356139d..fefdce8e77e4 100644
--- a/compiler/rustc_middle/src/thir.rs
+++ b/compiler/rustc_middle/src/thir.rs
@@ -16,17 +16,19 @@ use rustc_hir::RangeEnd;
 use rustc_index::newtype_index;
 use rustc_index::IndexVec;
 use rustc_middle::middle::region;
-use rustc_middle::mir::interpret::AllocId;
+use rustc_middle::mir::interpret::{AllocId, Scalar};
 use rustc_middle::mir::{self, BinOp, BorrowKind, FakeReadCause, Mutability, UnOp};
 use rustc_middle::ty::adjustment::PointerCoercion;
+use rustc_middle::ty::layout::IntegerExt;
 use rustc_middle::ty::{
     self, AdtDef, CanonicalUserType, CanonicalUserTypeAnnotation, FnSig, GenericArgsRef, List, Ty,
-    UpvarArgs,
+    TyCtxt, UpvarArgs,
 };
 use rustc_span::def_id::LocalDefId;
 use rustc_span::{sym, ErrorGuaranteed, Span, Symbol, DUMMY_SP};
-use rustc_target::abi::{FieldIdx, VariantIdx};
+use rustc_target::abi::{FieldIdx, Integer, Size, VariantIdx};
 use rustc_target::asm::InlineAsmRegOrRegClass;
+use std::cmp::Ordering;
 use std::fmt;
 use std::ops::Index;
 
@@ -810,12 +812,217 @@ pub enum PatKind<'tcx> {
     Error(ErrorGuaranteed),
 }
 
+/// A range pattern.
+/// The boundaries must be of the same type and that type must be numeric.
 #[derive(Clone, Debug, PartialEq, HashStable, TypeVisitable)]
 pub struct PatRange<'tcx> {
-    pub lo: mir::Const<'tcx>,
-    pub hi: mir::Const<'tcx>,
+    pub lo: PatRangeBoundary<'tcx>,
+    pub hi: PatRangeBoundary<'tcx>,
     #[type_visitable(ignore)]
     pub end: RangeEnd,
+    pub ty: Ty<'tcx>,
+}
+
+impl<'tcx> PatRange<'tcx> {
+    /// Whether this range covers the full extent of possible values (best-effort, we ignore floats).
+    #[inline]
+    pub fn is_full_range(&self, tcx: TyCtxt<'tcx>) -> Option<bool> {
+        let (min, max, size, bias) = match *self.ty.kind() {
+            ty::Char => (0, std::char::MAX as u128, Size::from_bits(32), 0),
+            ty::Int(ity) => {
+                let size = Integer::from_int_ty(&tcx, ity).size();
+                let max = size.truncate(u128::MAX);
+                let bias = 1u128 << (size.bits() - 1);
+                (0, max, size, bias)
+            }
+            ty::Uint(uty) => {
+                let size = Integer::from_uint_ty(&tcx, uty).size();
+                let max = size.unsigned_int_max();
+                (0, max, size, 0)
+            }
+            _ => return None,
+        };
+
+        // We want to compare ranges numerically, but the order of the bitwise representation of
+        // signed integers does not match their numeric order. Thus, to correct the ordering, we
+        // need to shift the range of signed integers to correct the comparison. This is achieved by
+        // XORing with a bias (see pattern/deconstruct_pat.rs for another pertinent example of this
+        // pattern).
+        //
+        // Also, for performance, it's important to only do the second `try_to_bits` if necessary.
+        let lo_is_min = match self.lo {
+            PatRangeBoundary::Finite(value) => {
+                let lo = value.try_to_bits(size).unwrap() ^ bias;
+                lo <= min
+            }
+        };
+        if lo_is_min {
+            let hi_is_max = match self.hi {
+                PatRangeBoundary::Finite(value) => {
+                    let hi = value.try_to_bits(size).unwrap() ^ bias;
+                    hi > max || hi == max && self.end == RangeEnd::Included
+                }
+            };
+            if hi_is_max {
+                return Some(true);
+            }
+        }
+        Some(false)
+    }
+
+    #[inline]
+    pub fn contains(
+        &self,
+        value: mir::Const<'tcx>,
+        tcx: TyCtxt<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+    ) -> Option<bool> {
+        use Ordering::*;
+        debug_assert_eq!(self.ty, value.ty());
+        let ty = self.ty;
+        let value = PatRangeBoundary::Finite(value);
+        // For performance, it's important to only do the second comparison if necessary.
+        Some(
+            match self.lo.compare_with(value, ty, tcx, param_env)? {
+                Less | Equal => true,
+                Greater => false,
+            } && match value.compare_with(self.hi, ty, tcx, param_env)? {
+                Less => true,
+                Equal => self.end == RangeEnd::Included,
+                Greater => false,
+            },
+        )
+    }
+
+    #[inline]
+    pub fn overlaps(
+        &self,
+        other: &Self,
+        tcx: TyCtxt<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+    ) -> Option<bool> {
+        use Ordering::*;
+        debug_assert_eq!(self.ty, other.ty);
+        // For performance, it's important to only do the second comparison if necessary.
+        Some(
+            match other.lo.compare_with(self.hi, self.ty, tcx, param_env)? {
+                Less => true,
+                Equal => self.end == RangeEnd::Included,
+                Greater => false,
+            } && match self.lo.compare_with(other.hi, self.ty, tcx, param_env)? {
+                Less => true,
+                Equal => other.end == RangeEnd::Included,
+                Greater => false,
+            },
+        )
+    }
+}
+
+impl<'tcx> fmt::Display for PatRange<'tcx> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let PatRangeBoundary::Finite(value) = &self.lo;
+        write!(f, "{value}")?;
+        write!(f, "{}", self.end)?;
+        let PatRangeBoundary::Finite(value) = &self.hi;
+        write!(f, "{value}")?;
+        Ok(())
+    }
+}
+
+/// A (possibly open) boundary of a range pattern.
+/// If present, the const must be of a numeric type.
+#[derive(Copy, Clone, Debug, PartialEq, HashStable, TypeVisitable)]
+pub enum PatRangeBoundary<'tcx> {
+    Finite(mir::Const<'tcx>),
+}
+
+impl<'tcx> PatRangeBoundary<'tcx> {
+    #[inline]
+    pub fn lower_bound(ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Self {
+        // Unwrap is ok because the type is known to be numeric.
+        let c = ty.numeric_min_val(tcx).unwrap();
+        let value = mir::Const::from_ty_const(c, tcx);
+        Self::Finite(value)
+    }
+    #[inline]
+    pub fn upper_bound(ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Self {
+        // Unwrap is ok because the type is known to be numeric.
+        let c = ty.numeric_max_val(tcx).unwrap();
+        let value = mir::Const::from_ty_const(c, tcx);
+        Self::Finite(value)
+    }
+
+    #[inline]
+    pub fn to_const(self, _ty: Ty<'tcx>, _tcx: TyCtxt<'tcx>) -> mir::Const<'tcx> {
+        match self {
+            Self::Finite(value) => value,
+        }
+    }
+    pub fn eval_bits(
+        self,
+        _ty: Ty<'tcx>,
+        tcx: TyCtxt<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+    ) -> u128 {
+        match self {
+            Self::Finite(value) => value.eval_bits(tcx, param_env),
+        }
+    }
+
+    #[instrument(skip(tcx, param_env), level = "debug", ret)]
+    pub fn compare_with(
+        self,
+        other: Self,
+        ty: Ty<'tcx>,
+        tcx: TyCtxt<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+    ) -> Option<Ordering> {
+        use PatRangeBoundary::*;
+        match (self, other) {
+            // This code is hot when compiling matches with many ranges. So we
+            // special-case extraction of evaluated scalars for speed, for types where
+            // raw data comparisons are appropriate. E.g. `unicode-normalization` has
+            // many ranges such as '\u{037A}'..='\u{037F}', and chars can be compared
+            // in this way.
+            (Finite(mir::Const::Ty(a)), Finite(mir::Const::Ty(b)))
+                if matches!(ty.kind(), ty::Uint(_) | ty::Char) =>
+            {
+                return Some(a.kind().cmp(&b.kind()));
+            }
+            (
+                Finite(mir::Const::Val(mir::ConstValue::Scalar(Scalar::Int(a)), _)),
+                Finite(mir::Const::Val(mir::ConstValue::Scalar(Scalar::Int(b)), _)),
+            ) if matches!(ty.kind(), ty::Uint(_) | ty::Char) => return Some(a.cmp(&b)),
+            _ => {}
+        }
+
+        let a = self.eval_bits(ty, tcx, param_env);
+        let b = other.eval_bits(ty, tcx, param_env);
+
+        match ty.kind() {
+            ty::Float(ty::FloatTy::F32) => {
+                use rustc_apfloat::Float;
+                let a = rustc_apfloat::ieee::Single::from_bits(a);
+                let b = rustc_apfloat::ieee::Single::from_bits(b);
+                a.partial_cmp(&b)
+            }
+            ty::Float(ty::FloatTy::F64) => {
+                use rustc_apfloat::Float;
+                let a = rustc_apfloat::ieee::Double::from_bits(a);
+                let b = rustc_apfloat::ieee::Double::from_bits(b);
+                a.partial_cmp(&b)
+            }
+            ty::Int(ity) => {
+                use rustc_middle::ty::layout::IntegerExt;
+                let size = rustc_target::abi::Integer::from_int_ty(&tcx, *ity).size();
+                let a = size.sign_extend(a) as i128;
+                let b = size.sign_extend(b) as i128;
+                Some(a.cmp(&b))
+            }
+            ty::Uint(_) | ty::Char => Some(a.cmp(&b)),
+            _ => bug!(),
+        }
+    }
 }
 
 impl<'tcx> fmt::Display for Pat<'tcx> {
@@ -944,11 +1151,7 @@ impl<'tcx> fmt::Display for Pat<'tcx> {
             PatKind::InlineConstant { def: _, ref subpattern } => {
                 write!(f, "{} (from inline const)", subpattern)
             }
-            PatKind::Range(box PatRange { lo, hi, end }) => {
-                write!(f, "{lo}")?;
-                write!(f, "{end}")?;
-                write!(f, "{hi}")
-            }
+            PatKind::Range(ref range) => write!(f, "{range}"),
             PatKind::Slice { ref prefix, ref slice, ref suffix }
             | PatKind::Array { ref prefix, ref slice, ref suffix } => {
                 write!(f, "[")?;
diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index be48c0e8926b..a2a74dafc5f0 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -19,7 +19,7 @@ use rustc_index::bit_set::GrowableBitSet;
 use rustc_macros::HashStable;
 use rustc_session::Limit;
 use rustc_span::sym;
-use rustc_target::abi::{Integer, IntegerType, Size};
+use rustc_target::abi::{Integer, IntegerType, Primitive, Size};
 use rustc_target::spec::abi::Abi;
 use smallvec::SmallVec;
 use std::{fmt, iter};
@@ -917,54 +917,62 @@ impl<'tcx> TypeFolder<TyCtxt<'tcx>> for OpaqueTypeExpander<'tcx> {
 }
 
 impl<'tcx> Ty<'tcx> {
+    /// Returns the `Size` for primitive types (bool, uint, int, char, float).
+    pub fn primitive_size(self, tcx: TyCtxt<'tcx>) -> Size {
+        match *self.kind() {
+            ty::Bool => Size::from_bytes(1),
+            ty::Char => Size::from_bytes(4),
+            ty::Int(ity) => Integer::from_int_ty(&tcx, ity).size(),
+            ty::Uint(uty) => Integer::from_uint_ty(&tcx, uty).size(),
+            ty::Float(ty::FloatTy::F32) => Primitive::F32.size(&tcx),
+            ty::Float(ty::FloatTy::F64) => Primitive::F64.size(&tcx),
+            _ => bug!("non primitive type"),
+        }
+    }
+
     pub fn int_size_and_signed(self, tcx: TyCtxt<'tcx>) -> (Size, bool) {
-        let (int, signed) = match *self.kind() {
-            ty::Int(ity) => (Integer::from_int_ty(&tcx, ity), true),
-            ty::Uint(uty) => (Integer::from_uint_ty(&tcx, uty), false),
+        match *self.kind() {
+            ty::Int(ity) => (Integer::from_int_ty(&tcx, ity).size(), true),
+            ty::Uint(uty) => (Integer::from_uint_ty(&tcx, uty).size(), false),
             _ => bug!("non integer discriminant"),
-        };
-        (int.size(), signed)
+        }
+    }
+
+    /// Returns the minimum and maximum values for the given numeric type (including `char`s) or
+    /// returns `None` if the type is not numeric.
+    pub fn numeric_min_and_max_as_bits(self, tcx: TyCtxt<'tcx>) -> Option<(u128, u128)> {
+        use rustc_apfloat::ieee::{Double, Single};
+        Some(match self.kind() {
+            ty::Int(_) | ty::Uint(_) => {
+                let (size, signed) = self.int_size_and_signed(tcx);
+                let min = if signed { size.truncate(size.signed_int_min() as u128) } else { 0 };
+                let max =
+                    if signed { size.signed_int_max() as u128 } else { size.unsigned_int_max() };
+                (min, max)
+            }
+            ty::Char => (0, std::char::MAX as u128),
+            ty::Float(ty::FloatTy::F32) => {
+                ((-Single::INFINITY).to_bits(), Single::INFINITY.to_bits())
+            }
+            ty::Float(ty::FloatTy::F64) => {
+                ((-Double::INFINITY).to_bits(), Double::INFINITY.to_bits())
+            }
+            _ => return None,
+        })
     }
 
     /// Returns the maximum value for the given numeric type (including `char`s)
     /// or returns `None` if the type is not numeric.
     pub fn numeric_max_val(self, tcx: TyCtxt<'tcx>) -> Option<ty::Const<'tcx>> {
-        let val = match self.kind() {
-            ty::Int(_) | ty::Uint(_) => {
-                let (size, signed) = self.int_size_and_signed(tcx);
-                let val =
-                    if signed { size.signed_int_max() as u128 } else { size.unsigned_int_max() };
-                Some(val)
-            }
-            ty::Char => Some(std::char::MAX as u128),
-            ty::Float(fty) => Some(match fty {
-                ty::FloatTy::F32 => rustc_apfloat::ieee::Single::INFINITY.to_bits(),
-                ty::FloatTy::F64 => rustc_apfloat::ieee::Double::INFINITY.to_bits(),
-            }),
-            _ => None,
-        };
-
-        val.map(|v| ty::Const::from_bits(tcx, v, ty::ParamEnv::empty().and(self)))
+        self.numeric_min_and_max_as_bits(tcx)
+            .map(|(_, max)| ty::Const::from_bits(tcx, max, ty::ParamEnv::empty().and(self)))
     }
 
     /// Returns the minimum value for the given numeric type (including `char`s)
     /// or returns `None` if the type is not numeric.
     pub fn numeric_min_val(self, tcx: TyCtxt<'tcx>) -> Option<ty::Const<'tcx>> {
-        let val = match self.kind() {
-            ty::Int(_) | ty::Uint(_) => {
-                let (size, signed) = self.int_size_and_signed(tcx);
-                let val = if signed { size.truncate(size.signed_int_min() as u128) } else { 0 };
-                Some(val)
-            }
-            ty::Char => Some(0),
-            ty::Float(fty) => Some(match fty {
-                ty::FloatTy::F32 => (-::rustc_apfloat::ieee::Single::INFINITY).to_bits(),
-                ty::FloatTy::F64 => (-::rustc_apfloat::ieee::Double::INFINITY).to_bits(),
-            }),
-            _ => None,
-        };
-
-        val.map(|v| ty::Const::from_bits(tcx, v, ty::ParamEnv::empty().and(self)))
+        self.numeric_min_and_max_as_bits(tcx)
+            .map(|(min, _)| ty::Const::from_bits(tcx, min, ty::ParamEnv::empty().and(self)))
     }
 
     /// Checks whether values of this type `T` are *moved* or *copied*
diff --git a/compiler/rustc_mir_build/src/build/matches/mod.rs b/compiler/rustc_mir_build/src/build/matches/mod.rs
index 1cf8c202ea4b..2e30cdd2c390 100644
--- a/compiler/rustc_mir_build/src/build/matches/mod.rs
+++ b/compiler/rustc_mir_build/src/build/matches/mod.rs
@@ -1027,7 +1027,7 @@ enum TestKind<'tcx> {
         ty: Ty<'tcx>,
     },
 
-    /// Test whether the value falls within an inclusive or exclusive range
+    /// Test whether the value falls within an inclusive or exclusive range.
     Range(Box<PatRange<'tcx>>),
 
     /// Test that the length of the slice is equal to `len`.
diff --git a/compiler/rustc_mir_build/src/build/matches/simplify.rs b/compiler/rustc_mir_build/src/build/matches/simplify.rs
index 32573b4d53af..6a40c8d840bd 100644
--- a/compiler/rustc_mir_build/src/build/matches/simplify.rs
+++ b/compiler/rustc_mir_build/src/build/matches/simplify.rs
@@ -15,11 +15,7 @@
 use crate::build::expr::as_place::PlaceBuilder;
 use crate::build::matches::{Ascription, Binding, Candidate, MatchPair};
 use crate::build::Builder;
-use rustc_hir::RangeEnd;
 use rustc_middle::thir::{self, *};
-use rustc_middle::ty;
-use rustc_middle::ty::layout::IntegerExt;
-use rustc_target::abi::{Integer, Size};
 
 use std::mem;
 
@@ -148,7 +144,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         match_pair: MatchPair<'pat, 'tcx>,
         candidate: &mut Candidate<'pat, 'tcx>,
     ) -> Result<(), MatchPair<'pat, 'tcx>> {
-        let tcx = self.tcx;
         match match_pair.pattern.kind {
             PatKind::AscribeUserType {
                 ref subpattern,
@@ -210,41 +205,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 Ok(())
             }
 
-            PatKind::Range(box PatRange { lo, hi, end }) => {
-                let (range, bias) = match *lo.ty().kind() {
-                    ty::Char => {
-                        (Some(('\u{0000}' as u128, '\u{10FFFF}' as u128, Size::from_bits(32))), 0)
-                    }
-                    ty::Int(ity) => {
-                        let size = Integer::from_int_ty(&tcx, ity).size();
-                        let max = size.truncate(u128::MAX);
-                        let bias = 1u128 << (size.bits() - 1);
-                        (Some((0, max, size)), bias)
-                    }
-                    ty::Uint(uty) => {
-                        let size = Integer::from_uint_ty(&tcx, uty).size();
-                        let max = size.truncate(u128::MAX);
-                        (Some((0, max, size)), 0)
-                    }
-                    _ => (None, 0),
-                };
-                if let Some((min, max, sz)) = range {
-                    // We want to compare ranges numerically, but the order of the bitwise
-                    // representation of signed integers does not match their numeric order. Thus,
-                    // to correct the ordering, we need to shift the range of signed integers to
-                    // correct the comparison. This is achieved by XORing with a bias (see
-                    // pattern/_match.rs for another pertinent example of this pattern).
-                    //
-                    // Also, for performance, it's important to only do the second
-                    // `try_to_bits` if necessary.
-                    let lo = lo.try_to_bits(sz).unwrap() ^ bias;
-                    if lo <= min {
-                        let hi = hi.try_to_bits(sz).unwrap() ^ bias;
-                        if hi > max || hi == max && end == RangeEnd::Included {
-                            // Irrefutable pattern match.
-                            return Ok(());
-                        }
-                    }
+            PatKind::Range(ref range) => {
+                if let Some(true) = range.is_full_range(self.tcx) {
+                    // Irrefutable pattern match.
+                    return Ok(());
                 }
                 Err(match_pair)
             }
diff --git a/compiler/rustc_mir_build/src/build/matches/test.rs b/compiler/rustc_mir_build/src/build/matches/test.rs
index 5e7db7413df4..bdd4f2011ebe 100644
--- a/compiler/rustc_mir_build/src/build/matches/test.rs
+++ b/compiler/rustc_mir_build/src/build/matches/test.rs
@@ -8,7 +8,6 @@
 use crate::build::expr::as_place::PlaceBuilder;
 use crate::build::matches::{Candidate, MatchPair, Test, TestKind};
 use crate::build::Builder;
-use crate::thir::pattern::compare_const_vals;
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_hir::{LangItem, RangeEnd};
 use rustc_index::bit_set::BitSet;
@@ -59,8 +58,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             },
 
             PatKind::Range(ref range) => {
-                assert_eq!(range.lo.ty(), match_pair.pattern.ty);
-                assert_eq!(range.hi.ty(), match_pair.pattern.ty);
+                assert_eq!(range.ty, match_pair.pattern.ty);
                 Test { span: match_pair.pattern.span, kind: TestKind::Range(range.clone()) }
             }
 
@@ -309,11 +307,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 }
             }
 
-            TestKind::Range(box PatRange { lo, hi, ref end }) => {
+            TestKind::Range(ref range) => {
                 let lower_bound_success = self.cfg.start_new_block();
                 let target_blocks = make_target_blocks(self);
 
                 // Test `val` by computing `lo <= val && val <= hi`, using primitive comparisons.
+                // FIXME: skip useless comparison when the range is half-open.
+                let lo = range.lo.to_const(range.ty, self.tcx);
+                let hi = range.hi.to_const(range.ty, self.tcx);
                 let lo = self.literal_operand(test.span, lo);
                 let hi = self.literal_operand(test.span, hi);
                 let val = Operand::Copy(place);
@@ -330,7 +331,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                     lo,
                     val.clone(),
                 );
-                let op = match *end {
+                let op = match range.end {
                     RangeEnd::Included => BinOp::Le,
                     RangeEnd::Excluded => BinOp::Lt,
                 };
@@ -698,34 +699,18 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
             }
 
             (TestKind::Range(test), PatKind::Range(pat)) => {
-                use std::cmp::Ordering::*;
-
                 if test == pat {
                     self.candidate_without_match_pair(match_pair_index, candidate);
                     return Some(0);
                 }
 
-                // For performance, it's important to only do the second
-                // `compare_const_vals` if necessary.
-                let no_overlap = if matches!(
-                    (compare_const_vals(self.tcx, test.hi, pat.lo, self.param_env)?, test.end),
-                    (Less, _) | (Equal, RangeEnd::Excluded) // test < pat
-                ) || matches!(
-                    (compare_const_vals(self.tcx, test.lo, pat.hi, self.param_env)?, pat.end),
-                    (Greater, _) | (Equal, RangeEnd::Excluded) // test > pat
-                ) {
-                    Some(1)
-                } else {
-                    None
-                };
-
                 // If the testing range does not overlap with pattern range,
                 // the pattern can be matched only if this test fails.
-                no_overlap
+                if !test.overlaps(pat, self.tcx, self.param_env)? { Some(1) } else { None }
             }
 
             (TestKind::Range(range), &PatKind::Constant { value }) => {
-                if let Some(false) = self.const_range_contains(&*range, value) {
+                if !range.contains(value, self.tcx, self.param_env)? {
                     // `value` is not contained in the testing range,
                     // so `value` can be matched only if this test fails.
                     Some(1)
@@ -817,27 +802,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
         span_bug!(match_pair.pattern.span, "simplifiable pattern found: {:?}", match_pair.pattern)
     }
 
-    fn const_range_contains(&self, range: &PatRange<'tcx>, value: Const<'tcx>) -> Option<bool> {
-        use std::cmp::Ordering::*;
-
-        // For performance, it's important to only do the second
-        // `compare_const_vals` if necessary.
-        Some(
-            matches!(compare_const_vals(self.tcx, range.lo, value, self.param_env)?, Less | Equal)
-                && matches!(
-                    (compare_const_vals(self.tcx, value, range.hi, self.param_env)?, range.end),
-                    (Less, _) | (Equal, RangeEnd::Included)
-                ),
-        )
-    }
-
     fn values_not_contained_in_range(
         &self,
         range: &PatRange<'tcx>,
         options: &FxIndexMap<Const<'tcx>, u128>,
     ) -> Option<bool> {
         for &val in options.keys() {
-            if self.const_range_contains(range, val)? {
+            if range.contains(val, self.tcx, self.param_env)? {
                 return Some(false);
             }
         }
diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
index 186c77795e49..f93daa29afb5 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
@@ -57,7 +57,7 @@ use rustc_hir::RangeEnd;
 use rustc_index::Idx;
 use rustc_middle::middle::stability::EvalResult;
 use rustc_middle::mir;
-use rustc_middle::thir::{FieldPat, Pat, PatKind, PatRange};
+use rustc_middle::thir::{FieldPat, Pat, PatKind, PatRange, PatRangeBoundary};
 use rustc_middle::ty::layout::IntegerExt;
 use rustc_middle::ty::{self, Ty, TyCtxt, VariantDef};
 use rustc_span::{Span, DUMMY_SP};
@@ -278,19 +278,20 @@ impl IntRange {
         let (lo, hi) = self.boundaries();
 
         let bias = IntRange::signed_bias(tcx, ty);
-        let (lo, hi) = (lo ^ bias, hi ^ bias);
+        let (lo_bits, hi_bits) = (lo ^ bias, hi ^ bias);
 
         let env = ty::ParamEnv::empty().and(ty);
-        let lo_const = mir::Const::from_bits(tcx, lo, env);
-        let hi_const = mir::Const::from_bits(tcx, hi, env);
+        let lo_const = mir::Const::from_bits(tcx, lo_bits, env);
+        let hi_const = mir::Const::from_bits(tcx, hi_bits, env);
 
-        let kind = if lo == hi {
+        let kind = if lo_bits == hi_bits {
             PatKind::Constant { value: lo_const }
         } else {
             PatKind::Range(Box::new(PatRange {
-                lo: lo_const,
-                hi: hi_const,
+                lo: PatRangeBoundary::Finite(lo_const),
+                hi: PatRangeBoundary::Finite(hi_const),
                 end: RangeEnd::Included,
+                ty,
             }))
         };
 
@@ -1387,11 +1388,12 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
                     }
                 }
             }
-            PatKind::Range(box PatRange { lo, hi, end }) => {
+            PatKind::Range(box PatRange { lo, hi, end, .. }) => {
                 use rustc_apfloat::Float;
-                let ty = lo.ty();
-                let lo = lo.try_eval_bits(cx.tcx, cx.param_env).unwrap();
-                let hi = hi.try_eval_bits(cx.tcx, cx.param_env).unwrap();
+                let ty = pat.ty;
+                // FIXME: handle half-open ranges
+                let lo = lo.eval_bits(ty, cx.tcx, cx.param_env);
+                let hi = hi.eval_bits(ty, cx.tcx, cx.param_env);
                 ctor = match ty.kind() {
                     ty::Char | ty::Int(_) | ty::Uint(_) => {
                         IntRange(IntRange::from_range(cx.tcx, lo, hi, ty, *end))
diff --git a/compiler/rustc_mir_build/src/thir/pattern/mod.rs b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
index dd71ab1f8e55..c530c0a772c9 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
@@ -17,11 +17,11 @@ use rustc_hir::def::{CtorOf, DefKind, Res};
 use rustc_hir::pat_util::EnumerateAndAdjustIterator;
 use rustc_hir::RangeEnd;
 use rustc_index::Idx;
-use rustc_middle::mir::interpret::{
-    ErrorHandled, GlobalId, LitToConstError, LitToConstInput, Scalar,
-};
+use rustc_middle::mir::interpret::{ErrorHandled, GlobalId, LitToConstError, LitToConstInput};
 use rustc_middle::mir::{self, BorrowKind, Const, Mutability, UserTypeProjection};
-use rustc_middle::thir::{Ascription, BindingMode, FieldPat, LocalVarId, Pat, PatKind, PatRange};
+use rustc_middle::thir::{
+    Ascription, BindingMode, FieldPat, LocalVarId, Pat, PatKind, PatRange, PatRangeBoundary,
+};
 use rustc_middle::ty::layout::IntegerExt;
 use rustc_middle::ty::{
     self, AdtDef, CanonicalUserTypeAnnotation, GenericArg, GenericArgsRef, Region, Ty, TyCtxt,
@@ -90,7 +90,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         &mut self,
         expr: Option<&'tcx hir::Expr<'tcx>>,
     ) -> Result<
-        (Option<mir::Const<'tcx>>, Option<Ascription<'tcx>>, Option<LocalDefId>),
+        (Option<PatRangeBoundary<'tcx>>, Option<Ascription<'tcx>>, Option<LocalDefId>),
         ErrorGuaranteed,
     > {
         match expr {
@@ -113,7 +113,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                     );
                     return Err(self.tcx.sess.delay_span_bug(expr.span, msg));
                 };
-                Ok((Some(value), ascr, inline_const))
+                Ok((Some(PatRangeBoundary::Finite(value)), ascr, inline_const))
             }
         }
     }
@@ -187,31 +187,23 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         let (lo, lo_ascr, lo_inline) = self.lower_pattern_range_endpoint(lo_expr)?;
         let (hi, hi_ascr, hi_inline) = self.lower_pattern_range_endpoint(hi_expr)?;
 
-        let lo = lo.unwrap_or_else(|| {
-            // Unwrap is ok because the type is known to be numeric.
-            let lo = ty.numeric_min_val(self.tcx).unwrap();
-            mir::Const::from_ty_const(lo, self.tcx)
-        });
-        let hi = hi.unwrap_or_else(|| {
-            // Unwrap is ok because the type is known to be numeric.
-            let hi = ty.numeric_max_val(self.tcx).unwrap();
-            mir::Const::from_ty_const(hi, self.tcx)
-        });
-        assert_eq!(lo.ty(), ty);
-        assert_eq!(hi.ty(), ty);
+        let lo = lo.unwrap_or_else(|| PatRangeBoundary::lower_bound(ty, self.tcx));
+        let hi = hi.unwrap_or_else(|| PatRangeBoundary::upper_bound(ty, self.tcx));
 
-        let cmp = compare_const_vals(self.tcx, lo, hi, self.param_env);
+        let cmp = lo.compare_with(hi, ty, self.tcx, self.param_env);
         let mut kind = match (end, cmp) {
             // `x..y` where `x < y`.
             // Non-empty because the range includes at least `x`.
             (RangeEnd::Excluded, Some(Ordering::Less)) => {
-                PatKind::Range(Box::new(PatRange { lo, hi, end }))
+                PatKind::Range(Box::new(PatRange { lo, hi, end, ty }))
             }
             // `x..=y` where `x == y`.
-            (RangeEnd::Included, Some(Ordering::Equal)) => PatKind::Constant { value: lo },
+            (RangeEnd::Included, Some(Ordering::Equal)) => {
+                PatKind::Constant { value: lo.to_const(ty, self.tcx) }
+            }
             // `x..=y` where `x < y`.
             (RangeEnd::Included, Some(Ordering::Less)) => {
-                PatKind::Range(Box::new(PatRange { lo, hi, end }))
+                PatKind::Range(Box::new(PatRange { lo, hi, end, ty }))
             }
             // `x..y` where `x >= y`, or `x..=y` where `x > y`. The range is empty => error.
             _ => {
@@ -851,59 +843,3 @@ impl<'tcx> PatternFoldable<'tcx> for PatKind<'tcx> {
         }
     }
 }
-
-#[instrument(skip(tcx), level = "debug")]
-pub(crate) fn compare_const_vals<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    a: mir::Const<'tcx>,
-    b: mir::Const<'tcx>,
-    param_env: ty::ParamEnv<'tcx>,
-) -> Option<Ordering> {
-    assert_eq!(a.ty(), b.ty());
-
-    let ty = a.ty();
-
-    // This code is hot when compiling matches with many ranges. So we
-    // special-case extraction of evaluated scalars for speed, for types where
-    // raw data comparisons are appropriate. E.g. `unicode-normalization` has
-    // many ranges such as '\u{037A}'..='\u{037F}', and chars can be compared
-    // in this way.
-    match ty.kind() {
-        ty::Float(_) | ty::Int(_) => {} // require special handling, see below
-        _ => match (a, b) {
-            (
-                mir::Const::Val(mir::ConstValue::Scalar(Scalar::Int(a)), _a_ty),
-                mir::Const::Val(mir::ConstValue::Scalar(Scalar::Int(b)), _b_ty),
-            ) => return Some(a.cmp(&b)),
-            (mir::Const::Ty(a), mir::Const::Ty(b)) => {
-                return Some(a.kind().cmp(&b.kind()));
-            }
-            _ => {}
-        },
-    }
-
-    let a = a.eval_bits(tcx, param_env);
-    let b = b.eval_bits(tcx, param_env);
-
-    use rustc_apfloat::Float;
-    match *ty.kind() {
-        ty::Float(ty::FloatTy::F32) => {
-            let a = rustc_apfloat::ieee::Single::from_bits(a);
-            let b = rustc_apfloat::ieee::Single::from_bits(b);
-            a.partial_cmp(&b)
-        }
-        ty::Float(ty::FloatTy::F64) => {
-            let a = rustc_apfloat::ieee::Double::from_bits(a);
-            let b = rustc_apfloat::ieee::Double::from_bits(b);
-            a.partial_cmp(&b)
-        }
-        ty::Int(ity) => {
-            use rustc_middle::ty::layout::IntegerExt;
-            let size = rustc_target::abi::Integer::from_int_ty(&tcx, ity).size();
-            let a = size.sign_extend(a);
-            let b = size.sign_extend(b);
-            Some((a as i128).cmp(&(b as i128)))
-        }
-        _ => Some(a.cmp(&b)),
-    }
-}

From 0ba6c4ab676c7845de2d3b9209139e02cbfe8fae Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Fri, 13 Oct 2023 01:22:37 +0200
Subject: [PATCH 155/297] Propagate half-open ranges through THIR

---
 compiler/rustc_middle/src/thir.rs             | 78 ++++++++++++-------
 .../rustc_mir_build/src/thir/pattern/mod.rs   | 29 +++----
 2 files changed, 67 insertions(+), 40 deletions(-)

diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs
index fefdce8e77e4..9457b62f22c1 100644
--- a/compiler/rustc_middle/src/thir.rs
+++ b/compiler/rustc_middle/src/thir.rs
@@ -851,17 +851,21 @@ impl<'tcx> PatRange<'tcx> {
         //
         // Also, for performance, it's important to only do the second `try_to_bits` if necessary.
         let lo_is_min = match self.lo {
+            PatRangeBoundary::NegInfinity => true,
             PatRangeBoundary::Finite(value) => {
                 let lo = value.try_to_bits(size).unwrap() ^ bias;
                 lo <= min
             }
+            PatRangeBoundary::PosInfinity => false,
         };
         if lo_is_min {
             let hi_is_max = match self.hi {
+                PatRangeBoundary::NegInfinity => false,
                 PatRangeBoundary::Finite(value) => {
                     let hi = value.try_to_bits(size).unwrap() ^ bias;
                     hi > max || hi == max && self.end == RangeEnd::Included
                 }
+                PatRangeBoundary::PosInfinity => true,
             };
             if hi_is_max {
                 return Some(true);
@@ -920,11 +924,16 @@ impl<'tcx> PatRange<'tcx> {
 
 impl<'tcx> fmt::Display for PatRange<'tcx> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let PatRangeBoundary::Finite(value) = &self.lo;
-        write!(f, "{value}")?;
-        write!(f, "{}", self.end)?;
-        let PatRangeBoundary::Finite(value) = &self.hi;
-        write!(f, "{value}")?;
+        if let PatRangeBoundary::Finite(value) = &self.lo {
+            write!(f, "{value}")?;
+        }
+        if let PatRangeBoundary::Finite(value) = &self.hi {
+            write!(f, "{}", self.end)?;
+            write!(f, "{value}")?;
+        } else {
+            // `0..` is parsed as an inclusive range, we must display it correctly.
+            write!(f, "..")?;
+        }
         Ok(())
     }
 }
@@ -934,38 +943,49 @@ impl<'tcx> fmt::Display for PatRange<'tcx> {
 #[derive(Copy, Clone, Debug, PartialEq, HashStable, TypeVisitable)]
 pub enum PatRangeBoundary<'tcx> {
     Finite(mir::Const<'tcx>),
+    NegInfinity,
+    PosInfinity,
 }
 
 impl<'tcx> PatRangeBoundary<'tcx> {
     #[inline]
-    pub fn lower_bound(ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Self {
-        // Unwrap is ok because the type is known to be numeric.
-        let c = ty.numeric_min_val(tcx).unwrap();
-        let value = mir::Const::from_ty_const(c, tcx);
-        Self::Finite(value)
+    pub fn is_finite(self) -> bool {
+        matches!(self, Self::Finite(..))
     }
     #[inline]
-    pub fn upper_bound(ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Self {
-        // Unwrap is ok because the type is known to be numeric.
-        let c = ty.numeric_max_val(tcx).unwrap();
-        let value = mir::Const::from_ty_const(c, tcx);
-        Self::Finite(value)
-    }
-
-    #[inline]
-    pub fn to_const(self, _ty: Ty<'tcx>, _tcx: TyCtxt<'tcx>) -> mir::Const<'tcx> {
+    pub fn as_finite(self) -> Option<mir::Const<'tcx>> {
         match self {
-            Self::Finite(value) => value,
+            Self::Finite(value) => Some(value),
+            Self::NegInfinity | Self::PosInfinity => None,
         }
     }
-    pub fn eval_bits(
-        self,
-        _ty: Ty<'tcx>,
-        tcx: TyCtxt<'tcx>,
-        param_env: ty::ParamEnv<'tcx>,
-    ) -> u128 {
+    #[inline]
+    pub fn to_const(self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> mir::Const<'tcx> {
+        match self {
+            Self::Finite(value) => value,
+            Self::NegInfinity => {
+                // Unwrap is ok because the type is known to be numeric.
+                let c = ty.numeric_min_val(tcx).unwrap();
+                mir::Const::from_ty_const(c, tcx)
+            }
+            Self::PosInfinity => {
+                // Unwrap is ok because the type is known to be numeric.
+                let c = ty.numeric_max_val(tcx).unwrap();
+                mir::Const::from_ty_const(c, tcx)
+            }
+        }
+    }
+    pub fn eval_bits(self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>) -> u128 {
         match self {
             Self::Finite(value) => value.eval_bits(tcx, param_env),
+            Self::NegInfinity => {
+                // Unwrap is ok because the type is known to be numeric.
+                ty.numeric_min_and_max_as_bits(tcx).unwrap().0
+            }
+            Self::PosInfinity => {
+                // Unwrap is ok because the type is known to be numeric.
+                ty.numeric_min_and_max_as_bits(tcx).unwrap().1
+            }
         }
     }
 
@@ -979,6 +999,12 @@ impl<'tcx> PatRangeBoundary<'tcx> {
     ) -> Option<Ordering> {
         use PatRangeBoundary::*;
         match (self, other) {
+            // When comparing with infinities, we must remember that `0u8..` and `0u8..=255`
+            // describe the same range. These two shortcuts are ok, but for the rest we must check
+            // bit values.
+            (PosInfinity, PosInfinity) => return Some(Ordering::Equal),
+            (NegInfinity, NegInfinity) => return Some(Ordering::Equal),
+
             // This code is hot when compiling matches with many ranges. So we
             // special-case extraction of evaluated scalars for speed, for types where
             // raw data comparisons are appropriate. E.g. `unicode-normalization` has
diff --git a/compiler/rustc_mir_build/src/thir/pattern/mod.rs b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
index c530c0a772c9..0811ab6a0a6b 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/mod.rs
@@ -187,24 +187,25 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
         let (lo, lo_ascr, lo_inline) = self.lower_pattern_range_endpoint(lo_expr)?;
         let (hi, hi_ascr, hi_inline) = self.lower_pattern_range_endpoint(hi_expr)?;
 
-        let lo = lo.unwrap_or_else(|| PatRangeBoundary::lower_bound(ty, self.tcx));
-        let hi = hi.unwrap_or_else(|| PatRangeBoundary::upper_bound(ty, self.tcx));
+        let lo = lo.unwrap_or(PatRangeBoundary::NegInfinity);
+        let hi = hi.unwrap_or(PatRangeBoundary::PosInfinity);
 
         let cmp = lo.compare_with(hi, ty, self.tcx, self.param_env);
-        let mut kind = match (end, cmp) {
+        let mut kind = PatKind::Range(Box::new(PatRange { lo, hi, end, ty }));
+        match (end, cmp) {
             // `x..y` where `x < y`.
-            // Non-empty because the range includes at least `x`.
-            (RangeEnd::Excluded, Some(Ordering::Less)) => {
-                PatKind::Range(Box::new(PatRange { lo, hi, end, ty }))
-            }
-            // `x..=y` where `x == y`.
-            (RangeEnd::Included, Some(Ordering::Equal)) => {
-                PatKind::Constant { value: lo.to_const(ty, self.tcx) }
-            }
+            (RangeEnd::Excluded, Some(Ordering::Less)) => {}
             // `x..=y` where `x < y`.
-            (RangeEnd::Included, Some(Ordering::Less)) => {
-                PatKind::Range(Box::new(PatRange { lo, hi, end, ty }))
+            (RangeEnd::Included, Some(Ordering::Less)) => {}
+            // `x..=y` where `x == y` and `x` and `y` are finite.
+            (RangeEnd::Included, Some(Ordering::Equal)) if lo.is_finite() && hi.is_finite() => {
+                kind = PatKind::Constant { value: lo.as_finite().unwrap() };
             }
+            // `..=x` where `x == ty::MIN`.
+            (RangeEnd::Included, Some(Ordering::Equal)) if !lo.is_finite() => {}
+            // `x..` where `x == ty::MAX` (yes, `x..` gives `RangeEnd::Included` since it is meant
+            // to include `ty::MAX`).
+            (RangeEnd::Included, Some(Ordering::Equal)) if !hi.is_finite() => {}
             // `x..y` where `x >= y`, or `x..=y` where `x > y`. The range is empty => error.
             _ => {
                 // Emit a more appropriate message if there was overflow.
@@ -223,7 +224,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> {
                 };
                 return Err(e);
             }
-        };
+        }
 
         // If we are handling a range with associated constants (e.g.
         // `Foo::<'a>::A..=Foo::B`), we need to put the ascriptions for the associated

From a5c67f4107530506c7bc1752cfc7437fa7ff933a Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Thu, 12 Oct 2023 16:51:27 +0200
Subject: [PATCH 156/297] Don't use `IntRange` for booleans

---
 .../src/thir/pattern/deconstruct_pat.rs       | 70 +++++++++++++++----
 1 file changed, 56 insertions(+), 14 deletions(-)

diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
index f93daa29afb5..c13307aa419b 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
@@ -110,7 +110,7 @@ pub(crate) struct IntRange {
 impl IntRange {
     #[inline]
     pub(super) fn is_integral(ty: Ty<'_>) -> bool {
-        matches!(ty.kind(), ty::Char | ty::Int(_) | ty::Uint(_) | ty::Bool)
+        matches!(ty.kind(), ty::Char | ty::Int(_) | ty::Uint(_))
     }
 
     pub(super) fn is_singleton(&self) -> bool {
@@ -299,8 +299,8 @@ impl IntRange {
     }
 }
 
-/// Note: this is often not what we want: e.g. `false` is converted into the range `0..=0` and
-/// would be displayed as such. To render properly, convert to a pattern first.
+/// Note: this will render signed ranges incorrectly. To render properly, convert to a pattern
+/// first.
 impl fmt::Debug for IntRange {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let (lo, hi) = self.boundaries();
@@ -541,6 +541,8 @@ pub(super) enum Constructor<'tcx> {
     Single,
     /// Enum variants.
     Variant(VariantIdx),
+    /// Booleans
+    Bool(bool),
     /// Ranges of integer literal values (`2`, `2..=5` or `2..5`).
     IntRange(IntRange),
     /// Ranges of floating-point literal values (`2.0..=5.2`).
@@ -581,6 +583,12 @@ impl<'tcx> Constructor<'tcx> {
             _ => None,
         }
     }
+    fn as_bool(&self) -> Option<bool> {
+        match self {
+            Bool(b) => Some(*b),
+            _ => None,
+        }
+    }
     pub(super) fn as_int_range(&self) -> Option<&IntRange> {
         match self {
             IntRange(range) => Some(range),
@@ -625,10 +633,11 @@ impl<'tcx> Constructor<'tcx> {
                 _ => bug!("Unexpected type for `Single` constructor: {:?}", pcx.ty),
             },
             Slice(slice) => slice.arity(),
-            Str(..)
+            Bool(..)
+            | IntRange(..)
             | F32Range(..)
             | F64Range(..)
-            | IntRange(..)
+            | Str(..)
             | Opaque
             | NonExhaustive
             | Hidden
@@ -744,6 +753,7 @@ impl<'tcx> Constructor<'tcx> {
 
             (Single, Single) => true,
             (Variant(self_id), Variant(other_id)) => self_id == other_id,
+            (Bool(self_b), Bool(other_b)) => self_b == other_b,
 
             (IntRange(self_range), IntRange(other_range)) => self_range.is_subrange(other_range),
             (F32Range(self_from, self_to, self_end), F32Range(other_from, other_to, other_end)) => {
@@ -796,9 +806,10 @@ pub(super) enum ConstructorSet {
         hidden_variants: Vec<VariantIdx>,
         non_exhaustive: bool,
     },
+    /// Booleans.
+    Bool,
     /// The type is spanned by integer values. The range or ranges give the set of allowed values.
     /// The second range is only useful for `char`.
-    /// This is reused for bool. FIXME: don't.
     /// `non_exhaustive` is used when the range is not allowed to be matched exhaustively (that's
     /// for usize/isize).
     Integers { range_1: IntRange, range_2: Option<IntRange>, non_exhaustive: bool },
@@ -848,9 +859,7 @@ impl ConstructorSet {
         // Invariant: this is `Uninhabited` if and only if the type is uninhabited (as determined by
         // `cx.is_uninhabited()`).
         match ty.kind() {
-            ty::Bool => {
-                Self::Integers { range_1: make_range(0, 1), range_2: None, non_exhaustive: false }
-            }
+            ty::Bool => Self::Bool,
             ty::Char => {
                 // The valid Unicode Scalar Value ranges.
                 Self::Integers {
@@ -1010,6 +1019,27 @@ impl ConstructorSet {
                     missing.push(NonExhaustive);
                 }
             }
+            ConstructorSet::Bool => {
+                let mut seen_false = false;
+                let mut seen_true = false;
+                for b in seen.map(|ctor| ctor.as_bool().unwrap()) {
+                    if b {
+                        seen_true = true;
+                    } else {
+                        seen_false = true;
+                    }
+                }
+                if seen_false {
+                    present.push(Bool(false));
+                } else {
+                    missing.push(Bool(false));
+                }
+                if seen_true {
+                    present.push(Bool(true));
+                } else {
+                    missing.push(Bool(true));
+                }
+            }
             ConstructorSet::Integers { range_1, range_2, non_exhaustive } => {
                 let seen_ranges: Vec<_> =
                     seen.map(|ctor| ctor.as_int_range().unwrap().clone()).collect();
@@ -1205,10 +1235,11 @@ impl<'p, 'tcx> Fields<'p, 'tcx> {
                 }
                 _ => bug!("bad slice pattern {:?} {:?}", constructor, pcx),
             },
-            Str(..)
+            Bool(..)
+            | IntRange(..)
             | F32Range(..)
             | F64Range(..)
-            | IntRange(..)
+            | Str(..)
             | Opaque
             | NonExhaustive
             | Hidden
@@ -1337,7 +1368,14 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
             }
             PatKind::Constant { value } => {
                 match pat.ty.kind() {
-                    ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) => {
+                    ty::Bool => {
+                        ctor = match value.try_eval_bool(cx.tcx, cx.param_env) {
+                            Some(b) => Bool(b),
+                            None => Opaque,
+                        };
+                        fields = Fields::empty();
+                    }
+                    ty::Char | ty::Int(_) | ty::Uint(_) => {
                         ctor = match value.try_eval_bits(cx.tcx, cx.param_env) {
                             Some(bits) => IntRange(IntRange::from_bits(cx.tcx, pat.ty, bits)),
                             None => Opaque,
@@ -1616,9 +1654,11 @@ impl<'p, 'tcx> fmt::Debug for DeconstructedPat<'p, 'tcx> {
                 }
                 write!(f, "]")
             }
+            Bool(b) => write!(f, "{b}"),
+            // Best-effort, will render signed ranges incorrectly
+            IntRange(range) => write!(f, "{range:?}"),
             F32Range(lo, hi, end) => write!(f, "{lo}{end}{hi}"),
             F64Range(lo, hi, end) => write!(f, "{lo}{end}{hi}"),
-            IntRange(range) => write!(f, "{range:?}"), // Best-effort, will render e.g. `false` as `0..=0`
             Str(value) => write!(f, "{value}"),
             Opaque => write!(f, "<constant pattern>"),
             Or => {
@@ -1668,10 +1708,13 @@ impl<'tcx> WitnessPat<'tcx> {
         self.ty
     }
 
+    /// Convert back to a `thir::Pat` for diagnostic purposes.
     pub(crate) fn to_pat(&self, cx: &MatchCheckCtxt<'_, 'tcx>) -> Pat<'tcx> {
         let is_wildcard = |pat: &Pat<'_>| matches!(pat.kind, PatKind::Wild);
         let mut subpatterns = self.iter_fields().map(|p| Box::new(p.to_pat(cx)));
         let kind = match &self.ctor {
+            Bool(b) => PatKind::Constant { value: mir::Const::from_bool(cx.tcx, *b) },
+            IntRange(range) => return range.to_pat(cx.tcx, self.ty),
             Single | Variant(_) => match self.ty.kind() {
                 ty::Tuple(..) => PatKind::Leaf {
                     subpatterns: subpatterns
@@ -1741,7 +1784,6 @@ impl<'tcx> WitnessPat<'tcx> {
                 }
             }
             &Str(value) => PatKind::Constant { value },
-            IntRange(range) => return range.to_pat(cx.tcx, self.ty),
             Wildcard | NonExhaustive | Hidden => PatKind::Wild,
             Missing { .. } => bug!(
                 "trying to convert a `Missing` constructor into a `Pat`; this is probably a bug,

From 9bc4c378ab3bbdd330ccd1268fae478a0f0905f2 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Thu, 12 Oct 2023 16:19:02 +0200
Subject: [PATCH 157/297] Inline `RangeInclusive` into `IntRange`

---
 .../src/thir/pattern/deconstruct_pat.rs       | 56 ++++++++-----------
 .../src/thir/pattern/usefulness.rs            | 18 +++---
 2 files changed, 33 insertions(+), 41 deletions(-)

diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
index c13307aa419b..249273c3595f 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
@@ -46,7 +46,6 @@ use std::cell::Cell;
 use std::cmp::{self, max, min, Ordering};
 use std::fmt;
 use std::iter::once;
-use std::ops::RangeInclusive;
 
 use smallvec::{smallvec, SmallVec};
 
@@ -102,9 +101,10 @@ enum Presence {
 ///
 /// `IntRange` is never used to encode an empty range or a "range" that wraps
 /// around the (offset) space: i.e., `range.lo <= range.hi`.
-#[derive(Clone, PartialEq, Eq)]
+#[derive(Clone, Copy, PartialEq, Eq)]
 pub(crate) struct IntRange {
-    range: RangeInclusive<u128>,
+    pub(crate) lo: u128,
+    pub(crate) hi: u128,
 }
 
 impl IntRange {
@@ -114,20 +114,16 @@ impl IntRange {
     }
 
     pub(super) fn is_singleton(&self) -> bool {
-        self.range.start() == self.range.end()
-    }
-
-    pub(super) fn boundaries(&self) -> (u128, u128) {
-        (*self.range.start(), *self.range.end())
+        self.lo == self.hi
     }
 
     #[inline]
     fn from_bits<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, bits: u128) -> IntRange {
         let bias = IntRange::signed_bias(tcx, ty);
-        // Perform a shift if the underlying types are signed,
-        // which makes the interval arithmetic simpler.
+        // Perform a shift if the underlying types are signed, which makes the interval arithmetic
+        // type-independent.
         let val = bits ^ bias;
-        IntRange { range: val..=val }
+        IntRange { lo: val, hi: val }
     }
 
     #[inline]
@@ -138,16 +134,17 @@ impl IntRange {
         ty: Ty<'tcx>,
         end: RangeEnd,
     ) -> IntRange {
-        // Perform a shift if the underlying types are signed,
-        // which makes the interval arithmetic simpler.
+        // Perform a shift if the underlying types are signed, which makes the interval arithmetic
+        // type-independent.
         let bias = IntRange::signed_bias(tcx, ty);
         let (lo, hi) = (lo ^ bias, hi ^ bias);
         let offset = (end == RangeEnd::Excluded) as u128;
-        if lo > hi || (lo == hi && end == RangeEnd::Excluded) {
+        let hi = hi - offset;
+        if lo > hi {
             // This should have been caught earlier by E0030.
-            bug!("malformed range pattern: {}..={}", lo, (hi - offset));
+            bug!("malformed range pattern: {lo}..={hi}");
         }
-        IntRange { range: lo..=(hi - offset) }
+        IntRange { lo, hi }
     }
 
     // The return value of `signed_bias` should be XORed with an endpoint to encode/decode it.
@@ -162,14 +159,12 @@ impl IntRange {
     }
 
     fn is_subrange(&self, other: &Self) -> bool {
-        other.range.start() <= self.range.start() && self.range.end() <= other.range.end()
+        other.lo <= self.lo && self.hi <= other.hi
     }
 
     fn intersection(&self, other: &Self) -> Option<Self> {
-        let (lo, hi) = self.boundaries();
-        let (other_lo, other_hi) = other.boundaries();
-        if lo <= other_hi && other_lo <= hi {
-            Some(IntRange { range: max(lo, other_lo)..=min(hi, other_hi) })
+        if self.lo <= other.hi && other.lo <= self.hi {
+            Some(IntRange { lo: max(self.lo, other.lo), hi: min(self.hi, other.hi) })
         } else {
             None
         }
@@ -216,9 +211,8 @@ impl IntRange {
 
         fn unpack_intrange(range: IntRange) -> [IntBoundary; 2] {
             use IntBoundary::*;
-            let (lo, hi) = range.boundaries();
-            let lo = JustBefore(lo);
-            let hi = match hi.checked_add(1) {
+            let lo = JustBefore(range.lo);
+            let hi = match range.hi.checked_add(1) {
                 Some(m) => JustBefore(m),
                 None => AfterMax,
             };
@@ -264,21 +258,19 @@ impl IntRange {
                 use IntBoundary::*;
                 use Presence::*;
                 let presence = if paren_count > 0 { Seen } else { Unseen };
-                let range = match (prev_bdy, bdy) {
-                    (JustBefore(n), JustBefore(m)) if n < m => n..=(m - 1),
-                    (JustBefore(n), AfterMax) => n..=u128::MAX,
+                let (lo, hi) = match (prev_bdy, bdy) {
+                    (JustBefore(n), JustBefore(m)) if n < m => (n, m - 1),
+                    (JustBefore(n), AfterMax) => (n, u128::MAX),
                     _ => unreachable!(), // Ruled out by the sorting and filtering we did
                 };
-                (presence, IntRange { range })
+                (presence, IntRange { lo, hi })
             })
     }
 
     /// Only used for displaying the range.
     pub(super) fn to_pat<'tcx>(&self, tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Pat<'tcx> {
-        let (lo, hi) = self.boundaries();
-
         let bias = IntRange::signed_bias(tcx, ty);
-        let (lo_bits, hi_bits) = (lo ^ bias, hi ^ bias);
+        let (lo_bits, hi_bits) = (self.lo ^ bias, self.hi ^ bias);
 
         let env = ty::ParamEnv::empty().and(ty);
         let lo_const = mir::Const::from_bits(tcx, lo_bits, env);
@@ -303,7 +295,7 @@ impl IntRange {
 /// first.
 impl fmt::Debug for IntRange {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let (lo, hi) = self.boundaries();
+        let (lo, hi) = (self.lo, self.hi);
         write!(f, "{lo}")?;
         write!(f, "{}", RangeEnd::Included)?;
         write!(f, "{hi}")
diff --git a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
index 25e0f3ceaa47..4b314535c893 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
@@ -1031,9 +1031,10 @@ fn lint_overlapping_range_endpoints<'p, 'tcx>(
         let split_int_ranges = set.present.iter().filter_map(|c| c.as_int_range());
         for overlap_range in split_int_ranges.clone() {
             if overlap_range.is_singleton() {
-                let overlap: u128 = overlap_range.boundaries().0;
-                // Spans of ranges that start or end with the overlap.
+                let overlap: u128 = overlap_range.lo;
+                // Ranges that look like `lo..=overlap`.
                 let mut prefixes: SmallVec<[_; 1]> = Default::default();
+                // Ranges that look like `overlap..=hi`.
                 let mut suffixes: SmallVec<[_; 1]> = Default::default();
                 // Iterate on patterns that contained `overlap`.
                 for pat in column.iter() {
@@ -1043,17 +1044,16 @@ fn lint_overlapping_range_endpoints<'p, 'tcx>(
                         // Don't lint when one of the ranges is a singleton.
                         continue;
                     }
-                    let (start, end) = this_range.boundaries();
-                    if start == overlap {
-                        // `this_range` looks like `overlap..=end`; it overlaps with any ranges that
-                        // look like `start..=overlap`.
+                    if this_range.lo == overlap {
+                        // `this_range` looks like `overlap..=this_range.hi`; it overlaps with any
+                        // ranges that look like `lo..=overlap`.
                         if !prefixes.is_empty() {
                             emit_lint(overlap_range, this_span, &prefixes);
                         }
                         suffixes.push(this_span)
-                    } else if end == overlap {
-                        // `this_range` looks like `start..=overlap`; it overlaps with any ranges
-                        // that look like `overlap..=end`.
+                    } else if this_range.hi == overlap {
+                        // `this_range` looks like `this_range.lo..=overlap`; it overlaps with any
+                        // ranges that look like `overlap..=hi`.
                         if !suffixes.is_empty() {
                             emit_lint(overlap_range, this_span, &suffixes);
                         }

From 6f35ae6f9bc32f591262c6eee94452e3eb2918d5 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Thu, 12 Oct 2023 19:47:33 +0200
Subject: [PATCH 158/297] Propagate half-open ranges through exhaustiveness
 checking

---
 .../src/thir/pattern/deconstruct_pat.rs       | 285 +++++++++++-------
 .../src/thir/pattern/usefulness.rs            |   7 +-
 2 files changed, 174 insertions(+), 118 deletions(-)

diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
index 249273c3595f..00a8bd687738 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
@@ -63,6 +63,7 @@ use rustc_span::{Span, DUMMY_SP};
 use rustc_target::abi::{FieldIdx, Integer, VariantIdx, FIRST_VARIANT};
 
 use self::Constructor::*;
+use self::MaybeInfiniteInt::*;
 use self::SliceKind::*;
 
 use super::usefulness::{MatchCheckCtxt, PatCtxt};
@@ -91,63 +92,21 @@ enum Presence {
     Seen,
 }
 
-/// An inclusive interval, used for precise integer exhaustiveness checking.
-/// `IntRange`s always store a contiguous range. This means that values are
-/// encoded such that `0` encodes the minimum value for the integer,
-/// regardless of the signedness.
-/// For example, the pattern `-128..=127i8` is encoded as `0..=255`.
-/// This makes comparisons and arithmetic on interval endpoints much more
-/// straightforward. See `signed_bias` for details.
-///
-/// `IntRange` is never used to encode an empty range or a "range" that wraps
-/// around the (offset) space: i.e., `range.lo <= range.hi`.
-#[derive(Clone, Copy, PartialEq, Eq)]
-pub(crate) struct IntRange {
-    pub(crate) lo: u128,
-    pub(crate) hi: u128,
+/// A possibly infinite integer. Values are encoded such that the ordering on `u128` matches the
+/// natural order on the original type. For example, `-128i8` is encoded as `0` and `127i8` as
+/// `255`. See `signed_bias` for details.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub(crate) enum MaybeInfiniteInt {
+    NegInfinity,
+    /// Encoded value. DO NOT CONSTRUCT BY HAND; use `new_finite`.
+    Finite(u128),
+    /// The integer after `u128::MAX`. Used when we switch to exclusive ranges in `IntRange::split`.
+    JustAfterMax,
+    PosInfinity,
 }
 
-impl IntRange {
-    #[inline]
-    pub(super) fn is_integral(ty: Ty<'_>) -> bool {
-        matches!(ty.kind(), ty::Char | ty::Int(_) | ty::Uint(_))
-    }
-
-    pub(super) fn is_singleton(&self) -> bool {
-        self.lo == self.hi
-    }
-
-    #[inline]
-    fn from_bits<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, bits: u128) -> IntRange {
-        let bias = IntRange::signed_bias(tcx, ty);
-        // Perform a shift if the underlying types are signed, which makes the interval arithmetic
-        // type-independent.
-        let val = bits ^ bias;
-        IntRange { lo: val, hi: val }
-    }
-
-    #[inline]
-    fn from_range<'tcx>(
-        tcx: TyCtxt<'tcx>,
-        lo: u128,
-        hi: u128,
-        ty: Ty<'tcx>,
-        end: RangeEnd,
-    ) -> IntRange {
-        // Perform a shift if the underlying types are signed, which makes the interval arithmetic
-        // type-independent.
-        let bias = IntRange::signed_bias(tcx, ty);
-        let (lo, hi) = (lo ^ bias, hi ^ bias);
-        let offset = (end == RangeEnd::Excluded) as u128;
-        let hi = hi - offset;
-        if lo > hi {
-            // This should have been caught earlier by E0030.
-            bug!("malformed range pattern: {lo}..={hi}");
-        }
-        IntRange { lo, hi }
-    }
-
-    // The return value of `signed_bias` should be XORed with an endpoint to encode/decode it.
+impl MaybeInfiniteInt {
+    // The return value of `signed_bias` should be XORed with a value to encode/decode it.
     fn signed_bias(tcx: TyCtxt<'_>, ty: Ty<'_>) -> u128 {
         match *ty.kind() {
             ty::Int(ity) => {
@@ -158,6 +117,107 @@ impl IntRange {
         }
     }
 
+    fn new_finite(tcx: TyCtxt<'_>, ty: Ty<'_>, bits: u128) -> Self {
+        let bias = Self::signed_bias(tcx, ty);
+        // Perform a shift if the underlying types are signed, which makes the interval arithmetic
+        // type-independent.
+        let x = bits ^ bias;
+        Finite(x)
+    }
+    fn from_pat_range_bdy<'tcx>(
+        bdy: PatRangeBoundary<'tcx>,
+        ty: Ty<'tcx>,
+        tcx: TyCtxt<'tcx>,
+        param_env: ty::ParamEnv<'tcx>,
+    ) -> Self {
+        match bdy {
+            PatRangeBoundary::NegInfinity => NegInfinity,
+            PatRangeBoundary::Finite(value) => {
+                let bits = value.eval_bits(tcx, param_env);
+                Self::new_finite(tcx, ty, bits)
+            }
+            PatRangeBoundary::PosInfinity => PosInfinity,
+        }
+    }
+    fn to_pat_range_bdy<'tcx>(self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> PatRangeBoundary<'tcx> {
+        match self {
+            NegInfinity => PatRangeBoundary::NegInfinity,
+            Finite(x) => {
+                let bias = Self::signed_bias(tcx, ty);
+                let bits = x ^ bias;
+                let env = ty::ParamEnv::empty().and(ty);
+                let value = mir::Const::from_bits(tcx, bits, env);
+                PatRangeBoundary::Finite(value)
+            }
+            JustAfterMax | PosInfinity => PatRangeBoundary::PosInfinity,
+        }
+    }
+
+    fn minus_one(self) -> Self {
+        match self {
+            Finite(n) => match n.checked_sub(1) {
+                Some(m) => Finite(m),
+                None => NegInfinity,
+            },
+            JustAfterMax => Finite(u128::MAX),
+            x => x,
+        }
+    }
+    fn plus_one(self) -> Self {
+        match self {
+            Finite(n) => match n.checked_add(1) {
+                Some(m) => Finite(m),
+                None => JustAfterMax,
+            },
+            x => x,
+        }
+    }
+}
+
+/// An inclusive interval, used for precise integer exhaustiveness checking.
+/// `IntRange`s always store a contiguous range.
+///
+/// `IntRange` is never used to encode an empty range or a "range" that wraps
+/// around the (offset) space: i.e., `range.lo <= range.hi`.
+///
+/// The range can have open ends.
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub(crate) struct IntRange {
+    pub(crate) lo: MaybeInfiniteInt, // Must not be `PosInfinity`.
+    pub(crate) hi: MaybeInfiniteInt, // Must not be `NegInfinity`.
+}
+
+impl IntRange {
+    #[inline]
+    pub(super) fn is_integral(ty: Ty<'_>) -> bool {
+        matches!(ty.kind(), ty::Char | ty::Int(_) | ty::Uint(_))
+    }
+
+    /// Best effort; will not know that e.g. `255u8..` is a singleton.
+    pub(super) fn is_singleton(&self) -> bool {
+        // Since `lo` and `hi` can't be the same `Infinity`, this correctly only detects a
+        // `Finite(x)` singleton.
+        self.lo == self.hi
+    }
+
+    #[inline]
+    fn from_bits<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, bits: u128) -> IntRange {
+        let x = MaybeInfiniteInt::new_finite(tcx, ty, bits);
+        IntRange { lo: x, hi: x }
+    }
+
+    #[inline]
+    fn from_range(lo: MaybeInfiniteInt, mut hi: MaybeInfiniteInt, end: RangeEnd) -> IntRange {
+        if end == RangeEnd::Excluded {
+            hi = hi.minus_one();
+        }
+        if lo > hi {
+            // This should have been caught earlier by E0030.
+            bug!("malformed range pattern: {lo:?}..={hi:?}");
+        }
+        IntRange { lo, hi }
+    }
+
     fn is_subrange(&self, other: &Self) -> bool {
         other.lo <= self.lo && self.hi <= other.hi
     }
@@ -201,29 +261,16 @@ impl IntRange {
         &self,
         column_ranges: impl Iterator<Item = IntRange>,
     ) -> impl Iterator<Item = (Presence, IntRange)> {
-        /// Represents a boundary between 2 integers. Because the intervals spanning boundaries must be
-        /// able to cover every integer, we need to be able to represent 2^128 + 1 such boundaries.
-        #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
-        enum IntBoundary {
-            JustBefore(u128),
-            AfterMax,
-        }
-
-        fn unpack_intrange(range: IntRange) -> [IntBoundary; 2] {
-            use IntBoundary::*;
-            let lo = JustBefore(range.lo);
-            let hi = match range.hi.checked_add(1) {
-                Some(m) => JustBefore(m),
-                None => AfterMax,
-            };
-            [lo, hi]
+        // Make the range into an exclusive range.
+        fn unpack_intrange(range: IntRange) -> [MaybeInfiniteInt; 2] {
+            [range.lo, range.hi.plus_one()]
         }
 
         // The boundaries of ranges in `column_ranges` intersected with `self`.
         // We do parenthesis matching for input ranges. A boundary counts as +1 if it starts
         // a range and -1 if it ends it. When the count is > 0 between two boundaries, we
         // are within an input range.
-        let mut boundaries: Vec<(IntBoundary, isize)> = column_ranges
+        let mut boundaries: Vec<(MaybeInfiniteInt, isize)> = column_ranges
             .filter_map(|r| self.intersection(&r))
             .map(unpack_intrange)
             .flat_map(|[lo, hi]| [(lo, 1), (hi, -1)])
@@ -233,7 +280,7 @@ impl IntRange {
         // the accumulated count between distinct boundary values.
         boundaries.sort_unstable();
 
-        let [self_start, self_end] = unpack_intrange(self.clone());
+        let [self_start, self_end] = unpack_intrange(*self);
         // Accumulate parenthesis counts.
         let mut paren_counter = 0isize;
         // Gather pairs of adjacent boundaries.
@@ -255,36 +302,26 @@ impl IntRange {
             .filter(|&(prev_bdy, _, bdy)| prev_bdy != bdy)
             // Convert back to ranges.
             .map(move |(prev_bdy, paren_count, bdy)| {
-                use IntBoundary::*;
                 use Presence::*;
                 let presence = if paren_count > 0 { Seen } else { Unseen };
-                let (lo, hi) = match (prev_bdy, bdy) {
-                    (JustBefore(n), JustBefore(m)) if n < m => (n, m - 1),
-                    (JustBefore(n), AfterMax) => (n, u128::MAX),
-                    _ => unreachable!(), // Ruled out by the sorting and filtering we did
-                };
-                (presence, IntRange { lo, hi })
+                // Turn back into an inclusive range.
+                let range = IntRange::from_range(prev_bdy, bdy, RangeEnd::Excluded);
+                (presence, range)
             })
     }
 
     /// Only used for displaying the range.
-    pub(super) fn to_pat<'tcx>(&self, tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Pat<'tcx> {
-        let bias = IntRange::signed_bias(tcx, ty);
-        let (lo_bits, hi_bits) = (self.lo ^ bias, self.hi ^ bias);
+    pub(super) fn to_pat<'tcx>(&self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Pat<'tcx> {
+        let lo = self.lo.to_pat_range_bdy(ty, tcx);
+        let hi = self.hi.to_pat_range_bdy(ty, tcx);
 
-        let env = ty::ParamEnv::empty().and(ty);
-        let lo_const = mir::Const::from_bits(tcx, lo_bits, env);
-        let hi_const = mir::Const::from_bits(tcx, hi_bits, env);
-
-        let kind = if lo_bits == hi_bits {
-            PatKind::Constant { value: lo_const }
+        let kind = if self.is_singleton() {
+            let value = lo.as_finite().unwrap();
+            PatKind::Constant { value }
+        } else if matches!((self.lo, self.hi), (NegInfinity, PosInfinity)) {
+            PatKind::Wild
         } else {
-            PatKind::Range(Box::new(PatRange {
-                lo: PatRangeBoundary::Finite(lo_const),
-                hi: PatRangeBoundary::Finite(hi_const),
-                end: RangeEnd::Included,
-                ty,
-            }))
+            PatKind::Range(Box::new(PatRange { lo, hi, end: RangeEnd::Included, ty }))
         };
 
         Pat { ty, span: DUMMY_SP, kind }
@@ -295,10 +332,14 @@ impl IntRange {
 /// first.
 impl fmt::Debug for IntRange {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let (lo, hi) = (self.lo, self.hi);
-        write!(f, "{lo}")?;
+        if let Finite(lo) = self.lo {
+            write!(f, "{lo}")?;
+        }
         write!(f, "{}", RangeEnd::Included)?;
-        write!(f, "{hi}")
+        if let Finite(hi) = self.hi {
+            write!(f, "{hi}")?;
+        }
+        Ok(())
     }
 }
 
@@ -840,8 +881,13 @@ pub(super) struct SplitConstructorSet<'tcx> {
 impl ConstructorSet {
     #[instrument(level = "debug", skip(cx), ret)]
     pub(super) fn for_ty<'p, 'tcx>(cx: &MatchCheckCtxt<'p, 'tcx>, ty: Ty<'tcx>) -> Self {
-        let make_range =
-            |start, end| IntRange::from_range(cx.tcx, start, end, ty, RangeEnd::Included);
+        let make_range = |start, end| {
+            IntRange::from_range(
+                MaybeInfiniteInt::new_finite(cx.tcx, ty, start),
+                MaybeInfiniteInt::new_finite(cx.tcx, ty, end),
+                RangeEnd::Included,
+            )
+        };
         // This determines the set of all possible constructors for the type `ty`. For numbers,
         // arrays and slices we use ranges and variable-length slices when appropriate.
         //
@@ -1419,24 +1465,33 @@ impl<'p, 'tcx> DeconstructedPat<'p, 'tcx> {
                 }
             }
             PatKind::Range(box PatRange { lo, hi, end, .. }) => {
-                use rustc_apfloat::Float;
                 let ty = pat.ty;
-                // FIXME: handle half-open ranges
-                let lo = lo.eval_bits(ty, cx.tcx, cx.param_env);
-                let hi = hi.eval_bits(ty, cx.tcx, cx.param_env);
                 ctor = match ty.kind() {
                     ty::Char | ty::Int(_) | ty::Uint(_) => {
-                        IntRange(IntRange::from_range(cx.tcx, lo, hi, ty, *end))
+                        let lo =
+                            MaybeInfiniteInt::from_pat_range_bdy(*lo, ty, cx.tcx, cx.param_env);
+                        let hi =
+                            MaybeInfiniteInt::from_pat_range_bdy(*hi, ty, cx.tcx, cx.param_env);
+                        IntRange(IntRange::from_range(lo, hi, *end))
                     }
-                    ty::Float(ty::FloatTy::F32) => {
-                        let lo = rustc_apfloat::ieee::Single::from_bits(lo);
-                        let hi = rustc_apfloat::ieee::Single::from_bits(hi);
-                        F32Range(lo, hi, *end)
-                    }
-                    ty::Float(ty::FloatTy::F64) => {
-                        let lo = rustc_apfloat::ieee::Double::from_bits(lo);
-                        let hi = rustc_apfloat::ieee::Double::from_bits(hi);
-                        F64Range(lo, hi, *end)
+                    ty::Float(fty) => {
+                        use rustc_apfloat::Float;
+                        let lo = lo.as_finite().map(|c| c.eval_bits(cx.tcx, cx.param_env));
+                        let hi = hi.as_finite().map(|c| c.eval_bits(cx.tcx, cx.param_env));
+                        match fty {
+                            ty::FloatTy::F32 => {
+                                use rustc_apfloat::ieee::Single;
+                                let lo = lo.map(Single::from_bits).unwrap_or(-Single::INFINITY);
+                                let hi = hi.map(Single::from_bits).unwrap_or(Single::INFINITY);
+                                F32Range(lo, hi, *end)
+                            }
+                            ty::FloatTy::F64 => {
+                                use rustc_apfloat::ieee::Double;
+                                let lo = lo.map(Double::from_bits).unwrap_or(-Double::INFINITY);
+                                let hi = hi.map(Double::from_bits).unwrap_or(Double::INFINITY);
+                                F64Range(lo, hi, *end)
+                            }
+                        }
                     }
                     _ => bug!("invalid type for range pattern: {}", ty),
                 };
@@ -1706,7 +1761,7 @@ impl<'tcx> WitnessPat<'tcx> {
         let mut subpatterns = self.iter_fields().map(|p| Box::new(p.to_pat(cx)));
         let kind = match &self.ctor {
             Bool(b) => PatKind::Constant { value: mir::Const::from_bool(cx.tcx, *b) },
-            IntRange(range) => return range.to_pat(cx.tcx, self.ty),
+            IntRange(range) => return range.to_pat(self.ty, cx.tcx),
             Single | Variant(_) => match self.ty.kind() {
                 ty::Tuple(..) => PatKind::Leaf {
                     subpatterns: subpatterns
diff --git a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
index 4b314535c893..3a210f2587ec 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
@@ -308,7 +308,8 @@
 use self::ArmType::*;
 use self::Usefulness::*;
 use super::deconstruct_pat::{
-    Constructor, ConstructorSet, DeconstructedPat, IntRange, SplitConstructorSet, WitnessPat,
+    Constructor, ConstructorSet, DeconstructedPat, IntRange, MaybeInfiniteInt, SplitConstructorSet,
+    WitnessPat,
 };
 use crate::errors::{NonExhaustiveOmittedPattern, Overlap, OverlappingRangeEndpoints, Uncovered};
 
@@ -1013,7 +1014,7 @@ fn lint_overlapping_range_endpoints<'p, 'tcx>(
 
     if IntRange::is_integral(ty) {
         let emit_lint = |overlap: &IntRange, this_span: Span, overlapped_spans: &[Span]| {
-            let overlap_as_pat = overlap.to_pat(cx.tcx, ty);
+            let overlap_as_pat = overlap.to_pat(ty, cx.tcx);
             let overlaps: Vec<_> = overlapped_spans
                 .iter()
                 .copied()
@@ -1031,7 +1032,7 @@ fn lint_overlapping_range_endpoints<'p, 'tcx>(
         let split_int_ranges = set.present.iter().filter_map(|c| c.as_int_range());
         for overlap_range in split_int_ranges.clone() {
             if overlap_range.is_singleton() {
-                let overlap: u128 = overlap_range.lo;
+                let overlap: MaybeInfiniteInt = overlap_range.lo;
                 // Ranges that look like `lo..=overlap`.
                 let mut prefixes: SmallVec<[_; 1]> = Default::default();
                 // Ranges that look like `overlap..=hi`.

From a4875ae1e250895a00356efc2bd5e1cf8a609802 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Fri, 13 Oct 2023 00:20:06 +0200
Subject: [PATCH 159/297] Match usize/isize exhaustively

---
 .../src/thir/pattern/check_match.rs           |  27 ++-
 .../src/thir/pattern/deconstruct_pat.rs       | 136 ++++++++----
 ...ture-gate-precise_pointer_size_matching.rs |  10 +-
 ...-gate-precise_pointer_size_matching.stderr |  18 +-
 .../pointer-sized-int.allow.stderr            |   2 +-
 .../pointer-sized-int.deny.stderr             | 194 +++++++-----------
 .../integer-ranges/pointer-sized-int.rs       |   4 -
 .../precise_pointer_matching-message.rs       |  10 +-
 .../precise_pointer_matching-message.stderr   |  18 +-
 ...2-types-containing-non-exhaustive-types.rs |  16 +-
 ...pes-containing-non-exhaustive-types.stderr |  68 +++---
 .../non-exhaustive-pattern-witness.rs         |  53 +++--
 .../non-exhaustive-pattern-witness.stderr     |  50 ++---
 .../usefulness/refutable-pattern-errors.rs    |   4 +-
 .../refutable-pattern-errors.stderr           |   4 +-
 .../usefulness/refutable-pattern-in-fn-arg.rs |   2 +-
 .../refutable-pattern-in-fn-arg.stderr        |   2 +-
 .../tuple-struct-nonexhaustive.stderr         |  10 +-
 18 files changed, 321 insertions(+), 307 deletions(-)

diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
index 9156af3425ae..f26f80105ee0 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
@@ -703,14 +703,21 @@ fn report_arm_reachability<'p, 'tcx>(
 }
 
 fn collect_non_exhaustive_tys<'tcx>(
+    tcx: TyCtxt<'tcx>,
     pat: &WitnessPat<'tcx>,
     non_exhaustive_tys: &mut FxHashSet<Ty<'tcx>>,
 ) {
     if matches!(pat.ctor(), Constructor::NonExhaustive) {
         non_exhaustive_tys.insert(pat.ty());
     }
+    if let Constructor::IntRange(range) = pat.ctor() {
+        if range.is_beyond_boundaries(pat.ty(), tcx) {
+            // The range denotes the values before `isize::MIN` or the values after `usize::MAX`/`isize::MAX`.
+            non_exhaustive_tys.insert(pat.ty());
+        }
+    }
     pat.iter_fields()
-        .for_each(|field_pat| collect_non_exhaustive_tys(field_pat, non_exhaustive_tys))
+        .for_each(|field_pat| collect_non_exhaustive_tys(tcx, field_pat, non_exhaustive_tys))
 }
 
 /// Report that a match is not exhaustive.
@@ -764,16 +771,24 @@ fn non_exhaustive_match<'p, 'tcx>(
     adt_defined_here(cx, &mut err, scrut_ty, &witnesses);
     err.note(format!("the matched value is of type `{}`", scrut_ty));
 
-    if !is_empty_match && witnesses.len() == 1 {
+    if !is_empty_match {
         let mut non_exhaustive_tys = FxHashSet::default();
-        collect_non_exhaustive_tys(&witnesses[0], &mut non_exhaustive_tys);
+        // Look at the first witness.
+        collect_non_exhaustive_tys(cx.tcx, &witnesses[0], &mut non_exhaustive_tys);
 
         for ty in non_exhaustive_tys {
             if ty.is_ptr_sized_integral() {
-                err.note(format!(
-                    "`{ty}` does not have a fixed maximum value, so a wildcard `_` is necessary to match \
-                         exhaustively",
+                if ty == cx.tcx.types.usize {
+                    err.note(format!(
+                        "`{ty}` does not have a fixed maximum value, so half-open ranges are necessary to match \
+                             exhaustively",
                     ));
+                } else if ty == cx.tcx.types.isize {
+                    err.note(format!(
+                        "`{ty}` does not have fixed minimum and maximum values, so half-open ranges are necessary to match \
+                             exhaustively",
+                    ));
+                }
                 if cx.tcx.sess.is_nightly_build() {
                     err.help(format!(
                             "add `#![feature(precise_pointer_size_matching)]` to the crate attributes to \
diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
index 00a8bd687738..5e9179896f79 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
@@ -56,6 +56,7 @@ use rustc_hir::RangeEnd;
 use rustc_index::Idx;
 use rustc_middle::middle::stability::EvalResult;
 use rustc_middle::mir;
+use rustc_middle::mir::interpret::Scalar;
 use rustc_middle::thir::{FieldPat, Pat, PatKind, PatRange, PatRangeBoundary};
 use rustc_middle::ty::layout::IntegerExt;
 use rustc_middle::ty::{self, Ty, TyCtxt, VariantDef};
@@ -139,20 +140,32 @@ impl MaybeInfiniteInt {
             PatRangeBoundary::PosInfinity => PosInfinity,
         }
     }
+    // This could change from finite to infinite if we got `usize::MAX+1` after range splitting.
     fn to_pat_range_bdy<'tcx>(self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> PatRangeBoundary<'tcx> {
         match self {
             NegInfinity => PatRangeBoundary::NegInfinity,
             Finite(x) => {
                 let bias = Self::signed_bias(tcx, ty);
                 let bits = x ^ bias;
-                let env = ty::ParamEnv::empty().and(ty);
-                let value = mir::Const::from_bits(tcx, bits, env);
-                PatRangeBoundary::Finite(value)
+                let size = ty.primitive_size(tcx);
+                match Scalar::try_from_uint(bits, size) {
+                    Some(scalar) => {
+                        let value = mir::Const::from_scalar(tcx, scalar, ty);
+                        PatRangeBoundary::Finite(value)
+                    }
+                    // The value doesn't fit. Since `x >= 0` and 0 always encodes the minimum value
+                    // for a type, the problem isn't that the value is too small. So it must be too
+                    // large.
+                    None => PatRangeBoundary::PosInfinity,
+                }
             }
             JustAfterMax | PosInfinity => PatRangeBoundary::PosInfinity,
         }
     }
 
+    fn is_finite(self) -> bool {
+        matches!(self, Finite(_))
+    }
     fn minus_one(self) -> Self {
         match self {
             Finite(n) => match n.checked_sub(1) {
@@ -169,22 +182,24 @@ impl MaybeInfiniteInt {
                 Some(m) => Finite(m),
                 None => JustAfterMax,
             },
+            JustAfterMax => bug!(),
             x => x,
         }
     }
 }
 
-/// An inclusive interval, used for precise integer exhaustiveness checking.
-/// `IntRange`s always store a contiguous range.
+/// An inclusive interval, used for precise integer exhaustiveness checking. `IntRange`s always
+/// store a contiguous range.
 ///
-/// `IntRange` is never used to encode an empty range or a "range" that wraps
-/// around the (offset) space: i.e., `range.lo <= range.hi`.
+/// `IntRange` is never used to encode an empty range or a "range" that wraps around the (offset)
+/// space: i.e., `range.lo <= range.hi`.
 ///
-/// The range can have open ends.
+/// Note: the range can be `NegInfinity..=NegInfinity` or `PosInfinity..=PosInfinity` to represent
+/// the values before `isize::MIN` and after `isize::MAX`/`usize::MAX`.
 #[derive(Clone, Copy, PartialEq, Eq)]
 pub(crate) struct IntRange {
-    pub(crate) lo: MaybeInfiniteInt, // Must not be `PosInfinity`.
-    pub(crate) hi: MaybeInfiniteInt, // Must not be `NegInfinity`.
+    pub(crate) lo: MaybeInfiniteInt,
+    pub(crate) hi: MaybeInfiniteInt,
 }
 
 impl IntRange {
@@ -195,9 +210,7 @@ impl IntRange {
 
     /// Best effort; will not know that e.g. `255u8..` is a singleton.
     pub(super) fn is_singleton(&self) -> bool {
-        // Since `lo` and `hi` can't be the same `Infinity`, this correctly only detects a
-        // `Finite(x)` singleton.
-        self.lo == self.hi
+        self.lo == self.hi && self.lo.is_finite()
     }
 
     #[inline]
@@ -310,18 +323,49 @@ impl IntRange {
             })
     }
 
+    /// Whether the range denotes the values before `isize::MIN` or the values after
+    /// `usize::MAX`/`isize::MAX`.
+    pub(crate) fn is_beyond_boundaries<'tcx>(&self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> bool {
+        // First check if we are usize/isize to avoid unnecessary `to_pat_range_bdy`.
+        ty.is_ptr_sized_integral() && !tcx.features().precise_pointer_size_matching && {
+            let lo = self.lo.to_pat_range_bdy(ty, tcx);
+            let hi = self.hi.to_pat_range_bdy(ty, tcx);
+            matches!(lo, PatRangeBoundary::PosInfinity)
+                || matches!(hi, PatRangeBoundary::NegInfinity)
+        }
+    }
     /// Only used for displaying the range.
     pub(super) fn to_pat<'tcx>(&self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Pat<'tcx> {
-        let lo = self.lo.to_pat_range_bdy(ty, tcx);
-        let hi = self.hi.to_pat_range_bdy(ty, tcx);
-
-        let kind = if self.is_singleton() {
+        let kind = if matches!((self.lo, self.hi), (NegInfinity, PosInfinity)) {
+            PatKind::Wild
+        } else if self.is_singleton() {
+            let lo = self.lo.to_pat_range_bdy(ty, tcx);
             let value = lo.as_finite().unwrap();
             PatKind::Constant { value }
-        } else if matches!((self.lo, self.hi), (NegInfinity, PosInfinity)) {
-            PatKind::Wild
         } else {
-            PatKind::Range(Box::new(PatRange { lo, hi, end: RangeEnd::Included, ty }))
+            let mut lo = self.lo.to_pat_range_bdy(ty, tcx);
+            let mut hi = self.hi.to_pat_range_bdy(ty, tcx);
+            let end = if hi.is_finite() {
+                RangeEnd::Included
+            } else {
+                // `0..=` isn't a valid pattern.
+                RangeEnd::Excluded
+            };
+            if matches!(hi, PatRangeBoundary::NegInfinity) {
+                // The range denotes the values before `isize::MIN`.
+                let c = ty.numeric_min_val(tcx).unwrap();
+                let value = mir::Const::from_ty_const(c, tcx);
+                hi = PatRangeBoundary::Finite(value);
+            }
+            if matches!(lo, PatRangeBoundary::PosInfinity) {
+                // The range denotes the values after `usize::MAX`/`isize::MAX`.
+                // We represent this as `usize::MAX..` which is slightly incorrect but probably
+                // clear enough.
+                let c = ty.numeric_max_val(tcx).unwrap();
+                let value = mir::Const::from_ty_const(c, tcx);
+                lo = PatRangeBoundary::Finite(value);
+            }
+            PatKind::Range(Box::new(PatRange { lo, hi, end, ty }))
         };
 
         Pat { ty, span: DUMMY_SP, kind }
@@ -843,9 +887,7 @@ pub(super) enum ConstructorSet {
     Bool,
     /// The type is spanned by integer values. The range or ranges give the set of allowed values.
     /// The second range is only useful for `char`.
-    /// `non_exhaustive` is used when the range is not allowed to be matched exhaustively (that's
-    /// for usize/isize).
-    Integers { range_1: IntRange, range_2: Option<IntRange>, non_exhaustive: bool },
+    Integers { range_1: IntRange, range_2: Option<IntRange> },
     /// The type is matched by slices. The usize is the compile-time length of the array, if known.
     Slice(Option<usize>),
     /// The type is matched by slices whose elements are uninhabited.
@@ -903,27 +945,37 @@ impl ConstructorSet {
                 Self::Integers {
                     range_1: make_range('\u{0000}' as u128, '\u{D7FF}' as u128),
                     range_2: Some(make_range('\u{E000}' as u128, '\u{10FFFF}' as u128)),
-                    non_exhaustive: false,
                 }
             }
             &ty::Int(ity) => {
-                // `usize`/`isize` are not allowed to be matched exhaustively unless the
-                // `precise_pointer_size_matching` feature is enabled.
-                let non_exhaustive =
-                    ty.is_ptr_sized_integral() && !cx.tcx.features().precise_pointer_size_matching;
-                let bits = Integer::from_int_ty(&cx.tcx, ity).size().bits() as u128;
-                let min = 1u128 << (bits - 1);
-                let max = min - 1;
-                Self::Integers { range_1: make_range(min, max), non_exhaustive, range_2: None }
+                let range = if ty.is_ptr_sized_integral()
+                    && !cx.tcx.features().precise_pointer_size_matching
+                {
+                    // The min/max values of `isize` are not allowed to be observed unless the
+                    // `precise_pointer_size_matching` feature is enabled.
+                    IntRange { lo: NegInfinity, hi: PosInfinity }
+                } else {
+                    let bits = Integer::from_int_ty(&cx.tcx, ity).size().bits() as u128;
+                    let min = 1u128 << (bits - 1);
+                    let max = min - 1;
+                    make_range(min, max)
+                };
+                Self::Integers { range_1: range, range_2: None }
             }
             &ty::Uint(uty) => {
-                // `usize`/`isize` are not allowed to be matched exhaustively unless the
-                // `precise_pointer_size_matching` feature is enabled.
-                let non_exhaustive =
-                    ty.is_ptr_sized_integral() && !cx.tcx.features().precise_pointer_size_matching;
-                let size = Integer::from_uint_ty(&cx.tcx, uty).size();
-                let max = size.truncate(u128::MAX);
-                Self::Integers { range_1: make_range(0, max), non_exhaustive, range_2: None }
+                let range = if ty.is_ptr_sized_integral()
+                    && !cx.tcx.features().precise_pointer_size_matching
+                {
+                    // The max value of `usize` is not allowed to be observed unless the
+                    // `precise_pointer_size_matching` feature is enabled.
+                    let lo = MaybeInfiniteInt::new_finite(cx.tcx, ty, 0);
+                    IntRange { lo, hi: PosInfinity }
+                } else {
+                    let size = Integer::from_uint_ty(&cx.tcx, uty).size();
+                    let max = size.truncate(u128::MAX);
+                    make_range(0, max)
+                };
+                Self::Integers { range_1: range, range_2: None }
             }
             ty::Array(sub_ty, len) if len.try_eval_target_usize(cx.tcx, cx.param_env).is_some() => {
                 let len = len.eval_target_usize(cx.tcx, cx.param_env) as usize;
@@ -1078,7 +1130,7 @@ impl ConstructorSet {
                     missing.push(Bool(true));
                 }
             }
-            ConstructorSet::Integers { range_1, range_2, non_exhaustive } => {
+            ConstructorSet::Integers { range_1, range_2 } => {
                 let seen_ranges: Vec<_> =
                     seen.map(|ctor| ctor.as_int_range().unwrap().clone()).collect();
                 for (seen, splitted_range) in range_1.split(seen_ranges.iter().cloned()) {
@@ -1095,10 +1147,6 @@ impl ConstructorSet {
                         }
                     }
                 }
-
-                if *non_exhaustive {
-                    missing.push(NonExhaustive);
-                }
             }
             &ConstructorSet::Slice(array_len) => {
                 let seen_slices = seen.map(|c| c.as_slice().unwrap());
diff --git a/tests/ui/feature-gates/feature-gate-precise_pointer_size_matching.rs b/tests/ui/feature-gates/feature-gate-precise_pointer_size_matching.rs
index 4c77180b767a..b4dc1fd45564 100644
--- a/tests/ui/feature-gates/feature-gate-precise_pointer_size_matching.rs
+++ b/tests/ui/feature-gates/feature-gate-precise_pointer_size_matching.rs
@@ -1,17 +1,17 @@
 fn main() {
     match 0usize {
-        //~^ ERROR non-exhaustive patterns: `_` not covered
-        //~| NOTE pattern `_` not covered
+        //~^ ERROR non-exhaustive patterns: `usize::MAX..` not covered
+        //~| NOTE pattern `usize::MAX..` not covered
         //~| NOTE the matched value is of type `usize`
         //~| NOTE `usize` does not have a fixed maximum value
         0..=usize::MAX => {}
     }
 
     match 0isize {
-        //~^ ERROR non-exhaustive patterns: `_` not covered
-        //~| NOTE pattern `_` not covered
+        //~^ ERROR non-exhaustive patterns: `..isize::MIN` and `isize::MAX..` not covered
+        //~| NOTE patterns `..isize::MIN` and `isize::MAX..` not covered
         //~| NOTE the matched value is of type `isize`
-        //~| NOTE `isize` does not have a fixed maximum value
+        //~| NOTE `isize` does not have fixed minimum and maximum values
         isize::MIN..=isize::MAX => {}
     }
 }
diff --git a/tests/ui/feature-gates/feature-gate-precise_pointer_size_matching.stderr b/tests/ui/feature-gates/feature-gate-precise_pointer_size_matching.stderr
index 853b57052ace..8694924e52fe 100644
--- a/tests/ui/feature-gates/feature-gate-precise_pointer_size_matching.stderr
+++ b/tests/ui/feature-gates/feature-gate-precise_pointer_size_matching.stderr
@@ -1,31 +1,31 @@
-error[E0004]: non-exhaustive patterns: `_` not covered
+error[E0004]: non-exhaustive patterns: `usize::MAX..` not covered
   --> $DIR/feature-gate-precise_pointer_size_matching.rs:2:11
    |
 LL |     match 0usize {
-   |           ^^^^^^ pattern `_` not covered
+   |           ^^^^^^ pattern `usize::MAX..` not covered
    |
    = note: the matched value is of type `usize`
-   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `usize` does not have a fixed maximum value, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
 LL ~         0..=usize::MAX => {},
-LL +         _ => todo!()
+LL +         usize::MAX.. => todo!()
    |
 
-error[E0004]: non-exhaustive patterns: `_` not covered
+error[E0004]: non-exhaustive patterns: `..isize::MIN` and `isize::MAX..` not covered
   --> $DIR/feature-gate-precise_pointer_size_matching.rs:10:11
    |
 LL |     match 0isize {
-   |           ^^^^^^ pattern `_` not covered
+   |           ^^^^^^ patterns `..isize::MIN` and `isize::MAX..` not covered
    |
    = note: the matched value is of type `isize`
-   = note: `isize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `isize` does not have fixed minimum and maximum values, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `isize` matching
-help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
    |
 LL ~         isize::MIN..=isize::MAX => {},
-LL +         _ => todo!()
+LL +         ..isize::MIN | isize::MAX.. => todo!()
    |
 
 error: aborting due to 2 previous errors
diff --git a/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.allow.stderr b/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.allow.stderr
index 12a928bf0180..7f26c93aa28a 100644
--- a/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.allow.stderr
+++ b/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.allow.stderr
@@ -1,5 +1,5 @@
 error[E0004]: non-exhaustive patterns: type `usize` is non-empty
-  --> $DIR/pointer-sized-int.rs:58:11
+  --> $DIR/pointer-sized-int.rs:54:11
    |
 LL |     match 7usize {}
    |           ^^^^^^
diff --git a/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.deny.stderr b/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.deny.stderr
index 7fa9eb8eda66..d16ec5412db1 100644
--- a/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.deny.stderr
+++ b/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.deny.stderr
@@ -1,218 +1,162 @@
-error[E0004]: non-exhaustive patterns: `_` not covered
+error[E0004]: non-exhaustive patterns: `usize::MAX..` not covered
   --> $DIR/pointer-sized-int.rs:14:11
    |
 LL |     match 0usize {
-   |           ^^^^^^ pattern `_` not covered
+   |           ^^^^^^ pattern `usize::MAX..` not covered
    |
    = note: the matched value is of type `usize`
-   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `usize` does not have a fixed maximum value, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
 LL ~         0 ..= usize::MAX => {},
-LL +         _ => todo!()
+LL +         usize::MAX.. => todo!()
    |
 
-error[E0004]: non-exhaustive patterns: `_` not covered
+error[E0004]: non-exhaustive patterns: `..isize::MIN` and `isize::MAX..` not covered
   --> $DIR/pointer-sized-int.rs:19:11
    |
 LL |     match 0isize {
-   |           ^^^^^^ pattern `_` not covered
+   |           ^^^^^^ patterns `..isize::MIN` and `isize::MAX..` not covered
    |
    = note: the matched value is of type `isize`
-   = note: `isize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `isize` does not have fixed minimum and maximum values, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `isize` matching
-help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
    |
 LL ~         isize::MIN ..= isize::MAX => {},
-LL +         _ => todo!()
+LL +         ..isize::MIN | isize::MAX.. => todo!()
    |
 
-error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/pointer-sized-int.rs:24:8
-   |
-LL |     m!(0usize, 0..);
-   |        ^^^^^^ pattern `_` not covered
-   |
-   = note: the matched value is of type `usize`
-   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
-   = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
-help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
-   |
-LL |         match $s { $($t)+ => {}, _ => todo!() }
-   |                                ++++++++++++++
-
-error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/pointer-sized-int.rs:26:8
+error[E0004]: non-exhaustive patterns: `usize::MAX..` not covered
+  --> $DIR/pointer-sized-int.rs:25:8
    |
 LL |     m!(0usize, 0..=usize::MAX);
-   |        ^^^^^^ pattern `_` not covered
+   |        ^^^^^^ pattern `usize::MAX..` not covered
    |
    = note: the matched value is of type `usize`
-   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `usize` does not have a fixed maximum value, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL |         match $s { $($t)+ => {}, _ => todo!() }
-   |                                ++++++++++++++
+LL |         match $s { $($t)+ => {}, usize::MAX.. => todo!() }
+   |                                +++++++++++++++++++++++++
 
-error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/pointer-sized-int.rs:28:8
+error[E0004]: non-exhaustive patterns: `usize::MAX..` not covered
+  --> $DIR/pointer-sized-int.rs:27:8
    |
 LL |     m!(0usize, 0..5 | 5..=usize::MAX);
-   |        ^^^^^^ pattern `_` not covered
+   |        ^^^^^^ pattern `usize::MAX..` not covered
    |
    = note: the matched value is of type `usize`
-   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `usize` does not have a fixed maximum value, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL |         match $s { $($t)+ => {}, _ => todo!() }
-   |                                ++++++++++++++
+LL |         match $s { $($t)+ => {}, usize::MAX.. => todo!() }
+   |                                +++++++++++++++++++++++++
 
-error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/pointer-sized-int.rs:30:8
+error[E0004]: non-exhaustive patterns: `usize::MAX..` not covered
+  --> $DIR/pointer-sized-int.rs:29:8
    |
 LL |     m!(0usize, 0..usize::MAX | usize::MAX);
-   |        ^^^^^^ pattern `_` not covered
+   |        ^^^^^^ pattern `usize::MAX..` not covered
    |
    = note: the matched value is of type `usize`
-   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `usize` does not have a fixed maximum value, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL |         match $s { $($t)+ => {}, _ => todo!() }
-   |                                ++++++++++++++
+LL |         match $s { $($t)+ => {}, usize::MAX.. => todo!() }
+   |                                +++++++++++++++++++++++++
 
-error[E0004]: non-exhaustive patterns: `(_, _)` not covered
-  --> $DIR/pointer-sized-int.rs:32:8
+error[E0004]: non-exhaustive patterns: `(usize::MAX.., _)` not covered
+  --> $DIR/pointer-sized-int.rs:31:8
    |
 LL |     m!((0usize, true), (0..5, true) | (5..=usize::MAX, true) | (0..=usize::MAX, false));
-   |        ^^^^^^^^^^^^^^ pattern `(_, _)` not covered
+   |        ^^^^^^^^^^^^^^ pattern `(usize::MAX.., _)` not covered
    |
    = note: the matched value is of type `(usize, bool)`
-   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `usize` does not have a fixed maximum value, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
-LL |         match $s { $($t)+ => {}, (_, _) => todo!() }
-   |                                +++++++++++++++++++
+LL |         match $s { $($t)+ => {}, (usize::MAX.., _) => todo!() }
+   |                                ++++++++++++++++++++++++++++++
 
-error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/pointer-sized-int.rs:34:8
-   |
-LL |     m!(0usize, 0..=usize::MAX | usize::MAX..);
-   |        ^^^^^^ pattern `_` not covered
-   |
-   = note: the matched value is of type `usize`
-   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
-   = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
-help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
-   |
-LL |         match $s { $($t)+ => {}, _ => todo!() }
-   |                                ++++++++++++++
-
-error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/pointer-sized-int.rs:37:8
-   |
-LL |     m!(0isize, ..0 | 0..);
-   |        ^^^^^^ pattern `_` not covered
-   |
-   = note: the matched value is of type `isize`
-   = note: `isize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
-   = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `isize` matching
-help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
-   |
-LL |         match $s { $($t)+ => {}, _ => todo!() }
-   |                                ++++++++++++++
-
-error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/pointer-sized-int.rs:39:8
+error[E0004]: non-exhaustive patterns: `..isize::MIN` and `isize::MAX..` not covered
+  --> $DIR/pointer-sized-int.rs:36:8
    |
 LL |     m!(0isize, isize::MIN..=isize::MAX);
-   |        ^^^^^^ pattern `_` not covered
+   |        ^^^^^^ patterns `..isize::MIN` and `isize::MAX..` not covered
    |
    = note: the matched value is of type `isize`
-   = note: `isize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `isize` does not have fixed minimum and maximum values, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `isize` matching
-help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
    |
-LL |         match $s { $($t)+ => {}, _ => todo!() }
-   |                                ++++++++++++++
+LL |         match $s { $($t)+ => {}, ..isize::MIN | isize::MAX.. => todo!() }
+   |                                ++++++++++++++++++++++++++++++++++++++++
 
-error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/pointer-sized-int.rs:41:8
+error[E0004]: non-exhaustive patterns: `..isize::MIN` and `isize::MAX..` not covered
+  --> $DIR/pointer-sized-int.rs:38:8
    |
 LL |     m!(0isize, isize::MIN..5 | 5..=isize::MAX);
-   |        ^^^^^^ pattern `_` not covered
+   |        ^^^^^^ patterns `..isize::MIN` and `isize::MAX..` not covered
    |
    = note: the matched value is of type `isize`
-   = note: `isize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `isize` does not have fixed minimum and maximum values, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `isize` matching
-help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
    |
-LL |         match $s { $($t)+ => {}, _ => todo!() }
-   |                                ++++++++++++++
+LL |         match $s { $($t)+ => {}, ..isize::MIN | isize::MAX.. => todo!() }
+   |                                ++++++++++++++++++++++++++++++++++++++++
 
-error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/pointer-sized-int.rs:43:8
+error[E0004]: non-exhaustive patterns: `..isize::MIN` and `isize::MAX..` not covered
+  --> $DIR/pointer-sized-int.rs:40:8
    |
 LL |     m!(0isize, isize::MIN..isize::MAX | isize::MAX);
-   |        ^^^^^^ pattern `_` not covered
+   |        ^^^^^^ patterns `..isize::MIN` and `isize::MAX..` not covered
    |
    = note: the matched value is of type `isize`
-   = note: `isize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `isize` does not have fixed minimum and maximum values, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `isize` matching
-help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
    |
-LL |         match $s { $($t)+ => {}, _ => todo!() }
-   |                                ++++++++++++++
+LL |         match $s { $($t)+ => {}, ..isize::MIN | isize::MAX.. => todo!() }
+   |                                ++++++++++++++++++++++++++++++++++++++++
 
-error[E0004]: non-exhaustive patterns: `(_, _)` not covered
-  --> $DIR/pointer-sized-int.rs:45:8
+error[E0004]: non-exhaustive patterns: `(..isize::MIN, _)` and `(isize::MAX.., _)` not covered
+  --> $DIR/pointer-sized-int.rs:42:8
    |
 LL |     m!((0isize, true), (isize::MIN..5, true)
-   |        ^^^^^^^^^^^^^^ pattern `(_, _)` not covered
+   |        ^^^^^^^^^^^^^^ patterns `(..isize::MIN, _)` and `(isize::MAX.., _)` not covered
    |
    = note: the matched value is of type `(isize, bool)`
-   = note: `isize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `isize` does not have fixed minimum and maximum values, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `isize` matching
-help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
    |
-LL |         match $s { $($t)+ => {}, (_, _) => todo!() }
-   |                                +++++++++++++++++++
+LL |         match $s { $($t)+ => {}, (..isize::MIN, _) | (isize::MAX.., _) => todo!() }
+   |                                ++++++++++++++++++++++++++++++++++++++++++++++++++
 
-error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/pointer-sized-int.rs:48:8
-   |
-LL |     m!(0isize, ..=isize::MIN | isize::MIN..=isize::MAX | isize::MAX..);
-   |        ^^^^^^ pattern `_` not covered
-   |
-   = note: the matched value is of type `isize`
-   = note: `isize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
-   = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `isize` matching
-help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
-   |
-LL |         match $s { $($t)+ => {}, _ => todo!() }
-   |                                ++++++++++++++
-
-error[E0004]: non-exhaustive patterns: `_` not covered
-  --> $DIR/pointer-sized-int.rs:51:11
+error[E0004]: non-exhaustive patterns: `..isize::MIN` and `isize::MAX..` not covered
+  --> $DIR/pointer-sized-int.rs:47:11
    |
 LL |     match 0isize {
-   |           ^^^^^^ pattern `_` not covered
+   |           ^^^^^^ patterns `..isize::MIN` and `isize::MAX..` not covered
    |
    = note: the matched value is of type `isize`
-   = note: `isize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `isize` does not have fixed minimum and maximum values, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `isize` matching
-help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
    |
 LL ~         1 ..= isize::MAX => {},
-LL +         _ => todo!()
+LL +         ..isize::MIN | isize::MAX.. => todo!()
    |
 
 error[E0004]: non-exhaustive patterns: type `usize` is non-empty
-  --> $DIR/pointer-sized-int.rs:58:11
+  --> $DIR/pointer-sized-int.rs:54:11
    |
 LL |     match 7usize {}
    |           ^^^^^^
@@ -225,6 +169,6 @@ LL +         _ => todo!(),
 LL +     }
    |
 
-error: aborting due to 16 previous errors
+error: aborting due to 12 previous errors
 
 For more information about this error, try `rustc --explain E0004`.
diff --git a/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.rs b/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.rs
index b55f8f3e5168..20a3cbe127f4 100644
--- a/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.rs
+++ b/tests/ui/pattern/usefulness/integer-ranges/pointer-sized-int.rs
@@ -22,7 +22,6 @@ fn main() {
     }
 
     m!(0usize, 0..);
-    //[deny]~^ ERROR non-exhaustive patterns
     m!(0usize, 0..=usize::MAX);
     //[deny]~^ ERROR non-exhaustive patterns
     m!(0usize, 0..5 | 5..=usize::MAX);
@@ -32,10 +31,8 @@ fn main() {
     m!((0usize, true), (0..5, true) | (5..=usize::MAX, true) | (0..=usize::MAX, false));
     //[deny]~^ ERROR non-exhaustive patterns
     m!(0usize, 0..=usize::MAX | usize::MAX..);
-    //[deny]~^ ERROR non-exhaustive patterns
 
     m!(0isize, ..0 | 0..);
-    //[deny]~^ ERROR non-exhaustive patterns
     m!(0isize, isize::MIN..=isize::MAX);
     //[deny]~^ ERROR non-exhaustive patterns
     m!(0isize, isize::MIN..5 | 5..=isize::MAX);
@@ -46,7 +43,6 @@ fn main() {
         | (5..=isize::MAX, true) | (isize::MIN..=isize::MAX, false));
     //[deny]~^^ ERROR non-exhaustive patterns
     m!(0isize, ..=isize::MIN | isize::MIN..=isize::MAX | isize::MAX..);
-    //[deny]~^ ERROR non-exhaustive patterns
 
     match 0isize {
         //[deny]~^ ERROR non-exhaustive patterns
diff --git a/tests/ui/pattern/usefulness/integer-ranges/precise_pointer_matching-message.rs b/tests/ui/pattern/usefulness/integer-ranges/precise_pointer_matching-message.rs
index a2aa655ca541..d60f479c0d15 100644
--- a/tests/ui/pattern/usefulness/integer-ranges/precise_pointer_matching-message.rs
+++ b/tests/ui/pattern/usefulness/integer-ranges/precise_pointer_matching-message.rs
@@ -1,18 +1,18 @@
 // This tests that the lint message explains the reason for the error.
 fn main() {
     match 0usize {
-        //~^ ERROR non-exhaustive patterns: `_` not covered
-        //~| NOTE pattern `_` not covered
+        //~^ ERROR non-exhaustive patterns: `usize::MAX..` not covered
+        //~| NOTE pattern `usize::MAX..` not covered
         //~| NOTE the matched value is of type `usize`
         //~| NOTE `usize` does not have a fixed maximum value
         0..=usize::MAX => {}
     }
 
     match 0isize {
-        //~^ ERROR non-exhaustive patterns: `_` not covered
-        //~| NOTE pattern `_` not covered
+        //~^ ERROR non-exhaustive patterns: `..isize::MIN` and `isize::MAX..` not covered
+        //~| NOTE patterns `..isize::MIN` and `isize::MAX..` not covered
         //~| NOTE the matched value is of type `isize`
-        //~| NOTE `isize` does not have a fixed maximum value
+        //~| NOTE `isize` does not have fixed minimum and maximum values
         isize::MIN..=isize::MAX => {}
     }
 }
diff --git a/tests/ui/pattern/usefulness/integer-ranges/precise_pointer_matching-message.stderr b/tests/ui/pattern/usefulness/integer-ranges/precise_pointer_matching-message.stderr
index b80411b26b04..a7f93648ed3d 100644
--- a/tests/ui/pattern/usefulness/integer-ranges/precise_pointer_matching-message.stderr
+++ b/tests/ui/pattern/usefulness/integer-ranges/precise_pointer_matching-message.stderr
@@ -1,31 +1,31 @@
-error[E0004]: non-exhaustive patterns: `_` not covered
+error[E0004]: non-exhaustive patterns: `usize::MAX..` not covered
   --> $DIR/precise_pointer_matching-message.rs:3:11
    |
 LL |     match 0usize {
-   |           ^^^^^^ pattern `_` not covered
+   |           ^^^^^^ pattern `usize::MAX..` not covered
    |
    = note: the matched value is of type `usize`
-   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `usize` does not have a fixed maximum value, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
 LL ~         0..=usize::MAX => {},
-LL +         _ => todo!()
+LL +         usize::MAX.. => todo!()
    |
 
-error[E0004]: non-exhaustive patterns: `_` not covered
+error[E0004]: non-exhaustive patterns: `..isize::MIN` and `isize::MAX..` not covered
   --> $DIR/precise_pointer_matching-message.rs:11:11
    |
 LL |     match 0isize {
-   |           ^^^^^^ pattern `_` not covered
+   |           ^^^^^^ patterns `..isize::MIN` and `isize::MAX..` not covered
    |
    = note: the matched value is of type `isize`
-   = note: `isize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `isize` does not have fixed minimum and maximum values, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `isize` matching
-help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
    |
 LL ~         isize::MIN..=isize::MAX => {},
-LL +         _ => todo!()
+LL +         ..isize::MIN | isize::MAX.. => todo!()
    |
 
 error: aborting due to 2 previous errors
diff --git a/tests/ui/pattern/usefulness/issue-85222-types-containing-non-exhaustive-types.rs b/tests/ui/pattern/usefulness/issue-85222-types-containing-non-exhaustive-types.rs
index 8f58227ee2c6..6cbcfed709f9 100644
--- a/tests/ui/pattern/usefulness/issue-85222-types-containing-non-exhaustive-types.rs
+++ b/tests/ui/pattern/usefulness/issue-85222-types-containing-non-exhaustive-types.rs
@@ -6,19 +6,19 @@ struct B<T, U>(T, U);
 
 fn main() {
     match 0 {
-        //~^ ERROR non-exhaustive patterns: `_` not covered [E0004]
+        //~^ ERROR non-exhaustive patterns: `usize::MAX..` not covered [E0004]
         0 => (),
         1..=usize::MAX => (),
     }
 
     match (0usize, 0usize) {
-        //~^ ERROR non-exhaustive patterns: `(_, _)` not covered [E0004]
+        //~^ ERROR non-exhaustive patterns: `(usize::MAX.., _)` not covered [E0004]
         (0, 0) => (),
         (1..=usize::MAX, 1..=usize::MAX) => (),
     }
 
     match (0isize, 0usize) {
-        //~^ ERROR non-exhaustive patterns: `(_, _)` not covered [E0004]
+        //~^ ERROR non-exhaustive patterns: `(..isize::MIN, _)` and `(isize::MAX.., _)` not covered [E0004]
         (isize::MIN..=isize::MAX, 0) => (),
         (isize::MIN..=isize::MAX, 1..=usize::MAX) => (),
     }
@@ -30,14 +30,14 @@ fn main() {
     }
 
     match Some(4) {
-        //~^ ERROR non-exhaustive patterns: `Some(_)` not covered
+        //~^ ERROR non-exhaustive patterns: `Some(usize::MAX..)` not covered
         Some(0) => (),
         Some(1..=usize::MAX) => (),
         None => (),
     }
 
     match Some(Some(Some(0))) {
-        //~^ ERROR non-exhaustive patterns: `Some(Some(Some(_)))` not covered
+        //~^ ERROR non-exhaustive patterns: `Some(Some(Some(usize::MAX..)))` not covered
         Some(Some(Some(0))) => (),
         Some(Some(Some(1..=usize::MAX))) => (),
         Some(Some(None)) => (),
@@ -46,13 +46,13 @@ fn main() {
     }
 
     match (A { a: 0usize }) {
-        //~^ ERROR non-exhaustive patterns: `A { .. }` not covered [E0004]
+        //~^ ERROR non-exhaustive patterns: `A { a: usize::MAX.. }` not covered [E0004]
         A { a: 0 } => (),
         A { a: 1..=usize::MAX } => (),
     }
 
     match B(0isize, 0usize) {
-        //~^ ERROR non-exhaustive patterns: `B(_, _)` not covered [E0004]
+        //~^ ERROR non-exhaustive patterns: `B(..isize::MIN, _)` and `B(isize::MAX.., _)` not covered [E0004]
         B(isize::MIN..=isize::MAX, 0) => (),
         B(isize::MIN..=isize::MAX, 1..=usize::MAX) => (),
     }
@@ -60,7 +60,7 @@ fn main() {
     // Should report only the note about usize not having fixed max value and not report
     // report the note about isize
     match B(0isize, 0usize) {
-        //~^ ERROR non-exhaustive patterns: `B(_, _)` not covered [E0004]
+        //~^ ERROR non-exhaustive patterns: `B(_, usize::MAX..)` not covered [E0004]
         B(_, 0) => (),
         B(_, 1..=usize::MAX) => (),
     }
diff --git a/tests/ui/pattern/usefulness/issue-85222-types-containing-non-exhaustive-types.stderr b/tests/ui/pattern/usefulness/issue-85222-types-containing-non-exhaustive-types.stderr
index ea1d99e20ae1..556efcda516a 100644
--- a/tests/ui/pattern/usefulness/issue-85222-types-containing-non-exhaustive-types.stderr
+++ b/tests/ui/pattern/usefulness/issue-85222-types-containing-non-exhaustive-types.stderr
@@ -1,46 +1,46 @@
-error[E0004]: non-exhaustive patterns: `_` not covered
+error[E0004]: non-exhaustive patterns: `usize::MAX..` not covered
   --> $DIR/issue-85222-types-containing-non-exhaustive-types.rs:8:11
    |
 LL |     match 0 {
-   |           ^ pattern `_` not covered
+   |           ^ pattern `usize::MAX..` not covered
    |
    = note: the matched value is of type `usize`
-   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `usize` does not have a fixed maximum value, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
 LL ~         1..=usize::MAX => (),
-LL ~         _ => todo!(),
+LL ~         usize::MAX.. => todo!(),
    |
 
-error[E0004]: non-exhaustive patterns: `(_, _)` not covered
+error[E0004]: non-exhaustive patterns: `(usize::MAX.., _)` not covered
   --> $DIR/issue-85222-types-containing-non-exhaustive-types.rs:14:11
    |
 LL |     match (0usize, 0usize) {
-   |           ^^^^^^^^^^^^^^^^ pattern `(_, _)` not covered
+   |           ^^^^^^^^^^^^^^^^ pattern `(usize::MAX.., _)` not covered
    |
    = note: the matched value is of type `(usize, usize)`
-   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `usize` does not have a fixed maximum value, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
 LL ~         (1..=usize::MAX, 1..=usize::MAX) => (),
-LL ~         (_, _) => todo!(),
+LL ~         (usize::MAX.., _) => todo!(),
    |
 
-error[E0004]: non-exhaustive patterns: `(_, _)` not covered
+error[E0004]: non-exhaustive patterns: `(..isize::MIN, _)` and `(isize::MAX.., _)` not covered
   --> $DIR/issue-85222-types-containing-non-exhaustive-types.rs:20:11
    |
 LL |     match (0isize, 0usize) {
-   |           ^^^^^^^^^^^^^^^^ pattern `(_, _)` not covered
+   |           ^^^^^^^^^^^^^^^^ patterns `(..isize::MIN, _)` and `(isize::MAX.., _)` not covered
    |
    = note: the matched value is of type `(isize, usize)`
-   = note: `isize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `isize` does not have fixed minimum and maximum values, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `isize` matching
-help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
    |
 LL ~         (isize::MIN..=isize::MAX, 1..=usize::MAX) => (),
-LL ~         (_, _) => todo!(),
+LL ~         (..isize::MIN, _) | (isize::MAX.., _) => todo!(),
    |
 
 error[E0004]: non-exhaustive patterns: `Some(_)` not covered
@@ -61,11 +61,11 @@ LL ~         None => {},
 LL +         Some(_) => todo!()
    |
 
-error[E0004]: non-exhaustive patterns: `Some(_)` not covered
+error[E0004]: non-exhaustive patterns: `Some(usize::MAX..)` not covered
   --> $DIR/issue-85222-types-containing-non-exhaustive-types.rs:32:11
    |
 LL |     match Some(4) {
-   |           ^^^^^^^ pattern `Some(_)` not covered
+   |           ^^^^^^^ pattern `Some(usize::MAX..)` not covered
    |
 note: `Option<usize>` defined here
   --> $SRC_DIR/core/src/option.rs:LL:COL
@@ -73,19 +73,19 @@ note: `Option<usize>` defined here
    |
    = note: not covered
    = note: the matched value is of type `Option<usize>`
-   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `usize` does not have a fixed maximum value, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
 LL ~         None => (),
-LL ~         Some(_) => todo!(),
+LL ~         Some(usize::MAX..) => todo!(),
    |
 
-error[E0004]: non-exhaustive patterns: `Some(Some(Some(_)))` not covered
+error[E0004]: non-exhaustive patterns: `Some(Some(Some(usize::MAX..)))` not covered
   --> $DIR/issue-85222-types-containing-non-exhaustive-types.rs:39:11
    |
 LL |     match Some(Some(Some(0))) {
-   |           ^^^^^^^^^^^^^^^^^^^ pattern `Some(Some(Some(_)))` not covered
+   |           ^^^^^^^^^^^^^^^^^^^ pattern `Some(Some(Some(usize::MAX..)))` not covered
    |
 note: `Option<Option<Option<usize>>>` defined here
   --> $SRC_DIR/core/src/option.rs:LL:COL
@@ -97,19 +97,19 @@ note: `Option<Option<Option<usize>>>` defined here
    |
    = note: not covered
    = note: the matched value is of type `Option<Option<Option<usize>>>`
-   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `usize` does not have a fixed maximum value, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
 LL ~         None => (),
-LL ~         Some(Some(Some(_))) => todo!(),
+LL ~         Some(Some(Some(usize::MAX..))) => todo!(),
    |
 
-error[E0004]: non-exhaustive patterns: `A { .. }` not covered
+error[E0004]: non-exhaustive patterns: `A { a: usize::MAX.. }` not covered
   --> $DIR/issue-85222-types-containing-non-exhaustive-types.rs:48:11
    |
 LL |     match (A { a: 0usize }) {
-   |           ^^^^^^^^^^^^^^^^^ pattern `A { .. }` not covered
+   |           ^^^^^^^^^^^^^^^^^ pattern `A { a: usize::MAX.. }` not covered
    |
 note: `A<usize>` defined here
   --> $DIR/issue-85222-types-containing-non-exhaustive-types.rs:1:8
@@ -117,19 +117,19 @@ note: `A<usize>` defined here
 LL | struct A<T> {
    |        ^
    = note: the matched value is of type `A<usize>`
-   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `usize` does not have a fixed maximum value, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
 LL ~         A { a: 1..=usize::MAX } => (),
-LL ~         A { .. } => todo!(),
+LL ~         A { a: usize::MAX.. } => todo!(),
    |
 
-error[E0004]: non-exhaustive patterns: `B(_, _)` not covered
+error[E0004]: non-exhaustive patterns: `B(..isize::MIN, _)` and `B(isize::MAX.., _)` not covered
   --> $DIR/issue-85222-types-containing-non-exhaustive-types.rs:54:11
    |
 LL |     match B(0isize, 0usize) {
-   |           ^^^^^^^^^^^^^^^^^ pattern `B(_, _)` not covered
+   |           ^^^^^^^^^^^^^^^^^ patterns `B(..isize::MIN, _)` and `B(isize::MAX.., _)` not covered
    |
 note: `B<isize, usize>` defined here
   --> $DIR/issue-85222-types-containing-non-exhaustive-types.rs:5:8
@@ -137,19 +137,19 @@ note: `B<isize, usize>` defined here
 LL | struct B<T, U>(T, U);
    |        ^
    = note: the matched value is of type `B<isize, usize>`
-   = note: `isize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `isize` does not have fixed minimum and maximum values, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `isize` matching
-help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
    |
 LL ~         B(isize::MIN..=isize::MAX, 1..=usize::MAX) => (),
-LL ~         B(_, _) => todo!(),
+LL ~         B(..isize::MIN, _) | B(isize::MAX.., _) => todo!(),
    |
 
-error[E0004]: non-exhaustive patterns: `B(_, _)` not covered
+error[E0004]: non-exhaustive patterns: `B(_, usize::MAX..)` not covered
   --> $DIR/issue-85222-types-containing-non-exhaustive-types.rs:62:11
    |
 LL |     match B(0isize, 0usize) {
-   |           ^^^^^^^^^^^^^^^^^ pattern `B(_, _)` not covered
+   |           ^^^^^^^^^^^^^^^^^ pattern `B(_, usize::MAX..)` not covered
    |
 note: `B<isize, usize>` defined here
   --> $DIR/issue-85222-types-containing-non-exhaustive-types.rs:5:8
@@ -157,12 +157,12 @@ note: `B<isize, usize>` defined here
 LL | struct B<T, U>(T, U);
    |        ^
    = note: the matched value is of type `B<isize, usize>`
-   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
+   = note: `usize` does not have a fixed maximum value, so half-open ranges are necessary to match exhaustively
    = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
 LL ~         B(_, 1..=usize::MAX) => (),
-LL ~         B(_, _) => todo!(),
+LL ~         B(_, usize::MAX..) => todo!(),
    |
 
 error: aborting due to 9 previous errors
diff --git a/tests/ui/pattern/usefulness/non-exhaustive-pattern-witness.rs b/tests/ui/pattern/usefulness/non-exhaustive-pattern-witness.rs
index 4bd34421922b..9e60d4f41a1f 100644
--- a/tests/ui/pattern/usefulness/non-exhaustive-pattern-witness.rs
+++ b/tests/ui/pattern/usefulness/non-exhaustive-pattern-witness.rs
@@ -1,88 +1,101 @@
 struct Foo {
     first: bool,
-    second: Option<[usize; 4]>
+    second: Option<[usize; 4]>,
 }
 
 fn struct_with_a_nested_enum_and_vector() {
     match (Foo { first: true, second: None }) {
-//~^ ERROR non-exhaustive patterns: `Foo { first: false, second: Some([_, _, _, _]) }` not covered
+        //~^ ERROR non-exhaustive patterns: `Foo { first: false, second: Some([0_usize, _, _, _]) }` and `Foo { first: false, second: Some([2_usize.., _, _, _]) }` not covered
         Foo { first: true, second: None } => (),
         Foo { first: true, second: Some(_) } => (),
         Foo { first: false, second: None } => (),
-        Foo { first: false, second: Some([1, 2, 3, 4]) } => ()
+        Foo { first: false, second: Some([1, 2, 3, 4]) } => (),
     }
 }
 
 enum Color {
     Red,
     Green,
-    CustomRGBA { a: bool, r: u8, g: u8, b: u8 }
+    CustomRGBA { a: bool, r: u8, g: u8, b: u8 },
 }
 
 fn enum_with_single_missing_variant() {
     match Color::Red {
-    //~^ ERROR non-exhaustive patterns: `Color::Red` not covered
+        //~^ ERROR non-exhaustive patterns: `Color::Red` not covered
         Color::CustomRGBA { .. } => (),
-        Color::Green => ()
+        Color::Green => (),
     }
 }
 
 enum Direction {
-    North, East, South, West
+    North,
+    East,
+    South,
+    West,
 }
 
 fn enum_with_multiple_missing_variants() {
     match Direction::North {
-    //~^ ERROR non-exhaustive patterns: `Direction::East`, `Direction::South` and `Direction::West` not covered
-        Direction::North => ()
+        //~^ ERROR non-exhaustive patterns: `Direction::East`, `Direction::South` and `Direction::West` not covered
+        Direction::North => (),
     }
 }
 
 enum ExcessiveEnum {
-    First, Second, Third, Fourth, Fifth, Sixth, Seventh, Eighth, Ninth, Tenth, Eleventh, Twelfth
+    First,
+    Second,
+    Third,
+    Fourth,
+    Fifth,
+    Sixth,
+    Seventh,
+    Eighth,
+    Ninth,
+    Tenth,
+    Eleventh,
+    Twelfth,
 }
 
 fn enum_with_excessive_missing_variants() {
     match ExcessiveEnum::First {
-    //~^ ERROR `ExcessiveEnum::Second`, `ExcessiveEnum::Third`, `ExcessiveEnum::Fourth` and 8 more not covered
-
-        ExcessiveEnum::First => ()
+        //~^ ERROR `ExcessiveEnum::Second`, `ExcessiveEnum::Third`, `ExcessiveEnum::Fourth` and 8 more not covered
+        ExcessiveEnum::First => (),
     }
 }
 
 fn enum_struct_variant() {
     match Color::Red {
-    //~^ ERROR non-exhaustive patterns: `Color::CustomRGBA { a: true, .. }` not covered
+        //~^ ERROR non-exhaustive patterns: `Color::CustomRGBA { a: true, .. }` not covered
         Color::Red => (),
         Color::Green => (),
         Color::CustomRGBA { a: false, r: _, g: _, b: 0 } => (),
-        Color::CustomRGBA { a: false, r: _, g: _, b: _ } => ()
+        Color::CustomRGBA { a: false, r: _, g: _, b: _ } => (),
     }
 }
 
 enum Enum {
     First,
-    Second(bool)
+    Second(bool),
 }
 
 fn vectors_with_nested_enums() {
     let x: &'static [Enum] = &[Enum::First, Enum::Second(false)];
     match *x {
-    //~^ ERROR non-exhaustive patterns: `[Enum::Second(true), Enum::Second(false)]` not covered
+        //~^ ERROR non-exhaustive patterns: `[Enum::Second(true), Enum::Second(false)]` not covered
         [] => (),
         [_] => (),
         [Enum::First, _] => (),
         [Enum::Second(true), Enum::First] => (),
         [Enum::Second(true), Enum::Second(true)] => (),
         [Enum::Second(false), _] => (),
-        [_, _, ref tail @ .., _] => ()
+        [_, _, ref tail @ .., _] => (),
     }
 }
 
 fn missing_nil() {
     match ((), false) {
-    //~^ ERROR non-exhaustive patterns: `((), false)` not covered
-        ((), true) => ()
+        //~^ ERROR non-exhaustive patterns: `((), false)` not covered
+        ((), true) => (),
     }
 }
 
diff --git a/tests/ui/pattern/usefulness/non-exhaustive-pattern-witness.stderr b/tests/ui/pattern/usefulness/non-exhaustive-pattern-witness.stderr
index d798ec722ddd..f914b98d9231 100644
--- a/tests/ui/pattern/usefulness/non-exhaustive-pattern-witness.stderr
+++ b/tests/ui/pattern/usefulness/non-exhaustive-pattern-witness.stderr
@@ -1,8 +1,8 @@
-error[E0004]: non-exhaustive patterns: `Foo { first: false, second: Some([_, _, _, _]) }` not covered
+error[E0004]: non-exhaustive patterns: `Foo { first: false, second: Some([0_usize, _, _, _]) }` and `Foo { first: false, second: Some([2_usize.., _, _, _]) }` not covered
   --> $DIR/non-exhaustive-pattern-witness.rs:7:11
    |
 LL |     match (Foo { first: true, second: None }) {
-   |           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ pattern `Foo { first: false, second: Some([_, _, _, _]) }` not covered
+   |           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ patterns `Foo { first: false, second: Some([0_usize, _, _, _]) }` and `Foo { first: false, second: Some([2_usize.., _, _, _]) }` not covered
    |
 note: `Foo` defined here
   --> $DIR/non-exhaustive-pattern-witness.rs:1:8
@@ -10,12 +10,10 @@ note: `Foo` defined here
 LL | struct Foo {
    |        ^^^
    = note: the matched value is of type `Foo`
-   = note: `usize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
-   = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `usize` matching
-help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
    |
 LL ~         Foo { first: false, second: Some([1, 2, 3, 4]) } => (),
-LL +         Foo { first: false, second: Some([_, _, _, _]) } => todo!()
+LL ~         Foo { first: false, second: Some([0_usize, _, _, _]) } | Foo { first: false, second: Some([2_usize.., _, _, _]) } => todo!(),
    |
 
 error[E0004]: non-exhaustive patterns: `Color::Red` not covered
@@ -35,40 +33,42 @@ LL |     Red,
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
 LL ~         Color::Green => (),
-LL +         Color::Red => todo!()
+LL ~         Color::Red => todo!(),
    |
 
 error[E0004]: non-exhaustive patterns: `Direction::East`, `Direction::South` and `Direction::West` not covered
-  --> $DIR/non-exhaustive-pattern-witness.rs:35:11
+  --> $DIR/non-exhaustive-pattern-witness.rs:38:11
    |
 LL |     match Direction::North {
    |           ^^^^^^^^^^^^^^^^ patterns `Direction::East`, `Direction::South` and `Direction::West` not covered
    |
 note: `Direction` defined here
-  --> $DIR/non-exhaustive-pattern-witness.rs:31:12
+  --> $DIR/non-exhaustive-pattern-witness.rs:32:5
    |
 LL | enum Direction {
    |      ---------
-LL |     North, East, South, West
-   |            ^^^^  ^^^^^  ^^^^ not covered
-   |            |     |
-   |            |     not covered
-   |            not covered
+LL |     North,
+LL |     East,
+   |     ^^^^ not covered
+LL |     South,
+   |     ^^^^^ not covered
+LL |     West,
+   |     ^^^^ not covered
    = note: the matched value is of type `Direction`
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
    |
 LL ~         Direction::North => (),
-LL +         Direction::East | Direction::South | Direction::West => todo!()
+LL ~         Direction::East | Direction::South | Direction::West => todo!(),
    |
 
 error[E0004]: non-exhaustive patterns: `ExcessiveEnum::Second`, `ExcessiveEnum::Third`, `ExcessiveEnum::Fourth` and 8 more not covered
-  --> $DIR/non-exhaustive-pattern-witness.rs:46:11
+  --> $DIR/non-exhaustive-pattern-witness.rs:60:11
    |
 LL |     match ExcessiveEnum::First {
    |           ^^^^^^^^^^^^^^^^^^^^ patterns `ExcessiveEnum::Second`, `ExcessiveEnum::Third`, `ExcessiveEnum::Fourth` and 8 more not covered
    |
 note: `ExcessiveEnum` defined here
-  --> $DIR/non-exhaustive-pattern-witness.rs:41:6
+  --> $DIR/non-exhaustive-pattern-witness.rs:44:6
    |
 LL | enum ExcessiveEnum {
    |      ^^^^^^^^^^^^^
@@ -76,11 +76,11 @@ LL | enum ExcessiveEnum {
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown, or multiple match arms
    |
 LL ~         ExcessiveEnum::First => (),
-LL +         _ => todo!()
+LL ~         _ => todo!(),
    |
 
 error[E0004]: non-exhaustive patterns: `Color::CustomRGBA { a: true, .. }` not covered
-  --> $DIR/non-exhaustive-pattern-witness.rs:54:11
+  --> $DIR/non-exhaustive-pattern-witness.rs:67:11
    |
 LL |     match Color::Red {
    |           ^^^^^^^^^^ pattern `Color::CustomRGBA { a: true, .. }` not covered
@@ -91,17 +91,17 @@ note: `Color` defined here
 LL | enum Color {
    |      -----
 ...
-LL |     CustomRGBA { a: bool, r: u8, g: u8, b: u8 }
+LL |     CustomRGBA { a: bool, r: u8, g: u8, b: u8 },
    |     ^^^^^^^^^^ not covered
    = note: the matched value is of type `Color`
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
 LL ~         Color::CustomRGBA { a: false, r: _, g: _, b: _ } => (),
-LL +         Color::CustomRGBA { a: true, .. } => todo!()
+LL ~         Color::CustomRGBA { a: true, .. } => todo!(),
    |
 
 error[E0004]: non-exhaustive patterns: `[Enum::Second(true), Enum::Second(false)]` not covered
-  --> $DIR/non-exhaustive-pattern-witness.rs:70:11
+  --> $DIR/non-exhaustive-pattern-witness.rs:83:11
    |
 LL |     match *x {
    |           ^^ pattern `[Enum::Second(true), Enum::Second(false)]` not covered
@@ -110,11 +110,11 @@ LL |     match *x {
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
 LL ~         [_, _, ref tail @ .., _] => (),
-LL +         [Enum::Second(true), Enum::Second(false)] => todo!()
+LL ~         [Enum::Second(true), Enum::Second(false)] => todo!(),
    |
 
 error[E0004]: non-exhaustive patterns: `((), false)` not covered
-  --> $DIR/non-exhaustive-pattern-witness.rs:83:11
+  --> $DIR/non-exhaustive-pattern-witness.rs:96:11
    |
 LL |     match ((), false) {
    |           ^^^^^^^^^^^ pattern `((), false)` not covered
@@ -123,7 +123,7 @@ LL |     match ((), false) {
 help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
    |
 LL ~         ((), true) => (),
-LL +         ((), false) => todo!()
+LL ~         ((), false) => todo!(),
    |
 
 error: aborting due to 7 previous errors
diff --git a/tests/ui/pattern/usefulness/refutable-pattern-errors.rs b/tests/ui/pattern/usefulness/refutable-pattern-errors.rs
index 7a3e991d5931..7603da1bb2ce 100644
--- a/tests/ui/pattern/usefulness/refutable-pattern-errors.rs
+++ b/tests/ui/pattern/usefulness/refutable-pattern-errors.rs
@@ -1,6 +1,6 @@
-fn func((1, (Some(1), 2..=3)): (isize, (Option<isize>, isize))) { }
+fn func((1, (Some(1), 2..=3)): (isize, (Option<isize>, isize))) {}
 //~^ ERROR refutable pattern in function argument
-//~| `(_, _)` not covered
+//~| `(..=0_isize, _)` and `(2_isize.., _)` not covered
 
 fn main() {
     let (1, (Some(1), 2..=3)) = (1, (None, 2));
diff --git a/tests/ui/pattern/usefulness/refutable-pattern-errors.stderr b/tests/ui/pattern/usefulness/refutable-pattern-errors.stderr
index beb51a4d4504..e66cd1130238 100644
--- a/tests/ui/pattern/usefulness/refutable-pattern-errors.stderr
+++ b/tests/ui/pattern/usefulness/refutable-pattern-errors.stderr
@@ -1,8 +1,8 @@
 error[E0005]: refutable pattern in function argument
   --> $DIR/refutable-pattern-errors.rs:1:9
    |
-LL | fn func((1, (Some(1), 2..=3)): (isize, (Option<isize>, isize))) { }
-   |         ^^^^^^^^^^^^^^^^^^^^^ pattern `(_, _)` not covered
+LL | fn func((1, (Some(1), 2..=3)): (isize, (Option<isize>, isize))) {}
+   |         ^^^^^^^^^^^^^^^^^^^^^ patterns `(..=0_isize, _)` and `(2_isize.., _)` not covered
    |
    = note: the matched value is of type `(isize, (Option<isize>, isize))`
 
diff --git a/tests/ui/pattern/usefulness/refutable-pattern-in-fn-arg.rs b/tests/ui/pattern/usefulness/refutable-pattern-in-fn-arg.rs
index 17dc38ab25d9..4203dd94d43a 100644
--- a/tests/ui/pattern/usefulness/refutable-pattern-in-fn-arg.rs
+++ b/tests/ui/pattern/usefulness/refutable-pattern-in-fn-arg.rs
@@ -1,6 +1,6 @@
 fn main() {
     let f = |3: isize| println!("hello");
     //~^ ERROR refutable pattern in function argument
-    //~| `_` not covered
+    //~| `..=2_isize` and `4_isize..` not covered
     f(4);
 }
diff --git a/tests/ui/pattern/usefulness/refutable-pattern-in-fn-arg.stderr b/tests/ui/pattern/usefulness/refutable-pattern-in-fn-arg.stderr
index ab3f6f69fb16..01f077909e8f 100644
--- a/tests/ui/pattern/usefulness/refutable-pattern-in-fn-arg.stderr
+++ b/tests/ui/pattern/usefulness/refutable-pattern-in-fn-arg.stderr
@@ -2,7 +2,7 @@ error[E0005]: refutable pattern in function argument
   --> $DIR/refutable-pattern-in-fn-arg.rs:2:14
    |
 LL |     let f = |3: isize| println!("hello");
-   |              ^ pattern `_` not covered
+   |              ^ patterns `..=2_isize` and `4_isize..` not covered
    |
    = note: the matched value is of type `isize`
 help: alternatively, you could prepend the pattern with an underscore to define a new named variable; identifiers cannot begin with digits
diff --git a/tests/ui/pattern/usefulness/tuple-struct-nonexhaustive.stderr b/tests/ui/pattern/usefulness/tuple-struct-nonexhaustive.stderr
index 50c7fc889f42..ef707ed4aa40 100644
--- a/tests/ui/pattern/usefulness/tuple-struct-nonexhaustive.stderr
+++ b/tests/ui/pattern/usefulness/tuple-struct-nonexhaustive.stderr
@@ -1,8 +1,8 @@
-error[E0004]: non-exhaustive patterns: `Foo(_, _)` not covered
+error[E0004]: non-exhaustive patterns: `Foo(..=0_isize, _)` and `Foo(3_isize.., _)` not covered
   --> $DIR/tuple-struct-nonexhaustive.rs:5:11
    |
 LL |     match x {
-   |           ^ pattern `Foo(_, _)` not covered
+   |           ^ patterns `Foo(..=0_isize, _)` and `Foo(3_isize.., _)` not covered
    |
 note: `Foo` defined here
   --> $DIR/tuple-struct-nonexhaustive.rs:1:8
@@ -10,12 +10,10 @@ note: `Foo` defined here
 LL | struct Foo(isize, isize);
    |        ^^^
    = note: the matched value is of type `Foo`
-   = note: `isize` does not have a fixed maximum value, so a wildcard `_` is necessary to match exhaustively
-   = help: add `#![feature(precise_pointer_size_matching)]` to the crate attributes to enable precise `isize` matching
-help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern or an explicit pattern as shown
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern, a match arm with multiple or-patterns as shown, or multiple match arms
    |
 LL ~         Foo(2, b) => println!("{}", b),
-LL +         Foo(_, _) => todo!()
+LL +         Foo(..=0_isize, _) | Foo(3_isize.., _) => todo!()
    |
 
 error: aborting due to previous error

From feb769a5c9e3954a4dd0a83d321b2996a1ac5667 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Sat, 21 Oct 2023 20:16:10 +0200
Subject: [PATCH 160/297] s/to_pat/to_diagnostic_pat/

---
 compiler/rustc_mir_build/src/errors.rs        | 12 +++++--
 .../src/thir/pattern/check_match.rs           |  8 ++---
 .../src/thir/pattern/deconstruct_pat.rs       | 32 +++++++++++--------
 .../src/thir/pattern/usefulness.rs            |  2 +-
 4 files changed, 33 insertions(+), 21 deletions(-)

diff --git a/compiler/rustc_mir_build/src/errors.rs b/compiler/rustc_mir_build/src/errors.rs
index 730670a8369c..5bfce3ab5100 100644
--- a/compiler/rustc_mir_build/src/errors.rs
+++ b/compiler/rustc_mir_build/src/errors.rs
@@ -807,13 +807,19 @@ impl<'tcx> Uncovered<'tcx> {
         cx: &MatchCheckCtxt<'p, 'tcx>,
         witnesses: Vec<WitnessPat<'tcx>>,
     ) -> Self {
-        let witness_1 = witnesses.get(0).unwrap().to_pat(cx);
+        let witness_1 = witnesses.get(0).unwrap().to_diagnostic_pat(cx);
         Self {
             span,
             count: witnesses.len(),
             // Substitute dummy values if witnesses is smaller than 3. These will never be read.
-            witness_2: witnesses.get(1).map(|w| w.to_pat(cx)).unwrap_or_else(|| witness_1.clone()),
-            witness_3: witnesses.get(2).map(|w| w.to_pat(cx)).unwrap_or_else(|| witness_1.clone()),
+            witness_2: witnesses
+                .get(1)
+                .map(|w| w.to_diagnostic_pat(cx))
+                .unwrap_or_else(|| witness_1.clone()),
+            witness_3: witnesses
+                .get(2)
+                .map(|w| w.to_diagnostic_pat(cx))
+                .unwrap_or_else(|| witness_1.clone()),
             witness_1,
             remainder: witnesses.len().saturating_sub(3),
         }
diff --git a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
index f26f80105ee0..933653e708e9 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/check_match.rs
@@ -760,7 +760,7 @@ fn non_exhaustive_match<'p, 'tcx>(
         pattern = if witnesses.len() < 4 {
             witnesses
                 .iter()
-                .map(|witness| witness.to_pat(cx).to_string())
+                .map(|witness| witness.to_diagnostic_pat(cx).to_string())
                 .collect::<Vec<String>>()
                 .join(" | ")
         } else {
@@ -915,13 +915,13 @@ pub(crate) fn joined_uncovered_patterns<'p, 'tcx>(
     witnesses: &[WitnessPat<'tcx>],
 ) -> String {
     const LIMIT: usize = 3;
-    let pat_to_str = |pat: &WitnessPat<'tcx>| pat.to_pat(cx).to_string();
+    let pat_to_str = |pat: &WitnessPat<'tcx>| pat.to_diagnostic_pat(cx).to_string();
     match witnesses {
         [] => bug!(),
-        [witness] => format!("`{}`", witness.to_pat(cx)),
+        [witness] => format!("`{}`", witness.to_diagnostic_pat(cx)),
         [head @ .., tail] if head.len() < LIMIT => {
             let head: Vec<_> = head.iter().map(pat_to_str).collect();
-            format!("`{}` and `{}`", head.join("`, `"), tail.to_pat(cx))
+            format!("`{}` and `{}`", head.join("`, `"), tail.to_diagnostic_pat(cx))
         }
         _ => {
             let (head, tail) = witnesses.split_at(LIMIT);
diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
index 5e9179896f79..8d9998d17ed5 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
@@ -140,8 +140,13 @@ impl MaybeInfiniteInt {
             PatRangeBoundary::PosInfinity => PosInfinity,
         }
     }
-    // This could change from finite to infinite if we got `usize::MAX+1` after range splitting.
-    fn to_pat_range_bdy<'tcx>(self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> PatRangeBoundary<'tcx> {
+    /// Used only for diagnostics.
+    /// This could change from finite to infinite if we got `usize::MAX+1` after range splitting.
+    fn to_diagnostic_pat_range_bdy<'tcx>(
+        self,
+        ty: Ty<'tcx>,
+        tcx: TyCtxt<'tcx>,
+    ) -> PatRangeBoundary<'tcx> {
         match self {
             NegInfinity => PatRangeBoundary::NegInfinity,
             Finite(x) => {
@@ -326,25 +331,25 @@ impl IntRange {
     /// Whether the range denotes the values before `isize::MIN` or the values after
     /// `usize::MAX`/`isize::MAX`.
     pub(crate) fn is_beyond_boundaries<'tcx>(&self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> bool {
-        // First check if we are usize/isize to avoid unnecessary `to_pat_range_bdy`.
+        // First check if we are usize/isize to avoid unnecessary `to_diagnostic_pat_range_bdy`.
         ty.is_ptr_sized_integral() && !tcx.features().precise_pointer_size_matching && {
-            let lo = self.lo.to_pat_range_bdy(ty, tcx);
-            let hi = self.hi.to_pat_range_bdy(ty, tcx);
+            let lo = self.lo.to_diagnostic_pat_range_bdy(ty, tcx);
+            let hi = self.hi.to_diagnostic_pat_range_bdy(ty, tcx);
             matches!(lo, PatRangeBoundary::PosInfinity)
                 || matches!(hi, PatRangeBoundary::NegInfinity)
         }
     }
     /// Only used for displaying the range.
-    pub(super) fn to_pat<'tcx>(&self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Pat<'tcx> {
+    pub(super) fn to_diagnostic_pat<'tcx>(&self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> Pat<'tcx> {
         let kind = if matches!((self.lo, self.hi), (NegInfinity, PosInfinity)) {
             PatKind::Wild
         } else if self.is_singleton() {
-            let lo = self.lo.to_pat_range_bdy(ty, tcx);
+            let lo = self.lo.to_diagnostic_pat_range_bdy(ty, tcx);
             let value = lo.as_finite().unwrap();
             PatKind::Constant { value }
         } else {
-            let mut lo = self.lo.to_pat_range_bdy(ty, tcx);
-            let mut hi = self.hi.to_pat_range_bdy(ty, tcx);
+            let mut lo = self.lo.to_diagnostic_pat_range_bdy(ty, tcx);
+            let mut hi = self.hi.to_diagnostic_pat_range_bdy(ty, tcx);
             let end = if hi.is_finite() {
                 RangeEnd::Included
             } else {
@@ -1803,13 +1808,14 @@ impl<'tcx> WitnessPat<'tcx> {
         self.ty
     }
 
-    /// Convert back to a `thir::Pat` for diagnostic purposes.
-    pub(crate) fn to_pat(&self, cx: &MatchCheckCtxt<'_, 'tcx>) -> Pat<'tcx> {
+    /// Convert back to a `thir::Pat` for diagnostic purposes. This panics for patterns that don't
+    /// appear in diagnostics, like float ranges.
+    pub(crate) fn to_diagnostic_pat(&self, cx: &MatchCheckCtxt<'_, 'tcx>) -> Pat<'tcx> {
         let is_wildcard = |pat: &Pat<'_>| matches!(pat.kind, PatKind::Wild);
-        let mut subpatterns = self.iter_fields().map(|p| Box::new(p.to_pat(cx)));
+        let mut subpatterns = self.iter_fields().map(|p| Box::new(p.to_diagnostic_pat(cx)));
         let kind = match &self.ctor {
             Bool(b) => PatKind::Constant { value: mir::Const::from_bool(cx.tcx, *b) },
-            IntRange(range) => return range.to_pat(self.ty, cx.tcx),
+            IntRange(range) => return range.to_diagnostic_pat(self.ty, cx.tcx),
             Single | Variant(_) => match self.ty.kind() {
                 ty::Tuple(..) => PatKind::Leaf {
                     subpatterns: subpatterns
diff --git a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
index 3a210f2587ec..c0920a3ef5a5 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
@@ -1014,7 +1014,7 @@ fn lint_overlapping_range_endpoints<'p, 'tcx>(
 
     if IntRange::is_integral(ty) {
         let emit_lint = |overlap: &IntRange, this_span: Span, overlapped_spans: &[Span]| {
-            let overlap_as_pat = overlap.to_pat(ty, cx.tcx);
+            let overlap_as_pat = overlap.to_diagnostic_pat(ty, cx.tcx);
             let overlaps: Vec<_> = overlapped_spans
                 .iter()
                 .copied()

From 35fe75d8f3800a77cf23a6d7b23d9c28311c5dfb Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Sat, 21 Oct 2023 20:16:48 +0200
Subject: [PATCH 161/297] Make IntRange exclusive

---
 .../src/thir/pattern/deconstruct_pat.rs       | 120 ++++++++++--------
 .../src/thir/pattern/usefulness.rs            |   2 +-
 2 files changed, 65 insertions(+), 57 deletions(-)

diff --git a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
index 8d9998d17ed5..0c7c2c6f9b4b 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/deconstruct_pat.rs
@@ -101,7 +101,7 @@ pub(crate) enum MaybeInfiniteInt {
     NegInfinity,
     /// Encoded value. DO NOT CONSTRUCT BY HAND; use `new_finite`.
     Finite(u128),
-    /// The integer after `u128::MAX`. Used when we switch to exclusive ranges in `IntRange::split`.
+    /// The integer after `u128::MAX`. We need it to represent `x..=u128::MAX` as an exclusive range.
     JustAfterMax,
     PosInfinity,
 }
@@ -140,8 +140,11 @@ impl MaybeInfiniteInt {
             PatRangeBoundary::PosInfinity => PosInfinity,
         }
     }
+
     /// Used only for diagnostics.
-    /// This could change from finite to infinite if we got `usize::MAX+1` after range splitting.
+    /// Note: it is possible to get `isize/usize::MAX+1` here, as explained in the doc for
+    /// [`IntRange::split`]. This cannot be represented as a `Const`, so we represent it with
+    /// `PosInfinity`.
     fn to_diagnostic_pat_range_bdy<'tcx>(
         self,
         ty: Ty<'tcx>,
@@ -168,20 +171,19 @@ impl MaybeInfiniteInt {
         }
     }
 
-    fn is_finite(self) -> bool {
-        matches!(self, Finite(_))
-    }
-    fn minus_one(self) -> Self {
+    /// Note: this will not turn a finite value into an infinite one or vice-versa.
+    pub(crate) fn minus_one(self) -> Self {
         match self {
             Finite(n) => match n.checked_sub(1) {
                 Some(m) => Finite(m),
-                None => NegInfinity,
+                None => bug!(),
             },
             JustAfterMax => Finite(u128::MAX),
             x => x,
         }
     }
-    fn plus_one(self) -> Self {
+    /// Note: this will not turn a finite value into an infinite one or vice-versa.
+    pub(crate) fn plus_one(self) -> Self {
         match self {
             Finite(n) => match n.checked_add(1) {
                 Some(m) => Finite(m),
@@ -193,18 +195,15 @@ impl MaybeInfiniteInt {
     }
 }
 
-/// An inclusive interval, used for precise integer exhaustiveness checking. `IntRange`s always
+/// An exclusive interval, used for precise integer exhaustiveness checking. `IntRange`s always
 /// store a contiguous range.
 ///
 /// `IntRange` is never used to encode an empty range or a "range" that wraps around the (offset)
-/// space: i.e., `range.lo <= range.hi`.
-///
-/// Note: the range can be `NegInfinity..=NegInfinity` or `PosInfinity..=PosInfinity` to represent
-/// the values before `isize::MIN` and after `isize::MAX`/`usize::MAX`.
+/// space: i.e., `range.lo < range.hi`.
 #[derive(Clone, Copy, PartialEq, Eq)]
 pub(crate) struct IntRange {
-    pub(crate) lo: MaybeInfiniteInt,
-    pub(crate) hi: MaybeInfiniteInt,
+    pub(crate) lo: MaybeInfiniteInt, // Must not be `PosInfinity`.
+    pub(crate) hi: MaybeInfiniteInt, // Must not be `NegInfinity`.
 }
 
 impl IntRange {
@@ -215,23 +214,25 @@ impl IntRange {
 
     /// Best effort; will not know that e.g. `255u8..` is a singleton.
     pub(super) fn is_singleton(&self) -> bool {
-        self.lo == self.hi && self.lo.is_finite()
+        // Since `lo` and `hi` can't be the same `Infinity` and `plus_one` never changes from finite
+        // to infinite, this correctly only detects ranges that contain exacly one `Finite(x)`.
+        self.lo.plus_one() == self.hi
     }
 
     #[inline]
     fn from_bits<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, bits: u128) -> IntRange {
         let x = MaybeInfiniteInt::new_finite(tcx, ty, bits);
-        IntRange { lo: x, hi: x }
+        IntRange { lo: x, hi: x.plus_one() }
     }
 
     #[inline]
     fn from_range(lo: MaybeInfiniteInt, mut hi: MaybeInfiniteInt, end: RangeEnd) -> IntRange {
-        if end == RangeEnd::Excluded {
-            hi = hi.minus_one();
+        if end == RangeEnd::Included {
+            hi = hi.plus_one();
         }
-        if lo > hi {
+        if lo >= hi {
             // This should have been caught earlier by E0030.
-            bug!("malformed range pattern: {lo:?}..={hi:?}");
+            bug!("malformed range pattern: {lo:?}..{hi:?}");
         }
         IntRange { lo, hi }
     }
@@ -241,7 +242,7 @@ impl IntRange {
     }
 
     fn intersection(&self, other: &Self) -> Option<Self> {
-        if self.lo <= other.hi && other.lo <= self.hi {
+        if self.lo < other.hi && other.lo < self.hi {
             Some(IntRange { lo: max(self.lo, other.lo), hi: min(self.hi, other.hi) })
         } else {
             None
@@ -275,38 +276,45 @@ impl IntRange {
     /// ```
     /// where each sequence of dashes is an output range, and dashes outside parentheses are marked
     /// as `Presence::Missing`.
+    ///
+    /// ## `isize`/`usize`
+    ///
+    /// Whereas a wildcard of type `i32` stands for the range `i32::MIN..=i32::MAX`, a `usize`
+    /// wildcard stands for `0..PosInfinity` and a `isize` wildcard stands for
+    /// `NegInfinity..PosInfinity`. In other words, as far as `IntRange` is concerned, there are
+    /// values before `isize::MIN` and after `usize::MAX`/`isize::MAX`.
+    /// This is to avoid e.g. `0..(u32::MAX as usize)` from being exhaustive on one architecture and
+    /// not others. See discussions around the `precise_pointer_size_matching` feature for more
+    /// details.
+    ///
+    /// These infinities affect splitting subtly: it is possible to get `NegInfinity..0` and
+    /// `usize::MAX+1..PosInfinity` in the output. Diagnostics must be careful to handle these
+    /// fictitious ranges sensibly.
     fn split(
         &self,
         column_ranges: impl Iterator<Item = IntRange>,
     ) -> impl Iterator<Item = (Presence, IntRange)> {
-        // Make the range into an exclusive range.
-        fn unpack_intrange(range: IntRange) -> [MaybeInfiniteInt; 2] {
-            [range.lo, range.hi.plus_one()]
-        }
-
         // The boundaries of ranges in `column_ranges` intersected with `self`.
         // We do parenthesis matching for input ranges. A boundary counts as +1 if it starts
         // a range and -1 if it ends it. When the count is > 0 between two boundaries, we
         // are within an input range.
         let mut boundaries: Vec<(MaybeInfiniteInt, isize)> = column_ranges
             .filter_map(|r| self.intersection(&r))
-            .map(unpack_intrange)
-            .flat_map(|[lo, hi]| [(lo, 1), (hi, -1)])
+            .flat_map(|r| [(r.lo, 1), (r.hi, -1)])
             .collect();
         // We sort by boundary, and for each boundary we sort the "closing parentheses" first. The
         // order of +1/-1 for a same boundary value is actually irrelevant, because we only look at
         // the accumulated count between distinct boundary values.
         boundaries.sort_unstable();
 
-        let [self_start, self_end] = unpack_intrange(*self);
         // Accumulate parenthesis counts.
         let mut paren_counter = 0isize;
         // Gather pairs of adjacent boundaries.
-        let mut prev_bdy = self_start;
+        let mut prev_bdy = self.lo;
         boundaries
             .into_iter()
             // End with the end of the range. The count is ignored.
-            .chain(once((self_end, 0)))
+            .chain(once((self.hi, 0)))
             // List pairs of adjacent boundaries and the count between them.
             .map(move |(bdy, delta)| {
                 // `delta` affects the count as we cross `bdy`, so the relevant count between
@@ -322,21 +330,22 @@ impl IntRange {
             .map(move |(prev_bdy, paren_count, bdy)| {
                 use Presence::*;
                 let presence = if paren_count > 0 { Seen } else { Unseen };
-                // Turn back into an inclusive range.
-                let range = IntRange::from_range(prev_bdy, bdy, RangeEnd::Excluded);
+                let range = IntRange { lo: prev_bdy, hi: bdy };
                 (presence, range)
             })
     }
 
-    /// Whether the range denotes the values before `isize::MIN` or the values after
-    /// `usize::MAX`/`isize::MAX`.
+    /// Whether the range denotes the fictitious values before `isize::MIN` or after
+    /// `usize::MAX`/`isize::MAX` (see doc of [`IntRange::split`] for why these exist).
     pub(crate) fn is_beyond_boundaries<'tcx>(&self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> bool {
-        // First check if we are usize/isize to avoid unnecessary `to_diagnostic_pat_range_bdy`.
         ty.is_ptr_sized_integral() && !tcx.features().precise_pointer_size_matching && {
+            // The two invalid ranges are `NegInfinity..isize::MIN` (represented as
+            // `NegInfinity..0`), and `{u,i}size::MAX+1..PosInfinity`. `to_diagnostic_pat_range_bdy`
+            // converts `MAX+1` to `PosInfinity`, and we couldn't have `PosInfinity` in `self.lo`
+            // otherwise.
             let lo = self.lo.to_diagnostic_pat_range_bdy(ty, tcx);
-            let hi = self.hi.to_diagnostic_pat_range_bdy(ty, tcx);
             matches!(lo, PatRangeBoundary::PosInfinity)
-                || matches!(hi, PatRangeBoundary::NegInfinity)
+                || matches!(self.hi, MaybeInfiniteInt::Finite(0))
         }
     }
     /// Only used for displaying the range.
@@ -348,28 +357,27 @@ impl IntRange {
             let value = lo.as_finite().unwrap();
             PatKind::Constant { value }
         } else {
+            // We convert to an inclusive range for diagnostics.
+            let mut end = RangeEnd::Included;
             let mut lo = self.lo.to_diagnostic_pat_range_bdy(ty, tcx);
-            let mut hi = self.hi.to_diagnostic_pat_range_bdy(ty, tcx);
-            let end = if hi.is_finite() {
-                RangeEnd::Included
-            } else {
-                // `0..=` isn't a valid pattern.
-                RangeEnd::Excluded
-            };
-            if matches!(hi, PatRangeBoundary::NegInfinity) {
-                // The range denotes the values before `isize::MIN`.
-                let c = ty.numeric_min_val(tcx).unwrap();
-                let value = mir::Const::from_ty_const(c, tcx);
-                hi = PatRangeBoundary::Finite(value);
-            }
             if matches!(lo, PatRangeBoundary::PosInfinity) {
-                // The range denotes the values after `usize::MAX`/`isize::MAX`.
-                // We represent this as `usize::MAX..` which is slightly incorrect but probably
-                // clear enough.
+                // The only reason to get `PosInfinity` here is the special case where
+                // `to_diagnostic_pat_range_bdy` found `{u,i}size::MAX+1`. So the range denotes the
+                // fictitious values after `{u,i}size::MAX` (see [`IntRange::split`] for why we do
+                // this). We show this to the user as `usize::MAX..` which is slightly incorrect but
+                // probably clear enough.
                 let c = ty.numeric_max_val(tcx).unwrap();
                 let value = mir::Const::from_ty_const(c, tcx);
                 lo = PatRangeBoundary::Finite(value);
             }
+            let hi = if matches!(self.hi, MaybeInfiniteInt::Finite(0)) {
+                // The range encodes `..ty::MIN`, so we can't convert it to an inclusive range.
+                end = RangeEnd::Excluded;
+                self.hi
+            } else {
+                self.hi.minus_one()
+            };
+            let hi = hi.to_diagnostic_pat_range_bdy(ty, tcx);
             PatKind::Range(Box::new(PatRange { lo, hi, end, ty }))
         };
 
@@ -384,7 +392,7 @@ impl fmt::Debug for IntRange {
         if let Finite(lo) = self.lo {
             write!(f, "{lo}")?;
         }
-        write!(f, "{}", RangeEnd::Included)?;
+        write!(f, "{}", RangeEnd::Excluded)?;
         if let Finite(hi) = self.hi {
             write!(f, "{hi}")?;
         }
diff --git a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
index c0920a3ef5a5..27bf5b92552f 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
@@ -1052,7 +1052,7 @@ fn lint_overlapping_range_endpoints<'p, 'tcx>(
                             emit_lint(overlap_range, this_span, &prefixes);
                         }
                         suffixes.push(this_span)
-                    } else if this_range.hi == overlap {
+                    } else if this_range.hi == overlap.plus_one() {
                         // `this_range` looks like `this_range.lo..=overlap`; it overlaps with any
                         // ranges that look like `overlap..=hi`.
                         if !suffixes.is_empty() {

From 24be43356e31d6b0ee2ec8bf912c0a325f236a1d Mon Sep 17 00:00:00 2001
From: Camille Gillot <gillot.camille@gmail.com>
Date: Fri, 27 Oct 2023 20:51:25 +0200
Subject: [PATCH 162/297] Apply suggestions from code review

Co-authored-by: Ralf Jung <post@ralfj.de>
---
 compiler/rustc_middle/src/mir/consts.rs | 11 +++++++----
 compiler/rustc_mir_transform/src/gvn.rs |  7 +++++--
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/compiler/rustc_middle/src/mir/consts.rs b/compiler/rustc_middle/src/mir/consts.rs
index 9ddf3903d04c..3cb2e349ce08 100644
--- a/compiler/rustc_middle/src/mir/consts.rs
+++ b/compiler/rustc_middle/src/mir/consts.rs
@@ -174,10 +174,13 @@ impl<'tcx> ConstValue<'tcx> {
     }
 
     /// Check if a constant may contain provenance information. This is used by MIR opts.
+    /// Can return `true` even if there is no provenance.
     pub fn may_have_provenance(&self, tcx: TyCtxt<'tcx>, size: Size) -> bool {
         match *self {
             ConstValue::ZeroSized | ConstValue::Scalar(Scalar::Int(_)) => return false,
             ConstValue::Scalar(Scalar::Ptr(..)) => return true,
+            // It's hard to find out the part of the allocation we point to;
+            // just conservatively check everything.
             ConstValue::Slice { data, meta: _ } => !data.inner().provenance().ptrs().is_empty(),
             ConstValue::Indirect { alloc_id, offset } => !tcx
                 .global_alloc(alloc_id)
@@ -504,10 +507,10 @@ impl<'tcx> Const<'tcx> {
     /// Return true if any evaluation of this constant always returns the same value,
     /// taking into account even pointer identity tests.
     pub fn is_deterministic(&self) -> bool {
-        // Some constants may contain pointers. We need to preserve the provenance of these
-        // pointers, but not all constants guarantee this:
-        // - valtrees purposefully do not;
-        // - ConstValue::Slice does not either.
+        // Some constants may generate fresh allocations for pointers they contain,
+        // so using the same constant twice can yield two different results:
+        // - valtrees purposefully generate new allocations
+        // - ConstValue::Slice also generate new allocations
         match self {
             Const::Ty(c) => match c.kind() {
                 ty::ConstKind::Param(..) => true,
diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index 39065962c695..de0dc25808b3 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -917,6 +917,7 @@ fn op_to_prop_const<'tcx>(
 
         // Do not try interning a value that contains provenance.
         // Due to https://github.com/rust-lang/rust/issues/79738, doing so could lead to bugs.
+        // FIXME: remove this hack once that issue is fixed.
         let alloc_ref = ecx.get_ptr_alloc(mplace.ptr(), size).ok()??;
         if alloc_ref.has_provenance() {
             return None;
@@ -928,6 +929,8 @@ fn op_to_prop_const<'tcx>(
         if matches!(ecx.tcx.global_alloc(alloc_id), GlobalAlloc::Memory(_)) {
             // `alloc_id` may point to a static. Codegen will choke on an `Indirect` with anything
             // by `GlobalAlloc::Memory`, so do fall through to copying if needed.
+            // FIXME: find a way to treat this more uniformly
+            // (probably by fixing codegen)
             return Some(ConstValue::Indirect { alloc_id, offset });
         }
     }
@@ -939,7 +942,7 @@ fn op_to_prop_const<'tcx>(
 
     // Check that we do not leak a pointer.
     // Those pointers may lose part of their identity in codegen.
-    // See https://github.com/rust-lang/rust/issues/79738.
+    // FIXME: remove this hack once https://github.com/rust-lang/rust/issues/79738 is fixed.
     if ecx.tcx.global_alloc(alloc_id).unwrap_memory().inner().provenance().ptrs().is_empty() {
         return Some(value);
     }
@@ -969,7 +972,7 @@ impl<'tcx> VnState<'_, 'tcx> {
 
         // Check that we do not leak a pointer.
         // Those pointers may lose part of their identity in codegen.
-        // See https://github.com/rust-lang/rust/issues/79738.
+        // FIXME: remove this hack once https://github.com/rust-lang/rust/issues/79738 is fixed.
         assert!(!value.may_have_provenance(self.tcx, op.layout.size));
 
         let const_ = Const::Val(value, op.layout.ty);

From aaa4e541ee1ce764eb58f848563bca0b8a330f72 Mon Sep 17 00:00:00 2001
From: bjorn3 <17426603+bjorn3@users.noreply.github.com>
Date: Fri, 27 Oct 2023 18:56:46 +0000
Subject: [PATCH 163/297] Explicitly mark which targets to distribute cg_clif
 for in CI

This avoids needlessly building cg_clif for other targets and makes it
easier for the dist code to determine if it should distribute cg_clif as
component.
---
 .github/workflows/ci.yml                   | 14 ++++++++++----
 src/bootstrap/src/core/build_steps/dist.rs |  4 +++-
 src/ci/github-actions/ci.yml               | 10 ++++++++++
 src/ci/run.sh                              |  6 ++++--
 4 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f91e6816ae42..ddeb489c647d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -193,8 +193,9 @@ jobs:
             os: ubuntu-20.04-8core-32gb
             env: {}
           - name: dist-aarch64-linux
+            env:
+              CODEGEN_BACKENDS: "llvm,cranelift"
             os: ubuntu-20.04-8core-32gb
-            env: {}
           - name: dist-android
             os: ubuntu-20.04-8core-32gb
             env: {}
@@ -244,15 +245,18 @@ jobs:
             os: ubuntu-20.04-8core-32gb
             env: {}
           - name: dist-x86_64-linux
+            env:
+              CODEGEN_BACKENDS: "llvm,cranelift"
             os: ubuntu-20.04-16core-64gb
-            env: {}
           - name: dist-x86_64-linux-alt
             env:
               IMAGE: dist-x86_64-linux
+              CODEGEN_BACKENDS: "llvm,cranelift"
             os: ubuntu-20.04-16core-64gb
           - name: dist-x86_64-musl
+            env:
+              CODEGEN_BACKENDS: "llvm,cranelift"
             os: ubuntu-20.04-8core-32gb
-            env: {}
           - name: dist-x86_64-netbsd
             os: ubuntu-20.04-8core-32gb
             env: {}
@@ -316,6 +320,7 @@ jobs:
               NO_DEBUG_ASSERTIONS: 1
               NO_OVERFLOW_CHECKS: 1
               DIST_REQUIRE_ALL_TOOLS: 1
+              CODEGEN_BACKENDS: "llvm,cranelift"
             os: macos-13
           - name: dist-apple-various
             env:
@@ -554,8 +559,9 @@ jobs:
       matrix:
         include:
           - name: dist-x86_64-linux
+            env:
+              CODEGEN_BACKENDS: "llvm,cranelift"
             os: ubuntu-20.04-16core-64gb
-            env: {}
     timeout-minutes: 600
     runs-on: "${{ matrix.os }}"
     steps:
diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs
index acab3cffe077..b578c5ec2950 100644
--- a/src/bootstrap/src/core/build_steps/dist.rs
+++ b/src/bootstrap/src/core/build_steps/dist.rs
@@ -1624,7 +1624,9 @@ impl Step for Extended {
                     prepare(tool);
                 }
             }
-            prepare("rustc-codegen-cranelift");
+            if builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("cranelift")) {
+                prepare("rustc-codegen-cranelift");
+            }
             // create an 'uninstall' package
             builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755);
             pkgbuild("uninstall");
diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml
index 2577682c57c6..bbba6600581e 100644
--- a/src/ci/github-actions/ci.yml
+++ b/src/ci/github-actions/ci.yml
@@ -359,6 +359,8 @@ jobs:
             <<: *job-linux-8c
 
           - name: dist-aarch64-linux
+            env:
+              CODEGEN_BACKENDS: llvm,cranelift
             <<: *job-linux-8c
 
           - name: dist-android
@@ -411,14 +413,19 @@ jobs:
 
           - &dist-x86_64-linux
             name: dist-x86_64-linux
+            env:
+              CODEGEN_BACKENDS: llvm,cranelift
             <<: *job-linux-16c
 
           - name: dist-x86_64-linux-alt
             env:
               IMAGE: dist-x86_64-linux
+              CODEGEN_BACKENDS: llvm,cranelift
             <<: *job-linux-16c
 
           - name: dist-x86_64-musl
+            env:
+              CODEGEN_BACKENDS: llvm,cranelift
             <<: *job-linux-8c
 
           - name: dist-x86_64-netbsd
@@ -501,6 +508,7 @@ jobs:
               NO_DEBUG_ASSERTIONS: 1
               NO_OVERFLOW_CHECKS: 1
               DIST_REQUIRE_ALL_TOOLS: 1
+              CODEGEN_BACKENDS: llvm,cranelift
             <<: *job-macos-xl
 
           - name: dist-apple-various
@@ -697,6 +705,8 @@ jobs:
         include:
           - &dist-x86_64-linux
             name: dist-x86_64-linux
+            env:
+              CODEGEN_BACKENDS: llvm,cranelift
             <<: *job-linux-16c
 
   master:
diff --git a/src/ci/run.sh b/src/ci/run.sh
index bcfe1fc04104..9a63bb5c91c9 100755
--- a/src/ci/run.sh
+++ b/src/ci/run.sh
@@ -84,8 +84,6 @@ if [ "$DIST_SRC" = "" ]; then
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-dist-src"
 fi
 
-RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift"
-
 # Always set the release channel for bootstrap; this is normally not important (i.e., only dist
 # builds would seem to matter) but in practice bootstrap wants to know whether we're targeting
 # master, beta, or stable with a build to determine whether to run some checks (notably toolstate).
@@ -107,6 +105,7 @@ if [ "$DEPLOY$DEPLOY_ALT" = "1" ]; then
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
   fi
 
+  RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=${CODEGEN_BACKENDS:-llvm}"
 else
   # We almost always want debug assertions enabled, but sometimes this takes too
   # long for too little benefit, so we just turn them off.
@@ -127,6 +126,9 @@ else
 
   RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir"
 
+  # Test the Cranelift backend in CI. Bootstrap knows which targets to run tests on.
+  RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift"
+
   # We enable this for non-dist builders, since those aren't trying to produce
   # fresh binaries. We currently don't entirely support distributing a fresh
   # copy of the compiler (including llvm tools, etc.) if we haven't actually

From a65d99d08702b9ceea0d153e432f5bd5f58a783a Mon Sep 17 00:00:00 2001
From: Milo <50248166+Milo123459@users.noreply.github.com>
Date: Fri, 27 Oct 2023 20:14:49 +0100
Subject: [PATCH 164/297] Update type_err_ctxt_ext.rs

Co-authored-by: Esteban Kuber <estebank@users.noreply.github.com>
---
 .../src/traits/error_reporting/type_err_ctxt_ext.rs           | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
index f83ee576c294..3e7b71320821 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
@@ -437,10 +437,10 @@ impl<'tcx> TypeErrCtxtExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
                                     format!(" in `{t}`"),
                                     format!("within `{t}`, "),
                                     s.map(|s| (format!("within this `{t}`"), s)),
-                                    file.and_then(|file| Some(format!(
+                                    file.map(|file| format!(
                                         "the full trait has been written to '{}'",
                                         file.display(),
-                                    )))
+                                    ))
                                 )
                             })
                             .unwrap_or_default();

From 98c469ce933612e9688f6b5a2ed9f83c1bb028ac Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Thu, 26 Oct 2023 17:12:57 +1100
Subject: [PATCH 165/297] Remove an unneeded dependency.

---
 Cargo.lock                          | 1 -
 compiler/rustc_interface/Cargo.toml | 1 -
 2 files changed, 2 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index ff21d5f8c086..bd121921b0b9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4054,7 +4054,6 @@ dependencies = [
  "rustc_hir_analysis",
  "rustc_hir_typeck",
  "rustc_incremental",
- "rustc_index",
  "rustc_lint",
  "rustc_macros",
  "rustc_metadata",
diff --git a/compiler/rustc_interface/Cargo.toml b/compiler/rustc_interface/Cargo.toml
index ae008674d012..b280665057cc 100644
--- a/compiler/rustc_interface/Cargo.toml
+++ b/compiler/rustc_interface/Cargo.toml
@@ -26,7 +26,6 @@ rustc_middle = { path = "../rustc_middle" }
 rustc_ast_lowering = { path = "../rustc_ast_lowering" }
 rustc_ast_passes = { path = "../rustc_ast_passes" }
 rustc_incremental = { path = "../rustc_incremental" }
-rustc_index = { path = "../rustc_index" }
 rustc_traits = { path = "../rustc_traits" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }

From 3feec48d70efc479ca0c14d65d89f001330b4fc1 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Fri, 27 Oct 2023 08:24:49 +1100
Subject: [PATCH 166/297] Fix a comment.

---
 compiler/rustc_interface/src/callbacks.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler/rustc_interface/src/callbacks.rs b/compiler/rustc_interface/src/callbacks.rs
index 45b1aeb4a5c3..6fa989bb96ce 100644
--- a/compiler/rustc_interface/src/callbacks.rs
+++ b/compiler/rustc_interface/src/callbacks.rs
@@ -26,7 +26,7 @@ fn track_span_parent(def_id: rustc_span::def_id::LocalDefId) {
     })
 }
 
-/// This is a callback from `rustc_ast` as it cannot access the implicit state
+/// This is a callback from `rustc_errors` as it cannot access the implicit state
 /// in `rustc_middle` otherwise. It is used when diagnostic messages are
 /// emitted and stores them in the current query, if there is one.
 fn track_diagnostic(diagnostic: &mut Diagnostic, f: &mut dyn FnMut(&mut Diagnostic)) {

From 2142d014ab9010ed87b3d5c865190bd1210c9073 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Fri, 27 Oct 2023 15:40:21 +1100
Subject: [PATCH 167/297] Streamline `rustc_interface` tests.

In `test_edition_parsing`, change the
`build_session_options_and_crate_config` call to
`build_session_options`, because the config isn't used.

That leaves a single call site for
`build_session_options_and_crate_config`, so just inline and remove it.
---
 compiler/rustc_interface/src/tests.rs | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index 3d191669b1ad..444c184dc2dd 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -39,18 +39,10 @@ use std::sync::Arc;
 
 type CfgSpecs = FxHashSet<(String, Option<String>)>;
 
-fn build_session_options_and_crate_config(
-    handler: &mut EarlyErrorHandler,
-    matches: getopts::Matches,
-) -> (Options, CfgSpecs) {
-    let sessopts = build_session_options(handler, &matches);
-    let cfg = parse_cfgspecs(handler, matches.opt_strs("cfg"));
-    (sessopts, cfg)
-}
-
 fn mk_session(handler: &mut EarlyErrorHandler, matches: getopts::Matches) -> (Session, CfgSpecs) {
     let registry = registry::Registry::new(&[]);
-    let (sessopts, cfg) = build_session_options_and_crate_config(handler, matches);
+    let sessopts = build_session_options(handler, &matches);
+    let cfg = parse_cfgspecs(handler, matches.opt_strs("cfg"));
     let temps_dir = sessopts.unstable_opts.temps_dir.as_deref().map(PathBuf::from);
     let io = CompilerIO {
         input: Input::Str { name: FileName::Custom(String::new()), input: String::new() },
@@ -880,6 +872,6 @@ fn test_edition_parsing() {
     let mut handler = EarlyErrorHandler::new(ErrorOutputType::default());
 
     let matches = optgroups().parse(&["--edition=2018".to_string()]).unwrap();
-    let (sessopts, _) = build_session_options_and_crate_config(&mut handler, matches);
+    let sessopts = build_session_options(&mut handler, &matches);
     assert!(sessopts.edition == Edition::Edition2018)
 }

From 32986d895f9105de00a87d7f3d8897477ab64803 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Fri, 27 Oct 2023 15:49:09 +1100
Subject: [PATCH 168/297] Change `CrateConfig` from `FxIndexSet` to
 `FxHashSet`.

Because its order doesn't matter. This is well demonstrated by
`to_crate_config`, which creates a `CrateConfig` from an `FxHashSet`.
---
 compiler/rustc_session/src/parse.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_session/src/parse.rs b/compiler/rustc_session/src/parse.rs
index abb0ab5630c1..316b9cc7acb1 100644
--- a/compiler/rustc_session/src/parse.rs
+++ b/compiler/rustc_session/src/parse.rs
@@ -9,7 +9,7 @@ use crate::lint::{
     builtin::UNSTABLE_SYNTAX_PRE_EXPANSION, BufferedEarlyLint, BuiltinLintDiagnostics, Lint, LintId,
 };
 use rustc_ast::node_id::NodeId;
-use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
+use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_data_structures::sync::{AppendOnlyVec, Lock, Lrc};
 use rustc_errors::{emitter::SilentEmitter, Handler};
 use rustc_errors::{
@@ -27,7 +27,7 @@ use std::str;
 
 /// The set of keys (and, optionally, values) that define the compilation
 /// environment of the crate, used to drive conditional compilation.
-pub type CrateConfig = FxIndexSet<(Symbol, Option<Symbol>)>;
+pub type CrateConfig = FxHashSet<(Symbol, Option<Symbol>)>;
 pub type CrateCheckConfig = CheckCfg<Symbol>;
 
 /// Collected spans during parsing for places where a certain feature was
@@ -237,7 +237,7 @@ impl ParseSess {
         Self {
             span_diagnostic: handler,
             unstable_features: UnstableFeatures::from_environment(None),
-            config: FxIndexSet::default(),
+            config: FxHashSet::default(),
             check_config: CrateCheckConfig::default(),
             edition: ExpnId::root().expn_data().edition,
             raw_identifier_spans: Default::default(),

From 75e415ba86cc16ee5c0fd3c9007c94ea98995ab3 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Fri, 27 Oct 2023 15:54:14 +1100
Subject: [PATCH 169/297] Optimize `parse_cfgspecs`.

In `parse_cfg`, we now construct a `FxHashSet<String>` directly instead of
constructing a `FxHashSet<Symbol>` and then immediately converting it to a
`FxHashSet<String>`(!)

(The type names made this behaviour non-obvious. The next commit will
make the type names clearer.)
---
 compiler/rustc_interface/src/interface.rs | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index 87badcbdf15b..26d7d26c31cb 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -16,7 +16,7 @@ use rustc_parse::maybe_new_parser_from_source_str;
 use rustc_query_impl::QueryCtxt;
 use rustc_query_system::query::print_query_stack;
 use rustc_session::config::{self, CheckCfg, ExpectedValues, Input, OutFileName, OutputFilenames};
-use rustc_session::parse::{CrateConfig, ParseSess};
+use rustc_session::parse::ParseSess;
 use rustc_session::CompilerIO;
 use rustc_session::Session;
 use rustc_session::{lint, EarlyErrorHandler};
@@ -67,7 +67,7 @@ pub fn parse_cfgspecs(
     cfgspecs: Vec<String>,
 ) -> FxHashSet<(String, Option<String>)> {
     rustc_span::create_default_session_if_not_set_then(move |_| {
-        let cfg = cfgspecs
+        cfgspecs
             .into_iter()
             .map(|s| {
                 let sess = ParseSess::with_silent_emitter(Some(format!(
@@ -97,7 +97,10 @@ pub fn parse_cfgspecs(
                                 }
                                 MetaItemKind::NameValue(..) | MetaItemKind::Word => {
                                     let ident = meta_item.ident().expect("multi-segment cfg key");
-                                    return (ident.name, meta_item.value_str());
+                                    return (
+                                        ident.name.to_string(),
+                                        meta_item.value_str().map(|sym| sym.to_string()),
+                                    );
                                 }
                             }
                         }
@@ -118,8 +121,7 @@ pub fn parse_cfgspecs(
                     error!(r#"expected `key` or `key="value"`"#);
                 }
             })
-            .collect::<CrateConfig>();
-        cfg.into_iter().map(|(a, b)| (a.to_string(), b.map(|b| b.to_string()))).collect()
+            .collect::<FxHashSet<_>>()
     })
 }
 

From 5e5499715714a383d76a3e69807e22093b616c66 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Fri, 27 Oct 2023 15:58:02 +1100
Subject: [PATCH 170/297] Clean up config mess.

`parse_cfgspecs` and `parse_check_cfg` run very early, before the main
interner is running. They each use a short-lived interner and convert
all interned symbols to strings in their output data structures. Once
the main interner starts up, these data structures get converted into
new data structures that are identical except with the strings converted
to symbols.

All is not obvious from the current code, which is a mess, particularly
with inconsistent naming that obscures the parallel string/symbol data
structures. This commit clean things up a lot.

- The existing `CheckCfg` type is generic, allowing both
  `CheckCfg<String>` and `CheckCfg<Symbol>` forms. This is really
  useful, but it defaults to `String`. The commit removes the default so
  we have to use `CheckCfg<String>` and `CheckCfg<Symbol>` explicitly,
  which makes things clearer.

- Introduces `Cfg`, which is generic over `String` and `Symbol`, similar
  to `CheckCfg`.

- Renames some things.
  - `parse_cfgspecs` -> `parse_cfg`
  - `CfgSpecs` -> `Cfg<String>`, plus it's used in more places, rather
    than the underlying `FxHashSet` type.
  - `CrateConfig` -> `Cfg<Symbol>`.
  - `CrateCheckConfig` -> `CheckCfg<Symbol>`

- Adds some comments explaining the string-to-symbol conversions.

- `to_crate_check_config`, which converts `CheckCfg<String>` to
  `CheckCfg<Symbol>`, is inlined and removed and combined with the
  overly-general `CheckCfg::map_data` to produce
  `CheckCfg::<String>::intern`.

- `build_configuration` now does the `Cfg<String>`-to-`Cfg<Symbol>`
  conversion, so callers don't need to, which removes the need for
  `to_crate_config`.

The diff for two of the fields in `Config` is a good example of the
improved clarity:
```
-    pub crate_cfg: FxHashSet<(String, Option<String>)>,
-    pub crate_check_cfg: CheckCfg,
+    pub crate_cfg: Cfg<String>,
+    pub crate_check_cfg: CheckCfg<String>,
```
Compare that with the diff for the corresponding fields in `ParseSess`,
and the relationship to `Config` is much clearer than before:
```
-    pub config: CrateConfig,
-    pub check_config: CrateCheckConfig,
+    pub config: Cfg<Symbol>,
+    pub check_config: CheckCfg<Symbol>,
```
---
 compiler/rustc_driver_impl/src/lib.rs     |  2 +-
 compiler/rustc_interface/src/interface.rs | 27 +++++++------
 compiler/rustc_interface/src/tests.rs     | 21 +++++-----
 compiler/rustc_interface/src/util.rs      | 20 +++++-----
 compiler/rustc_session/src/config.rs      | 47 +++++++++++------------
 compiler/rustc_session/src/parse.rs       | 15 +++-----
 src/librustdoc/core.rs                    |  2 +-
 src/librustdoc/doctest.rs                 |  2 +-
 8 files changed, 65 insertions(+), 71 deletions(-)

diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs
index 331843ab0515..635930742fc8 100644
--- a/compiler/rustc_driver_impl/src/lib.rs
+++ b/compiler/rustc_driver_impl/src/lib.rs
@@ -318,7 +318,7 @@ fn run_compiler(
         return Ok(());
     }
 
-    let cfg = interface::parse_cfgspecs(&early_error_handler, matches.opt_strs("cfg"));
+    let cfg = interface::parse_cfg(&early_error_handler, matches.opt_strs("cfg"));
     let check_cfg = interface::parse_check_cfg(&early_error_handler, matches.opt_strs("check-cfg"));
     let (odir, ofile) = make_output(&matches);
     let mut config = interface::Config {
diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index 26d7d26c31cb..7672523292b4 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -15,7 +15,9 @@ use rustc_middle::{bug, ty};
 use rustc_parse::maybe_new_parser_from_source_str;
 use rustc_query_impl::QueryCtxt;
 use rustc_query_system::query::print_query_stack;
-use rustc_session::config::{self, CheckCfg, ExpectedValues, Input, OutFileName, OutputFilenames};
+use rustc_session::config::{
+    self, Cfg, CheckCfg, ExpectedValues, Input, OutFileName, OutputFilenames,
+};
 use rustc_session::parse::ParseSess;
 use rustc_session::CompilerIO;
 use rustc_session::Session;
@@ -61,14 +63,13 @@ impl Compiler {
     }
 }
 
-/// Converts strings provided as `--cfg [cfgspec]` into a `crate_cfg`.
-pub fn parse_cfgspecs(
-    handler: &EarlyErrorHandler,
-    cfgspecs: Vec<String>,
-) -> FxHashSet<(String, Option<String>)> {
+/// Converts strings provided as `--cfg [cfgspec]` into a `Cfg`.
+pub fn parse_cfg(handler: &EarlyErrorHandler, cfgs: Vec<String>) -> Cfg<String> {
+    // This creates a short-lived `SessionGlobals`, containing an interner. The
+    // parsed values are converted from symbols to strings before exiting
+    // because the symbols are meaningless once the interner is gone.
     rustc_span::create_default_session_if_not_set_then(move |_| {
-        cfgspecs
-            .into_iter()
+        cfgs.into_iter()
             .map(|s| {
                 let sess = ParseSess::with_silent_emitter(Some(format!(
                     "this error occurred on the command line: `--cfg={s}`"
@@ -121,12 +122,14 @@ pub fn parse_cfgspecs(
                     error!(r#"expected `key` or `key="value"`"#);
                 }
             })
-            .collect::<FxHashSet<_>>()
+            .collect::<Cfg<String>>()
     })
 }
 
 /// Converts strings provided as `--check-cfg [specs]` into a `CheckCfg`.
-pub fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -> CheckCfg {
+pub fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -> CheckCfg<String> {
+    // The comment about `SessionGlobals` and symbols in `parse_cfg` above
+    // applies here too.
     rustc_span::create_default_session_if_not_set_then(move |_| {
         // If any --check-cfg is passed then exhaustive_values and exhaustive_names
         // are enabled by default.
@@ -374,8 +377,8 @@ pub struct Config {
     pub opts: config::Options,
 
     /// cfg! configuration in addition to the default ones
-    pub crate_cfg: FxHashSet<(String, Option<String>)>,
-    pub crate_check_cfg: CheckCfg,
+    pub crate_cfg: Cfg<String>,
+    pub crate_check_cfg: CheckCfg<String>,
 
     pub input: Input,
     pub output_dir: Option<PathBuf>,
diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index 444c184dc2dd..26cd83ac9687 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -1,17 +1,16 @@
 #![allow(rustc::bad_opt_access)]
-use crate::interface::parse_cfgspecs;
-
-use rustc_data_structures::fx::FxHashSet;
+use crate::interface::parse_cfg;
 use rustc_data_structures::profiling::TimePassesFormat;
 use rustc_errors::{emitter::HumanReadableErrorType, registry, ColorConfig};
 use rustc_session::config::rustc_optgroups;
+use rustc_session::config::Cfg;
 use rustc_session::config::DebugInfo;
 use rustc_session::config::Input;
 use rustc_session::config::InstrumentXRay;
 use rustc_session::config::LinkSelfContained;
 use rustc_session::config::Polonius;
 use rustc_session::config::TraitSolver;
-use rustc_session::config::{build_configuration, build_session_options, to_crate_config};
+use rustc_session::config::{build_configuration, build_session_options};
 use rustc_session::config::{
     BranchProtection, Externs, OomStrategy, OutFileName, OutputType, OutputTypes, PAuthKey, PacRet,
     ProcMacroExecutionStrategy, SymbolManglingVersion, WasiExecModel,
@@ -31,18 +30,18 @@ use rustc_span::FileName;
 use rustc_span::SourceFileHashAlgorithm;
 use rustc_target::spec::{CodeModel, LinkerFlavorCli, MergeFunctions, PanicStrategy, RelocModel};
 use rustc_target::spec::{RelroLevel, SanitizerSet, SplitDebuginfo, StackProtector, TlsModel};
-
 use std::collections::{BTreeMap, BTreeSet};
 use std::num::NonZeroUsize;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
-type CfgSpecs = FxHashSet<(String, Option<String>)>;
-
-fn mk_session(handler: &mut EarlyErrorHandler, matches: getopts::Matches) -> (Session, CfgSpecs) {
+fn mk_session(
+    handler: &mut EarlyErrorHandler,
+    matches: getopts::Matches,
+) -> (Session, Cfg<String>) {
     let registry = registry::Registry::new(&[]);
     let sessopts = build_session_options(handler, &matches);
-    let cfg = parse_cfgspecs(handler, matches.opt_strs("cfg"));
+    let cfg = parse_cfg(handler, matches.opt_strs("cfg"));
     let temps_dir = sessopts.unstable_opts.temps_dir.as_deref().map(PathBuf::from);
     let io = CompilerIO {
         input: Input::Str { name: FileName::Custom(String::new()), input: String::new() },
@@ -133,7 +132,7 @@ fn test_switch_implies_cfg_test() {
         let matches = optgroups().parse(&["--test".to_string()]).unwrap();
         let mut handler = EarlyErrorHandler::new(ErrorOutputType::default());
         let (sess, cfg) = mk_session(&mut handler, matches);
-        let cfg = build_configuration(&sess, to_crate_config(cfg));
+        let cfg = build_configuration(&sess, cfg);
         assert!(cfg.contains(&(sym::test, None)));
     });
 }
@@ -145,7 +144,7 @@ fn test_switch_implies_cfg_test_unless_cfg_test() {
         let matches = optgroups().parse(&["--test".to_string(), "--cfg=test".to_string()]).unwrap();
         let mut handler = EarlyErrorHandler::new(ErrorOutputType::default());
         let (sess, cfg) = mk_session(&mut handler, matches);
-        let cfg = build_configuration(&sess, to_crate_config(cfg));
+        let cfg = build_configuration(&sess, cfg);
         let mut test_items = cfg.iter().filter(|&&(name, _)| name == sym::test);
         assert!(test_items.next().is_some());
         assert!(test_items.next().is_none());
diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs
index d2455a036ccd..4d0be65697a4 100644
--- a/compiler/rustc_interface/src/util.rs
+++ b/compiler/rustc_interface/src/util.rs
@@ -3,18 +3,17 @@ use info;
 use libloading::Library;
 use rustc_ast as ast;
 use rustc_codegen_ssa::traits::CodegenBackend;
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_data_structures::fx::FxHashMap;
 #[cfg(parallel_compiler)]
 use rustc_data_structures::sync;
 use rustc_errors::registry::Registry;
 use rustc_parse::validate_attr;
 use rustc_session as session;
-use rustc_session::config::CheckCfg;
-use rustc_session::config::{self, CrateType};
-use rustc_session::config::{OutFileName, OutputFilenames, OutputTypes};
+use rustc_session::config::{
+    self, Cfg, CheckCfg, CrateType, OutFileName, OutputFilenames, OutputTypes,
+};
 use rustc_session::filesearch::sysroot_candidates;
 use rustc_session::lint::{self, BuiltinLintDiagnostics, LintBuffer};
-use rustc_session::parse::CrateConfig;
 use rustc_session::{filesearch, output, Session};
 use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::edition::Edition;
@@ -38,7 +37,7 @@ pub type MakeBackendFn = fn() -> Box<dyn CodegenBackend>;
 /// This is performed by checking whether a set of permitted features
 /// is available on the target machine, by querying the codegen backend.
 pub fn add_configuration(
-    cfg: &mut CrateConfig,
+    cfg: &mut Cfg<Symbol>,
     sess: &mut Session,
     codegen_backend: &dyn CodegenBackend,
 ) {
@@ -60,8 +59,8 @@ pub fn add_configuration(
 pub fn create_session(
     handler: &EarlyErrorHandler,
     sopts: config::Options,
-    cfg: FxHashSet<(String, Option<String>)>,
-    check_cfg: CheckCfg,
+    cfg: Cfg<String>,
+    check_cfg: CheckCfg<String>,
     locale_resources: &'static [&'static str],
     file_loader: Option<Box<dyn FileLoader + Send + Sync + 'static>>,
     io: CompilerIO,
@@ -121,12 +120,13 @@ pub fn create_session(
 
     codegen_backend.init(&sess);
 
-    let mut cfg = config::build_configuration(&sess, config::to_crate_config(cfg));
+    let mut cfg = config::build_configuration(&sess, cfg);
     add_configuration(&mut cfg, &mut sess, &*codegen_backend);
 
-    let mut check_cfg = config::to_crate_check_config(check_cfg);
+    let mut check_cfg = check_cfg.intern();
     check_cfg.fill_well_known(&sess.target);
 
+    // These configs use symbols, rather than strings.
     sess.parse_sess.config = cfg;
     sess.parse_sess.check_config = check_cfg;
 
diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index 7aced414ed67..fef4dfba6d22 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -16,7 +16,6 @@ use rustc_target::spec::LinkSelfContainedComponents;
 use rustc_target::spec::{PanicStrategy, RelocModel, SanitizerSet, SplitDebuginfo};
 use rustc_target::spec::{Target, TargetTriple, TargetWarnings, TARGETS};
 
-use crate::parse::{CrateCheckConfig, CrateConfig};
 use rustc_feature::UnstableFeatures;
 use rustc_span::edition::{Edition, DEFAULT_EDITION, EDITION_NAME_LIST, LATEST_STABLE_EDITION};
 use rustc_span::source_map::{FileName, FilePathMapping};
@@ -1248,8 +1247,8 @@ pub const fn default_lib_output() -> CrateType {
     CrateType::Rlib
 }
 
-fn default_configuration(sess: &Session) -> CrateConfig {
-    // NOTE: This should be kept in sync with `CrateCheckConfig::fill_well_known` below.
+fn default_configuration(sess: &Session) -> Cfg<Symbol> {
+    // NOTE: This should be kept in sync with `CheckCfg::<Symbol>::fill_well_known` below.
     let end = &sess.target.endian;
     let arch = &sess.target.arch;
     let wordsz = sess.target.pointer_width.to_string();
@@ -1265,7 +1264,7 @@ fn default_configuration(sess: &Session) -> CrateConfig {
         sess.emit_fatal(err);
     });
 
-    let mut ret = CrateConfig::default();
+    let mut ret = Cfg::default();
     ret.reserve(7); // the minimum number of insertions
     // Target bindings.
     ret.insert((sym::target_os, Some(Symbol::intern(os))));
@@ -1358,15 +1357,14 @@ fn default_configuration(sess: &Session) -> CrateConfig {
     ret
 }
 
-/// Converts the crate `cfg!` configuration from `String` to `Symbol`.
-/// `rustc_interface::interface::Config` accepts this in the compiler configuration,
-/// but the symbol interner is not yet set up then, so we must convert it later.
-pub fn to_crate_config(cfg: FxHashSet<(String, Option<String>)>) -> CrateConfig {
-    cfg.into_iter().map(|(a, b)| (Symbol::intern(&a), b.map(|b| Symbol::intern(&b)))).collect()
-}
+/// The parsed `--cfg` options that define the compilation environment of the
+/// crate, used to drive conditional compilation. `T` is always `String` or
+/// `Symbol`. Strings are used temporarily very early on. Once the the main
+/// symbol interner is running, they are converted to symbols.
+pub type Cfg<T> = FxHashSet<(T, Option<T>)>;
 
-/// The parsed `--check-cfg` options
-pub struct CheckCfg<T = String> {
+/// The parsed `--check-cfg` options. The `<T>` structure is similar to `Cfg`.
+pub struct CheckCfg<T> {
     /// Is well known names activated
     pub exhaustive_names: bool,
     /// Is well known values activated
@@ -1385,8 +1383,8 @@ impl<T> Default for CheckCfg<T> {
     }
 }
 
-impl<T> CheckCfg<T> {
-    fn map_data<O: Eq + Hash>(self, f: impl Fn(T) -> O) -> CheckCfg<O> {
+impl CheckCfg<String> {
+    pub fn intern(self) -> CheckCfg<Symbol> {
         CheckCfg {
             exhaustive_names: self.exhaustive_names,
             exhaustive_values: self.exhaustive_values,
@@ -1395,10 +1393,10 @@ impl<T> CheckCfg<T> {
                 .into_iter()
                 .map(|(name, values)| {
                     (
-                        f(name),
+                        Symbol::intern(&name),
                         match values {
                             ExpectedValues::Some(values) => ExpectedValues::Some(
-                                values.into_iter().map(|b| b.map(|b| f(b))).collect(),
+                                values.into_iter().map(|b| b.map(|b| Symbol::intern(&b))).collect(),
                             ),
                             ExpectedValues::Any => ExpectedValues::Any,
                         },
@@ -1441,14 +1439,7 @@ impl<'a, T: Eq + Hash + Copy + 'a> Extend<&'a T> for ExpectedValues<T> {
     }
 }
 
-/// Converts the crate `--check-cfg` options from `String` to `Symbol`.
-/// `rustc_interface::interface::Config` accepts this in the compiler configuration,
-/// but the symbol interner is not yet set up then, so we must convert it later.
-pub fn to_crate_check_config(cfg: CheckCfg) -> CrateCheckConfig {
-    cfg.map_data(|s| Symbol::intern(&s))
-}
-
-impl CrateCheckConfig {
+impl CheckCfg<Symbol> {
     pub fn fill_well_known(&mut self, current_target: &Target) {
         if !self.exhaustive_values && !self.exhaustive_names {
             return;
@@ -1588,7 +1579,13 @@ impl CrateCheckConfig {
     }
 }
 
-pub fn build_configuration(sess: &Session, mut user_cfg: CrateConfig) -> CrateConfig {
+pub fn build_configuration(sess: &Session, user_cfg: Cfg<String>) -> Cfg<Symbol> {
+    // We can now intern these strings.
+    let mut user_cfg: Cfg<Symbol> = user_cfg
+        .into_iter()
+        .map(|(a, b)| (Symbol::intern(&a), b.map(|b| Symbol::intern(&b))))
+        .collect();
+
     // Combine the configuration requested by the session (command line) with
     // some default and generated configuration items.
     let default_cfg = default_configuration(sess);
diff --git a/compiler/rustc_session/src/parse.rs b/compiler/rustc_session/src/parse.rs
index 316b9cc7acb1..5b98ee5d992d 100644
--- a/compiler/rustc_session/src/parse.rs
+++ b/compiler/rustc_session/src/parse.rs
@@ -1,7 +1,7 @@
 //! Contains `ParseSess` which holds state living beyond what one `Parser` might.
 //! It also serves as an input to the parser itself.
 
-use crate::config::CheckCfg;
+use crate::config::{Cfg, CheckCfg};
 use crate::errors::{
     CliFeatureDiagnosticHelp, FeatureDiagnosticForIssue, FeatureDiagnosticHelp, FeatureGateError,
 };
@@ -25,11 +25,6 @@ use rustc_span::{Span, Symbol};
 use rustc_ast::attr::AttrIdGenerator;
 use std::str;
 
-/// The set of keys (and, optionally, values) that define the compilation
-/// environment of the crate, used to drive conditional compilation.
-pub type CrateConfig = FxHashSet<(Symbol, Option<Symbol>)>;
-pub type CrateCheckConfig = CheckCfg<Symbol>;
-
 /// Collected spans during parsing for places where a certain feature was
 /// used and should be feature gated accordingly in `check_crate`.
 #[derive(Default)]
@@ -193,8 +188,8 @@ pub fn add_feature_diagnostics_for_issue(
 pub struct ParseSess {
     pub span_diagnostic: Handler,
     pub unstable_features: UnstableFeatures,
-    pub config: CrateConfig,
-    pub check_config: CrateCheckConfig,
+    pub config: Cfg<Symbol>,
+    pub check_config: CheckCfg<Symbol>,
     pub edition: Edition,
     /// Places where raw identifiers were used. This is used to avoid complaining about idents
     /// clashing with keywords in new editions.
@@ -237,8 +232,8 @@ impl ParseSess {
         Self {
             span_diagnostic: handler,
             unstable_features: UnstableFeatures::from_environment(None),
-            config: FxHashSet::default(),
-            check_config: CrateCheckConfig::default(),
+            config: Cfg::default(),
+            check_config: CheckCfg::default(),
             edition: ExpnId::root().expn_data().edition,
             raw_identifier_spans: Default::default(),
             bad_unicode_identifiers: Lock::new(Default::default()),
diff --git a/src/librustdoc/core.rs b/src/librustdoc/core.rs
index 94e557dcfdb5..bcc61df7c4d8 100644
--- a/src/librustdoc/core.rs
+++ b/src/librustdoc/core.rs
@@ -255,7 +255,7 @@ pub(crate) fn create_config(
 
     interface::Config {
         opts: sessopts,
-        crate_cfg: interface::parse_cfgspecs(handler, cfgs),
+        crate_cfg: interface::parse_cfg(handler, cfgs),
         crate_check_cfg: interface::parse_check_cfg(handler, check_cfgs),
         input,
         output_file: None,
diff --git a/src/librustdoc/doctest.rs b/src/librustdoc/doctest.rs
index 97913345e8fb..c61c98e4de9c 100644
--- a/src/librustdoc/doctest.rs
+++ b/src/librustdoc/doctest.rs
@@ -92,7 +92,7 @@ pub(crate) fn run(options: RustdocOptions) -> Result<(), ErrorGuaranteed> {
     cfgs.push("doctest".to_owned());
     let config = interface::Config {
         opts: sessopts,
-        crate_cfg: interface::parse_cfgspecs(&early_error_handler, cfgs),
+        crate_cfg: interface::parse_cfg(&early_error_handler, cfgs),
         crate_check_cfg: interface::parse_check_cfg(
             &early_error_handler,
             options.check_cfgs.clone(),

From 5438004766fc0f9b123608425b8ec7111f536640 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Sat, 28 Oct 2023 09:22:30 +1100
Subject: [PATCH 171/297] Change `Cfg<T>` to an `FxIndexSet`.

Despite what I claimed in an earlier commit, the ordering does matter to
some degree. Using `FxIndexSet` prevents changes to the error message
order in `tests/ui/check-cfg/mix.rs`.
---
 compiler/rustc_session/src/config.rs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index fef4dfba6d22..854e2b779935 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -9,7 +9,7 @@ use crate::utils::{CanonicalizedPath, NativeLib, NativeLibKind};
 use crate::{lint, HashStableContext};
 use crate::{EarlyErrorHandler, Session};
 
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
 use rustc_data_structures::stable_hasher::{StableOrd, ToStableHashKey};
 use rustc_target::abi::Align;
 use rustc_target::spec::LinkSelfContainedComponents;
@@ -1361,7 +1361,10 @@ fn default_configuration(sess: &Session) -> Cfg<Symbol> {
 /// crate, used to drive conditional compilation. `T` is always `String` or
 /// `Symbol`. Strings are used temporarily very early on. Once the the main
 /// symbol interner is running, they are converted to symbols.
-pub type Cfg<T> = FxHashSet<(T, Option<T>)>;
+///
+/// An `FxIndexSet` is used to ensure deterministic ordering of error messages
+/// relating to `--cfg`.
+pub type Cfg<T> = FxIndexSet<(T, Option<T>)>;
 
 /// The parsed `--check-cfg` options. The `<T>` structure is similar to `Cfg`.
 pub struct CheckCfg<T> {

From 975b98b8160adf429336170d838911cae8d66de4 Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Fri, 20 Oct 2023 21:12:24 +1100
Subject: [PATCH 172/297] coverage: Regression test for #17012 (bad counter IDs
 in mappings)

---
 .../status-quo/bad_counter_ids.cov-map        | 98 +++++++++++++++++++
 .../status-quo/bad_counter_ids.rs             | 66 +++++++++++++
 tests/run-coverage/bad_counter_ids.coverage   | 68 +++++++++++++
 tests/run-coverage/bad_counter_ids.rs         | 66 +++++++++++++
 4 files changed, 298 insertions(+)
 create mode 100644 tests/coverage-map/status-quo/bad_counter_ids.cov-map
 create mode 100644 tests/coverage-map/status-quo/bad_counter_ids.rs
 create mode 100644 tests/run-coverage/bad_counter_ids.coverage
 create mode 100644 tests/run-coverage/bad_counter_ids.rs

diff --git a/tests/coverage-map/status-quo/bad_counter_ids.cov-map b/tests/coverage-map/status-quo/bad_counter_ids.cov-map
new file mode 100644
index 000000000000..2142bf35d5e7
--- /dev/null
+++ b/tests/coverage-map/status-quo/bad_counter_ids.cov-map
@@ -0,0 +1,98 @@
+Function name: <bad_counter_ids::Foo as core::cmp::PartialEq>::eq
+Raw bytes (9): 0x[01, 01, 00, 01, 01, 0c, 11, 00, 1a]
+Number of files: 1
+- file 0 => global file 1
+Number of expressions: 0
+Number of file 0 mappings: 1
+- Code(Counter(0)) at (prev + 12, 17) to (start + 0, 26)
+
+Function name: <bad_counter_ids::Foo as core::fmt::Debug>::fmt
+Raw bytes (9): 0x[01, 01, 00, 01, 01, 0c, 0a, 00, 0f]
+Number of files: 1
+- file 0 => global file 1
+Number of expressions: 0
+Number of file 0 mappings: 1
+- Code(Counter(0)) at (prev + 12, 10) to (start + 0, 15)
+
+Function name: bad_counter_ids::eq_bad
+Raw bytes (14): 0x[01, 01, 00, 02, 01, 23, 01, 02, 1f, 00, 03, 01, 00, 02]
+Number of files: 1
+- file 0 => global file 1
+Number of expressions: 0
+Number of file 0 mappings: 2
+- Code(Counter(0)) at (prev + 35, 1) to (start + 2, 31)
+- Code(Zero) at (prev + 3, 1) to (start + 0, 2)
+
+Function name: bad_counter_ids::eq_bad_message
+Raw bytes (21): 0x[01, 01, 01, 01, 05, 03, 01, 28, 01, 02, 0f, 02, 02, 20, 00, 2b, 00, 01, 01, 00, 02]
+Number of files: 1
+- file 0 => global file 1
+Number of expressions: 1
+- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
+Number of file 0 mappings: 3
+- Code(Counter(0)) at (prev + 40, 1) to (start + 2, 15)
+- Code(Expression(0, Sub)) at (prev + 2, 32) to (start + 0, 43)
+    = (c0 - c1)
+- Code(Zero) at (prev + 1, 1) to (start + 0, 2)
+
+Function name: bad_counter_ids::eq_good
+Raw bytes (14): 0x[01, 01, 00, 02, 01, 0f, 01, 02, 1f, 05, 03, 01, 00, 02]
+Number of files: 1
+- file 0 => global file 1
+Number of expressions: 0
+Number of file 0 mappings: 2
+- Code(Counter(0)) at (prev + 15, 1) to (start + 2, 31)
+- Code(Counter(1)) at (prev + 3, 1) to (start + 0, 2)
+
+Function name: bad_counter_ids::eq_good_message
+Raw bytes (19): 0x[01, 01, 00, 03, 01, 14, 01, 02, 0f, 00, 02, 20, 00, 2b, 05, 01, 01, 00, 02]
+Number of files: 1
+- file 0 => global file 1
+Number of expressions: 0
+Number of file 0 mappings: 3
+- Code(Counter(0)) at (prev + 20, 1) to (start + 2, 15)
+- Code(Zero) at (prev + 2, 32) to (start + 0, 43)
+- Code(Counter(1)) at (prev + 1, 1) to (start + 0, 2)
+
+Function name: bad_counter_ids::ne_bad
+Raw bytes (14): 0x[01, 01, 00, 02, 01, 2d, 01, 02, 1f, 00, 03, 01, 00, 02]
+Number of files: 1
+- file 0 => global file 1
+Number of expressions: 0
+Number of file 0 mappings: 2
+- Code(Counter(0)) at (prev + 45, 1) to (start + 2, 31)
+- Code(Zero) at (prev + 3, 1) to (start + 0, 2)
+
+Function name: bad_counter_ids::ne_bad_message
+Raw bytes (19): 0x[01, 01, 00, 03, 01, 32, 01, 02, 0f, 05, 02, 20, 00, 2b, 00, 01, 01, 00, 02]
+Number of files: 1
+- file 0 => global file 1
+Number of expressions: 0
+Number of file 0 mappings: 3
+- Code(Counter(0)) at (prev + 50, 1) to (start + 2, 15)
+- Code(Counter(1)) at (prev + 2, 32) to (start + 0, 43)
+- Code(Zero) at (prev + 1, 1) to (start + 0, 2)
+
+Function name: bad_counter_ids::ne_good
+Raw bytes (16): 0x[01, 01, 01, 01, 05, 02, 01, 19, 01, 02, 1f, 02, 03, 01, 00, 02]
+Number of files: 1
+- file 0 => global file 1
+Number of expressions: 1
+- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
+Number of file 0 mappings: 2
+- Code(Counter(0)) at (prev + 25, 1) to (start + 2, 31)
+- Code(Expression(0, Sub)) at (prev + 3, 1) to (start + 0, 2)
+    = (c0 - c1)
+
+Function name: bad_counter_ids::ne_good_message
+Raw bytes (21): 0x[01, 01, 01, 01, 05, 03, 01, 1e, 01, 02, 0f, 00, 02, 20, 00, 2b, 02, 01, 01, 00, 02]
+Number of files: 1
+- file 0 => global file 1
+Number of expressions: 1
+- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
+Number of file 0 mappings: 3
+- Code(Counter(0)) at (prev + 30, 1) to (start + 2, 15)
+- Code(Zero) at (prev + 2, 32) to (start + 0, 43)
+- Code(Expression(0, Sub)) at (prev + 1, 1) to (start + 0, 2)
+    = (c0 - c1)
+
diff --git a/tests/coverage-map/status-quo/bad_counter_ids.rs b/tests/coverage-map/status-quo/bad_counter_ids.rs
new file mode 100644
index 000000000000..ef5460102b70
--- /dev/null
+++ b/tests/coverage-map/status-quo/bad_counter_ids.rs
@@ -0,0 +1,66 @@
+#![feature(coverage_attribute)]
+// compile-flags: --edition=2021 -Copt-level=0 -Zmir-opt-level=3
+
+// Regression test for <https://github.com/rust-lang/rust/issues/117012>.
+//
+// If some coverage counters were removed by MIR optimizations, we need to take
+// care not to refer to those counter IDs in coverage mappings, and instead
+// replace them with a constant zero value. If we don't, `llvm-cov` might see
+// a too-large counter ID and silently discard the entire function from its
+// coverage reports.
+
+#[derive(Debug, PartialEq, Eq)]
+struct Foo(u32);
+
+fn eq_good() {
+    println!("a");
+    assert_eq!(Foo(1), Foo(1));
+}
+
+fn eq_good_message() {
+    println!("b");
+    assert_eq!(Foo(1), Foo(1), "message b");
+}
+
+fn ne_good() {
+    println!("c");
+    assert_ne!(Foo(1), Foo(3));
+}
+
+fn ne_good_message() {
+    println!("d");
+    assert_ne!(Foo(1), Foo(3), "message d");
+}
+
+fn eq_bad() {
+    println!("e");
+    assert_eq!(Foo(1), Foo(3));
+}
+
+fn eq_bad_message() {
+    println!("f");
+    assert_eq!(Foo(1), Foo(3), "message f");
+}
+
+fn ne_bad() {
+    println!("g");
+    assert_ne!(Foo(1), Foo(1));
+}
+
+fn ne_bad_message() {
+    println!("h");
+    assert_ne!(Foo(1), Foo(1), "message h");
+}
+
+#[coverage(off)]
+fn main() {
+    eq_good();
+    eq_good_message();
+    ne_good();
+    ne_good_message();
+
+    assert!(std::panic::catch_unwind(eq_bad).is_err());
+    assert!(std::panic::catch_unwind(eq_bad_message).is_err());
+    assert!(std::panic::catch_unwind(ne_bad).is_err());
+    assert!(std::panic::catch_unwind(ne_bad_message).is_err());
+}
diff --git a/tests/run-coverage/bad_counter_ids.coverage b/tests/run-coverage/bad_counter_ids.coverage
new file mode 100644
index 000000000000..763f04d099e9
--- /dev/null
+++ b/tests/run-coverage/bad_counter_ids.coverage
@@ -0,0 +1,68 @@
+   LL|       |#![feature(coverage_attribute)]
+   LL|       |// compile-flags: --edition=2021 -Copt-level=0 -Zmir-opt-level=3
+   LL|       |
+   LL|       |// Regression test for <https://github.com/rust-lang/rust/issues/117012>.
+   LL|       |//
+   LL|       |// If some coverage counters were removed by MIR optimizations, we need to take
+   LL|       |// care not to refer to those counter IDs in coverage mappings, and instead
+   LL|       |// replace them with a constant zero value. If we don't, `llvm-cov` might see
+   LL|       |// a too-large counter ID and silently discard the entire function from its
+   LL|       |// coverage reports.
+   LL|       |
+   LL|      8|#[derive(Debug, PartialEq, Eq)]
+   LL|       |struct Foo(u32);
+   LL|       |
+   LL|      1|fn eq_good() {
+   LL|      1|    println!("a");
+   LL|      1|    assert_eq!(Foo(1), Foo(1));
+   LL|      1|}
+   LL|       |
+   LL|      1|fn eq_good_message() {
+   LL|      1|    println!("b");
+   LL|      1|    assert_eq!(Foo(1), Foo(1), "message b");
+                                             ^0
+   LL|      1|}
+   LL|       |
+   LL|       |fn ne_good() {
+   LL|       |    println!("c");
+   LL|       |    assert_ne!(Foo(1), Foo(3));
+   LL|       |}
+   LL|       |
+   LL|       |fn ne_good_message() {
+   LL|       |    println!("d");
+   LL|       |    assert_ne!(Foo(1), Foo(3), "message d");
+   LL|       |}
+   LL|       |
+   LL|      1|fn eq_bad() {
+   LL|      1|    println!("e");
+   LL|      1|    assert_eq!(Foo(1), Foo(3));
+   LL|      0|}
+   LL|       |
+   LL|       |fn eq_bad_message() {
+   LL|       |    println!("f");
+   LL|       |    assert_eq!(Foo(1), Foo(3), "message f");
+   LL|       |}
+   LL|       |
+   LL|      1|fn ne_bad() {
+   LL|      1|    println!("g");
+   LL|      1|    assert_ne!(Foo(1), Foo(1));
+   LL|      0|}
+   LL|       |
+   LL|      1|fn ne_bad_message() {
+   LL|      1|    println!("h");
+   LL|      1|    assert_ne!(Foo(1), Foo(1), "message h");
+   LL|      0|}
+   LL|       |
+   LL|       |#[coverage(off)]
+   LL|       |fn main() {
+   LL|       |    eq_good();
+   LL|       |    eq_good_message();
+   LL|       |    ne_good();
+   LL|       |    ne_good_message();
+   LL|       |
+   LL|       |    assert!(std::panic::catch_unwind(eq_bad).is_err());
+   LL|       |    assert!(std::panic::catch_unwind(eq_bad_message).is_err());
+   LL|       |    assert!(std::panic::catch_unwind(ne_bad).is_err());
+   LL|       |    assert!(std::panic::catch_unwind(ne_bad_message).is_err());
+   LL|       |}
+
diff --git a/tests/run-coverage/bad_counter_ids.rs b/tests/run-coverage/bad_counter_ids.rs
new file mode 100644
index 000000000000..ef5460102b70
--- /dev/null
+++ b/tests/run-coverage/bad_counter_ids.rs
@@ -0,0 +1,66 @@
+#![feature(coverage_attribute)]
+// compile-flags: --edition=2021 -Copt-level=0 -Zmir-opt-level=3
+
+// Regression test for <https://github.com/rust-lang/rust/issues/117012>.
+//
+// If some coverage counters were removed by MIR optimizations, we need to take
+// care not to refer to those counter IDs in coverage mappings, and instead
+// replace them with a constant zero value. If we don't, `llvm-cov` might see
+// a too-large counter ID and silently discard the entire function from its
+// coverage reports.
+
+#[derive(Debug, PartialEq, Eq)]
+struct Foo(u32);
+
+fn eq_good() {
+    println!("a");
+    assert_eq!(Foo(1), Foo(1));
+}
+
+fn eq_good_message() {
+    println!("b");
+    assert_eq!(Foo(1), Foo(1), "message b");
+}
+
+fn ne_good() {
+    println!("c");
+    assert_ne!(Foo(1), Foo(3));
+}
+
+fn ne_good_message() {
+    println!("d");
+    assert_ne!(Foo(1), Foo(3), "message d");
+}
+
+fn eq_bad() {
+    println!("e");
+    assert_eq!(Foo(1), Foo(3));
+}
+
+fn eq_bad_message() {
+    println!("f");
+    assert_eq!(Foo(1), Foo(3), "message f");
+}
+
+fn ne_bad() {
+    println!("g");
+    assert_ne!(Foo(1), Foo(1));
+}
+
+fn ne_bad_message() {
+    println!("h");
+    assert_ne!(Foo(1), Foo(1), "message h");
+}
+
+#[coverage(off)]
+fn main() {
+    eq_good();
+    eq_good_message();
+    ne_good();
+    ne_good_message();
+
+    assert!(std::panic::catch_unwind(eq_bad).is_err());
+    assert!(std::panic::catch_unwind(eq_bad_message).is_err());
+    assert!(std::panic::catch_unwind(ne_bad).is_err());
+    assert!(std::panic::catch_unwind(ne_bad_message).is_err());
+}

From 230dd5b8c71c1d176a6705bb458a473411788f68 Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Tue, 24 Oct 2023 22:48:59 +1100
Subject: [PATCH 173/297] coverage: Consistently remove unused counter IDs from
 expressions/mappings

---
 .../src/coverageinfo/map_data.rs              |  72 +++++----
 tests/coverage-map/fn_sig_into_try.cov-map    |  30 ++--
 .../status-quo/bad_counter_ids.cov-map        |  18 +--
 .../status-quo/inline-dead.cov-map            |  18 +--
 .../status-quo/issue-84561.cov-map            |  56 +++----
 .../status-quo/loops_branches.cov-map         | 138 +++++++++---------
 .../status-quo/sort_groups.cov-map            |  10 +-
 .../status-quo/tight_inf_loop.cov-map         |   6 +-
 tests/coverage-map/status-quo/while.cov-map   |  10 +-
 tests/run-coverage/bad_counter_ids.coverage   |  25 ++--
 10 files changed, 197 insertions(+), 186 deletions(-)

diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs
index 93a8a4b1d5e8..cd67fafb8e4e 100644
--- a/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs
+++ b/compiler/rustc_codegen_llvm/src/coverageinfo/map_data.rs
@@ -102,7 +102,7 @@ impl<'tcx> FunctionCoverageCollector<'tcx> {
         // have zero as both of their operands, and will therefore always have
         // a value of zero. Other expressions that refer to these as operands
         // can have those operands replaced with `CovTerm::Zero`.
-        let mut zero_expressions = FxIndexSet::default();
+        let mut zero_expressions = ZeroExpressions::default();
 
         // Simplify a copy of each expression based on lower-numbered expressions,
         // and then update the set of always-zero expressions if necessary.
@@ -131,16 +131,16 @@ impl<'tcx> FunctionCoverageCollector<'tcx> {
                 )
             };
 
-            // If an operand refers to an expression that is always zero, then
-            // that operand can be replaced with `CovTerm::Zero`.
-            let maybe_set_operand_to_zero = |operand: &mut CovTerm| match *operand {
-                CovTerm::Expression(id) => {
+            // If an operand refers to a counter or expression that is always
+            // zero, then that operand can be replaced with `CovTerm::Zero`.
+            let maybe_set_operand_to_zero = |operand: &mut CovTerm| {
+                if let CovTerm::Expression(id) = *operand {
                     assert_operand_expression_is_lower(id);
-                    if zero_expressions.contains(&id) {
-                        *operand = CovTerm::Zero;
-                    }
                 }
-                _ => (),
+
+                if is_zero_term(&self.counters_seen, &zero_expressions, *operand) {
+                    *operand = CovTerm::Zero;
+                }
             };
             maybe_set_operand_to_zero(&mut lhs);
             maybe_set_operand_to_zero(&mut rhs);
@@ -159,7 +159,7 @@ impl<'tcx> FunctionCoverageCollector<'tcx> {
             }
         }
 
-        ZeroExpressions(zero_expressions)
+        zero_expressions
     }
 
     pub(crate) fn into_finished(self) -> FunctionCoverage<'tcx> {
@@ -205,19 +205,14 @@ impl<'tcx> FunctionCoverage<'tcx> {
         // thing on the Rust side unless we're confident we can do much better.
         // (See `CounterExpressionsMinimizer` in `CoverageMappingWriter.cpp`.)
 
-        let counter_from_operand = |operand: CovTerm| match operand {
-            CovTerm::Expression(id) if self.zero_expressions.contains(id) => Counter::ZERO,
-            _ => Counter::from_term(operand),
-        };
-
         self.function_coverage_info.expressions.iter().map(move |&Expression { lhs, op, rhs }| {
             CounterExpression {
-                lhs: counter_from_operand(lhs),
+                lhs: self.counter_for_term(lhs),
                 kind: match op {
                     Op::Add => ExprKind::Add,
                     Op::Subtract => ExprKind::Subtract,
                 },
-                rhs: counter_from_operand(rhs),
+                rhs: self.counter_for_term(rhs),
             }
         })
     }
@@ -227,34 +222,49 @@ impl<'tcx> FunctionCoverage<'tcx> {
     pub(crate) fn counter_regions(
         &self,
     ) -> impl Iterator<Item = (Counter, &CodeRegion)> + ExactSizeIterator {
-        // Historically, mappings were stored directly in counter/expression
-        // statements in MIR, and MIR optimizations would sometimes remove them.
-        // That's mostly no longer true, so now we detect cases where that would
-        // have happened, and zero out the corresponding mappings here instead.
-        let counter_for_term = move |term: CovTerm| {
-            let force_to_zero = match term {
-                CovTerm::Counter(id) => !self.counters_seen.contains(id),
-                CovTerm::Expression(id) => self.zero_expressions.contains(id),
-                CovTerm::Zero => false,
-            };
-            if force_to_zero { Counter::ZERO } else { Counter::from_term(term) }
-        };
-
         self.function_coverage_info.mappings.iter().map(move |mapping| {
             let &Mapping { term, ref code_region } = mapping;
-            let counter = counter_for_term(term);
+            let counter = self.counter_for_term(term);
             (counter, code_region)
         })
     }
+
+    fn counter_for_term(&self, term: CovTerm) -> Counter {
+        if is_zero_term(&self.counters_seen, &self.zero_expressions, term) {
+            Counter::ZERO
+        } else {
+            Counter::from_term(term)
+        }
+    }
 }
 
 /// Set of expression IDs that are known to always evaluate to zero.
 /// Any mapping or expression operand that refers to these expressions can have
 /// that reference replaced with a constant zero value.
+#[derive(Default)]
 struct ZeroExpressions(FxIndexSet<ExpressionId>);
 
 impl ZeroExpressions {
+    fn insert(&mut self, id: ExpressionId) {
+        self.0.insert(id);
+    }
+
     fn contains(&self, id: ExpressionId) -> bool {
         self.0.contains(&id)
     }
 }
+
+/// Returns `true` if the given term is known to have a value of zero, taking
+/// into account knowledge of which counters are unused and which expressions
+/// are always zero.
+fn is_zero_term(
+    counters_seen: &BitSet<CounterId>,
+    zero_expressions: &ZeroExpressions,
+    term: CovTerm,
+) -> bool {
+    match term {
+        CovTerm::Zero => true,
+        CovTerm::Counter(id) => !counters_seen.contains(id),
+        CovTerm::Expression(id) => zero_expressions.contains(id),
+    }
+}
diff --git a/tests/coverage-map/fn_sig_into_try.cov-map b/tests/coverage-map/fn_sig_into_try.cov-map
index 4672e7c1ce90..6e26c61aac9f 100644
--- a/tests/coverage-map/fn_sig_into_try.cov-map
+++ b/tests/coverage-map/fn_sig_into_try.cov-map
@@ -7,47 +7,47 @@ Number of file 0 mappings: 1
 - Code(Counter(0)) at (prev + 10, 1) to (start + 4, 2)
 
 Function name: fn_sig_into_try::b
-Raw bytes (28): 0x[01, 01, 02, 01, 05, 05, 02, 04, 01, 10, 01, 02, 0f, 00, 02, 0f, 00, 10, 02, 01, 05, 00, 0c, 07, 01, 01, 00, 02]
+Raw bytes (28): 0x[01, 01, 02, 01, 00, 00, 02, 04, 01, 10, 01, 02, 0f, 00, 02, 0f, 00, 10, 02, 01, 05, 00, 0c, 07, 01, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 2
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Expression(0, Sub)
+- expression 0 operands: lhs = Counter(0), rhs = Zero
+- expression 1 operands: lhs = Zero, rhs = Expression(0, Sub)
 Number of file 0 mappings: 4
 - Code(Counter(0)) at (prev + 16, 1) to (start + 2, 15)
 - Code(Zero) at (prev + 2, 15) to (start + 0, 16)
 - Code(Expression(0, Sub)) at (prev + 1, 5) to (start + 0, 12)
-    = (c0 - c1)
+    = (c0 - Zero)
 - Code(Expression(1, Add)) at (prev + 1, 1) to (start + 0, 2)
-    = (c1 + (c0 - c1))
+    = (Zero + (c0 - Zero))
 
 Function name: fn_sig_into_try::c
-Raw bytes (28): 0x[01, 01, 02, 01, 05, 05, 02, 04, 01, 16, 01, 02, 17, 00, 02, 17, 00, 18, 02, 01, 05, 00, 0c, 07, 01, 01, 00, 02]
+Raw bytes (28): 0x[01, 01, 02, 01, 00, 00, 02, 04, 01, 16, 01, 02, 17, 00, 02, 17, 00, 18, 02, 01, 05, 00, 0c, 07, 01, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 2
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Expression(0, Sub)
+- expression 0 operands: lhs = Counter(0), rhs = Zero
+- expression 1 operands: lhs = Zero, rhs = Expression(0, Sub)
 Number of file 0 mappings: 4
 - Code(Counter(0)) at (prev + 22, 1) to (start + 2, 23)
 - Code(Zero) at (prev + 2, 23) to (start + 0, 24)
 - Code(Expression(0, Sub)) at (prev + 1, 5) to (start + 0, 12)
-    = (c0 - c1)
+    = (c0 - Zero)
 - Code(Expression(1, Add)) at (prev + 1, 1) to (start + 0, 2)
-    = (c1 + (c0 - c1))
+    = (Zero + (c0 - Zero))
 
 Function name: fn_sig_into_try::d
-Raw bytes (28): 0x[01, 01, 02, 01, 05, 05, 02, 04, 01, 1c, 01, 03, 0f, 00, 03, 0f, 00, 10, 02, 01, 05, 00, 0c, 07, 01, 01, 00, 02]
+Raw bytes (28): 0x[01, 01, 02, 01, 00, 00, 02, 04, 01, 1c, 01, 03, 0f, 00, 03, 0f, 00, 10, 02, 01, 05, 00, 0c, 07, 01, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 2
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Expression(0, Sub)
+- expression 0 operands: lhs = Counter(0), rhs = Zero
+- expression 1 operands: lhs = Zero, rhs = Expression(0, Sub)
 Number of file 0 mappings: 4
 - Code(Counter(0)) at (prev + 28, 1) to (start + 3, 15)
 - Code(Zero) at (prev + 3, 15) to (start + 0, 16)
 - Code(Expression(0, Sub)) at (prev + 1, 5) to (start + 0, 12)
-    = (c0 - c1)
+    = (c0 - Zero)
 - Code(Expression(1, Add)) at (prev + 1, 1) to (start + 0, 2)
-    = (c1 + (c0 - c1))
+    = (Zero + (c0 - Zero))
 
diff --git a/tests/coverage-map/status-quo/bad_counter_ids.cov-map b/tests/coverage-map/status-quo/bad_counter_ids.cov-map
index 2142bf35d5e7..0b8081acfa6d 100644
--- a/tests/coverage-map/status-quo/bad_counter_ids.cov-map
+++ b/tests/coverage-map/status-quo/bad_counter_ids.cov-map
@@ -24,15 +24,15 @@ Number of file 0 mappings: 2
 - Code(Zero) at (prev + 3, 1) to (start + 0, 2)
 
 Function name: bad_counter_ids::eq_bad_message
-Raw bytes (21): 0x[01, 01, 01, 01, 05, 03, 01, 28, 01, 02, 0f, 02, 02, 20, 00, 2b, 00, 01, 01, 00, 02]
+Raw bytes (21): 0x[01, 01, 01, 01, 00, 03, 01, 28, 01, 02, 0f, 02, 02, 20, 00, 2b, 00, 01, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 1
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
+- expression 0 operands: lhs = Counter(0), rhs = Zero
 Number of file 0 mappings: 3
 - Code(Counter(0)) at (prev + 40, 1) to (start + 2, 15)
 - Code(Expression(0, Sub)) at (prev + 2, 32) to (start + 0, 43)
-    = (c0 - c1)
+    = (c0 - Zero)
 - Code(Zero) at (prev + 1, 1) to (start + 0, 2)
 
 Function name: bad_counter_ids::eq_good
@@ -74,25 +74,25 @@ Number of file 0 mappings: 3
 - Code(Zero) at (prev + 1, 1) to (start + 0, 2)
 
 Function name: bad_counter_ids::ne_good
-Raw bytes (16): 0x[01, 01, 01, 01, 05, 02, 01, 19, 01, 02, 1f, 02, 03, 01, 00, 02]
+Raw bytes (16): 0x[01, 01, 01, 01, 00, 02, 01, 19, 01, 02, 1f, 02, 03, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 1
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
+- expression 0 operands: lhs = Counter(0), rhs = Zero
 Number of file 0 mappings: 2
 - Code(Counter(0)) at (prev + 25, 1) to (start + 2, 31)
 - Code(Expression(0, Sub)) at (prev + 3, 1) to (start + 0, 2)
-    = (c0 - c1)
+    = (c0 - Zero)
 
 Function name: bad_counter_ids::ne_good_message
-Raw bytes (21): 0x[01, 01, 01, 01, 05, 03, 01, 1e, 01, 02, 0f, 00, 02, 20, 00, 2b, 02, 01, 01, 00, 02]
+Raw bytes (21): 0x[01, 01, 01, 01, 00, 03, 01, 1e, 01, 02, 0f, 00, 02, 20, 00, 2b, 02, 01, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 1
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
+- expression 0 operands: lhs = Counter(0), rhs = Zero
 Number of file 0 mappings: 3
 - Code(Counter(0)) at (prev + 30, 1) to (start + 2, 15)
 - Code(Zero) at (prev + 2, 32) to (start + 0, 43)
 - Code(Expression(0, Sub)) at (prev + 1, 1) to (start + 0, 2)
-    = (c0 - c1)
+    = (c0 - Zero)
 
diff --git a/tests/coverage-map/status-quo/inline-dead.cov-map b/tests/coverage-map/status-quo/inline-dead.cov-map
index 06b64da5723b..958b423f24cf 100644
--- a/tests/coverage-map/status-quo/inline-dead.cov-map
+++ b/tests/coverage-map/status-quo/inline-dead.cov-map
@@ -7,19 +7,19 @@ Number of file 0 mappings: 1
 - Code(Zero) at (prev + 25, 1) to (start + 2, 2)
 
 Function name: inline_dead::live::<false>
-Raw bytes (28): 0x[01, 01, 02, 01, 05, 05, 02, 04, 01, 10, 01, 01, 09, 00, 02, 09, 00, 0f, 02, 02, 09, 00, 0a, 07, 02, 01, 00, 02]
+Raw bytes (28): 0x[01, 01, 02, 01, 00, 00, 02, 04, 01, 10, 01, 01, 09, 00, 02, 09, 00, 0f, 02, 02, 09, 00, 0a, 07, 02, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 2
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Expression(0, Sub)
+- expression 0 operands: lhs = Counter(0), rhs = Zero
+- expression 1 operands: lhs = Zero, rhs = Expression(0, Sub)
 Number of file 0 mappings: 4
 - Code(Counter(0)) at (prev + 16, 1) to (start + 1, 9)
 - Code(Zero) at (prev + 2, 9) to (start + 0, 15)
 - Code(Expression(0, Sub)) at (prev + 2, 9) to (start + 0, 10)
-    = (c0 - c1)
+    = (c0 - Zero)
 - Code(Expression(1, Add)) at (prev + 2, 1) to (start + 0, 2)
-    = (c1 + (c0 - c1))
+    = (Zero + (c0 - Zero))
 
 Function name: inline_dead::main
 Raw bytes (14): 0x[01, 01, 00, 02, 01, 04, 01, 03, 0d, 01, 07, 06, 02, 02]
@@ -31,15 +31,15 @@ Number of file 0 mappings: 2
 - Code(Counter(0)) at (prev + 7, 6) to (start + 2, 2)
 
 Function name: inline_dead::main::{closure#0}
-Raw bytes (23): 0x[01, 01, 02, 09, 06, 01, 05, 03, 01, 07, 17, 00, 18, 00, 02, 0d, 00, 0e, 03, 02, 05, 00, 06]
+Raw bytes (23): 0x[01, 01, 02, 00, 06, 01, 00, 03, 01, 07, 17, 00, 18, 00, 02, 0d, 00, 0e, 03, 02, 05, 00, 06]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 2
-- expression 0 operands: lhs = Counter(2), rhs = Expression(1, Sub)
-- expression 1 operands: lhs = Counter(0), rhs = Counter(1)
+- expression 0 operands: lhs = Zero, rhs = Expression(1, Sub)
+- expression 1 operands: lhs = Counter(0), rhs = Zero
 Number of file 0 mappings: 3
 - Code(Counter(0)) at (prev + 7, 23) to (start + 0, 24)
 - Code(Zero) at (prev + 2, 13) to (start + 0, 14)
 - Code(Expression(0, Add)) at (prev + 2, 5) to (start + 0, 6)
-    = (c2 + (c0 - c1))
+    = (Zero + (c0 - Zero))
 
diff --git a/tests/coverage-map/status-quo/issue-84561.cov-map b/tests/coverage-map/status-quo/issue-84561.cov-map
index 76340b1a78c7..82582b309bfc 100644
--- a/tests/coverage-map/status-quo/issue-84561.cov-map
+++ b/tests/coverage-map/status-quo/issue-84561.cov-map
@@ -85,47 +85,47 @@ Number of file 0 mappings: 1
 - Code(Counter(0)) at (prev + 165, 9) to (start + 2, 10)
 
 Function name: issue_84561::test3
-Raw bytes (437): 0x[01, 01, 41, 05, 09, 0d, 11, 15, 19, 12, 1d, 15, 19, 21, 25, 1e, 29, 21, 25, 31, 39, 3d, 41, 2e, 45, 3d, 41, 42, 49, 45, 4d, 3f, 51, 42, 49, 45, 4d, 5d, 8a, 01, 8f, 01, 5d, 92, 01, 55, 51, 59, 92, 01, 55, 51, 59, 8f, 01, 5d, 92, 01, 55, 51, 59, 87, 01, 61, 5d, 8a, 01, 8f, 01, 5d, 92, 01, 55, 51, 59, 82, 01, 65, 87, 01, 61, 5d, 8a, 01, 8f, 01, 5d, 92, 01, 55, 51, 59, 75, f6, 01, fb, 01, 79, 71, fe, 01, 82, 02, 71, 69, 6d, 71, fe, 01, 82, 02, 71, 69, 6d, 69, 6d, 82, 02, 71, 69, 6d, fb, 01, 79, 71, fe, 01, 82, 02, 71, 69, 6d, f3, 01, 7d, 75, f6, 01, fb, 01, 79, 71, fe, 01, 82, 02, 71, 69, 6d, ee, 01, 81, 01, f3, 01, 7d, 75, f6, 01, fb, 01, 79, 71, fe, 01, 82, 02, 71, 69, 6d, 33, 01, 06, 01, 03, 1c, 05, 04, 09, 01, 1c, 02, 02, 05, 04, 1f, 0d, 05, 05, 00, 1f, 06, 01, 05, 00, 1f, 15, 01, 09, 01, 1c, 12, 02, 05, 00, 1f, 0e, 01, 05, 00, 0f, 00, 00, 20, 00, 30, 21, 01, 05, 03, 0f, 00, 03, 20, 00, 30, 00, 00, 33, 00, 41, 00, 00, 4b, 00, 5a, 1e, 01, 05, 00, 0f, 00, 05, 09, 03, 10, 00, 05, 0d, 00, 1b, 00, 02, 0d, 00, 1c, 1a, 04, 09, 05, 06, 31, 06, 05, 03, 06, 22, 04, 05, 03, 06, 3d, 04, 09, 04, 06, 2e, 05, 08, 00, 0f, 45, 01, 09, 03, 0a, 2a, 05, 09, 03, 0a, 3f, 05, 08, 00, 0f, 51, 01, 09, 00, 13, 00, 03, 0d, 00, 1d, 3a, 03, 09, 00, 13, 00, 03, 0d, 00, 1d, 87, 01, 03, 05, 00, 0f, 8f, 01, 01, 0c, 00, 13, 5d, 01, 0d, 00, 13, 8a, 01, 02, 0d, 00, 13, 82, 01, 04, 05, 02, 13, 65, 03, 0d, 00, 13, 7e, 02, 0d, 00, 13, f3, 01, 03, 05, 00, 0f, 69, 01, 0c, 00, 13, 6d, 01, 0d, 03, 0e, 75, 04, 0d, 00, 13, fb, 01, 02, 0d, 00, 17, 82, 02, 01, 14, 00, 1b, 71, 01, 15, 00, 1b, fe, 01, 02, 15, 00, 1b, f6, 01, 04, 0d, 00, 13, 7d, 03, 09, 00, 19, ee, 01, 02, 05, 00, 0f, ea, 01, 03, 09, 00, 22, 00, 02, 05, 00, 0f, 00, 03, 09, 00, 2c, 00, 02, 01, 00, 02]
+Raw bytes (436): 0x[01, 01, 41, 05, 09, 0d, 00, 15, 19, 12, 00, 15, 19, 21, 00, 1e, 00, 21, 00, 31, 00, 3d, 41, 2e, 45, 3d, 41, 42, 49, 45, 00, 3f, 51, 42, 49, 45, 00, 5d, 8a, 01, 8f, 01, 5d, 92, 01, 55, 51, 00, 92, 01, 55, 51, 00, 8f, 01, 5d, 92, 01, 55, 51, 00, 87, 01, 61, 5d, 8a, 01, 8f, 01, 5d, 92, 01, 55, 51, 00, 82, 01, 65, 87, 01, 61, 5d, 8a, 01, 8f, 01, 5d, 92, 01, 55, 51, 00, 75, f6, 01, fb, 01, 79, 71, fe, 01, 82, 02, 71, 69, 6d, 71, fe, 01, 82, 02, 71, 69, 6d, 69, 6d, 82, 02, 71, 69, 6d, fb, 01, 79, 71, fe, 01, 82, 02, 71, 69, 6d, f3, 01, 7d, 75, f6, 01, fb, 01, 79, 71, fe, 01, 82, 02, 71, 69, 6d, ee, 01, 00, f3, 01, 7d, 75, f6, 01, fb, 01, 79, 71, fe, 01, 82, 02, 71, 69, 6d, 33, 01, 06, 01, 03, 1c, 05, 04, 09, 01, 1c, 02, 02, 05, 04, 1f, 0d, 05, 05, 00, 1f, 06, 01, 05, 00, 1f, 15, 01, 09, 01, 1c, 12, 02, 05, 00, 1f, 0e, 01, 05, 00, 0f, 00, 00, 20, 00, 30, 21, 01, 05, 03, 0f, 00, 03, 20, 00, 30, 00, 00, 33, 00, 41, 00, 00, 4b, 00, 5a, 1e, 01, 05, 00, 0f, 00, 05, 09, 03, 10, 00, 05, 0d, 00, 1b, 00, 02, 0d, 00, 1c, 1a, 04, 09, 05, 06, 31, 06, 05, 03, 06, 22, 04, 05, 03, 06, 3d, 04, 09, 04, 06, 2e, 05, 08, 00, 0f, 45, 01, 09, 03, 0a, 2a, 05, 09, 03, 0a, 3f, 05, 08, 00, 0f, 51, 01, 09, 00, 13, 00, 03, 0d, 00, 1d, 3a, 03, 09, 00, 13, 00, 03, 0d, 00, 1d, 87, 01, 03, 05, 00, 0f, 8f, 01, 01, 0c, 00, 13, 5d, 01, 0d, 00, 13, 8a, 01, 02, 0d, 00, 13, 82, 01, 04, 05, 02, 13, 65, 03, 0d, 00, 13, 7e, 02, 0d, 00, 13, f3, 01, 03, 05, 00, 0f, 69, 01, 0c, 00, 13, 6d, 01, 0d, 03, 0e, 75, 04, 0d, 00, 13, fb, 01, 02, 0d, 00, 17, 82, 02, 01, 14, 00, 1b, 71, 01, 15, 00, 1b, fe, 01, 02, 15, 00, 1b, f6, 01, 04, 0d, 00, 13, 7d, 03, 09, 00, 19, ee, 01, 02, 05, 00, 0f, ea, 01, 03, 09, 00, 22, 00, 02, 05, 00, 0f, 00, 03, 09, 00, 2c, 00, 02, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 65
 - expression 0 operands: lhs = Counter(1), rhs = Counter(2)
-- expression 1 operands: lhs = Counter(3), rhs = Counter(4)
+- expression 1 operands: lhs = Counter(3), rhs = Zero
 - expression 2 operands: lhs = Counter(5), rhs = Counter(6)
-- expression 3 operands: lhs = Expression(4, Sub), rhs = Counter(7)
+- expression 3 operands: lhs = Expression(4, Sub), rhs = Zero
 - expression 4 operands: lhs = Counter(5), rhs = Counter(6)
-- expression 5 operands: lhs = Counter(8), rhs = Counter(9)
-- expression 6 operands: lhs = Expression(7, Sub), rhs = Counter(10)
-- expression 7 operands: lhs = Counter(8), rhs = Counter(9)
-- expression 8 operands: lhs = Counter(12), rhs = Counter(14)
+- expression 5 operands: lhs = Counter(8), rhs = Zero
+- expression 6 operands: lhs = Expression(7, Sub), rhs = Zero
+- expression 7 operands: lhs = Counter(8), rhs = Zero
+- expression 8 operands: lhs = Counter(12), rhs = Zero
 - expression 9 operands: lhs = Counter(15), rhs = Counter(16)
 - expression 10 operands: lhs = Expression(11, Sub), rhs = Counter(17)
 - expression 11 operands: lhs = Counter(15), rhs = Counter(16)
 - expression 12 operands: lhs = Expression(16, Sub), rhs = Counter(18)
-- expression 13 operands: lhs = Counter(17), rhs = Counter(19)
+- expression 13 operands: lhs = Counter(17), rhs = Zero
 - expression 14 operands: lhs = Expression(15, Add), rhs = Counter(20)
 - expression 15 operands: lhs = Expression(16, Sub), rhs = Counter(18)
-- expression 16 operands: lhs = Counter(17), rhs = Counter(19)
+- expression 16 operands: lhs = Counter(17), rhs = Zero
 - expression 17 operands: lhs = Counter(23), rhs = Expression(34, Sub)
 - expression 18 operands: lhs = Expression(35, Add), rhs = Counter(23)
 - expression 19 operands: lhs = Expression(36, Sub), rhs = Counter(21)
-- expression 20 operands: lhs = Counter(20), rhs = Counter(22)
+- expression 20 operands: lhs = Counter(20), rhs = Zero
 - expression 21 operands: lhs = Expression(36, Sub), rhs = Counter(21)
-- expression 22 operands: lhs = Counter(20), rhs = Counter(22)
+- expression 22 operands: lhs = Counter(20), rhs = Zero
 - expression 23 operands: lhs = Expression(35, Add), rhs = Counter(23)
 - expression 24 operands: lhs = Expression(36, Sub), rhs = Counter(21)
-- expression 25 operands: lhs = Counter(20), rhs = Counter(22)
+- expression 25 operands: lhs = Counter(20), rhs = Zero
 - expression 26 operands: lhs = Expression(33, Add), rhs = Counter(24)
 - expression 27 operands: lhs = Counter(23), rhs = Expression(34, Sub)
 - expression 28 operands: lhs = Expression(35, Add), rhs = Counter(23)
 - expression 29 operands: lhs = Expression(36, Sub), rhs = Counter(21)
-- expression 30 operands: lhs = Counter(20), rhs = Counter(22)
+- expression 30 operands: lhs = Counter(20), rhs = Zero
 - expression 31 operands: lhs = Expression(32, Sub), rhs = Counter(25)
 - expression 32 operands: lhs = Expression(33, Add), rhs = Counter(24)
 - expression 33 operands: lhs = Counter(23), rhs = Expression(34, Sub)
 - expression 34 operands: lhs = Expression(35, Add), rhs = Counter(23)
 - expression 35 operands: lhs = Expression(36, Sub), rhs = Counter(21)
-- expression 36 operands: lhs = Counter(20), rhs = Counter(22)
+- expression 36 operands: lhs = Counter(20), rhs = Zero
 - expression 37 operands: lhs = Counter(29), rhs = Expression(61, Sub)
 - expression 38 operands: lhs = Expression(62, Add), rhs = Counter(30)
 - expression 39 operands: lhs = Counter(28), rhs = Expression(63, Sub)
@@ -147,7 +147,7 @@ Number of expressions: 65
 - expression 55 operands: lhs = Counter(28), rhs = Expression(63, Sub)
 - expression 56 operands: lhs = Expression(64, Sub), rhs = Counter(28)
 - expression 57 operands: lhs = Counter(26), rhs = Counter(27)
-- expression 58 operands: lhs = Expression(59, Sub), rhs = Counter(32)
+- expression 58 operands: lhs = Expression(59, Sub), rhs = Zero
 - expression 59 operands: lhs = Expression(60, Add), rhs = Counter(31)
 - expression 60 operands: lhs = Counter(29), rhs = Expression(61, Sub)
 - expression 61 operands: lhs = Expression(62, Add), rhs = Counter(30)
@@ -161,27 +161,27 @@ Number of file 0 mappings: 51
     = (c1 - c2)
 - Code(Counter(3)) at (prev + 5, 5) to (start + 0, 31)
 - Code(Expression(1, Sub)) at (prev + 1, 5) to (start + 0, 31)
-    = (c3 - c4)
+    = (c3 - Zero)
 - Code(Counter(5)) at (prev + 1, 9) to (start + 1, 28)
 - Code(Expression(4, Sub)) at (prev + 2, 5) to (start + 0, 31)
     = (c5 - c6)
 - Code(Expression(3, Sub)) at (prev + 1, 5) to (start + 0, 15)
-    = ((c5 - c6) - c7)
+    = ((c5 - c6) - Zero)
 - Code(Zero) at (prev + 0, 32) to (start + 0, 48)
 - Code(Counter(8)) at (prev + 1, 5) to (start + 3, 15)
 - Code(Zero) at (prev + 3, 32) to (start + 0, 48)
 - Code(Zero) at (prev + 0, 51) to (start + 0, 65)
 - Code(Zero) at (prev + 0, 75) to (start + 0, 90)
 - Code(Expression(7, Sub)) at (prev + 1, 5) to (start + 0, 15)
-    = (c8 - c9)
+    = (c8 - Zero)
 - Code(Zero) at (prev + 5, 9) to (start + 3, 16)
 - Code(Zero) at (prev + 5, 13) to (start + 0, 27)
 - Code(Zero) at (prev + 2, 13) to (start + 0, 28)
 - Code(Expression(6, Sub)) at (prev + 4, 9) to (start + 5, 6)
-    = ((c8 - c9) - c10)
+    = ((c8 - Zero) - Zero)
 - Code(Counter(12)) at (prev + 6, 5) to (start + 3, 6)
 - Code(Expression(8, Sub)) at (prev + 4, 5) to (start + 3, 6)
-    = (c12 - c14)
+    = (c12 - Zero)
 - Code(Counter(15)) at (prev + 4, 9) to (start + 4, 6)
 - Code(Expression(11, Sub)) at (prev + 5, 8) to (start + 0, 15)
     = (c15 - c16)
@@ -189,24 +189,24 @@ Number of file 0 mappings: 51
 - Code(Expression(10, Sub)) at (prev + 5, 9) to (start + 3, 10)
     = ((c15 - c16) - c17)
 - Code(Expression(15, Add)) at (prev + 5, 8) to (start + 0, 15)
-    = ((c17 - c19) + c18)
+    = ((c17 - Zero) + c18)
 - Code(Counter(20)) at (prev + 1, 9) to (start + 0, 19)
 - Code(Zero) at (prev + 3, 13) to (start + 0, 29)
 - Code(Expression(14, Sub)) at (prev + 3, 9) to (start + 0, 19)
-    = (((c17 - c19) + c18) - c20)
+    = (((c17 - Zero) + c18) - c20)
 - Code(Zero) at (prev + 3, 13) to (start + 0, 29)
 - Code(Expression(33, Add)) at (prev + 3, 5) to (start + 0, 15)
-    = (c23 + (((c20 - c22) + c21) - c23))
+    = (c23 + (((c20 - Zero) + c21) - c23))
 - Code(Expression(35, Add)) at (prev + 1, 12) to (start + 0, 19)
-    = ((c20 - c22) + c21)
+    = ((c20 - Zero) + c21)
 - Code(Counter(23)) at (prev + 1, 13) to (start + 0, 19)
 - Code(Expression(34, Sub)) at (prev + 2, 13) to (start + 0, 19)
-    = (((c20 - c22) + c21) - c23)
+    = (((c20 - Zero) + c21) - c23)
 - Code(Expression(32, Sub)) at (prev + 4, 5) to (start + 2, 19)
-    = ((c23 + (((c20 - c22) + c21) - c23)) - c24)
+    = ((c23 + (((c20 - Zero) + c21) - c23)) - c24)
 - Code(Counter(25)) at (prev + 3, 13) to (start + 0, 19)
 - Code(Expression(31, Sub)) at (prev + 2, 13) to (start + 0, 19)
-    = (((c23 + (((c20 - c22) + c21) - c23)) - c24) - c25)
+    = (((c23 + (((c20 - Zero) + c21) - c23)) - c24) - c25)
 - Code(Expression(60, Add)) at (prev + 3, 5) to (start + 0, 15)
     = (c29 + ((c28 + ((c26 - c27) - c28)) - c30))
 - Code(Counter(26)) at (prev + 1, 12) to (start + 0, 19)
@@ -225,7 +225,7 @@ Number of file 0 mappings: 51
 - Code(Expression(59, Sub)) at (prev + 2, 5) to (start + 0, 15)
     = ((c29 + ((c28 + ((c26 - c27) - c28)) - c30)) - c31)
 - Code(Expression(58, Sub)) at (prev + 3, 9) to (start + 0, 34)
-    = (((c29 + ((c28 + ((c26 - c27) - c28)) - c30)) - c31) - c32)
+    = (((c29 + ((c28 + ((c26 - c27) - c28)) - c30)) - c31) - Zero)
 - Code(Zero) at (prev + 2, 5) to (start + 0, 15)
 - Code(Zero) at (prev + 3, 9) to (start + 0, 44)
 - Code(Zero) at (prev + 2, 1) to (start + 0, 2)
diff --git a/tests/coverage-map/status-quo/loops_branches.cov-map b/tests/coverage-map/status-quo/loops_branches.cov-map
index ebace8cbd713..813583a9de75 100644
--- a/tests/coverage-map/status-quo/loops_branches.cov-map
+++ b/tests/coverage-map/status-quo/loops_branches.cov-map
@@ -1,177 +1,177 @@
 Function name: <loops_branches::DebugTest as core::fmt::Debug>::fmt
-Raw bytes (249): 0x[01, 01, 31, 05, 09, 00, 02, bb, 01, 19, bf, 01, c3, 01, 0d, 00, 11, 15, bf, 01, c3, 01, 0d, 00, 11, 15, bb, 01, 19, bf, 01, c3, 01, 0d, 00, 11, 15, b6, 01, 15, bb, 01, 19, bf, 01, c3, 01, 0d, 00, 11, 15, b2, 01, 1d, b6, 01, 15, bb, 01, 19, bf, 01, c3, 01, 0d, 00, 11, 15, 00, ae, 01, b2, 01, 1d, b6, 01, 15, bb, 01, 19, bf, 01, c3, 01, 0d, 00, 11, 15, ab, 01, 11, 00, ae, 01, b2, 01, 1d, b6, 01, 15, bb, 01, 19, bf, 01, c3, 01, 0d, 00, 11, 15, 25, a3, 01, a6, 01, 19, ab, 01, 11, 00, ae, 01, b2, 01, 1d, b6, 01, 15, bb, 01, 19, bf, 01, c3, 01, 0d, 00, 11, 15, 14, 01, 09, 05, 01, 10, 05, 02, 10, 00, 15, 00, 01, 17, 00, 1b, 00, 00, 1c, 00, 1e, 02, 01, 0e, 00, 0f, 07, 01, 0d, 00, 1e, 25, 00, 1e, 00, 1f, 00, 01, 10, 01, 0a, b6, 01, 03, 0d, 00, 0e, bb, 01, 00, 12, 00, 17, b6, 01, 01, 10, 00, 14, b2, 01, 01, 14, 00, 19, 00, 01, 1b, 00, 1f, 00, 00, 20, 00, 22, ae, 01, 01, 12, 00, 13, ab, 01, 01, 11, 00, 22, a6, 01, 00, 22, 00, 23, 00, 01, 14, 01, 0e, 19, 03, 09, 00, 0f, 9f, 01, 01, 05, 00, 06]
+Raw bytes (249): 0x[01, 01, 31, 05, 00, 00, 02, bb, 01, 19, bf, 01, c3, 01, 0d, 00, 11, 00, bf, 01, c3, 01, 0d, 00, 11, 00, bb, 01, 19, bf, 01, c3, 01, 0d, 00, 11, 00, b6, 01, 00, bb, 01, 19, bf, 01, c3, 01, 0d, 00, 11, 00, b2, 01, 00, b6, 01, 00, bb, 01, 19, bf, 01, c3, 01, 0d, 00, 11, 00, 00, ae, 01, b2, 01, 00, b6, 01, 00, bb, 01, 19, bf, 01, c3, 01, 0d, 00, 11, 00, ab, 01, 11, 00, ae, 01, b2, 01, 00, b6, 01, 00, bb, 01, 19, bf, 01, c3, 01, 0d, 00, 11, 00, 25, a3, 01, a6, 01, 19, ab, 01, 11, 00, ae, 01, b2, 01, 00, b6, 01, 00, bb, 01, 19, bf, 01, c3, 01, 0d, 00, 11, 00, 14, 01, 09, 05, 01, 10, 05, 02, 10, 00, 15, 00, 01, 17, 00, 1b, 00, 00, 1c, 00, 1e, 02, 01, 0e, 00, 0f, 07, 01, 0d, 00, 1e, 25, 00, 1e, 00, 1f, 00, 01, 10, 01, 0a, b6, 01, 03, 0d, 00, 0e, bb, 01, 00, 12, 00, 17, b6, 01, 01, 10, 00, 14, b2, 01, 01, 14, 00, 19, 00, 01, 1b, 00, 1f, 00, 00, 20, 00, 22, ae, 01, 01, 12, 00, 13, ab, 01, 01, 11, 00, 22, a6, 01, 00, 22, 00, 23, 00, 01, 14, 01, 0e, 19, 03, 09, 00, 0f, 9f, 01, 01, 05, 00, 06]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 49
-- expression 0 operands: lhs = Counter(1), rhs = Counter(2)
+- expression 0 operands: lhs = Counter(1), rhs = Zero
 - expression 1 operands: lhs = Zero, rhs = Expression(0, Sub)
 - expression 2 operands: lhs = Expression(46, Add), rhs = Counter(6)
 - expression 3 operands: lhs = Expression(47, Add), rhs = Expression(48, Add)
 - expression 4 operands: lhs = Counter(3), rhs = Zero
-- expression 5 operands: lhs = Counter(4), rhs = Counter(5)
+- expression 5 operands: lhs = Counter(4), rhs = Zero
 - expression 6 operands: lhs = Expression(47, Add), rhs = Expression(48, Add)
 - expression 7 operands: lhs = Counter(3), rhs = Zero
-- expression 8 operands: lhs = Counter(4), rhs = Counter(5)
+- expression 8 operands: lhs = Counter(4), rhs = Zero
 - expression 9 operands: lhs = Expression(46, Add), rhs = Counter(6)
 - expression 10 operands: lhs = Expression(47, Add), rhs = Expression(48, Add)
 - expression 11 operands: lhs = Counter(3), rhs = Zero
-- expression 12 operands: lhs = Counter(4), rhs = Counter(5)
-- expression 13 operands: lhs = Expression(45, Sub), rhs = Counter(5)
+- expression 12 operands: lhs = Counter(4), rhs = Zero
+- expression 13 operands: lhs = Expression(45, Sub), rhs = Zero
 - expression 14 operands: lhs = Expression(46, Add), rhs = Counter(6)
 - expression 15 operands: lhs = Expression(47, Add), rhs = Expression(48, Add)
 - expression 16 operands: lhs = Counter(3), rhs = Zero
-- expression 17 operands: lhs = Counter(4), rhs = Counter(5)
-- expression 18 operands: lhs = Expression(44, Sub), rhs = Counter(7)
-- expression 19 operands: lhs = Expression(45, Sub), rhs = Counter(5)
+- expression 17 operands: lhs = Counter(4), rhs = Zero
+- expression 18 operands: lhs = Expression(44, Sub), rhs = Zero
+- expression 19 operands: lhs = Expression(45, Sub), rhs = Zero
 - expression 20 operands: lhs = Expression(46, Add), rhs = Counter(6)
 - expression 21 operands: lhs = Expression(47, Add), rhs = Expression(48, Add)
 - expression 22 operands: lhs = Counter(3), rhs = Zero
-- expression 23 operands: lhs = Counter(4), rhs = Counter(5)
+- expression 23 operands: lhs = Counter(4), rhs = Zero
 - expression 24 operands: lhs = Zero, rhs = Expression(43, Sub)
-- expression 25 operands: lhs = Expression(44, Sub), rhs = Counter(7)
-- expression 26 operands: lhs = Expression(45, Sub), rhs = Counter(5)
+- expression 25 operands: lhs = Expression(44, Sub), rhs = Zero
+- expression 26 operands: lhs = Expression(45, Sub), rhs = Zero
 - expression 27 operands: lhs = Expression(46, Add), rhs = Counter(6)
 - expression 28 operands: lhs = Expression(47, Add), rhs = Expression(48, Add)
 - expression 29 operands: lhs = Counter(3), rhs = Zero
-- expression 30 operands: lhs = Counter(4), rhs = Counter(5)
+- expression 30 operands: lhs = Counter(4), rhs = Zero
 - expression 31 operands: lhs = Expression(42, Add), rhs = Counter(4)
 - expression 32 operands: lhs = Zero, rhs = Expression(43, Sub)
-- expression 33 operands: lhs = Expression(44, Sub), rhs = Counter(7)
-- expression 34 operands: lhs = Expression(45, Sub), rhs = Counter(5)
+- expression 33 operands: lhs = Expression(44, Sub), rhs = Zero
+- expression 34 operands: lhs = Expression(45, Sub), rhs = Zero
 - expression 35 operands: lhs = Expression(46, Add), rhs = Counter(6)
 - expression 36 operands: lhs = Expression(47, Add), rhs = Expression(48, Add)
 - expression 37 operands: lhs = Counter(3), rhs = Zero
-- expression 38 operands: lhs = Counter(4), rhs = Counter(5)
+- expression 38 operands: lhs = Counter(4), rhs = Zero
 - expression 39 operands: lhs = Counter(9), rhs = Expression(40, Add)
 - expression 40 operands: lhs = Expression(41, Sub), rhs = Counter(6)
 - expression 41 operands: lhs = Expression(42, Add), rhs = Counter(4)
 - expression 42 operands: lhs = Zero, rhs = Expression(43, Sub)
-- expression 43 operands: lhs = Expression(44, Sub), rhs = Counter(7)
-- expression 44 operands: lhs = Expression(45, Sub), rhs = Counter(5)
+- expression 43 operands: lhs = Expression(44, Sub), rhs = Zero
+- expression 44 operands: lhs = Expression(45, Sub), rhs = Zero
 - expression 45 operands: lhs = Expression(46, Add), rhs = Counter(6)
 - expression 46 operands: lhs = Expression(47, Add), rhs = Expression(48, Add)
 - expression 47 operands: lhs = Counter(3), rhs = Zero
-- expression 48 operands: lhs = Counter(4), rhs = Counter(5)
+- expression 48 operands: lhs = Counter(4), rhs = Zero
 Number of file 0 mappings: 20
 - Code(Counter(0)) at (prev + 9, 5) to (start + 1, 16)
 - Code(Counter(1)) at (prev + 2, 16) to (start + 0, 21)
 - Code(Zero) at (prev + 1, 23) to (start + 0, 27)
 - Code(Zero) at (prev + 0, 28) to (start + 0, 30)
 - Code(Expression(0, Sub)) at (prev + 1, 14) to (start + 0, 15)
-    = (c1 - c2)
+    = (c1 - Zero)
 - Code(Expression(1, Add)) at (prev + 1, 13) to (start + 0, 30)
-    = (Zero + (c1 - c2))
+    = (Zero + (c1 - Zero))
 - Code(Counter(9)) at (prev + 0, 30) to (start + 0, 31)
 - Code(Zero) at (prev + 1, 16) to (start + 1, 10)
 - Code(Expression(45, Sub)) at (prev + 3, 13) to (start + 0, 14)
-    = (((c3 + Zero) + (c4 + c5)) - c6)
+    = (((c3 + Zero) + (c4 + Zero)) - c6)
 - Code(Expression(46, Add)) at (prev + 0, 18) to (start + 0, 23)
-    = ((c3 + Zero) + (c4 + c5))
+    = ((c3 + Zero) + (c4 + Zero))
 - Code(Expression(45, Sub)) at (prev + 1, 16) to (start + 0, 20)
-    = (((c3 + Zero) + (c4 + c5)) - c6)
+    = (((c3 + Zero) + (c4 + Zero)) - c6)
 - Code(Expression(44, Sub)) at (prev + 1, 20) to (start + 0, 25)
-    = ((((c3 + Zero) + (c4 + c5)) - c6) - c5)
+    = ((((c3 + Zero) + (c4 + Zero)) - c6) - Zero)
 - Code(Zero) at (prev + 1, 27) to (start + 0, 31)
 - Code(Zero) at (prev + 0, 32) to (start + 0, 34)
 - Code(Expression(43, Sub)) at (prev + 1, 18) to (start + 0, 19)
-    = (((((c3 + Zero) + (c4 + c5)) - c6) - c5) - c7)
+    = (((((c3 + Zero) + (c4 + Zero)) - c6) - Zero) - Zero)
 - Code(Expression(42, Add)) at (prev + 1, 17) to (start + 0, 34)
-    = (Zero + (((((c3 + Zero) + (c4 + c5)) - c6) - c5) - c7))
+    = (Zero + (((((c3 + Zero) + (c4 + Zero)) - c6) - Zero) - Zero))
 - Code(Expression(41, Sub)) at (prev + 0, 34) to (start + 0, 35)
-    = ((Zero + (((((c3 + Zero) + (c4 + c5)) - c6) - c5) - c7)) - c4)
+    = ((Zero + (((((c3 + Zero) + (c4 + Zero)) - c6) - Zero) - Zero)) - c4)
 - Code(Zero) at (prev + 1, 20) to (start + 1, 14)
 - Code(Counter(6)) at (prev + 3, 9) to (start + 0, 15)
 - Code(Expression(39, Add)) at (prev + 1, 5) to (start + 0, 6)
-    = (c9 + (((Zero + (((((c3 + Zero) + (c4 + c5)) - c6) - c5) - c7)) - c4) + c6))
+    = (c9 + (((Zero + (((((c3 + Zero) + (c4 + Zero)) - c6) - Zero) - Zero)) - c4) + c6))
 
 Function name: <loops_branches::DisplayTest as core::fmt::Display>::fmt
-Raw bytes (253): 0x[01, 01, 33, 01, 05, 02, 09, 00, 0e, 02, 09, bf, 01, 19, c3, 01, c7, 01, 05, 0d, 11, 15, c3, 01, c7, 01, 05, 0d, 11, 15, bf, 01, 19, c3, 01, c7, 01, 05, 0d, 11, 15, ba, 01, 11, bf, 01, 19, c3, 01, c7, 01, 05, 0d, 11, 15, b6, 01, 1d, ba, 01, 11, bf, 01, 19, c3, 01, c7, 01, 05, 0d, 11, 15, 00, b2, 01, b6, 01, 1d, ba, 01, 11, bf, 01, 19, c3, 01, c7, 01, 05, 0d, 11, 15, af, 01, 15, 00, b2, 01, b6, 01, 1d, ba, 01, 11, bf, 01, 19, c3, 01, c7, 01, 05, 0d, 11, 15, aa, 01, cb, 01, af, 01, 15, 00, b2, 01, b6, 01, 1d, ba, 01, 11, bf, 01, 19, c3, 01, c7, 01, 05, 0d, 11, 15, 19, 25, 14, 01, 22, 05, 01, 11, 00, 01, 12, 01, 0a, 02, 02, 10, 00, 15, 00, 01, 17, 00, 1b, 00, 00, 1c, 00, 1e, 0e, 01, 0e, 00, 0f, 0b, 01, 0d, 00, 1e, 25, 00, 1e, 00, 1f, ba, 01, 02, 0d, 00, 0e, bf, 01, 00, 12, 00, 17, ba, 01, 01, 10, 00, 15, 00, 00, 16, 01, 0e, b6, 01, 02, 14, 00, 19, 00, 01, 1b, 00, 1f, 00, 00, 20, 00, 22, b2, 01, 01, 12, 00, 13, af, 01, 01, 11, 00, 22, aa, 01, 00, 22, 00, 23, 19, 03, 09, 00, 0f, a7, 01, 01, 05, 00, 06]
+Raw bytes (253): 0x[01, 01, 33, 01, 00, 02, 00, 00, 0e, 02, 00, bf, 01, 19, c3, 01, c7, 01, 00, 0d, 00, 15, c3, 01, c7, 01, 00, 0d, 00, 15, bf, 01, 19, c3, 01, c7, 01, 00, 0d, 00, 15, ba, 01, 00, bf, 01, 19, c3, 01, c7, 01, 00, 0d, 00, 15, b6, 01, 00, ba, 01, 00, bf, 01, 19, c3, 01, c7, 01, 00, 0d, 00, 15, 00, b2, 01, b6, 01, 00, ba, 01, 00, bf, 01, 19, c3, 01, c7, 01, 00, 0d, 00, 15, af, 01, 15, 00, b2, 01, b6, 01, 00, ba, 01, 00, bf, 01, 19, c3, 01, c7, 01, 00, 0d, 00, 15, aa, 01, cb, 01, af, 01, 15, 00, b2, 01, b6, 01, 00, ba, 01, 00, bf, 01, 19, c3, 01, c7, 01, 00, 0d, 00, 15, 19, 25, 14, 01, 22, 05, 01, 11, 00, 01, 12, 01, 0a, 02, 02, 10, 00, 15, 00, 01, 17, 00, 1b, 00, 00, 1c, 00, 1e, 0e, 01, 0e, 00, 0f, 0b, 01, 0d, 00, 1e, 25, 00, 1e, 00, 1f, ba, 01, 02, 0d, 00, 0e, bf, 01, 00, 12, 00, 17, ba, 01, 01, 10, 00, 15, 00, 00, 16, 01, 0e, b6, 01, 02, 14, 00, 19, 00, 01, 1b, 00, 1f, 00, 00, 20, 00, 22, b2, 01, 01, 12, 00, 13, af, 01, 01, 11, 00, 22, aa, 01, 00, 22, 00, 23, 19, 03, 09, 00, 0f, a7, 01, 01, 05, 00, 06]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 51
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Expression(0, Sub), rhs = Counter(2)
+- expression 0 operands: lhs = Counter(0), rhs = Zero
+- expression 1 operands: lhs = Expression(0, Sub), rhs = Zero
 - expression 2 operands: lhs = Zero, rhs = Expression(3, Sub)
-- expression 3 operands: lhs = Expression(0, Sub), rhs = Counter(2)
+- expression 3 operands: lhs = Expression(0, Sub), rhs = Zero
 - expression 4 operands: lhs = Expression(47, Add), rhs = Counter(6)
 - expression 5 operands: lhs = Expression(48, Add), rhs = Expression(49, Add)
-- expression 6 operands: lhs = Counter(1), rhs = Counter(3)
-- expression 7 operands: lhs = Counter(4), rhs = Counter(5)
+- expression 6 operands: lhs = Zero, rhs = Counter(3)
+- expression 7 operands: lhs = Zero, rhs = Counter(5)
 - expression 8 operands: lhs = Expression(48, Add), rhs = Expression(49, Add)
-- expression 9 operands: lhs = Counter(1), rhs = Counter(3)
-- expression 10 operands: lhs = Counter(4), rhs = Counter(5)
+- expression 9 operands: lhs = Zero, rhs = Counter(3)
+- expression 10 operands: lhs = Zero, rhs = Counter(5)
 - expression 11 operands: lhs = Expression(47, Add), rhs = Counter(6)
 - expression 12 operands: lhs = Expression(48, Add), rhs = Expression(49, Add)
-- expression 13 operands: lhs = Counter(1), rhs = Counter(3)
-- expression 14 operands: lhs = Counter(4), rhs = Counter(5)
-- expression 15 operands: lhs = Expression(46, Sub), rhs = Counter(4)
+- expression 13 operands: lhs = Zero, rhs = Counter(3)
+- expression 14 operands: lhs = Zero, rhs = Counter(5)
+- expression 15 operands: lhs = Expression(46, Sub), rhs = Zero
 - expression 16 operands: lhs = Expression(47, Add), rhs = Counter(6)
 - expression 17 operands: lhs = Expression(48, Add), rhs = Expression(49, Add)
-- expression 18 operands: lhs = Counter(1), rhs = Counter(3)
-- expression 19 operands: lhs = Counter(4), rhs = Counter(5)
-- expression 20 operands: lhs = Expression(45, Sub), rhs = Counter(7)
-- expression 21 operands: lhs = Expression(46, Sub), rhs = Counter(4)
+- expression 18 operands: lhs = Zero, rhs = Counter(3)
+- expression 19 operands: lhs = Zero, rhs = Counter(5)
+- expression 20 operands: lhs = Expression(45, Sub), rhs = Zero
+- expression 21 operands: lhs = Expression(46, Sub), rhs = Zero
 - expression 22 operands: lhs = Expression(47, Add), rhs = Counter(6)
 - expression 23 operands: lhs = Expression(48, Add), rhs = Expression(49, Add)
-- expression 24 operands: lhs = Counter(1), rhs = Counter(3)
-- expression 25 operands: lhs = Counter(4), rhs = Counter(5)
+- expression 24 operands: lhs = Zero, rhs = Counter(3)
+- expression 25 operands: lhs = Zero, rhs = Counter(5)
 - expression 26 operands: lhs = Zero, rhs = Expression(44, Sub)
-- expression 27 operands: lhs = Expression(45, Sub), rhs = Counter(7)
-- expression 28 operands: lhs = Expression(46, Sub), rhs = Counter(4)
+- expression 27 operands: lhs = Expression(45, Sub), rhs = Zero
+- expression 28 operands: lhs = Expression(46, Sub), rhs = Zero
 - expression 29 operands: lhs = Expression(47, Add), rhs = Counter(6)
 - expression 30 operands: lhs = Expression(48, Add), rhs = Expression(49, Add)
-- expression 31 operands: lhs = Counter(1), rhs = Counter(3)
-- expression 32 operands: lhs = Counter(4), rhs = Counter(5)
+- expression 31 operands: lhs = Zero, rhs = Counter(3)
+- expression 32 operands: lhs = Zero, rhs = Counter(5)
 - expression 33 operands: lhs = Expression(43, Add), rhs = Counter(5)
 - expression 34 operands: lhs = Zero, rhs = Expression(44, Sub)
-- expression 35 operands: lhs = Expression(45, Sub), rhs = Counter(7)
-- expression 36 operands: lhs = Expression(46, Sub), rhs = Counter(4)
+- expression 35 operands: lhs = Expression(45, Sub), rhs = Zero
+- expression 36 operands: lhs = Expression(46, Sub), rhs = Zero
 - expression 37 operands: lhs = Expression(47, Add), rhs = Counter(6)
 - expression 38 operands: lhs = Expression(48, Add), rhs = Expression(49, Add)
-- expression 39 operands: lhs = Counter(1), rhs = Counter(3)
-- expression 40 operands: lhs = Counter(4), rhs = Counter(5)
+- expression 39 operands: lhs = Zero, rhs = Counter(3)
+- expression 40 operands: lhs = Zero, rhs = Counter(5)
 - expression 41 operands: lhs = Expression(42, Sub), rhs = Expression(50, Add)
 - expression 42 operands: lhs = Expression(43, Add), rhs = Counter(5)
 - expression 43 operands: lhs = Zero, rhs = Expression(44, Sub)
-- expression 44 operands: lhs = Expression(45, Sub), rhs = Counter(7)
-- expression 45 operands: lhs = Expression(46, Sub), rhs = Counter(4)
+- expression 44 operands: lhs = Expression(45, Sub), rhs = Zero
+- expression 45 operands: lhs = Expression(46, Sub), rhs = Zero
 - expression 46 operands: lhs = Expression(47, Add), rhs = Counter(6)
 - expression 47 operands: lhs = Expression(48, Add), rhs = Expression(49, Add)
-- expression 48 operands: lhs = Counter(1), rhs = Counter(3)
-- expression 49 operands: lhs = Counter(4), rhs = Counter(5)
+- expression 48 operands: lhs = Zero, rhs = Counter(3)
+- expression 49 operands: lhs = Zero, rhs = Counter(5)
 - expression 50 operands: lhs = Counter(6), rhs = Counter(9)
 Number of file 0 mappings: 20
 - Code(Counter(0)) at (prev + 34, 5) to (start + 1, 17)
 - Code(Zero) at (prev + 1, 18) to (start + 1, 10)
 - Code(Expression(0, Sub)) at (prev + 2, 16) to (start + 0, 21)
-    = (c0 - c1)
+    = (c0 - Zero)
 - Code(Zero) at (prev + 1, 23) to (start + 0, 27)
 - Code(Zero) at (prev + 0, 28) to (start + 0, 30)
 - Code(Expression(3, Sub)) at (prev + 1, 14) to (start + 0, 15)
-    = ((c0 - c1) - c2)
+    = ((c0 - Zero) - Zero)
 - Code(Expression(2, Add)) at (prev + 1, 13) to (start + 0, 30)
-    = (Zero + ((c0 - c1) - c2))
+    = (Zero + ((c0 - Zero) - Zero))
 - Code(Counter(9)) at (prev + 0, 30) to (start + 0, 31)
 - Code(Expression(46, Sub)) at (prev + 2, 13) to (start + 0, 14)
-    = (((c1 + c3) + (c4 + c5)) - c6)
+    = (((Zero + c3) + (Zero + c5)) - c6)
 - Code(Expression(47, Add)) at (prev + 0, 18) to (start + 0, 23)
-    = ((c1 + c3) + (c4 + c5))
+    = ((Zero + c3) + (Zero + c5))
 - Code(Expression(46, Sub)) at (prev + 1, 16) to (start + 0, 21)
-    = (((c1 + c3) + (c4 + c5)) - c6)
+    = (((Zero + c3) + (Zero + c5)) - c6)
 - Code(Zero) at (prev + 0, 22) to (start + 1, 14)
 - Code(Expression(45, Sub)) at (prev + 2, 20) to (start + 0, 25)
-    = ((((c1 + c3) + (c4 + c5)) - c6) - c4)
+    = ((((Zero + c3) + (Zero + c5)) - c6) - Zero)
 - Code(Zero) at (prev + 1, 27) to (start + 0, 31)
 - Code(Zero) at (prev + 0, 32) to (start + 0, 34)
 - Code(Expression(44, Sub)) at (prev + 1, 18) to (start + 0, 19)
-    = (((((c1 + c3) + (c4 + c5)) - c6) - c4) - c7)
+    = (((((Zero + c3) + (Zero + c5)) - c6) - Zero) - Zero)
 - Code(Expression(43, Add)) at (prev + 1, 17) to (start + 0, 34)
-    = (Zero + (((((c1 + c3) + (c4 + c5)) - c6) - c4) - c7))
+    = (Zero + (((((Zero + c3) + (Zero + c5)) - c6) - Zero) - Zero))
 - Code(Expression(42, Sub)) at (prev + 0, 34) to (start + 0, 35)
-    = ((Zero + (((((c1 + c3) + (c4 + c5)) - c6) - c4) - c7)) - c5)
+    = ((Zero + (((((Zero + c3) + (Zero + c5)) - c6) - Zero) - Zero)) - c5)
 - Code(Counter(6)) at (prev + 3, 9) to (start + 0, 15)
 - Code(Expression(41, Add)) at (prev + 1, 5) to (start + 0, 6)
-    = (((Zero + (((((c1 + c3) + (c4 + c5)) - c6) - c4) - c7)) - c5) + (c6 + c9))
+    = (((Zero + (((((Zero + c3) + (Zero + c5)) - c6) - Zero) - Zero)) - c5) + (c6 + c9))
 
 Function name: loops_branches::main
 Raw bytes (9): 0x[01, 01, 00, 01, 01, 37, 01, 05, 02]
diff --git a/tests/coverage-map/status-quo/sort_groups.cov-map b/tests/coverage-map/status-quo/sort_groups.cov-map
index 7156a66cf898..db027f3dc324 100644
--- a/tests/coverage-map/status-quo/sort_groups.cov-map
+++ b/tests/coverage-map/status-quo/sort_groups.cov-map
@@ -44,19 +44,19 @@ Number of file 0 mappings: 4
     = (c1 + (c0 - c1))
 
 Function name: sort_groups::main
-Raw bytes (28): 0x[01, 01, 02, 01, 05, 05, 02, 04, 01, 06, 01, 04, 0d, 00, 04, 0e, 02, 06, 02, 02, 06, 00, 07, 07, 01, 05, 02, 02]
+Raw bytes (28): 0x[01, 01, 02, 01, 00, 00, 02, 04, 01, 06, 01, 04, 0d, 00, 04, 0e, 02, 06, 02, 02, 06, 00, 07, 07, 01, 05, 02, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 2
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Counter(1), rhs = Expression(0, Sub)
+- expression 0 operands: lhs = Counter(0), rhs = Zero
+- expression 1 operands: lhs = Zero, rhs = Expression(0, Sub)
 Number of file 0 mappings: 4
 - Code(Counter(0)) at (prev + 6, 1) to (start + 4, 13)
 - Code(Zero) at (prev + 4, 14) to (start + 2, 6)
 - Code(Expression(0, Sub)) at (prev + 2, 6) to (start + 0, 7)
-    = (c0 - c1)
+    = (c0 - Zero)
 - Code(Expression(1, Add)) at (prev + 1, 5) to (start + 2, 2)
-    = (c1 + (c0 - c1))
+    = (Zero + (c0 - Zero))
 
 Function name: sort_groups::other_fn
 Raw bytes (9): 0x[01, 01, 00, 01, 01, 17, 01, 00, 11]
diff --git a/tests/coverage-map/status-quo/tight_inf_loop.cov-map b/tests/coverage-map/status-quo/tight_inf_loop.cov-map
index 76884212c14f..7fe3146b0809 100644
--- a/tests/coverage-map/status-quo/tight_inf_loop.cov-map
+++ b/tests/coverage-map/status-quo/tight_inf_loop.cov-map
@@ -1,12 +1,12 @@
 Function name: tight_inf_loop::main
-Raw bytes (21): 0x[01, 01, 01, 01, 05, 03, 01, 01, 01, 01, 0d, 00, 02, 09, 00, 10, 02, 01, 06, 01, 02]
+Raw bytes (21): 0x[01, 01, 01, 01, 00, 03, 01, 01, 01, 01, 0d, 00, 02, 09, 00, 10, 02, 01, 06, 01, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 1
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
+- expression 0 operands: lhs = Counter(0), rhs = Zero
 Number of file 0 mappings: 3
 - Code(Counter(0)) at (prev + 1, 1) to (start + 1, 13)
 - Code(Zero) at (prev + 2, 9) to (start + 0, 16)
 - Code(Expression(0, Sub)) at (prev + 1, 6) to (start + 1, 2)
-    = (c0 - c1)
+    = (c0 - Zero)
 
diff --git a/tests/coverage-map/status-quo/while.cov-map b/tests/coverage-map/status-quo/while.cov-map
index cfd2be96a0db..af250f3fb710 100644
--- a/tests/coverage-map/status-quo/while.cov-map
+++ b/tests/coverage-map/status-quo/while.cov-map
@@ -1,15 +1,15 @@
 Function name: while::main
-Raw bytes (28): 0x[01, 01, 02, 01, 05, 03, 05, 04, 01, 01, 01, 01, 10, 03, 02, 0b, 00, 14, 00, 00, 15, 01, 06, 06, 02, 01, 00, 02]
+Raw bytes (28): 0x[01, 01, 02, 01, 00, 03, 00, 04, 01, 01, 01, 01, 10, 03, 02, 0b, 00, 14, 00, 00, 15, 01, 06, 06, 02, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
 Number of expressions: 2
-- expression 0 operands: lhs = Counter(0), rhs = Counter(1)
-- expression 1 operands: lhs = Expression(0, Add), rhs = Counter(1)
+- expression 0 operands: lhs = Counter(0), rhs = Zero
+- expression 1 operands: lhs = Expression(0, Add), rhs = Zero
 Number of file 0 mappings: 4
 - Code(Counter(0)) at (prev + 1, 1) to (start + 1, 16)
 - Code(Expression(0, Add)) at (prev + 2, 11) to (start + 0, 20)
-    = (c0 + c1)
+    = (c0 + Zero)
 - Code(Zero) at (prev + 0, 21) to (start + 1, 6)
 - Code(Expression(1, Sub)) at (prev + 2, 1) to (start + 0, 2)
-    = ((c0 + c1) - c1)
+    = ((c0 + Zero) - Zero)
 
diff --git a/tests/run-coverage/bad_counter_ids.coverage b/tests/run-coverage/bad_counter_ids.coverage
index 763f04d099e9..d69ebf160ea6 100644
--- a/tests/run-coverage/bad_counter_ids.coverage
+++ b/tests/run-coverage/bad_counter_ids.coverage
@@ -23,25 +23,26 @@
                                              ^0
    LL|      1|}
    LL|       |
-   LL|       |fn ne_good() {
-   LL|       |    println!("c");
-   LL|       |    assert_ne!(Foo(1), Foo(3));
-   LL|       |}
+   LL|      1|fn ne_good() {
+   LL|      1|    println!("c");
+   LL|      1|    assert_ne!(Foo(1), Foo(3));
+   LL|      1|}
    LL|       |
-   LL|       |fn ne_good_message() {
-   LL|       |    println!("d");
-   LL|       |    assert_ne!(Foo(1), Foo(3), "message d");
-   LL|       |}
+   LL|      1|fn ne_good_message() {
+   LL|      1|    println!("d");
+   LL|      1|    assert_ne!(Foo(1), Foo(3), "message d");
+                                             ^0
+   LL|      1|}
    LL|       |
    LL|      1|fn eq_bad() {
    LL|      1|    println!("e");
    LL|      1|    assert_eq!(Foo(1), Foo(3));
    LL|      0|}
    LL|       |
-   LL|       |fn eq_bad_message() {
-   LL|       |    println!("f");
-   LL|       |    assert_eq!(Foo(1), Foo(3), "message f");
-   LL|       |}
+   LL|      1|fn eq_bad_message() {
+   LL|      1|    println!("f");
+   LL|      1|    assert_eq!(Foo(1), Foo(3), "message f");
+   LL|      0|}
    LL|       |
    LL|      1|fn ne_bad() {
    LL|      1|    println!("g");

From fe00cfef57fcd54d89a09b69819eb66cf90edf70 Mon Sep 17 00:00:00 2001
From: Mu001999 <mu001999@outlook.com>
Date: Sat, 28 Oct 2023 08:53:51 +0800
Subject: [PATCH 174/297] restore snapshot when parse_param_general

---
 compiler/rustc_parse/src/parser/item.rs |  2 ++
 tests/ui/parser/issue-116781.rs         |  8 ++++++++
 tests/ui/parser/issue-116781.stderr     | 16 ++++++++++++++++
 3 files changed, 26 insertions(+)
 create mode 100644 tests/ui/parser/issue-116781.rs
 create mode 100644 tests/ui/parser/issue-116781.stderr

diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
index 982f601c0d5a..a4aaf9290dc3 100644
--- a/compiler/rustc_parse/src/parser/item.rs
+++ b/compiler/rustc_parse/src/parser/item.rs
@@ -2494,9 +2494,11 @@ impl<'a> Parser<'a> {
         // Parse the arguments, starting out with `self` being allowed...
         let (mut params, _) = self.parse_paren_comma_seq(|p| {
             p.recover_diff_marker();
+            let snapshot = p.create_snapshot_for_diagnostic();
             let param = p.parse_param_general(req_name, first_param).or_else(|mut e| {
                 e.emit();
                 let lo = p.prev_token.span;
+                p.restore_snapshot(snapshot);
                 // Skip every token until next possible arg or end.
                 p.eat_to_tokens(&[&token::Comma, &token::CloseDelim(Delimiter::Parenthesis)]);
                 // Create a placeholder argument for proper arg count (issue #34264).
diff --git a/tests/ui/parser/issue-116781.rs b/tests/ui/parser/issue-116781.rs
new file mode 100644
index 000000000000..0e951d2eaa44
--- /dev/null
+++ b/tests/ui/parser/issue-116781.rs
@@ -0,0 +1,8 @@
+#[derive(Debug)]
+struct Foo {
+    #[cfg(all())]
+    field: fn(($),), //~ ERROR expected pattern, found `$`
+    //~^ ERROR expected pattern, found `$`
+}
+
+fn main() {}
diff --git a/tests/ui/parser/issue-116781.stderr b/tests/ui/parser/issue-116781.stderr
new file mode 100644
index 000000000000..1a77b60a50dc
--- /dev/null
+++ b/tests/ui/parser/issue-116781.stderr
@@ -0,0 +1,16 @@
+error: expected pattern, found `$`
+  --> $DIR/issue-116781.rs:4:16
+   |
+LL |     field: fn(($),),
+   |                ^ expected pattern
+
+error: expected pattern, found `$`
+  --> $DIR/issue-116781.rs:4:16
+   |
+LL |     field: fn(($),),
+   |                ^ expected pattern
+   |
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+
+error: aborting due to 2 previous errors
+

From 64678d46678a8bb6ecdb8a6d71f4203858a4a2b2 Mon Sep 17 00:00:00 2001
From: Ralf Jung <post@ralfj.de>
Date: Sat, 28 Oct 2023 11:34:13 +0200
Subject: [PATCH 175/297] -Zunpretty help: add missing possible values

---
 compiler/rustc_session/src/options.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index fd473acbd3c3..ab170e1eac89 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -1902,6 +1902,7 @@ written to standard error output)"),
         `hir` (the HIR), `hir,identified`,
         `hir,typed` (HIR with types for each node),
         `hir-tree` (dump the raw HIR),
+        `thir-tree`, `thir-flat`,
         `mir` (the MIR), or `mir-cfg` (graphviz formatted MIR)"),
     unsound_mir_opts: bool = (false, parse_bool, [TRACKED],
         "enable unsound and buggy MIR optimizations (default: no)"),

From b329c69f6cf04be0bfc247ce07fac71104156244 Mon Sep 17 00:00:00 2001
From: Ralf Jung <post@ralfj.de>
Date: Sat, 28 Oct 2023 11:54:04 +0200
Subject: [PATCH 176/297] memcpy assumptions: link to source showing that GCC
 makes the same assumption

---
 library/core/src/lib.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs
index 0d00899c4de3..f472f61b0b99 100644
--- a/library/core/src/lib.rs
+++ b/library/core/src/lib.rs
@@ -26,7 +26,8 @@
 //!   assumptions about their semantics: For `memcpy`, `memmove`, `memset`, `memcmp`, and `bcmp`, if
 //!   the `n` parameter is 0, the function is assumed to not be UB. Furthermore, for `memcpy`, if
 //!   source and target pointer are equal, the function is assumed to not be UB.
-//!   (Note that these are [standard assumptions](https://reviews.llvm.org/D86993) among compilers.)
+//!   (Note that these are standard assumptions among compilers:
+//!   [clang](https://reviews.llvm.org/D86993) and [GCC](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=32667) do the same.)
 //!   These functions are often provided by the system libc, but can also be provided by the
 //!   [compiler-builtins crate](https://crates.io/crates/compiler_builtins).
 //!   Note that the library does not guarantee that it will always make these assumptions, so Rust

From a510288f0a89406ca0487cc0b1f41eb5d432208c Mon Sep 17 00:00:00 2001
From: Havard Eidnes <he@NetBSD.org>
Date: Sat, 28 Oct 2023 12:14:30 +0000
Subject: [PATCH 177/297] i586_unknown_netbsd.rs: drop "-m32" flag insertion to
 gcc.

This triggers a consistency check in rust (that all linker flavours
must have identical arguments), and on NetBSD/i386, the 32-bitness
is implicitly chosen through the chosen toolchain, and appears to
not be required.  So drop it, and also drop the imports of the
now-no-longer-used identifiers.
---
 compiler/rustc_target/src/spec/i586_unknown_netbsd.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/compiler/rustc_target/src/spec/i586_unknown_netbsd.rs b/compiler/rustc_target/src/spec/i586_unknown_netbsd.rs
index 957cb38b7933..0d8bdc3f89f7 100644
--- a/compiler/rustc_target/src/spec/i586_unknown_netbsd.rs
+++ b/compiler/rustc_target/src/spec/i586_unknown_netbsd.rs
@@ -1,10 +1,9 @@
-use crate::spec::{Cc, LinkerFlavor, Lld, StackProbeType, Target, TargetOptions};
+use crate::spec::{StackProbeType, Target, TargetOptions};
 
 pub fn target() -> Target {
     let mut base = super::netbsd_base::opts();
     base.cpu = "pentium".into();
     base.max_atomic_width = Some(64);
-    base.pre_link_args.entry(LinkerFlavor::Gnu(Cc::Yes, Lld::No)).or_default().push("-m32".into());
     base.stack_probes = StackProbeType::Call;
 
     Target {

From d9ddad3921e24ff819a35ad030a0b7a9f7b098c6 Mon Sep 17 00:00:00 2001
From: Havard Eidnes <he@NetBSD.org>
Date: Sat, 28 Oct 2023 13:29:00 +0000
Subject: [PATCH 178/297] i586-unknown-netbsd: add entry in
 platform-support.md.

---
 src/doc/rustc/src/platform-support.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/doc/rustc/src/platform-support.md b/src/doc/rustc/src/platform-support.md
index 4e55d3604302..f0be501fbbec 100644
--- a/src/doc/rustc/src/platform-support.md
+++ b/src/doc/rustc/src/platform-support.md
@@ -152,6 +152,7 @@ target | std | notes
 `i586-pc-windows-msvc` | * | 32-bit Windows w/o SSE [^x86_32-floats-x87]
 `i586-unknown-linux-gnu` | ✓ | 32-bit Linux w/o SSE (kernel 3.2, glibc 2.17) [^x86_32-floats-x87]
 `i586-unknown-linux-musl` | ✓ | 32-bit Linux w/o SSE, MUSL [^x86_32-floats-x87]
+[`i586-unknown-netbsd`](platform-support/netbsd.md) |  ✓ | 32-bir x86, restricted to Pentium
 [`i686-linux-android`](platform-support/android.md) | ✓ | 32-bit x86 Android [^x86_32-floats-return-ABI]
 `i686-unknown-freebsd` | ✓ | 32-bit FreeBSD [^x86_32-floats-return-ABI]
 `i686-unknown-linux-musl` | ✓ | 32-bit Linux with MUSL [^x86_32-floats-return-ABI]

From 4dd7568a97058c0a2d17f6ace15e42403f93cea1 Mon Sep 17 00:00:00 2001
From: coekjan <cn_yzr@qq.com>
Date: Sat, 28 Oct 2023 21:30:43 +0800
Subject: [PATCH 179/297] mark constructor of `BinaryHeap` as const fn

---
 library/alloc/src/collections/binary_heap/mod.rs | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/library/alloc/src/collections/binary_heap/mod.rs b/library/alloc/src/collections/binary_heap/mod.rs
index 66573b90db96..61c5950b0273 100644
--- a/library/alloc/src/collections/binary_heap/mod.rs
+++ b/library/alloc/src/collections/binary_heap/mod.rs
@@ -434,8 +434,9 @@ impl<T: Ord> BinaryHeap<T> {
     /// heap.push(4);
     /// ```
     #[stable(feature = "rust1", since = "1.0.0")]
+    #[rustc_const_unstable(feature = "const_binary_heap_constructor", issue = "112353")]
     #[must_use]
-    pub fn new() -> BinaryHeap<T> {
+    pub const fn new() -> BinaryHeap<T> {
         BinaryHeap { data: vec![] }
     }
 
@@ -477,8 +478,9 @@ impl<T: Ord, A: Allocator> BinaryHeap<T, A> {
     /// heap.push(4);
     /// ```
     #[unstable(feature = "allocator_api", issue = "32838")]
+    #[rustc_const_unstable(feature = "const_binary_heap_constructor", issue = "112353")]
     #[must_use]
-    pub fn new_in(alloc: A) -> BinaryHeap<T, A> {
+    pub const fn new_in(alloc: A) -> BinaryHeap<T, A> {
         BinaryHeap { data: Vec::new_in(alloc) }
     }
 

From 351d532a27c56f7af82feaa7d612c412fd5c221c Mon Sep 17 00:00:00 2001
From: Ralf Jung <post@ralfj.de>
Date: Sat, 28 Oct 2023 15:39:54 +0200
Subject: [PATCH 180/297] interpret: call caller_location logic the same way
 codegen does, and share some code

---
 .../rustc_codegen_cranelift/src/common.rs     |  11 +-
 compiler/rustc_codegen_ssa/src/mir/block.rs   |   9 +-
 .../src/const_eval/eval_queries.rs            |   2 +-
 .../src/const_eval/machine.rs                 |  19 +++
 .../rustc_const_eval/src/const_eval/mod.rs    |  18 +--
 .../src/interpret/eval_context.rs             |  62 ++++++++
 .../src/interpret/intrinsics.rs               |   8 +-
 .../interpret/intrinsics/caller_location.rs   | 136 ------------------
 compiler/rustc_const_eval/src/lib.rs          |   2 +-
 .../src/util/caller_location.rs               |  63 ++++++++
 compiler/rustc_const_eval/src/util/mod.rs     |   1 +
 compiler/rustc_middle/src/mir/consts.rs       |  18 +++
 12 files changed, 173 insertions(+), 176 deletions(-)
 delete mode 100644 compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs
 create mode 100644 compiler/rustc_const_eval/src/util/caller_location.rs

diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs
index 9771f44f62cf..38dba016c23e 100644
--- a/compiler/rustc_codegen_cranelift/src/common.rs
+++ b/compiler/rustc_codegen_cranelift/src/common.rs
@@ -433,16 +433,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
     // Note: must be kept in sync with get_caller_location from cg_ssa
     pub(crate) fn get_caller_location(&mut self, mut source_info: mir::SourceInfo) -> CValue<'tcx> {
         let span_to_caller_location = |fx: &mut FunctionCx<'_, '_, 'tcx>, span: Span| {
-            use rustc_session::RemapFileNameExt;
-            let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
-            let caller = fx.tcx.sess.source_map().lookup_char_pos(topmost.lo());
-            let const_loc = fx.tcx.const_caller_location((
-                rustc_span::symbol::Symbol::intern(
-                    &caller.file.name.for_codegen(&fx.tcx.sess).to_string_lossy(),
-                ),
-                caller.line as u32,
-                caller.col_display as u32 + 1,
-            ));
+            let const_loc = fx.tcx.span_as_caller_location(span);
             crate::constant::codegen_const_value(fx, const_loc, fx.tcx.caller_location_ty())
         };
 
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index caade768795f..cbe3c7368967 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -1454,14 +1454,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let tcx = bx.tcx();
 
         let mut span_to_caller_location = |span: Span| {
-            use rustc_session::RemapFileNameExt;
-            let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
-            let caller = tcx.sess.source_map().lookup_char_pos(topmost.lo());
-            let const_loc = tcx.const_caller_location((
-                Symbol::intern(&caller.file.name.for_codegen(self.cx.sess()).to_string_lossy()),
-                caller.line as u32,
-                caller.col_display as u32 + 1,
-            ));
+            let const_loc = tcx.span_as_caller_location(span);
             OperandRef::from_const(bx, const_loc, bx.tcx().caller_location_ty())
         };
 
diff --git a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
index 46e00d0f176b..669838308a55 100644
--- a/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
+++ b/compiler/rustc_const_eval/src/const_eval/eval_queries.rs
@@ -90,7 +90,7 @@ fn eval_body_using_ecx<'mir, 'tcx>(
 /// that inform us about the generic bounds of the constant. E.g., using an associated constant
 /// of a function's generic parameter will require knowledge about the bounds on the generic
 /// parameter. These bounds are passed to `mk_eval_cx` via the `ParamEnv` argument.
-pub(super) fn mk_eval_cx<'mir, 'tcx>(
+pub(crate) fn mk_eval_cx<'mir, 'tcx>(
     tcx: TyCtxt<'tcx>,
     root_span: Span,
     param_env: ty::ParamEnv<'tcx>,
diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs
index 0d0ebe6f390b..4b447229c5f6 100644
--- a/compiler/rustc_const_eval/src/const_eval/machine.rs
+++ b/compiler/rustc_const_eval/src/const_eval/machine.rs
@@ -4,6 +4,7 @@ use rustc_middle::mir;
 use rustc_middle::mir::interpret::PointerArithmetic;
 use rustc_middle::ty::layout::{FnAbiOf, TyAndLayout};
 use rustc_middle::ty::{self, TyCtxt};
+use rustc_span::Span;
 use std::borrow::Borrow;
 use std::hash::Hash;
 use std::ops::ControlFlow;
@@ -181,6 +182,24 @@ impl interpret::MayLeak for ! {
 }
 
 impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
+    fn location_triple_for_span(&self, span: Span) -> (Symbol, u32, u32) {
+        let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
+        let caller = self.tcx.sess.source_map().lookup_char_pos(topmost.lo());
+
+        use rustc_session::{config::RemapPathScopeComponents, RemapFileNameExt};
+        (
+            Symbol::intern(
+                &caller
+                    .file
+                    .name
+                    .for_scope(&self.tcx.sess, RemapPathScopeComponents::DIAGNOSTICS)
+                    .to_string_lossy(),
+            ),
+            u32::try_from(caller.line).unwrap(),
+            u32::try_from(caller.col_display).unwrap().checked_add(1).unwrap(),
+        )
+    }
+
     /// "Intercept" a function call, because we have something special to do for it.
     /// All `#[rustc_do_not_const_check]` functions should be hooked here.
     /// If this returns `Some` function, which may be `instance` or a different function with
diff --git a/compiler/rustc_const_eval/src/const_eval/mod.rs b/compiler/rustc_const_eval/src/const_eval/mod.rs
index bcbe996be7d8..d6ee6975cddd 100644
--- a/compiler/rustc_const_eval/src/const_eval/mod.rs
+++ b/compiler/rustc_const_eval/src/const_eval/mod.rs
@@ -1,12 +1,12 @@
 // Not in interpret to make sure we do not use private implementation details
 
 use crate::errors::MaxNumNodesInConstErr;
-use crate::interpret::{intern_const_alloc_recursive, InternKind, InterpCx, Scalar};
+use crate::interpret::InterpCx;
 use rustc_middle::mir;
 use rustc_middle::mir::interpret::{EvalToValTreeResult, GlobalId};
 use rustc_middle::query::TyCtxtAt;
 use rustc_middle::ty::{self, Ty, TyCtxt};
-use rustc_span::{source_map::DUMMY_SP, symbol::Symbol};
+use rustc_span::source_map::DUMMY_SP;
 
 mod error;
 mod eval_queries;
@@ -20,20 +20,6 @@ pub use fn_queries::*;
 pub use machine::*;
 pub(crate) use valtrees::{const_to_valtree_inner, valtree_to_const_value};
 
-pub(crate) fn const_caller_location(
-    tcx: TyCtxt<'_>,
-    (file, line, col): (Symbol, u32, u32),
-) -> mir::ConstValue<'_> {
-    trace!("const_caller_location: {}:{}:{}", file, line, col);
-    let mut ecx = mk_eval_cx(tcx, DUMMY_SP, ty::ParamEnv::reveal_all(), CanAccessStatics::No);
-
-    let loc_place = ecx.alloc_caller_location(file, line, col);
-    if intern_const_alloc_recursive(&mut ecx, InternKind::Constant, &loc_place).is_err() {
-        bug!("intern_const_alloc_recursive should not error in this case")
-    }
-    mir::ConstValue::Scalar(Scalar::from_maybe_pointer(loc_place.ptr(), &tcx))
-}
-
 // We forbid type-level constants that contain more than `VALTREE_MAX_NODES` nodes.
 const VALTREE_MAX_NODES: usize = 100000;
 
diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs
index ec0af79459cb..f3c19495a61f 100644
--- a/compiler/rustc_const_eval/src/interpret/eval_context.rs
+++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs
@@ -595,6 +595,68 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         }
     }
 
+    /// Walks up the callstack from the intrinsic's callsite, searching for the first callsite in a
+    /// frame which is not `#[track_caller]`. This is the fancy version of `cur_span`.
+    pub(crate) fn find_closest_untracked_caller_location(&self) -> Span {
+        for frame in self.stack().iter().rev() {
+            debug!("find_closest_untracked_caller_location: checking frame {:?}", frame.instance);
+
+            // Assert that the frame we look at is actually executing code currently
+            // (`loc` is `Right` when we are unwinding and the frame does not require cleanup).
+            let loc = frame.loc.left().unwrap();
+
+            // This could be a non-`Call` terminator (such as `Drop`), or not a terminator at all
+            // (such as `box`). Use the normal span by default.
+            let mut source_info = *frame.body.source_info(loc);
+
+            // If this is a `Call` terminator, use the `fn_span` instead.
+            let block = &frame.body.basic_blocks[loc.block];
+            if loc.statement_index == block.statements.len() {
+                debug!(
+                    "find_closest_untracked_caller_location: got terminator {:?} ({:?})",
+                    block.terminator(),
+                    block.terminator().kind,
+                );
+                if let mir::TerminatorKind::Call { fn_span, .. } = block.terminator().kind {
+                    source_info.span = fn_span;
+                }
+            }
+
+            // Note: this must be kept in sync with get_caller_location from cg_ssa.
+
+            // Walk up the `SourceScope`s, in case some of them are from MIR inlining.
+            // If so, the starting `source_info.span` is in the innermost inlined
+            // function, and will be replaced with outer callsite spans as long
+            // as the inlined functions were `#[track_caller]`.
+            loop {
+                let scope_data = &frame.body.source_scopes[source_info.scope];
+
+                if let Some((callee, callsite_span)) = scope_data.inlined {
+                    // Stop inside the most nested non-`#[track_caller]` function,
+                    // before ever reaching its caller (which is irrelevant).
+                    if !callee.def.requires_caller_location(*self.tcx) {
+                        return source_info.span;
+                    }
+                    source_info.span = callsite_span;
+                }
+
+                // Skip past all of the parents with `inlined: None`.
+                match scope_data.inlined_parent_scope {
+                    Some(parent) => source_info.scope = parent,
+                    None => break,
+                }
+            }
+
+            // Stop inside the most nested non-`#[track_caller]` function,
+            // before ever reaching its caller (which is irrelevant).
+            if !frame.instance.def.requires_caller_location(*self.tcx) {
+                return source_info.span;
+            }
+        }
+
+        span_bug!(self.cur_span(), "no non-`#[track_caller]` frame found")
+    }
+
     #[inline(always)]
     pub fn layout_of_local(
         &self,
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
index c97207a61ac3..8fd04a687915 100644
--- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
@@ -22,8 +22,6 @@ use super::{
 
 use crate::fluent_generated as fluent;
 
-mod caller_location;
-
 fn numeric_intrinsic<Prov>(name: Symbol, bits: u128, kind: Primitive) -> Scalar<Prov> {
     let size = match kind {
         Primitive::Int(integer, _) => integer.size(),
@@ -130,8 +128,10 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
         match intrinsic_name {
             sym::caller_location => {
                 let span = self.find_closest_untracked_caller_location();
-                let location = self.alloc_caller_location_for_span(span);
-                self.write_immediate(location.to_ref(self), dest)?;
+                let val = self.tcx.span_as_caller_location(span);
+                let val =
+                    self.const_val_to_op(val, self.tcx.caller_location_ty(), Some(dest.layout))?;
+                self.copy_op(&val, dest, /* allow_transmute */ false)?;
             }
 
             sym::min_align_of_val | sym::size_of_val => {
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs
deleted file mode 100644
index 16b7decf9c43..000000000000
--- a/compiler/rustc_const_eval/src/interpret/intrinsics/caller_location.rs
+++ /dev/null
@@ -1,136 +0,0 @@
-use rustc_ast::Mutability;
-use rustc_hir::lang_items::LangItem;
-use rustc_middle::mir::TerminatorKind;
-use rustc_middle::ty::layout::LayoutOf;
-use rustc_span::{Span, Symbol};
-
-use crate::interpret::{
-    intrinsics::{InterpCx, Machine},
-    MPlaceTy, MemoryKind, Scalar,
-};
-
-impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
-    /// Walks up the callstack from the intrinsic's callsite, searching for the first callsite in a
-    /// frame which is not `#[track_caller]`.
-    pub(crate) fn find_closest_untracked_caller_location(&self) -> Span {
-        for frame in self.stack().iter().rev() {
-            debug!("find_closest_untracked_caller_location: checking frame {:?}", frame.instance);
-
-            // Assert that the frame we look at is actually executing code currently
-            // (`loc` is `Right` when we are unwinding and the frame does not require cleanup).
-            let loc = frame.loc.left().unwrap();
-
-            // This could be a non-`Call` terminator (such as `Drop`), or not a terminator at all
-            // (such as `box`). Use the normal span by default.
-            let mut source_info = *frame.body.source_info(loc);
-
-            // If this is a `Call` terminator, use the `fn_span` instead.
-            let block = &frame.body.basic_blocks[loc.block];
-            if loc.statement_index == block.statements.len() {
-                debug!(
-                    "find_closest_untracked_caller_location: got terminator {:?} ({:?})",
-                    block.terminator(),
-                    block.terminator().kind
-                );
-                if let TerminatorKind::Call { fn_span, .. } = block.terminator().kind {
-                    source_info.span = fn_span;
-                }
-            }
-
-            // Walk up the `SourceScope`s, in case some of them are from MIR inlining.
-            // If so, the starting `source_info.span` is in the innermost inlined
-            // function, and will be replaced with outer callsite spans as long
-            // as the inlined functions were `#[track_caller]`.
-            loop {
-                let scope_data = &frame.body.source_scopes[source_info.scope];
-
-                if let Some((callee, callsite_span)) = scope_data.inlined {
-                    // Stop inside the most nested non-`#[track_caller]` function,
-                    // before ever reaching its caller (which is irrelevant).
-                    if !callee.def.requires_caller_location(*self.tcx) {
-                        return source_info.span;
-                    }
-                    source_info.span = callsite_span;
-                }
-
-                // Skip past all of the parents with `inlined: None`.
-                match scope_data.inlined_parent_scope {
-                    Some(parent) => source_info.scope = parent,
-                    None => break,
-                }
-            }
-
-            // Stop inside the most nested non-`#[track_caller]` function,
-            // before ever reaching its caller (which is irrelevant).
-            if !frame.instance.def.requires_caller_location(*self.tcx) {
-                return source_info.span;
-            }
-        }
-
-        span_bug!(self.cur_span(), "no non-`#[track_caller]` frame found")
-    }
-
-    /// Allocate a `const core::panic::Location` with the provided filename and line/column numbers.
-    pub(crate) fn alloc_caller_location(
-        &mut self,
-        filename: Symbol,
-        line: u32,
-        col: u32,
-    ) -> MPlaceTy<'tcx, M::Provenance> {
-        let loc_details = self.tcx.sess.opts.unstable_opts.location_detail;
-        // This can fail if rustc runs out of memory right here. Trying to emit an error would be
-        // pointless, since that would require allocating more memory than these short strings.
-        let file = if loc_details.file {
-            self.allocate_str(filename.as_str(), MemoryKind::CallerLocation, Mutability::Not)
-                .unwrap()
-        } else {
-            // FIXME: This creates a new allocation each time. It might be preferable to
-            // perform this allocation only once, and re-use the `MPlaceTy`.
-            // See https://github.com/rust-lang/rust/pull/89920#discussion_r730012398
-            self.allocate_str("<redacted>", MemoryKind::CallerLocation, Mutability::Not).unwrap()
-        };
-        let line = if loc_details.line { Scalar::from_u32(line) } else { Scalar::from_u32(0) };
-        let col = if loc_details.column { Scalar::from_u32(col) } else { Scalar::from_u32(0) };
-
-        // Allocate memory for `CallerLocation` struct.
-        let loc_ty = self
-            .tcx
-            .type_of(self.tcx.require_lang_item(LangItem::PanicLocation, None))
-            .instantiate(*self.tcx, self.tcx.mk_args(&[self.tcx.lifetimes.re_erased.into()]));
-        let loc_layout = self.layout_of(loc_ty).unwrap();
-        let location = self.allocate(loc_layout, MemoryKind::CallerLocation).unwrap();
-
-        // Initialize fields.
-        self.write_immediate(file.to_ref(self), &self.project_field(&location, 0).unwrap())
-            .expect("writing to memory we just allocated cannot fail");
-        self.write_scalar(line, &self.project_field(&location, 1).unwrap())
-            .expect("writing to memory we just allocated cannot fail");
-        self.write_scalar(col, &self.project_field(&location, 2).unwrap())
-            .expect("writing to memory we just allocated cannot fail");
-
-        location
-    }
-
-    pub(crate) fn location_triple_for_span(&self, span: Span) -> (Symbol, u32, u32) {
-        let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
-        let caller = self.tcx.sess.source_map().lookup_char_pos(topmost.lo());
-
-        use rustc_session::{config::RemapPathScopeComponents, RemapFileNameExt};
-        (
-            Symbol::intern(
-                &caller
-                    .file
-                    .name
-                    .for_scope(&self.tcx.sess, RemapPathScopeComponents::DIAGNOSTICS)
-                    .to_string_lossy(),
-            ),
-            u32::try_from(caller.line).unwrap(),
-            u32::try_from(caller.col_display).unwrap().checked_add(1).unwrap(),
-        )
-    }
-
-    pub fn alloc_caller_location_for_span(&mut self, span: Span) -> MPlaceTy<'tcx, M::Provenance> {
-        let (file, line, column) = self.location_triple_for_span(span);
-        self.alloc_caller_location(file, line, column)
-    }
-}
diff --git a/compiler/rustc_const_eval/src/lib.rs b/compiler/rustc_const_eval/src/lib.rs
index 1fd5723f2775..7fc87ed88810 100644
--- a/compiler/rustc_const_eval/src/lib.rs
+++ b/compiler/rustc_const_eval/src/lib.rs
@@ -49,7 +49,7 @@ pub fn provide(providers: &mut Providers) {
     const_eval::provide(providers);
     providers.eval_to_const_value_raw = const_eval::eval_to_const_value_raw_provider;
     providers.eval_to_allocation_raw = const_eval::eval_to_allocation_raw_provider;
-    providers.const_caller_location = const_eval::const_caller_location;
+    providers.const_caller_location = util::caller_location::const_caller_location_provider;
     providers.eval_to_valtree = |tcx, param_env_and_value| {
         let (param_env, raw) = param_env_and_value.into_parts();
         const_eval::eval_to_valtree(tcx, param_env, raw)
diff --git a/compiler/rustc_const_eval/src/util/caller_location.rs b/compiler/rustc_const_eval/src/util/caller_location.rs
new file mode 100644
index 000000000000..d67863d75d66
--- /dev/null
+++ b/compiler/rustc_const_eval/src/util/caller_location.rs
@@ -0,0 +1,63 @@
+use rustc_hir::LangItem;
+use rustc_middle::mir;
+use rustc_middle::ty::layout::LayoutOf;
+use rustc_middle::ty::{self, TyCtxt};
+use rustc_span::{source_map::DUMMY_SP, symbol::Symbol};
+use rustc_type_ir::Mutability;
+
+use crate::const_eval::{mk_eval_cx, CanAccessStatics, CompileTimeEvalContext};
+use crate::interpret::*;
+
+/// Allocate a `const core::panic::Location` with the provided filename and line/column numbers.
+fn alloc_caller_location<'mir, 'tcx>(
+    ecx: &mut CompileTimeEvalContext<'mir, 'tcx>,
+    filename: Symbol,
+    line: u32,
+    col: u32,
+) -> MPlaceTy<'tcx> {
+    let loc_details = ecx.tcx.sess.opts.unstable_opts.location_detail;
+    // This can fail if rustc runs out of memory right here. Trying to emit an error would be
+    // pointless, since that would require allocating more memory than these short strings.
+    let file = if loc_details.file {
+        ecx.allocate_str(filename.as_str(), MemoryKind::CallerLocation, Mutability::Not).unwrap()
+    } else {
+        // FIXME: This creates a new allocation each time. It might be preferable to
+        // perform this allocation only once, and re-use the `MPlaceTy`.
+        // See https://github.com/rust-lang/rust/pull/89920#discussion_r730012398
+        ecx.allocate_str("<redacted>", MemoryKind::CallerLocation, Mutability::Not).unwrap()
+    };
+    let line = if loc_details.line { Scalar::from_u32(line) } else { Scalar::from_u32(0) };
+    let col = if loc_details.column { Scalar::from_u32(col) } else { Scalar::from_u32(0) };
+
+    // Allocate memory for `CallerLocation` struct.
+    let loc_ty = ecx
+        .tcx
+        .type_of(ecx.tcx.require_lang_item(LangItem::PanicLocation, None))
+        .instantiate(*ecx.tcx, ecx.tcx.mk_args(&[ecx.tcx.lifetimes.re_erased.into()]));
+    let loc_layout = ecx.layout_of(loc_ty).unwrap();
+    let location = ecx.allocate(loc_layout, MemoryKind::CallerLocation).unwrap();
+
+    // Initialize fields.
+    ecx.write_immediate(file.to_ref(ecx), &ecx.project_field(&location, 0).unwrap())
+        .expect("writing to memory we just allocated cannot fail");
+    ecx.write_scalar(line, &ecx.project_field(&location, 1).unwrap())
+        .expect("writing to memory we just allocated cannot fail");
+    ecx.write_scalar(col, &ecx.project_field(&location, 2).unwrap())
+        .expect("writing to memory we just allocated cannot fail");
+
+    location
+}
+
+pub(crate) fn const_caller_location_provider(
+    tcx: TyCtxt<'_>,
+    (file, line, col): (Symbol, u32, u32),
+) -> mir::ConstValue<'_> {
+    trace!("const_caller_location: {}:{}:{}", file, line, col);
+    let mut ecx = mk_eval_cx(tcx, DUMMY_SP, ty::ParamEnv::reveal_all(), CanAccessStatics::No);
+
+    let loc_place = alloc_caller_location(&mut ecx, file, line, col);
+    if intern_const_alloc_recursive(&mut ecx, InternKind::Constant, &loc_place).is_err() {
+        bug!("intern_const_alloc_recursive should not error in this case")
+    }
+    mir::ConstValue::Scalar(Scalar::from_maybe_pointer(loc_place.ptr(), &tcx))
+}
diff --git a/compiler/rustc_const_eval/src/util/mod.rs b/compiler/rustc_const_eval/src/util/mod.rs
index 040b3071e6f8..1e58bd645cdd 100644
--- a/compiler/rustc_const_eval/src/util/mod.rs
+++ b/compiler/rustc_const_eval/src/util/mod.rs
@@ -1,6 +1,7 @@
 use rustc_middle::mir;
 
 mod alignment;
+pub(crate) mod caller_location;
 mod check_validity_requirement;
 mod compare_types;
 mod type_name;
diff --git a/compiler/rustc_middle/src/mir/consts.rs b/compiler/rustc_middle/src/mir/consts.rs
index 4e429f316e80..47865bae0d2d 100644
--- a/compiler/rustc_middle/src/mir/consts.rs
+++ b/compiler/rustc_middle/src/mir/consts.rs
@@ -3,6 +3,7 @@ use std::fmt::{self, Debug, Display, Formatter};
 use rustc_hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
 use rustc_hir::{self as hir};
+use rustc_session::RemapFileNameExt;
 use rustc_span::Span;
 use rustc_target::abi::{HasDataLayout, Size};
 
@@ -529,3 +530,20 @@ impl<'tcx> Display for Const<'tcx> {
         }
     }
 }
+
+///////////////////////////////////////////////////////////////////////////
+/// Const-related utilities
+
+impl<'tcx> TyCtxt<'tcx> {
+    pub fn span_as_caller_location(self, span: Span) -> ConstValue<'tcx> {
+        let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
+        let caller = self.sess.source_map().lookup_char_pos(topmost.lo());
+        self.const_caller_location((
+            rustc_span::symbol::Symbol::intern(
+                &caller.file.name.for_codegen(&self.sess).to_string_lossy(),
+            ),
+            caller.line as u32,
+            caller.col_display as u32 + 1,
+        ))
+    }
+}

From 04fa124feb6e3f78fe9bea9fac7b4cbd033c9160 Mon Sep 17 00:00:00 2001
From: Ralf Jung <post@ralfj.de>
Date: Sat, 28 Oct 2023 16:16:15 +0200
Subject: [PATCH 181/297] share the track_caller handling within a mir::Body

---
 .../rustc_codegen_cranelift/src/common.rs     | 37 +++--------------
 compiler/rustc_codegen_ssa/src/mir/block.rs   | 36 ++---------------
 .../src/interpret/eval_context.rs             | 40 +++++--------------
 compiler/rustc_middle/src/mir/mod.rs          | 34 ++++++++++++++++
 4 files changed, 54 insertions(+), 93 deletions(-)

diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs
index 38dba016c23e..63562d335089 100644
--- a/compiler/rustc_codegen_cranelift/src/common.rs
+++ b/compiler/rustc_codegen_cranelift/src/common.rs
@@ -430,38 +430,11 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
         }
     }
 
-    // Note: must be kept in sync with get_caller_location from cg_ssa
-    pub(crate) fn get_caller_location(&mut self, mut source_info: mir::SourceInfo) -> CValue<'tcx> {
-        let span_to_caller_location = |fx: &mut FunctionCx<'_, '_, 'tcx>, span: Span| {
-            let const_loc = fx.tcx.span_as_caller_location(span);
-            crate::constant::codegen_const_value(fx, const_loc, fx.tcx.caller_location_ty())
-        };
-
-        // Walk up the `SourceScope`s, in case some of them are from MIR inlining.
-        // If so, the starting `source_info.span` is in the innermost inlined
-        // function, and will be replaced with outer callsite spans as long
-        // as the inlined functions were `#[track_caller]`.
-        loop {
-            let scope_data = &self.mir.source_scopes[source_info.scope];
-
-            if let Some((callee, callsite_span)) = scope_data.inlined {
-                // Stop inside the most nested non-`#[track_caller]` function,
-                // before ever reaching its caller (which is irrelevant).
-                if !callee.def.requires_caller_location(self.tcx) {
-                    return span_to_caller_location(self, source_info.span);
-                }
-                source_info.span = callsite_span;
-            }
-
-            // Skip past all of the parents with `inlined: None`.
-            match scope_data.inlined_parent_scope {
-                Some(parent) => source_info.scope = parent,
-                None => break,
-            }
-        }
-
-        // No inlined `SourceScope`s, or all of them were `#[track_caller]`.
-        self.caller_location.unwrap_or_else(|| span_to_caller_location(self, source_info.span))
+    pub(crate) fn get_caller_location(&mut self, source_info: mir::SourceInfo) -> CValue<'tcx> {
+        self.mir.caller_location_span(source_info, self.caller_location, self.tcx, |span| {
+            let const_loc = self.tcx.span_as_caller_location(span);
+            crate::constant::codegen_const_value(self, const_loc, self.tcx.caller_location_ty())
+        })
     }
 
     pub(crate) fn anonymous_str(&mut self, msg: &str) -> Value {
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
index cbe3c7368967..22afb4063267 100644
--- a/compiler/rustc_codegen_ssa/src/mir/block.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -1449,40 +1449,12 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
     fn get_caller_location(
         &mut self,
         bx: &mut Bx,
-        mut source_info: mir::SourceInfo,
+        source_info: mir::SourceInfo,
     ) -> OperandRef<'tcx, Bx::Value> {
-        let tcx = bx.tcx();
-
-        let mut span_to_caller_location = |span: Span| {
-            let const_loc = tcx.span_as_caller_location(span);
+        self.mir.caller_location_span(source_info, self.caller_location, bx.tcx(), |span: Span| {
+            let const_loc = bx.tcx().span_as_caller_location(span);
             OperandRef::from_const(bx, const_loc, bx.tcx().caller_location_ty())
-        };
-
-        // Walk up the `SourceScope`s, in case some of them are from MIR inlining.
-        // If so, the starting `source_info.span` is in the innermost inlined
-        // function, and will be replaced with outer callsite spans as long
-        // as the inlined functions were `#[track_caller]`.
-        loop {
-            let scope_data = &self.mir.source_scopes[source_info.scope];
-
-            if let Some((callee, callsite_span)) = scope_data.inlined {
-                // Stop inside the most nested non-`#[track_caller]` function,
-                // before ever reaching its caller (which is irrelevant).
-                if !callee.def.requires_caller_location(tcx) {
-                    return span_to_caller_location(source_info.span);
-                }
-                source_info.span = callsite_span;
-            }
-
-            // Skip past all of the parents with `inlined: None`.
-            match scope_data.inlined_parent_scope {
-                Some(parent) => source_info.scope = parent,
-                None => break,
-            }
-        }
-
-        // No inlined `SourceScope`s, or all of them were `#[track_caller]`.
-        self.caller_location.unwrap_or_else(|| span_to_caller_location(source_info.span))
+        })
     }
 
     fn get_personality_slot(&mut self, bx: &mut Bx) -> PlaceRef<'tcx, Bx::Value> {
diff --git a/compiler/rustc_const_eval/src/interpret/eval_context.rs b/compiler/rustc_const_eval/src/interpret/eval_context.rs
index f3c19495a61f..07cab5e3400e 100644
--- a/compiler/rustc_const_eval/src/interpret/eval_context.rs
+++ b/compiler/rustc_const_eval/src/interpret/eval_context.rs
@@ -622,35 +622,17 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                 }
             }
 
-            // Note: this must be kept in sync with get_caller_location from cg_ssa.
-
-            // Walk up the `SourceScope`s, in case some of them are from MIR inlining.
-            // If so, the starting `source_info.span` is in the innermost inlined
-            // function, and will be replaced with outer callsite spans as long
-            // as the inlined functions were `#[track_caller]`.
-            loop {
-                let scope_data = &frame.body.source_scopes[source_info.scope];
-
-                if let Some((callee, callsite_span)) = scope_data.inlined {
-                    // Stop inside the most nested non-`#[track_caller]` function,
-                    // before ever reaching its caller (which is irrelevant).
-                    if !callee.def.requires_caller_location(*self.tcx) {
-                        return source_info.span;
-                    }
-                    source_info.span = callsite_span;
-                }
-
-                // Skip past all of the parents with `inlined: None`.
-                match scope_data.inlined_parent_scope {
-                    Some(parent) => source_info.scope = parent,
-                    None => break,
-                }
-            }
-
-            // Stop inside the most nested non-`#[track_caller]` function,
-            // before ever reaching its caller (which is irrelevant).
-            if !frame.instance.def.requires_caller_location(*self.tcx) {
-                return source_info.span;
+            let caller_location = if frame.instance.def.requires_caller_location(*self.tcx) {
+                // We use `Err(())` as indication that we should continue up the call stack since
+                // this is a `#[track_caller]` function.
+                Some(Err(()))
+            } else {
+                None
+            };
+            if let Ok(span) =
+                frame.body.caller_location_span(source_info, caller_location, *self.tcx, Ok)
+            {
+                return span;
             }
         }
 
diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs
index a85af7c3fb51..7054cede2d87 100644
--- a/compiler/rustc_middle/src/mir/mod.rs
+++ b/compiler/rustc_middle/src/mir/mod.rs
@@ -578,6 +578,40 @@ impl<'tcx> Body<'tcx> {
     pub fn is_custom_mir(&self) -> bool {
         self.injection_phase.is_some()
     }
+
+    /// For a `Location` in this scope, determine what the "caller location" at that point is. This
+    /// is interesting because of inlining: the `#[track_caller]` attribute of inlined functions
+    /// must be honored. Falls back to the `tracked_caller` value for `#[track_caller]` functions,
+    /// or the function's scope.
+    pub fn caller_location_span<T>(
+        &self,
+        mut source_info: SourceInfo,
+        caller_location: Option<T>,
+        tcx: TyCtxt<'tcx>,
+        from_span: impl FnOnce(Span) -> T,
+    ) -> T {
+        loop {
+            let scope_data = &self.source_scopes[source_info.scope];
+
+            if let Some((callee, callsite_span)) = scope_data.inlined {
+                // Stop inside the most nested non-`#[track_caller]` function,
+                // before ever reaching its caller (which is irrelevant).
+                if !callee.def.requires_caller_location(tcx) {
+                    return from_span(source_info.span);
+                }
+                source_info.span = callsite_span;
+            }
+
+            // Skip past all of the parents with `inlined: None`.
+            match scope_data.inlined_parent_scope {
+                Some(parent) => source_info.scope = parent,
+                None => break,
+            }
+        }
+
+        // No inlined `SourceScope`s, or all of them were `#[track_caller]`.
+        caller_location.unwrap_or_else(|| from_span(source_info.span))
+    }
 }
 
 #[derive(Copy, Clone, PartialEq, Eq, Debug, TyEncodable, TyDecodable, HashStable)]

From f7985afe4f7ea41661acb7e8b30725734b8643f9 Mon Sep 17 00:00:00 2001
From: Ralf Jung <post@ralfj.de>
Date: Sat, 28 Oct 2023 16:50:40 +0200
Subject: [PATCH 182/297] explain why we don't inline when target features
 differ

---
 compiler/rustc_mir_transform/src/inline.rs | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs
index 277060573bcb..793dcf0d994c 100644
--- a/compiler/rustc_mir_transform/src/inline.rs
+++ b/compiler/rustc_mir_transform/src/inline.rs
@@ -439,6 +439,11 @@ impl<'tcx> Inliner<'tcx> {
         }
 
         if callee_attrs.target_features != self.codegen_fn_attrs.target_features {
+            // In general it is not correct to inline a callee with target features that are a
+            // subset of the caller. This is because the callee might contain calls, and the ABI of
+            // those calls depends on the target features of the surrounding function. By moving a
+            // `Call` terminator from one MIR body to another with more target features, we might
+            // change the ABI of that call!
             return Err("incompatible target features");
         }
 

From 3bb0c94eb6d62e6773da3bcd2043ce85937d6cfc Mon Sep 17 00:00:00 2001
From: onur-ozkan <work@onurozkan.dev>
Date: Sat, 28 Oct 2023 19:36:09 +0300
Subject: [PATCH 183/297] change default output mode of `BootstrapCommand`

Signed-off-by: onur-ozkan <work@onurozkan.dev>
---
 src/bootstrap/src/utils/exec.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/bootstrap/src/utils/exec.rs b/src/bootstrap/src/utils/exec.rs
index 9c1c21cd9589..0aede2022bad 100644
--- a/src/bootstrap/src/utils/exec.rs
+++ b/src/bootstrap/src/utils/exec.rs
@@ -54,7 +54,7 @@ impl<'a> From<&'a mut Command> for BootstrapCommand<'a> {
         Self {
             command,
             failure_behavior: BehaviorOnFailure::Exit,
-            output_mode: OutputMode::SuppressOnSuccess,
+            output_mode: OutputMode::PrintAll,
         }
     }
 }

From f5fa36fbb72fe6d676adcb9bb7c1cba869c81c00 Mon Sep 17 00:00:00 2001
From: Havard Eidnes <he@NetBSD.org>
Date: Sat, 28 Oct 2023 16:50:14 +0000
Subject: [PATCH 184/297] i586-unknown-netbsd platform-support.md: fix typo.

---
 src/doc/rustc/src/platform-support.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/doc/rustc/src/platform-support.md b/src/doc/rustc/src/platform-support.md
index f0be501fbbec..ef373298efa1 100644
--- a/src/doc/rustc/src/platform-support.md
+++ b/src/doc/rustc/src/platform-support.md
@@ -152,7 +152,7 @@ target | std | notes
 `i586-pc-windows-msvc` | * | 32-bit Windows w/o SSE [^x86_32-floats-x87]
 `i586-unknown-linux-gnu` | ✓ | 32-bit Linux w/o SSE (kernel 3.2, glibc 2.17) [^x86_32-floats-x87]
 `i586-unknown-linux-musl` | ✓ | 32-bit Linux w/o SSE, MUSL [^x86_32-floats-x87]
-[`i586-unknown-netbsd`](platform-support/netbsd.md) |  ✓ | 32-bir x86, restricted to Pentium
+[`i586-unknown-netbsd`](platform-support/netbsd.md) |  ✓ | 32-bit x86, restricted to Pentium
 [`i686-linux-android`](platform-support/android.md) | ✓ | 32-bit x86 Android [^x86_32-floats-return-ABI]
 `i686-unknown-freebsd` | ✓ | 32-bit FreeBSD [^x86_32-floats-return-ABI]
 `i686-unknown-linux-musl` | ✓ | 32-bit Linux with MUSL [^x86_32-floats-return-ABI]

From 236f6ba261593e9704af8fe188ace73b3e395594 Mon Sep 17 00:00:00 2001
From: onur-ozkan <work@onurozkan.dev>
Date: Sat, 28 Oct 2023 20:02:41 +0300
Subject: [PATCH 185/297] set `BootstrapCommand` output mode for submodules

Signed-off-by: onur-ozkan <work@onurozkan.dev>
---
 src/bootstrap/src/lib.rs | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs
index 8dd1a698dfa8..d7f49a6d11b9 100644
--- a/src/bootstrap/src/lib.rs
+++ b/src/bootstrap/src/lib.rs
@@ -586,7 +586,11 @@ impl Build {
                     .args(&["diff-index", "--quiet", "HEAD"])
                     .current_dir(&absolute_path),
             )
-            .allow_failure(),
+            .allow_failure()
+            .output_mode(match self.is_verbose() {
+                true => OutputMode::PrintAll,
+                false => OutputMode::PrintOutput,
+            }),
         );
         if has_local_modifications {
             self.run(Command::new("git").args(&["stash", "push"]).current_dir(&absolute_path));

From 56643ec19e79ae1efba18ec9698dd297b1efaa57 Mon Sep 17 00:00:00 2001
From: Nilstrieb <48135649+Nilstrieb@users.noreply.github.com>
Date: Sat, 28 Oct 2023 20:21:31 +0200
Subject: [PATCH 186/297] Remove needless print ctx defs

---
 compiler/rustc_middle/src/ty/print/pretty.rs | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index 433ac33f1b82..8c76db60e3a7 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -53,7 +53,6 @@ macro_rules! p {
 }
 macro_rules! define_scoped_cx {
     ($cx:ident) => {
-        #[allow(unused_macros)]
         macro_rules! scoped_cx {
             () => {
                 $cx
@@ -408,8 +407,6 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
         def_id: DefId,
         callers: &mut Vec<DefId>,
     ) -> Result<bool, PrintError> {
-        define_scoped_cx!(self);
-
         debug!("try_print_visible_def_path: def_id={:?}", def_id);
 
         // If `def_id` is a direct or injected extern crate, return the
@@ -1868,8 +1865,6 @@ impl<'tcx> Printer<'tcx> for FmtPrinter<'_, 'tcx> {
         def_id: DefId,
         args: &'tcx [GenericArg<'tcx>],
     ) -> Result<(), PrintError> {
-        define_scoped_cx!(self);
-
         if args.is_empty() {
             match self.try_print_trimmed_def_path(def_id)? {
                 true => return Ok(()),
@@ -2401,8 +2396,6 @@ impl<'tcx> FmtPrinter<'_, 'tcx> {
             let _ = write!(cx, "{cont}");
         };
 
-        define_scoped_cx!(self);
-
         let possible_names = ('a'..='z').rev().map(|s| Symbol::intern(&format!("'{s}")));
 
         let mut available_names = possible_names

From 4dada601c17301ce897af77dbdf0375d657234a0 Mon Sep 17 00:00:00 2001
From: Nilstrieb <48135649+Nilstrieb@users.noreply.github.com>
Date: Sat, 28 Oct 2023 20:22:47 +0200
Subject: [PATCH 187/297] Move macros to usage

---
 compiler/rustc_middle/src/ty/print/pretty.rs | 80 ++++++++++----------
 1 file changed, 40 insertions(+), 40 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index 8c76db60e3a7..0bdebc41842c 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -2623,46 +2623,6 @@ where
     }
 }
 
-macro_rules! forward_display_to_print {
-    ($($ty:ty),+) => {
-        // Some of the $ty arguments may not actually use 'tcx
-        $(#[allow(unused_lifetimes)] impl<'tcx> fmt::Display for $ty {
-            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-                ty::tls::with(|tcx| {
-                    let mut cx = FmtPrinter::new(tcx, Namespace::TypeNS);
-                    tcx.lift(*self)
-                        .expect("could not lift for printing")
-                        .print(&mut cx)?;
-                    f.write_str(&cx.into_buffer())?;
-                    Ok(())
-                })
-            }
-        })+
-    };
-}
-
-macro_rules! define_print_and_forward_display {
-    (($self:ident, $cx:ident): $($ty:ty $print:block)+) => {
-        define_print!(($self, $cx): $($ty $print)*);
-        forward_display_to_print!($($ty),+);
-    };
-}
-
-macro_rules! define_print {
-    (($self:ident, $cx:ident): $($ty:ty $print:block)+) => {
-        $(impl<'tcx, P: PrettyPrinter<'tcx>> Print<'tcx, P> for $ty {
-            fn print(&$self, $cx: &mut P) -> Result<(), PrintError> {
-                #[allow(unused_mut)]
-                let mut $cx = $cx;
-                define_scoped_cx!($cx);
-                let _: () = $print;
-                #[allow(unreachable_code)]
-                Ok(())
-            }
-        })+
-    };
-}
-
 /// Wrapper type for `ty::TraitRef` which opts-in to pretty printing only
 /// the trait path. That is, it will print `Trait<U>` instead of
 /// `<T as Trait<U>>`.
@@ -2737,6 +2697,46 @@ pub struct PrintClosureAsImpl<'tcx> {
     pub closure: ty::ClosureArgs<'tcx>,
 }
 
+macro_rules! forward_display_to_print {
+    ($($ty:ty),+) => {
+        // Some of the $ty arguments may not actually use 'tcx
+        $(#[allow(unused_lifetimes)] impl<'tcx> fmt::Display for $ty {
+            fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+                ty::tls::with(|tcx| {
+                    let mut cx = FmtPrinter::new(tcx, Namespace::TypeNS);
+                    tcx.lift(*self)
+                        .expect("could not lift for printing")
+                        .print(&mut cx)?;
+                    f.write_str(&cx.into_buffer())?;
+                    Ok(())
+                })
+            }
+        })+
+    };
+}
+
+macro_rules! define_print {
+    (($self:ident, $cx:ident): $($ty:ty $print:block)+) => {
+        $(impl<'tcx, P: PrettyPrinter<'tcx>> Print<'tcx, P> for $ty {
+            fn print(&$self, $cx: &mut P) -> Result<(), PrintError> {
+                #[allow(unused_mut)]
+                let mut $cx = $cx;
+                define_scoped_cx!($cx);
+                let _: () = $print;
+                #[allow(unreachable_code)]
+                Ok(())
+            }
+        })+
+    };
+}
+
+macro_rules! define_print_and_forward_display {
+    (($self:ident, $cx:ident): $($ty:ty $print:block)+) => {
+        define_print!(($self, $cx): $($ty $print)*);
+        forward_display_to_print!($($ty),+);
+    };
+}
+
 forward_display_to_print! {
     ty::Region<'tcx>,
     Ty<'tcx>,

From 4e2bbfef3eaddf399e33caeaa3976005855dc322 Mon Sep 17 00:00:00 2001
From: Nilstrieb <48135649+Nilstrieb@users.noreply.github.com>
Date: Sat, 28 Oct 2023 20:25:43 +0200
Subject: [PATCH 188/297] Remove needless `allow`s

---
 compiler/rustc_middle/src/ty/print/mod.rs    | 2 --
 compiler/rustc_middle/src/ty/print/pretty.rs | 3 ---
 2 files changed, 5 deletions(-)

diff --git a/compiler/rustc_middle/src/ty/print/mod.rs b/compiler/rustc_middle/src/ty/print/mod.rs
index 80af8a925539..6bbc8f70f515 100644
--- a/compiler/rustc_middle/src/ty/print/mod.rs
+++ b/compiler/rustc_middle/src/ty/print/mod.rs
@@ -12,8 +12,6 @@ pub use self::pretty::*;
 
 pub type PrintError = std::fmt::Error;
 
-// FIXME(eddyb) false positive, the lifetime parameters are used with `P:  Printer<...>`.
-#[allow(unused_lifetimes)]
 pub trait Print<'tcx, P> {
     fn print(&self, cx: &mut P) -> Result<(), PrintError>;
 }
diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index 0bdebc41842c..799ba2c35ec8 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -2719,11 +2719,8 @@ macro_rules! define_print {
     (($self:ident, $cx:ident): $($ty:ty $print:block)+) => {
         $(impl<'tcx, P: PrettyPrinter<'tcx>> Print<'tcx, P> for $ty {
             fn print(&$self, $cx: &mut P) -> Result<(), PrintError> {
-                #[allow(unused_mut)]
-                let mut $cx = $cx;
                 define_scoped_cx!($cx);
                 let _: () = $print;
-                #[allow(unreachable_code)]
                 Ok(())
             }
         })+

From 794bf8a850b3e83705845de398b51547d5366bb1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9my=20Rakic?= <remy.rakic+github@gmail.com>
Date: Sat, 28 Oct 2023 19:00:49 +0000
Subject: [PATCH 189/297] pass `CODEGEN_BACKENDS` to docker

---
 src/ci/docker/run.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh
index 99e3ce199f45..0b535532f695 100755
--- a/src/ci/docker/run.sh
+++ b/src/ci/docker/run.sh
@@ -288,6 +288,7 @@ docker \
   --env DIST_TRY_BUILD \
   --env PR_CI_JOB \
   --env OBJDIR_ON_HOST="$objdir" \
+  --env CODEGEN_BACKENDS \
   --init \
   --rm \
   rust-ci \

From aefd7ace91b58eb8cf00817de24a42ae2b3956d3 Mon Sep 17 00:00:00 2001
From: bjorn3 <17426603+bjorn3@users.noreply.github.com>
Date: Sat, 28 Oct 2023 19:11:51 +0000
Subject: [PATCH 190/297] Ensure the rustc-codegen-cranelift-preview component
 is never empty

Either generate it with the actual codegen backend dylib or omit it
entirely when the cranelift backend is disabled for this build.
---
 src/bootstrap/src/core/build_steps/dist.rs | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs
index b578c5ec2950..197ce78d8859 100644
--- a/src/bootstrap/src/core/build_steps/dist.rs
+++ b/src/bootstrap/src/core/build_steps/dist.rs
@@ -1305,6 +1305,10 @@ impl Step for CodegenBackend {
             return None;
         }
 
+        if !builder.config.rust_codegen_backends.contains(&self.backend) {
+            return None;
+        }
+
         if self.backend == "cranelift" {
             if !target_supports_cranelift_backend(self.compiler.host) {
                 builder.info("target not supported by rustc_codegen_cranelift. skipping");
@@ -1343,12 +1347,15 @@ impl Step for CodegenBackend {
         let backends_dst = PathBuf::from("lib").join(&backends_rel);
 
         let backend_name = format!("rustc_codegen_{}", backend);
+        let mut found_backend = false;
         for backend in fs::read_dir(&backends_src).unwrap() {
             let file_name = backend.unwrap().file_name();
             if file_name.to_str().unwrap().contains(&backend_name) {
                 tarball.add_file(backends_src.join(file_name), &backends_dst, 0o644);
+                found_backend = true;
             }
         }
+        assert!(found_backend);
 
         Some(tarball.generate())
     }

From 552abdce505a3aef1f25654697e4c1d695e11c63 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tomasz=20Mi=C4=85sko?= <tomasz.miasko@gmail.com>
Date: Sun, 29 Oct 2023 00:00:00 +0000
Subject: [PATCH 191/297] Rename a few remaining references to abort terminator

Follow up to e3f2edc75bf2becb57d7d770bba20606da1c4224
---
 compiler/rustc_borrowck/src/type_check/mod.rs | 2 +-
 compiler/rustc_middle/src/mir/pretty.rs       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs
index 9eb02be2f155..8477686d4f57 100644
--- a/compiler/rustc_borrowck/src/type_check/mod.rs
+++ b/compiler/rustc_borrowck/src/type_check/mod.rs
@@ -1640,7 +1640,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             }
             TerminatorKind::UnwindTerminate(_) => {
                 if !is_cleanup {
-                    span_mirbug!(self, block_data, "abort on non-cleanup block!")
+                    span_mirbug!(self, block_data, "terminate on non-cleanup block!")
                 }
             }
             TerminatorKind::Return => {
diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs
index 12057f5e1cb6..5f4ff22bc49c 100644
--- a/compiler/rustc_middle/src/mir/pretty.rs
+++ b/compiler/rustc_middle/src/mir/pretty.rs
@@ -785,7 +785,7 @@ impl<'tcx> TerminatorKind<'tcx> {
             CoroutineDrop => write!(fmt, "coroutine_drop"),
             UnwindResume => write!(fmt, "resume"),
             UnwindTerminate(reason) => {
-                write!(fmt, "abort({})", reason.as_short_str())
+                write!(fmt, "terminate({})", reason.as_short_str())
             }
             Yield { value, resume_arg, .. } => write!(fmt, "{resume_arg:?} = yield({value:?})"),
             Unreachable => write!(fmt, "unreachable"),

From 7a504cc68a56bfaa7855016c81ced9e6b1320fc5 Mon Sep 17 00:00:00 2001
From: git-bruh <e817509a-8ee9-4332-b0ad-3a6bdf9ab63f@aleeas.com>
Date: Tue, 19 Sep 2023 19:47:50 +0530
Subject: [PATCH 192/297] Don't use LFS64 symbols on musl

Simplify #[cfg] blocks

fmt

don't try to use the more appropriate direntry on musl
---
 library/std/src/os/linux/fs.rs |  9 ++++++++-
 library/std/src/sys/unix/fd.rs | 24 ++++++++++++++++++++----
 library/std/src/sys/unix/fs.rs | 23 ++++++++++++++---------
 3 files changed, 42 insertions(+), 14 deletions(-)

diff --git a/library/std/src/os/linux/fs.rs b/library/std/src/os/linux/fs.rs
index 479bbcc17a89..ab0b2a3eda3f 100644
--- a/library/std/src/os/linux/fs.rs
+++ b/library/std/src/os/linux/fs.rs
@@ -329,7 +329,14 @@ pub trait MetadataExt {
 impl MetadataExt for Metadata {
     #[allow(deprecated)]
     fn as_raw_stat(&self) -> &raw::stat {
-        unsafe { &*(self.as_inner().as_inner() as *const libc::stat64 as *const raw::stat) }
+        #[cfg(target_env = "musl")]
+        unsafe {
+            &*(self.as_inner().as_inner() as *const libc::stat as *const raw::stat)
+        }
+        #[cfg(not(target_env = "musl"))]
+        unsafe {
+            &*(self.as_inner().as_inner() as *const libc::stat64 as *const raw::stat)
+        }
     }
     fn st_dev(&self) -> u64 {
         self.as_inner().as_inner().st_dev as u64
diff --git a/library/std/src/sys/unix/fd.rs b/library/std/src/sys/unix/fd.rs
index 6c4f408426a9..bf1fb3123c4c 100644
--- a/library/std/src/sys/unix/fd.rs
+++ b/library/std/src/sys/unix/fd.rs
@@ -126,9 +126,17 @@ impl FileDesc {
     }
 
     pub fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result<usize> {
-        #[cfg(not(any(target_os = "linux", target_os = "android", target_os = "hurd")))]
+        #[cfg(not(any(
+            all(target_os = "linux", not(target_env = "musl")),
+            target_os = "android",
+            target_os = "hurd"
+        )))]
         use libc::pread as pread64;
-        #[cfg(any(target_os = "linux", target_os = "android", target_os = "hurd"))]
+        #[cfg(any(
+            all(target_os = "linux", not(target_env = "musl")),
+            target_os = "android",
+            target_os = "hurd"
+        ))]
         use libc::pread64;
 
         unsafe {
@@ -285,9 +293,17 @@ impl FileDesc {
     }
 
     pub fn write_at(&self, buf: &[u8], offset: u64) -> io::Result<usize> {
-        #[cfg(not(any(target_os = "linux", target_os = "android", target_os = "hurd")))]
+        #[cfg(not(any(
+            all(target_os = "linux", not(target_env = "musl")),
+            target_os = "android",
+            target_os = "hurd"
+        )))]
         use libc::pwrite as pwrite64;
-        #[cfg(any(target_os = "linux", target_os = "android", target_os = "hurd"))]
+        #[cfg(any(
+            all(target_os = "linux", not(target_env = "musl")),
+            target_os = "android",
+            target_os = "hurd"
+        ))]
         use libc::pwrite64;
 
         unsafe {
diff --git a/library/std/src/sys/unix/fs.rs b/library/std/src/sys/unix/fs.rs
index e1c58f2ba3c3..40eb910fdc38 100644
--- a/library/std/src/sys/unix/fs.rs
+++ b/library/std/src/sys/unix/fs.rs
@@ -40,13 +40,17 @@ use libc::{c_int, mode_t};
 ))]
 use libc::c_char;
 #[cfg(any(
-    target_os = "linux",
+    all(target_os = "linux", not(target_env = "musl")),
     target_os = "emscripten",
     target_os = "android",
-    target_os = "hurd",
+    target_os = "hurd"
 ))]
 use libc::dirfd;
-#[cfg(any(target_os = "linux", target_os = "emscripten", target_os = "hurd"))]
+#[cfg(any(
+    all(target_os = "linux", not(target_env = "musl")),
+    target_os = "emscripten",
+    target_os = "hurd"
+))]
 use libc::fstatat64;
 #[cfg(any(
     target_os = "android",
@@ -57,9 +61,10 @@ use libc::fstatat64;
     target_os = "aix",
     target_os = "nto",
     target_os = "vita",
+    all(target_os = "linux", target_env = "musl"),
 ))]
 use libc::readdir as readdir64;
-#[cfg(any(target_os = "linux", target_os = "hurd"))]
+#[cfg(any(all(target_os = "linux", not(target_env = "musl")), target_os = "hurd"))]
 use libc::readdir64;
 #[cfg(any(target_os = "emscripten", target_os = "l4re"))]
 use libc::readdir64_r;
@@ -84,7 +89,7 @@ use libc::{
     lstat as lstat64, off64_t, open as open64, stat as stat64,
 };
 #[cfg(not(any(
-    target_os = "linux",
+    all(target_os = "linux", not(target_env = "musl")),
     target_os = "emscripten",
     target_os = "l4re",
     target_os = "android",
@@ -95,7 +100,7 @@ use libc::{
     lstat as lstat64, off_t as off64_t, open as open64, stat as stat64,
 };
 #[cfg(any(
-    target_os = "linux",
+    all(target_os = "linux", not(target_env = "musl")),
     target_os = "emscripten",
     target_os = "l4re",
     target_os = "hurd"
@@ -853,10 +858,10 @@ impl DirEntry {
 
     #[cfg(all(
         any(
-            target_os = "linux",
+            all(target_os = "linux", not(target_env = "musl")),
             target_os = "emscripten",
             target_os = "android",
-            target_os = "hurd",
+            target_os = "hurd"
         ),
         not(miri)
     ))]
@@ -882,7 +887,7 @@ impl DirEntry {
 
     #[cfg(any(
         not(any(
-            target_os = "linux",
+            all(target_os = "linux", not(target_env = "musl")),
             target_os = "emscripten",
             target_os = "android",
             target_os = "hurd",

From 3784adcd64bb6a2a871dcb47de527cd6aa0953e4 Mon Sep 17 00:00:00 2001
From: Noah Lev <camelidcamel@gmail.com>
Date: Sun, 29 Oct 2023 01:44:12 -0400
Subject: [PATCH 193/297] rustdoc: Use `ThinVec` in `GenericParamDefKind`

This should hopefully reduce memory usage and improve performance since
these vectors are often empty (and `GenericParamDefKind` is constructed
*a lot*).
---
 src/librustdoc/clean/mod.rs      | 12 ++++++------
 src/librustdoc/clean/simplify.rs |  2 +-
 src/librustdoc/clean/types.rs    |  8 ++++----
 3 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/librustdoc/clean/mod.rs b/src/librustdoc/clean/mod.rs
index f2447b877ca8..4d4c102e69ea 100644
--- a/src/librustdoc/clean/mod.rs
+++ b/src/librustdoc/clean/mod.rs
@@ -498,7 +498,7 @@ fn clean_generic_param_def<'tcx>(
 ) -> GenericParamDef {
     let (name, kind) = match def.kind {
         ty::GenericParamDefKind::Lifetime => {
-            (def.name, GenericParamDefKind::Lifetime { outlives: vec![] })
+            (def.name, GenericParamDefKind::Lifetime { outlives: ThinVec::new() })
         }
         ty::GenericParamDefKind::Type { has_default, synthetic, .. } => {
             let default = if has_default {
@@ -515,7 +515,7 @@ fn clean_generic_param_def<'tcx>(
                 def.name,
                 GenericParamDefKind::Type {
                     did: def.def_id,
-                    bounds: vec![], // These are filled in from the where-clauses.
+                    bounds: ThinVec::new(), // These are filled in from the where-clauses.
                     default: default.map(Box::new),
                     synthetic,
                 },
@@ -567,7 +567,7 @@ fn clean_generic_param<'tcx>(
                     })
                     .collect()
             } else {
-                Vec::new()
+                ThinVec::new()
             };
             (param.name.ident().name, GenericParamDefKind::Lifetime { outlives })
         }
@@ -580,7 +580,7 @@ fn clean_generic_param<'tcx>(
                     .filter_map(|x| clean_generic_bound(x, cx))
                     .collect()
             } else {
-                Vec::new()
+                ThinVec::new()
             };
             (
                 param.name.ident().name,
@@ -636,7 +636,7 @@ pub(crate) fn clean_generics<'tcx>(
             match param.kind {
                 GenericParamDefKind::Lifetime { .. } => unreachable!(),
                 GenericParamDefKind::Type { did, ref bounds, .. } => {
-                    cx.impl_trait_bounds.insert(did.into(), bounds.clone());
+                    cx.impl_trait_bounds.insert(did.into(), bounds.to_vec());
                 }
                 GenericParamDefKind::Const { .. } => unreachable!(),
             }
@@ -3146,7 +3146,7 @@ fn clean_bound_vars<'tcx>(
                 name,
                 kind: GenericParamDefKind::Type {
                     did,
-                    bounds: Vec::new(),
+                    bounds: ThinVec::new(),
                     default: None,
                     synthetic: false,
                 },
diff --git a/src/librustdoc/clean/simplify.rs b/src/librustdoc/clean/simplify.rs
index eea8c1906c5b..627f15e67ac3 100644
--- a/src/librustdoc/clean/simplify.rs
+++ b/src/librustdoc/clean/simplify.rs
@@ -145,7 +145,7 @@ pub(crate) fn move_bounds_to_generic_parameters(generics: &mut clean::Generics)
                 ..
             }) = generics.params.iter_mut().find(|param| &param.name == arg)
         {
-            param_bounds.append(bounds);
+            param_bounds.extend(bounds.drain(..));
         } else if let WherePredicate::RegionPredicate { lifetime: Lifetime(arg), bounds } = &mut pred
             && let Some(GenericParamDef {
                 kind: GenericParamDefKind::Lifetime { outlives: param_bounds },
diff --git a/src/librustdoc/clean/types.rs b/src/librustdoc/clean/types.rs
index 449aac4cfc8b..455228d04efa 100644
--- a/src/librustdoc/clean/types.rs
+++ b/src/librustdoc/clean/types.rs
@@ -1325,8 +1325,8 @@ impl WherePredicate {
 
 #[derive(Clone, PartialEq, Eq, Debug, Hash)]
 pub(crate) enum GenericParamDefKind {
-    Lifetime { outlives: Vec<Lifetime> },
-    Type { did: DefId, bounds: Vec<GenericBound>, default: Option<Box<Type>>, synthetic: bool },
+    Lifetime { outlives: ThinVec<Lifetime> },
+    Type { did: DefId, bounds: ThinVec<GenericBound>, default: Option<Box<Type>>, synthetic: bool },
     Const { ty: Box<Type>, default: Option<Box<String>>, is_host_effect: bool },
 }
 
@@ -1344,7 +1344,7 @@ pub(crate) struct GenericParamDef {
 
 impl GenericParamDef {
     pub(crate) fn lifetime(name: Symbol) -> Self {
-        Self { name, kind: GenericParamDefKind::Lifetime { outlives: Vec::new() } }
+        Self { name, kind: GenericParamDefKind::Lifetime { outlives: ThinVec::new() } }
     }
 
     pub(crate) fn is_synthetic_param(&self) -> bool {
@@ -2521,7 +2521,7 @@ mod size_asserts {
     static_assert_size!(DocFragment, 32);
     static_assert_size!(GenericArg, 32);
     static_assert_size!(GenericArgs, 32);
-    static_assert_size!(GenericParamDef, 56);
+    static_assert_size!(GenericParamDef, 40);
     static_assert_size!(Generics, 16);
     static_assert_size!(Item, 56);
     static_assert_size!(ItemKind, 56);

From 82b447a0cc47c5b476fb85eaeba865c97490f93f Mon Sep 17 00:00:00 2001
From: Havard Eidnes <he@NetBSD.org>
Date: Sun, 29 Oct 2023 12:39:30 +0000
Subject: [PATCH 194/297] Add support for mipsel-unknown-netbsd, 32-bit LE
 mips.

---
 .../src/spec/mipsel_unknown_netbsd.rs         | 21 +++++++++++++++++++
 compiler/rustc_target/src/spec/mod.rs         |  1 +
 src/bootstrap/src/core/build_steps/llvm.rs    |  6 ++++++
 src/doc/rustc/src/platform-support.md         |  1 +
 4 files changed, 29 insertions(+)
 create mode 100644 compiler/rustc_target/src/spec/mipsel_unknown_netbsd.rs

diff --git a/compiler/rustc_target/src/spec/mipsel_unknown_netbsd.rs b/compiler/rustc_target/src/spec/mipsel_unknown_netbsd.rs
new file mode 100644
index 000000000000..74b324d57a08
--- /dev/null
+++ b/compiler/rustc_target/src/spec/mipsel_unknown_netbsd.rs
@@ -0,0 +1,21 @@
+use crate::abi::Endian;
+use crate::spec::{Target, TargetOptions};
+
+pub fn target() -> Target {
+    let mut base = super::netbsd_base::opts();
+    base.max_atomic_width = Some(32);
+    base.cpu = "mips32".into();
+
+    Target {
+       llvm_target: "mipsel-unknown-netbsd".into(),
+       pointer_width: 32,
+       data_layout: "e-m:m-p:32:32-i8:8:32-i16:16:32-i64:64-n32-S64".into(),
+       arch: "mips".into(),
+       options: TargetOptions {
+           features: "+soft-float".into(),
+           mcount: "__mcount".into(),
+           endian: Endian::Little,
+           ..base
+       },
+    }
+}
diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs
index ccf297615521..48ca5fcf3ad0 100644
--- a/compiler/rustc_target/src/spec/mod.rs
+++ b/compiler/rustc_target/src/spec/mod.rs
@@ -1566,6 +1566,7 @@ supported_targets! {
     ("armv7-unknown-netbsd-eabihf", armv7_unknown_netbsd_eabihf),
     ("i586-unknown-netbsd", i586_unknown_netbsd),
     ("i686-unknown-netbsd", i686_unknown_netbsd),
+    ("mipsel-unknown-netbsd", mipsel_unknown_netbsd),
     ("powerpc-unknown-netbsd", powerpc_unknown_netbsd),
     ("riscv64gc-unknown-netbsd", riscv64gc_unknown_netbsd),
     ("sparc64-unknown-netbsd", sparc64_unknown_netbsd),
diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs
index 24351118a5aa..5b4543077216 100644
--- a/src/bootstrap/src/core/build_steps/llvm.rs
+++ b/src/bootstrap/src/core/build_steps/llvm.rs
@@ -396,6 +396,12 @@ impl Step for Llvm {
             ldflags.shared.push(" -latomic");
         }
 
+        if target.starts_with("mips") && target.contains("netbsd") {
+            // LLVM wants 64-bit atomics, while mipsel is 32-bit only, so needs -latomic
+            ldflags.exe.push(" -latomic");
+            ldflags.shared.push(" -latomic");
+        }
+
         if target.contains("msvc") {
             cfg.define("LLVM_USE_CRT_DEBUG", "MT");
             cfg.define("LLVM_USE_CRT_RELEASE", "MT");
diff --git a/src/doc/rustc/src/platform-support.md b/src/doc/rustc/src/platform-support.md
index 68ec0f348dc5..6a979869a59a 100644
--- a/src/doc/rustc/src/platform-support.md
+++ b/src/doc/rustc/src/platform-support.md
@@ -287,6 +287,7 @@ target | std | host | notes
 `mips64el-unknown-linux-muslabi64` | ✓ |  | MIPS64 (little endian) Linux, N64 ABI, musl libc
 `mipsel-unknown-linux-gnu` | ✓ | ✓ | MIPS (little endian) Linux (kernel 4.4, glibc 2.23)
 `mipsel-unknown-linux-musl` | ✓ |  | MIPS (little endian) Linux with musl libc
+[`mipsel-unknown-netbsd`](platform-support/netbsd.md) | ✓ | ✓ | 32-bit MIPS (LE), requires mips32 cpu support
 `mipsel-sony-psp` | * |  | MIPS (LE) Sony PlayStation Portable (PSP)
 [`mipsel-sony-psx`](platform-support/mipsel-sony-psx.md) | * |  | MIPS (LE) Sony PlayStation 1 (PSX)
 `mipsel-unknown-linux-uclibc` | ✓ |  | MIPS (LE) Linux with uClibc

From f656accdf534bc58a021e2d7f04bf5900e2ba8ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9my=20Rakic?= <remy.rakic+github@gmail.com>
Date: Sun, 29 Oct 2023 13:12:02 +0000
Subject: [PATCH 195/297] handle dry runs in `dist::CodegenBackend`

self.number_of_times_dry_runs_have_caused_issues += 1;
---
 src/bootstrap/src/core/build_steps/dist.rs | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs
index 197ce78d8859..6e80c55c8ce3 100644
--- a/src/bootstrap/src/core/build_steps/dist.rs
+++ b/src/bootstrap/src/core/build_steps/dist.rs
@@ -1298,6 +1298,10 @@ impl Step for CodegenBackend {
     }
 
     fn run(self, builder: &Builder<'_>) -> Option<GeneratedTarball> {
+        if builder.config.dry_run() {
+            return None;
+        }
+
         // This prevents rustc_codegen_cranelift from being built for "dist"
         // or "install" on the stable/beta channels. It is not yet stable and
         // should not be included.

From 2e7364a58613a8465763b068785bf5655846483f Mon Sep 17 00:00:00 2001
From: Ben Kimock <kimockb@gmail.com>
Date: Wed, 25 Oct 2023 19:46:06 -0400
Subject: [PATCH 196/297] Increase the reach of panic_immediate_abort

---
 library/alloc/Cargo.toml              |  2 +
 library/alloc/src/alloc.rs            |  9 +++-
 library/alloc/src/raw_vec.rs          |  2 +-
 library/alloc/src/vec/mod.rs          | 11 +++--
 library/core/src/cell.rs              |  4 +-
 library/core/src/panic/unwind_safe.rs |  1 +
 library/core/src/str/mod.rs           |  6 +++
 library/std/Cargo.toml                |  2 +-
 library/std/src/panicking.rs          | 60 ++++++++++++++++++++++++++-
 9 files changed, 87 insertions(+), 10 deletions(-)

diff --git a/library/alloc/Cargo.toml b/library/alloc/Cargo.toml
index e5f828c4c0b3..63aec14f4810 100644
--- a/library/alloc/Cargo.toml
+++ b/library/alloc/Cargo.toml
@@ -36,3 +36,5 @@ compiler-builtins-c = ["compiler_builtins/c"]
 compiler-builtins-no-asm = ["compiler_builtins/no-asm"]
 compiler-builtins-mangled-names = ["compiler_builtins/mangled-names"]
 compiler-builtins-weak-intrinsics = ["compiler_builtins/weak-intrinsics"]
+# Make panics and failed asserts immediately abort without formatting any message
+panic_immediate_abort = []
diff --git a/library/alloc/src/alloc.rs b/library/alloc/src/alloc.rs
index a548de814c53..2499f1053d86 100644
--- a/library/alloc/src/alloc.rs
+++ b/library/alloc/src/alloc.rs
@@ -377,13 +377,20 @@ pub const fn handle_alloc_error(layout: Layout) -> ! {
         panic!("allocation failed");
     }
 
+    #[inline]
     fn rt_error(layout: Layout) -> ! {
         unsafe {
             __rust_alloc_error_handler(layout.size(), layout.align());
         }
     }
 
-    unsafe { core::intrinsics::const_eval_select((layout,), ct_error, rt_error) }
+    #[cfg(not(feature = "panic_immediate_abort"))]
+    unsafe {
+        core::intrinsics::const_eval_select((layout,), ct_error, rt_error)
+    }
+
+    #[cfg(feature = "panic_immediate_abort")]
+    ct_error(layout)
 }
 
 // For alloc test `std::alloc::handle_alloc_error` can be used directly.
diff --git a/library/alloc/src/raw_vec.rs b/library/alloc/src/raw_vec.rs
index 625b67a79ad0..bd66ad612196 100644
--- a/library/alloc/src/raw_vec.rs
+++ b/library/alloc/src/raw_vec.rs
@@ -530,7 +530,7 @@ fn alloc_guard(alloc_size: usize) -> Result<(), TryReserveError> {
 // ensure that the code generation related to these panics is minimal as there's
 // only one location which panics rather than a bunch throughout the module.
 #[cfg(not(no_global_oom_handling))]
-#[inline(never)]
+#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never))]
 fn capacity_overflow() -> ! {
     panic!("capacity overflow");
 }
diff --git a/library/alloc/src/vec/mod.rs b/library/alloc/src/vec/mod.rs
index 3b12c1bee0b7..6c78d65f1c94 100644
--- a/library/alloc/src/vec/mod.rs
+++ b/library/alloc/src/vec/mod.rs
@@ -1447,7 +1447,8 @@ impl<T, A: Allocator> Vec<T, A> {
     #[stable(feature = "rust1", since = "1.0.0")]
     pub fn swap_remove(&mut self, index: usize) -> T {
         #[cold]
-        #[inline(never)]
+        #[cfg_attr(not(feature = "panic_immediate_abort"), inline(never))]
+        #[track_caller]
         fn assert_failed(index: usize, len: usize) -> ! {
             panic!("swap_remove index (is {index}) should be < len (is {len})");
         }
@@ -1488,7 +1489,8 @@ impl<T, A: Allocator> Vec<T, A> {
     #[stable(feature = "rust1", since = "1.0.0")]
     pub fn insert(&mut self, index: usize, element: T) {
         #[cold]
-        #[inline(never)]
+        #[cfg_attr(not(feature = "panic_immediate_abort"), inline(never))]
+        #[track_caller]
         fn assert_failed(index: usize, len: usize) -> ! {
             panic!("insertion index (is {index}) should be <= len (is {len})");
         }
@@ -1549,7 +1551,7 @@ impl<T, A: Allocator> Vec<T, A> {
     #[track_caller]
     pub fn remove(&mut self, index: usize) -> T {
         #[cold]
-        #[inline(never)]
+        #[cfg_attr(not(feature = "panic_immediate_abort"), inline(never))]
         #[track_caller]
         fn assert_failed(index: usize, len: usize) -> ! {
             panic!("removal index (is {index}) should be < len (is {len})");
@@ -2148,7 +2150,8 @@ impl<T, A: Allocator> Vec<T, A> {
         A: Clone,
     {
         #[cold]
-        #[inline(never)]
+        #[cfg_attr(not(feature = "panic_immediate_abort"), inline(never))]
+        #[track_caller]
         fn assert_failed(at: usize, len: usize) -> ! {
             panic!("`at` split index (is {at}) should be <= len (is {len})");
         }
diff --git a/library/core/src/cell.rs b/library/core/src/cell.rs
index f3d8b54d4e94..0978b3c92805 100644
--- a/library/core/src/cell.rs
+++ b/library/core/src/cell.rs
@@ -755,7 +755,7 @@ impl Display for BorrowMutError {
 }
 
 // This ensures the panicking code is outlined from `borrow_mut` for `RefCell`.
-#[inline(never)]
+#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never))]
 #[track_caller]
 #[cold]
 fn panic_already_borrowed(err: BorrowMutError) -> ! {
@@ -763,7 +763,7 @@ fn panic_already_borrowed(err: BorrowMutError) -> ! {
 }
 
 // This ensures the panicking code is outlined from `borrow` for `RefCell`.
-#[inline(never)]
+#[cfg_attr(not(feature = "panic_immediate_abort"), inline(never))]
 #[track_caller]
 #[cold]
 fn panic_already_mutably_borrowed(err: BorrowError) -> ! {
diff --git a/library/core/src/panic/unwind_safe.rs b/library/core/src/panic/unwind_safe.rs
index 7e7b6b4dbe9b..6a53909a8f12 100644
--- a/library/core/src/panic/unwind_safe.rs
+++ b/library/core/src/panic/unwind_safe.rs
@@ -267,6 +267,7 @@ impl<T> DerefMut for AssertUnwindSafe<T> {
 impl<R, F: FnOnce() -> R> FnOnce<()> for AssertUnwindSafe<F> {
     type Output = R;
 
+    #[inline]
     extern "rust-call" fn call_once(self, _args: ()) -> R {
         (self.0)()
     }
diff --git a/library/core/src/str/mod.rs b/library/core/src/str/mod.rs
index 6aba9b645094..27178328be5c 100644
--- a/library/core/src/str/mod.rs
+++ b/library/core/src/str/mod.rs
@@ -81,6 +81,7 @@ use iter::{MatchesInternal, SplitNInternal};
 #[cold]
 #[track_caller]
 #[rustc_allow_const_fn_unstable(const_eval_select)]
+#[cfg(not(feature = "panic_immediate_abort"))]
 const fn slice_error_fail(s: &str, begin: usize, end: usize) -> ! {
     // SAFETY: panics for both branches
     unsafe {
@@ -92,6 +93,11 @@ const fn slice_error_fail(s: &str, begin: usize, end: usize) -> ! {
     }
 }
 
+#[cfg(feature = "panic_immediate_abort")]
+const fn slice_error_fail(s: &str, begin: usize, end: usize) -> ! {
+    slice_error_fail_ct(s, begin, end)
+}
+
 #[track_caller]
 const fn slice_error_fail_ct(_: &str, _: usize, _: usize) -> ! {
     panic!("failed to slice string");
diff --git a/library/std/Cargo.toml b/library/std/Cargo.toml
index 3b57f864e20d..2eb7608c7ae6 100644
--- a/library/std/Cargo.toml
+++ b/library/std/Cargo.toml
@@ -72,7 +72,7 @@ llvm-libunwind = ["unwind/llvm-libunwind"]
 system-llvm-libunwind = ["unwind/system-llvm-libunwind"]
 
 # Make panics and failed asserts immediately abort without formatting any message
-panic_immediate_abort = ["core/panic_immediate_abort"]
+panic_immediate_abort = ["core/panic_immediate_abort", "alloc/panic_immediate_abort"]
 
 # Enable std_detect default features for stdarch/crates/std_detect:
 # https://github.com/rust-lang/stdarch/blob/master/crates/std_detect/Cargo.toml
diff --git a/library/std/src/panicking.rs b/library/std/src/panicking.rs
index d7a2baa1ff57..55f4917a9370 100644
--- a/library/std/src/panicking.rs
+++ b/library/std/src/panicking.rs
@@ -295,12 +295,53 @@ fn default_hook(info: &PanicInfo<'_>) {
 
 #[cfg(not(test))]
 #[doc(hidden)]
+#[cfg(feature = "panic_immediate_abort")]
+#[unstable(feature = "update_panic_count", issue = "none")]
+pub mod panic_count {
+    /// A reason for forcing an immediate abort on panic.
+    #[derive(Debug)]
+    pub enum MustAbort {
+        AlwaysAbort,
+        PanicInHook,
+    }
+
+    #[inline]
+    pub fn increase(run_panic_hook: bool) -> Option<MustAbort> {
+        None
+    }
+
+    #[inline]
+    pub fn finished_panic_hook() {}
+
+    #[inline]
+    pub fn decrease() {}
+
+    #[inline]
+    pub fn set_always_abort() {}
+
+    // Disregards ALWAYS_ABORT_FLAG
+    #[inline]
+    #[must_use]
+    pub fn get_count() -> usize {
+        0
+    }
+
+    #[must_use]
+    #[inline]
+    pub fn count_is_zero() -> bool {
+        true
+    }
+}
+
+#[cfg(not(test))]
+#[doc(hidden)]
+#[cfg(not(feature = "panic_immediate_abort"))]
 #[unstable(feature = "update_panic_count", issue = "none")]
 pub mod panic_count {
     use crate::cell::Cell;
     use crate::sync::atomic::{AtomicUsize, Ordering};
 
-    pub const ALWAYS_ABORT_FLAG: usize = 1 << (usize::BITS - 1);
+    const ALWAYS_ABORT_FLAG: usize = 1 << (usize::BITS - 1);
 
     /// A reason for forcing an immediate abort on panic.
     #[derive(Debug)]
@@ -421,6 +462,13 @@ pub mod panic_count {
 pub use realstd::rt::panic_count;
 
 /// Invoke a closure, capturing the cause of an unwinding panic if one occurs.
+#[cfg(feature = "panic_immediate_abort")]
+pub unsafe fn r#try<R, F: FnOnce() -> R>(f: F) -> Result<R, Box<dyn Any + Send>> {
+    Ok(f())
+}
+
+/// Invoke a closure, capturing the cause of an unwinding panic if one occurs.
+#[cfg(not(feature = "panic_immediate_abort"))]
 pub unsafe fn r#try<R, F: FnOnce() -> R>(f: F) -> Result<R, Box<dyn Any + Send>> {
     union Data<F, R> {
         f: ManuallyDrop<F>,
@@ -755,6 +803,7 @@ fn rust_panic_with_hook(
 
 /// This is the entry point for `resume_unwind`.
 /// It just forwards the payload to the panic runtime.
+#[cfg_attr(feature = "panic_immediate_abort", inline)]
 pub fn rust_panic_without_hook(payload: Box<dyn Any + Send>) -> ! {
     panic_count::increase(false);
 
@@ -777,7 +826,16 @@ pub fn rust_panic_without_hook(payload: Box<dyn Any + Send>) -> ! {
 /// yer breakpoints.
 #[inline(never)]
 #[cfg_attr(not(test), rustc_std_internal_symbol)]
+#[cfg(not(feature = "panic_immediate_abort"))]
 fn rust_panic(msg: &mut dyn PanicPayload) -> ! {
     let code = unsafe { __rust_start_panic(msg) };
     rtabort!("failed to initiate panic, error {code}")
 }
+
+#[cfg_attr(not(test), rustc_std_internal_symbol)]
+#[cfg(feature = "panic_immediate_abort")]
+fn rust_panic(_: &mut dyn PanicPayload) -> ! {
+    unsafe {
+        crate::intrinsics::abort();
+    }
+}

From 5e6c313cafd636bf54973d64875ebeba635dc612 Mon Sep 17 00:00:00 2001
From: Havard Eidnes <he@NetBSD.org>
Date: Sun, 29 Oct 2023 13:53:24 +0000
Subject: [PATCH 197/297] mipsel_unknown_netbsd.rs: fix indentation.

---
 .../src/spec/mipsel_unknown_netbsd.rs         | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/compiler/rustc_target/src/spec/mipsel_unknown_netbsd.rs b/compiler/rustc_target/src/spec/mipsel_unknown_netbsd.rs
index 74b324d57a08..651358d6422c 100644
--- a/compiler/rustc_target/src/spec/mipsel_unknown_netbsd.rs
+++ b/compiler/rustc_target/src/spec/mipsel_unknown_netbsd.rs
@@ -7,15 +7,15 @@ pub fn target() -> Target {
     base.cpu = "mips32".into();
 
     Target {
-       llvm_target: "mipsel-unknown-netbsd".into(),
-       pointer_width: 32,
-       data_layout: "e-m:m-p:32:32-i8:8:32-i16:16:32-i64:64-n32-S64".into(),
-       arch: "mips".into(),
-       options: TargetOptions {
-           features: "+soft-float".into(),
-           mcount: "__mcount".into(),
-           endian: Endian::Little,
-           ..base
-       },
+        llvm_target: "mipsel-unknown-netbsd".into(),
+        pointer_width: 32,
+        data_layout: "e-m:m-p:32:32-i8:8:32-i16:16:32-i64:64-n32-S64".into(),
+        arch: "mips".into(),
+        options: TargetOptions {
+            features: "+soft-float".into(),
+            mcount: "__mcount".into(),
+            endian: Endian::Little,
+            ..base
+        },
     }
 }

From 827a6d8d9f0b84fec8fd15c336d845a05435889d Mon Sep 17 00:00:00 2001
From: bjorn3 <17426603+bjorn3@users.noreply.github.com>
Date: Sun, 29 Oct 2023 16:06:32 +0000
Subject: [PATCH 198/297] Use the LLVM rustc backend as external assembler

The LLVM backend is generally available, while the gnu assembler is not
on Windows and many other platforms by default.
---
 src/global_asm.rs | 70 +++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 58 insertions(+), 12 deletions(-)

diff --git a/src/global_asm.rs b/src/global_asm.rs
index 6692d1b85eac..7c2138bf0cfd 100644
--- a/src/global_asm.rs
+++ b/src/global_asm.rs
@@ -134,20 +134,66 @@ pub(crate) fn compile_global_asm(
         .join("\n");
     global_asm.push('\n');
 
-    let output_object_file = config.output_filenames.temp_path(OutputType::Object, Some(cgu_name));
+    let global_asm_object_file = add_file_stem_postfix(
+        config.output_filenames.temp_path(OutputType::Object, Some(cgu_name)),
+        ".asm",
+    );
 
     // Assemble `global_asm`
-    let global_asm_object_file = add_file_stem_postfix(output_object_file, ".asm");
-    let mut child = Command::new(&config.assembler)
-        .arg("-o")
-        .arg(&global_asm_object_file)
-        .stdin(Stdio::piped())
-        .spawn()
-        .expect("Failed to spawn `as`.");
-    child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap();
-    let status = child.wait().expect("Failed to wait for `as`.");
-    if !status.success() {
-        return Err(format!("Failed to assemble `{}`", global_asm));
+    if false {
+        let mut child = Command::new(&config.assembler)
+            .arg("-o")
+            .arg(&global_asm_object_file)
+            .stdin(Stdio::piped())
+            .spawn()
+            .expect("Failed to spawn `as`.");
+        child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap();
+        let status = child.wait().expect("Failed to wait for `as`.");
+        if !status.success() {
+            return Err(format!("Failed to assemble `{}`", global_asm));
+        }
+    } else {
+        let mut child = Command::new(std::env::current_exe().unwrap())
+            .arg("--target")
+            .arg(&config.target)
+            .arg("--crate-type")
+            .arg("staticlib")
+            .arg("--emit")
+            .arg("obj")
+            .arg("-o")
+            .arg(&global_asm_object_file)
+            .arg("-")
+            .arg("-Abad_asm_style")
+            .stdin(Stdio::piped())
+            .spawn()
+            .expect("Failed to spawn `as`.");
+        let mut stdin = child.stdin.take().unwrap();
+        stdin
+            .write_all(
+                br####"
+                #![feature(decl_macro, no_core, rustc_attrs)]
+                #![allow(internal_features)]
+                #![no_core]
+                #[rustc_builtin_macro]
+                #[rustc_macro_transparency = "semitransparent"]
+                macro global_asm() { /* compiler built-in */ }
+                global_asm!(r###"
+                "####,
+            )
+            .unwrap();
+        stdin.write_all(global_asm.as_bytes()).unwrap();
+        stdin
+            .write_all(
+                br####"
+                "###);
+                "####,
+            )
+            .unwrap();
+        std::mem::drop(stdin);
+        let status = child.wait().expect("Failed to wait for `as`.");
+        if !status.success() {
+            return Err(format!("Failed to assemble `{}`", global_asm));
+        }
     }
 
     Ok(Some(global_asm_object_file))

From b1e705146a26f5b650360edf2ecf05fec1315c33 Mon Sep 17 00:00:00 2001
From: bjorn3 <17426603+bjorn3@users.noreply.github.com>
Date: Sun, 29 Oct 2023 16:08:16 +0000
Subject: [PATCH 199/297] Unconditionally handle int $$0x29 using a shim

---
 src/inline_asm.rs | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/inline_asm.rs b/src/inline_asm.rs
index 0517c609337b..b2e27399055e 100644
--- a/src/inline_asm.rs
+++ b/src/inline_asm.rs
@@ -45,6 +45,14 @@ pub(crate) fn codegen_inline_asm<'tcx>(
 ) {
     // FIXME add .eh_frame unwind info directives
 
+    // Used by panic_abort on Windows, but uses a syntax which only happens to work with
+    // asm!() by accident and breaks with the GNU assembler as well as global_asm!() for
+    // the LLVM backend.
+    if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) {
+        fx.bcx.ins().trap(TrapCode::User(1));
+        return;
+    }
+
     if !asm_supported(fx.tcx) {
         if template.is_empty() {
             let destination_block = fx.get_block(destination.unwrap());
@@ -52,12 +60,6 @@ pub(crate) fn codegen_inline_asm<'tcx>(
             return;
         }
 
-        // Used by panic_abort
-        if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) {
-            fx.bcx.ins().trap(TrapCode::User(1));
-            return;
-        }
-
         // Used by stdarch
         if template[0] == InlineAsmTemplatePiece::String("mov ".to_string())
             && matches!(

From 35453ac0eb2fc906181903ae32a41df070f11458 Mon Sep 17 00:00:00 2001
From: bjorn3 <17426603+bjorn3@users.noreply.github.com>
Date: Sun, 29 Oct 2023 16:09:39 +0000
Subject: [PATCH 200/297] Stabilize inline asm usage on all platforms

But exclude sym operands for now as they are somewhat broken.
---
 Cargo.toml              |   4 +-
 src/global_asm.rs       |  42 ++++----
 src/inline_asm.rs       | 206 ++--------------------------------------
 src/intrinsics/cpuid.rs |  74 ---------------
 src/intrinsics/mod.rs   |   2 -
 5 files changed, 29 insertions(+), 299 deletions(-)
 delete mode 100644 src/intrinsics/cpuid.rs

diff --git a/Cargo.toml b/Cargo.toml
index f2ce714e8ff1..f422d53380fe 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -35,9 +35,9 @@ smallvec = "1.8.1"
 
 [features]
 # Enable features not ready to be enabled when compiling as part of rustc
-unstable-features = ["jit", "inline_asm"]
+unstable-features = ["jit", "inline_asm_sym"]
 jit = ["cranelift-jit", "libloading"]
-inline_asm = []
+inline_asm_sym = []
 
 [package.metadata.rust-analyzer]
 rustc_private = true
diff --git a/src/global_asm.rs b/src/global_asm.rs
index 7c2138bf0cfd..332a7ca9bf24 100644
--- a/src/global_asm.rs
+++ b/src/global_asm.rs
@@ -46,6 +46,13 @@ pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String,
                             global_asm.push_str(&string);
                         }
                         InlineAsmOperand::SymFn { anon_const } => {
+                            if cfg!(not(feature = "inline_asm_sym")) {
+                                tcx.sess.span_err(
+                                    item.span,
+                                    "asm! and global_asm! sym operands are not yet supported",
+                                );
+                            }
+
                             let ty = tcx.typeck_body(anon_const.body).node_type(anon_const.hir_id);
                             let instance = match ty.kind() {
                                 &ty::FnDef(def_id, args) => Instance::new(def_id, args),
@@ -57,6 +64,13 @@ pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String,
                             global_asm.push_str(symbol.name);
                         }
                         InlineAsmOperand::SymStatic { path: _, def_id } => {
+                            if cfg!(not(feature = "inline_asm_sym")) {
+                                tcx.sess.span_err(
+                                    item.span,
+                                    "asm! and global_asm! sym operands are not yet supported",
+                                );
+                            }
+
                             let instance = Instance::mono(tcx, def_id).polymorphize(tcx);
                             let symbol = tcx.symbol_name(instance);
                             global_asm.push_str(symbol.name);
@@ -81,22 +95,23 @@ pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String,
     }
 }
 
-pub(crate) fn asm_supported(tcx: TyCtxt<'_>) -> bool {
-    cfg!(feature = "inline_asm") && !tcx.sess.target.is_like_windows
-}
-
 #[derive(Debug)]
 pub(crate) struct GlobalAsmConfig {
-    asm_enabled: bool,
     assembler: PathBuf,
+    target: String,
     pub(crate) output_filenames: Arc<OutputFilenames>,
 }
 
 impl GlobalAsmConfig {
     pub(crate) fn new(tcx: TyCtxt<'_>) -> Self {
         GlobalAsmConfig {
-            asm_enabled: asm_supported(tcx),
             assembler: crate::toolchain::get_toolchain_binary(tcx.sess, "as"),
+            target: match &tcx.sess.opts.target_triple {
+                rustc_target::spec::TargetTriple::TargetTriple(triple) => triple.clone(),
+                rustc_target::spec::TargetTriple::TargetJson { path_for_rustdoc, .. } => {
+                    path_for_rustdoc.to_str().unwrap().to_owned()
+                }
+            },
             output_filenames: tcx.output_filenames(()).clone(),
         }
     }
@@ -111,21 +126,6 @@ pub(crate) fn compile_global_asm(
         return Ok(None);
     }
 
-    if !config.asm_enabled {
-        if global_asm.contains("__rust_probestack") {
-            return Ok(None);
-        }
-
-        if cfg!(not(feature = "inline_asm")) {
-            return Err(
-                "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift"
-                    .to_owned(),
-            );
-        } else {
-            return Err("asm! and global_asm! are not yet supported on Windows".to_owned());
-        }
-    }
-
     // Remove all LLVM style comments
     let mut global_asm = global_asm
         .lines()
diff --git a/src/inline_asm.rs b/src/inline_asm.rs
index b2e27399055e..331649b2ec24 100644
--- a/src/inline_asm.rs
+++ b/src/inline_asm.rs
@@ -8,7 +8,6 @@ use rustc_span::sym;
 use rustc_target::asm::*;
 use target_lexicon::BinaryFormat;
 
-use crate::global_asm::asm_supported;
 use crate::prelude::*;
 
 enum CInlineAsmOperand<'tcx> {
@@ -53,205 +52,6 @@ pub(crate) fn codegen_inline_asm<'tcx>(
         return;
     }
 
-    if !asm_supported(fx.tcx) {
-        if template.is_empty() {
-            let destination_block = fx.get_block(destination.unwrap());
-            fx.bcx.ins().jump(destination_block, &[]);
-            return;
-        }
-
-        // Used by stdarch
-        if template[0] == InlineAsmTemplatePiece::String("mov ".to_string())
-            && matches!(
-                template[1],
-                InlineAsmTemplatePiece::Placeholder {
-                    operand_idx: 0,
-                    modifier: Some('r'),
-                    span: _
-                }
-            )
-            && template[2] == InlineAsmTemplatePiece::String(", rbx".to_string())
-            && template[3] == InlineAsmTemplatePiece::String("\n".to_string())
-            && template[4] == InlineAsmTemplatePiece::String("cpuid".to_string())
-            && template[5] == InlineAsmTemplatePiece::String("\n".to_string())
-            && template[6] == InlineAsmTemplatePiece::String("xchg ".to_string())
-            && matches!(
-                template[7],
-                InlineAsmTemplatePiece::Placeholder {
-                    operand_idx: 0,
-                    modifier: Some('r'),
-                    span: _
-                }
-            )
-            && template[8] == InlineAsmTemplatePiece::String(", rbx".to_string())
-        {
-            assert_eq!(operands.len(), 4);
-            let (leaf, eax_place) = match operands[1] {
-                InlineAsmOperand::InOut {
-                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)),
-                    late: _,
-                    ref in_value,
-                    out_place: Some(out_place),
-                } => (
-                    crate::base::codegen_operand(fx, in_value).load_scalar(fx),
-                    crate::base::codegen_place(fx, out_place),
-                ),
-                _ => unreachable!(),
-            };
-            let ebx_place = match operands[0] {
-                InlineAsmOperand::Out {
-                    reg:
-                        InlineAsmRegOrRegClass::RegClass(InlineAsmRegClass::X86(
-                            X86InlineAsmRegClass::reg,
-                        )),
-                    late: _,
-                    place: Some(place),
-                } => crate::base::codegen_place(fx, place),
-                _ => unreachable!(),
-            };
-            let (sub_leaf, ecx_place) = match operands[2] {
-                InlineAsmOperand::InOut {
-                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)),
-                    late: _,
-                    ref in_value,
-                    out_place: Some(out_place),
-                } => (
-                    crate::base::codegen_operand(fx, in_value).load_scalar(fx),
-                    crate::base::codegen_place(fx, out_place),
-                ),
-                _ => unreachable!(),
-            };
-            let edx_place = match operands[3] {
-                InlineAsmOperand::Out {
-                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)),
-                    late: _,
-                    place: Some(place),
-                } => crate::base::codegen_place(fx, place),
-                _ => unreachable!(),
-            };
-
-            let (eax, ebx, ecx, edx) = crate::intrinsics::codegen_cpuid_call(fx, leaf, sub_leaf);
-
-            eax_place.write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32)));
-            ebx_place.write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32)));
-            ecx_place.write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32)));
-            edx_place.write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32)));
-            let destination_block = fx.get_block(destination.unwrap());
-            fx.bcx.ins().jump(destination_block, &[]);
-            return;
-        }
-
-        // Used by compiler-builtins
-        if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") {
-            // ___chkstk, ___chkstk_ms and __alloca are only used on Windows
-            crate::trap::trap_unimplemented(fx, "Stack probes are not supported");
-            return;
-        } else if fx.tcx.symbol_name(fx.instance).name == "__alloca" {
-            crate::trap::trap_unimplemented(fx, "Alloca is not supported");
-            return;
-        }
-
-        // Used by core::hint::spin_loop()
-        if template[0]
-            == InlineAsmTemplatePiece::String(".insn i 0x0F, 0, x0, x0, 0x010".to_string())
-            && template.len() == 1
-        {
-            let destination_block = fx.get_block(destination.unwrap());
-            fx.bcx.ins().jump(destination_block, &[]);
-            return;
-        }
-
-        // Used by measureme
-        if template[0] == InlineAsmTemplatePiece::String("xor %eax, %eax".to_string())
-            && template[1] == InlineAsmTemplatePiece::String("\n".to_string())
-            && template[2] == InlineAsmTemplatePiece::String("mov %rbx, ".to_string())
-            && matches!(
-                template[3],
-                InlineAsmTemplatePiece::Placeholder {
-                    operand_idx: 0,
-                    modifier: Some('r'),
-                    span: _
-                }
-            )
-            && template[4] == InlineAsmTemplatePiece::String("\n".to_string())
-            && template[5] == InlineAsmTemplatePiece::String("cpuid".to_string())
-            && template[6] == InlineAsmTemplatePiece::String("\n".to_string())
-            && template[7] == InlineAsmTemplatePiece::String("mov ".to_string())
-            && matches!(
-                template[8],
-                InlineAsmTemplatePiece::Placeholder {
-                    operand_idx: 0,
-                    modifier: Some('r'),
-                    span: _
-                }
-            )
-            && template[9] == InlineAsmTemplatePiece::String(", %rbx".to_string())
-        {
-            let destination_block = fx.get_block(destination.unwrap());
-            fx.bcx.ins().jump(destination_block, &[]);
-            return;
-        } else if template[0] == InlineAsmTemplatePiece::String("rdpmc".to_string()) {
-            // Return zero dummy values for all performance counters
-            match operands[0] {
-                InlineAsmOperand::In {
-                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)),
-                    value: _,
-                } => {}
-                _ => unreachable!(),
-            };
-            let lo = match operands[1] {
-                InlineAsmOperand::Out {
-                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)),
-                    late: true,
-                    place: Some(place),
-                } => crate::base::codegen_place(fx, place),
-                _ => unreachable!(),
-            };
-            let hi = match operands[2] {
-                InlineAsmOperand::Out {
-                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)),
-                    late: true,
-                    place: Some(place),
-                } => crate::base::codegen_place(fx, place),
-                _ => unreachable!(),
-            };
-
-            let u32_layout = fx.layout_of(fx.tcx.types.u32);
-            let zero = fx.bcx.ins().iconst(types::I32, 0);
-            lo.write_cvalue(fx, CValue::by_val(zero, u32_layout));
-            hi.write_cvalue(fx, CValue::by_val(zero, u32_layout));
-
-            let destination_block = fx.get_block(destination.unwrap());
-            fx.bcx.ins().jump(destination_block, &[]);
-            return;
-        } else if template[0] == InlineAsmTemplatePiece::String("lock xadd ".to_string())
-            && matches!(
-                template[1],
-                InlineAsmTemplatePiece::Placeholder { operand_idx: 1, modifier: None, span: _ }
-            )
-            && template[2] == InlineAsmTemplatePiece::String(", (".to_string())
-            && matches!(
-                template[3],
-                InlineAsmTemplatePiece::Placeholder { operand_idx: 0, modifier: None, span: _ }
-            )
-            && template[4] == InlineAsmTemplatePiece::String(")".to_string())
-        {
-            let destination_block = fx.get_block(destination.unwrap());
-            fx.bcx.ins().jump(destination_block, &[]);
-            return;
-        }
-
-        if cfg!(not(feature = "inline_asm")) {
-            fx.tcx.sess.span_err(
-                span,
-                "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift",
-            );
-        } else {
-            fx.tcx.sess.span_err(span, "asm! and global_asm! are not yet supported on Windows");
-        }
-        return;
-    }
-
     let operands = operands
         .into_iter()
         .map(|operand| match *operand {
@@ -282,6 +82,12 @@ pub(crate) fn codegen_inline_asm<'tcx>(
                 CInlineAsmOperand::Const { value }
             }
             InlineAsmOperand::SymFn { ref value } => {
+                if cfg!(not(feature = "inline_asm_sym")) {
+                    fx.tcx
+                        .sess
+                        .span_err(span, "asm! and global_asm! sym operands are not yet supported");
+                }
+
                 let const_ = fx.monomorphize(value.const_);
                 if let ty::FnDef(def_id, args) = *const_.ty().kind() {
                     let instance = ty::Instance::resolve_for_fn_ptr(
diff --git a/src/intrinsics/cpuid.rs b/src/intrinsics/cpuid.rs
deleted file mode 100644
index 5120b89c4e8b..000000000000
--- a/src/intrinsics/cpuid.rs
+++ /dev/null
@@ -1,74 +0,0 @@
-//! Emulation of a subset of the cpuid x86 instruction.
-
-use crate::prelude::*;
-
-/// Emulates a subset of the cpuid x86 instruction.
-///
-/// This emulates an intel cpu with sse and sse2 support, but which doesn't support anything else.
-pub(crate) fn codegen_cpuid_call<'tcx>(
-    fx: &mut FunctionCx<'_, '_, 'tcx>,
-    leaf: Value,
-    _sub_leaf: Value,
-) -> (Value, Value, Value, Value) {
-    let leaf_0 = fx.bcx.create_block();
-    let leaf_1 = fx.bcx.create_block();
-    let leaf_7 = fx.bcx.create_block();
-    let leaf_8000_0000 = fx.bcx.create_block();
-    let leaf_8000_0001 = fx.bcx.create_block();
-    let unsupported_leaf = fx.bcx.create_block();
-
-    let dest = fx.bcx.create_block();
-    let eax = fx.bcx.append_block_param(dest, types::I32);
-    let ebx = fx.bcx.append_block_param(dest, types::I32);
-    let ecx = fx.bcx.append_block_param(dest, types::I32);
-    let edx = fx.bcx.append_block_param(dest, types::I32);
-
-    let mut switch = cranelift_frontend::Switch::new();
-    switch.set_entry(0, leaf_0);
-    switch.set_entry(1, leaf_1);
-    switch.set_entry(7, leaf_7);
-    switch.set_entry(0x8000_0000, leaf_8000_0000);
-    switch.set_entry(0x8000_0001, leaf_8000_0001);
-    switch.emit(&mut fx.bcx, leaf, unsupported_leaf);
-
-    fx.bcx.switch_to_block(leaf_0);
-    let max_basic_leaf = fx.bcx.ins().iconst(types::I32, 1);
-    let vend0 = fx.bcx.ins().iconst(types::I32, i64::from(u32::from_le_bytes(*b"Genu")));
-    let vend2 = fx.bcx.ins().iconst(types::I32, i64::from(u32::from_le_bytes(*b"ineI")));
-    let vend1 = fx.bcx.ins().iconst(types::I32, i64::from(u32::from_le_bytes(*b"ntel")));
-    fx.bcx.ins().jump(dest, &[max_basic_leaf, vend0, vend1, vend2]);
-
-    fx.bcx.switch_to_block(leaf_1);
-    let cpu_signature = fx.bcx.ins().iconst(types::I32, 0);
-    let additional_information = fx.bcx.ins().iconst(types::I32, 0);
-    let ecx_features = fx.bcx.ins().iconst(types::I32, 0);
-    let edx_features = fx.bcx.ins().iconst(types::I32, 1 << 25 /* sse */ | 1 << 26 /* sse2 */);
-    fx.bcx.ins().jump(dest, &[cpu_signature, additional_information, ecx_features, edx_features]);
-
-    fx.bcx.switch_to_block(leaf_7);
-    // This leaf technically has subleaves, but we just return zero for all subleaves.
-    let zero = fx.bcx.ins().iconst(types::I32, 0);
-    fx.bcx.ins().jump(dest, &[zero, zero, zero, zero]);
-
-    fx.bcx.switch_to_block(leaf_8000_0000);
-    let extended_max_basic_leaf = fx.bcx.ins().iconst(types::I32, 0);
-    let zero = fx.bcx.ins().iconst(types::I32, 0);
-    fx.bcx.ins().jump(dest, &[extended_max_basic_leaf, zero, zero, zero]);
-
-    fx.bcx.switch_to_block(leaf_8000_0001);
-    let zero = fx.bcx.ins().iconst(types::I32, 0);
-    let proc_info_ecx = fx.bcx.ins().iconst(types::I32, 0);
-    let proc_info_edx = fx.bcx.ins().iconst(types::I32, 0);
-    fx.bcx.ins().jump(dest, &[zero, zero, proc_info_ecx, proc_info_edx]);
-
-    fx.bcx.switch_to_block(unsupported_leaf);
-    crate::trap::trap_unimplemented(
-        fx,
-        "__cpuid_count arch intrinsic doesn't yet support specified leaf",
-    );
-
-    fx.bcx.switch_to_block(dest);
-    fx.bcx.ins().nop();
-
-    (eax, ebx, ecx, edx)
-}
diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs
index e94091e6a25e..83d5d53624eb 100644
--- a/src/intrinsics/mod.rs
+++ b/src/intrinsics/mod.rs
@@ -12,7 +12,6 @@ macro_rules! intrinsic_args {
     }
 }
 
-mod cpuid;
 mod llvm;
 mod llvm_aarch64;
 mod llvm_x86;
@@ -25,7 +24,6 @@ use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
 use rustc_middle::ty::GenericArgsRef;
 use rustc_span::symbol::{kw, sym, Symbol};
 
-pub(crate) use self::cpuid::codegen_cpuid_call;
 pub(crate) use self::llvm::codegen_llvm_intrinsic_call;
 use crate::prelude::*;
 

From e53b18f033b114829d7e47b0aac7c8477b59d5e0 Mon Sep 17 00:00:00 2001
From: Ben Kimock <kimockb@gmail.com>
Date: Sun, 29 Oct 2023 13:20:51 -0400
Subject: [PATCH 201/297] Enable cross-crate-inlining when MIR inlining is
 enabled

---
 compiler/rustc_mir_transform/src/cross_crate_inline.rs | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs
index 24d081f2ac9a..4d0e261ed1fd 100644
--- a/compiler/rustc_mir_transform/src/cross_crate_inline.rs
+++ b/compiler/rustc_mir_transform/src/cross_crate_inline.rs
@@ -1,3 +1,5 @@
+use crate::inline;
+use crate::pass_manager as pm;
 use rustc_attr::InlineAttr;
 use rustc_hir::def::DefKind;
 use rustc_hir::def_id::LocalDefId;
@@ -40,8 +42,11 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
         return false;
     }
 
-    // Don't do any inference unless optimizations are enabled.
-    if matches!(tcx.sess.opts.optimize, OptLevel::No) {
+    // Don't do any inference if codegen optimizations are disabled and also MIR inlining is not
+    // enabled. This ensures that we do inference even if someone only passes -Zinline-mir,
+    // which is less confusing than having to also enable -Copt-level=1.
+    if matches!(tcx.sess.opts.optimize, OptLevel::No) && !pm::should_run_pass(tcx, &inline::Inline)
+    {
         return false;
     }
 

From 69c6aa560cfc1991822fdab1579153f6a25c392f Mon Sep 17 00:00:00 2001
From: bjorn3 <17426603+bjorn3@users.noreply.github.com>
Date: Sun, 29 Oct 2023 18:53:38 +0000
Subject: [PATCH 202/297] Avoid infinite recursion when cranelift is the
 default codegen backend

---
 scripts/setup_rust_fork.sh | 4 +++-
 scripts/test_bootstrap.sh  | 4 +++-
 src/global_asm.rs          | 3 ++-
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh
index 3e48fb006dea..bbb8a010d965 100644
--- a/scripts/setup_rust_fork.sh
+++ b/scripts/setup_rust_fork.sh
@@ -4,7 +4,9 @@ set -e
 # Compiletest expects all standard library paths to start with /rustc/FAKE_PREFIX.
 # CG_CLIF_STDLIB_REMAP_PATH_PREFIX will cause cg_clif's build system to pass
 # --remap-path-prefix to handle this.
-CG_CLIF_STDLIB_REMAP_PATH_PREFIX=/rustc/FAKE_PREFIX ./y.sh build
+# CG_CLIF_FORCE_GNU_AS will force usage of as instead of the LLVM backend of rustc as we
+# the LLVM backend isn't compiled in here.
+CG_CLIF_FORCE_GNU_AS=1 CG_CLIF_STDLIB_REMAP_PATH_PREFIX=/rustc/FAKE_PREFIX ./y.sh build
 
 echo "[SETUP] Rust fork"
 git clone https://github.com/rust-lang/rust.git || true
diff --git a/scripts/test_bootstrap.sh b/scripts/test_bootstrap.sh
index 791d457993de..a8f6d7a20248 100755
--- a/scripts/test_bootstrap.sh
+++ b/scripts/test_bootstrap.sh
@@ -11,5 +11,7 @@ rm -r compiler/rustc_codegen_cranelift/{Cargo.*,src}
 cp ../Cargo.* compiler/rustc_codegen_cranelift/
 cp -r ../src compiler/rustc_codegen_cranelift/src
 
-./x.py build --stage 1 library/std
+# CG_CLIF_FORCE_GNU_AS will force usage of as instead of the LLVM backend of rustc as we
+# the LLVM backend isn't compiled in here.
+CG_CLIF_FORCE_GNU_AS=1 ./x.py build --stage 1 library/std
 popd
diff --git a/src/global_asm.rs b/src/global_asm.rs
index 332a7ca9bf24..b14007f4e523 100644
--- a/src/global_asm.rs
+++ b/src/global_asm.rs
@@ -140,7 +140,7 @@ pub(crate) fn compile_global_asm(
     );
 
     // Assemble `global_asm`
-    if false {
+    if option_env!("CG_CLIF_FORCE_GNU_AS").is_some() {
         let mut child = Command::new(&config.assembler)
             .arg("-o")
             .arg(&global_asm_object_file)
@@ -164,6 +164,7 @@ pub(crate) fn compile_global_asm(
             .arg(&global_asm_object_file)
             .arg("-")
             .arg("-Abad_asm_style")
+            .arg("-Zcodegen-backend=llvm")
             .stdin(Stdio::piped())
             .spawn()
             .expect("Failed to spawn `as`.");

From dde58803fd6cbb270c7a437f36a8a3a29fbef679 Mon Sep 17 00:00:00 2001
From: bjorn3 <17426603+bjorn3@users.noreply.github.com>
Date: Sun, 29 Oct 2023 20:29:52 +0000
Subject: [PATCH 203/297] Rustup to rustc 1.75.0-nightly (e5cfc5547 2023-10-28)

---
 patches/stdlib-lock.toml | 17 -----------------
 rust-toolchain           |  2 +-
 2 files changed, 1 insertion(+), 18 deletions(-)

diff --git a/patches/stdlib-lock.toml b/patches/stdlib-lock.toml
index f10b4d6b9d4f..9902bca8eab2 100644
--- a/patches/stdlib-lock.toml
+++ b/patches/stdlib-lock.toml
@@ -40,22 +40,6 @@ version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "56fc6cf8dc8c4158eed8649f9b8b0ea1518eb62b544fe9490d66fa0b349eafe9"
 
-[[package]]
-name = "auxv"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e50430f9beb8effb02399fa81c76eeaa26b05e4f03b09285cad8d079c1af5a3d"
-dependencies = [
- "byteorder",
- "gcc",
-]
-
-[[package]]
-name = "byteorder"
-version = "1.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
-
 [[package]]
 name = "cc"
 version = "1.0.79"
@@ -388,7 +372,6 @@ dependencies = [
 name = "std_detect"
 version = "0.1.5"
 dependencies = [
- "auxv",
  "cfg-if",
  "compiler_builtins",
  "cupid",
diff --git a/rust-toolchain b/rust-toolchain
index 3735ac1c17b1..7e3eaacf8ef0 100644
--- a/rust-toolchain
+++ b/rust-toolchain
@@ -1,3 +1,3 @@
 [toolchain]
-channel = "nightly-2023-10-21"
+channel = "nightly-2023-10-29"
 components = ["rust-src", "rustc-dev", "llvm-tools"]

From 28e60de4879786a02c215f83f8ba96f474bcbcaa Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 23 Oct 2023 11:13:39 +1100
Subject: [PATCH 204/297] Remove `memoffset` dependency from
 `rustc_query_impl`.

The comment explains it's for `unstable_offset_of`, but `offset_of` is
now stable.
---
 Cargo.lock                           | 1 -
 compiler/rustc_query_impl/Cargo.toml | 3 ---
 2 files changed, 4 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 0761268c9d41..2b1e918e2f5e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4412,7 +4412,6 @@ version = "0.0.0"
 dependencies = [
  "field-offset",
  "measureme",
- "memoffset",
  "rustc-rayon-core",
  "rustc_data_structures",
  "rustc_errors",
diff --git a/compiler/rustc_query_impl/Cargo.toml b/compiler/rustc_query_impl/Cargo.toml
index a350e8b2e3a4..dac86116d5b6 100644
--- a/compiler/rustc_query_impl/Cargo.toml
+++ b/compiler/rustc_query_impl/Cargo.toml
@@ -22,8 +22,5 @@ rustc_span = { path = "../rustc_span" }
 thin-vec = "0.2.12"
 tracing = "0.1"
 
-# Not used directly, but included to enable the unstable_offset_of feature
-memoffset = { version = "0.9.0", features = ["unstable_offset_of"] }
-
 [features]
 rustc_use_parallel_compiler = ["rustc-rayon-core", "rustc_query_system/rustc_use_parallel_compiler"]

From c5613258bb3f10907ba1127f363102bf842637cd Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Sun, 29 Oct 2023 17:45:01 -0400
Subject: [PATCH 205/297] Ignore RPIT duplicated lifetimes in
 opaque_types_defined_by

---
 compiler/rustc_middle/src/ty/util.rs              |  2 +-
 compiler/rustc_ty_utils/src/opaque_types.rs       |  9 ++++++++-
 ...plicate-lifetimes-from-rpit-containing-tait.rs | 13 +++++++++++++
 ...licate-lifetimes-from-rpit-containing-tait2.rs | 15 +++++++++++++++
 4 files changed, 37 insertions(+), 2 deletions(-)
 create mode 100644 tests/ui/type-alias-impl-trait/duplicate-lifetimes-from-rpit-containing-tait.rs
 create mode 100644 tests/ui/type-alias-impl-trait/duplicate-lifetimes-from-rpit-containing-tait2.rs

diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs
index d29d1902404f..c155f83734e3 100644
--- a/compiler/rustc_middle/src/ty/util.rs
+++ b/compiler/rustc_middle/src/ty/util.rs
@@ -460,7 +460,7 @@ impl<'tcx> TyCtxt<'tcx> {
     /// Checks whether each generic argument is simply a unique generic parameter.
     pub fn uses_unique_generic_params(
         self,
-        args: GenericArgsRef<'tcx>,
+        args: &[ty::GenericArg<'tcx>],
         ignore_regions: CheckRegions,
     ) -> Result<(), NotUniqueParam<'tcx>> {
         let mut seen = GrowableBitSet::default();
diff --git a/compiler/rustc_ty_utils/src/opaque_types.rs b/compiler/rustc_ty_utils/src/opaque_types.rs
index d1a154fe20d8..f7d54566cb70 100644
--- a/compiler/rustc_ty_utils/src/opaque_types.rs
+++ b/compiler/rustc_ty_utils/src/opaque_types.rs
@@ -154,7 +154,14 @@ impl<'tcx> TypeVisitor<TyCtxt<'tcx>> for OpaqueTypeCollector<'tcx> {
 
                 self.opaques.push(alias_ty.def_id.expect_local());
 
-                match self.tcx.uses_unique_generic_params(alias_ty.args, CheckRegions::Bound) {
+                let parent_count = self.tcx.generics_of(alias_ty.def_id).parent_count;
+                // Only check that the parent generics of the TAIT/RPIT are unique.
+                // the args owned by the opaque are going to always be duplicate
+                // lifetime params for RPITs, and empty for TAITs.
+                match self
+                    .tcx
+                    .uses_unique_generic_params(&alias_ty.args[..parent_count], CheckRegions::Bound)
+                {
                     Ok(()) => {
                         // FIXME: implement higher kinded lifetime bounds on nested opaque types. They are not
                         // supported at all, so this is sound to do, but once we want to support them, you'll
diff --git a/tests/ui/type-alias-impl-trait/duplicate-lifetimes-from-rpit-containing-tait.rs b/tests/ui/type-alias-impl-trait/duplicate-lifetimes-from-rpit-containing-tait.rs
new file mode 100644
index 000000000000..4c56fe2d1dce
--- /dev/null
+++ b/tests/ui/type-alias-impl-trait/duplicate-lifetimes-from-rpit-containing-tait.rs
@@ -0,0 +1,13 @@
+// check-pass
+
+#![feature(type_alias_impl_trait)]
+
+type Opaque<'lt> = impl Sized + 'lt;
+
+fn test<'a>(
+    arg: impl Iterator<Item = &'a u8>,
+) -> impl Iterator<Item = Opaque<'a>> {
+    arg
+}
+
+fn main() {}
diff --git a/tests/ui/type-alias-impl-trait/duplicate-lifetimes-from-rpit-containing-tait2.rs b/tests/ui/type-alias-impl-trait/duplicate-lifetimes-from-rpit-containing-tait2.rs
new file mode 100644
index 000000000000..97f8c799fc56
--- /dev/null
+++ b/tests/ui/type-alias-impl-trait/duplicate-lifetimes-from-rpit-containing-tait2.rs
@@ -0,0 +1,15 @@
+// check-pass
+// edition: 2021
+
+#![feature(type_alias_impl_trait)]
+
+struct Foo<'a>(&'a ());
+
+impl<'a> Foo<'a> {
+    async fn new() -> () {
+        type T = impl Sized;
+        let _: T = ();
+    }
+}
+
+fn main() {}

From 8ff624a9f29b4f545984bddf732efe21e42dfa40 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Fri, 20 Oct 2023 13:37:29 +1100
Subject: [PATCH 206/297] Clean up `rustc_*/Cargo.toml`.

- Sort dependencies and features sections.
- Add `tidy` markers to the sorted sections so they stay sorted.
- Remove empty `[lib`] sections.
- Remove "See more keys..." comments.

Excluded files:
- rustc_codegen_{cranelift,gcc}, because they're external.
- rustc_lexer, because it has external use.
- stable_mir, because it has external use.
---
 compiler/rustc/Cargo.toml                 | 13 ++++--
 compiler/rustc_abi/Cargo.toml             |  8 +++-
 compiler/rustc_arena/Cargo.toml           |  2 +
 compiler/rustc_ast/Cargo.toml             |  6 +--
 compiler/rustc_ast_lowering/Cargo.toml    |  6 ++-
 compiler/rustc_ast_passes/Cargo.toml      |  4 +-
 compiler/rustc_ast_pretty/Cargo.toml      |  4 +-
 compiler/rustc_attr/Cargo.toml            |  4 +-
 compiler/rustc_baked_icu_data/Cargo.toml  |  4 ++
 compiler/rustc_borrowck/Cargo.toml        | 12 +++---
 compiler/rustc_builtin_macros/Cargo.toml  |  6 ++-
 compiler/rustc_codegen_llvm/Cargo.toml    | 17 ++++----
 compiler/rustc_codegen_ssa/Cargo.toml     | 31 ++++++++-------
 compiler/rustc_const_eval/Cargo.toml      | 10 ++---
 compiler/rustc_data_structures/Cargo.toml | 22 +++++------
 compiler/rustc_driver/Cargo.toml          |  2 +
 compiler/rustc_driver_impl/Cargo.toml     |  6 ++-
 compiler/rustc_error_codes/Cargo.toml     |  4 ++
 compiler/rustc_error_messages/Cargo.toml  | 14 ++++---
 compiler/rustc_errors/Cargo.toml          | 28 +++++++------
 compiler/rustc_expand/Cargo.toml          |  6 ++-
 compiler/rustc_feature/Cargo.toml         |  4 +-
 compiler/rustc_fluent_macro/Cargo.toml    |  4 +-
 compiler/rustc_fs_util/Cargo.toml         |  4 ++
 compiler/rustc_graphviz/Cargo.toml        |  4 ++
 compiler/rustc_hir/Cargo.toml             | 16 ++++----
 compiler/rustc_hir_analysis/Cargo.toml    | 22 ++++++-----
 compiler/rustc_hir_pretty/Cargo.toml      |  8 ++--
 compiler/rustc_hir_typeck/Cargo.toml      | 16 ++++----
 compiler/rustc_incremental/Cargo.toml     |  6 +--
 compiler/rustc_index/Cargo.toml           |  8 ++--
 compiler/rustc_infer/Cargo.toml           |  8 ++--
 compiler/rustc_interface/Cargo.toml       | 48 ++++++++++++-----------
 compiler/rustc_lexer/Cargo.toml           |  1 -
 compiler/rustc_lint/Cargo.toml            | 28 +++++++------
 compiler/rustc_lint_defs/Cargo.toml       | 12 +++---
 compiler/rustc_llvm/Cargo.toml            |  4 ++
 compiler/rustc_log/Cargo.toml             |  8 +++-
 compiler/rustc_macros/Cargo.toml          |  6 ++-
 compiler/rustc_metadata/Cargo.toml        | 20 +++++-----
 compiler/rustc_middle/Cargo.toml          | 15 +++----
 compiler/rustc_mir_build/Cargo.toml       | 14 +++----
 compiler/rustc_mir_dataflow/Cargo.toml    | 10 ++---
 compiler/rustc_mir_transform/Cargo.toml   | 22 ++++++-----
 compiler/rustc_monomorphize/Cargo.toml    | 12 +++---
 compiler/rustc_parse/Cargo.toml           |  4 +-
 compiler/rustc_parse_format/Cargo.toml    |  4 +-
 compiler/rustc_passes/Cargo.toml          | 28 +++++++------
 compiler/rustc_plugin_impl/Cargo.toml     |  8 ++--
 compiler/rustc_privacy/Cargo.toml         |  6 ++-
 compiler/rustc_query_impl/Cargo.toml      |  9 +++--
 compiler/rustc_query_system/Cargo.toml    |  8 ++--
 compiler/rustc_resolve/Cargo.toml         |  4 +-
 compiler/rustc_serialize/Cargo.toml       |  4 ++
 compiler/rustc_session/Cargo.toml         | 20 ++++++----
 compiler/rustc_smir/Cargo.toml            |  4 +-
 compiler/rustc_span/Cargo.toml            | 18 ++++-----
 compiler/rustc_symbol_mangling/Cargo.toml | 19 +++++----
 compiler/rustc_target/Cargo.toml          | 14 ++++---
 compiler/rustc_trait_selection/Cargo.toml | 14 +++----
 compiler/rustc_traits/Cargo.toml          |  8 ++--
 compiler/rustc_transmute/Cargo.toml       | 12 +++---
 compiler/rustc_ty_utils/Cargo.toml        | 12 +++---
 compiler/rustc_type_ir/Cargo.toml         |  8 ++--
 64 files changed, 407 insertions(+), 306 deletions(-)

diff --git a/compiler/rustc/Cargo.toml b/compiler/rustc/Cargo.toml
index dcb165f9fdbf..3cb56a7d3121 100644
--- a/compiler/rustc/Cargo.toml
+++ b/compiler/rustc/Cargo.toml
@@ -4,16 +4,21 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
-rustc_driver = { path = "../rustc_driver" }
-rustc_driver_impl = { path = "../rustc_driver_impl" }
+# tidy-alphabetical-start
 
 # Make sure rustc_codegen_ssa ends up in the sysroot, because this
 # crate is intended to be used by codegen backends, which may not be in-tree.
 rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }
+
+rustc_driver = { path = "../rustc_driver" }
+rustc_driver_impl = { path = "../rustc_driver_impl" }
+
 # Make sure rustc_smir ends up in the sysroot, because this
-# crate is intended to be used by stable MIR consumers, which are not in-tree
+# crate is intended to be used by stable MIR consumers, which are not in-tree.
 rustc_smir = { path = "../rustc_smir" }
+
 stable_mir = { path = "../stable_mir" }
+# tidy-alphabetical-end
 
 [dependencies.jemalloc-sys]
 version = "0.5.0"
@@ -21,7 +26,9 @@ optional = true
 features = ['unprefixed_malloc_on_supported_platforms']
 
 [features]
+# tidy-alphabetical-start
 jemalloc = ['jemalloc-sys']
 llvm = ['rustc_driver_impl/llvm']
 max_level_info = ['rustc_driver_impl/max_level_info']
 rustc_use_parallel_compiler = ['rustc_driver_impl/rustc_use_parallel_compiler']
+# tidy-alphabetical-end
diff --git a/compiler/rustc_abi/Cargo.toml b/compiler/rustc_abi/Cargo.toml
index c43fd745e8f5..e549724b1c0a 100644
--- a/compiler/rustc_abi/Cargo.toml
+++ b/compiler/rustc_abi/Cargo.toml
@@ -4,18 +4,20 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
+# tidy-alphabetical-start
 bitflags = "1.2.1"
-tracing = "0.1"
 rand = { version = "0.8.4", default-features = false, optional = true }
 rand_xoshiro = { version = "0.6.0", optional = true }
 rustc_data_structures = { path = "../rustc_data_structures", optional = true  }
 rustc_index = { path = "../rustc_index", default-features = false }
 rustc_macros = { path = "../rustc_macros", optional = true }
 rustc_serialize = { path = "../rustc_serialize", optional = true  }
+tracing = "0.1"
+# tidy-alphabetical-end
 
 [features]
+# tidy-alphabetical-start
 default = ["nightly", "randomize"]
-randomize = ["rand", "rand_xoshiro", "nightly"]
 # rust-analyzer depends on this crate and we therefore require it to built on a stable toolchain
 # without depending on rustc_data_structures, rustc_macros and rustc_serialize
 nightly = [
@@ -24,3 +26,5 @@ nightly = [
     "rustc_macros",
     "rustc_serialize",
 ]
+randomize = ["rand", "rand_xoshiro", "nightly"]
+# tidy-alphabetical-end
diff --git a/compiler/rustc_arena/Cargo.toml b/compiler/rustc_arena/Cargo.toml
index 5c2aee6b47f9..382ab2b07754 100644
--- a/compiler/rustc_arena/Cargo.toml
+++ b/compiler/rustc_arena/Cargo.toml
@@ -4,4 +4,6 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
+# tidy-alphabetical-start
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
+# tidy-alphabetical-end
diff --git a/compiler/rustc_ast/Cargo.toml b/compiler/rustc_ast/Cargo.toml
index e0948471acb5..59e3d85589a2 100644
--- a/compiler/rustc_ast/Cargo.toml
+++ b/compiler/rustc_ast/Cargo.toml
@@ -3,9 +3,8 @@ name = "rustc_ast"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 bitflags = "1.2.1"
 memchr = "2.5.0"
 rustc_data_structures = { path = "../rustc_data_structures" }
@@ -14,8 +13,9 @@ rustc_lexer = { path = "../rustc_lexer" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_span = { path = "../rustc_span" }
-# depends on Mutability and Movability, which could be uplifted into a common crate.
+# For Mutability and Movability, which could be uplifted into a common crate.
 rustc_type_ir = { path = "../rustc_type_ir" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 thin-vec = "0.2.12"
 tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_ast_lowering/Cargo.toml b/compiler/rustc_ast_lowering/Cargo.toml
index 6b0da2565050..8cc4521e0a78 100644
--- a/compiler/rustc_ast_lowering/Cargo.toml
+++ b/compiler/rustc_ast_lowering/Cargo.toml
@@ -7,18 +7,20 @@ edition = "2021"
 doctest = false
 
 [dependencies]
+# tidy-alphabetical-start
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
-rustc_hir = { path = "../rustc_hir" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
+rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
-rustc_middle = { path = "../rustc_middle" }
 rustc_macros = { path = "../rustc_macros" }
+rustc_middle = { path = "../rustc_middle" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 thin-vec = "0.2.12"
 tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_ast_passes/Cargo.toml b/compiler/rustc_ast_passes/Cargo.toml
index eb7361235207..c1ebfb3e6d03 100644
--- a/compiler/rustc_ast_passes/Cargo.toml
+++ b/compiler/rustc_ast_passes/Cargo.toml
@@ -4,6 +4,7 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
+# tidy-alphabetical-start
 itertools = "0.10.1"
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
@@ -11,11 +12,12 @@ rustc_attr = { path = "../rustc_attr" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_feature = { path = "../rustc_feature" }
-rustc_macros = { path = "../rustc_macros" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
+rustc_macros = { path = "../rustc_macros" }
 rustc_parse = { path = "../rustc_parse" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 thin-vec = "0.2.12"
 tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_ast_pretty/Cargo.toml b/compiler/rustc_ast_pretty/Cargo.toml
index 980a8fa93a97..af1524c8baa7 100644
--- a/compiler/rustc_ast_pretty/Cargo.toml
+++ b/compiler/rustc_ast_pretty/Cargo.toml
@@ -3,9 +3,9 @@ name = "rustc_ast_pretty"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 rustc_ast = { path = "../rustc_ast" }
 rustc_span = { path = "../rustc_span" }
 thin-vec = "0.2.12"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_attr/Cargo.toml b/compiler/rustc_attr/Cargo.toml
index a14d27968179..d33416d20038 100644
--- a/compiler/rustc_attr/Cargo.toml
+++ b/compiler/rustc_attr/Cargo.toml
@@ -3,9 +3,8 @@ name = "rustc_attr"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_data_structures = { path = "../rustc_data_structures" }
@@ -17,3 +16,4 @@ rustc_macros = { path = "../rustc_macros" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
+# tidy-alphabetical-end
diff --git a/compiler/rustc_baked_icu_data/Cargo.toml b/compiler/rustc_baked_icu_data/Cargo.toml
index d3a307675ac4..fb54937a098f 100644
--- a/compiler/rustc_baked_icu_data/Cargo.toml
+++ b/compiler/rustc_baked_icu_data/Cargo.toml
@@ -4,11 +4,15 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
+# tidy-alphabetical-start
 icu_list = "1.2"
 icu_locid = "1.2"
 icu_provider = "1.2"
 icu_provider_adapters = "1.2"
 zerovec = "0.9.4"
+# tidy-alphabetical-end
 
 [features]
+# tidy-alphabetical-start
 rustc_use_parallel_compiler = ['icu_provider/sync']
+# tidy-alphabetical-end
diff --git a/compiler/rustc_borrowck/Cargo.toml b/compiler/rustc_borrowck/Cargo.toml
index 56a9deb6aab1..636817a7ce10 100644
--- a/compiler/rustc_borrowck/Cargo.toml
+++ b/compiler/rustc_borrowck/Cargo.toml
@@ -3,19 +3,16 @@ name = "rustc_borrowck"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 either = "1.5.0"
 itertools = "0.10.1"
-tracing = "0.1"
 polonius-engine = "0.13.0"
-smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
+rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_graphviz = { path = "../rustc_graphviz" }
 rustc_hir = { path = "../rustc_hir" }
-rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_index = { path = "../rustc_index" }
 rustc_infer = { path = "../rustc_infer" }
 rustc_lexer = { path = "../rustc_lexer" }
@@ -24,7 +21,10 @@ rustc_middle = { path = "../rustc_middle" }
 rustc_mir_dataflow = { path = "../rustc_mir_dataflow" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
+rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 rustc_trait_selection = { path = "../rustc_trait_selection" }
 rustc_traits = { path = "../rustc_traits" }
-rustc_span = { path = "../rustc_span" }
+smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
+tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_builtin_macros/Cargo.toml b/compiler/rustc_builtin_macros/Cargo.toml
index 44012e802aa7..21b87be4b81d 100644
--- a/compiler/rustc_builtin_macros/Cargo.toml
+++ b/compiler/rustc_builtin_macros/Cargo.toml
@@ -7,6 +7,7 @@ edition = "2021"
 doctest = false
 
 [dependencies]
+# tidy-alphabetical-start
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_attr = { path = "../rustc_attr" }
@@ -14,16 +15,17 @@ rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_expand = { path = "../rustc_expand" }
 rustc_feature = { path = "../rustc_feature" }
+rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_index = { path = "../rustc_index" }
 rustc_lexer = { path = "../rustc_lexer" }
 rustc_lint_defs = { path = "../rustc_lint_defs" }
 rustc_macros = { path = "../rustc_macros" }
-rustc_fluent_macro = { path = "../rustc_fluent_macro" }
-rustc_parse_format = { path = "../rustc_parse_format" }
 rustc_parse = { path = "../rustc_parse" }
+rustc_parse_format = { path = "../rustc_parse_format" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 thin-vec = "0.2.12"
 tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml
index e864337e5dcb..1d309eb908e3 100644
--- a/compiler/rustc_codegen_llvm/Cargo.toml
+++ b/compiler/rustc_codegen_llvm/Cargo.toml
@@ -7,18 +7,15 @@ edition = "2021"
 test = false
 
 [dependencies]
+# tidy-alphabetical-start
 bitflags = "1.0"
 cstr = "0.2"
 itertools = "0.10.5"
 libc = "0.2"
 measureme = "10.0.0"
-object = { version = "0.32.0", default-features = false, features = [
-    "std",
-    "read",
-] }
-tracing = "0.1"
-rustc_middle = { path = "../rustc_middle" }
+object = { version = "0.32.0", default-features = false, features = ["std", "read"] }
 rustc-demangle = "0.1.21"
+rustc_ast = { path = "../rustc_ast" }
 rustc_attr = { path = "../rustc_attr" }
 rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }
 rustc_data_structures = { path = "../rustc_data_structures" }
@@ -30,12 +27,14 @@ rustc_index = { path = "../rustc_index" }
 rustc_llvm = { path = "../rustc_llvm" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_metadata = { path = "../rustc_metadata" }
+rustc_middle = { path = "../rustc_middle" }
 rustc_query_system = { path = "../rustc_query_system" }
 rustc_session = { path = "../rustc_session" }
+rustc_span = { path = "../rustc_span" }
 rustc_symbol_mangling = { path = "../rustc_symbol_mangling" }
 rustc_target = { path = "../rustc_target" }
-smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-rustc_ast = { path = "../rustc_ast" }
-rustc_span = { path = "../rustc_span" }
 serde = { version = "1", features = [ "derive" ]}
 serde_json = "1"
+smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
+tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_codegen_ssa/Cargo.toml b/compiler/rustc_codegen_ssa/Cargo.toml
index 454e2f806763..4dae49f81a38 100644
--- a/compiler/rustc_codegen_ssa/Cargo.toml
+++ b/compiler/rustc_codegen_ssa/Cargo.toml
@@ -4,43 +4,46 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
+# tidy-alphabetical-start
 ar_archive_writer = "0.1.5"
 bitflags = "1.2.1"
 cc = "1.0.69"
 itertools = "0.10.1"
-tracing = "0.1"
 jobserver = "0.1.22"
-tempfile = "3.2"
-thorin-dwp = "0.7"
 pathdiff = "0.2.0"
-serde_json = "1.0.59"
-smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 regex = "1.4"
-thin-vec = "0.2.12"
-
-rustc_serialize = { path = "../rustc_serialize" }
 rustc_arena = { path = "../rustc_arena" }
 rustc_ast = { path = "../rustc_ast" }
-rustc_span = { path = "../rustc_span" }
-rustc_middle = { path = "../rustc_middle" }
-rustc_type_ir = { path = "../rustc_type_ir" }
 rustc_attr = { path = "../rustc_attr" }
-rustc_fluent_macro = { path = "../rustc_fluent_macro" }
-rustc_symbol_mangling = { path = "../rustc_symbol_mangling" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
+rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_fs_util = { path = "../rustc_fs_util" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_incremental = { path = "../rustc_incremental" }
 rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_metadata = { path = "../rustc_metadata" }
+rustc_middle = { path = "../rustc_middle" }
 rustc_query_system = { path = "../rustc_query_system" }
-rustc_target = { path = "../rustc_target" }
+rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
+rustc_span = { path = "../rustc_span" }
+rustc_symbol_mangling = { path = "../rustc_symbol_mangling" }
+rustc_target = { path = "../rustc_target" }
+rustc_type_ir = { path = "../rustc_type_ir" }
+serde_json = "1.0.59"
+smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
+tempfile = "3.2"
+thin-vec = "0.2.12"
+thorin-dwp = "0.7"
+tracing = "0.1"
+# tidy-alphabetical-end
 
 [target.'cfg(unix)'.dependencies]
+# tidy-alphabetical-start
 libc = "0.2.50"
+# tidy-alphabetical-end
 
 [dependencies.object]
 version = "0.32.0"
diff --git a/compiler/rustc_const_eval/Cargo.toml b/compiler/rustc_const_eval/Cargo.toml
index 4e47fed86404..c4f8841d71c6 100644
--- a/compiler/rustc_const_eval/Cargo.toml
+++ b/compiler/rustc_const_eval/Cargo.toml
@@ -3,25 +3,25 @@ name = "rustc_const_eval"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
-tracing = "0.1"
+# tidy-alphabetical-start
 either = "1"
 rustc_apfloat = "0.2.0"
 rustc_ast = { path = "../rustc_ast" }
 rustc_attr = { path = "../rustc_attr" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
-rustc_hir = { path = "../rustc_hir" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
+rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
 rustc_infer = { path = "../rustc_infer" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_mir_dataflow = { path = "../rustc_mir_dataflow" }
 rustc_session = { path = "../rustc_session" }
+rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 rustc_trait_selection = { path = "../rustc_trait_selection" }
-rustc_span = { path = "../rustc_span" }
 rustc_type_ir = { path = "../rustc_type_ir" }
+tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml
index dce4e199e17e..2701fdbbd77f 100644
--- a/compiler/rustc_data_structures/Cargo.toml
+++ b/compiler/rustc_data_structures/Cargo.toml
@@ -3,35 +3,31 @@ name = "rustc_data_structures"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 arrayvec = { version = "0.7", default-features = false }
 bitflags = "1.2.1"
+elsa = "=1.7.1"
 ena = "0.14.2"
 indexmap = { version = "2.0.0" }
+itertools = "0.10.1"
 jobserver_crate = { version = "0.1.13", package = "jobserver" }
 libc = "0.2"
 measureme = "10.0.0"
-rustc-rayon-core = { version = "0.5.0", optional = true }
+rustc-hash = "1.1.0"
 rustc-rayon = { version = "0.5.0", optional = true }
+rustc-rayon-core = { version = "0.5.0", optional = true }
 rustc_arena = { path = "../rustc_arena" }
 rustc_graphviz = { path = "../rustc_graphviz" }
-rustc-hash = "1.1.0"
 rustc_index = { path = "../rustc_index", package = "rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_serialize = { path = "../rustc_serialize" }
-smallvec = { version = "1.8.1", features = [
-    "const_generics",
-    "union",
-    "may_dangle",
-] }
+smallvec = { version = "1.8.1", features = ["const_generics", "union", "may_dangle"] }
 stacker = "0.1.15"
 tempfile = "3.2"
 thin-vec = "0.2.12"
 tracing = "0.1"
-elsa = "=1.7.1"
-itertools = "0.10.1"
+# tidy-alphabetical-end
 
 [dependencies.parking_lot]
 version = "0.12"
@@ -47,7 +43,11 @@ features = [
 ]
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
+# tidy-alphabetical-start
 memmap2 = "0.2.1"
+# tidy-alphabetical-end
 
 [features]
+# tidy-alphabetical-start
 rustc_use_parallel_compiler = ["indexmap/rustc-rayon", "rustc-rayon", "rustc-rayon-core"]
+# tidy-alphabetical-end
diff --git a/compiler/rustc_driver/Cargo.toml b/compiler/rustc_driver/Cargo.toml
index d7c295418ba6..ae9712ad66d8 100644
--- a/compiler/rustc_driver/Cargo.toml
+++ b/compiler/rustc_driver/Cargo.toml
@@ -7,4 +7,6 @@ edition = "2021"
 crate-type = ["dylib"]
 
 [dependencies]
+# tidy-alphabetical-start
 rustc_driver_impl = { path = "../rustc_driver_impl" }
+# tidy-alphabetical-end
diff --git a/compiler/rustc_driver_impl/Cargo.toml b/compiler/rustc_driver_impl/Cargo.toml
index d931a8dab9b8..252803e3cbaa 100644
--- a/compiler/rustc_driver_impl/Cargo.toml
+++ b/compiler/rustc_driver_impl/Cargo.toml
@@ -3,8 +3,6 @@ name = "rustc_driver_impl"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
 # tidy-alphabetical-start
 rustc_ast = { path = "../rustc_ast" }
@@ -57,7 +55,9 @@ tracing = { version = "0.1.35" }
 # tidy-alphabetical-end
 
 [target.'cfg(unix)'.dependencies]
+# tidy-alphabetical-start
 libc = "0.2"
+# tidy-alphabetical-end
 
 [target.'cfg(windows)'.dependencies.windows]
 version = "0.48.0"
@@ -66,6 +66,7 @@ features = [
 ]
 
 [features]
+# tidy-alphabetical-start
 llvm = ['rustc_interface/llvm']
 max_level_info = ['rustc_log/max_level_info']
 rustc_use_parallel_compiler = [
@@ -73,3 +74,4 @@ rustc_use_parallel_compiler = [
     'rustc_interface/rustc_use_parallel_compiler',
     'rustc_middle/rustc_use_parallel_compiler'
 ]
+# tidy-alphabetical-end
diff --git a/compiler/rustc_error_codes/Cargo.toml b/compiler/rustc_error_codes/Cargo.toml
index 7d5f3e4672a0..de668b81b7e0 100644
--- a/compiler/rustc_error_codes/Cargo.toml
+++ b/compiler/rustc_error_codes/Cargo.toml
@@ -2,3 +2,7 @@
 name = "rustc_error_codes"
 version = "0.0.0"
 edition = "2021"
+
+[dependencies]
+# tidy-alphabetical-start
+# tidy-alphabetical-end
diff --git a/compiler/rustc_error_messages/Cargo.toml b/compiler/rustc_error_messages/Cargo.toml
index 4df5a8d48fd1..1969feed48ff 100644
--- a/compiler/rustc_error_messages/Cargo.toml
+++ b/compiler/rustc_error_messages/Cargo.toml
@@ -3,23 +3,25 @@ name = "rustc_error_messages"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 fluent-bundle = "0.15.2"
 fluent-syntax = "0.11"
+icu_list = "1.2"
+icu_locid = "1.2"
+icu_provider_adapters = "1.2"
 intl-memoizer = "0.5.1"
 rustc_baked_icu_data = { path = "../rustc_baked_icu_data" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
+rustc_macros = { path = "../rustc_macros" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_span = { path = "../rustc_span" }
-rustc_macros = { path = "../rustc_macros" }
 tracing = "0.1"
 unic-langid = { version = "0.9.0", features = ["macros"] }
-icu_list = "1.2"
-icu_locid = "1.2"
-icu_provider_adapters = "1.2"
+# tidy-alphabetical-end
 
 [features]
+# tidy-alphabetical-start
 rustc_use_parallel_compiler = ['rustc_baked_icu_data/rustc_use_parallel_compiler']
+# tidy-alphabetical-end
diff --git a/compiler/rustc_errors/Cargo.toml b/compiler/rustc_errors/Cargo.toml
index faab9f09da83..fc3ff835a813 100644
--- a/compiler/rustc_errors/Cargo.toml
+++ b/compiler/rustc_errors/Cargo.toml
@@ -3,29 +3,29 @@ name = "rustc_errors"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
-tracing = "0.1"
+# tidy-alphabetical-start
+annotate-snippets = "0.9"
+derive_setters = "0.1.6"
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
+rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_error_messages = { path = "../rustc_error_messages" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
-rustc_serialize = { path = "../rustc_serialize" }
-rustc_span = { path = "../rustc_span" }
-rustc_macros = { path = "../rustc_macros" }
-rustc_data_structures = { path = "../rustc_data_structures" }
-rustc_target = { path = "../rustc_target" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_lint_defs = { path = "../rustc_lint_defs" }
+rustc_macros = { path = "../rustc_macros" }
+rustc_serialize = { path = "../rustc_serialize" }
+rustc_span = { path = "../rustc_span" }
+rustc_target = { path = "../rustc_target" }
 rustc_type_ir = { path = "../rustc_type_ir" }
-unicode-width = "0.1.4"
-termcolor = "1.2.0"
-annotate-snippets = "0.9"
-termize = "0.1.1"
 serde = { version = "1.0.125", features = [ "derive" ] }
 serde_json = "1.0.59"
-derive_setters = "0.1.6"
+termcolor = "1.2.0"
+termize = "0.1.1"
+tracing = "0.1"
+unicode-width = "0.1.4"
+# tidy-alphabetical-end
 
 [target.'cfg(windows)'.dependencies.windows]
 version = "0.48.0"
@@ -36,4 +36,6 @@ features = [
 ]
 
 [features]
+# tidy-alphabetical-start
 rustc_use_parallel_compiler = ['rustc_error_messages/rustc_use_parallel_compiler']
+# tidy-alphabetical-end
diff --git a/compiler/rustc_expand/Cargo.toml b/compiler/rustc_expand/Cargo.toml
index 02da5b5dc53f..9189a501aa5f 100644
--- a/compiler/rustc_expand/Cargo.toml
+++ b/compiler/rustc_expand/Cargo.toml
@@ -8,9 +8,10 @@ build = false
 doctest = false
 
 [dependencies]
+# tidy-alphabetical-start
 crossbeam-channel = "0.5.0"
-rustc_ast_passes = { path = "../rustc_ast_passes" }
 rustc_ast = { path = "../rustc_ast" }
+rustc_ast_passes = { path = "../rustc_ast_passes" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_attr = { path = "../rustc_attr" }
 rustc_data_structures = { path = "../rustc_data_structures" }
@@ -25,6 +26,7 @@ rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
+termcolor = "1.2"
 thin-vec = "0.2.12"
 tracing = "0.1"
-termcolor = "1.2"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_feature/Cargo.toml b/compiler/rustc_feature/Cargo.toml
index 6f6468646fc7..9df320e1279e 100644
--- a/compiler/rustc_feature/Cargo.toml
+++ b/compiler/rustc_feature/Cargo.toml
@@ -3,8 +3,8 @@ name = "rustc_feature"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_span = { path = "../rustc_span" }
+# tidy-alphabetical-end
diff --git a/compiler/rustc_fluent_macro/Cargo.toml b/compiler/rustc_fluent_macro/Cargo.toml
index 60b8e1e3786d..872dd29a7a8a 100644
--- a/compiler/rustc_fluent_macro/Cargo.toml
+++ b/compiler/rustc_fluent_macro/Cargo.toml
@@ -7,10 +7,12 @@ edition = "2021"
 proc-macro = true
 
 [dependencies]
+# tidy-alphabetical-start
 annotate-snippets = "0.9"
 fluent-bundle = "0.15.2"
 fluent-syntax = "0.11"
-syn = { version = "2", features = ["full"] }
 proc-macro2 = "1"
 quote = "1"
+syn = { version = "2", features = ["full"] }
 unic-langid = { version = "0.9.0", features = ["macros"] }
+# tidy-alphabetical-end
diff --git a/compiler/rustc_fs_util/Cargo.toml b/compiler/rustc_fs_util/Cargo.toml
index 34c3fe2a005c..4b76200c06c5 100644
--- a/compiler/rustc_fs_util/Cargo.toml
+++ b/compiler/rustc_fs_util/Cargo.toml
@@ -2,3 +2,7 @@
 name = "rustc_fs_util"
 version = "0.0.0"
 edition = "2021"
+
+[dependencies]
+# tidy-alphabetical-start
+# tidy-alphabetical-end
diff --git a/compiler/rustc_graphviz/Cargo.toml b/compiler/rustc_graphviz/Cargo.toml
index d657fdb1a778..780004ae3fbc 100644
--- a/compiler/rustc_graphviz/Cargo.toml
+++ b/compiler/rustc_graphviz/Cargo.toml
@@ -2,3 +2,7 @@
 name = "rustc_graphviz"
 version = "0.0.0"
 edition = "2021"
+
+[dependencies]
+# tidy-alphabetical-start
+# tidy-alphabetical-end
diff --git a/compiler/rustc_hir/Cargo.toml b/compiler/rustc_hir/Cargo.toml
index 129f8d235adb..a72c4d0f18bc 100644
--- a/compiler/rustc_hir/Cargo.toml
+++ b/compiler/rustc_hir/Cargo.toml
@@ -3,18 +3,18 @@ name = "rustc_hir"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
+odht = { version = "0.3.1", features = ["nightly"] }
 rustc_arena = { path = "../rustc_arena" }
-rustc_target = { path = "../rustc_target" }
-rustc_macros = { path = "../rustc_macros" }
+rustc_ast = { path = "../rustc_ast" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_error_messages = { path = "../rustc_error_messages" }
 rustc_index = { path = "../rustc_index" }
-rustc_span = { path = "../rustc_span" }
+rustc_macros = { path = "../rustc_macros" }
 rustc_serialize = { path = "../rustc_serialize" }
-rustc_ast = { path = "../rustc_ast" }
-tracing = "0.1"
+rustc_span = { path = "../rustc_span" }
+rustc_target = { path = "../rustc_target" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-odht = { version = "0.3.1", features = ["nightly"] }
+tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_hir_analysis/Cargo.toml b/compiler/rustc_hir_analysis/Cargo.toml
index 4eb94d5bc634..b671bebeb050 100644
--- a/compiler/rustc_hir_analysis/Cargo.toml
+++ b/compiler/rustc_hir_analysis/Cargo.toml
@@ -8,23 +8,25 @@ test = false
 doctest = false
 
 [dependencies]
+# tidy-alphabetical-start
 rustc_arena = { path = "../rustc_arena" }
-rustc_macros = { path = "../rustc_macros" }
-rustc_middle = { path = "../rustc_middle" }
+rustc_ast = { path = "../rustc_ast" }
 rustc_attr = { path = "../rustc_attr" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
-rustc_hir = { path = "../rustc_hir" }
+rustc_feature = { path = "../rustc_feature" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
-rustc_target = { path = "../rustc_target" }
-rustc_session = { path = "../rustc_session" }
-smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-rustc_ast = { path = "../rustc_ast" }
-rustc_span = { path = "../rustc_span" }
+rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
 rustc_infer = { path = "../rustc_infer" }
-rustc_trait_selection = { path = "../rustc_trait_selection" }
 rustc_lint_defs = { path = "../rustc_lint_defs" }
+rustc_macros = { path = "../rustc_macros" }
+rustc_middle = { path = "../rustc_middle" }
+rustc_session = { path = "../rustc_session" }
+rustc_span = { path = "../rustc_span" }
+rustc_target = { path = "../rustc_target" }
+rustc_trait_selection = { path = "../rustc_trait_selection" }
 rustc_type_ir = { path = "../rustc_type_ir" }
-rustc_feature = { path = "../rustc_feature" }
+smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_hir_pretty/Cargo.toml b/compiler/rustc_hir_pretty/Cargo.toml
index 1ea7be1ae7b5..aacf41b6eb73 100644
--- a/compiler/rustc_hir_pretty/Cargo.toml
+++ b/compiler/rustc_hir_pretty/Cargo.toml
@@ -3,11 +3,11 @@ name = "rustc_hir_pretty"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
+rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_hir = { path = "../rustc_hir" }
-rustc_target = { path = "../rustc_target" }
 rustc_span = { path = "../rustc_span" }
-rustc_ast = { path = "../rustc_ast" }
+rustc_target = { path = "../rustc_target" }
+# tidy-alphabetical-end
diff --git a/compiler/rustc_hir_typeck/Cargo.toml b/compiler/rustc_hir_typeck/Cargo.toml
index 0666eeee4d38..0062889d2445 100644
--- a/compiler/rustc_hir_typeck/Cargo.toml
+++ b/compiler/rustc_hir_typeck/Cargo.toml
@@ -3,28 +3,28 @@ name = "rustc_hir_typeck"
 version = "0.0.0"
 edition = "2021"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies]
-smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-tracing = "0.1"
+# tidy-alphabetical-start
 rustc_ast = { path = "../rustc_ast" }
 rustc_attr = { path = "../rustc_attr" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
-rustc_graphviz = { path = "../rustc_graphviz" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
-rustc_index = { path = "../rustc_index" }
-rustc_infer = { path = "../rustc_infer" }
+rustc_graphviz = { path = "../rustc_graphviz" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_hir_analysis = { path = "../rustc_hir_analysis" }
 rustc_hir_pretty = { path = "../rustc_hir_pretty" }
+rustc_index = { path = "../rustc_index" }
+rustc_infer = { path = "../rustc_infer" }
 rustc_lint = { path = "../rustc_lint" }
-rustc_middle = { path = "../rustc_middle" }
 rustc_macros = { path = "../rustc_macros" }
+rustc_middle = { path = "../rustc_middle" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 rustc_trait_selection = { path = "../rustc_trait_selection" }
 rustc_type_ir = { path = "../rustc_type_ir" }
+smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
+tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_incremental/Cargo.toml b/compiler/rustc_incremental/Cargo.toml
index 59a0623c1e1b..46a63b02e846 100644
--- a/compiler/rustc_incremental/Cargo.toml
+++ b/compiler/rustc_incremental/Cargo.toml
@@ -3,15 +3,14 @@ name = "rustc_incremental"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 rand = "0.8.4"
 rustc_ast = { path = "../rustc_ast" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
-rustc_fs_util = { path = "../rustc_fs_util" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
+rustc_fs_util = { path = "../rustc_fs_util" }
 rustc_graphviz = { path = "../rustc_graphviz" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_macros = { path = "../rustc_macros" }
@@ -21,3 +20,4 @@ rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 thin-vec = "0.2.12"
 tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_index/Cargo.toml b/compiler/rustc_index/Cargo.toml
index e1cda5a9edda..856f8a67dd61 100644
--- a/compiler/rustc_index/Cargo.toml
+++ b/compiler/rustc_index/Cargo.toml
@@ -3,14 +3,16 @@ name = "rustc_index"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 arrayvec = { version = "0.7", default-features = false }
-rustc_serialize = { path = "../rustc_serialize", optional = true }
 rustc_macros = { path = "../rustc_macros", optional = true }
+rustc_serialize = { path = "../rustc_serialize", optional = true }
 smallvec = "1.8.1"
+# tidy-alphabetical-end
 
 [features]
+# tidy-alphabetical-start
 default = ["nightly"]
 nightly = ["rustc_serialize", "rustc_macros"]
+# tidy-alphabetical-end
diff --git a/compiler/rustc_infer/Cargo.toml b/compiler/rustc_infer/Cargo.toml
index 9dd5868adc71..00251a192264 100644
--- a/compiler/rustc_infer/Cargo.toml
+++ b/compiler/rustc_infer/Cargo.toml
@@ -7,15 +7,17 @@ edition = "2021"
 doctest = false
 
 [dependencies]
-tracing = "0.1"
-rustc_middle = { path = "../rustc_middle" }
+# tidy-alphabetical-start
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
-rustc_hir = { path = "../rustc_hir" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
+rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
+rustc_middle = { path = "../rustc_middle" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
+tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_interface/Cargo.toml b/compiler/rustc_interface/Cargo.toml
index b280665057cc..a73152601b35 100644
--- a/compiler/rustc_interface/Cargo.toml
+++ b/compiler/rustc_interface/Cargo.toml
@@ -3,54 +3,56 @@ name = "rustc_interface"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 libloading = "0.7.1"
-tracing = "0.1"
-rustc-rayon-core = { version = "0.5.0", optional = true }
 rustc-rayon = { version = "0.5.0", optional = true }
+rustc-rayon-core = { version = "0.5.0", optional = true }
 rustc_ast = { path = "../rustc_ast" }
+rustc_ast_lowering = { path = "../rustc_ast_lowering" }
+rustc_ast_passes = { path = "../rustc_ast_passes" }
 rustc_attr = { path = "../rustc_attr" }
 rustc_borrowck = { path = "../rustc_borrowck" }
 rustc_builtin_macros = { path = "../rustc_builtin_macros" }
+rustc_codegen_llvm = { path = "../rustc_codegen_llvm", optional = true }
+rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }
+rustc_const_eval = { path = "../rustc_const_eval" }
+rustc_data_structures = { path = "../rustc_data_structures" }
+rustc_errors = { path = "../rustc_errors" }
 rustc_expand = { path = "../rustc_expand" }
 rustc_feature = { path = "../rustc_feature" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_fs_util = { path = "../rustc_fs_util" }
-rustc_macros = { path = "../rustc_macros" }
-rustc_parse = { path = "../rustc_parse" }
-rustc_session = { path = "../rustc_session" }
-rustc_span = { path = "../rustc_span" }
-rustc_middle = { path = "../rustc_middle" }
-rustc_ast_lowering = { path = "../rustc_ast_lowering" }
-rustc_ast_passes = { path = "../rustc_ast_passes" }
-rustc_incremental = { path = "../rustc_incremental" }
-rustc_traits = { path = "../rustc_traits" }
-rustc_data_structures = { path = "../rustc_data_structures" }
-rustc_codegen_ssa = { path = "../rustc_codegen_ssa" }
-rustc_symbol_mangling = { path = "../rustc_symbol_mangling" }
-rustc_codegen_llvm = { path = "../rustc_codegen_llvm", optional = true }
 rustc_hir = { path = "../rustc_hir" }
+rustc_hir_analysis = { path = "../rustc_hir_analysis" }
+rustc_hir_typeck = { path = "../rustc_hir_typeck" }
+rustc_incremental = { path = "../rustc_incremental" }
+rustc_lint = { path = "../rustc_lint" }
+rustc_macros = { path = "../rustc_macros" }
 rustc_metadata = { path = "../rustc_metadata" }
-rustc_const_eval = { path = "../rustc_const_eval" }
+rustc_middle = { path = "../rustc_middle" }
 rustc_mir_build = { path = "../rustc_mir_build" }
 rustc_mir_transform = { path = "../rustc_mir_transform" }
 rustc_monomorphize = { path = "../rustc_monomorphize" }
+rustc_parse = { path = "../rustc_parse" }
 rustc_passes = { path = "../rustc_passes" }
-rustc_hir_analysis = { path = "../rustc_hir_analysis" }
-rustc_hir_typeck = { path = "../rustc_hir_typeck" }
-rustc_lint = { path = "../rustc_lint" }
-rustc_errors = { path = "../rustc_errors" }
 rustc_plugin_impl = { path = "../rustc_plugin_impl" }
 rustc_privacy = { path = "../rustc_privacy" }
-rustc_query_system = { path = "../rustc_query_system" }
 rustc_query_impl = { path = "../rustc_query_impl" }
+rustc_query_system = { path = "../rustc_query_system" }
 rustc_resolve = { path = "../rustc_resolve" }
+rustc_session = { path = "../rustc_session" }
+rustc_span = { path = "../rustc_span" }
+rustc_symbol_mangling = { path = "../rustc_symbol_mangling" }
 rustc_target = { path = "../rustc_target" }
 rustc_trait_selection = { path = "../rustc_trait_selection" }
+rustc_traits = { path = "../rustc_traits" }
 rustc_ty_utils = { path = "../rustc_ty_utils" }
+tracing = "0.1"
+# tidy-alphabetical-end
 
 [features]
+# tidy-alphabetical-start
 llvm = ['rustc_codegen_llvm']
 rustc_use_parallel_compiler = ['rustc-rayon', 'rustc-rayon-core', 'rustc_query_impl/rustc_use_parallel_compiler', 'rustc_errors/rustc_use_parallel_compiler']
+# tidy-alphabetical-end
diff --git a/compiler/rustc_lexer/Cargo.toml b/compiler/rustc_lexer/Cargo.toml
index 373a8970de83..84b9e2922955 100644
--- a/compiler/rustc_lexer/Cargo.toml
+++ b/compiler/rustc_lexer/Cargo.toml
@@ -3,7 +3,6 @@ name = "rustc_lexer"
 version = "0.0.0"
 license = "MIT OR Apache-2.0"
 edition = "2021"
-
 repository = "https://github.com/rust-lang/rust/"
 description = """
 Rust lexer used by rustc. No stability guarantees are provided.
diff --git a/compiler/rustc_lint/Cargo.toml b/compiler/rustc_lint/Cargo.toml
index 539eea3d816a..fa1133e7780f 100644
--- a/compiler/rustc_lint/Cargo.toml
+++ b/compiler/rustc_lint/Cargo.toml
@@ -4,23 +4,25 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
-tracing = "0.1"
-unicode-security = "0.1.0"
-rustc_middle = { path = "../rustc_middle" }
+# tidy-alphabetical-start
+rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_attr = { path = "../rustc_attr" }
-rustc_errors = { path = "../rustc_errors" }
-rustc_hir = { path = "../rustc_hir" }
-rustc_fluent_macro = { path = "../rustc_fluent_macro" }
-rustc_target = { path = "../rustc_target" }
-rustc_ast = { path = "../rustc_ast" }
-rustc_span = { path = "../rustc_span" }
 rustc_data_structures = { path = "../rustc_data_structures" }
+rustc_errors = { path = "../rustc_errors" }
 rustc_feature = { path = "../rustc_feature" }
+rustc_fluent_macro = { path = "../rustc_fluent_macro" }
+rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
-rustc_session = { path = "../rustc_session" }
-rustc_trait_selection = { path = "../rustc_trait_selection" }
-rustc_parse_format = { path = "../rustc_parse_format" }
 rustc_infer = { path = "../rustc_infer" }
-rustc_type_ir = { path = "../rustc_type_ir" }
 rustc_macros = { path = "../rustc_macros" }
+rustc_middle = { path = "../rustc_middle" }
+rustc_parse_format = { path = "../rustc_parse_format" }
+rustc_session = { path = "../rustc_session" }
+rustc_span = { path = "../rustc_span" }
+rustc_target = { path = "../rustc_target" }
+rustc_trait_selection = { path = "../rustc_trait_selection" }
+rustc_type_ir = { path = "../rustc_type_ir" }
+tracing = "0.1"
+unicode-security = "0.1.0"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_lint_defs/Cargo.toml b/compiler/rustc_lint_defs/Cargo.toml
index 2bf34d82f395..eb2a184ef84b 100644
--- a/compiler/rustc_lint_defs/Cargo.toml
+++ b/compiler/rustc_lint_defs/Cargo.toml
@@ -4,12 +4,14 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
-serde = { version = "1.0.125", features = ["derive"] }
+# tidy-alphabetical-start
 rustc_ast = { path = "../rustc_ast" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_error_messages = { path = "../rustc_error_messages" }
-rustc_span = { path = "../rustc_span" }
-rustc_serialize = { path = "../rustc_serialize" }
-rustc_macros = { path = "../rustc_macros" }
-rustc_target = { path = "../rustc_target" }
 rustc_hir = { path = "../rustc_hir" }
+rustc_macros = { path = "../rustc_macros" }
+rustc_serialize = { path = "../rustc_serialize" }
+rustc_span = { path = "../rustc_span" }
+rustc_target = { path = "../rustc_target" }
+serde = { version = "1.0.125", features = ["derive"] }
+# tidy-alphabetical-end
diff --git a/compiler/rustc_llvm/Cargo.toml b/compiler/rustc_llvm/Cargo.toml
index 3880f25a9ba9..58e219e5a463 100644
--- a/compiler/rustc_llvm/Cargo.toml
+++ b/compiler/rustc_llvm/Cargo.toml
@@ -4,7 +4,11 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
+# tidy-alphabetical-start
 libc = "0.2.73"
+# tidy-alphabetical-end
 
 [build-dependencies]
+# tidy-alphabetical-start
 cc = "1.0.69"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_log/Cargo.toml b/compiler/rustc_log/Cargo.toml
index aa6e46cd8de7..6009a43e9855 100644
--- a/compiler/rustc_log/Cargo.toml
+++ b/compiler/rustc_log/Cargo.toml
@@ -4,13 +4,19 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
+# tidy-alphabetical-start
 tracing = "0.1.28"
+tracing-core = "=0.1.30" # FIXME(Nilstrieb) tracing has a deadlock: https://github.com/tokio-rs/tracing/issues/2635
 tracing-subscriber = { version = "0.3.3", default-features = false, features = ["fmt", "env-filter", "smallvec", "parking_lot", "ansi"] }
 tracing-tree = "0.2.0"
-tracing-core = "=0.1.30" # FIXME(Nilstrieb) tracing has a deadlock: https://github.com/tokio-rs/tracing/issues/2635
+# tidy-alphabetical-end
 
 [dev-dependencies]
+# tidy-alphabetical-start
 rustc_span = { path = "../rustc_span" }
+# tidy-alphabetical-end
 
 [features]
+# tidy-alphabetical-start
 max_level_info = ['tracing/max_level_info']
+# tidy-alphabetical-end
diff --git a/compiler/rustc_macros/Cargo.toml b/compiler/rustc_macros/Cargo.toml
index 6e7e19a24024..d8d2bef49640 100644
--- a/compiler/rustc_macros/Cargo.toml
+++ b/compiler/rustc_macros/Cargo.toml
@@ -7,7 +7,9 @@ edition = "2021"
 proc-macro = true
 
 [dependencies]
-synstructure = "0.13.0"
-syn = { version = "2.0.9", features = ["full"] }
+# tidy-alphabetical-start
 proc-macro2 = "1"
 quote = "1"
+syn = { version = "2.0.9", features = ["full"] }
+synstructure = "0.13.0"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_metadata/Cargo.toml b/compiler/rustc_metadata/Cargo.toml
index 840111c31b4d..a4012592c099 100644
--- a/compiler/rustc_metadata/Cargo.toml
+++ b/compiler/rustc_metadata/Cargo.toml
@@ -3,30 +3,30 @@ name = "rustc_metadata"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 bitflags = "1.2.1"
 libloading = "0.7.1"
 odht = { version = "0.3.1", features = ["nightly"] }
-snap = "1"
-tracing = "0.1"
-tempfile = "3.2"
-rustc_middle = { path = "../rustc_middle" }
+rustc_ast = { path = "../rustc_ast" }
 rustc_attr = { path = "../rustc_attr" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
+rustc_expand = { path = "../rustc_expand" }
 rustc_feature = { path = "../rustc_feature" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_fs_util = { path = "../rustc_fs_util" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_hir_pretty = { path = "../rustc_hir_pretty" }
-rustc_target = { path = "../rustc_target" }
 rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
+rustc_middle = { path = "../rustc_middle" }
 rustc_serialize = { path = "../rustc_serialize" }
-rustc_ast = { path = "../rustc_ast" }
-rustc_expand = { path = "../rustc_expand" }
-rustc_span = { path = "../rustc_span" }
 rustc_session = { path = "../rustc_session" }
+rustc_span = { path = "../rustc_span" }
+rustc_target = { path = "../rustc_target" }
 rustc_type_ir = { path = "../rustc_type_ir" }
+snap = "1"
+tempfile = "3.2"
+tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_middle/Cargo.toml b/compiler/rustc_middle/Cargo.toml
index bb8e774cea3d..d356984c1e9d 100644
--- a/compiler/rustc_middle/Cargo.toml
+++ b/compiler/rustc_middle/Cargo.toml
@@ -3,24 +3,24 @@ name = "rustc_middle"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 bitflags = "1.2.1"
 derive_more = "0.99.17"
 either = "1.5.0"
-gsgdt = "0.1.2"
 field-offset = "0.3.5"
+gsgdt = "0.1.2"
 measureme = "10.0.0"
 polonius-engine = "0.13.0"
+rustc-rayon = { version = "0.5.0", optional = true }
+rustc-rayon-core = { version = "0.5.0", optional = true }
 rustc_apfloat = "0.2.0"
 rustc_arena = { path = "../rustc_arena" }
 rustc_ast = { path = "../rustc_ast" }
 rustc_attr = { path = "../rustc_attr" }
 rustc_data_structures = { path = "../rustc_data_structures" }
+rustc_error_messages = { path = "../rustc_error_messages" } # Used for intra-doc links
 rustc_errors = { path = "../rustc_errors" }
-# Used for intra-doc links
-rustc_error_messages = { path = "../rustc_error_messages" }
 rustc_feature = { path = "../rustc_feature" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_graphviz = { path = "../rustc_graphviz" }
@@ -28,8 +28,6 @@ rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_query_system = { path = "../rustc_query_system" }
-rustc-rayon-core = { version = "0.5.0", optional = true }
-rustc-rayon = { version = "0.5.0", optional = true }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
@@ -38,6 +36,9 @@ rustc_type_ir = { path = "../rustc_type_ir" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 thin-vec = "0.2.12"
 tracing = "0.1"
+# tidy-alphabetical-end
 
 [features]
+# tidy-alphabetical-start
 rustc_use_parallel_compiler = ["rustc-rayon", "rustc-rayon-core"]
+# tidy-alphabetical-end
diff --git a/compiler/rustc_mir_build/Cargo.toml b/compiler/rustc_mir_build/Cargo.toml
index c7e2c625ce57..6dceacd75a54 100644
--- a/compiler/rustc_mir_build/Cargo.toml
+++ b/compiler/rustc_mir_build/Cargo.toml
@@ -3,25 +3,25 @@ name = "rustc_mir_build"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
-rustc_arena = { path = "../rustc_arena" }
-tracing = "0.1"
+# tidy-alphabetical-start
 either = "1"
-rustc_middle = { path = "../rustc_middle" }
 rustc_apfloat = "0.2.0"
+rustc_arena = { path = "../rustc_arena" }
+rustc_ast = { path = "../rustc_ast" }
 rustc_data_structures = { path = "../rustc_data_structures" }
-rustc_index = { path = "../rustc_index" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_hir = { path = "../rustc_hir" }
+rustc_index = { path = "../rustc_index" }
 rustc_infer = { path = "../rustc_infer" }
 rustc_macros = { path = "../rustc_macros" }
+rustc_middle = { path = "../rustc_middle" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 rustc_trait_selection = { path = "../rustc_trait_selection" }
-rustc_ast = { path = "../rustc_ast" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
+tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_mir_dataflow/Cargo.toml b/compiler/rustc_mir_dataflow/Cargo.toml
index 4a296bb33670..61664eb2a6fb 100644
--- a/compiler/rustc_mir_dataflow/Cargo.toml
+++ b/compiler/rustc_mir_dataflow/Cargo.toml
@@ -3,13 +3,10 @@ name = "rustc_mir_dataflow"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 polonius-engine = "0.13.0"
 regex = "1"
-smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-tracing = "0.1"
 rustc_ast = { path = "../rustc_ast" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
@@ -20,5 +17,8 @@ rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_serialize = { path = "../rustc_serialize" }
-rustc_target = { path = "../rustc_target" }
 rustc_span = { path = "../rustc_span" }
+rustc_target = { path = "../rustc_target" }
+smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
+tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_mir_transform/Cargo.toml b/compiler/rustc_mir_transform/Cargo.toml
index 9ec0bb4ab94a..0448e9d276d4 100644
--- a/compiler/rustc_mir_transform/Cargo.toml
+++ b/compiler/rustc_mir_transform/Cargo.toml
@@ -3,31 +3,33 @@ name = "rustc_mir_transform"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
-itertools = "0.10.1"
-smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
-tracing = "0.1"
+# tidy-alphabetical-start
 either = "1"
-rustc_ast = { path = "../rustc_ast" }
+itertools = "0.10.1"
 rustc_arena = { path = "../rustc_arena" }
+rustc_ast = { path = "../rustc_ast" }
 rustc_attr = { path = "../rustc_attr" }
+rustc_const_eval = { path = "../rustc_const_eval" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
+rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
+rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
-rustc_const_eval = { path = "../rustc_const_eval" }
 rustc_mir_build = { path = "../rustc_mir_build" }
 rustc_mir_dataflow = { path = "../rustc_mir_dataflow" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
+rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 rustc_trait_selection = { path = "../rustc_trait_selection" }
-rustc_span = { path = "../rustc_span" }
-rustc_fluent_macro = { path = "../rustc_fluent_macro" }
-rustc_macros = { path = "../rustc_macros" }
+smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
+tracing = "0.1"
+# tidy-alphabetical-end
 
 [dev-dependencies]
+# tidy-alphabetical-start
 coverage_test_macros = { path = "src/coverage/test_macros" }
+# tidy-alphabetical-end
diff --git a/compiler/rustc_monomorphize/Cargo.toml b/compiler/rustc_monomorphize/Cargo.toml
index fe097424e8ad..c7f1b9fa7845 100644
--- a/compiler/rustc_monomorphize/Cargo.toml
+++ b/compiler/rustc_monomorphize/Cargo.toml
@@ -3,18 +3,18 @@ name = "rustc_monomorphize"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
-serde = "1"
-serde_json = "1"
-tracing = "0.1"
+# tidy-alphabetical-start
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
-rustc_hir = { path = "../rustc_hir" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
+rustc_hir = { path = "../rustc_hir" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
+serde = "1"
+serde_json = "1"
+tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_parse/Cargo.toml b/compiler/rustc_parse/Cargo.toml
index 1bd9f66290db..02f9f35f0f56 100644
--- a/compiler/rustc_parse/Cargo.toml
+++ b/compiler/rustc_parse/Cargo.toml
@@ -3,9 +3,8 @@ name = "rustc_parse"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 bitflags = "1.0"
 rustc_ast = { path = "../rustc_ast" }
 rustc_ast_pretty = { path = "../rustc_ast_pretty" }
@@ -21,3 +20,4 @@ thin-vec = "0.2.12"
 tracing = "0.1"
 unicode-normalization = "0.1.11"
 unicode-width = "0.1.4"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_parse_format/Cargo.toml b/compiler/rustc_parse_format/Cargo.toml
index 143303532392..707c4e318474 100644
--- a/compiler/rustc_parse_format/Cargo.toml
+++ b/compiler/rustc_parse_format/Cargo.toml
@@ -4,5 +4,7 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
-rustc_lexer = { path = "../rustc_lexer" }
+# tidy-alphabetical-start
 rustc_index = { path = "../rustc_index", default-features = false }
+rustc_lexer = { path = "../rustc_lexer" }
+# tidy-alphabetical-end
diff --git a/compiler/rustc_passes/Cargo.toml b/compiler/rustc_passes/Cargo.toml
index 0413b5b4fb92..40c3811e054e 100644
--- a/compiler/rustc_passes/Cargo.toml
+++ b/compiler/rustc_passes/Cargo.toml
@@ -4,23 +4,25 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
-tracing = "0.1"
+# tidy-alphabetical-start
 itertools = "0.10.1"
-rustc_middle = { path = "../rustc_middle" }
+rustc_ast = { path = "../rustc_ast" }
+rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_attr = { path = "../rustc_attr" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_expand = { path = "../rustc_expand" }
-rustc_hir = { path = "../rustc_hir" }
-rustc_fluent_macro = { path = "../rustc_fluent_macro" }
-rustc_index = { path = "../rustc_index" }
-rustc_session = { path = "../rustc_session" }
-rustc_target = { path = "../rustc_target" }
-rustc_macros = { path = "../rustc_macros" }
-rustc_ast = { path = "../rustc_ast" }
-rustc_serialize = { path = "../rustc_serialize" }
-rustc_span = { path = "../rustc_span" }
-rustc_lexer = { path = "../rustc_lexer" }
-rustc_ast_pretty = { path = "../rustc_ast_pretty" }
 rustc_feature = { path = "../rustc_feature" }
+rustc_fluent_macro = { path = "../rustc_fluent_macro" }
+rustc_hir = { path = "../rustc_hir" }
+rustc_index = { path = "../rustc_index" }
+rustc_lexer = { path = "../rustc_lexer" }
+rustc_macros = { path = "../rustc_macros" }
+rustc_middle = { path = "../rustc_middle" }
+rustc_serialize = { path = "../rustc_serialize" }
+rustc_session = { path = "../rustc_session" }
+rustc_span = { path = "../rustc_span" }
+rustc_target = { path = "../rustc_target" }
 rustc_trait_selection = { path = "../rustc_trait_selection" }
+tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_plugin_impl/Cargo.toml b/compiler/rustc_plugin_impl/Cargo.toml
index c930b3365fdd..a0762e815e3f 100644
--- a/compiler/rustc_plugin_impl/Cargo.toml
+++ b/compiler/rustc_plugin_impl/Cargo.toml
@@ -4,15 +4,15 @@ version = "0.0.0"
 build = false
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 libloading = "0.7.1"
+rustc_ast = { path = "../rustc_ast" }
 rustc_errors = { path = "../rustc_errors" }
+rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_lint = { path = "../rustc_lint" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_metadata = { path = "../rustc_metadata" }
-rustc_fluent_macro = { path = "../rustc_fluent_macro" }
-rustc_ast = { path = "../rustc_ast" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
+# tidy-alphabetical-end
diff --git a/compiler/rustc_privacy/Cargo.toml b/compiler/rustc_privacy/Cargo.toml
index 08c4067705cb..90803c20d431 100644
--- a/compiler/rustc_privacy/Cargo.toml
+++ b/compiler/rustc_privacy/Cargo.toml
@@ -4,15 +4,17 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
+# tidy-alphabetical-start
 rustc_ast = { path = "../rustc_ast" }
 rustc_attr = { path = "../rustc_attr" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
-rustc_hir = { path = "../rustc_hir" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
+rustc_hir = { path = "../rustc_hir" }
+rustc_hir_analysis = { path = "../rustc_hir_analysis" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
-rustc_hir_analysis = { path = "../rustc_hir_analysis" }
 tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_query_impl/Cargo.toml b/compiler/rustc_query_impl/Cargo.toml
index dac86116d5b6..88eb46030697 100644
--- a/compiler/rustc_query_impl/Cargo.toml
+++ b/compiler/rustc_query_impl/Cargo.toml
@@ -3,24 +3,25 @@ name = "rustc_query_impl"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
-
 [dependencies]
+# tidy-alphabetical-start
 field-offset = "0.3.5"
 measureme = "10.0.0"
+rustc-rayon-core = { version = "0.5.0", optional = true }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
 rustc_middle = { path = "../rustc_middle" }
 rustc_query_system = { path = "../rustc_query_system" }
-rustc-rayon-core = { version = "0.5.0", optional = true }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
 thin-vec = "0.2.12"
 tracing = "0.1"
+# tidy-alphabetical-end
 
 [features]
+# tidy-alphabetical-start
 rustc_use_parallel_compiler = ["rustc-rayon-core", "rustc_query_system/rustc_use_parallel_compiler"]
+# tidy-alphabetical-end
diff --git a/compiler/rustc_query_system/Cargo.toml b/compiler/rustc_query_system/Cargo.toml
index 584355df8024..6ea87a4a633e 100644
--- a/compiler/rustc_query_system/Cargo.toml
+++ b/compiler/rustc_query_system/Cargo.toml
@@ -3,10 +3,10 @@ name = "rustc_query_system"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 parking_lot = "0.12"
+rustc-rayon-core = { version = "0.5.0", optional = true }
 rustc_ast = { path = "../rustc_ast" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
@@ -15,7 +15,6 @@ rustc_fluent_macro = { path = "../rustc_fluent_macro" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
-rustc-rayon-core = { version = "0.5.0", optional = true }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
@@ -24,6 +23,9 @@ rustc_type_ir = { path = "../rustc_type_ir" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 thin-vec = "0.2.12"
 tracing = "0.1"
+# tidy-alphabetical-end
 
 [features]
+# tidy-alphabetical-start
 rustc_use_parallel_compiler = ["rustc-rayon-core"]
+# tidy-alphabetical-end
diff --git a/compiler/rustc_resolve/Cargo.toml b/compiler/rustc_resolve/Cargo.toml
index 46da0aa2853e..ff9d6d8739f2 100644
--- a/compiler/rustc_resolve/Cargo.toml
+++ b/compiler/rustc_resolve/Cargo.toml
@@ -3,9 +3,8 @@ name = "rustc_resolve"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 bitflags = "1.2.1"
 pulldown-cmark = { version = "0.9.3", default-features = false }
 rustc_arena = { path = "../rustc_arena" }
@@ -28,3 +27,4 @@ rustc_span = { path = "../rustc_span" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 thin-vec = "0.2.12"
 tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_serialize/Cargo.toml b/compiler/rustc_serialize/Cargo.toml
index 46b923e8c7b7..8bf98c16361e 100644
--- a/compiler/rustc_serialize/Cargo.toml
+++ b/compiler/rustc_serialize/Cargo.toml
@@ -4,10 +4,14 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
+# tidy-alphabetical-start
 indexmap = "2.0.0"
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 thin-vec = "0.2.12"
+# tidy-alphabetical-end
 
 [dev-dependencies]
+# tidy-alphabetical-start
 rustc_macros = { path = "../rustc_macros" }
 tempfile = "3.2"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_session/Cargo.toml b/compiler/rustc_session/Cargo.toml
index e26d25d9a412..1f51dd6c975e 100644
--- a/compiler/rustc_session/Cargo.toml
+++ b/compiler/rustc_session/Cargo.toml
@@ -4,26 +4,30 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
+# tidy-alphabetical-start
 bitflags = "1.2.1"
 getopts = "0.2"
-rustc_macros = { path = "../rustc_macros" }
-tracing = "0.1"
+rustc_ast = { path = "../rustc_ast" }
+rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
 rustc_feature = { path = "../rustc_feature" }
-rustc_hir = { path = "../rustc_hir" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
-rustc_target = { path = "../rustc_target" }
-rustc_serialize = { path = "../rustc_serialize" }
-rustc_data_structures = { path = "../rustc_data_structures" }
-rustc_span = { path = "../rustc_span" }
 rustc_fs_util = { path = "../rustc_fs_util" }
-rustc_ast = { path = "../rustc_ast" }
+rustc_hir = { path = "../rustc_hir" }
 rustc_lint_defs = { path = "../rustc_lint_defs" }
+rustc_macros = { path = "../rustc_macros" }
+rustc_serialize = { path = "../rustc_serialize" }
+rustc_span = { path = "../rustc_span" }
+rustc_target = { path = "../rustc_target" }
 smallvec = "1.8.1"
 termize = "0.1.1"
+tracing = "0.1"
+# tidy-alphabetical-end
 
 [target.'cfg(unix)'.dependencies]
+# tidy-alphabetical-start
 libc = "0.2"
+# tidy-alphabetical-end
 
 [target.'cfg(windows)'.dependencies.windows]
 version = "0.48.0"
diff --git a/compiler/rustc_smir/Cargo.toml b/compiler/rustc_smir/Cargo.toml
index 47dd7372f3d1..836ea046ffea 100644
--- a/compiler/rustc_smir/Cargo.toml
+++ b/compiler/rustc_smir/Cargo.toml
@@ -4,6 +4,7 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
+# tidy-alphabetical-start
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_hir = { path = "../rustc_hir" }
 rustc_middle = { path = "../rustc_middle" }
@@ -12,5 +13,4 @@ rustc_target = { path = "../rustc_target" }
 scoped-tls = "1.0"
 stable_mir = {path = "../stable_mir" }
 tracing = "0.1"
-
-[features]
+# tidy-alphabetical-end
diff --git a/compiler/rustc_span/Cargo.toml b/compiler/rustc_span/Cargo.toml
index 31c2a56faa53..99de91a068ad 100644
--- a/compiler/rustc_span/Cargo.toml
+++ b/compiler/rustc_span/Cargo.toml
@@ -3,18 +3,18 @@ name = "rustc_span"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
-rustc_serialize = { path = "../rustc_serialize" }
-rustc_macros = { path = "../rustc_macros" }
+# tidy-alphabetical-start
+indexmap = { version = "2.0.0" }
+md5 = { package = "md-5", version = "0.10.0" }
+rustc_arena = { path = "../rustc_arena" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_index = { path = "../rustc_index" }
-rustc_arena = { path = "../rustc_arena" }
+rustc_macros = { path = "../rustc_macros" }
+rustc_serialize = { path = "../rustc_serialize" }
 scoped-tls = "1.0"
-unicode-width = "0.1.4"
-tracing = "0.1"
 sha1 = "0.10.0"
 sha2 = "0.10.1"
-md5 = { package = "md-5", version = "0.10.0" }
-indexmap = { version = "2.0.0" }
+tracing = "0.1"
+unicode-width = "0.1.4"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_symbol_mangling/Cargo.toml b/compiler/rustc_symbol_mangling/Cargo.toml
index d53bc5b6a8e0..ff3f1ad646fa 100644
--- a/compiler/rustc_symbol_mangling/Cargo.toml
+++ b/compiler/rustc_symbol_mangling/Cargo.toml
@@ -3,19 +3,18 @@ name = "rustc_symbol_mangling"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 bitflags = "1.2.1"
-tracing = "0.1"
 punycode = "0.4.0"
 rustc-demangle = "0.1.21"
-twox-hash = "1.6.3"
-
-rustc_span = { path = "../rustc_span" }
-rustc_middle = { path = "../rustc_middle" }
-rustc_hir = { path = "../rustc_hir" }
-rustc_target = { path = "../rustc_target" }
 rustc_data_structures = { path = "../rustc_data_structures" }
-rustc_session = { path = "../rustc_session" }
 rustc_errors = { path = "../rustc_errors" }
+rustc_hir = { path = "../rustc_hir" }
+rustc_middle = { path = "../rustc_middle" }
+rustc_session = { path = "../rustc_session" }
+rustc_span = { path = "../rustc_span" }
+rustc_target = { path = "../rustc_target" }
+tracing = "0.1"
+twox-hash = "1.6.3"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_target/Cargo.toml b/compiler/rustc_target/Cargo.toml
index 779d03cd094d..94dfeb12dc98 100644
--- a/compiler/rustc_target/Cargo.toml
+++ b/compiler/rustc_target/Cargo.toml
@@ -4,19 +4,23 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
+# tidy-alphabetical-start
 bitflags = "1.2.1"
-tracing = "0.1"
-serde_json = "1.0.59"
-rustc_fs_util = { path = "../rustc_fs_util" }
 rustc_abi = { path = "../rustc_abi" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_feature = { path = "../rustc_feature" }
+rustc_fs_util = { path = "../rustc_fs_util" }
+rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_span = { path = "../rustc_span" }
-rustc_index = { path = "../rustc_index" }
+serde_json = "1.0.59"
+tracing = "0.1"
+# tidy-alphabetical-end
 
 [dependencies.object]
-version = "0.32.0"
+# tidy-alphabetical-start
 default-features = false
 features = ["elf", "macho"]
+version = "0.32.0"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_trait_selection/Cargo.toml b/compiler/rustc_trait_selection/Cargo.toml
index 83605627d6f4..667ee3d4e1c6 100644
--- a/compiler/rustc_trait_selection/Cargo.toml
+++ b/compiler/rustc_trait_selection/Cargo.toml
@@ -3,21 +3,19 @@ name = "rustc_trait_selection"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
-rustc_parse_format = { path = "../rustc_parse_format" }
-tracing = "0.1"
-rustc_attr = { path = "../rustc_attr" }
-rustc_middle = { path = "../rustc_middle" }
+# tidy-alphabetical-start
 rustc_ast = { path = "../rustc_ast" }
+rustc_attr = { path = "../rustc_attr" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
-rustc_hir = { path = "../rustc_hir" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
+rustc_hir = { path = "../rustc_hir" }
 rustc_index = { path = "../rustc_index" }
 rustc_infer = { path = "../rustc_infer" }
 rustc_macros = { path = "../rustc_macros" }
+rustc_middle = { path = "../rustc_middle" }
+rustc_parse_format = { path = "../rustc_parse_format" }
 rustc_query_system = { path = "../rustc_query_system" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
@@ -25,3 +23,5 @@ rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 rustc_transmute = { path = "../rustc_transmute", features = ["rustc"] }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
+tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_traits/Cargo.toml b/compiler/rustc_traits/Cargo.toml
index 0cdc978a3d02..9c788116d98c 100644
--- a/compiler/rustc_traits/Cargo.toml
+++ b/compiler/rustc_traits/Cargo.toml
@@ -4,10 +4,12 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
-tracing = "0.1"
-rustc_middle = { path = "../rustc_middle" }
+# tidy-alphabetical-start
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_hir = { path = "../rustc_hir" }
-rustc_span = { path = "../rustc_span" }
 rustc_infer = { path = "../rustc_infer" }
+rustc_middle = { path = "../rustc_middle" }
+rustc_span = { path = "../rustc_span" }
 rustc_trait_selection = { path = "../rustc_trait_selection" }
+tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_transmute/Cargo.toml b/compiler/rustc_transmute/Cargo.toml
index c2b2730c328f..07420985a851 100644
--- a/compiler/rustc_transmute/Cargo.toml
+++ b/compiler/rustc_transmute/Cargo.toml
@@ -3,10 +3,8 @@ name = "rustc_transmute"
 version = "0.0.0"
 edition = "2021"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
 [dependencies]
-tracing = "0.1"
+# tidy-alphabetical-start
 rustc_data_structures = { path = "../rustc_data_structures"}
 rustc_hir = { path = "../rustc_hir", optional = true}
 rustc_infer = { path = "../rustc_infer", optional = true}
@@ -14,16 +12,20 @@ rustc_macros = { path = "../rustc_macros", optional = true}
 rustc_middle = { path = "../rustc_middle", optional = true}
 rustc_span = { path = "../rustc_span", optional = true}
 rustc_target = { path = "../rustc_target", optional = true}
+tracing = "0.1"
+# tidy-alphabetical-end
 
 [features]
 rustc = [
-    "rustc_middle",
     "rustc_hir",
     "rustc_infer",
     "rustc_macros",
+    "rustc_middle",
     "rustc_span",
     "rustc_target",
 ]
 
 [dev-dependencies]
-itertools = "0.10.1"
\ No newline at end of file
+# tidy-alphabetical-start
+itertools = "0.10.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_ty_utils/Cargo.toml b/compiler/rustc_ty_utils/Cargo.toml
index 50dac3a37a4a..7e00f1e0c422 100644
--- a/compiler/rustc_ty_utils/Cargo.toml
+++ b/compiler/rustc_ty_utils/Cargo.toml
@@ -4,18 +4,20 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
-tracing = "0.1"
+# tidy-alphabetical-start
 itertools = "0.10.1"
-rustc_middle = { path = "../rustc_middle" }
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_errors = { path = "../rustc_errors" }
-rustc_hir = { path = "../rustc_hir" }
 rustc_fluent_macro = { path = "../rustc_fluent_macro" }
+rustc_hir = { path = "../rustc_hir" }
+rustc_index = { path = "../rustc_index" }
 rustc_infer = { path = "../rustc_infer" }
 rustc_macros = { path = "../rustc_macros" }
-rustc_span = { path = "../rustc_span" }
+rustc_middle = { path = "../rustc_middle" }
 rustc_session = { path = "../rustc_session" }
+rustc_span = { path = "../rustc_span" }
 rustc_target = { path = "../rustc_target" }
 rustc_trait_selection = { path = "../rustc_trait_selection" }
 rustc_type_ir = { path = "../rustc_type_ir" }
-rustc_index = { path = "../rustc_index" }
+tracing = "0.1"
+# tidy-alphabetical-end
diff --git a/compiler/rustc_type_ir/Cargo.toml b/compiler/rustc_type_ir/Cargo.toml
index c4008e9b6129..484757e2fc1a 100644
--- a/compiler/rustc_type_ir/Cargo.toml
+++ b/compiler/rustc_type_ir/Cargo.toml
@@ -3,12 +3,12 @@ name = "rustc_type_ir"
 version = "0.0.0"
 edition = "2021"
 
-[lib]
-
 [dependencies]
+# tidy-alphabetical-start
 bitflags = "1.2.1"
-rustc_index = { path = "../rustc_index" }
-rustc_serialize = { path = "../rustc_serialize" }
 rustc_data_structures = { path = "../rustc_data_structures" }
+rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
+rustc_serialize = { path = "../rustc_serialize" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
+# tidy-alphabetical-end

From 51e22be682e4ed5864e36400cbbc10836ee52c7e Mon Sep 17 00:00:00 2001
From: "Alexis (Poliorcetics) Bourget" <ab_github@poliorcetiq.eu>
Date: Sat, 1 Jul 2023 19:13:00 +0200
Subject: [PATCH 207/297] feat: render Object Safety informations non-object
 safe traits

---
 src/librustdoc/clean/types.rs            |  3 +++
 src/librustdoc/html/render/print_item.rs | 14 ++++++++++++++
 src/librustdoc/html/render/sidebar.rs    |  8 ++++++++
 3 files changed, 25 insertions(+)

diff --git a/src/librustdoc/clean/types.rs b/src/librustdoc/clean/types.rs
index 455228d04efa..b773977eb9ca 100644
--- a/src/librustdoc/clean/types.rs
+++ b/src/librustdoc/clean/types.rs
@@ -1455,6 +1455,9 @@ impl Trait {
     pub(crate) fn unsafety(&self, tcx: TyCtxt<'_>) -> hir::Unsafety {
         tcx.trait_def(self.def_id).unsafety
     }
+    pub(crate) fn is_object_safe(&self, tcx: TyCtxt<'_>) -> bool {
+        tcx.check_is_object_safe(self.def_id)
+    }
 }
 
 #[derive(Clone, Debug)]
diff --git a/src/librustdoc/html/render/print_item.rs b/src/librustdoc/html/render/print_item.rs
index fdf455690617..4d518a3e20b9 100644
--- a/src/librustdoc/html/render/print_item.rs
+++ b/src/librustdoc/html/render/print_item.rs
@@ -959,6 +959,20 @@ fn item_trait(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, t: &clean:
     let cloned_shared = Rc::clone(&cx.shared);
     let cache = &cloned_shared.cache;
     let mut extern_crates = FxHashSet::default();
+
+    if !t.is_object_safe(cx.tcx()) {
+        write_small_section_header(
+            w,
+            "object-safety",
+            "Object Safety",
+            &format!("<div class=\"object-safety-info\">This trait is <b>not</b> \
+                <a href=\"{base}/reference/items/traits.html#object-safety\">\
+                object safe</a>.</div>",
+                base = crate::clean::utils::DOC_RUST_LANG_ORG_CHANNEL
+            ),
+        );
+    }
+
     if let Some(implementors) = cache.implementors.get(&it.item_id.expect_def_id()) {
         // The DefId is for the first Type found with that name. The bool is
         // if any Types with the same name but different DefId have been found.
diff --git a/src/librustdoc/html/render/sidebar.rs b/src/librustdoc/html/render/sidebar.rs
index 4e8d88c55b64..ba4aaaff5a74 100644
--- a/src/librustdoc/html/render/sidebar.rs
+++ b/src/librustdoc/html/render/sidebar.rs
@@ -218,6 +218,14 @@ fn sidebar_trait<'a>(
     .map(|(id, title, items)| LinkBlock::new(Link::new(id, title), "", items))
     .collect();
     sidebar_assoc_items(cx, it, &mut blocks);
+
+    if !t.is_object_safe(cx.tcx()) {
+        blocks.push(LinkBlock::forced(
+            Link::new("object-safety", "Object Safety"),
+            "object-safety-note",
+        ));
+    }
+
     blocks.push(LinkBlock::forced(Link::new("implementors", "Implementors"), "impl"));
     if t.is_auto(cx.tcx()) {
         blocks.push(LinkBlock::forced(

From 40556b968ca044108d5f28d7a8810821c8be7d9d Mon Sep 17 00:00:00 2001
From: "Alexis (Poliorcetics) Bourget" <ab_github@poliorcetiq.eu>
Date: Sat, 8 Jul 2023 17:29:05 +0200
Subject: [PATCH 208/297] feat: Add 'object-safety' ID to init_id_map() in
 rustdoc

---
 src/librustdoc/html/markdown.rs          | 1 +
 src/librustdoc/html/render/print_item.rs | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/librustdoc/html/markdown.rs b/src/librustdoc/html/markdown.rs
index d24e6e5faf52..aa728c26afcf 100644
--- a/src/librustdoc/html/markdown.rs
+++ b/src/librustdoc/html/markdown.rs
@@ -2024,6 +2024,7 @@ fn init_id_map() -> FxHashMap<Cow<'static, str>, usize> {
     map.insert("required-associated-consts".into(), 1);
     map.insert("required-methods".into(), 1);
     map.insert("provided-methods".into(), 1);
+    map.insert("object-safety".into(), 1);
     map.insert("implementors".into(), 1);
     map.insert("synthetic-implementors".into(), 1);
     map.insert("implementations-list".into(), 1);
diff --git a/src/librustdoc/html/render/print_item.rs b/src/librustdoc/html/render/print_item.rs
index 4d518a3e20b9..c852f01f450a 100644
--- a/src/librustdoc/html/render/print_item.rs
+++ b/src/librustdoc/html/render/print_item.rs
@@ -965,7 +965,8 @@ fn item_trait(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, t: &clean:
             w,
             "object-safety",
             "Object Safety",
-            &format!("<div class=\"object-safety-info\">This trait is <b>not</b> \
+            &format!(
+                "<div class=\"object-safety-info\">This trait is <b>not</b> \
                 <a href=\"{base}/reference/items/traits.html#object-safety\">\
                 object safe</a>.</div>",
                 base = crate::clean::utils::DOC_RUST_LANG_ORG_CHANNEL

From a119158eb39a5e251d0d309a261c11743b996ce0 Mon Sep 17 00:00:00 2001
From: "Alexis (Poliorcetics) Bourget" <ab_github@poliorcetiq.eu>
Date: Sat, 1 Jul 2023 19:13:26 +0200
Subject: [PATCH 209/297] tests: object-safety section in traits

---
 tests/rustdoc/sidebar-items.rs     |  7 +++++++
 tests/rustdoc/trait-object-safe.rs | 27 +++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)
 create mode 100644 tests/rustdoc/trait-object-safe.rs

diff --git a/tests/rustdoc/sidebar-items.rs b/tests/rustdoc/sidebar-items.rs
index 6f7afa59bddc..b746f6982644 100644
--- a/tests/rustdoc/sidebar-items.rs
+++ b/tests/rustdoc/sidebar-items.rs
@@ -14,6 +14,7 @@
 // @has - '//*[@class="sidebar-elems"]//section//a' 'Output'
 // @has - '//div[@class="sidebar-elems"]//h3/a[@href="#provided-associated-types"]' 'Provided Associated Types'
 // @has - '//*[@class="sidebar-elems"]//section//a' 'Extra'
+// @has - '//div[@class="sidebar-elems"]//h3/a[@href="#object-safety"]' 'Object Safety'
 pub trait Foo {
     const FOO: usize;
     const BAR: u32 = 0;
@@ -24,6 +25,12 @@ pub trait Foo {
     fn bar() -> Self::Output;
 }
 
+// @has foo/trait.Safe.html
+// @!has - '//div[@class="sidebar-elems"]//h3/a[@href="#object-safety"]' ''
+pub trait Safe {
+    fn access(&self);
+}
+
 // @has foo/struct.Bar.html
 // @has - '//div[@class="sidebar-elems"]//h3/a[@href="#fields"]' 'Fields'
 // @has - '//*[@class="sidebar-elems"]//section//a[@href="#structfield.f"]' 'f'
diff --git a/tests/rustdoc/trait-object-safe.rs b/tests/rustdoc/trait-object-safe.rs
new file mode 100644
index 000000000000..818843f75583
--- /dev/null
+++ b/tests/rustdoc/trait-object-safe.rs
@@ -0,0 +1,27 @@
+#![crate_name = "foo"]
+
+// @has 'foo/trait.Unsafe.html'
+// @has - '//*[@class="object-safety-info"]' 'This trait is not object safe.'
+// @has - '//*[@id="object-safety"]' 'Object Safety'
+pub trait Unsafe {
+    fn foo() -> Self;
+}
+
+// @has 'foo/trait.Unsafe2.html'
+// @has - '//*[@class="object-safety-info"]' 'This trait is not object safe.'
+// @has - '//*[@id="object-safety"]' 'Object Safety'
+pub trait Unsafe2<T> {
+    fn foo(i: T);
+}
+
+// @has 'foo/trait.Safe.html'
+// @!has - '//*[@class="object-safety-info"]' ''
+// @!has - '//*[@id="object-safety"]' ''
+pub trait Safe {
+    fn foo(&self);
+}
+
+// @has 'foo/struct.Foo.html'
+// @!has - '//*[@class="object-safety-info"]' ''
+// @!has - '//*[@id="object-safety"]' ''
+pub struct Foo;

From 8c04999226e4be90c46c370580c9e8cd7f0ccb05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Esteban=20K=C3=BCber?= <esteban@kuber.com.ar>
Date: Tue, 24 Oct 2023 16:45:04 +0000
Subject: [PATCH 210/297] On object safety error, mention new enum as
 alternative

When we encounter a `dyn Trait` that isn't object safe, look for its
implementors. If there's one, mention using it directly If there are
less than 9, mention the possibility of creating a new enum and using
that instead.

Account for object unsafe `impl Trait on dyn Trait {}`.  Make a
distinction between public and sealed traits.

Fix #80194.
---
 .../src/traits/error_reporting/mod.rs         | 64 ++++++++++++++++++-
 .../object-safety-err-ret.stderr              |  1 +
 ...gate-dispatch-from-dyn-missing-impl.stderr |  2 +
 .../gat-in-trait-path.base.stderr             |  3 +
 .../issue-76535.base.stderr                   |  4 ++
 .../issue-79422.base.stderr                   |  6 ++
 .../in-trait/foreign-dyn-error.stderr         |  1 +
 .../impl-trait/in-trait/object-safety.stderr  |  4 ++
 ...-trait-in-return-position-dyn-trait.stderr |  6 ++
 tests/ui/issues/issue-19380.stderr            |  2 +
 tests/ui/object-safety/issue-19538.stderr     |  2 +
 .../object-safety-issue-22040.stderr          |  1 +
 .../object-safety-no-static.curr.stderr       |  1 +
 ...-no-static.object_safe_for_dispatch.stderr |  1 +
 ...ary-self-types-not-object-safe.curr.stderr |  2 +
 ...bject-safe.object_safe_for_dispatch.stderr |  1 +
 tests/ui/traits/issue-38604.stderr            |  2 +
 tests/ui/traits/item-privacy.stderr           |  1 +
 .../supertrait-object-safety.stderr           |  3 +
 tests/ui/traits/object/safety.stderr          |  2 +
 tests/ui/traits/test-2.stderr                 |  9 +++
 ...ter-defaults-referencing-Self-ppaux.stderr |  1 +
 .../wf/wf-convert-unsafe-trait-obj-box.stderr |  3 +
 .../ui/wf/wf-convert-unsafe-trait-obj.stderr  |  3 +
 tests/ui/wf/wf-unsafe-trait-obj-match.stderr  |  6 ++
 25 files changed, 130 insertions(+), 1 deletion(-)

diff --git a/compiler/rustc_infer/src/traits/error_reporting/mod.rs b/compiler/rustc_infer/src/traits/error_reporting/mod.rs
index 7a335827f370..32966011932d 100644
--- a/compiler/rustc_infer/src/traits/error_reporting/mod.rs
+++ b/compiler/rustc_infer/src/traits/error_reporting/mod.rs
@@ -5,7 +5,8 @@ use rustc_data_structures::fx::FxIndexSet;
 use rustc_errors::{struct_span_err, DiagnosticBuilder, ErrorGuaranteed, MultiSpan};
 use rustc_hir as hir;
 use rustc_hir::def_id::{DefId, LocalDefId};
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::ty::print::with_no_trimmed_paths;
+use rustc_middle::ty::{self, TyCtxt};
 use rustc_span::Span;
 use std::fmt;
 use std::iter;
@@ -108,5 +109,66 @@ pub fn report_object_safety_error<'tcx>(
             violation.solution(&mut err);
         }
     }
+
+    let impls_of = tcx.trait_impls_of(trait_def_id);
+    let impls = if impls_of.blanket_impls().is_empty() {
+        impls_of
+            .non_blanket_impls()
+            .values()
+            .flatten()
+            .filter(|def_id| {
+                !matches!(tcx.type_of(*def_id).instantiate_identity().kind(), ty::Dynamic(..))
+            })
+            .collect::<Vec<_>>()
+    } else {
+        vec![]
+    };
+    let externally_visible = if !impls.is_empty()
+        && let Some(def_id) = trait_def_id.as_local()
+        && tcx.effective_visibilities(()).is_exported(def_id)
+    {
+        true
+    } else {
+        false
+    };
+    match &impls[..] {
+        [] => {}
+        _ if impls.len() > 9 => {}
+        [only] if externally_visible => {
+            err.help(with_no_trimmed_paths!(format!(
+                "only type `{}` is seen to implement the trait in this crate, consider using it \
+                 directly instead",
+                tcx.type_of(*only).instantiate_identity(),
+            )));
+        }
+        [only] => {
+            err.help(with_no_trimmed_paths!(format!(
+                "only type `{}` implements the trait, consider using it directly instead",
+                tcx.type_of(*only).instantiate_identity(),
+            )));
+        }
+        impls => {
+            let types = impls
+                .iter()
+                .map(|t| {
+                    with_no_trimmed_paths!(format!("  {}", tcx.type_of(*t).instantiate_identity(),))
+                })
+                .collect::<Vec<_>>();
+            err.help(format!(
+                "the following types implement the trait, consider defining an enum where each \
+                 variant holds one of these types, implementing `{}` for this new enum and using \
+                 it instead:\n{}",
+                trait_str,
+                types.join("\n"),
+            ));
+        }
+    }
+    if externally_visible {
+        err.note(format!(
+            "`{trait_str}` can be implemented in other crates; if you want to support your users \
+             passing their own types here, you can't refer to a specific type",
+        ));
+    }
+
     err
 }
diff --git a/tests/ui/const-generics/generic_const_exprs/object-safety-err-ret.stderr b/tests/ui/const-generics/generic_const_exprs/object-safety-err-ret.stderr
index 7ce2b9ac95a5..b7ec657120c6 100644
--- a/tests/ui/const-generics/generic_const_exprs/object-safety-err-ret.stderr
+++ b/tests/ui/const-generics/generic_const_exprs/object-safety-err-ret.stderr
@@ -15,6 +15,7 @@ LL |     fn test(&self) -> [u8; bar::<Self>()];
    |        ...because method `test` references the `Self` type in its `where` clause
    = help: consider moving `test` to another trait
    = help: consider moving `test` to another trait
+   = help: only type `()` implements the trait, consider using it directly instead
 
 error: aborting due to previous error
 
diff --git a/tests/ui/feature-gates/feature-gate-dispatch-from-dyn-missing-impl.stderr b/tests/ui/feature-gates/feature-gate-dispatch-from-dyn-missing-impl.stderr
index 303700c7ab4f..00b8c0eef989 100644
--- a/tests/ui/feature-gates/feature-gate-dispatch-from-dyn-missing-impl.stderr
+++ b/tests/ui/feature-gates/feature-gate-dispatch-from-dyn-missing-impl.stderr
@@ -14,6 +14,7 @@ LL | trait Trait {
    |       ----- this trait cannot be made into an object...
 LL |     fn ptr(self: Ptr<Self>);
    |                  ^^^^^^^^^ ...because method `ptr`'s `self` parameter cannot be dispatched on
+   = help: only type `i32` implements the trait, consider using it directly instead
 
 error[E0038]: the trait `Trait` cannot be made into an object
   --> $DIR/feature-gate-dispatch-from-dyn-missing-impl.rs:32:5
@@ -31,6 +32,7 @@ LL | trait Trait {
    |       ----- this trait cannot be made into an object...
 LL |     fn ptr(self: Ptr<Self>);
    |                  ^^^^^^^^^ ...because method `ptr`'s `self` parameter cannot be dispatched on
+   = help: only type `i32` implements the trait, consider using it directly instead
    = note: required for the cast from `Ptr<{integer}>` to `Ptr<dyn Trait>`
 
 error: aborting due to 2 previous errors
diff --git a/tests/ui/generic-associated-types/gat-in-trait-path.base.stderr b/tests/ui/generic-associated-types/gat-in-trait-path.base.stderr
index fd54faaf37cb..9013d4295300 100644
--- a/tests/ui/generic-associated-types/gat-in-trait-path.base.stderr
+++ b/tests/ui/generic-associated-types/gat-in-trait-path.base.stderr
@@ -12,6 +12,9 @@ LL | trait Foo {
 LL |     type A<'a> where Self: 'a;
    |          ^ ...because it contains the generic associated type `A`
    = help: consider moving `A` to another trait
+   = help: the following types implement the trait, consider defining an enum where each variant holds one of these types, implementing `Foo` for this new enum and using it instead:
+             Fooy
+             Fooer<T>
 
 error: aborting due to previous error
 
diff --git a/tests/ui/generic-associated-types/issue-76535.base.stderr b/tests/ui/generic-associated-types/issue-76535.base.stderr
index 370329b9f837..bb14e297174a 100644
--- a/tests/ui/generic-associated-types/issue-76535.base.stderr
+++ b/tests/ui/generic-associated-types/issue-76535.base.stderr
@@ -28,6 +28,8 @@ LL | pub trait SuperTrait {
 LL |     type SubType<'a>: SubTrait where Self: 'a;
    |          ^^^^^^^ ...because it contains the generic associated type `SubType`
    = help: consider moving `SubType` to another trait
+   = help: only type `SuperStruct` is seen to implement the trait in this crate, consider using it directly instead
+   = note: `SuperTrait` can be implemented in other crates; if you want to support your users passing their own types here, you can't refer to a specific type
 
 error[E0038]: the trait `SuperTrait` cannot be made into an object
   --> $DIR/issue-76535.rs:39:57
@@ -43,6 +45,8 @@ LL | pub trait SuperTrait {
 LL |     type SubType<'a>: SubTrait where Self: 'a;
    |          ^^^^^^^ ...because it contains the generic associated type `SubType`
    = help: consider moving `SubType` to another trait
+   = help: only type `SuperStruct` is seen to implement the trait in this crate, consider using it directly instead
+   = note: `SuperTrait` can be implemented in other crates; if you want to support your users passing their own types here, you can't refer to a specific type
    = note: required for the cast from `Box<SuperStruct>` to `Box<dyn SuperTrait<SubType = SubStruct<'_>>>`
 
 error: aborting due to 3 previous errors
diff --git a/tests/ui/generic-associated-types/issue-79422.base.stderr b/tests/ui/generic-associated-types/issue-79422.base.stderr
index ad704f5e9f01..bcc6382cf7cd 100644
--- a/tests/ui/generic-associated-types/issue-79422.base.stderr
+++ b/tests/ui/generic-associated-types/issue-79422.base.stderr
@@ -28,6 +28,9 @@ LL | trait MapLike<K, V> {
 LL |     type VRefCont<'a>: RefCont<'a, V> where Self: 'a;
    |          ^^^^^^^^ ...because it contains the generic associated type `VRefCont`
    = help: consider moving `VRefCont` to another trait
+   = help: the following types implement the trait, consider defining an enum where each variant holds one of these types, implementing `MapLike` for this new enum and using it instead:
+             std::collections::BTreeMap<K, V>
+             Source
 
 error[E0038]: the trait `MapLike` cannot be made into an object
   --> $DIR/issue-79422.rs:44:13
@@ -43,6 +46,9 @@ LL | trait MapLike<K, V> {
 LL |     type VRefCont<'a>: RefCont<'a, V> where Self: 'a;
    |          ^^^^^^^^ ...because it contains the generic associated type `VRefCont`
    = help: consider moving `VRefCont` to another trait
+   = help: the following types implement the trait, consider defining an enum where each variant holds one of these types, implementing `MapLike` for this new enum and using it instead:
+             std::collections::BTreeMap<K, V>
+             Source
    = note: required for the cast from `Box<BTreeMap<u8, u8>>` to `Box<dyn MapLike<u8, u8, VRefCont = (dyn RefCont<'_, u8> + 'static)>>`
 
 error: aborting due to 3 previous errors
diff --git a/tests/ui/impl-trait/in-trait/foreign-dyn-error.stderr b/tests/ui/impl-trait/in-trait/foreign-dyn-error.stderr
index bfb2be8cbc1a..f29ec95d5940 100644
--- a/tests/ui/impl-trait/in-trait/foreign-dyn-error.stderr
+++ b/tests/ui/impl-trait/in-trait/foreign-dyn-error.stderr
@@ -9,6 +9,7 @@ note: for a trait to be "object safe" it needs to allow building a vtable to all
    |
 LL |     fn bar(self) -> impl Deref<Target = impl Sized>;
    |                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the trait cannot be made into an object because method `bar` references an `impl Trait` type in its return type
+   = help: only type `rpitit::Foreign` implements the trait, consider using it directly instead
 
 error: aborting due to previous error
 
diff --git a/tests/ui/impl-trait/in-trait/object-safety.stderr b/tests/ui/impl-trait/in-trait/object-safety.stderr
index 3271cb18d9fb..a7be0516cd32 100644
--- a/tests/ui/impl-trait/in-trait/object-safety.stderr
+++ b/tests/ui/impl-trait/in-trait/object-safety.stderr
@@ -12,6 +12,7 @@ LL | trait Foo {
 LL |     fn baz(&self) -> impl Debug;
    |                      ^^^^^^^^^^ ...because method `baz` references an `impl Trait` type in its return type
    = help: consider moving `baz` to another trait
+   = help: only type `u32` implements the trait, consider using it directly instead
 
 error[E0038]: the trait `Foo` cannot be made into an object
   --> $DIR/object-safety.rs:17:15
@@ -27,6 +28,7 @@ LL | trait Foo {
 LL |     fn baz(&self) -> impl Debug;
    |                      ^^^^^^^^^^ ...because method `baz` references an `impl Trait` type in its return type
    = help: consider moving `baz` to another trait
+   = help: only type `u32` implements the trait, consider using it directly instead
 
 error[E0038]: the trait `Foo` cannot be made into an object
   --> $DIR/object-safety.rs:17:13
@@ -42,6 +44,7 @@ LL | trait Foo {
 LL |     fn baz(&self) -> impl Debug;
    |                      ^^^^^^^^^^ ...because method `baz` references an `impl Trait` type in its return type
    = help: consider moving `baz` to another trait
+   = help: only type `u32` implements the trait, consider using it directly instead
 
 error[E0038]: the trait `Foo` cannot be made into an object
   --> $DIR/object-safety.rs:14:13
@@ -57,6 +60,7 @@ LL | trait Foo {
 LL |     fn baz(&self) -> impl Debug;
    |                      ^^^^^^^^^^ ...because method `baz` references an `impl Trait` type in its return type
    = help: consider moving `baz` to another trait
+   = help: only type `u32` implements the trait, consider using it directly instead
    = note: required for the cast from `Box<u32>` to `Box<dyn Foo>`
 
 error: aborting due to 4 previous errors
diff --git a/tests/ui/impl-trait/object-unsafe-trait-in-return-position-dyn-trait.stderr b/tests/ui/impl-trait/object-unsafe-trait-in-return-position-dyn-trait.stderr
index 687dbe65e6c3..37c96d9bc4e9 100644
--- a/tests/ui/impl-trait/object-unsafe-trait-in-return-position-dyn-trait.stderr
+++ b/tests/ui/impl-trait/object-unsafe-trait-in-return-position-dyn-trait.stderr
@@ -11,6 +11,9 @@ LL | trait NotObjectSafe {
    |       ------------- this trait cannot be made into an object...
 LL |     fn foo() -> Self;
    |        ^^^ ...because associated function `foo` has no `self` parameter
+   = help: the following types implement the trait, consider defining an enum where each variant holds one of these types, implementing `NotObjectSafe` for this new enum and using it instead:
+             A
+             B
 help: consider turning `foo` into a method by giving it a `&self` argument
    |
 LL |     fn foo(&self) -> Self;
@@ -33,6 +36,9 @@ LL | trait NotObjectSafe {
    |       ------------- this trait cannot be made into an object...
 LL |     fn foo() -> Self;
    |        ^^^ ...because associated function `foo` has no `self` parameter
+   = help: the following types implement the trait, consider defining an enum where each variant holds one of these types, implementing `NotObjectSafe` for this new enum and using it instead:
+             A
+             B
 help: consider turning `foo` into a method by giving it a `&self` argument
    |
 LL |     fn foo(&self) -> Self;
diff --git a/tests/ui/issues/issue-19380.stderr b/tests/ui/issues/issue-19380.stderr
index 37e280fbcc72..f6244d9d44f3 100644
--- a/tests/ui/issues/issue-19380.stderr
+++ b/tests/ui/issues/issue-19380.stderr
@@ -11,6 +11,7 @@ LL | trait Qiz {
    |       --- this trait cannot be made into an object...
 LL |   fn qiz();
    |      ^^^ ...because associated function `qiz` has no `self` parameter
+   = help: only type `Foo` implements the trait, consider using it directly instead
 help: consider turning `qiz` into a method by giving it a `&self` argument
    |
 LL |   fn qiz(&self);
@@ -33,6 +34,7 @@ LL | trait Qiz {
    |       --- this trait cannot be made into an object...
 LL |   fn qiz();
    |      ^^^ ...because associated function `qiz` has no `self` parameter
+   = help: only type `Foo` implements the trait, consider using it directly instead
    = note: required for the cast from `&Foo` to `&'static (dyn Qiz + 'static)`
 help: consider turning `qiz` into a method by giving it a `&self` argument
    |
diff --git a/tests/ui/object-safety/issue-19538.stderr b/tests/ui/object-safety/issue-19538.stderr
index 183245b23223..31657501e258 100644
--- a/tests/ui/object-safety/issue-19538.stderr
+++ b/tests/ui/object-safety/issue-19538.stderr
@@ -13,6 +13,7 @@ LL |     fn foo<T>(&self, val: T);
 LL | trait Bar: Foo { }
    |       --- this trait cannot be made into an object...
    = help: consider moving `foo` to another trait
+   = help: only type `Thing` implements the trait, consider using it directly instead
 
 error[E0038]: the trait `Bar` cannot be made into an object
   --> $DIR/issue-19538.rs:17:30
@@ -29,6 +30,7 @@ LL |     fn foo<T>(&self, val: T);
 LL | trait Bar: Foo { }
    |       --- this trait cannot be made into an object...
    = help: consider moving `foo` to another trait
+   = help: only type `Thing` implements the trait, consider using it directly instead
    = note: required for the cast from `&mut Thing` to `&mut dyn Bar`
 
 error: aborting due to 2 previous errors
diff --git a/tests/ui/object-safety/object-safety-issue-22040.stderr b/tests/ui/object-safety/object-safety-issue-22040.stderr
index 0262d536246d..2e59d88bdaf9 100644
--- a/tests/ui/object-safety/object-safety-issue-22040.stderr
+++ b/tests/ui/object-safety/object-safety-issue-22040.stderr
@@ -11,6 +11,7 @@ LL | trait Expr: Debug + PartialEq {
    |       ----          ^^^^^^^^^ ...because it uses `Self` as a type parameter
    |       |
    |       this trait cannot be made into an object...
+   = help: only type `SExpr<'x>` implements the trait, consider using it directly instead
 
 error: aborting due to previous error
 
diff --git a/tests/ui/object-safety/object-safety-no-static.curr.stderr b/tests/ui/object-safety/object-safety-no-static.curr.stderr
index 1b025229e543..b40470b457b3 100644
--- a/tests/ui/object-safety/object-safety-no-static.curr.stderr
+++ b/tests/ui/object-safety/object-safety-no-static.curr.stderr
@@ -11,6 +11,7 @@ LL | trait Foo {
    |       --- this trait cannot be made into an object...
 LL |     fn foo() {}
    |        ^^^ ...because associated function `foo` has no `self` parameter
+   = help: only type `Bar` implements the trait, consider using it directly instead
 help: consider turning `foo` into a method by giving it a `&self` argument
    |
 LL |     fn foo(&self) {}
diff --git a/tests/ui/object-safety/object-safety-no-static.object_safe_for_dispatch.stderr b/tests/ui/object-safety/object-safety-no-static.object_safe_for_dispatch.stderr
index befcef952a85..1eae9a9b9daf 100644
--- a/tests/ui/object-safety/object-safety-no-static.object_safe_for_dispatch.stderr
+++ b/tests/ui/object-safety/object-safety-no-static.object_safe_for_dispatch.stderr
@@ -11,6 +11,7 @@ LL | trait Foo {
    |       --- this trait cannot be made into an object...
 LL |     fn foo() {}
    |        ^^^ ...because associated function `foo` has no `self` parameter
+   = help: only type `Bar` implements the trait, consider using it directly instead
    = note: required for the cast from `Box<Bar>` to `Box<dyn Foo>`
 help: consider turning `foo` into a method by giving it a `&self` argument
    |
diff --git a/tests/ui/self/arbitrary-self-types-not-object-safe.curr.stderr b/tests/ui/self/arbitrary-self-types-not-object-safe.curr.stderr
index 13591f5b6351..fdd18c6b37b5 100644
--- a/tests/ui/self/arbitrary-self-types-not-object-safe.curr.stderr
+++ b/tests/ui/self/arbitrary-self-types-not-object-safe.curr.stderr
@@ -14,6 +14,7 @@ LL | trait Foo {
    |       --- this trait cannot be made into an object...
 LL |     fn foo(self: &Rc<Self>) -> usize;
    |                  ^^^^^^^^^ ...because method `foo`'s `self` parameter cannot be dispatched on
+   = help: only type `usize` implements the trait, consider using it directly instead
 
 error[E0038]: the trait `Foo` cannot be made into an object
   --> $DIR/arbitrary-self-types-not-object-safe.rs:33:13
@@ -31,6 +32,7 @@ LL | trait Foo {
    |       --- this trait cannot be made into an object...
 LL |     fn foo(self: &Rc<Self>) -> usize;
    |                  ^^^^^^^^^ ...because method `foo`'s `self` parameter cannot be dispatched on
+   = help: only type `usize` implements the trait, consider using it directly instead
    = note: required for the cast from `Rc<usize>` to `Rc<dyn Foo>`
 
 error: aborting due to 2 previous errors
diff --git a/tests/ui/self/arbitrary-self-types-not-object-safe.object_safe_for_dispatch.stderr b/tests/ui/self/arbitrary-self-types-not-object-safe.object_safe_for_dispatch.stderr
index 593f705353a5..0a567ddcc2e8 100644
--- a/tests/ui/self/arbitrary-self-types-not-object-safe.object_safe_for_dispatch.stderr
+++ b/tests/ui/self/arbitrary-self-types-not-object-safe.object_safe_for_dispatch.stderr
@@ -14,6 +14,7 @@ LL | trait Foo {
    |       --- this trait cannot be made into an object...
 LL |     fn foo(self: &Rc<Self>) -> usize;
    |                  ^^^^^^^^^ ...because method `foo`'s `self` parameter cannot be dispatched on
+   = help: only type `usize` implements the trait, consider using it directly instead
    = note: required for the cast from `Rc<usize>` to `Rc<dyn Foo>`
 
 error: aborting due to previous error
diff --git a/tests/ui/traits/issue-38604.stderr b/tests/ui/traits/issue-38604.stderr
index d53276024306..3ab9af21bc45 100644
--- a/tests/ui/traits/issue-38604.stderr
+++ b/tests/ui/traits/issue-38604.stderr
@@ -11,6 +11,7 @@ LL | trait Foo where u32: Q<Self> {
    |       ---            ^^^^^^^ ...because it uses `Self` as a type parameter
    |       |
    |       this trait cannot be made into an object...
+   = help: only type `()` implements the trait, consider using it directly instead
 
 error[E0038]: the trait `Foo` cannot be made into an object
   --> $DIR/issue-38604.rs:15:9
@@ -25,6 +26,7 @@ LL | trait Foo where u32: Q<Self> {
    |       ---            ^^^^^^^ ...because it uses `Self` as a type parameter
    |       |
    |       this trait cannot be made into an object...
+   = help: only type `()` implements the trait, consider using it directly instead
    = note: required for the cast from `Box<()>` to `Box<dyn Foo>`
 
 error: aborting due to 2 previous errors
diff --git a/tests/ui/traits/item-privacy.stderr b/tests/ui/traits/item-privacy.stderr
index af2e07632121..d74be7726602 100644
--- a/tests/ui/traits/item-privacy.stderr
+++ b/tests/ui/traits/item-privacy.stderr
@@ -143,6 +143,7 @@ LL |         const C: u8 = 0;
    = help: consider moving `C` to another trait
    = help: consider moving `A` to another trait
    = help: consider moving `B` to another trait
+   = help: only type `S` implements the trait, consider using it directly instead
 
 error[E0223]: ambiguous associated type
   --> $DIR/item-privacy.rs:115:12
diff --git a/tests/ui/traits/non_lifetime_binders/supertrait-object-safety.stderr b/tests/ui/traits/non_lifetime_binders/supertrait-object-safety.stderr
index d56519223f4d..b6e540c5ffcb 100644
--- a/tests/ui/traits/non_lifetime_binders/supertrait-object-safety.stderr
+++ b/tests/ui/traits/non_lifetime_binders/supertrait-object-safety.stderr
@@ -20,6 +20,7 @@ LL | trait Foo: for<T> Bar<T> {}
    |       ---  ^^^^^^^^^^^^^ ...because where clause cannot reference non-lifetime `for<...>` variables
    |       |
    |       this trait cannot be made into an object...
+   = help: only type `()` implements the trait, consider using it directly instead
    = note: required for the cast from `&()` to `&dyn Foo`
 
 error[E0038]: the trait `Foo` cannot be made into an object
@@ -35,6 +36,7 @@ LL | trait Foo: for<T> Bar<T> {}
    |       ---  ^^^^^^^^^^^^^ ...because where clause cannot reference non-lifetime `for<...>` variables
    |       |
    |       this trait cannot be made into an object...
+   = help: only type `()` implements the trait, consider using it directly instead
 
 error[E0038]: the trait `Foo` cannot be made into an object
   --> $DIR/supertrait-object-safety.rs:22:5
@@ -49,6 +51,7 @@ LL | trait Foo: for<T> Bar<T> {}
    |       ---  ^^^^^^^^^^^^^ ...because where clause cannot reference non-lifetime `for<...>` variables
    |       |
    |       this trait cannot be made into an object...
+   = help: only type `()` implements the trait, consider using it directly instead
 
 error: aborting due to 3 previous errors; 1 warning emitted
 
diff --git a/tests/ui/traits/object/safety.stderr b/tests/ui/traits/object/safety.stderr
index a51b69759381..19a46a502c2e 100644
--- a/tests/ui/traits/object/safety.stderr
+++ b/tests/ui/traits/object/safety.stderr
@@ -11,6 +11,7 @@ LL | trait Tr {
    |       -- this trait cannot be made into an object...
 LL |     fn foo();
    |        ^^^ ...because associated function `foo` has no `self` parameter
+   = help: only type `St` implements the trait, consider using it directly instead
    = note: required for the cast from `&St` to `&dyn Tr`
 help: consider turning `foo` into a method by giving it a `&self` argument
    |
@@ -34,6 +35,7 @@ LL | trait Tr {
    |       -- this trait cannot be made into an object...
 LL |     fn foo();
    |        ^^^ ...because associated function `foo` has no `self` parameter
+   = help: only type `St` implements the trait, consider using it directly instead
 help: consider turning `foo` into a method by giving it a `&self` argument
    |
 LL |     fn foo(&self);
diff --git a/tests/ui/traits/test-2.stderr b/tests/ui/traits/test-2.stderr
index 74a0fc42708e..3972e539776c 100644
--- a/tests/ui/traits/test-2.stderr
+++ b/tests/ui/traits/test-2.stderr
@@ -42,6 +42,9 @@ LL | trait bar { fn dup(&self) -> Self; fn blah<X>(&self); }
    |       this trait cannot be made into an object...
    = help: consider moving `dup` to another trait
    = help: consider moving `blah` to another trait
+   = help: the following types implement the trait, consider defining an enum where each variant holds one of these types, implementing `bar` for this new enum and using it instead:
+             i32
+             u32
 
 error[E0038]: the trait `bar` cannot be made into an object
   --> $DIR/test-2.rs:13:5
@@ -59,6 +62,9 @@ LL | trait bar { fn dup(&self) -> Self; fn blah<X>(&self); }
    |       this trait cannot be made into an object...
    = help: consider moving `dup` to another trait
    = help: consider moving `blah` to another trait
+   = help: the following types implement the trait, consider defining an enum where each variant holds one of these types, implementing `bar` for this new enum and using it instead:
+             i32
+             u32
 
 error[E0038]: the trait `bar` cannot be made into an object
   --> $DIR/test-2.rs:13:6
@@ -76,6 +82,9 @@ LL | trait bar { fn dup(&self) -> Self; fn blah<X>(&self); }
    |       this trait cannot be made into an object...
    = help: consider moving `dup` to another trait
    = help: consider moving `blah` to another trait
+   = help: the following types implement the trait, consider defining an enum where each variant holds one of these types, implementing `bar` for this new enum and using it instead:
+             i32
+             u32
    = note: required for the cast from `Box<{integer}>` to `Box<dyn bar>`
 
 error: aborting due to 5 previous errors
diff --git a/tests/ui/type/type-parameter-defaults-referencing-Self-ppaux.stderr b/tests/ui/type/type-parameter-defaults-referencing-Self-ppaux.stderr
index 8a296dc7ee6e..4412c49eaddd 100644
--- a/tests/ui/type/type-parameter-defaults-referencing-Self-ppaux.stderr
+++ b/tests/ui/type/type-parameter-defaults-referencing-Self-ppaux.stderr
@@ -24,6 +24,7 @@ LL | trait MyAdd<Rhs=Self> { fn add(&self, other: &Rhs) -> Self; }
    |       |
    |       this trait cannot be made into an object...
    = help: consider moving `add` to another trait
+   = help: only type `i32` implements the trait, consider using it directly instead
 
 error: aborting due to 2 previous errors
 
diff --git a/tests/ui/wf/wf-convert-unsafe-trait-obj-box.stderr b/tests/ui/wf/wf-convert-unsafe-trait-obj-box.stderr
index 40a25c7df6ba..85f5073364ff 100644
--- a/tests/ui/wf/wf-convert-unsafe-trait-obj-box.stderr
+++ b/tests/ui/wf/wf-convert-unsafe-trait-obj-box.stderr
@@ -11,6 +11,7 @@ LL | trait Trait: Sized {}
    |       -----  ^^^^^ ...because it requires `Self: Sized`
    |       |
    |       this trait cannot be made into an object...
+   = help: only type `S` implements the trait, consider using it directly instead
    = note: required for the cast from `Box<S>` to `Box<dyn Trait>`
 
 error[E0038]: the trait `Trait` cannot be made into an object
@@ -26,6 +27,7 @@ LL | trait Trait: Sized {}
    |       -----  ^^^^^ ...because it requires `Self: Sized`
    |       |
    |       this trait cannot be made into an object...
+   = help: only type `S` implements the trait, consider using it directly instead
    = note: required for the cast from `Box<S>` to `Box<(dyn Trait + 'static)>`
 
 error[E0038]: the trait `Trait` cannot be made into an object
@@ -41,6 +43,7 @@ LL | trait Trait: Sized {}
    |       -----  ^^^^^ ...because it requires `Self: Sized`
    |       |
    |       this trait cannot be made into an object...
+   = help: only type `S` implements the trait, consider using it directly instead
    = note: required for the cast from `Box<S>` to `Box<dyn Trait>`
 
 error: aborting due to 3 previous errors
diff --git a/tests/ui/wf/wf-convert-unsafe-trait-obj.stderr b/tests/ui/wf/wf-convert-unsafe-trait-obj.stderr
index e2c71df2feb4..a2a196316495 100644
--- a/tests/ui/wf/wf-convert-unsafe-trait-obj.stderr
+++ b/tests/ui/wf/wf-convert-unsafe-trait-obj.stderr
@@ -11,6 +11,7 @@ LL | trait Trait: Sized {}
    |       -----  ^^^^^ ...because it requires `Self: Sized`
    |       |
    |       this trait cannot be made into an object...
+   = help: only type `S` implements the trait, consider using it directly instead
    = note: required for the cast from `&S` to `&dyn Trait`
 
 error[E0038]: the trait `Trait` cannot be made into an object
@@ -26,6 +27,7 @@ LL | trait Trait: Sized {}
    |       -----  ^^^^^ ...because it requires `Self: Sized`
    |       |
    |       this trait cannot be made into an object...
+   = help: only type `S` implements the trait, consider using it directly instead
    = note: required for the cast from `&S` to `&dyn Trait`
 
 error[E0038]: the trait `Trait` cannot be made into an object
@@ -41,6 +43,7 @@ LL | trait Trait: Sized {}
    |       -----  ^^^^^ ...because it requires `Self: Sized`
    |       |
    |       this trait cannot be made into an object...
+   = help: only type `S` implements the trait, consider using it directly instead
    = note: required for the cast from `&S` to `&dyn Trait`
 
 error: aborting due to 3 previous errors
diff --git a/tests/ui/wf/wf-unsafe-trait-obj-match.stderr b/tests/ui/wf/wf-unsafe-trait-obj-match.stderr
index d5b23572ff57..a0279774abeb 100644
--- a/tests/ui/wf/wf-unsafe-trait-obj-match.stderr
+++ b/tests/ui/wf/wf-unsafe-trait-obj-match.stderr
@@ -29,6 +29,9 @@ LL | trait Trait: Sized {}
    |       -----  ^^^^^ ...because it requires `Self: Sized`
    |       |
    |       this trait cannot be made into an object...
+   = help: the following types implement the trait, consider defining an enum where each variant holds one of these types, implementing `Trait` for this new enum and using it instead:
+             S
+             R
    = note: required for the cast from `&S` to `&dyn Trait`
 
 error[E0038]: the trait `Trait` cannot be made into an object
@@ -48,6 +51,9 @@ LL | trait Trait: Sized {}
    |       -----  ^^^^^ ...because it requires `Self: Sized`
    |       |
    |       this trait cannot be made into an object...
+   = help: the following types implement the trait, consider defining an enum where each variant holds one of these types, implementing `Trait` for this new enum and using it instead:
+             S
+             R
    = note: required for the cast from `&R` to `&dyn Trait`
 
 error: aborting due to 3 previous errors

From b589f474410829046f2f37ff31bd7035b87251fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Esteban=20K=C3=BCber?= <esteban@kuber.com.ar>
Date: Fri, 27 Oct 2023 16:11:43 +0000
Subject: [PATCH 211/297] Account for `ref` and `mut` in the wrong place for
 pattern ident renaming

If the user writes `S { ref field: name }` instead of
`S { field: ref name }`, we suggest the correct code.

Fix #72298.
---
 compiler/rustc_parse/src/parser/pat.rs        | 34 ++++++++++-
 ...rrect-placement-of-pattern-modifiers.fixed | 18 ++++++
 ...ncorrect-placement-of-pattern-modifiers.rs | 18 ++++++
 ...rect-placement-of-pattern-modifiers.stderr | 58 +++++++++++++++++++
 4 files changed, 127 insertions(+), 1 deletion(-)
 create mode 100644 tests/ui/pattern/incorrect-placement-of-pattern-modifiers.fixed
 create mode 100644 tests/ui/pattern/incorrect-placement-of-pattern-modifiers.rs
 create mode 100644 tests/ui/pattern/incorrect-placement-of-pattern-modifiers.stderr

diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs
index 3e4e92789107..3378e4d46b7b 100644
--- a/compiler/rustc_parse/src/parser/pat.rs
+++ b/compiler/rustc_parse/src/parser/pat.rs
@@ -967,11 +967,12 @@ impl<'a> Parser<'a> {
 
             // check that a comma comes after every field
             if !ate_comma {
-                let err = ExpectedCommaAfterPatternField { span: self.token.span }
+                let mut err = ExpectedCommaAfterPatternField { span: self.token.span }
                     .into_diagnostic(&self.sess.span_diagnostic);
                 if let Some(mut delayed) = delayed_err {
                     delayed.emit();
                 }
+                self.recover_misplaced_pattern_modifiers(&fields, &mut err);
                 return Err(err);
             }
             ate_comma = false;
@@ -1109,6 +1110,37 @@ impl<'a> Parser<'a> {
         Ok((fields, etc))
     }
 
+    /// If the user writes `S { ref field: name }` instead of `S { field: ref name }`, we suggest
+    /// the correct code.
+    fn recover_misplaced_pattern_modifiers(
+        &self,
+        fields: &ThinVec<PatField>,
+        err: &mut DiagnosticBuilder<'a, ErrorGuaranteed>,
+    ) {
+        if let Some(last) = fields.iter().last()
+            && last.is_shorthand
+            && let PatKind::Ident(binding, ident, None) = last.pat.kind
+            && binding != BindingAnnotation::NONE
+            && self.token == token::Colon
+            // We found `ref mut? ident:`, try to parse a `name,` or `name }`.
+            && let Some(name_span) = self.look_ahead(1, |t| t.is_ident().then(|| t.span))
+            && self.look_ahead(2, |t| {
+                t == &token::Comma || t == &token::CloseDelim(Delimiter::Brace)
+            })
+        {
+            let span = last.pat.span.with_hi(ident.span.lo());
+            // We have `S { ref field: name }` instead of `S { field: ref name }`
+            err.multipart_suggestion(
+                "the pattern modifiers belong after the `:`",
+                vec![
+                    (span, String::new()),
+                    (name_span.shrink_to_lo(), binding.prefix_str().to_string()),
+                ],
+                Applicability::MachineApplicable,
+            );
+        }
+    }
+
     /// Recover on `...` or `_` as if it were `..` to avoid further errors.
     /// See issue #46718.
     fn recover_bad_dot_dot(&self) {
diff --git a/tests/ui/pattern/incorrect-placement-of-pattern-modifiers.fixed b/tests/ui/pattern/incorrect-placement-of-pattern-modifiers.fixed
new file mode 100644
index 000000000000..cf6c2a24fdf5
--- /dev/null
+++ b/tests/ui/pattern/incorrect-placement-of-pattern-modifiers.fixed
@@ -0,0 +1,18 @@
+// run-rustfix
+struct S {
+    field_name: (),
+}
+
+fn main() {
+    match (S {field_name: ()}) {
+        S {field_name: ref _foo} => {} //~ ERROR expected `,`
+    }
+    match (S {field_name: ()}) {
+        S {field_name: mut _foo} => {} //~ ERROR expected `,`
+    }
+    match (S {field_name: ()}) {
+        S {field_name: ref mut _foo} => {} //~ ERROR expected `,`
+    }
+    // Verify that we recover enough to run typeck.
+    let _: usize = 3usize; //~ ERROR mismatched types
+}
diff --git a/tests/ui/pattern/incorrect-placement-of-pattern-modifiers.rs b/tests/ui/pattern/incorrect-placement-of-pattern-modifiers.rs
new file mode 100644
index 000000000000..98772c1188ed
--- /dev/null
+++ b/tests/ui/pattern/incorrect-placement-of-pattern-modifiers.rs
@@ -0,0 +1,18 @@
+// run-rustfix
+struct S {
+    field_name: (),
+}
+
+fn main() {
+    match (S {field_name: ()}) {
+        S {ref field_name: _foo} => {} //~ ERROR expected `,`
+    }
+    match (S {field_name: ()}) {
+        S {mut field_name: _foo} => {} //~ ERROR expected `,`
+    }
+    match (S {field_name: ()}) {
+        S {ref mut field_name: _foo} => {} //~ ERROR expected `,`
+    }
+    // Verify that we recover enough to run typeck.
+    let _: usize = 3u8; //~ ERROR mismatched types
+}
diff --git a/tests/ui/pattern/incorrect-placement-of-pattern-modifiers.stderr b/tests/ui/pattern/incorrect-placement-of-pattern-modifiers.stderr
new file mode 100644
index 000000000000..e80789253c0b
--- /dev/null
+++ b/tests/ui/pattern/incorrect-placement-of-pattern-modifiers.stderr
@@ -0,0 +1,58 @@
+error: expected `,`
+  --> $DIR/incorrect-placement-of-pattern-modifiers.rs:8:26
+   |
+LL |         S {ref field_name: _foo} => {}
+   |         -                ^
+   |         |
+   |         while parsing the fields for this pattern
+   |
+help: the pattern modifiers belong after the `:`
+   |
+LL -         S {ref field_name: _foo} => {}
+LL +         S {field_name: ref _foo} => {}
+   |
+
+error: expected `,`
+  --> $DIR/incorrect-placement-of-pattern-modifiers.rs:11:26
+   |
+LL |         S {mut field_name: _foo} => {}
+   |         -                ^
+   |         |
+   |         while parsing the fields for this pattern
+   |
+help: the pattern modifiers belong after the `:`
+   |
+LL -         S {mut field_name: _foo} => {}
+LL +         S {field_name: mut _foo} => {}
+   |
+
+error: expected `,`
+  --> $DIR/incorrect-placement-of-pattern-modifiers.rs:14:30
+   |
+LL |         S {ref mut field_name: _foo} => {}
+   |         -                    ^
+   |         |
+   |         while parsing the fields for this pattern
+   |
+help: the pattern modifiers belong after the `:`
+   |
+LL -         S {ref mut field_name: _foo} => {}
+LL +         S {field_name: ref mut _foo} => {}
+   |
+
+error[E0308]: mismatched types
+  --> $DIR/incorrect-placement-of-pattern-modifiers.rs:17:20
+   |
+LL |     let _: usize = 3u8;
+   |            -----   ^^^ expected `usize`, found `u8`
+   |            |
+   |            expected due to this
+   |
+help: change the type of the numeric literal from `u8` to `usize`
+   |
+LL |     let _: usize = 3usize;
+   |                     ~~~~~
+
+error: aborting due to 4 previous errors
+
+For more information about this error, try `rustc --explain E0308`.

From 50ca5ef07f893d5bd797c2b35f51f0ed301605c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Esteban=20K=C3=BCber?= <esteban@kuber.com.ar>
Date: Fri, 13 Oct 2023 22:43:48 +0000
Subject: [PATCH 212/297] When encountering unclosed delimiters during parsing,
 check for diff markers

Fix #116252.
---
 compiler/rustc_parse/src/lexer/mod.rs         | 14 ++--
 compiler/rustc_parse/src/lexer/tokentrees.rs  | 64 +++++++++++++------
 .../rustc_parse/src/parser/diagnostics.rs     | 12 +++-
 .../diff-markers/unclosed-delims-in-macro.rs  |  9 +++
 .../unclosed-delims-in-macro.stderr           | 18 ++++++
 .../ui/parser/diff-markers/unclosed-delims.rs | 14 ++++
 .../diff-markers/unclosed-delims.stderr       | 18 ++++++
 7 files changed, 122 insertions(+), 27 deletions(-)
 create mode 100644 tests/ui/parser/diff-markers/unclosed-delims-in-macro.rs
 create mode 100644 tests/ui/parser/diff-markers/unclosed-delims-in-macro.stderr
 create mode 100644 tests/ui/parser/diff-markers/unclosed-delims.rs
 create mode 100644 tests/ui/parser/diff-markers/unclosed-delims.stderr

diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs
index a375a1d69cde..f2eed5c9be51 100644
--- a/compiler/rustc_parse/src/lexer/mod.rs
+++ b/compiler/rustc_parse/src/lexer/mod.rs
@@ -64,10 +64,10 @@ pub(crate) fn parse_token_trees<'a>(
         override_span,
         nbsp_is_whitespace: false,
     };
-    let (token_trees, unmatched_delims) =
+    let (stream, res, unmatched_delims) =
         tokentrees::TokenTreesReader::parse_all_token_trees(string_reader);
-    match token_trees {
-        Ok(stream) if unmatched_delims.is_empty() => Ok(stream),
+    match res {
+        Ok(()) if unmatched_delims.is_empty() => Ok(stream),
         _ => {
             // Return error if there are unmatched delimiters or unclosed delimiters.
             // We emit delimiter mismatch errors first, then emit the unclosing delimiter mismatch
@@ -79,9 +79,11 @@ pub(crate) fn parse_token_trees<'a>(
                     err.buffer(&mut buffer);
                 }
             }
-            if let Err(err) = token_trees {
-                // Add unclosing delimiter error
-                err.buffer(&mut buffer);
+            if let Err(errs) = res {
+                // Add unclosing delimiter or diff marker errors
+                for err in errs {
+                    err.buffer(&mut buffer);
+                }
             }
             Err(buffer)
         }
diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs
index 1d9dbfe4b89d..31d91fe80bd6 100644
--- a/compiler/rustc_parse/src/lexer/tokentrees.rs
+++ b/compiler/rustc_parse/src/lexer/tokentrees.rs
@@ -5,7 +5,7 @@ use super::{StringReader, UnmatchedDelim};
 use rustc_ast::token::{self, Delimiter, Token};
 use rustc_ast::tokenstream::{DelimSpan, Spacing, TokenStream, TokenTree};
 use rustc_ast_pretty::pprust::token_to_string;
-use rustc_errors::{PErr, PResult};
+use rustc_errors::PErr;
 
 pub(super) struct TokenTreesReader<'a> {
     string_reader: StringReader<'a>,
@@ -18,36 +18,42 @@ pub(super) struct TokenTreesReader<'a> {
 impl<'a> TokenTreesReader<'a> {
     pub(super) fn parse_all_token_trees(
         string_reader: StringReader<'a>,
-    ) -> (PResult<'a, TokenStream>, Vec<UnmatchedDelim>) {
+    ) -> (TokenStream, Result<(), Vec<PErr<'a>>>, Vec<UnmatchedDelim>) {
         let mut tt_reader = TokenTreesReader {
             string_reader,
             token: Token::dummy(),
             diag_info: TokenTreeDiagInfo::default(),
         };
-        let res = tt_reader.parse_token_trees(/* is_delimited */ false);
-        (res, tt_reader.diag_info.unmatched_delims)
+        let (stream, res) = tt_reader.parse_token_trees(/* is_delimited */ false);
+        (stream, res, tt_reader.diag_info.unmatched_delims)
     }
 
     // Parse a stream of tokens into a list of `TokenTree`s.
-    fn parse_token_trees(&mut self, is_delimited: bool) -> PResult<'a, TokenStream> {
+    fn parse_token_trees(
+        &mut self,
+        is_delimited: bool,
+    ) -> (TokenStream, Result<(), Vec<PErr<'a>>>) {
         self.token = self.string_reader.next_token().0;
         let mut buf = Vec::new();
         loop {
             match self.token.kind {
-                token::OpenDelim(delim) => buf.push(self.parse_token_tree_open_delim(delim)?),
+                token::OpenDelim(delim) => {
+                    buf.push(match self.parse_token_tree_open_delim(delim) {
+                        Ok(val) => val,
+                        Err(errs) => return (TokenStream::new(buf), Err(errs)),
+                    })
+                }
                 token::CloseDelim(delim) => {
-                    return if is_delimited {
-                        Ok(TokenStream::new(buf))
-                    } else {
-                        Err(self.close_delim_err(delim))
-                    };
+                    return (
+                        TokenStream::new(buf),
+                        if is_delimited { Ok(()) } else { Err(vec![self.close_delim_err(delim)]) },
+                    );
                 }
                 token::Eof => {
-                    return if is_delimited {
-                        Err(self.eof_err())
-                    } else {
-                        Ok(TokenStream::new(buf))
-                    };
+                    return (
+                        TokenStream::new(buf),
+                        if is_delimited { Err(vec![self.eof_err()]) } else { Ok(()) },
+                    );
                 }
                 _ => {
                     // Get the next normal token. This might require getting multiple adjacent
@@ -97,7 +103,10 @@ impl<'a> TokenTreesReader<'a> {
         err
     }
 
-    fn parse_token_tree_open_delim(&mut self, open_delim: Delimiter) -> PResult<'a, TokenTree> {
+    fn parse_token_tree_open_delim(
+        &mut self,
+        open_delim: Delimiter,
+    ) -> Result<TokenTree, Vec<PErr<'a>>> {
         // The span for beginning of the delimited section
         let pre_span = self.token.span;
 
@@ -106,7 +115,26 @@ impl<'a> TokenTreesReader<'a> {
         // Parse the token trees within the delimiters.
         // We stop at any delimiter so we can try to recover if the user
         // uses an incorrect delimiter.
-        let tts = self.parse_token_trees(/* is_delimited */ true)?;
+        let (tts, res) = self.parse_token_trees(/* is_delimited */ true);
+        if let Err(mut errs) = res {
+            // If there are unclosed delims, see if there are diff markers and if so, point them
+            // out instead of complaining about the unclosed delims.
+            let mut parser = crate::stream_to_parser(self.string_reader.sess, tts, None);
+            let mut diff_errs = vec![];
+            while parser.token != token::Eof {
+                if let Err(diff_err) = parser.err_diff_marker() {
+                    diff_errs.push(diff_err);
+                }
+                parser.bump();
+            }
+            if !diff_errs.is_empty() {
+                errs.iter_mut().for_each(|err| {
+                    err.delay_as_bug();
+                });
+                return Err(diff_errs);
+            }
+            return Err(errs);
+        }
 
         // Expand to cover the entire delimited token tree
         let delim_span = DelimSpan::from_pair(pre_span, self.token.span);
diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs
index 7b5bb319ed8d..2a8eb6edd23f 100644
--- a/compiler/rustc_parse/src/parser/diagnostics.rs
+++ b/compiler/rustc_parse/src/parser/diagnostics.rs
@@ -2808,8 +2808,15 @@ impl<'a> Parser<'a> {
     }
 
     pub fn recover_diff_marker(&mut self) {
+        if let Err(mut err) = self.err_diff_marker() {
+            err.emit();
+            FatalError.raise();
+        }
+    }
+
+    pub fn err_diff_marker(&mut self) -> PResult<'a, ()> {
         let Some(start) = self.diff_marker(&TokenKind::BinOp(token::Shl), &TokenKind::Lt) else {
-            return;
+            return Ok(());
         };
         let mut spans = Vec::with_capacity(3);
         spans.push(start);
@@ -2856,8 +2863,7 @@ impl<'a> Parser<'a> {
             "for an explanation on these markers from the `git` documentation, visit \
              <https://git-scm.com/book/en/v2/Git-Tools-Advanced-Merging#_checking_out_conflicts>",
         );
-        err.emit();
-        FatalError.raise()
+        Err(err)
     }
 
     /// Parse and throw away a parenthesized comma separated
diff --git a/tests/ui/parser/diff-markers/unclosed-delims-in-macro.rs b/tests/ui/parser/diff-markers/unclosed-delims-in-macro.rs
new file mode 100644
index 000000000000..da1774acea54
--- /dev/null
+++ b/tests/ui/parser/diff-markers/unclosed-delims-in-macro.rs
@@ -0,0 +1,9 @@
+macro_rules! foo {
+<<<<<<< HEAD
+    //~^ ERROR encountered diff marker
+    () {
+=======
+    () { //
+>>>>>>> 7a4f13c blah blah blah
+    }
+}
diff --git a/tests/ui/parser/diff-markers/unclosed-delims-in-macro.stderr b/tests/ui/parser/diff-markers/unclosed-delims-in-macro.stderr
new file mode 100644
index 000000000000..e0b6f1b5eb80
--- /dev/null
+++ b/tests/ui/parser/diff-markers/unclosed-delims-in-macro.stderr
@@ -0,0 +1,18 @@
+error: encountered diff marker
+  --> $DIR/unclosed-delims-in-macro.rs:2:1
+   |
+LL | <<<<<<< HEAD
+   | ^^^^^^^ after this is the code before the merge
+...
+LL | =======
+   | -------
+LL |     () { //
+LL | >>>>>>> 7a4f13c blah blah blah
+   | ^^^^^^^ above this are the incoming code changes
+   |
+   = help: if you're having merge conflicts after pulling new code, the top section is the code you already had and the bottom section is the remote code
+   = help: if you're in the middle of a rebase, the top section is the code being rebased onto and the bottom section is the code coming from the current commit being rebased
+   = note: for an explanation on these markers from the `git` documentation, visit <https://git-scm.com/book/en/v2/Git-Tools-Advanced-Merging#_checking_out_conflicts>
+
+error: aborting due to previous error
+
diff --git a/tests/ui/parser/diff-markers/unclosed-delims.rs b/tests/ui/parser/diff-markers/unclosed-delims.rs
new file mode 100644
index 000000000000..653a605c28c6
--- /dev/null
+++ b/tests/ui/parser/diff-markers/unclosed-delims.rs
@@ -0,0 +1,14 @@
+mod tests {
+    #[test]
+<<<<<<< HEAD
+//~^ ERROR encountered diff marker
+//~| NOTE after this is the code before the merge
+//~| NOTE for an explanation on these markers
+    fn test1() {
+=======
+//~^ NOTE
+    fn test2() {
+>>>>>>> 7a4f13c blah blah blah
+//~^ NOTE above this are the incoming code changes
+    }
+}
diff --git a/tests/ui/parser/diff-markers/unclosed-delims.stderr b/tests/ui/parser/diff-markers/unclosed-delims.stderr
new file mode 100644
index 000000000000..67199179b392
--- /dev/null
+++ b/tests/ui/parser/diff-markers/unclosed-delims.stderr
@@ -0,0 +1,18 @@
+error: encountered diff marker
+  --> $DIR/unclosed-delims.rs:3:1
+   |
+LL | <<<<<<< HEAD
+   | ^^^^^^^ after this is the code before the merge
+...
+LL | =======
+   | -------
+...
+LL | >>>>>>> 7a4f13c blah blah blah
+   | ^^^^^^^ above this are the incoming code changes
+   |
+   = help: if you're having merge conflicts after pulling new code, the top section is the code you already had and the bottom section is the remote code
+   = help: if you're in the middle of a rebase, the top section is the code being rebased onto and the bottom section is the code coming from the current commit being rebased
+   = note: for an explanation on these markers from the `git` documentation, visit <https://git-scm.com/book/en/v2/Git-Tools-Advanced-Merging#_checking_out_conflicts>
+
+error: aborting due to previous error
+

From 6d69eb1f2e3d130a3e01412958f2387c1ad3b047 Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Sun, 29 Oct 2023 20:50:47 +1100
Subject: [PATCH 213/297] coverage: Replace manual debug indents with nested
 tracing spans

---
 .../src/coverage/counters.rs                  | 84 +++++--------------
 1 file changed, 19 insertions(+), 65 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs
index d07f59bc72af..2e3e560975f0 100644
--- a/compiler/rustc_mir_transform/src/coverage/counters.rs
+++ b/compiler/rustc_mir_transform/src/coverage/counters.rs
@@ -12,8 +12,6 @@ use rustc_middle::mir::coverage::*;
 
 use std::fmt::{self, Debug};
 
-const NESTED_INDENT: &str = "    ";
-
 /// The coverage counter or counter expression associated with a particular
 /// BCB node or BCB edge.
 #[derive(Clone)]
@@ -347,22 +345,17 @@ impl<'a> MakeBcbCounters<'a> {
     }
 
     fn get_or_make_counter_operand(&mut self, bcb: BasicCoverageBlock) -> Result<CovTerm, Error> {
-        self.recursive_get_or_make_counter_operand(bcb, 1)
+        self.recursive_get_or_make_counter_operand(bcb)
     }
 
+    #[instrument(level = "debug", skip(self))]
     fn recursive_get_or_make_counter_operand(
         &mut self,
         bcb: BasicCoverageBlock,
-        debug_indent_level: usize,
     ) -> Result<CovTerm, Error> {
         // If the BCB already has a counter, return it.
         if let Some(counter_kind) = &self.coverage_counters.bcb_counters[bcb] {
-            debug!(
-                "{}{:?} already has a counter: {:?}",
-                NESTED_INDENT.repeat(debug_indent_level),
-                bcb,
-                counter_kind,
-            );
+            debug!("{bcb:?} already has a counter: {counter_kind:?}");
             return Ok(counter_kind.as_term());
         }
 
@@ -373,20 +366,12 @@ impl<'a> MakeBcbCounters<'a> {
         if one_path_to_target || self.bcb_predecessors(bcb).contains(&bcb) {
             let counter_kind = self.coverage_counters.make_counter();
             if one_path_to_target {
-                debug!(
-                    "{}{:?} gets a new counter: {:?}",
-                    NESTED_INDENT.repeat(debug_indent_level),
-                    bcb,
-                    counter_kind,
-                );
+                debug!("{bcb:?} gets a new counter: {counter_kind:?}");
             } else {
                 debug!(
-                    "{}{:?} has itself as its own predecessor. It can't be part of its own \
-                    Expression sum, so it will get its own new counter: {:?}. (Note, the compiled \
-                    code will generate an infinite loop.)",
-                    NESTED_INDENT.repeat(debug_indent_level),
-                    bcb,
-                    counter_kind,
+                    "{bcb:?} has itself as its own predecessor. It can't be part of its own \
+                    Expression sum, so it will get its own new counter: {counter_kind:?}. \
+                    (Note, the compiled code will generate an infinite loop.)",
                 );
             }
             return self.coverage_counters.set_bcb_counter(bcb, counter_kind);
@@ -397,23 +382,13 @@ impl<'a> MakeBcbCounters<'a> {
         // counters for those incoming edges first, then call `make_expression()` to sum them up,
         // with additional intermediate expressions as needed.
         let mut predecessors = self.bcb_predecessors(bcb).to_owned().into_iter();
-        debug!(
-            "{}{:?} has multiple incoming edges and will get an expression that sums them up...",
-            NESTED_INDENT.repeat(debug_indent_level),
-            bcb,
-        );
-        let first_edge_counter_operand = self.recursive_get_or_make_edge_counter_operand(
-            predecessors.next().unwrap(),
-            bcb,
-            debug_indent_level + 1,
-        )?;
+        debug!("{bcb:?} has multiple incoming edges and will need a sum-up expression");
+        let first_edge_counter_operand =
+            self.recursive_get_or_make_edge_counter_operand(predecessors.next().unwrap(), bcb)?;
         let mut some_sumup_edge_counter_operand = None;
         for predecessor in predecessors {
-            let edge_counter_operand = self.recursive_get_or_make_edge_counter_operand(
-                predecessor,
-                bcb,
-                debug_indent_level + 1,
-            )?;
+            let edge_counter_operand =
+                self.recursive_get_or_make_edge_counter_operand(predecessor, bcb)?;
             if let Some(sumup_edge_counter_operand) =
                 some_sumup_edge_counter_operand.replace(edge_counter_operand)
             {
@@ -422,11 +397,7 @@ impl<'a> MakeBcbCounters<'a> {
                     Op::Add,
                     edge_counter_operand,
                 );
-                debug!(
-                    "{}new intermediate expression: {:?}",
-                    NESTED_INDENT.repeat(debug_indent_level),
-                    intermediate_expression
-                );
+                debug!("new intermediate expression: {intermediate_expression:?}");
                 let intermediate_expression_operand = intermediate_expression.as_term();
                 some_sumup_edge_counter_operand.replace(intermediate_expression_operand);
             }
@@ -436,12 +407,7 @@ impl<'a> MakeBcbCounters<'a> {
             Op::Add,
             some_sumup_edge_counter_operand.unwrap(),
         );
-        debug!(
-            "{}{:?} gets a new counter (sum of predecessor counters): {:?}",
-            NESTED_INDENT.repeat(debug_indent_level),
-            bcb,
-            counter_kind
-        );
+        debug!("{bcb:?} gets a new counter (sum of predecessor counters): {counter_kind:?}");
         self.coverage_counters.set_bcb_counter(bcb, counter_kind)
     }
 
@@ -450,45 +416,33 @@ impl<'a> MakeBcbCounters<'a> {
         from_bcb: BasicCoverageBlock,
         to_bcb: BasicCoverageBlock,
     ) -> Result<CovTerm, Error> {
-        self.recursive_get_or_make_edge_counter_operand(from_bcb, to_bcb, 1)
+        self.recursive_get_or_make_edge_counter_operand(from_bcb, to_bcb)
     }
 
+    #[instrument(level = "debug", skip(self))]
     fn recursive_get_or_make_edge_counter_operand(
         &mut self,
         from_bcb: BasicCoverageBlock,
         to_bcb: BasicCoverageBlock,
-        debug_indent_level: usize,
     ) -> Result<CovTerm, Error> {
         // If the source BCB has only one successor (assumed to be the given target), an edge
         // counter is unnecessary. Just get or make a counter for the source BCB.
         let successors = self.bcb_successors(from_bcb).iter();
         if successors.len() == 1 {
-            return self.recursive_get_or_make_counter_operand(from_bcb, debug_indent_level + 1);
+            return self.recursive_get_or_make_counter_operand(from_bcb);
         }
 
         // If the edge already has a counter, return it.
         if let Some(counter_kind) =
             self.coverage_counters.bcb_edge_counters.get(&(from_bcb, to_bcb))
         {
-            debug!(
-                "{}Edge {:?}->{:?} already has a counter: {:?}",
-                NESTED_INDENT.repeat(debug_indent_level),
-                from_bcb,
-                to_bcb,
-                counter_kind
-            );
+            debug!("Edge {from_bcb:?}->{to_bcb:?} already has a counter: {counter_kind:?}");
             return Ok(counter_kind.as_term());
         }
 
         // Make a new counter to count this edge.
         let counter_kind = self.coverage_counters.make_counter();
-        debug!(
-            "{}Edge {:?}->{:?} gets a new counter: {:?}",
-            NESTED_INDENT.repeat(debug_indent_level),
-            from_bcb,
-            to_bcb,
-            counter_kind
-        );
+        debug!("Edge {from_bcb:?}->{to_bcb:?} gets a new counter: {counter_kind:?}");
         self.coverage_counters.set_bcb_edge_counter(from_bcb, to_bcb, counter_kind)
     }
 

From 2f1be08473e07e15d848b21e159be6729ffd90d2 Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Sun, 29 Oct 2023 20:53:17 +1100
Subject: [PATCH 214/297] coverage: Inline the "recursive" worker methods for
 assigning counters

Now that we don't manually pass around indent levels, there's no need for these
worker methods to exist separately from their main callers.
---
 .../src/coverage/counters.rs                  | 26 ++++---------------
 1 file changed, 5 insertions(+), 21 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs
index 2e3e560975f0..8909de143335 100644
--- a/compiler/rustc_mir_transform/src/coverage/counters.rs
+++ b/compiler/rustc_mir_transform/src/coverage/counters.rs
@@ -344,15 +344,8 @@ impl<'a> MakeBcbCounters<'a> {
         Ok(())
     }
 
-    fn get_or_make_counter_operand(&mut self, bcb: BasicCoverageBlock) -> Result<CovTerm, Error> {
-        self.recursive_get_or_make_counter_operand(bcb)
-    }
-
     #[instrument(level = "debug", skip(self))]
-    fn recursive_get_or_make_counter_operand(
-        &mut self,
-        bcb: BasicCoverageBlock,
-    ) -> Result<CovTerm, Error> {
+    fn get_or_make_counter_operand(&mut self, bcb: BasicCoverageBlock) -> Result<CovTerm, Error> {
         // If the BCB already has a counter, return it.
         if let Some(counter_kind) = &self.coverage_counters.bcb_counters[bcb] {
             debug!("{bcb:?} already has a counter: {counter_kind:?}");
@@ -384,11 +377,10 @@ impl<'a> MakeBcbCounters<'a> {
         let mut predecessors = self.bcb_predecessors(bcb).to_owned().into_iter();
         debug!("{bcb:?} has multiple incoming edges and will need a sum-up expression");
         let first_edge_counter_operand =
-            self.recursive_get_or_make_edge_counter_operand(predecessors.next().unwrap(), bcb)?;
+            self.get_or_make_edge_counter_operand(predecessors.next().unwrap(), bcb)?;
         let mut some_sumup_edge_counter_operand = None;
         for predecessor in predecessors {
-            let edge_counter_operand =
-                self.recursive_get_or_make_edge_counter_operand(predecessor, bcb)?;
+            let edge_counter_operand = self.get_or_make_edge_counter_operand(predecessor, bcb)?;
             if let Some(sumup_edge_counter_operand) =
                 some_sumup_edge_counter_operand.replace(edge_counter_operand)
             {
@@ -411,16 +403,8 @@ impl<'a> MakeBcbCounters<'a> {
         self.coverage_counters.set_bcb_counter(bcb, counter_kind)
     }
 
-    fn get_or_make_edge_counter_operand(
-        &mut self,
-        from_bcb: BasicCoverageBlock,
-        to_bcb: BasicCoverageBlock,
-    ) -> Result<CovTerm, Error> {
-        self.recursive_get_or_make_edge_counter_operand(from_bcb, to_bcb)
-    }
-
     #[instrument(level = "debug", skip(self))]
-    fn recursive_get_or_make_edge_counter_operand(
+    fn get_or_make_edge_counter_operand(
         &mut self,
         from_bcb: BasicCoverageBlock,
         to_bcb: BasicCoverageBlock,
@@ -429,7 +413,7 @@ impl<'a> MakeBcbCounters<'a> {
         // counter is unnecessary. Just get or make a counter for the source BCB.
         let successors = self.bcb_successors(from_bcb).iter();
         if successors.len() == 1 {
-            return self.recursive_get_or_make_counter_operand(from_bcb);
+            return self.get_or_make_counter_operand(from_bcb);
         }
 
         // If the edge already has a counter, return it.

From 10c4734c79c7fc2cad772f0f013032ffeecbce1b Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Sun, 29 Oct 2023 21:23:47 +1100
Subject: [PATCH 215/297] coverage: Use a tracing span to group the parts of a
 sum-up expression

---
 compiler/rustc_mir_transform/src/coverage/counters.rs | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs
index 8909de143335..dcf603a4f77c 100644
--- a/compiler/rustc_mir_transform/src/coverage/counters.rs
+++ b/compiler/rustc_mir_transform/src/coverage/counters.rs
@@ -374,8 +374,9 @@ impl<'a> MakeBcbCounters<'a> {
         // counters and/or expressions of its incoming edges. This will recursively get or create
         // counters for those incoming edges first, then call `make_expression()` to sum them up,
         // with additional intermediate expressions as needed.
+        let _sumup_debug_span = debug_span!("(preparing sum-up expression)").entered();
+
         let mut predecessors = self.bcb_predecessors(bcb).to_owned().into_iter();
-        debug!("{bcb:?} has multiple incoming edges and will need a sum-up expression");
         let first_edge_counter_operand =
             self.get_or_make_edge_counter_operand(predecessors.next().unwrap(), bcb)?;
         let mut some_sumup_edge_counter_operand = None;
@@ -399,6 +400,8 @@ impl<'a> MakeBcbCounters<'a> {
             Op::Add,
             some_sumup_edge_counter_operand.unwrap(),
         );
+        drop(_sumup_debug_span);
+
         debug!("{bcb:?} gets a new counter (sum of predecessor counters): {counter_kind:?}");
         self.coverage_counters.set_bcb_counter(bcb, counter_kind)
     }

From 678e01a3fc1d21eb8ea4b281b3905c6d3340ce54 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 30 Oct 2023 13:37:34 +1100
Subject: [PATCH 216/297] Delay parsing of `--cfg` and `--check-cfg` options.

By storing the unparsed values in `Config` and then parsing them within
`run_compiler`, the parsing functions can use the main symbol interner,
and not create their own short-lived interners.

This change also eliminates the need for one `EarlyErrorHandler` in
rustdoc, because parsing errors can be reported by another, slightly
later `EarlyErrorHandler`.
---
 compiler/rustc_driver_impl/src/lib.rs     |   6 +-
 compiler/rustc_interface/src/interface.rs | 531 +++++++++++-----------
 src/librustdoc/core.rs                    |   7 +-
 src/librustdoc/doctest.rs                 |  11 +-
 src/librustdoc/lib.rs                     |   5 +-
 5 files changed, 271 insertions(+), 289 deletions(-)

diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs
index 37897b8d702b..e4cb3fd25cdb 100644
--- a/compiler/rustc_driver_impl/src/lib.rs
+++ b/compiler/rustc_driver_impl/src/lib.rs
@@ -317,13 +317,11 @@ fn run_compiler(
         return Ok(());
     }
 
-    let cfg = interface::parse_cfg(&early_error_handler, matches.opt_strs("cfg"));
-    let check_cfg = interface::parse_check_cfg(&early_error_handler, matches.opt_strs("check-cfg"));
     let (odir, ofile) = make_output(&matches);
     let mut config = interface::Config {
         opts: sopts,
-        crate_cfg: cfg,
-        crate_check_cfg: check_cfg,
+        crate_cfg: matches.opt_strs("cfg"),
+        crate_check_cfg: matches.opt_strs("check-cfg"),
         input: Input::File(PathBuf::new()),
         output_file: ofile,
         output_dir: odir,
diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index e703fe228c3b..6b51dfea9ecf 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -65,295 +65,286 @@ impl Compiler {
 
 /// Converts strings provided as `--cfg [cfgspec]` into a `Cfg`.
 pub fn parse_cfg(handler: &EarlyErrorHandler, cfgs: Vec<String>) -> Cfg<String> {
-    // This creates a short-lived `SessionGlobals`, containing an interner. The
-    // parsed values are converted from symbols to strings before exiting
-    // because the symbols are meaningless once the interner is gone.
-    rustc_span::create_default_session_if_not_set_then(move |_| {
-        cfgs.into_iter()
-            .map(|s| {
-                let sess = ParseSess::with_silent_emitter(Some(format!(
-                    "this error occurred on the command line: `--cfg={s}`"
-                )));
-                let filename = FileName::cfg_spec_source_code(&s);
-
-                macro_rules! error {
-                    ($reason: expr) => {
-                        handler.early_error(format!(
-                            concat!("invalid `--cfg` argument: `{}` (", $reason, ")"),
-                            s
-                        ));
-                    };
-                }
-
-                match maybe_new_parser_from_source_str(&sess, filename, s.to_string()) {
-                    Ok(mut parser) => match parser.parse_meta_item() {
-                        Ok(meta_item) if parser.token == token::Eof => {
-                            if meta_item.path.segments.len() != 1 {
-                                error!("argument key must be an identifier");
-                            }
-                            match &meta_item.kind {
-                                MetaItemKind::List(..) => {}
-                                MetaItemKind::NameValue(lit) if !lit.kind.is_str() => {
-                                    error!("argument value must be a string");
-                                }
-                                MetaItemKind::NameValue(..) | MetaItemKind::Word => {
-                                    let ident = meta_item.ident().expect("multi-segment cfg key");
-                                    return (
-                                        ident.name.to_string(),
-                                        meta_item.value_str().map(|sym| sym.to_string()),
-                                    );
-                                }
-                            }
-                        }
-                        Ok(..) => {}
-                        Err(err) => err.cancel(),
-                    },
-                    Err(errs) => drop(errs),
-                }
-
-                // If the user tried to use a key="value" flag, but is missing the quotes, provide
-                // a hint about how to resolve this.
-                if s.contains('=') && !s.contains("=\"") && !s.ends_with('"') {
-                    error!(concat!(
-                        r#"expected `key` or `key="value"`, ensure escaping is appropriate"#,
-                        r#" for your shell, try 'key="value"' or key=\"value\""#
-                    ));
-                } else {
-                    error!(r#"expected `key` or `key="value"`"#);
-                }
-            })
-            .collect::<Cfg<String>>()
-    })
-}
-
-/// Converts strings provided as `--check-cfg [specs]` into a `CheckCfg`.
-pub fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -> CheckCfg<String> {
-    // The comment about `SessionGlobals` and symbols in `parse_cfg` above
-    // applies here too.
-    rustc_span::create_default_session_if_not_set_then(move |_| {
-        // If any --check-cfg is passed then exhaustive_values and exhaustive_names
-        // are enabled by default.
-        let exhaustive_names = !specs.is_empty();
-        let exhaustive_values = !specs.is_empty();
-        let mut check_cfg = CheckCfg { exhaustive_names, exhaustive_values, ..CheckCfg::default() };
-
-        let mut old_syntax = None;
-        for s in specs {
+    cfgs.into_iter()
+        .map(|s| {
             let sess = ParseSess::with_silent_emitter(Some(format!(
-                "this error occurred on the command line: `--check-cfg={s}`"
+                "this error occurred on the command line: `--cfg={s}`"
             )));
             let filename = FileName::cfg_spec_source_code(&s);
 
             macro_rules! error {
-                ($reason:expr) => {
+                ($reason: expr) => {
                     handler.early_error(format!(
-                        concat!("invalid `--check-cfg` argument: `{}` (", $reason, ")"),
+                        concat!("invalid `--cfg` argument: `{}` (", $reason, ")"),
                         s
-                    ))
+                    ));
                 };
             }
 
-            let expected_error = || -> ! {
-                error!("expected `cfg(name, values(\"value1\", \"value2\", ... \"valueN\"))`")
-            };
-
-            let Ok(mut parser) = maybe_new_parser_from_source_str(&sess, filename, s.to_string())
-            else {
-                expected_error();
-            };
-
-            let meta_item = match parser.parse_meta_item() {
-                Ok(meta_item) if parser.token == token::Eof => meta_item,
-                Ok(..) => expected_error(),
-                Err(err) => {
-                    err.cancel();
-                    expected_error();
-                }
-            };
-
-            let Some(args) = meta_item.meta_item_list() else {
-                expected_error();
-            };
-
-            if meta_item.has_name(sym::names) {
-                // defaults are flipped for the old syntax
-                if old_syntax == None {
-                    check_cfg.exhaustive_names = false;
-                    check_cfg.exhaustive_values = false;
-                }
-                old_syntax = Some(true);
-
-                check_cfg.exhaustive_names = true;
-                for arg in args {
-                    if arg.is_word() && arg.ident().is_some() {
-                        let ident = arg.ident().expect("multi-segment cfg key");
-                        check_cfg
-                            .expecteds
-                            .entry(ident.name.to_string())
-                            .or_insert(ExpectedValues::Any);
-                    } else {
-                        error!("`names()` arguments must be simple identifiers");
-                    }
-                }
-            } else if meta_item.has_name(sym::values) {
-                // defaults are flipped for the old syntax
-                if old_syntax == None {
-                    check_cfg.exhaustive_names = false;
-                    check_cfg.exhaustive_values = false;
-                }
-                old_syntax = Some(true);
-
-                if let Some((name, values)) = args.split_first() {
-                    if name.is_word() && name.ident().is_some() {
-                        let ident = name.ident().expect("multi-segment cfg key");
-                        let expected_values = check_cfg
-                            .expecteds
-                            .entry(ident.name.to_string())
-                            .and_modify(|expected_values| match expected_values {
-                                ExpectedValues::Some(_) => {}
-                                ExpectedValues::Any => {
-                                    // handle the case where names(...) was done
-                                    // before values by changing to a list
-                                    *expected_values = ExpectedValues::Some(FxHashSet::default());
-                                }
-                            })
-                            .or_insert_with(|| ExpectedValues::Some(FxHashSet::default()));
-
-                        let ExpectedValues::Some(expected_values) = expected_values else {
-                            bug!("`expected_values` should be a list a values")
-                        };
-
-                        for val in values {
-                            if let Some(LitKind::Str(s, _)) = val.lit().map(|lit| &lit.kind) {
-                                expected_values.insert(Some(s.to_string()));
-                            } else {
-                                error!("`values()` arguments must be string literals");
+            match maybe_new_parser_from_source_str(&sess, filename, s.to_string()) {
+                Ok(mut parser) => match parser.parse_meta_item() {
+                    Ok(meta_item) if parser.token == token::Eof => {
+                        if meta_item.path.segments.len() != 1 {
+                            error!("argument key must be an identifier");
+                        }
+                        match &meta_item.kind {
+                            MetaItemKind::List(..) => {}
+                            MetaItemKind::NameValue(lit) if !lit.kind.is_str() => {
+                                error!("argument value must be a string");
                             }
-                        }
-
-                        if values.is_empty() {
-                            expected_values.insert(None);
-                        }
-                    } else {
-                        error!("`values()` first argument must be a simple identifier");
-                    }
-                } else if args.is_empty() {
-                    check_cfg.exhaustive_values = true;
-                } else {
-                    expected_error();
-                }
-            } else if meta_item.has_name(sym::cfg) {
-                old_syntax = Some(false);
-
-                let mut names = Vec::new();
-                let mut values: FxHashSet<_> = Default::default();
-
-                let mut any_specified = false;
-                let mut values_specified = false;
-                let mut values_any_specified = false;
-
-                for arg in args {
-                    if arg.is_word() && let Some(ident) = arg.ident() {
-                        if values_specified {
-                            error!("`cfg()` names cannot be after values");
-                        }
-                        names.push(ident);
-                    } else if arg.has_name(sym::any)
-                        && let Some(args) = arg.meta_item_list()
-                    {
-                        if any_specified {
-                            error!("`any()` cannot be specified multiple times");
-                        }
-                        any_specified = true;
-                        if !args.is_empty() {
-                            error!("`any()` must be empty");
-                        }
-                    } else if arg.has_name(sym::values)
-                        && let Some(args) = arg.meta_item_list()
-                    {
-                        if names.is_empty() {
-                            error!("`values()` cannot be specified before the names");
-                        } else if values_specified {
-                            error!("`values()` cannot be specified multiple times");
-                        }
-                        values_specified = true;
-
-                        for arg in args {
-                            if let Some(LitKind::Str(s, _)) =
-                                arg.lit().map(|lit| &lit.kind)
-                            {
-                                values.insert(Some(s.to_string()));
-                            } else if arg.has_name(sym::any)
-                                && let Some(args) = arg.meta_item_list()
-                            {
-                                if values_any_specified {
-                                    error!(
-                                        "`any()` in `values()` cannot be specified multiple times"
-                                    );
-                                }
-                                values_any_specified = true;
-                                if !args.is_empty() {
-                                    error!("`any()` must be empty");
-                                }
-                            } else {
-                                error!(
-                                    "`values()` arguments must be string literals or `any()`"
+                            MetaItemKind::NameValue(..) | MetaItemKind::Word => {
+                                let ident = meta_item.ident().expect("multi-segment cfg key");
+                                return (
+                                    ident.name.to_string(),
+                                    meta_item.value_str().map(|sym| sym.to_string()),
                                 );
                             }
                         }
-                    } else {
-                        error!(
-                            "`cfg()` arguments must be simple identifiers, `any()` or `values(...)`"
-                        );
                     }
-                }
+                    Ok(..) => {}
+                    Err(err) => err.cancel(),
+                },
+                Err(errs) => drop(errs),
+            }
 
-                if values.is_empty() && !values_any_specified && !any_specified {
-                    values.insert(None);
-                } else if !values.is_empty() && values_any_specified {
-                    error!(
-                        "`values()` arguments cannot specify string literals and `any()` at the same time"
-                    );
-                }
+            // If the user tried to use a key="value" flag, but is missing the quotes, provide
+            // a hint about how to resolve this.
+            if s.contains('=') && !s.contains("=\"") && !s.ends_with('"') {
+                error!(concat!(
+                    r#"expected `key` or `key="value"`, ensure escaping is appropriate"#,
+                    r#" for your shell, try 'key="value"' or key=\"value\""#
+                ));
+            } else {
+                error!(r#"expected `key` or `key="value"`"#);
+            }
+        })
+        .collect::<Cfg<String>>()
+}
 
-                if any_specified {
-                    if names.is_empty()
-                        && values.is_empty()
-                        && !values_specified
-                        && !values_any_specified
-                    {
-                        check_cfg.exhaustive_names = false;
-                    } else {
-                        error!("`cfg(any())` can only be provided in isolation");
+/// Converts strings provided as `--check-cfg [specs]` into a `CheckCfg`.
+pub fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -> CheckCfg<String> {
+    // If any --check-cfg is passed then exhaustive_values and exhaustive_names
+    // are enabled by default.
+    let exhaustive_names = !specs.is_empty();
+    let exhaustive_values = !specs.is_empty();
+    let mut check_cfg = CheckCfg { exhaustive_names, exhaustive_values, ..CheckCfg::default() };
+
+    let mut old_syntax = None;
+    for s in specs {
+        let sess = ParseSess::with_silent_emitter(Some(format!(
+            "this error occurred on the command line: `--check-cfg={s}`"
+        )));
+        let filename = FileName::cfg_spec_source_code(&s);
+
+        macro_rules! error {
+            ($reason:expr) => {
+                handler.early_error(format!(
+                    concat!("invalid `--check-cfg` argument: `{}` (", $reason, ")"),
+                    s
+                ))
+            };
+        }
+
+        let expected_error = || -> ! {
+            error!("expected `cfg(name, values(\"value1\", \"value2\", ... \"valueN\"))`")
+        };
+
+        let Ok(mut parser) = maybe_new_parser_from_source_str(&sess, filename, s.to_string())
+        else {
+            expected_error();
+        };
+
+        let meta_item = match parser.parse_meta_item() {
+            Ok(meta_item) if parser.token == token::Eof => meta_item,
+            Ok(..) => expected_error(),
+            Err(err) => {
+                err.cancel();
+                expected_error();
+            }
+        };
+
+        let Some(args) = meta_item.meta_item_list() else {
+            expected_error();
+        };
+
+        if meta_item.has_name(sym::names) {
+            // defaults are flipped for the old syntax
+            if old_syntax == None {
+                check_cfg.exhaustive_names = false;
+                check_cfg.exhaustive_values = false;
+            }
+            old_syntax = Some(true);
+
+            check_cfg.exhaustive_names = true;
+            for arg in args {
+                if arg.is_word() && arg.ident().is_some() {
+                    let ident = arg.ident().expect("multi-segment cfg key");
+                    check_cfg
+                        .expecteds
+                        .entry(ident.name.to_string())
+                        .or_insert(ExpectedValues::Any);
+                } else {
+                    error!("`names()` arguments must be simple identifiers");
+                }
+            }
+        } else if meta_item.has_name(sym::values) {
+            // defaults are flipped for the old syntax
+            if old_syntax == None {
+                check_cfg.exhaustive_names = false;
+                check_cfg.exhaustive_values = false;
+            }
+            old_syntax = Some(true);
+
+            if let Some((name, values)) = args.split_first() {
+                if name.is_word() && name.ident().is_some() {
+                    let ident = name.ident().expect("multi-segment cfg key");
+                    let expected_values = check_cfg
+                        .expecteds
+                        .entry(ident.name.to_string())
+                        .and_modify(|expected_values| match expected_values {
+                            ExpectedValues::Some(_) => {}
+                            ExpectedValues::Any => {
+                                // handle the case where names(...) was done
+                                // before values by changing to a list
+                                *expected_values = ExpectedValues::Some(FxHashSet::default());
+                            }
+                        })
+                        .or_insert_with(|| ExpectedValues::Some(FxHashSet::default()));
+
+                    let ExpectedValues::Some(expected_values) = expected_values else {
+                        bug!("`expected_values` should be a list a values")
+                    };
+
+                    for val in values {
+                        if let Some(LitKind::Str(s, _)) = val.lit().map(|lit| &lit.kind) {
+                            expected_values.insert(Some(s.to_string()));
+                        } else {
+                            error!("`values()` arguments must be string literals");
+                        }
+                    }
+
+                    if values.is_empty() {
+                        expected_values.insert(None);
                     }
                 } else {
-                    for name in names {
-                        check_cfg
-                            .expecteds
-                            .entry(name.to_string())
-                            .and_modify(|v| match v {
-                                ExpectedValues::Some(v) if !values_any_specified => {
-                                    v.extend(values.clone())
-                                }
-                                ExpectedValues::Some(_) => *v = ExpectedValues::Any,
-                                ExpectedValues::Any => {}
-                            })
-                            .or_insert_with(|| {
-                                if values_any_specified {
-                                    ExpectedValues::Any
-                                } else {
-                                    ExpectedValues::Some(values.clone())
-                                }
-                            });
-                    }
+                    error!("`values()` first argument must be a simple identifier");
                 }
+            } else if args.is_empty() {
+                check_cfg.exhaustive_values = true;
             } else {
                 expected_error();
             }
-        }
+        } else if meta_item.has_name(sym::cfg) {
+            old_syntax = Some(false);
 
-        check_cfg
-    })
+            let mut names = Vec::new();
+            let mut values: FxHashSet<_> = Default::default();
+
+            let mut any_specified = false;
+            let mut values_specified = false;
+            let mut values_any_specified = false;
+
+            for arg in args {
+                if arg.is_word() && let Some(ident) = arg.ident() {
+                    if values_specified {
+                        error!("`cfg()` names cannot be after values");
+                    }
+                    names.push(ident);
+                } else if arg.has_name(sym::any)
+                    && let Some(args) = arg.meta_item_list()
+                {
+                    if any_specified {
+                        error!("`any()` cannot be specified multiple times");
+                    }
+                    any_specified = true;
+                    if !args.is_empty() {
+                        error!("`any()` must be empty");
+                    }
+                } else if arg.has_name(sym::values)
+                    && let Some(args) = arg.meta_item_list()
+                {
+                    if names.is_empty() {
+                        error!("`values()` cannot be specified before the names");
+                    } else if values_specified {
+                        error!("`values()` cannot be specified multiple times");
+                    }
+                    values_specified = true;
+
+                    for arg in args {
+                        if let Some(LitKind::Str(s, _)) =
+                            arg.lit().map(|lit| &lit.kind)
+                        {
+                            values.insert(Some(s.to_string()));
+                        } else if arg.has_name(sym::any)
+                            && let Some(args) = arg.meta_item_list()
+                        {
+                            if values_any_specified {
+                                error!(
+                                    "`any()` in `values()` cannot be specified multiple times"
+                                );
+                            }
+                            values_any_specified = true;
+                            if !args.is_empty() {
+                                error!("`any()` must be empty");
+                            }
+                        } else {
+                            error!(
+                                "`values()` arguments must be string literals or `any()`"
+                            );
+                        }
+                    }
+                } else {
+                    error!(
+                        "`cfg()` arguments must be simple identifiers, `any()` or `values(...)`"
+                    );
+                }
+            }
+
+            if values.is_empty() && !values_any_specified && !any_specified {
+                values.insert(None);
+            } else if !values.is_empty() && values_any_specified {
+                error!(
+                    "`values()` arguments cannot specify string literals and `any()` at the same time"
+                );
+            }
+
+            if any_specified {
+                if names.is_empty()
+                    && values.is_empty()
+                    && !values_specified
+                    && !values_any_specified
+                {
+                    check_cfg.exhaustive_names = false;
+                } else {
+                    error!("`cfg(any())` can only be provided in isolation");
+                }
+            } else {
+                for name in names {
+                    check_cfg
+                        .expecteds
+                        .entry(name.to_string())
+                        .and_modify(|v| match v {
+                            ExpectedValues::Some(v) if !values_any_specified => {
+                                v.extend(values.clone())
+                            }
+                            ExpectedValues::Some(_) => *v = ExpectedValues::Any,
+                            ExpectedValues::Any => {}
+                        })
+                        .or_insert_with(|| {
+                            if values_any_specified {
+                                ExpectedValues::Any
+                            } else {
+                                ExpectedValues::Some(values.clone())
+                            }
+                        });
+                }
+            }
+        } else {
+            expected_error();
+        }
+    }
+
+    check_cfg
 }
 
 /// The compiler configuration
@@ -361,9 +352,9 @@ pub struct Config {
     /// Command line options
     pub opts: config::Options,
 
-    /// cfg! configuration in addition to the default ones
-    pub crate_cfg: Cfg<String>,
-    pub crate_check_cfg: CheckCfg<String>,
+    /// Unparsed cfg! configuration in addition to the default ones.
+    pub crate_cfg: Vec<String>,
+    pub crate_check_cfg: Vec<String>,
 
     pub input: Input,
     pub output_dir: Option<PathBuf>,
@@ -436,8 +427,8 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
             let (mut sess, codegen_backend) = util::create_session(
                 &handler,
                 config.opts,
-                config.crate_cfg,
-                config.crate_check_cfg,
+                parse_cfg(&handler, config.crate_cfg),
+                parse_check_cfg(&handler, config.crate_check_cfg),
                 config.locale_resources,
                 config.file_loader,
                 CompilerIO {
diff --git a/src/librustdoc/core.rs b/src/librustdoc/core.rs
index bcc61df7c4d8..17b117de5d47 100644
--- a/src/librustdoc/core.rs
+++ b/src/librustdoc/core.rs
@@ -14,8 +14,8 @@ use rustc_lint::{late_lint_mod, MissingDoc};
 use rustc_middle::hir::nested_filter;
 use rustc_middle::ty::{ParamEnv, Ty, TyCtxt};
 use rustc_session::config::{self, CrateType, ErrorOutputType, ResolveDocLinks};
+use rustc_session::lint;
 use rustc_session::Session;
-use rustc_session::{lint, EarlyErrorHandler};
 use rustc_span::symbol::sym;
 use rustc_span::{source_map, Span};
 
@@ -175,7 +175,6 @@ pub(crate) fn new_handler(
 
 /// Parse, resolve, and typecheck the given crate.
 pub(crate) fn create_config(
-    handler: &EarlyErrorHandler,
     RustdocOptions {
         input,
         crate_name,
@@ -255,8 +254,8 @@ pub(crate) fn create_config(
 
     interface::Config {
         opts: sessopts,
-        crate_cfg: interface::parse_cfg(handler, cfgs),
-        crate_check_cfg: interface::parse_check_cfg(handler, check_cfgs),
+        crate_cfg: cfgs,
+        crate_check_cfg: check_cfgs,
         input,
         output_file: None,
         output_dir: None,
diff --git a/src/librustdoc/doctest.rs b/src/librustdoc/doctest.rs
index c61c98e4de9c..2412865801d6 100644
--- a/src/librustdoc/doctest.rs
+++ b/src/librustdoc/doctest.rs
@@ -13,7 +13,7 @@ use rustc_parse::parser::attr::InnerAttrPolicy;
 use rustc_resolve::rustdoc::span_of_fragments;
 use rustc_session::config::{self, CrateType, ErrorOutputType};
 use rustc_session::parse::ParseSess;
-use rustc_session::{lint, EarlyErrorHandler, Session};
+use rustc_session::{lint, Session};
 use rustc_span::edition::Edition;
 use rustc_span::source_map::SourceMap;
 use rustc_span::symbol::sym;
@@ -85,18 +85,13 @@ pub(crate) fn run(options: RustdocOptions) -> Result<(), ErrorGuaranteed> {
         ..config::Options::default()
     };
 
-    let early_error_handler = EarlyErrorHandler::new(ErrorOutputType::default());
-
     let mut cfgs = options.cfgs.clone();
     cfgs.push("doc".to_owned());
     cfgs.push("doctest".to_owned());
     let config = interface::Config {
         opts: sessopts,
-        crate_cfg: interface::parse_cfg(&early_error_handler, cfgs),
-        crate_check_cfg: interface::parse_check_cfg(
-            &early_error_handler,
-            options.check_cfgs.clone(),
-        ),
+        crate_cfg: cfgs,
+        crate_check_cfg: options.check_cfgs.clone(),
         input,
         output_file: None,
         output_dir: None,
diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs
index 41aee06c8d26..dda06d4c9c1d 100644
--- a/src/librustdoc/lib.rs
+++ b/src/librustdoc/lib.rs
@@ -757,8 +757,7 @@ fn main_args(
         (false, true) => {
             let input = options.input.clone();
             let edition = options.edition;
-            let config =
-                core::create_config(handler, options, &render_options, using_internal_features);
+            let config = core::create_config(options, &render_options, using_internal_features);
 
             // `markdown::render` can invoke `doctest::make_test`, which
             // requires session globals and a thread pool, so we use
@@ -791,7 +790,7 @@ fn main_args(
     let scrape_examples_options = options.scrape_examples_options.clone();
     let bin_crate = options.bin_crate;
 
-    let config = core::create_config(handler, options, &render_options, using_internal_features);
+    let config = core::create_config(options, &render_options, using_internal_features);
 
     interface::run_compiler(config, |compiler| {
         let sess = compiler.session();

From bfcff7933e9be5c9ff31854896e9582b84b19d36 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 30 Oct 2023 13:52:03 +1100
Subject: [PATCH 217/297] Reduce exposure of cfg parsers.

---
 compiler/rustc_interface/src/interface.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index 6b51dfea9ecf..3b442697b5f3 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -64,7 +64,7 @@ impl Compiler {
 }
 
 /// Converts strings provided as `--cfg [cfgspec]` into a `Cfg`.
-pub fn parse_cfg(handler: &EarlyErrorHandler, cfgs: Vec<String>) -> Cfg<String> {
+pub(crate) fn parse_cfg(handler: &EarlyErrorHandler, cfgs: Vec<String>) -> Cfg<String> {
     cfgs.into_iter()
         .map(|s| {
             let sess = ParseSess::with_silent_emitter(Some(format!(
@@ -122,7 +122,7 @@ pub fn parse_cfg(handler: &EarlyErrorHandler, cfgs: Vec<String>) -> Cfg<String>
 }
 
 /// Converts strings provided as `--check-cfg [specs]` into a `CheckCfg`.
-pub fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -> CheckCfg<String> {
+pub(crate) fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -> CheckCfg<String> {
     // If any --check-cfg is passed then exhaustive_values and exhaustive_names
     // are enabled by default.
     let exhaustive_names = !specs.is_empty();

From 8e4ac980fd775d311bc63642c8d6b5739aa6f34c Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 30 Oct 2023 14:01:33 +1100
Subject: [PATCH 218/297] Change cfg parsers to produce symbols instead of
 strings.

---
 compiler/rustc_interface/src/interface.rs | 29 ++++++++------------
 compiler/rustc_interface/src/tests.rs     |  4 +--
 compiler/rustc_interface/src/util.rs      |  5 ++--
 compiler/rustc_session/src/config.rs      | 32 +----------------------
 4 files changed, 16 insertions(+), 54 deletions(-)

diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index 3b442697b5f3..cf887bc665bd 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -24,6 +24,7 @@ use rustc_session::Session;
 use rustc_session::{lint, EarlyErrorHandler};
 use rustc_span::source_map::{FileLoader, FileName};
 use rustc_span::symbol::sym;
+use rustc_span::Symbol;
 use std::path::PathBuf;
 use std::result;
 use std::sync::Arc;
@@ -64,7 +65,7 @@ impl Compiler {
 }
 
 /// Converts strings provided as `--cfg [cfgspec]` into a `Cfg`.
-pub(crate) fn parse_cfg(handler: &EarlyErrorHandler, cfgs: Vec<String>) -> Cfg<String> {
+pub(crate) fn parse_cfg(handler: &EarlyErrorHandler, cfgs: Vec<String>) -> Cfg<Symbol> {
     cfgs.into_iter()
         .map(|s| {
             let sess = ParseSess::with_silent_emitter(Some(format!(
@@ -94,10 +95,7 @@ pub(crate) fn parse_cfg(handler: &EarlyErrorHandler, cfgs: Vec<String>) -> Cfg<S
                             }
                             MetaItemKind::NameValue(..) | MetaItemKind::Word => {
                                 let ident = meta_item.ident().expect("multi-segment cfg key");
-                                return (
-                                    ident.name.to_string(),
-                                    meta_item.value_str().map(|sym| sym.to_string()),
-                                );
+                                return (ident.name, meta_item.value_str());
                             }
                         }
                     }
@@ -118,11 +116,11 @@ pub(crate) fn parse_cfg(handler: &EarlyErrorHandler, cfgs: Vec<String>) -> Cfg<S
                 error!(r#"expected `key` or `key="value"`"#);
             }
         })
-        .collect::<Cfg<String>>()
+        .collect::<Cfg<Symbol>>()
 }
 
 /// Converts strings provided as `--check-cfg [specs]` into a `CheckCfg`.
-pub(crate) fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -> CheckCfg<String> {
+pub(crate) fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -> CheckCfg<Symbol> {
     // If any --check-cfg is passed then exhaustive_values and exhaustive_names
     // are enabled by default.
     let exhaustive_names = !specs.is_empty();
@@ -179,10 +177,7 @@ pub(crate) fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -
             for arg in args {
                 if arg.is_word() && arg.ident().is_some() {
                     let ident = arg.ident().expect("multi-segment cfg key");
-                    check_cfg
-                        .expecteds
-                        .entry(ident.name.to_string())
-                        .or_insert(ExpectedValues::Any);
+                    check_cfg.expecteds.entry(ident.name).or_insert(ExpectedValues::Any);
                 } else {
                     error!("`names()` arguments must be simple identifiers");
                 }
@@ -200,7 +195,7 @@ pub(crate) fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -
                     let ident = name.ident().expect("multi-segment cfg key");
                     let expected_values = check_cfg
                         .expecteds
-                        .entry(ident.name.to_string())
+                        .entry(ident.name)
                         .and_modify(|expected_values| match expected_values {
                             ExpectedValues::Some(_) => {}
                             ExpectedValues::Any => {
@@ -217,7 +212,7 @@ pub(crate) fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -
 
                     for val in values {
                         if let Some(LitKind::Str(s, _)) = val.lit().map(|lit| &lit.kind) {
-                            expected_values.insert(Some(s.to_string()));
+                            expected_values.insert(Some(*s));
                         } else {
                             error!("`values()` arguments must be string literals");
                         }
@@ -271,10 +266,8 @@ pub(crate) fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -
                     values_specified = true;
 
                     for arg in args {
-                        if let Some(LitKind::Str(s, _)) =
-                            arg.lit().map(|lit| &lit.kind)
-                        {
-                            values.insert(Some(s.to_string()));
+                        if let Some(LitKind::Str(s, _)) = arg.lit().map(|lit| &lit.kind) {
+                            values.insert(Some(*s));
                         } else if arg.has_name(sym::any)
                             && let Some(args) = arg.meta_item_list()
                         {
@@ -322,7 +315,7 @@ pub(crate) fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -
                 for name in names {
                     check_cfg
                         .expecteds
-                        .entry(name.to_string())
+                        .entry(name.name)
                         .and_modify(|v| match v {
                             ExpectedValues::Some(v) if !values_any_specified => {
                                 v.extend(values.clone())
diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index 57ca709267a7..466aa21ad8e7 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -26,8 +26,8 @@ use rustc_session::{build_session, getopts, Session};
 use rustc_session::{CompilerIO, EarlyErrorHandler};
 use rustc_span::edition::{Edition, DEFAULT_EDITION};
 use rustc_span::symbol::sym;
-use rustc_span::FileName;
 use rustc_span::SourceFileHashAlgorithm;
+use rustc_span::{FileName, Symbol};
 use rustc_target::spec::{CodeModel, LinkerFlavorCli, MergeFunctions, PanicStrategy, RelocModel};
 use rustc_target::spec::{RelroLevel, SanitizerSet, SplitDebuginfo, StackProtector, TlsModel};
 use std::collections::{BTreeMap, BTreeSet};
@@ -38,7 +38,7 @@ use std::sync::Arc;
 fn mk_session(
     handler: &mut EarlyErrorHandler,
     matches: getopts::Matches,
-) -> (Session, Cfg<String>) {
+) -> (Session, Cfg<Symbol>) {
     let registry = registry::Registry::new(&[]);
     let sessopts = build_session_options(handler, &matches);
     let cfg = parse_cfg(handler, matches.opt_strs("cfg"));
diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs
index 4d0be65697a4..a974bdd3c0e6 100644
--- a/compiler/rustc_interface/src/util.rs
+++ b/compiler/rustc_interface/src/util.rs
@@ -59,8 +59,8 @@ pub fn add_configuration(
 pub fn create_session(
     handler: &EarlyErrorHandler,
     sopts: config::Options,
-    cfg: Cfg<String>,
-    check_cfg: CheckCfg<String>,
+    cfg: Cfg<Symbol>,
+    mut check_cfg: CheckCfg<Symbol>,
     locale_resources: &'static [&'static str],
     file_loader: Option<Box<dyn FileLoader + Send + Sync + 'static>>,
     io: CompilerIO,
@@ -123,7 +123,6 @@ pub fn create_session(
     let mut cfg = config::build_configuration(&sess, cfg);
     add_configuration(&mut cfg, &mut sess, &*codegen_backend);
 
-    let mut check_cfg = check_cfg.intern();
     check_cfg.fill_well_known(&sess.target);
 
     // These configs use symbols, rather than strings.
diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index 854e2b779935..4a0bdd511a37 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -1386,30 +1386,6 @@ impl<T> Default for CheckCfg<T> {
     }
 }
 
-impl CheckCfg<String> {
-    pub fn intern(self) -> CheckCfg<Symbol> {
-        CheckCfg {
-            exhaustive_names: self.exhaustive_names,
-            exhaustive_values: self.exhaustive_values,
-            expecteds: self
-                .expecteds
-                .into_iter()
-                .map(|(name, values)| {
-                    (
-                        Symbol::intern(&name),
-                        match values {
-                            ExpectedValues::Some(values) => ExpectedValues::Some(
-                                values.into_iter().map(|b| b.map(|b| Symbol::intern(&b))).collect(),
-                            ),
-                            ExpectedValues::Any => ExpectedValues::Any,
-                        },
-                    )
-                })
-                .collect(),
-        }
-    }
-}
-
 pub enum ExpectedValues<T> {
     Some(FxHashSet<Option<T>>),
     Any,
@@ -1582,13 +1558,7 @@ impl CheckCfg<Symbol> {
     }
 }
 
-pub fn build_configuration(sess: &Session, user_cfg: Cfg<String>) -> Cfg<Symbol> {
-    // We can now intern these strings.
-    let mut user_cfg: Cfg<Symbol> = user_cfg
-        .into_iter()
-        .map(|(a, b)| (Symbol::intern(&a), b.map(|b| Symbol::intern(&b))))
-        .collect();
-
+pub fn build_configuration(sess: &Session, mut user_cfg: Cfg<Symbol>) -> Cfg<Symbol> {
     // Combine the configuration requested by the session (command line) with
     // some default and generated configuration items.
     let default_cfg = default_configuration(sess);

From 5c6a12c1affd7759eb793f3bcd2a97577fab7ab4 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 30 Oct 2023 14:05:06 +1100
Subject: [PATCH 219/297] Make `Cfg` and `CheckCfg` non-generic.

They now only ever contains symbols.
---
 compiler/rustc_interface/src/interface.rs |  7 +++--
 compiler/rustc_interface/src/tests.rs     |  8 ++----
 compiler/rustc_interface/src/util.rs      | 10 +++-----
 compiler/rustc_session/src/config.rs      | 31 ++++++++---------------
 compiler/rustc_session/src/parse.rs       |  4 +--
 5 files changed, 20 insertions(+), 40 deletions(-)

diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index cf887bc665bd..66bcb199709e 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -24,7 +24,6 @@ use rustc_session::Session;
 use rustc_session::{lint, EarlyErrorHandler};
 use rustc_span::source_map::{FileLoader, FileName};
 use rustc_span::symbol::sym;
-use rustc_span::Symbol;
 use std::path::PathBuf;
 use std::result;
 use std::sync::Arc;
@@ -65,7 +64,7 @@ impl Compiler {
 }
 
 /// Converts strings provided as `--cfg [cfgspec]` into a `Cfg`.
-pub(crate) fn parse_cfg(handler: &EarlyErrorHandler, cfgs: Vec<String>) -> Cfg<Symbol> {
+pub(crate) fn parse_cfg(handler: &EarlyErrorHandler, cfgs: Vec<String>) -> Cfg {
     cfgs.into_iter()
         .map(|s| {
             let sess = ParseSess::with_silent_emitter(Some(format!(
@@ -116,11 +115,11 @@ pub(crate) fn parse_cfg(handler: &EarlyErrorHandler, cfgs: Vec<String>) -> Cfg<S
                 error!(r#"expected `key` or `key="value"`"#);
             }
         })
-        .collect::<Cfg<Symbol>>()
+        .collect::<Cfg>()
 }
 
 /// Converts strings provided as `--check-cfg [specs]` into a `CheckCfg`.
-pub(crate) fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -> CheckCfg<Symbol> {
+pub(crate) fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -> CheckCfg {
     // If any --check-cfg is passed then exhaustive_values and exhaustive_names
     // are enabled by default.
     let exhaustive_names = !specs.is_empty();
diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index 466aa21ad8e7..41ccb25f7893 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -26,8 +26,7 @@ use rustc_session::{build_session, getopts, Session};
 use rustc_session::{CompilerIO, EarlyErrorHandler};
 use rustc_span::edition::{Edition, DEFAULT_EDITION};
 use rustc_span::symbol::sym;
-use rustc_span::SourceFileHashAlgorithm;
-use rustc_span::{FileName, Symbol};
+use rustc_span::{FileName, SourceFileHashAlgorithm};
 use rustc_target::spec::{CodeModel, LinkerFlavorCli, MergeFunctions, PanicStrategy, RelocModel};
 use rustc_target::spec::{RelroLevel, SanitizerSet, SplitDebuginfo, StackProtector, TlsModel};
 use std::collections::{BTreeMap, BTreeSet};
@@ -35,10 +34,7 @@ use std::num::NonZeroUsize;
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
-fn mk_session(
-    handler: &mut EarlyErrorHandler,
-    matches: getopts::Matches,
-) -> (Session, Cfg<Symbol>) {
+fn mk_session(handler: &mut EarlyErrorHandler, matches: getopts::Matches) -> (Session, Cfg) {
     let registry = registry::Registry::new(&[]);
     let sessopts = build_session_options(handler, &matches);
     let cfg = parse_cfg(handler, matches.opt_strs("cfg"));
diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs
index a974bdd3c0e6..9bda685df013 100644
--- a/compiler/rustc_interface/src/util.rs
+++ b/compiler/rustc_interface/src/util.rs
@@ -36,11 +36,7 @@ pub type MakeBackendFn = fn() -> Box<dyn CodegenBackend>;
 ///
 /// This is performed by checking whether a set of permitted features
 /// is available on the target machine, by querying the codegen backend.
-pub fn add_configuration(
-    cfg: &mut Cfg<Symbol>,
-    sess: &mut Session,
-    codegen_backend: &dyn CodegenBackend,
-) {
+pub fn add_configuration(cfg: &mut Cfg, sess: &mut Session, codegen_backend: &dyn CodegenBackend) {
     let tf = sym::target_feature;
 
     let unstable_target_features = codegen_backend.target_features(sess, true);
@@ -59,8 +55,8 @@ pub fn add_configuration(
 pub fn create_session(
     handler: &EarlyErrorHandler,
     sopts: config::Options,
-    cfg: Cfg<Symbol>,
-    mut check_cfg: CheckCfg<Symbol>,
+    cfg: Cfg,
+    mut check_cfg: CheckCfg,
     locale_resources: &'static [&'static str],
     file_loader: Option<Box<dyn FileLoader + Send + Sync + 'static>>,
     io: CompilerIO,
diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs
index 4a0bdd511a37..a8ebab4ae333 100644
--- a/compiler/rustc_session/src/config.rs
+++ b/compiler/rustc_session/src/config.rs
@@ -1247,8 +1247,8 @@ pub const fn default_lib_output() -> CrateType {
     CrateType::Rlib
 }
 
-fn default_configuration(sess: &Session) -> Cfg<Symbol> {
-    // NOTE: This should be kept in sync with `CheckCfg::<Symbol>::fill_well_known` below.
+fn default_configuration(sess: &Session) -> Cfg {
+    // NOTE: This should be kept in sync with `CheckCfg::fill_well_known` below.
     let end = &sess.target.endian;
     let arch = &sess.target.arch;
     let wordsz = sess.target.pointer_width.to_string();
@@ -1358,32 +1358,21 @@ fn default_configuration(sess: &Session) -> Cfg<Symbol> {
 }
 
 /// The parsed `--cfg` options that define the compilation environment of the
-/// crate, used to drive conditional compilation. `T` is always `String` or
-/// `Symbol`. Strings are used temporarily very early on. Once the the main
-/// symbol interner is running, they are converted to symbols.
+/// crate, used to drive conditional compilation.
 ///
 /// An `FxIndexSet` is used to ensure deterministic ordering of error messages
 /// relating to `--cfg`.
-pub type Cfg<T> = FxIndexSet<(T, Option<T>)>;
+pub type Cfg = FxIndexSet<(Symbol, Option<Symbol>)>;
 
-/// The parsed `--check-cfg` options. The `<T>` structure is similar to `Cfg`.
-pub struct CheckCfg<T> {
+/// The parsed `--check-cfg` options.
+#[derive(Default)]
+pub struct CheckCfg {
     /// Is well known names activated
     pub exhaustive_names: bool,
     /// Is well known values activated
     pub exhaustive_values: bool,
     /// All the expected values for a config name
-    pub expecteds: FxHashMap<T, ExpectedValues<T>>,
-}
-
-impl<T> Default for CheckCfg<T> {
-    fn default() -> Self {
-        CheckCfg {
-            exhaustive_names: false,
-            exhaustive_values: false,
-            expecteds: FxHashMap::default(),
-        }
-    }
+    pub expecteds: FxHashMap<Symbol, ExpectedValues<Symbol>>,
 }
 
 pub enum ExpectedValues<T> {
@@ -1418,7 +1407,7 @@ impl<'a, T: Eq + Hash + Copy + 'a> Extend<&'a T> for ExpectedValues<T> {
     }
 }
 
-impl CheckCfg<Symbol> {
+impl CheckCfg {
     pub fn fill_well_known(&mut self, current_target: &Target) {
         if !self.exhaustive_values && !self.exhaustive_names {
             return;
@@ -1558,7 +1547,7 @@ impl CheckCfg<Symbol> {
     }
 }
 
-pub fn build_configuration(sess: &Session, mut user_cfg: Cfg<Symbol>) -> Cfg<Symbol> {
+pub fn build_configuration(sess: &Session, mut user_cfg: Cfg) -> Cfg {
     // Combine the configuration requested by the session (command line) with
     // some default and generated configuration items.
     let default_cfg = default_configuration(sess);
diff --git a/compiler/rustc_session/src/parse.rs b/compiler/rustc_session/src/parse.rs
index 5b98ee5d992d..4d20d6d41878 100644
--- a/compiler/rustc_session/src/parse.rs
+++ b/compiler/rustc_session/src/parse.rs
@@ -188,8 +188,8 @@ pub fn add_feature_diagnostics_for_issue(
 pub struct ParseSess {
     pub span_diagnostic: Handler,
     pub unstable_features: UnstableFeatures,
-    pub config: Cfg<Symbol>,
-    pub check_config: CheckCfg<Symbol>,
+    pub config: Cfg,
+    pub check_config: CheckCfg,
     pub edition: Edition,
     /// Places where raw identifiers were used. This is used to avoid complaining about idents
     /// clashing with keywords in new editions.

From 371f97257147c88d102dd501992b48ca568cb79b Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 30 Oct 2023 15:06:48 +1100
Subject: [PATCH 220/297] Improve readability of `parse_check_cfg`.

---
 compiler/rustc_interface/src/interface.rs | 39 ++++++++---------------
 1 file changed, 13 insertions(+), 26 deletions(-)

diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index 66bcb199709e..26034dbaf997 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -164,34 +164,31 @@ pub(crate) fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -
             expected_error();
         };
 
-        if meta_item.has_name(sym::names) {
+        let mut set_old_syntax = || {
             // defaults are flipped for the old syntax
             if old_syntax == None {
                 check_cfg.exhaustive_names = false;
                 check_cfg.exhaustive_values = false;
             }
             old_syntax = Some(true);
+        };
+
+        if meta_item.has_name(sym::names) {
+            set_old_syntax();
 
             check_cfg.exhaustive_names = true;
             for arg in args {
-                if arg.is_word() && arg.ident().is_some() {
-                    let ident = arg.ident().expect("multi-segment cfg key");
+                if arg.is_word() && let Some(ident) = arg.ident() {
                     check_cfg.expecteds.entry(ident.name).or_insert(ExpectedValues::Any);
                 } else {
                     error!("`names()` arguments must be simple identifiers");
                 }
             }
         } else if meta_item.has_name(sym::values) {
-            // defaults are flipped for the old syntax
-            if old_syntax == None {
-                check_cfg.exhaustive_names = false;
-                check_cfg.exhaustive_values = false;
-            }
-            old_syntax = Some(true);
+            set_old_syntax();
 
             if let Some((name, values)) = args.split_first() {
-                if name.is_word() && name.ident().is_some() {
-                    let ident = name.ident().expect("multi-segment cfg key");
+                if name.is_word() && let Some(ident) = name.ident() {
                     let expected_values = check_cfg
                         .expecteds
                         .entry(ident.name)
@@ -244,9 +241,7 @@ pub(crate) fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -
                         error!("`cfg()` names cannot be after values");
                     }
                     names.push(ident);
-                } else if arg.has_name(sym::any)
-                    && let Some(args) = arg.meta_item_list()
-                {
+                } else if arg.has_name(sym::any) && let Some(args) = arg.meta_item_list() {
                     if any_specified {
                         error!("`any()` cannot be specified multiple times");
                     }
@@ -254,9 +249,7 @@ pub(crate) fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -
                     if !args.is_empty() {
                         error!("`any()` must be empty");
                     }
-                } else if arg.has_name(sym::values)
-                    && let Some(args) = arg.meta_item_list()
-                {
+                } else if arg.has_name(sym::values) && let Some(args) = arg.meta_item_list() {
                     if names.is_empty() {
                         error!("`values()` cannot be specified before the names");
                     } else if values_specified {
@@ -267,22 +260,16 @@ pub(crate) fn parse_check_cfg(handler: &EarlyErrorHandler, specs: Vec<String>) -
                     for arg in args {
                         if let Some(LitKind::Str(s, _)) = arg.lit().map(|lit| &lit.kind) {
                             values.insert(Some(*s));
-                        } else if arg.has_name(sym::any)
-                            && let Some(args) = arg.meta_item_list()
-                        {
+                        } else if arg.has_name(sym::any) && let Some(args) = arg.meta_item_list() {
                             if values_any_specified {
-                                error!(
-                                    "`any()` in `values()` cannot be specified multiple times"
-                                );
+                                error!("`any()` in `values()` cannot be specified multiple times");
                             }
                             values_any_specified = true;
                             if !args.is_empty() {
                                 error!("`any()` must be empty");
                             }
                         } else {
-                            error!(
-                                "`values()` arguments must be string literals or `any()`"
-                            );
+                            error!("`values()` arguments must be string literals or `any()`");
                         }
                     }
                 } else {

From 85e56e81f8b5ba947d2522369019088e5bd5d6e1 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 30 Oct 2023 15:26:47 +1100
Subject: [PATCH 221/297] Remove out-of-date comment.

It was added in 51938c61f6f1b26e463f9071716f543543486e72, a commit with
a 7,720 line diff and a one line commit message. Even then the comment
was incorrect; there was a removed a `build_output_filenames` call with
a `&[]` argument in rustdoc, but the commit removed that call. In such a
large commit, it's easy for small errors to occur.
---
 compiler/rustc_interface/src/passes.rs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs
index 884afae23d80..1736a1f6d537 100644
--- a/compiler/rustc_interface/src/passes.rs
+++ b/compiler/rustc_interface/src/passes.rs
@@ -602,9 +602,7 @@ fn output_filenames(tcx: TyCtxt<'_>, (): ()) -> Arc<OutputFilenames> {
     let (_, krate) = &*tcx.resolver_for_lowering(()).borrow();
     let crate_name = tcx.crate_name(LOCAL_CRATE);
 
-    // FIXME: rustdoc passes &[] instead of &krate.attrs here
     let outputs = util::build_output_filenames(&krate.attrs, sess);
-
     let output_paths =
         generated_output_paths(tcx, &outputs, sess.io.output_file.is_some(), crate_name);
 

From a60d6438dc483f9d999716f70ea153d7246971b1 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 30 Oct 2023 15:33:05 +1100
Subject: [PATCH 222/297] Wrap some overlong comments.

---
 compiler/rustc_interface/src/passes.rs | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs
index 1736a1f6d537..9685b0777768 100644
--- a/compiler/rustc_interface/src/passes.rs
+++ b/compiler/rustc_interface/src/passes.rs
@@ -880,16 +880,18 @@ fn analysis(tcx: TyCtxt<'_>, (): ()) -> Result<()> {
 
             let trait_ref = ty::Binder::dummy(ty::TraitRef::identity(tcx, tr));
 
-            // A slightly edited version of the code in `rustc_trait_selection::traits::vtable::vtable_entries`,
-            // that works without self type and just counts number of entries.
+            // A slightly edited version of the code in
+            // `rustc_trait_selection::traits::vtable::vtable_entries`, that works without self
+            // type and just counts number of entries.
             //
-            // Note that this is technically wrong, for traits which have associated types in supertraits:
+            // Note that this is technically wrong, for traits which have associated types in
+            // supertraits:
             //
             //   trait A: AsRef<Self::T> + AsRef<()> { type T; }
             //
-            // Without self type we can't normalize `Self::T`, so we can't know if `AsRef<Self::T>` and
-            // `AsRef<()>` are the same trait, thus we assume that those are different, and potentially
-            // over-estimate how many vtable entries there are.
+            // Without self type we can't normalize `Self::T`, so we can't know if `AsRef<Self::T>`
+            // and `AsRef<()>` are the same trait, thus we assume that those are different, and
+            // potentially over-estimate how many vtable entries there are.
             //
             // Similarly this is wrong for traits that have methods with possibly-impossible bounds.
             // For example:
@@ -916,10 +918,10 @@ fn analysis(tcx: TyCtxt<'_>, (): ()) -> Result<()> {
                         let own_existential_entries =
                             tcx.own_existential_vtable_entries(trait_ref.def_id());
 
-                        // The original code here ignores the method if its predicates are impossible.
-                        // We can't really do that as, for example, all not trivial bounds on generic
-                        // parameters are impossible (since we don't know the parameters...),
-                        // see the comment above.
+                        // The original code here ignores the method if its predicates are
+                        // impossible. We can't really do that as, for example, all not trivial
+                        // bounds on generic parameters are impossible (since we don't know the
+                        // parameters...), see the comment above.
                         entries_ignoring_upcasting += own_existential_entries.len();
 
                         if emit_vptr {

From 1e5b2da94b238d911796afd88245286324acbf0e Mon Sep 17 00:00:00 2001
From: David Tolnay <dtolnay@gmail.com>
Date: Thu, 26 Oct 2023 10:08:39 -0700
Subject: [PATCH 223/297] Rename Since -> StableSince in preparation for a
 DeprecatedSince

---
 compiler/rustc_attr/src/builtin.rs            | 12 +++++-----
 compiler/rustc_passes/src/stability.rs        |  8 +++----
 src/librustdoc/clean/types.rs                 |  6 ++---
 src/librustdoc/html/render/mod.rs             | 22 +++++++++----------
 .../clippy_utils/src/qualify_min_const_fn.rs  |  8 +++----
 5 files changed, 28 insertions(+), 28 deletions(-)

diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs
index ff23e3da4380..592c9ff115ba 100644
--- a/compiler/rustc_attr/src/builtin.rs
+++ b/compiler/rustc_attr/src/builtin.rs
@@ -139,7 +139,7 @@ pub enum StabilityLevel {
     /// `#[stable]`
     Stable {
         /// Rust release which stabilized this feature.
-        since: Since,
+        since: StableSince,
         /// Is this item allowed to be referred to on stable, despite being contained in unstable
         /// modules?
         allowed_through_unstable_modules: bool,
@@ -149,7 +149,7 @@ pub enum StabilityLevel {
 /// Rust release in which a feature is stabilized.
 #[derive(Encodable, Decodable, PartialEq, Copy, Clone, Debug, Eq, Hash)]
 #[derive(HashStable_Generic)]
-pub enum Since {
+pub enum StableSince {
     Version(RustcVersion),
     /// Stabilized in the upcoming version, whatever number that is.
     Current,
@@ -378,16 +378,16 @@ fn parse_stability(sess: &Session, attr: &Attribute) -> Option<(Symbol, Stabilit
 
     let since = if let Some(since) = since {
         if since.as_str() == VERSION_PLACEHOLDER {
-            Since::Current
+            StableSince::Current
         } else if let Some(version) = parse_version(since) {
-            Since::Version(version)
+            StableSince::Version(version)
         } else {
             sess.emit_err(session_diagnostics::InvalidSince { span: attr.span });
-            Since::Err
+            StableSince::Err
         }
     } else {
         sess.emit_err(session_diagnostics::MissingSince { span: attr.span });
-        Since::Err
+        StableSince::Err
     };
 
     match feature {
diff --git a/compiler/rustc_passes/src/stability.rs b/compiler/rustc_passes/src/stability.rs
index 7bfb0742b8be..a3f0f2d65124 100644
--- a/compiler/rustc_passes/src/stability.rs
+++ b/compiler/rustc_passes/src/stability.rs
@@ -3,7 +3,7 @@
 
 use crate::errors;
 use rustc_attr::{
-    self as attr, ConstStability, Since, Stability, StabilityLevel, Unstable, UnstableReason,
+    self as attr, ConstStability, Stability, StabilityLevel, StableSince, Unstable, UnstableReason,
     VERSION_PLACEHOLDER,
 };
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap};
@@ -227,10 +227,10 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
                 (&depr.as_ref().and_then(|(d, _)| d.since), &stab.level)
             {
                 match stab_since {
-                    Since::Current => {
+                    StableSince::Current => {
                         self.tcx.sess.emit_err(errors::CannotStabilizeDeprecated { span, item_sp });
                     }
-                    Since::Version(stab_since) => {
+                    StableSince::Version(stab_since) => {
                         // Explicit version of iter::order::lt to handle parse errors properly
                         for (dep_v, stab_v) in iter::zip(
                             dep_since.as_str().split('.'),
@@ -260,7 +260,7 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
                             }
                         }
                     }
-                    Since::Err => {
+                    StableSince::Err => {
                         // An error already reported. Assume the unparseable stabilization
                         // version is older than the deprecation version.
                     }
diff --git a/src/librustdoc/clean/types.rs b/src/librustdoc/clean/types.rs
index 455228d04efa..ba55871c75df 100644
--- a/src/librustdoc/clean/types.rs
+++ b/src/librustdoc/clean/types.rs
@@ -12,7 +12,7 @@ use thin_vec::ThinVec;
 
 use rustc_ast as ast;
 use rustc_ast_pretty::pprust;
-use rustc_attr::{ConstStability, Deprecation, Since, Stability, StabilityLevel};
+use rustc_attr::{ConstStability, Deprecation, Stability, StabilityLevel, StableSince};
 use rustc_const_eval::const_eval::is_unstable_const_fn;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_hir as hir;
@@ -585,14 +585,14 @@ impl Item {
         })
     }
 
-    pub(crate) fn stable_since(&self, tcx: TyCtxt<'_>) -> Option<Since> {
+    pub(crate) fn stable_since(&self, tcx: TyCtxt<'_>) -> Option<StableSince> {
         match self.stability(tcx)?.level {
             StabilityLevel::Stable { since, .. } => Some(since),
             StabilityLevel::Unstable { .. } => None,
         }
     }
 
-    pub(crate) fn const_stable_since(&self, tcx: TyCtxt<'_>) -> Option<Since> {
+    pub(crate) fn const_stable_since(&self, tcx: TyCtxt<'_>) -> Option<StableSince> {
         match self.const_stability(tcx)?.level {
             StabilityLevel::Stable { since, .. } => Some(since),
             StabilityLevel::Unstable { .. } => None,
diff --git a/src/librustdoc/html/render/mod.rs b/src/librustdoc/html/render/mod.rs
index 0b6079b29334..7ab78e73ba77 100644
--- a/src/librustdoc/html/render/mod.rs
+++ b/src/librustdoc/html/render/mod.rs
@@ -48,7 +48,7 @@ use std::str;
 use std::string::ToString;
 
 use askama::Template;
-use rustc_attr::{ConstStability, Deprecation, Since, StabilityLevel};
+use rustc_attr::{ConstStability, Deprecation, StabilityLevel, StableSince};
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_hir::def_id::{DefId, DefIdSet};
@@ -912,10 +912,10 @@ fn assoc_method(
 /// consequence of the above rules.
 fn render_stability_since_raw_with_extra(
     w: &mut Buffer,
-    ver: Option<Since>,
+    ver: Option<StableSince>,
     const_stability: Option<ConstStability>,
-    containing_ver: Option<Since>,
-    containing_const_ver: Option<Since>,
+    containing_ver: Option<StableSince>,
+    containing_const_ver: Option<StableSince>,
     extra_class: &str,
 ) -> bool {
     let stable_version = if ver != containing_ver && let Some(ver) = &ver {
@@ -977,21 +977,21 @@ fn render_stability_since_raw_with_extra(
     !stability.is_empty()
 }
 
-fn since_to_string(since: &Since) -> Option<String> {
+fn since_to_string(since: &StableSince) -> Option<String> {
     match since {
-        Since::Version(since) => Some(since.to_string()),
-        Since::Current => Some(RustcVersion::CURRENT.to_string()),
-        Since::Err => None,
+        StableSince::Version(since) => Some(since.to_string()),
+        StableSince::Current => Some(RustcVersion::CURRENT.to_string()),
+        StableSince::Err => None,
     }
 }
 
 #[inline]
 fn render_stability_since_raw(
     w: &mut Buffer,
-    ver: Option<Since>,
+    ver: Option<StableSince>,
     const_stability: Option<ConstStability>,
-    containing_ver: Option<Since>,
-    containing_const_ver: Option<Since>,
+    containing_ver: Option<StableSince>,
+    containing_const_ver: Option<StableSince>,
 ) -> bool {
     render_stability_since_raw_with_extra(
         w,
diff --git a/src/tools/clippy/clippy_utils/src/qualify_min_const_fn.rs b/src/tools/clippy/clippy_utils/src/qualify_min_const_fn.rs
index 31f7b87de635..ef0e2d3e1b3d 100644
--- a/src/tools/clippy/clippy_utils/src/qualify_min_const_fn.rs
+++ b/src/tools/clippy/clippy_utils/src/qualify_min_const_fn.rs
@@ -5,7 +5,7 @@
 
 use crate::msrvs::Msrv;
 use hir::LangItem;
-use rustc_attr::Since;
+use rustc_attr::StableSince;
 use rustc_const_eval::transform::check_consts::ConstCx;
 use rustc_hir as hir;
 use rustc_hir::def_id::DefId;
@@ -372,9 +372,9 @@ fn is_const_fn(tcx: TyCtxt<'_>, def_id: DefId, msrv: &Msrv) -> bool {
                 // as a part of an unimplemented MSRV check https://github.com/rust-lang/rust/issues/65262.
 
                 let const_stab_rust_version = match since {
-                    Since::Version(version) => version,
-                    Since::Current => rustc_session::RustcVersion::CURRENT,
-                    Since::Err => return false,
+                    StableSince::Version(version) => version,
+                    StableSince::Current => rustc_session::RustcVersion::CURRENT,
+                    StableSince::Err => return false,
                 };
 
                 msrv.meets(RustcVersion::new(

From 95b0088e7c7a781bcd77a274ec73d7fb4f9074a2 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 30 Oct 2023 15:42:03 +1100
Subject: [PATCH 224/297] Remove `check_output`.

Using `find` and `any` from `std` makes the code shorter and clearer.
---
 compiler/rustc_interface/src/passes.rs | 24 +++---------------------
 1 file changed, 3 insertions(+), 21 deletions(-)

diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs
index 9685b0777768..2099a6d59cbb 100644
--- a/compiler/rustc_interface/src/passes.rs
+++ b/compiler/rustc_interface/src/passes.rs
@@ -392,34 +392,16 @@ fn generated_output_paths(
     out_filenames
 }
 
-// Runs `f` on every output file path and returns the first non-None result, or None if `f`
-// returns None for every file path.
-fn check_output<F, T>(output_paths: &[PathBuf], f: F) -> Option<T>
-where
-    F: Fn(&PathBuf) -> Option<T>,
-{
-    for output_path in output_paths {
-        if let Some(result) = f(output_path) {
-            return Some(result);
-        }
-    }
-    None
-}
-
 fn output_contains_path(output_paths: &[PathBuf], input_path: &Path) -> bool {
     let input_path = try_canonicalize(input_path).ok();
     if input_path.is_none() {
         return false;
     }
-    let check = |output_path: &PathBuf| {
-        if try_canonicalize(output_path).ok() == input_path { Some(()) } else { None }
-    };
-    check_output(output_paths, check).is_some()
+    output_paths.iter().any(|output_path| try_canonicalize(output_path).ok() == input_path)
 }
 
-fn output_conflicts_with_dir(output_paths: &[PathBuf]) -> Option<PathBuf> {
-    let check = |output_path: &PathBuf| output_path.is_dir().then(|| output_path.clone());
-    check_output(output_paths, check)
+fn output_conflicts_with_dir(output_paths: &[PathBuf]) -> Option<&PathBuf> {
+    output_paths.iter().find(|output_path| output_path.is_dir())
 }
 
 fn escape_dep_filename(filename: &str) -> String {

From 441068b613b48c8f6cb251530ef1c56039720fd3 Mon Sep 17 00:00:00 2001
From: Ayush Singh <ayushdevel1325@gmail.com>
Date: Wed, 25 Oct 2023 22:38:39 +0530
Subject: [PATCH 225/297] Use ImageDataType for allocation type

Signed-off-by: Ayush Singh <ayushdevel1325@gmail.com>
---
 library/std/src/sys/uefi/alloc.rs | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/library/std/src/sys/uefi/alloc.rs b/library/std/src/sys/uefi/alloc.rs
index 789e3cbd81ad..ad3904d82f3f 100644
--- a/library/std/src/sys/uefi/alloc.rs
+++ b/library/std/src/sys/uefi/alloc.rs
@@ -1,13 +1,17 @@
 //! Global Allocator for UEFI.
 //! Uses [r-efi-alloc](https://crates.io/crates/r-efi-alloc)
 
-use crate::alloc::{GlobalAlloc, Layout, System};
+use r_efi::protocols::loaded_image;
 
-const MEMORY_TYPE: u32 = r_efi::efi::LOADER_DATA;
+use crate::alloc::{GlobalAlloc, Layout, System};
+use crate::sync::OnceLock;
+use crate::sys::uefi::helpers;
 
 #[stable(feature = "alloc_system_type", since = "1.28.0")]
 unsafe impl GlobalAlloc for System {
     unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
+        static EFI_MEMORY_TYPE: OnceLock<u32> = OnceLock::new();
+
         // Return null pointer if boot services are not available
         if crate::os::uefi::env::boot_services().is_none() {
             return crate::ptr::null_mut();
@@ -15,8 +19,20 @@ unsafe impl GlobalAlloc for System {
 
         // If boot services is valid then SystemTable is not null.
         let system_table = crate::os::uefi::env::system_table().as_ptr().cast();
+
+        // Each loaded image has an image handle that supports `EFI_LOADED_IMAGE_PROTOCOL`. Thus, this
+        // will never fail.
+        let mem_type = EFI_MEMORY_TYPE.get_or_init(|| {
+            let protocol = helpers::image_handle_protocol::<loaded_image::Protocol>(
+                loaded_image::PROTOCOL_GUID,
+            )
+            .unwrap();
+            // Gives allocations the memory type that the data sections were loaded as.
+            unsafe { (*protocol.as_ptr()).image_data_type }
+        });
+
         // The caller must ensure non-0 layout
-        unsafe { r_efi_alloc::raw::alloc(system_table, layout, MEMORY_TYPE) }
+        unsafe { r_efi_alloc::raw::alloc(system_table, layout, *mem_type) }
     }
 
     unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {

From 5c7cf837395db7221bd1061eee230fd27f81e6ad Mon Sep 17 00:00:00 2001
From: David Tolnay <dtolnay@gmail.com>
Date: Sun, 29 Oct 2023 22:25:13 -0700
Subject: [PATCH 226/297] Fill in syntactically valid deprecation version in
 tests

---
 .../stability-attribute-sanity.rs                |  8 ++++----
 .../stability-attribute-sanity.stderr            | 16 ++++++++--------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/tests/ui/stability-attribute/stability-attribute-sanity.rs b/tests/ui/stability-attribute/stability-attribute-sanity.rs
index 8258b6f5ae0f..7030bcd3aba4 100644
--- a/tests/ui/stability-attribute/stability-attribute-sanity.rs
+++ b/tests/ui/stability-attribute/stability-attribute-sanity.rs
@@ -41,7 +41,7 @@ mod missing_version {
     fn f2() { }
 
     #[stable(feature = "a", since = "4.4.4")]
-    #[deprecated(since = "a")] //~ ERROR missing 'note' [E0543]
+    #[deprecated(since = "5.5.5")] //~ ERROR missing 'note' [E0543]
     fn f3() { }
 }
 
@@ -58,8 +58,8 @@ fn multiple2() { }
 fn multiple3() { }
 
 #[stable(feature = "e", since = "b")] //~ ERROR 'since' must be a Rust version number, such as "1.31.0"
-#[deprecated(since = "b", note = "text")]
-#[deprecated(since = "b", note = "text")] //~ ERROR multiple `deprecated` attributes
+#[deprecated(since = "5.5.5", note = "text")]
+#[deprecated(since = "5.5.5", note = "text")] //~ ERROR multiple `deprecated` attributes
 #[rustc_const_unstable(feature = "c", issue = "none")]
 #[rustc_const_unstable(feature = "d", issue = "none")] //~ ERROR multiple stability levels
 pub const fn multiple4() { }
@@ -69,7 +69,7 @@ pub const fn multiple4() { }
 #[deprecated(since = "invalid", note = "text")]
 fn invalid_deprecation_version() {}
 
-#[deprecated(since = "a", note = "text")]
+#[deprecated(since = "5.5.5", note = "text")]
 fn deprecated_without_unstable_or_stable() { }
 //~^^ ERROR deprecated attribute must be paired with either stable or unstable attribute
 
diff --git a/tests/ui/stability-attribute/stability-attribute-sanity.stderr b/tests/ui/stability-attribute/stability-attribute-sanity.stderr
index 955230742bd1..27cd451f9898 100644
--- a/tests/ui/stability-attribute/stability-attribute-sanity.stderr
+++ b/tests/ui/stability-attribute/stability-attribute-sanity.stderr
@@ -1,14 +1,14 @@
 error: multiple `deprecated` attributes
   --> $DIR/stability-attribute-sanity.rs:62:1
    |
-LL | #[deprecated(since = "b", note = "text")]
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: remove this attribute
+LL | #[deprecated(since = "5.5.5", note = "text")]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ help: remove this attribute
    |
 note: attribute also specified here
   --> $DIR/stability-attribute-sanity.rs:61:1
    |
-LL | #[deprecated(since = "b", note = "text")]
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+LL | #[deprecated(since = "5.5.5", note = "text")]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error[E0541]: unknown meta item 'reason'
   --> $DIR/stability-attribute-sanity.rs:8:46
@@ -73,8 +73,8 @@ LL |     #[deprecated(note = "a")]
 error[E0543]: missing 'note'
   --> $DIR/stability-attribute-sanity.rs:44:5
    |
-LL |     #[deprecated(since = "a")]
-   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^
+LL |     #[deprecated(since = "5.5.5")]
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error[E0544]: multiple stability levels
   --> $DIR/stability-attribute-sanity.rs:49:1
@@ -118,8 +118,8 @@ LL | fn invalid_deprecation_version() {}
 error[E0549]: deprecated attribute must be paired with either stable or unstable attribute
   --> $DIR/stability-attribute-sanity.rs:72:1
    |
-LL | #[deprecated(since = "a", note = "text")]
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+LL | #[deprecated(since = "5.5.5", note = "text")]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error[E0711]: feature `a` is declared stable since 1.0.0, but was previously declared stable since 4.4.4
   --> $DIR/stability-attribute-sanity.rs:67:1

From 2fe7d17bd9aabe3948b8693c2a3d4841ce9fbd14 Mon Sep 17 00:00:00 2001
From: David Tolnay <dtolnay@gmail.com>
Date: Thu, 26 Oct 2023 10:14:29 -0700
Subject: [PATCH 227/297] Store version of `deprecated` attribute in structured
 form

---
 compiler/rustc_attr/src/builtin.rs            | 61 ++++++++++++++-----
 compiler/rustc_middle/src/middle/stability.rs | 46 ++++----------
 compiler/rustc_passes/src/stability.rs        | 49 +++++----------
 src/librustdoc/html/render/mod.rs             | 13 ++--
 src/librustdoc/json/conversions.rs            |  3 +-
 .../stability-attribute-sanity.rs             |  5 +-
 .../stability-attribute-sanity.stderr         | 13 ++--
 7 files changed, 85 insertions(+), 105 deletions(-)

diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs
index 592c9ff115ba..62709ef4db95 100644
--- a/compiler/rustc_attr/src/builtin.rs
+++ b/compiler/rustc_attr/src/builtin.rs
@@ -13,6 +13,7 @@ use rustc_session::parse::{feature_err, ParseSess};
 use rustc_session::{RustcVersion, Session};
 use rustc_span::hygiene::Transparency;
 use rustc_span::{symbol::sym, symbol::Symbol, Span};
+use std::fmt::{self, Display};
 use std::num::NonZeroU32;
 
 use crate::session_diagnostics::{self, IncorrectReprFormatGenericCause};
@@ -720,17 +721,37 @@ pub fn eval_condition(
 
 #[derive(Copy, Debug, Encodable, Decodable, Clone, HashStable_Generic)]
 pub struct Deprecation {
-    pub since: Option<Symbol>,
+    pub since: Option<DeprecatedSince>,
     /// The note to issue a reason.
     pub note: Option<Symbol>,
     /// A text snippet used to completely replace any use of the deprecated item in an expression.
     ///
     /// This is currently unstable.
     pub suggestion: Option<Symbol>,
+}
 
-    /// Whether to treat the since attribute as being a Rust version identifier
-    /// (rather than an opaque string).
-    pub is_since_rustc_version: bool,
+/// Release in which an API is deprecated.
+#[derive(Copy, Debug, Encodable, Decodable, Clone, HashStable_Generic)]
+pub enum DeprecatedSince {
+    RustcVersion(RustcVersion),
+    /// Deprecated in the future ("to be determined").
+    Future,
+    /// `feature(staged_api)` is off, or it's on but the deprecation version
+    /// cannot be parsed as a RustcVersion. In the latter case, an error has
+    /// already been emitted. In the former case, deprecation versions outside
+    /// the standard library are allowed to be arbitrary strings, for better or
+    /// worse.
+    Symbol(Symbol),
+}
+
+impl Display for DeprecatedSince {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            DeprecatedSince::RustcVersion(since) => Display::fmt(since, formatter),
+            DeprecatedSince::Future => formatter.write_str("TBD"),
+            DeprecatedSince::Symbol(since) => Display::fmt(since, formatter),
+        }
+    }
 }
 
 /// Finds the deprecation attribute. `None` if none exists.
@@ -839,22 +860,30 @@ pub fn find_deprecation(
             }
         }
 
-        if is_rustc {
-            if since.is_none() {
-                sess.emit_err(session_diagnostics::MissingSince { span: attr.span });
-                continue;
+        let since = if let Some(since) = since {
+            if since.as_str() == "TBD" {
+                Some(DeprecatedSince::Future)
+            } else if !is_rustc {
+                Some(DeprecatedSince::Symbol(since))
+            } else if let Some(version) = parse_version(since) {
+                Some(DeprecatedSince::RustcVersion(version))
+            } else {
+                sess.emit_err(session_diagnostics::InvalidSince { span: attr.span });
+                Some(DeprecatedSince::Symbol(since))
             }
+        } else if is_rustc {
+            sess.emit_err(session_diagnostics::MissingSince { span: attr.span });
+            continue;
+        } else {
+            None
+        };
 
-            if note.is_none() {
-                sess.emit_err(session_diagnostics::MissingNote { span: attr.span });
-                continue;
-            }
+        if is_rustc && note.is_none() {
+            sess.emit_err(session_diagnostics::MissingNote { span: attr.span });
+            continue;
         }
 
-        depr = Some((
-            Deprecation { since, note, suggestion, is_since_rustc_version: is_rustc },
-            attr.span,
-        ));
+        depr = Some((Deprecation { since, note, suggestion }, attr.span));
     }
 
     depr
diff --git a/compiler/rustc_middle/src/middle/stability.rs b/compiler/rustc_middle/src/middle/stability.rs
index 20547696d5a7..b750352e13a6 100644
--- a/compiler/rustc_middle/src/middle/stability.rs
+++ b/compiler/rustc_middle/src/middle/stability.rs
@@ -5,7 +5,9 @@ pub use self::StabilityLevel::*;
 
 use crate::ty::{self, TyCtxt};
 use rustc_ast::NodeId;
-use rustc_attr::{self as attr, ConstStability, DefaultBodyStability, Deprecation, Stability};
+use rustc_attr::{
+    self as attr, ConstStability, DefaultBodyStability, DeprecatedSince, Deprecation, Stability,
+};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{Applicability, Diagnostic};
 use rustc_feature::GateIssue;
@@ -126,36 +128,14 @@ pub fn report_unstable(
 /// Checks whether an item marked with `deprecated(since="X")` is currently
 /// deprecated (i.e., whether X is not greater than the current rustc version).
 pub fn deprecation_in_effect(depr: &Deprecation) -> bool {
-    let is_since_rustc_version = depr.is_since_rustc_version;
-    let since = depr.since.as_ref().map(Symbol::as_str);
-
-    if !is_since_rustc_version {
+    match depr.since {
+        Some(DeprecatedSince::RustcVersion(since)) => since <= RustcVersion::CURRENT,
+        Some(DeprecatedSince::Future) => false,
         // The `since` field doesn't have semantic purpose without `#![staged_api]`.
-        return true;
+        Some(DeprecatedSince::Symbol(_)) => true,
+        // Assume deprecation is in effect if "since" field is missing.
+        None => true,
     }
-
-    if let Some(since) = since {
-        if since == "TBD" {
-            return false;
-        }
-
-        // We ignore non-integer components of the version (e.g., "nightly").
-        let since: Vec<u16> =
-            since.split(|c| c == '.' || c == '-').flat_map(|s| s.parse()).collect();
-
-        // We simply treat invalid `since` attributes as relating to a previous
-        // Rust version, thus always displaying the warning.
-        if since.len() != 3 {
-            return true;
-        }
-
-        let rustc = RustcVersion::CURRENT;
-        return since.as_slice() <= &[rustc.major, rustc.minor, rustc.patch];
-    };
-
-    // Assume deprecation is in effect if "since" field is missing
-    // or if we can't determine the current Rust version.
-    true
 }
 
 pub fn deprecation_suggestion(
@@ -180,7 +160,7 @@ fn deprecation_lint(is_in_effect: bool) -> &'static Lint {
 
 fn deprecation_message(
     is_in_effect: bool,
-    since: Option<Symbol>,
+    since: Option<DeprecatedSince>,
     note: Option<Symbol>,
     kind: &str,
     path: &str,
@@ -188,9 +168,7 @@ fn deprecation_message(
     let message = if is_in_effect {
         format!("use of deprecated {kind} `{path}`")
     } else {
-        let since = since.as_ref().map(Symbol::as_str);
-
-        if since == Some("TBD") {
+        if let Some(DeprecatedSince::Future) = since {
             format!("use of {kind} `{path}` that will be deprecated in a future Rust version")
         } else {
             format!(
@@ -381,7 +359,7 @@ impl<'tcx> TyCtxt<'tcx> {
                 // With #![staged_api], we want to emit down the whole
                 // hierarchy.
                 let depr_attr = &depr_entry.attr;
-                if !skip || depr_attr.is_since_rustc_version {
+                if !skip || matches!(depr_attr.since, Some(DeprecatedSince::RustcVersion(_))) {
                     // Calculating message for lint involves calling `self.def_path_str`.
                     // Which by default to calculate visible path will invoke expensive `visible_parent_map` query.
                     // So we skip message calculation altogether, if lint is allowed.
diff --git a/compiler/rustc_passes/src/stability.rs b/compiler/rustc_passes/src/stability.rs
index a3f0f2d65124..675fe6c5d42b 100644
--- a/compiler/rustc_passes/src/stability.rs
+++ b/compiler/rustc_passes/src/stability.rs
@@ -3,8 +3,8 @@
 
 use crate::errors;
 use rustc_attr::{
-    self as attr, ConstStability, Stability, StabilityLevel, StableSince, Unstable, UnstableReason,
-    VERSION_PLACEHOLDER,
+    self as attr, ConstStability, DeprecatedSince, Stability, StabilityLevel, StableSince,
+    Unstable, UnstableReason, VERSION_PLACEHOLDER,
 };
 use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap};
 use rustc_hir as hir;
@@ -24,8 +24,6 @@ use rustc_span::symbol::{sym, Symbol};
 use rustc_span::Span;
 use rustc_target::spec::abi::Abi;
 
-use std::cmp::Ordering;
-use std::iter;
 use std::mem::replace;
 use std::num::NonZeroU32;
 
@@ -198,7 +196,11 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
             }
         }
 
-        if let Some((rustc_attr::Deprecation { is_since_rustc_version: true, .. }, span)) = &depr {
+        if let Some((
+            rustc_attr::Deprecation { since: Some(DeprecatedSince::RustcVersion(_)), .. },
+            span,
+        )) = &depr
+        {
             if stab.is_none() {
                 self.tcx.sess.emit_err(errors::DeprecatedAttribute { span: *span });
             }
@@ -223,41 +225,20 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
 
             // Check if deprecated_since < stable_since. If it is,
             // this is *almost surely* an accident.
-            if let (&Some(dep_since), &attr::Stable { since: stab_since, .. }) =
-                (&depr.as_ref().and_then(|(d, _)| d.since), &stab.level)
+            if let (
+                &Some(DeprecatedSince::RustcVersion(dep_since)),
+                &attr::Stable { since: stab_since, .. },
+            ) = (&depr.as_ref().and_then(|(d, _)| d.since), &stab.level)
             {
                 match stab_since {
                     StableSince::Current => {
                         self.tcx.sess.emit_err(errors::CannotStabilizeDeprecated { span, item_sp });
                     }
                     StableSince::Version(stab_since) => {
-                        // Explicit version of iter::order::lt to handle parse errors properly
-                        for (dep_v, stab_v) in iter::zip(
-                            dep_since.as_str().split('.'),
-                            [stab_since.major, stab_since.minor, stab_since.patch],
-                        ) {
-                            match dep_v.parse::<u64>() {
-                                Ok(dep_vp) => match dep_vp.cmp(&u64::from(stab_v)) {
-                                    Ordering::Less => {
-                                        self.tcx.sess.emit_err(errors::CannotStabilizeDeprecated {
-                                            span,
-                                            item_sp,
-                                        });
-                                        break;
-                                    }
-                                    Ordering::Equal => continue,
-                                    Ordering::Greater => break,
-                                },
-                                Err(_) => {
-                                    if dep_v != "TBD" {
-                                        self.tcx.sess.emit_err(errors::InvalidDeprecationVersion {
-                                            span,
-                                            item_sp,
-                                        });
-                                    }
-                                    break;
-                                }
-                            }
+                        if dep_since < stab_since {
+                            self.tcx
+                                .sess
+                                .emit_err(errors::CannotStabilizeDeprecated { span, item_sp });
                         }
                     }
                     StableSince::Err => {
diff --git a/src/librustdoc/html/render/mod.rs b/src/librustdoc/html/render/mod.rs
index 7ab78e73ba77..e852d02b6acb 100644
--- a/src/librustdoc/html/render/mod.rs
+++ b/src/librustdoc/html/render/mod.rs
@@ -48,7 +48,7 @@ use std::str;
 use std::string::ToString;
 
 use askama::Template;
-use rustc_attr::{ConstStability, Deprecation, StabilityLevel, StableSince};
+use rustc_attr::{ConstStability, DeprecatedSince, Deprecation, StabilityLevel, StableSince};
 use rustc_data_structures::captures::Captures;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_hir::def_id::{DefId, DefIdSet};
@@ -617,21 +617,18 @@ fn short_item_info(
 ) -> Vec<ShortItemInfo> {
     let mut extra_info = vec![];
 
-    if let Some(depr @ Deprecation { note, since, is_since_rustc_version: _, suggestion: _ }) =
-        item.deprecation(cx.tcx())
-    {
+    if let Some(depr @ Deprecation { note, since, suggestion: _ }) = item.deprecation(cx.tcx()) {
         // We display deprecation messages for #[deprecated], but only display
         // the future-deprecation messages for rustc versions.
         let mut message = if let Some(since) = since {
-            let since = since.as_str();
             if !stability::deprecation_in_effect(&depr) {
-                if since == "TBD" {
+                if let DeprecatedSince::Future = since {
                     String::from("Deprecating in a future Rust version")
                 } else {
-                    format!("Deprecating in {}", Escape(since))
+                    format!("Deprecating in {}", Escape(&since.to_string()))
                 }
             } else {
-                format!("Deprecated since {}", Escape(since))
+                format!("Deprecated since {}", Escape(&since.to_string()))
             }
         } else {
             String::from("Deprecated")
diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs
index 563e0cffdddb..ff2e6b1ebc9a 100644
--- a/src/librustdoc/json/conversions.rs
+++ b/src/librustdoc/json/conversions.rs
@@ -138,8 +138,7 @@ where
 }
 
 pub(crate) fn from_deprecation(deprecation: rustc_attr::Deprecation) -> Deprecation {
-    #[rustfmt::skip]
-    let rustc_attr::Deprecation { since, note, is_since_rustc_version: _, suggestion: _ } = deprecation;
+    let rustc_attr::Deprecation { since, note, suggestion: _ } = deprecation;
     Deprecation { since: since.map(|s| s.to_string()), note: note.map(|s| s.to_string()) }
 }
 
diff --git a/tests/ui/stability-attribute/stability-attribute-sanity.rs b/tests/ui/stability-attribute/stability-attribute-sanity.rs
index 7030bcd3aba4..7857a0603bd4 100644
--- a/tests/ui/stability-attribute/stability-attribute-sanity.rs
+++ b/tests/ui/stability-attribute/stability-attribute-sanity.rs
@@ -64,9 +64,8 @@ fn multiple3() { }
 #[rustc_const_unstable(feature = "d", issue = "none")] //~ ERROR multiple stability levels
 pub const fn multiple4() { }
 
-#[stable(feature = "a", since = "1.0.0")] //~ ERROR invalid deprecation version found
-//~^ ERROR feature `a` is declared stable since 1.0.0
-#[deprecated(since = "invalid", note = "text")]
+#[stable(feature = "a", since = "1.0.0")] //~ ERROR feature `a` is declared stable since 1.0.0
+#[deprecated(since = "invalid", note = "text")] //~ ERROR 'since' must be a Rust version number, such as "1.31.0"
 fn invalid_deprecation_version() {}
 
 #[deprecated(since = "5.5.5", note = "text")]
diff --git a/tests/ui/stability-attribute/stability-attribute-sanity.stderr b/tests/ui/stability-attribute/stability-attribute-sanity.stderr
index 27cd451f9898..c614fc2b9f7f 100644
--- a/tests/ui/stability-attribute/stability-attribute-sanity.stderr
+++ b/tests/ui/stability-attribute/stability-attribute-sanity.stderr
@@ -106,17 +106,14 @@ error[E0544]: multiple stability levels
 LL | #[rustc_const_unstable(feature = "d", issue = "none")]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-error: invalid deprecation version found
-  --> $DIR/stability-attribute-sanity.rs:67:1
+error: 'since' must be a Rust version number, such as "1.31.0"
+  --> $DIR/stability-attribute-sanity.rs:68:1
    |
-LL | #[stable(feature = "a", since = "1.0.0")]
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ invalid deprecation version
-...
-LL | fn invalid_deprecation_version() {}
-   | ----------------------------------- the stability attribute annotates this item
+LL | #[deprecated(since = "invalid", note = "text")]
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error[E0549]: deprecated attribute must be paired with either stable or unstable attribute
-  --> $DIR/stability-attribute-sanity.rs:72:1
+  --> $DIR/stability-attribute-sanity.rs:71:1
    |
 LL | #[deprecated(since = "5.5.5", note = "text")]
    | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

From 8afb40b3a8083ebb73204cac12a057fea531dacc Mon Sep 17 00:00:00 2001
From: David Tolnay <dtolnay@gmail.com>
Date: Sun, 29 Oct 2023 22:50:49 -0700
Subject: [PATCH 228/297] Delete unused InvalidDeprecationVersion diagnostic

---
 compiler/rustc_passes/messages.ftl  |  5 -----
 compiler/rustc_passes/src/errors.rs | 10 ----------
 2 files changed, 15 deletions(-)

diff --git a/compiler/rustc_passes/messages.ftl b/compiler/rustc_passes/messages.ftl
index 026186cbe6cd..b6661fe721f2 100644
--- a/compiler/rustc_passes/messages.ftl
+++ b/compiler/rustc_passes/messages.ftl
@@ -396,11 +396,6 @@ passes_invalid_attr_at_crate_level =
 passes_invalid_attr_at_crate_level_item =
     the inner attribute doesn't annotate this {$kind}
 
-passes_invalid_deprecation_version =
-    invalid deprecation version found
-    .label = invalid deprecation version
-    .item = the stability attribute annotates this item
-
 passes_invalid_macro_export_arguments = `{$name}` isn't a valid `#[macro_export]` argument
 
 passes_invalid_macro_export_arguments_too_many_items = `#[macro_export]` can only take 1 or 0 arguments
diff --git a/compiler/rustc_passes/src/errors.rs b/compiler/rustc_passes/src/errors.rs
index eca0fb7748b1..e2a61c594109 100644
--- a/compiler/rustc_passes/src/errors.rs
+++ b/compiler/rustc_passes/src/errors.rs
@@ -1524,16 +1524,6 @@ pub struct CannotStabilizeDeprecated {
     pub item_sp: Span,
 }
 
-#[derive(Diagnostic)]
-#[diag(passes_invalid_deprecation_version)]
-pub struct InvalidDeprecationVersion {
-    #[primary_span]
-    #[label]
-    pub span: Span,
-    #[label(passes_item)]
-    pub item_sp: Span,
-}
-
 #[derive(Diagnostic)]
 #[diag(passes_missing_stability_attr)]
 pub struct MissingStabilityAttr<'a> {

From 0c8bdd0bf33c2313ad9d0be495ffde8f4532237a Mon Sep 17 00:00:00 2001
From: Gurinder Singh <frederick.the.fool@gmail.com>
Date: Mon, 30 Oct 2023 13:03:40 +0530
Subject: [PATCH 229/297] Fail typeck for illegal break-with-value

This is fixes the issue wherein typeck was succeeding for break-with-value
at illegal locations such as inside `while`, `while let` and `for` loops which
eventually caused an ICE during MIR interpetation for const eval.

Now we fail typeck for such code which prevents faulty MIR from being generated
and interpreted, thus fixing the ICE.
---
 compiler/rustc_hir_typeck/src/expr.rs         | 13 ++++++---
 .../issue-114529-illegal-break-with-value.rs  | 20 +++++++++++++
 ...sue-114529-illegal-break-with-value.stderr | 29 +++++++++++++++++++
 3 files changed, 58 insertions(+), 4 deletions(-)
 create mode 100644 tests/ui/typeck/issue-114529-illegal-break-with-value.rs
 create mode 100644 tests/ui/typeck/issue-114529-illegal-break-with-value.stderr

diff --git a/compiler/rustc_hir_typeck/src/expr.rs b/compiler/rustc_hir_typeck/src/expr.rs
index be225ceb8432..b1d6e4f0523b 100644
--- a/compiler/rustc_hir_typeck/src/expr.rs
+++ b/compiler/rustc_hir_typeck/src/expr.rs
@@ -625,10 +625,15 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     }
                 };
 
-                // If the loop context is not a `loop { }`, then break with
-                // a value is illegal, and `opt_coerce_to` will be `None`.
-                // Just set expectation to error in that case.
-                let coerce_to = opt_coerce_to.unwrap_or_else(|| Ty::new_misc_error(tcx));
+                let coerce_to = match opt_coerce_to {
+                    Some(c) => c,
+                    None => {
+                        // If the loop context is not a `loop { }`, then break with
+                        // a value is illegal, and `opt_coerce_to` will be `None`.
+                        // Return error in that case (#114529).
+                        return Ty::new_misc_error(tcx);
+                    }
+                };
 
                 // Recurse without `enclosing_breakables` borrowed.
                 e_ty = self.check_expr_with_hint(e, coerce_to);
diff --git a/tests/ui/typeck/issue-114529-illegal-break-with-value.rs b/tests/ui/typeck/issue-114529-illegal-break-with-value.rs
new file mode 100644
index 000000000000..613d1b6343a3
--- /dev/null
+++ b/tests/ui/typeck/issue-114529-illegal-break-with-value.rs
@@ -0,0 +1,20 @@
+// Regression test for issue #114529
+// Tests that we do not ICE during const eval for a
+// break-with-value in contexts where it is illegal
+
+#[allow(while_true)]
+fn main() {
+    [(); {
+        while true {
+            break 9; //~ ERROR `break` with value from a `while` loop
+        };
+        51
+    }];
+
+    [(); {
+        while let Some(v) = Some(9) {
+            break v; //~ ERROR `break` with value from a `while` loop
+        };
+        51
+    }];
+}
diff --git a/tests/ui/typeck/issue-114529-illegal-break-with-value.stderr b/tests/ui/typeck/issue-114529-illegal-break-with-value.stderr
new file mode 100644
index 000000000000..4d6c27bbbd03
--- /dev/null
+++ b/tests/ui/typeck/issue-114529-illegal-break-with-value.stderr
@@ -0,0 +1,29 @@
+error[E0571]: `break` with value from a `while` loop
+  --> $DIR/issue-114529-illegal-break-with-value.rs:9:13
+   |
+LL |         while true {
+   |         ---------- you can't `break` with a value in a `while` loop
+LL |             break 9;
+   |             ^^^^^^^ can only break with a value inside `loop` or breakable block
+   |
+help: use `break` on its own without a value inside this `while` loop
+   |
+LL |             break;
+   |             ~~~~~
+
+error[E0571]: `break` with value from a `while` loop
+  --> $DIR/issue-114529-illegal-break-with-value.rs:16:13
+   |
+LL |         while let Some(v) = Some(9) {
+   |         --------------------------- you can't `break` with a value in a `while` loop
+LL |             break v;
+   |             ^^^^^^^ can only break with a value inside `loop` or breakable block
+   |
+help: use `break` on its own without a value inside this `while` loop
+   |
+LL |             break;
+   |             ~~~~~
+
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0571`.

From 2409ea3c12e1712dd38a9dd4b7498c6fa23fb67f Mon Sep 17 00:00:00 2001
From: chenx97 <chenx97@aosc.io>
Date: Thu, 19 Oct 2023 18:03:26 +0800
Subject: [PATCH 230/297] bootstrap: bump fd-lock, clap and windows.

this also updates target-sensitive dependencies like rustix
and libc.
---
 src/bootstrap/Cargo.lock       | 129 +++++++++++++++------------------
 src/bootstrap/Cargo.toml       |   6 +-
 src/bootstrap/src/bin/rustc.rs |   5 +-
 src/bootstrap/src/utils/job.rs |  14 ++--
 4 files changed, 68 insertions(+), 86 deletions(-)

diff --git a/src/bootstrap/Cargo.lock b/src/bootstrap/Cargo.lock
index 19f666d04639..346bfb30fecf 100644
--- a/src/bootstrap/Cargo.lock
+++ b/src/bootstrap/Cargo.lock
@@ -29,6 +29,12 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "bitflags"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07"
+
 [[package]]
 name = "block-buffer"
 version = "0.10.2"
@@ -105,23 +111,21 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "clap"
-version = "4.2.4"
+version = "4.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "956ac1f6381d8d82ab4684768f89c0ea3afe66925ceadb4eeb3fc452ffc55d62"
+checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b"
 dependencies = [
  "clap_builder",
  "clap_derive",
- "once_cell",
 ]
 
 [[package]]
 name = "clap_builder"
-version = "4.2.4"
+version = "4.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84080e799e54cff944f4b4a4b0e71630b0e0443b25b985175c7dddc1a859b749"
+checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663"
 dependencies = [
  "anstyle",
- "bitflags",
  "clap_lex",
 ]
 
@@ -136,9 +140,9 @@ dependencies = [
 
 [[package]]
 name = "clap_derive"
-version = "4.2.0"
+version = "4.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4"
+checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442"
 dependencies = [
  "heck",
  "proc-macro2",
@@ -148,9 +152,9 @@ dependencies = [
 
 [[package]]
 name = "clap_lex"
-version = "0.4.1"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1"
+checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1"
 
 [[package]]
 name = "cmake"
@@ -253,30 +257,19 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
 
 [[package]]
 name = "errno"
-version = "0.3.0"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0"
+checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860"
 dependencies = [
- "errno-dragonfly",
  "libc",
  "windows-sys",
 ]
 
-[[package]]
-name = "errno-dragonfly"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
-dependencies = [
- "cc",
- "libc",
-]
-
 [[package]]
 name = "fd-lock"
-version = "3.0.11"
+version = "3.0.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9799aefb4a2e4a01cc47610b1dd47c18ab13d991f27bbcaed9296f5a53d5cbad"
+checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5"
 dependencies = [
  "cfg-if",
  "rustix",
@@ -339,12 +332,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "hermit-abi"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
-
 [[package]]
 name = "hex"
 version = "0.4.3"
@@ -378,17 +365,6 @@ dependencies = [
  "winapi-util",
 ]
 
-[[package]]
-name = "io-lifetimes"
-version = "1.0.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb"
-dependencies = [
- "hermit-abi 0.3.2",
- "libc",
- "windows-sys",
-]
-
 [[package]]
 name = "itoa"
 version = "1.0.2"
@@ -413,15 +389,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.140"
+version = "0.2.149"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c"
+checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b"
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.3.2"
+version = "0.4.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f508063cc7bb32987c71511216bd5a32be15bccb6a80b52df8b9d7f01fc3aa2"
+checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f"
 
 [[package]]
 name = "log"
@@ -473,7 +449,7 @@ version = "1.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
 dependencies = [
- "hermit-abi 0.1.19",
+ "hermit-abi",
  "libc",
 ]
 
@@ -565,7 +541,7 @@ version = "0.2.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42"
 dependencies = [
- "bitflags",
+ "bitflags 1.3.2",
 ]
 
 [[package]]
@@ -593,13 +569,12 @@ checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64"
 
 [[package]]
 name = "rustix"
-version = "0.37.6"
+version = "0.38.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d097081ed288dfe45699b72f5b5d648e5f15d64d900c7080273baa20c16a6849"
+checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed"
 dependencies = [
- "bitflags",
+ "bitflags 2.4.1",
  "errno",
- "io-lifetimes",
  "libc",
  "linux-raw-sys",
  "windows-sys",
@@ -797,27 +772,37 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
 [[package]]
 name = "windows"
-version = "0.46.0"
+version = "0.51.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25"
+checksum = "ca229916c5ee38c2f2bc1e9d8f04df975b4bd93f9955dc69fabb5d91270045c9"
+dependencies = [
+ "windows-core",
+ "windows-targets",
+]
+
+[[package]]
+name = "windows-core"
+version = "0.51.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64"
 dependencies = [
  "windows-targets",
 ]
 
 [[package]]
 name = "windows-sys"
-version = "0.45.0"
+version = "0.48.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
 dependencies = [
  "windows-targets",
 ]
 
 [[package]]
 name = "windows-targets"
-version = "0.42.2"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
+checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
 dependencies = [
  "windows_aarch64_gnullvm",
  "windows_aarch64_msvc",
@@ -830,45 +815,45 @@ dependencies = [
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.42.2"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
+checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.42.2"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
+checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.42.2"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
+checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.42.2"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
+checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.42.2"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
+checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.42.2"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
+checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.42.2"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
+checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
 
 [[package]]
 name = "xattr"
diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml
index bc025f444842..e4d359141cec 100644
--- a/src/bootstrap/Cargo.toml
+++ b/src/bootstrap/Cargo.toml
@@ -35,7 +35,7 @@ test = false
 [dependencies]
 build_helper = { path = "../tools/build_helper" }
 cc = "1.0.69"
-clap = { version = "4.2.4", default-features = false, features = ["std", "usage", "help", "derive", "error-context"] }
+clap = { version = "4.4.7", default-features = false, features = ["std", "usage", "help", "derive", "error-context"] }
 clap_complete = "4.4.3"
 cmake = "0.1.38"
 filetime = "0.2"
@@ -64,13 +64,13 @@ sysinfo = { version = "0.26.0", optional = true }
 
 # Solaris doesn't support flock() and thus fd-lock is not option now
 [target.'cfg(not(target_os = "solaris"))'.dependencies]
-fd-lock = "3.0.8"
+fd-lock = "3.0.13"
 
 [target.'cfg(windows)'.dependencies.junction]
 version = "1.0.0"
 
 [target.'cfg(windows)'.dependencies.windows]
-version = "0.46.0"
+version = "0.51.1"
 features = [
     "Win32_Foundation",
     "Win32_Security",
diff --git a/src/bootstrap/src/bin/rustc.rs b/src/bootstrap/src/bin/rustc.rs
index 04bf9723edd4..3041fd37ac80 100644
--- a/src/bootstrap/src/bin/rustc.rs
+++ b/src/bootstrap/src/bin/rustc.rs
@@ -300,10 +300,9 @@ fn format_rusage_data(child: Child) -> Option<String> {
             &mut user_filetime,
         )
     }
-    .ok()
     .ok()?;
-    unsafe { FileTimeToSystemTime(&user_filetime, &mut user_time) }.ok().ok()?;
-    unsafe { FileTimeToSystemTime(&kernel_filetime, &mut kernel_time) }.ok().ok()?;
+    unsafe { FileTimeToSystemTime(&user_filetime, &mut user_time) }.ok()?;
+    unsafe { FileTimeToSystemTime(&kernel_filetime, &mut kernel_time) }.ok()?;
 
     // Unlike on Linux with RUSAGE_CHILDREN, this will only return memory information for the process
     // with the given handle and none of that process's children.
diff --git a/src/bootstrap/src/utils/job.rs b/src/bootstrap/src/utils/job.rs
index 37235134a28c..c5c718a892f7 100644
--- a/src/bootstrap/src/utils/job.rs
+++ b/src/bootstrap/src/utils/job.rs
@@ -89,8 +89,7 @@ mod for_windows {
             JobObjectExtendedLimitInformation,
             &info as *const _ as *const c_void,
             mem::size_of_val(&info) as u32,
-        )
-        .ok();
+        );
         assert!(r.is_ok(), "{}", io::Error::last_os_error());
 
         // Assign our process to this job object. Note that if this fails, one very
@@ -102,9 +101,9 @@ mod for_windows {
         // Also note that nested jobs (why this might fail) are supported in recent
         // versions of Windows, but the version of Windows that our bots are running
         // at least don't support nested job objects.
-        let r = AssignProcessToJobObject(job, GetCurrentProcess()).ok();
+        let r = AssignProcessToJobObject(job, GetCurrentProcess());
         if r.is_err() {
-            CloseHandle(job);
+            CloseHandle(job).ok();
             return;
         }
 
@@ -131,7 +130,7 @@ mod for_windows {
                 // it might be better to improve the experience of the second case
                 // when users have interrupted the parent process and we haven't finish
                 // duplicating the handle yet. We just need close the job object if that occurs.
-                CloseHandle(job);
+                CloseHandle(job).ok();
                 return;
             }
         };
@@ -145,8 +144,7 @@ mod for_windows {
             0,
             false,
             DUPLICATE_SAME_ACCESS,
-        )
-        .ok();
+        );
 
         // If this failed, well at least we tried! An example of DuplicateHandle
         // failing in the past has been when the wrong python2 package spawned this
@@ -155,7 +153,7 @@ mod for_windows {
         // mode" here is that we only clean everything up when the build system
         // dies, not when the python parent does, so not too bad.
         if r.is_err() {
-            CloseHandle(job);
+            CloseHandle(job).ok();
         }
     }
 }

From 4b14048d6037a2da5601952a2dd36fe67d04218b Mon Sep 17 00:00:00 2001
From: onur-ozkan <work@onurozkan.dev>
Date: Mon, 30 Oct 2023 11:22:32 +0300
Subject: [PATCH 231/297] improve and fix x install

Fix: Write access check of `prefix` and `sysconfdir`
when DESTDIR is present.

Improvement: Instead of repeatedly reading `DESTDIR` within
each `fn prepare_dir` usage, read it once and pass it to
the `fn prepare_dir`.

Signed-off-by: onur-ozkan <work@onurozkan.dev>
---
 src/bootstrap/src/core/build_steps/install.rs | 52 +++++++++++--------
 1 file changed, 30 insertions(+), 22 deletions(-)

diff --git a/src/bootstrap/src/core/build_steps/install.rs b/src/bootstrap/src/core/build_steps/install.rs
index 391995b7c3b8..ed6b1c5154c8 100644
--- a/src/bootstrap/src/core/build_steps/install.rs
+++ b/src/bootstrap/src/core/build_steps/install.rs
@@ -71,16 +71,27 @@ fn install_sh(
 
     let prefix = default_path(&builder.config.prefix, "/usr/local");
     let sysconfdir = prefix.join(default_path(&builder.config.sysconfdir, "/etc"));
+    let destdir_env = env::var_os("DESTDIR").map(PathBuf::from);
 
-    // Sanity check for the user write access on prefix and sysconfdir
-    assert!(
-        is_dir_writable_for_user(&prefix),
-        "User doesn't have write access on `install.prefix` path in the `config.toml`.",
-    );
-    assert!(
-        is_dir_writable_for_user(&sysconfdir),
-        "User doesn't have write access on `install.sysconfdir` path in `config.toml`."
-    );
+    // Sanity checks on the write access of user.
+    //
+    // When the `DESTDIR` environment variable is present, there is no point to
+    // check write access for `prefix` and `sysconfdir` individually, as they
+    // are combined with the path from the `DESTDIR` environment variable. In
+    // this case, we only need to check the `DESTDIR` path, disregarding the
+    // `prefix` and `sysconfdir` paths.
+    if let Some(destdir) = &destdir_env {
+        assert!(is_dir_writable_for_user(destdir), "User doesn't have write access on DESTDIR.");
+    } else {
+        assert!(
+            is_dir_writable_for_user(&prefix),
+            "User doesn't have write access on `install.prefix` path in the `config.toml`.",
+        );
+        assert!(
+            is_dir_writable_for_user(&sysconfdir),
+            "User doesn't have write access on `install.sysconfdir` path in `config.toml`."
+        );
+    }
 
     let datadir = prefix.join(default_path(&builder.config.datadir, "share"));
     let docdir = prefix.join(default_path(&builder.config.docdir, "share/doc/rust"));
@@ -94,13 +105,13 @@ fn install_sh(
     let mut cmd = Command::new(SHELL);
     cmd.current_dir(&empty_dir)
         .arg(sanitize_sh(&tarball.decompressed_output().join("install.sh")))
-        .arg(format!("--prefix={}", prepare_dir(prefix)))
-        .arg(format!("--sysconfdir={}", prepare_dir(sysconfdir)))
-        .arg(format!("--datadir={}", prepare_dir(datadir)))
-        .arg(format!("--docdir={}", prepare_dir(docdir)))
-        .arg(format!("--bindir={}", prepare_dir(bindir)))
-        .arg(format!("--libdir={}", prepare_dir(libdir)))
-        .arg(format!("--mandir={}", prepare_dir(mandir)))
+        .arg(format!("--prefix={}", prepare_dir(&destdir_env, prefix)))
+        .arg(format!("--sysconfdir={}", prepare_dir(&destdir_env, sysconfdir)))
+        .arg(format!("--datadir={}", prepare_dir(&destdir_env, datadir)))
+        .arg(format!("--docdir={}", prepare_dir(&destdir_env, docdir)))
+        .arg(format!("--bindir={}", prepare_dir(&destdir_env, bindir)))
+        .arg(format!("--libdir={}", prepare_dir(&destdir_env, libdir)))
+        .arg(format!("--mandir={}", prepare_dir(&destdir_env, mandir)))
         .arg("--disable-ldconfig");
     builder.run(&mut cmd);
     t!(fs::remove_dir_all(&empty_dir));
@@ -110,19 +121,16 @@ fn default_path(config: &Option<PathBuf>, default: &str) -> PathBuf {
     config.as_ref().cloned().unwrap_or_else(|| PathBuf::from(default))
 }
 
-fn prepare_dir(mut path: PathBuf) -> String {
+fn prepare_dir(destdir_env: &Option<PathBuf>, mut path: PathBuf) -> String {
     // The DESTDIR environment variable is a standard way to install software in a subdirectory
     // while keeping the original directory structure, even if the prefix or other directories
     // contain absolute paths.
     //
     // More information on the environment variable is available here:
     // https://www.gnu.org/prep/standards/html_node/DESTDIR.html
-    if let Some(destdir) = env::var_os("DESTDIR").map(PathBuf::from) {
-        // Sanity check for the user write access on DESTDIR
-        assert!(is_dir_writable_for_user(&destdir), "User doesn't have write access on DESTDIR.");
-
+    if let Some(destdir) = destdir_env {
         let without_destdir = path.clone();
-        path = destdir;
+        path = destdir.clone();
         // Custom .join() which ignores disk roots.
         for part in without_destdir.components() {
             if let Component::Normal(s) = part {

From 3f7e50696f04e713311b841c4446dbf800ffaf6a Mon Sep 17 00:00:00 2001
From: Ralf Jung <post@ralfj.de>
Date: Mon, 30 Oct 2023 10:29:08 +0100
Subject: [PATCH 232/297] deduce_param_attrs: explain a read-only case

---
 compiler/rustc_mir_transform/src/deduce_param_attrs.rs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
index 79645310a398..990cfb05e607 100644
--- a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
+++ b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs
@@ -44,6 +44,7 @@ impl<'tcx> Visitor<'tcx> for DeduceReadOnly {
                 // Whether mutating though a `&raw const` is allowed is still undecided, so we
                 // disable any sketchy `readonly` optimizations for now.
                 // But we only need to do this if the pointer would point into the argument.
+                // IOW: for indirect places, like `&raw (*local).field`, this surely cannot mutate `local`.
                 !place.is_indirect()
             }
             PlaceContext::NonMutatingUse(..) | PlaceContext::NonUse(..) => {

From be8fd8b7d028c50c66b6300aa1bf692afe2e231c Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 30 Oct 2023 16:10:20 +1100
Subject: [PATCH 233/297] Streamline `collect_crate_types`.

- The early return can be right at the top.
- The control flow is simplified with `if let`.
- The `collect` isn't necessary.
- The "Unconditionally" comment is erroneously duplicated from
  `check_attr_crate_type`, and can be removed.
---
 compiler/rustc_interface/src/util.rs | 22 +++++++---------------
 1 file changed, 7 insertions(+), 15 deletions(-)

diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs
index 9bda685df013..5fde98c7c062 100644
--- a/compiler/rustc_interface/src/util.rs
+++ b/compiler/rustc_interface/src/util.rs
@@ -482,21 +482,6 @@ fn categorize_crate_type(s: Symbol) -> Option<CrateType> {
 }
 
 pub fn collect_crate_types(session: &Session, attrs: &[ast::Attribute]) -> Vec<CrateType> {
-    // Unconditionally collect crate types from attributes to make them used
-    let attr_types: Vec<CrateType> = attrs
-        .iter()
-        .filter_map(|a| {
-            if a.has_name(sym::crate_type) {
-                match a.value_str() {
-                    Some(s) => categorize_crate_type(s),
-                    _ => None,
-                }
-            } else {
-                None
-            }
-        })
-        .collect();
-
     // If we're generating a test executable, then ignore all other output
     // styles at all other locations
     if session.opts.test {
@@ -510,6 +495,13 @@ pub fn collect_crate_types(session: &Session, attrs: &[ast::Attribute]) -> Vec<C
     #[allow(rustc::bad_opt_access)]
     let mut base = session.opts.crate_types.clone();
     if base.is_empty() {
+        let attr_types = attrs.iter().filter_map(|a| {
+            if a.has_name(sym::crate_type) && let Some(s) = a.value_str() {
+                categorize_crate_type(s)
+            } else {
+                None
+            }
+        });
         base.extend(attr_types);
         if base.is_empty() {
             base.push(output::default_output_for_target(session));

From 90862f63f9bc9f68cf28af6be31b4600c996bdf3 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 30 Oct 2023 18:19:40 +1100
Subject: [PATCH 234/297] Remove an unnecessary `drop`.

---
 compiler/rustc_interface/src/callbacks.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/compiler/rustc_interface/src/callbacks.rs b/compiler/rustc_interface/src/callbacks.rs
index 6fa989bb96ce..ef00ced67ffe 100644
--- a/compiler/rustc_interface/src/callbacks.rs
+++ b/compiler/rustc_interface/src/callbacks.rs
@@ -33,9 +33,7 @@ fn track_diagnostic(diagnostic: &mut Diagnostic, f: &mut dyn FnMut(&mut Diagnost
     tls::with_context_opt(|icx| {
         if let Some(icx) = icx {
             if let Some(diagnostics) = icx.diagnostics {
-                let mut diagnostics = diagnostics.lock();
-                diagnostics.extend(Some(diagnostic.clone()));
-                std::mem::drop(diagnostics);
+                diagnostics.lock().extend(Some(diagnostic.clone()));
             }
 
             // Diagnostics are tracked, we can ignore the dependency.

From 0c381ec05afd1f51f12c9727fc05eead2400d500 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 30 Oct 2023 18:29:23 +1100
Subject: [PATCH 235/297] Streamline some `use` items.

---
 compiler/rustc_interface/src/tests.rs | 24 +++++++-----------------
 1 file changed, 7 insertions(+), 17 deletions(-)

diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs
index 41ccb25f7893..594283168c98 100644
--- a/compiler/rustc_interface/src/tests.rs
+++ b/compiler/rustc_interface/src/tests.rs
@@ -2,28 +2,18 @@
 use crate::interface::parse_cfg;
 use rustc_data_structures::profiling::TimePassesFormat;
 use rustc_errors::{emitter::HumanReadableErrorType, registry, ColorConfig};
-use rustc_session::config::rustc_optgroups;
-use rustc_session::config::Cfg;
-use rustc_session::config::DebugInfo;
-use rustc_session::config::Input;
-use rustc_session::config::InstrumentXRay;
-use rustc_session::config::LinkSelfContained;
-use rustc_session::config::Polonius;
-use rustc_session::config::TraitSolver;
-use rustc_session::config::{build_configuration, build_session_options};
 use rustc_session::config::{
-    BranchProtection, Externs, OomStrategy, OutFileName, OutputType, OutputTypes, PAuthKey, PacRet,
-    ProcMacroExecutionStrategy, SymbolManglingVersion, WasiExecModel,
+    build_configuration, build_session_options, rustc_optgroups, BranchProtection, CFGuard, Cfg,
+    DebugInfo, DumpMonoStatsFormat, ErrorOutputType, ExternEntry, ExternLocation, Externs, Input,
+    InstrumentCoverage, InstrumentXRay, LinkSelfContained, LinkerPluginLto, LocationDetail, LtoCli,
+    MirSpanview, OomStrategy, Options, OutFileName, OutputType, OutputTypes, PAuthKey, PacRet,
+    Passes, Polonius, ProcMacroExecutionStrategy, Strip, SwitchWithOptPath, SymbolManglingVersion,
+    TraitSolver, WasiExecModel,
 };
-use rustc_session::config::{CFGuard, ExternEntry, LinkerPluginLto, LtoCli, SwitchWithOptPath};
-use rustc_session::config::{DumpMonoStatsFormat, MirSpanview};
-use rustc_session::config::{ErrorOutputType, ExternLocation, LocationDetail, Options, Strip};
-use rustc_session::config::{InstrumentCoverage, Passes};
 use rustc_session::lint::Level;
 use rustc_session::search_paths::SearchPath;
 use rustc_session::utils::{CanonicalizedPath, NativeLib, NativeLibKind};
-use rustc_session::{build_session, getopts, Session};
-use rustc_session::{CompilerIO, EarlyErrorHandler};
+use rustc_session::{build_session, getopts, CompilerIO, EarlyErrorHandler, Session};
 use rustc_span::edition::{Edition, DEFAULT_EDITION};
 use rustc_span::symbol::sym;
 use rustc_span::{FileName, SourceFileHashAlgorithm};

From b48adef3fd122d178a11787ad4b85cbff0b79dee Mon Sep 17 00:00:00 2001
From: Ralf Jung <post@ralfj.de>
Date: Mon, 30 Oct 2023 11:07:50 +0100
Subject: [PATCH 236/297] update which targets we test Miri on

---
 .github/workflows/ci.yml                      |  1 +
 .../host-x86_64/x86_64-gnu-tools/Dockerfile   |  2 ++
 .../x86_64-gnu-tools/checktools.sh            | 29 +++++++++++++++++--
 src/ci/github-actions/ci.yml                  |  1 +
 4 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f91e6816ae42..5c6729e35624 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -377,6 +377,7 @@ jobs:
           - name: x86_64-msvc-ext
             env:
               SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
+              HOST_TARGET: x86_64-pc-windows-msvc
               RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json"
               DEPLOY_TOOLSTATES_JSON: toolstates-windows.json
             os: windows-2019-8core-32gb
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
index 85f2f84a44c3..b0eeff0c5761 100644
--- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile
@@ -83,6 +83,8 @@ ENV RUST_CONFIGURE_ARGS \
   --build=x86_64-unknown-linux-gnu \
   --save-toolstates=/tmp/toolstate/toolstates.json
 
+ENV HOST_TARGET x86_64-unknown-linux-gnu
+
 ENV SCRIPT /tmp/checktools.sh ../x.py && \
   NODE_PATH=`npm root -g` python3 ../x.py test tests/rustdoc-gui --stage 2 \
     --test-args "'--no-sandbox --jobs 1'"
diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh
index 7dde6370904b..821a09feb2d5 100755
--- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh
+++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh
@@ -1,4 +1,5 @@
 #!/bin/sh
+# ignore-tidy-linelength
 
 set -eu
 
@@ -26,8 +27,30 @@ python3 "$X_PY" test --stage 2 src/tools/clippy
 python3 "$X_PY" test --stage 2 src/tools/rustfmt
 python3 "$X_PY" test --stage 2 src/tools/miri
 # We natively run this script on x86_64-unknown-linux-gnu and x86_64-pc-windows-msvc.
-# Also cover some other targets (on both of these hosts) via cross-testing.
+# Also cover some other targets via cross-testing, in particular all tier 1 targets.
 export BOOTSTRAP_SKIP_TARGET_SANITY=1 # we don't need `cc` for these targets
-python3 "$X_PY" test --stage 2 src/tools/miri --target i686-pc-windows-msvc
-python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-apple-darwin
+case $HOST_TARGET in
+  x86_64-unknown-linux-gnu)
+    # Only this branch runs in PR CI.
+    # Fully test all main OSes, including a 32bit target.
+    python3 "$X_PY" test --stage 2 src/tools/miri --target x86_64-apple-darwin
+    python3 "$X_PY" test --stage 2 src/tools/miri --target i686-pc-windows-msvc
+    # Only run "pass" tests for the remaining targets, which is quite a bit faster.
+    python3 "$X_PY" test --stage 2 src/tools/miri --target x86_64-pc-windows-gnu --test-args pass
+    python3 "$X_PY" test --stage 2 src/tools/miri --target i686-unknown-linux-gnu --test-args pass
+    python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-unknown-linux-gnu --test-args pass
+    python3 "$X_PY" test --stage 2 src/tools/miri --target s390x-unknown-linux-gnu --test-args pass
+    ;;
+  x86_64-pc-windows-msvc)
+    # Strangely, Linux targets do not work here. cargo always says
+    # "error: cannot produce cdylib for ... as the target ... does not support these crate types".
+    # Only run "pass" tests, which is quite a bit faster.
+    python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-apple-darwin --test-args pass
+    python3 "$X_PY" test --stage 2 src/tools/miri --target i686-pc-windows-gnu --test-args pass
+    ;;
+  *)
+    echo "FATAL: unexpected host $HOST_TARGET"
+    exit 1
+    ;;
+esac
 unset BOOTSTRAP_SKIP_TARGET_SANITY
diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml
index 2577682c57c6..f849d8d9a8e2 100644
--- a/src/ci/github-actions/ci.yml
+++ b/src/ci/github-actions/ci.yml
@@ -574,6 +574,7 @@ jobs:
           - name: x86_64-msvc-ext
             env:
               SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
+              HOST_TARGET: x86_64-pc-windows-msvc
               RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json
               DEPLOY_TOOLSTATES_JSON: toolstates-windows.json
             <<: *job-windows-8c

From 745c600617bc5dd1405cb53b1f79e5f421d62b0f Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Mon, 30 Oct 2023 10:13:12 +0000
Subject: [PATCH 237/297] Talk about `gen fn` in diagnostics about `gen fn`

---
 compiler/rustc_parse/messages.ftl       | 4 ++--
 compiler/rustc_parse/src/errors.rs      | 4 ++--
 compiler/rustc_parse/src/parser/item.rs | 2 +-
 tests/ui/coroutine/gen_fn.e2024.stderr  | 4 ++--
 tests/ui/coroutine/gen_fn.rs            | 2 +-
 5 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/compiler/rustc_parse/messages.ftl b/compiler/rustc_parse/messages.ftl
index 9bedd7b8a338..9ba1f0a5df91 100644
--- a/compiler/rustc_parse/messages.ftl
+++ b/compiler/rustc_parse/messages.ftl
@@ -278,8 +278,8 @@ parse_found_expr_would_be_stmt = expected expression, found `{$token}`
 parse_function_body_equals_expr = function body cannot be `= expression;`
     .suggestion = surround the expression with `{"{"}` and `{"}"}` instead of `=` and `;`
 
-parse_gen_block = `gen` blocks are not yet implemented
-    .help = only the keyword is reserved for now
+parse_gen_fn = `gen` functions are not yet implemented
+    .help = for now you can use `gen {"{}"}` blocks and return `impl Iterator` instead
 
 parse_generic_args_in_pat_require_turbofish_syntax = generic args in patterns require the turbofish syntax
 
diff --git a/compiler/rustc_parse/src/errors.rs b/compiler/rustc_parse/src/errors.rs
index c0e94d15da0b..e5cd91a32a7b 100644
--- a/compiler/rustc_parse/src/errors.rs
+++ b/compiler/rustc_parse/src/errors.rs
@@ -521,9 +521,9 @@ pub(crate) struct CatchAfterTry {
 }
 
 #[derive(Diagnostic)]
-#[diag(parse_gen_block)]
+#[diag(parse_gen_fn)]
 #[help]
-pub(crate) struct GenBlock {
+pub(crate) struct GenFn {
     #[primary_span]
     pub span: Span,
 }
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
index a9f456d64cd1..55ad3f42938b 100644
--- a/compiler/rustc_parse/src/parser/item.rs
+++ b/compiler/rustc_parse/src/parser/item.rs
@@ -2372,7 +2372,7 @@ impl<'a> Parser<'a> {
         }
 
         if let Gen::Yes { span, .. } = genness {
-            self.sess.emit_err(errors::GenBlock { span });
+            self.sess.emit_err(errors::GenFn { span });
         }
 
         if !self.eat_keyword_case(kw::Fn, case) {
diff --git a/tests/ui/coroutine/gen_fn.e2024.stderr b/tests/ui/coroutine/gen_fn.e2024.stderr
index 41bd04d97691..388e10fd65e4 100644
--- a/tests/ui/coroutine/gen_fn.e2024.stderr
+++ b/tests/ui/coroutine/gen_fn.e2024.stderr
@@ -1,10 +1,10 @@
-error: `gen` blocks are not yet implemented
+error: `gen` functions are not yet implemented
   --> $DIR/gen_fn.rs:4:1
    |
 LL | gen fn foo() {}
    | ^^^
    |
-   = help: only the keyword is reserved for now
+   = help: for now you can use `gen {}` blocks and return `impl Iterator` instead
 
 error: aborting due to previous error
 
diff --git a/tests/ui/coroutine/gen_fn.rs b/tests/ui/coroutine/gen_fn.rs
index 9566660dfb56..da515f263b00 100644
--- a/tests/ui/coroutine/gen_fn.rs
+++ b/tests/ui/coroutine/gen_fn.rs
@@ -3,6 +3,6 @@
 
 gen fn foo() {}
 //[none]~^ ERROR: expected one of `#`, `async`, `const`, `default`, `extern`, `fn`, `pub`, `unsafe`, or `use`, found `gen`
-//[e2024]~^^ ERROR: `gen` blocks are not yet implemented
+//[e2024]~^^ ERROR: `gen` functions are not yet implemented
 
 fn main() {}

From bc926f7c33c5fa12a5d42c1c597245d047346fd6 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Mon, 30 Oct 2023 10:23:57 +0000
Subject: [PATCH 238/297] Add a custom panic message for resuming `gen` blocks
 after they panicked

---
 compiler/rustc_middle/messages.ftl          |  2 ++
 compiler/rustc_middle/src/mir/terminator.rs |  3 +--
 tests/ui/coroutine/gen_block_panic.rs       | 25 +++++++++++++++++++++
 tests/ui/coroutine/gen_block_panic.stderr   | 12 ++++++++++
 4 files changed, 40 insertions(+), 2 deletions(-)
 create mode 100644 tests/ui/coroutine/gen_block_panic.rs
 create mode 100644 tests/ui/coroutine/gen_block_panic.stderr

diff --git a/compiler/rustc_middle/messages.ftl b/compiler/rustc_middle/messages.ftl
index 37ff5bcf1e2d..27d555d7e26c 100644
--- a/compiler/rustc_middle/messages.ftl
+++ b/compiler/rustc_middle/messages.ftl
@@ -12,6 +12,8 @@ middle_assert_coroutine_resume_after_return = coroutine resumed after completion
 middle_assert_divide_by_zero =
     attempt to divide `{$val}` by zero
 
+middle_assert_gen_resume_after_panic = `gen` fn or block cannot be further iterated on after it panicked
+
 middle_assert_misaligned_ptr_deref =
     misaligned pointer dereference: address must be a multiple of {$required} but is {$found}
 
diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs
index c2aa015f4b7e..9dfbe1733cc5 100644
--- a/compiler/rustc_middle/src/mir/terminator.rs
+++ b/compiler/rustc_middle/src/mir/terminator.rs
@@ -250,8 +250,7 @@ impl<O> AssertKind<O> {
                 middle_assert_coroutine_resume_after_return
             }
             ResumedAfterPanic(CoroutineKind::Async(_)) => middle_assert_async_resume_after_panic,
-            // FIXME(gen_blocks): custom error message for `gen` blocks
-            ResumedAfterPanic(CoroutineKind::Gen(_)) => middle_assert_async_resume_after_panic,
+            ResumedAfterPanic(CoroutineKind::Gen(_)) => middle_assert_gen_resume_after_panic,
             ResumedAfterPanic(CoroutineKind::Coroutine) => {
                 middle_assert_coroutine_resume_after_panic
             }
diff --git a/tests/ui/coroutine/gen_block_panic.rs b/tests/ui/coroutine/gen_block_panic.rs
new file mode 100644
index 000000000000..33e2b4b265a1
--- /dev/null
+++ b/tests/ui/coroutine/gen_block_panic.rs
@@ -0,0 +1,25 @@
+//compile-flags: --edition 2024 -Zunstable-options
+// run-pass
+#![feature(gen_blocks)]
+
+fn main() {
+    let mut iter = gen {
+        yield 42;
+        panic!("foo");
+        yield 69; //~ WARN: unreachable statement
+    };
+    assert_eq!(iter.next(), Some(42));
+    let mut tmp = std::panic::AssertUnwindSafe(&mut iter);
+    match std::panic::catch_unwind(move || tmp.next()) {
+        Ok(_) => unreachable!(),
+        Err(err) => assert_eq!(*err.downcast::<&'static str>().unwrap(), "foo"),
+    }
+
+    match std::panic::catch_unwind(move || iter.next()) {
+        Ok(_) => unreachable!(),
+        Err(err) => assert_eq!(
+            *err.downcast::<&'static str>().unwrap(),
+            "`gen fn` should just keep returning `None` after panicking",
+        ),
+    }
+}
diff --git a/tests/ui/coroutine/gen_block_panic.stderr b/tests/ui/coroutine/gen_block_panic.stderr
new file mode 100644
index 000000000000..33d2a95cab8b
--- /dev/null
+++ b/tests/ui/coroutine/gen_block_panic.stderr
@@ -0,0 +1,12 @@
+warning: unreachable statement
+  --> $DIR/gen_block_panic.rs:9:9
+   |
+LL |         panic!("foo");
+   |         ------------- any code following this expression is unreachable
+LL |         yield 69;
+   |         ^^^^^^^^^ unreachable statement
+   |
+   = note: `#[warn(unreachable_code)]` on by default
+
+warning: 1 warning emitted
+

From a2486dba3b6cd64b082273a3e24a3e8b6b65dc1b Mon Sep 17 00:00:00 2001
From: Gurinder Singh <frederick.the.fool@gmail.com>
Date: Mon, 30 Oct 2023 19:04:55 +0530
Subject: [PATCH 239/297] Fix missing leading space in suggestion

For a local pattern with no space between `let` and `(` e.g.:

  let(_a) = 3;

we were previously suggesting this illegal code:

  let_a =3;

After this change the suggestion will instead be:

  let _a =3;

(Note the space after `let`)
---
 compiler/rustc_lint/src/unused.rs            |  2 +-
 tests/ui/lint/lint-unnecessary-parens.fixed  |  8 +++
 tests/ui/lint/lint-unnecessary-parens.rs     |  8 +++
 tests/ui/lint/lint-unnecessary-parens.stderr | 74 +++++++++++++++++++-
 4 files changed, 90 insertions(+), 2 deletions(-)

diff --git a/compiler/rustc_lint/src/unused.rs b/compiler/rustc_lint/src/unused.rs
index 6b31fb079e01..355855b8e2b3 100644
--- a/compiler/rustc_lint/src/unused.rs
+++ b/compiler/rustc_lint/src/unused.rs
@@ -1154,7 +1154,7 @@ impl EarlyLintPass for UnusedParens {
 
     fn check_stmt(&mut self, cx: &EarlyContext<'_>, s: &ast::Stmt) {
         if let StmtKind::Local(ref local) = s.kind {
-            self.check_unused_parens_pat(cx, &local.pat, true, false, (false, false));
+            self.check_unused_parens_pat(cx, &local.pat, true, false, (true, false));
         }
 
         <Self as UnusedDelimLint>::check_stmt(self, cx, s)
diff --git a/tests/ui/lint/lint-unnecessary-parens.fixed b/tests/ui/lint/lint-unnecessary-parens.fixed
index bafac05d8daa..b17914da6e6a 100644
--- a/tests/ui/lint/lint-unnecessary-parens.fixed
+++ b/tests/ui/lint/lint-unnecessary-parens.fixed
@@ -84,6 +84,14 @@ fn main() {
     _a = 0; //~ ERROR unnecessary parentheses around assigned value
     _a += 1; //~ ERROR unnecessary parentheses around assigned value
 
+    let mut _a = 3; //~ ERROR unnecessary parentheses around pattern
+    let mut _a = 3; //~ ERROR unnecessary parentheses around pattern
+    let mut _a = 3; //~ ERROR unnecessary parentheses around pattern
+
+    let _a = 3; //~ ERROR unnecessary parentheses around pattern
+    let _a = 3; //~ ERROR unnecessary parentheses around pattern
+    let _a = 3; //~ ERROR unnecessary parentheses around pattern
+
     let _a = baz!(3, 4);
     let _b = baz!(3);
 }
diff --git a/tests/ui/lint/lint-unnecessary-parens.rs b/tests/ui/lint/lint-unnecessary-parens.rs
index ce537a4dc1da..4cbd6562cd39 100644
--- a/tests/ui/lint/lint-unnecessary-parens.rs
+++ b/tests/ui/lint/lint-unnecessary-parens.rs
@@ -84,6 +84,14 @@ fn main() {
     _a = (0); //~ ERROR unnecessary parentheses around assigned value
     _a += (1); //~ ERROR unnecessary parentheses around assigned value
 
+    let(mut _a) = 3; //~ ERROR unnecessary parentheses around pattern
+    let (mut _a) = 3; //~ ERROR unnecessary parentheses around pattern
+    let( mut _a) = 3; //~ ERROR unnecessary parentheses around pattern
+
+    let(_a) = 3; //~ ERROR unnecessary parentheses around pattern
+    let (_a) = 3; //~ ERROR unnecessary parentheses around pattern
+    let( _a) = 3; //~ ERROR unnecessary parentheses around pattern
+
     let _a = baz!(3, 4);
     let _b = baz!(3);
 }
diff --git a/tests/ui/lint/lint-unnecessary-parens.stderr b/tests/ui/lint/lint-unnecessary-parens.stderr
index 2ad07530f8c8..ba7a78b8da1a 100644
--- a/tests/ui/lint/lint-unnecessary-parens.stderr
+++ b/tests/ui/lint/lint-unnecessary-parens.stderr
@@ -267,5 +267,77 @@ LL -     _a += (1);
 LL +     _a += 1;
    |
 
-error: aborting due to 22 previous errors
+error: unnecessary parentheses around pattern
+  --> $DIR/lint-unnecessary-parens.rs:87:8
+   |
+LL |     let(mut _a) = 3;
+   |        ^      ^
+   |
+help: remove these parentheses
+   |
+LL -     let(mut _a) = 3;
+LL +     let mut _a = 3;
+   |
+
+error: unnecessary parentheses around pattern
+  --> $DIR/lint-unnecessary-parens.rs:88:9
+   |
+LL |     let (mut _a) = 3;
+   |         ^      ^
+   |
+help: remove these parentheses
+   |
+LL -     let (mut _a) = 3;
+LL +     let mut _a = 3;
+   |
+
+error: unnecessary parentheses around pattern
+  --> $DIR/lint-unnecessary-parens.rs:89:8
+   |
+LL |     let( mut _a) = 3;
+   |        ^^      ^
+   |
+help: remove these parentheses
+   |
+LL -     let( mut _a) = 3;
+LL +     let mut _a = 3;
+   |
+
+error: unnecessary parentheses around pattern
+  --> $DIR/lint-unnecessary-parens.rs:91:8
+   |
+LL |     let(_a) = 3;
+   |        ^  ^
+   |
+help: remove these parentheses
+   |
+LL -     let(_a) = 3;
+LL +     let _a = 3;
+   |
+
+error: unnecessary parentheses around pattern
+  --> $DIR/lint-unnecessary-parens.rs:92:9
+   |
+LL |     let (_a) = 3;
+   |         ^  ^
+   |
+help: remove these parentheses
+   |
+LL -     let (_a) = 3;
+LL +     let _a = 3;
+   |
+
+error: unnecessary parentheses around pattern
+  --> $DIR/lint-unnecessary-parens.rs:93:8
+   |
+LL |     let( _a) = 3;
+   |        ^^  ^
+   |
+help: remove these parentheses
+   |
+LL -     let( _a) = 3;
+LL +     let _a = 3;
+   |
+
+error: aborting due to 28 previous errors
 

From 82f34fdd2313b27a4f3e28bed74163154e231b9e Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Mon, 30 Oct 2023 19:20:56 +0800
Subject: [PATCH 240/297] Fix #117284, Fix unused variables lint issue for args
 in macro

---
 compiler/rustc_passes/messages.ftl            |  3 ++
 compiler/rustc_passes/src/errors.rs           | 21 ++++++++++----
 compiler/rustc_passes/src/liveness.rs         | 25 ++++++++++++----
 .../lint/unused/issue-117284-arg-in-macro.rs  | 17 +++++++++++
 .../unused/issue-117284-arg-in-macro.stderr   | 29 +++++++++++++++++++
 5 files changed, 84 insertions(+), 11 deletions(-)
 create mode 100644 tests/ui/lint/unused/issue-117284-arg-in-macro.rs
 create mode 100644 tests/ui/lint/unused/issue-117284-arg-in-macro.stderr

diff --git a/compiler/rustc_passes/messages.ftl b/compiler/rustc_passes/messages.ftl
index 026186cbe6cd..a0a98dd85366 100644
--- a/compiler/rustc_passes/messages.ftl
+++ b/compiler/rustc_passes/messages.ftl
@@ -778,6 +778,8 @@ passes_unused_var_maybe_capture_ref = unused variable: `{$name}`
 passes_unused_var_remove_field = unused variable: `{$name}`
 passes_unused_var_remove_field_suggestion = try removing the field
 
+passes_unused_variable_args_in_macro = `{$name}` is captured in macro and introduced a unused variable
+
 passes_unused_variable_try_ignore = unused variable: `{$name}`
     .suggestion = try ignoring the field
 
@@ -785,6 +787,7 @@ passes_unused_variable_try_prefix = unused variable: `{$name}`
     .label = unused variable
     .suggestion = if this is intentional, prefix it with an underscore
 
+
 passes_used_compiler_linker =
     `used(compiler)` and `used(linker)` can't be used together
 
diff --git a/compiler/rustc_passes/src/errors.rs b/compiler/rustc_passes/src/errors.rs
index eca0fb7748b1..bc31b5db021d 100644
--- a/compiler/rustc_passes/src/errors.rs
+++ b/compiler/rustc_passes/src/errors.rs
@@ -1768,15 +1768,24 @@ pub struct UnusedVariableTryPrefix {
     #[subdiagnostic]
     pub string_interp: Vec<UnusedVariableStringInterp>,
     #[subdiagnostic]
-    pub sugg: UnusedVariableTryPrefixSugg,
+    pub sugg: UnusedVariableSugg,
+    pub name: String,
 }
 
 #[derive(Subdiagnostic)]
-#[multipart_suggestion(passes_suggestion, applicability = "machine-applicable")]
-pub struct UnusedVariableTryPrefixSugg {
-    #[suggestion_part(code = "_{name}")]
-    pub spans: Vec<Span>,
-    pub name: String,
+pub enum UnusedVariableSugg {
+    #[multipart_suggestion(passes_suggestion, applicability = "machine-applicable")]
+    TryPrefixSugg {
+        #[suggestion_part(code = "_{name}")]
+        spans: Vec<Span>,
+        name: String,
+    },
+    #[help(passes_unused_variable_args_in_macro)]
+    NoSugg {
+        #[primary_span]
+        span: Span,
+        name: String,
+    },
 }
 
 pub struct UnusedVariableStringInterp {
diff --git a/compiler/rustc_passes/src/liveness.rs b/compiler/rustc_passes/src/liveness.rs
index d068fe62473c..b73fb984c0ea 100644
--- a/compiler/rustc_passes/src/liveness.rs
+++ b/compiler/rustc_passes/src/liveness.rs
@@ -1580,7 +1580,6 @@ impl<'tcx> Liveness<'_, 'tcx> {
         opt_body: Option<&hir::Body<'_>>,
     ) {
         let first_hir_id = hir_ids_and_spans[0].0;
-
         if let Some(name) = self.should_warn(var).filter(|name| name != "self") {
             // annoying: for parameters in funcs like `fn(x: i32)
             // {ret}`, there is only one node, so asking about
@@ -1652,11 +1651,29 @@ impl<'tcx> Liveness<'_, 'tcx> {
                         },
                     );
                 } else {
+                    // #117284, when `pat_span` and `ident_span` have different contexts
+                    // we can't provide a good suggestion, instead we pointed out the spans from macro
+                    let from_macro = non_shorthands
+                        .iter()
+                        .find(|(_, pat_span, ident_span)| {
+                            pat_span.ctxt() != ident_span.ctxt() && pat_span.from_expansion()
+                        })
+                        .map(|(_, pat_span, _)| *pat_span);
                     let non_shorthands = non_shorthands
                         .into_iter()
                         .map(|(_, _, ident_span)| ident_span)
                         .collect::<Vec<_>>();
+
                     let suggestions = self.string_interp_suggestions(&name, opt_body);
+                    let sugg = if let Some(span) = from_macro {
+                        errors::UnusedVariableSugg::NoSugg { span, name: name.clone() }
+                    } else {
+                        errors::UnusedVariableSugg::TryPrefixSugg {
+                            spans: non_shorthands,
+                            name: name.clone(),
+                        }
+                    };
+
                     self.ir.tcx.emit_spanned_lint(
                         lint::builtin::UNUSED_VARIABLES,
                         first_hir_id,
@@ -1666,10 +1683,8 @@ impl<'tcx> Liveness<'_, 'tcx> {
                             .collect::<Vec<_>>(),
                         errors::UnusedVariableTryPrefix {
                             label: if !suggestions.is_empty() { Some(pat.span) } else { None },
-                            sugg: errors::UnusedVariableTryPrefixSugg {
-                                spans: non_shorthands,
-                                name,
-                            },
+                            name,
+                            sugg,
                             string_interp: suggestions,
                         },
                     );
diff --git a/tests/ui/lint/unused/issue-117284-arg-in-macro.rs b/tests/ui/lint/unused/issue-117284-arg-in-macro.rs
new file mode 100644
index 000000000000..eea0f4c594dc
--- /dev/null
+++ b/tests/ui/lint/unused/issue-117284-arg-in-macro.rs
@@ -0,0 +1,17 @@
+#![deny(unused_variables)]
+macro_rules! make_var {
+    ($struct:ident, $var:ident) => {
+        let $var = $struct.$var;
+    };
+}
+
+#[allow(unused)]
+struct MyStruct {
+    var: i32,
+}
+
+fn main() {
+    let s = MyStruct { var: 42 };
+    make_var!(s, var); //~ ERROR unused variable: `var`
+    let a = 1; //~ ERROR unused variable: `a`
+}
diff --git a/tests/ui/lint/unused/issue-117284-arg-in-macro.stderr b/tests/ui/lint/unused/issue-117284-arg-in-macro.stderr
new file mode 100644
index 000000000000..84efaa4f3687
--- /dev/null
+++ b/tests/ui/lint/unused/issue-117284-arg-in-macro.stderr
@@ -0,0 +1,29 @@
+error: unused variable: `var`
+  --> $DIR/issue-117284-arg-in-macro.rs:15:18
+   |
+LL |     make_var!(s, var);
+   |                  ^^^
+   |
+help: `var` is captured in macro and introduced a unused variable
+  --> $DIR/issue-117284-arg-in-macro.rs:4:13
+   |
+LL |         let $var = $struct.$var;
+   |             ^^^^
+...
+LL |     make_var!(s, var);
+   |     ----------------- in this macro invocation
+note: the lint level is defined here
+  --> $DIR/issue-117284-arg-in-macro.rs:1:9
+   |
+LL | #![deny(unused_variables)]
+   |         ^^^^^^^^^^^^^^^^
+   = note: this error originates in the macro `make_var` (in Nightly builds, run with -Z macro-backtrace for more info)
+
+error: unused variable: `a`
+  --> $DIR/issue-117284-arg-in-macro.rs:16:9
+   |
+LL |     let a = 1;
+   |         ^ help: if this is intentional, prefix it with an underscore: `_a`
+
+error: aborting due to 2 previous errors
+

From 972ee01b69378280b4911caaae0bd95ea8d790c3 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Mon, 30 Oct 2023 13:38:54 +0000
Subject: [PATCH 241/297] Add regression test

---
 .../nested_impl_trait_in_assoc_ty.rs          | 37 +++++++++++++++++++
 .../nested_impl_trait_in_assoc_ty.stderr      | 15 ++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 tests/ui/type-alias-impl-trait/nested_impl_trait_in_assoc_ty.rs
 create mode 100644 tests/ui/type-alias-impl-trait/nested_impl_trait_in_assoc_ty.stderr

diff --git a/tests/ui/type-alias-impl-trait/nested_impl_trait_in_assoc_ty.rs b/tests/ui/type-alias-impl-trait/nested_impl_trait_in_assoc_ty.rs
new file mode 100644
index 000000000000..859a6807c2ca
--- /dev/null
+++ b/tests/ui/type-alias-impl-trait/nested_impl_trait_in_assoc_ty.rs
@@ -0,0 +1,37 @@
+// edition: 2021
+
+#![feature(impl_trait_in_assoc_type)]
+
+use std::future::Future;
+
+pub struct MemtableLocalStateStore {
+    mem_table: MemTable,
+}
+
+impl LocalStateStore for MemtableLocalStateStore {
+    type IterStream<'a> = impl Sized + 'a where Self: 'a;
+
+    fn iter(&self) -> impl Future<Output = Self::IterStream<'_>> + '_ {
+        async move { merge_stream(self.mem_table.iter()) }
+    }
+}
+
+trait LocalStateStore {
+    type IterStream<'a>
+    where
+        Self: 'a;
+
+    fn iter(&self) -> impl Future<Output = Self::IterStream<'_>> + '_;
+}
+
+struct MemTable;
+
+impl MemTable {
+    fn iter<'a>(&'a self) -> impl Iterator<Item = &'a ()> {
+        std::iter::empty()
+    }
+}
+
+pub(crate) async fn merge_stream<'a>(mem_table_iter: impl Iterator<Item = &'a ()>) {}
+
+fn main() {}
diff --git a/tests/ui/type-alias-impl-trait/nested_impl_trait_in_assoc_ty.stderr b/tests/ui/type-alias-impl-trait/nested_impl_trait_in_assoc_ty.stderr
new file mode 100644
index 000000000000..30b3deca1e42
--- /dev/null
+++ b/tests/ui/type-alias-impl-trait/nested_impl_trait_in_assoc_ty.stderr
@@ -0,0 +1,15 @@
+error[E0792]: non-defining opaque type use in defining scope
+  --> $DIR/nested_impl_trait_in_assoc_ty.rs:15:9
+   |
+LL |         async move { merge_stream(self.mem_table.iter()) }
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ argument `'_` is not a generic parameter
+   |
+note: for this opaque type
+  --> $DIR/nested_impl_trait_in_assoc_ty.rs:35:1
+   |
+LL | pub(crate) async fn merge_stream<'a>(mem_table_iter: impl Iterator<Item = &'a ()>) {}
+   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0792`.

From 8d03e1394f90c43c25c00bcb6ba2dd621215cac4 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Mon, 30 Oct 2023 13:46:44 +0000
Subject: [PATCH 242/297] Don't treat closures/coroutines as part of the public
 API

---
 compiler/rustc_ty_utils/src/opaque_types.rs   |  1 +
 compiler/rustc_ty_utils/src/sig_types.rs      | 29 +++++--------------
 .../nested_impl_trait_in_assoc_ty.rs          |  7 +++++
 .../nested_impl_trait_in_assoc_ty.stderr      | 15 ----------
 4 files changed, 16 insertions(+), 36 deletions(-)
 delete mode 100644 tests/ui/type-alias-impl-trait/nested_impl_trait_in_assoc_ty.stderr

diff --git a/compiler/rustc_ty_utils/src/opaque_types.rs b/compiler/rustc_ty_utils/src/opaque_types.rs
index f7d54566cb70..9242a1a751bc 100644
--- a/compiler/rustc_ty_utils/src/opaque_types.rs
+++ b/compiler/rustc_ty_utils/src/opaque_types.rs
@@ -121,6 +121,7 @@ impl<'tcx> OpaqueTypeCollector<'tcx> {
 }
 
 impl<'tcx> super::sig_types::SpannedTypeVisitor<'tcx> for OpaqueTypeCollector<'tcx> {
+    #[instrument(skip(self), ret, level = "trace")]
     fn visit(&mut self, span: Span, value: impl TypeVisitable<TyCtxt<'tcx>>) -> ControlFlow<!> {
         self.visit_spanned(span, value);
         ControlFlow::Continue(())
diff --git a/compiler/rustc_ty_utils/src/sig_types.rs b/compiler/rustc_ty_utils/src/sig_types.rs
index 1ab39974e0f2..5d4aee318c61 100644
--- a/compiler/rustc_ty_utils/src/sig_types.rs
+++ b/compiler/rustc_ty_utils/src/sig_types.rs
@@ -4,7 +4,7 @@
 use std::ops::ControlFlow;
 
 use rustc_hir::{def::DefKind, def_id::LocalDefId};
-use rustc_middle::ty::{self, TyCtxt};
+use rustc_middle::ty::TyCtxt;
 use rustc_span::Span;
 use rustc_type_ir::visit::TypeVisitable;
 
@@ -25,24 +25,9 @@ pub fn walk_types<'tcx, V: SpannedTypeVisitor<'tcx>>(
     let kind = tcx.def_kind(item);
     trace!(?kind);
     match kind {
-        DefKind::Coroutine => {
-            match tcx.type_of(item).instantiate_identity().kind() {
-                ty::Coroutine(_, args, _) => visitor.visit(tcx.def_span(item), args.as_coroutine().sig())?,
-                _ => bug!(),
-            }
-            for (pred, span) in tcx.predicates_of(item).instantiate_identity(tcx) {
-                visitor.visit(span, pred)?;
-            }
-        }
-        // Walk over the signature of the function-like
-        DefKind::Closure | DefKind::AssocFn | DefKind::Fn => {
-            let ty_sig = match kind {
-                DefKind::Closure => match tcx.type_of(item).instantiate_identity().kind() {
-                    ty::Closure(_, args) => args.as_closure().sig(),
-                    _ => bug!(),
-                },
-                _ => tcx.fn_sig(item).instantiate_identity(),
-            };
+        // Walk over the signature of the function
+        DefKind::AssocFn | DefKind::Fn => {
+            let ty_sig = tcx.fn_sig(item).instantiate_identity();
             let hir_sig = tcx.hir().get_by_def_id(item).fn_decl().unwrap();
             // Walk over the inputs and outputs manually in order to get good spans for them.
             visitor.visit(hir_sig.output.span(), ty_sig.output());
@@ -79,8 +64,10 @@ pub fn walk_types<'tcx, V: SpannedTypeVisitor<'tcx>>(
                 visitor.visit(span, pred)?;
             }
         }
-        // Does not have a syntactical signature
-        DefKind::InlineConst => {}
+        // These are not part of a public API, they can only appear as hidden types, and there
+        // the interesting parts are solely in the signature of the containing item's opaque type
+        // or dyn type.
+        DefKind::InlineConst | DefKind::Closure | DefKind::Coroutine => {}
         DefKind::Impl { of_trait } => {
             if of_trait {
                 let span = tcx.hir().get_by_def_id(item).expect_item().expect_impl().of_trait.unwrap().path.span;
diff --git a/tests/ui/type-alias-impl-trait/nested_impl_trait_in_assoc_ty.rs b/tests/ui/type-alias-impl-trait/nested_impl_trait_in_assoc_ty.rs
index 859a6807c2ca..5f3dbaa17989 100644
--- a/tests/ui/type-alias-impl-trait/nested_impl_trait_in_assoc_ty.rs
+++ b/tests/ui/type-alias-impl-trait/nested_impl_trait_in_assoc_ty.rs
@@ -1,4 +1,11 @@
+//! This test checks that we do not walk types in async blocks for
+//! determining the opaque types that appear in a signature. async blocks,
+//! all other coroutines and closures are always private and not part of
+//! a signature. They become part of a signature via `dyn Trait` or `impl Trait`,
+//! which is something that we process abstractly without looking at its hidden
+//! types.
 // edition: 2021
+// check-pass
 
 #![feature(impl_trait_in_assoc_type)]
 
diff --git a/tests/ui/type-alias-impl-trait/nested_impl_trait_in_assoc_ty.stderr b/tests/ui/type-alias-impl-trait/nested_impl_trait_in_assoc_ty.stderr
deleted file mode 100644
index 30b3deca1e42..000000000000
--- a/tests/ui/type-alias-impl-trait/nested_impl_trait_in_assoc_ty.stderr
+++ /dev/null
@@ -1,15 +0,0 @@
-error[E0792]: non-defining opaque type use in defining scope
-  --> $DIR/nested_impl_trait_in_assoc_ty.rs:15:9
-   |
-LL |         async move { merge_stream(self.mem_table.iter()) }
-   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ argument `'_` is not a generic parameter
-   |
-note: for this opaque type
-  --> $DIR/nested_impl_trait_in_assoc_ty.rs:35:1
-   |
-LL | pub(crate) async fn merge_stream<'a>(mem_table_iter: impl Iterator<Item = &'a ()>) {}
-   | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-error: aborting due to previous error
-
-For more information about this error, try `rustc --explain E0792`.

From 251021c1ab095e6448cfe060e1d43a4bab2711f5 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Mon, 30 Oct 2023 13:47:08 +0000
Subject: [PATCH 243/297] Merge two equal match arms

---
 compiler/rustc_ty_utils/src/sig_types.rs | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/compiler/rustc_ty_utils/src/sig_types.rs b/compiler/rustc_ty_utils/src/sig_types.rs
index 5d4aee318c61..741ee8f0b001 100644
--- a/compiler/rustc_ty_utils/src/sig_types.rs
+++ b/compiler/rustc_ty_utils/src/sig_types.rs
@@ -82,12 +82,7 @@ pub fn walk_types<'tcx, V: SpannedTypeVisitor<'tcx>>(
             for (pred, span) in tcx.predicates_of(item).instantiate_identity(tcx) {
                 visitor.visit(span, pred)?;
             }}
-        DefKind::Trait => {
-            for (pred, span) in tcx.predicates_of(item).instantiate_identity(tcx) {
-                visitor.visit(span, pred)?;
-            }
-        }
-        DefKind::TraitAlias => {
+        DefKind::TraitAlias | DefKind::Trait => {
             for (pred, span) in tcx.predicates_of(item).instantiate_identity(tcx) {
                 visitor.visit(span, pred)?;
             }

From 43ff2a7e5095a9d79a3c4acde1208df6aa7bf296 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Mon, 30 Oct 2023 13:48:34 +0000
Subject: [PATCH 244/297] Some manual rustfmt as rustfmt is broken on this file

---
 compiler/rustc_ty_utils/src/sig_types.rs | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/compiler/rustc_ty_utils/src/sig_types.rs b/compiler/rustc_ty_utils/src/sig_types.rs
index 741ee8f0b001..ccdc6120196a 100644
--- a/compiler/rustc_ty_utils/src/sig_types.rs
+++ b/compiler/rustc_ty_utils/src/sig_types.rs
@@ -46,7 +46,7 @@ pub fn walk_types<'tcx, V: SpannedTypeVisitor<'tcx>>(
                 Some(ty) => ty.span,
                 _ => tcx.def_span(item),
             };
-            visitor.visit(span,  tcx.type_of(item).instantiate_identity());
+            visitor.visit(span, tcx.type_of(item).instantiate_identity());
             for (pred, span) in tcx.predicates_of(item).instantiate_identity(tcx) {
                 visitor.visit(span, pred)?;
             }
@@ -59,7 +59,7 @@ pub fn walk_types<'tcx, V: SpannedTypeVisitor<'tcx>>(
         // Look at field types
         DefKind::Struct | DefKind::Union | DefKind::Enum => {
             let span = tcx.def_ident_span(item).unwrap();
-            visitor.visit(span,  tcx.type_of(item).instantiate_identity());
+            visitor.visit(span, tcx.type_of(item).instantiate_identity());
             for (pred, span) in tcx.predicates_of(item).instantiate_identity(tcx) {
                 visitor.visit(span, pred)?;
             }
@@ -81,7 +81,8 @@ pub fn walk_types<'tcx, V: SpannedTypeVisitor<'tcx>>(
             visitor.visit(span, tcx.type_of(item).instantiate_identity());
             for (pred, span) in tcx.predicates_of(item).instantiate_identity(tcx) {
                 visitor.visit(span, pred)?;
-            }}
+            }
+        }
         DefKind::TraitAlias | DefKind::Trait => {
             for (pred, span) in tcx.predicates_of(item).instantiate_identity(tcx) {
                 visitor.visit(span, pred)?;

From 8076414f89275786ae04035151d54bb5b47bde9f Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Mon, 30 Oct 2023 10:14:43 -0400
Subject: [PATCH 245/297] Don't emit delayed good-path bugs on panic

---
 compiler/rustc_errors/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/compiler/rustc_errors/src/lib.rs b/compiler/rustc_errors/src/lib.rs
index 5e3fdf170bc0..dd462cc64865 100644
--- a/compiler/rustc_errors/src/lib.rs
+++ b/compiler/rustc_errors/src/lib.rs
@@ -556,7 +556,7 @@ impl Drop for HandlerInner {
         // instead of "require some error happened". Sadly that isn't ideal, as
         // lints can be `#[allow]`'d, potentially leading to this triggering.
         // Also, "good path" should be replaced with a better naming.
-        if !self.has_any_message() && !self.suppressed_expected_diag {
+        if !self.has_any_message() && !self.suppressed_expected_diag && !std::thread::panicking() {
             let bugs = std::mem::replace(&mut self.delayed_good_path_bugs, Vec::new());
             self.flush_delayed(
                 bugs,

From 8508e658958d02bc0c4bb62646a9cf9098ee53a8 Mon Sep 17 00:00:00 2001
From: Nicholas Bishop <nbishop@nbishop.net>
Date: Sun, 29 Oct 2023 17:04:52 -0400
Subject: [PATCH 246/297] Fix bad-c-variadic error being emitted multiple times

If a function incorrectly contains multiple `...` args, and is also not
foreign or `unsafe extern "C"`, only emit the latter error once.
---
 .../rustc_ast_passes/src/ast_validation.rs    | 18 +++++---
 compiler/rustc_ast_passes/src/errors.rs       |  2 +-
 tests/ui/c-variadic/issue-86053-1.stderr      | 10 +----
 .../variadic-ffi-semantic-restrictions.rs     |  2 -
 .../variadic-ffi-semantic-restrictions.stderr | 42 +++++++------------
 5 files changed, 31 insertions(+), 43 deletions(-)

diff --git a/compiler/rustc_ast_passes/src/ast_validation.rs b/compiler/rustc_ast_passes/src/ast_validation.rs
index 0477c7b2ba99..f59f7d389cda 100644
--- a/compiler/rustc_ast_passes/src/ast_validation.rs
+++ b/compiler/rustc_ast_passes/src/ast_validation.rs
@@ -485,6 +485,18 @@ impl<'a> AstValidator<'a> {
     /// Reject C-variadic type unless the function is foreign,
     /// or free and `unsafe extern "C"` semantically.
     fn check_c_variadic_type(&self, fk: FnKind<'a>) {
+        let variadic_spans: Vec<_> = fk
+            .decl()
+            .inputs
+            .iter()
+            .filter(|arg| matches!(arg.ty.kind, TyKind::CVarArgs))
+            .map(|arg| arg.span)
+            .collect();
+
+        if variadic_spans.is_empty() {
+            return;
+        }
+
         match (fk.ctxt(), fk.header()) {
             (Some(FnCtxt::Foreign), _) => return,
             (Some(FnCtxt::Free), Some(header)) => match header.ext {
@@ -499,11 +511,7 @@ impl<'a> AstValidator<'a> {
             _ => {}
         };
 
-        for Param { ty, span, .. } in &fk.decl().inputs {
-            if let TyKind::CVarArgs = ty.kind {
-                self.err_handler().emit_err(errors::BadCVariadic { span: *span });
-            }
-        }
+        self.err_handler().emit_err(errors::BadCVariadic { span: variadic_spans });
     }
 
     fn check_item_named(&self, ident: Ident, kind: &str) {
diff --git a/compiler/rustc_ast_passes/src/errors.rs b/compiler/rustc_ast_passes/src/errors.rs
index e74d94e43474..579f528e20f9 100644
--- a/compiler/rustc_ast_passes/src/errors.rs
+++ b/compiler/rustc_ast_passes/src/errors.rs
@@ -271,7 +271,7 @@ pub struct ExternItemAscii {
 #[diag(ast_passes_bad_c_variadic)]
 pub struct BadCVariadic {
     #[primary_span]
-    pub span: Span,
+    pub span: Vec<Span>,
 }
 
 #[derive(Diagnostic)]
diff --git a/tests/ui/c-variadic/issue-86053-1.stderr b/tests/ui/c-variadic/issue-86053-1.stderr
index 5a02f4aa93a9..69e19e1d4d26 100644
--- a/tests/ui/c-variadic/issue-86053-1.stderr
+++ b/tests/ui/c-variadic/issue-86053-1.stderr
@@ -50,13 +50,7 @@ error: only foreign or `unsafe extern "C"` functions may be C-variadic
   --> $DIR/issue-86053-1.rs:11:12
    |
 LL |     self , ... ,   self ,   self , ... ) where F : FnOnce ( & 'a & 'b usize ) {
-   |            ^^^
-
-error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/issue-86053-1.rs:11:36
-   |
-LL |     self , ... ,   self ,   self , ... ) where F : FnOnce ( & 'a & 'b usize ) {
-   |                                    ^^^
+   |            ^^^                     ^^^
 
 error[E0412]: cannot find type `F` in this scope
   --> $DIR/issue-86053-1.rs:11:48
@@ -76,6 +70,6 @@ help: you might be missing a type parameter
 LL | fn ordering4 < 'a , 'b, F     > ( a :            ,   self , self ,   self ,
    |                       +++
 
-error: aborting due to 11 previous errors
+error: aborting due to 10 previous errors
 
 For more information about this error, try `rustc --explain E0412`.
diff --git a/tests/ui/parser/variadic-ffi-semantic-restrictions.rs b/tests/ui/parser/variadic-ffi-semantic-restrictions.rs
index 0b61e267da80..8e382314ca27 100644
--- a/tests/ui/parser/variadic-ffi-semantic-restrictions.rs
+++ b/tests/ui/parser/variadic-ffi-semantic-restrictions.rs
@@ -49,11 +49,9 @@ impl X {
     //~| ERROR C-variadic function must be declared with at least one named argument
     fn i_f3(..., x: isize, ...) {}
     //~^ ERROR only foreign or `unsafe extern "C"` functions may be C-variadic
-    //~| ERROR only foreign or `unsafe extern "C"` functions may be C-variadic
     //~| ERROR `...` must be the last argument of a C-variadic function
     fn i_f4(..., x: isize, ...) {}
     //~^ ERROR only foreign or `unsafe extern "C"` functions may be C-variadic
-    //~| ERROR only foreign or `unsafe extern "C"` functions may be C-variadic
     //~| ERROR `...` must be the last argument of a C-variadic function
 }
 
diff --git a/tests/ui/parser/variadic-ffi-semantic-restrictions.stderr b/tests/ui/parser/variadic-ffi-semantic-restrictions.stderr
index f1cbbb279c84..95e381f06fc4 100644
--- a/tests/ui/parser/variadic-ffi-semantic-restrictions.stderr
+++ b/tests/ui/parser/variadic-ffi-semantic-restrictions.stderr
@@ -116,91 +116,79 @@ error: only foreign or `unsafe extern "C"` functions may be C-variadic
   --> $DIR/variadic-ffi-semantic-restrictions.rs:50:13
    |
 LL |     fn i_f3(..., x: isize, ...) {}
-   |             ^^^
-
-error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:50:28
-   |
-LL |     fn i_f3(..., x: isize, ...) {}
-   |                            ^^^
+   |             ^^^            ^^^
 
 error: `...` must be the last argument of a C-variadic function
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:54:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:53:13
    |
 LL |     fn i_f4(..., x: isize, ...) {}
    |             ^^^
 
 error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:54:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:53:13
    |
 LL |     fn i_f4(..., x: isize, ...) {}
-   |             ^^^
+   |             ^^^            ^^^
 
 error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:54:28
-   |
-LL |     fn i_f4(..., x: isize, ...) {}
-   |                            ^^^
-
-error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:61:23
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:59:23
    |
 LL |     fn t_f1(x: isize, ...) {}
    |                       ^^^
 
 error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:63:23
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:61:23
    |
 LL |     fn t_f2(x: isize, ...);
    |                       ^^^
 
 error: C-variadic function must be declared with at least one named argument
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:65:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:63:13
    |
 LL |     fn t_f3(...) {}
    |             ^^^
 
 error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:65:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:63:13
    |
 LL |     fn t_f3(...) {}
    |             ^^^
 
 error: C-variadic function must be declared with at least one named argument
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:68:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:66:13
    |
 LL |     fn t_f4(...);
    |             ^^^
 
 error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:68:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:66:13
    |
 LL |     fn t_f4(...);
    |             ^^^
 
 error: `...` must be the last argument of a C-variadic function
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:71:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:69:13
    |
 LL |     fn t_f5(..., x: isize) {}
    |             ^^^
 
 error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:71:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:69:13
    |
 LL |     fn t_f5(..., x: isize) {}
    |             ^^^
 
 error: `...` must be the last argument of a C-variadic function
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:74:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:72:13
    |
 LL |     fn t_f6(..., x: isize);
    |             ^^^
 
 error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:74:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:72:13
    |
 LL |     fn t_f6(..., x: isize);
    |             ^^^
 
-error: aborting due to 34 previous errors
+error: aborting due to 32 previous errors
 

From d5e836cf0ce49761524e9c8ffb121e186e774052 Mon Sep 17 00:00:00 2001
From: Nadrieril <nadrieril+git@gmail.com>
Date: Mon, 30 Oct 2023 15:31:00 +0100
Subject: [PATCH 247/297] Correctly handle nested or-patterns in column-wise
 analyses

---
 compiler/rustc_mir_build/src/thir/pattern/usefulness.rs | 2 +-
 tests/ui/or-patterns/exhaustiveness-pass.rs             | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
index 25e0f3ceaa47..59f0849d08b3 100644
--- a/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
+++ b/compiler/rustc_mir_build/src/thir/pattern/usefulness.rs
@@ -931,7 +931,7 @@ impl<'p, 'tcx> PatternColumn<'p, 'tcx> {
             let specialized = pat.specialize(pcx, &ctor);
             for (subpat, column) in specialized.iter().zip(&mut specialized_columns) {
                 if subpat.is_or_pat() {
-                    column.patterns.extend(subpat.iter_fields())
+                    column.patterns.extend(subpat.flatten_or_pat())
                 } else {
                     column.patterns.push(subpat)
                 }
diff --git a/tests/ui/or-patterns/exhaustiveness-pass.rs b/tests/ui/or-patterns/exhaustiveness-pass.rs
index e8c8a0e7ba52..428b9a19fe69 100644
--- a/tests/ui/or-patterns/exhaustiveness-pass.rs
+++ b/tests/ui/or-patterns/exhaustiveness-pass.rs
@@ -35,4 +35,10 @@ fn main() {
         ((0, 0) | (1, 0),) => {}
         _ => {}
     }
+
+    // This one caused ICE https://github.com/rust-lang/rust/issues/117378
+    match (0u8, 0) {
+        (x @ 0 | x @ (1 | 2), _) => {}
+        (3.., _) => {}
+    }
 }

From f91b5ceaf2568c38afc5a4b7c82459ca3d857998 Mon Sep 17 00:00:00 2001
From: Nicholas Bishop <nbishop@nbishop.net>
Date: Sun, 29 Oct 2023 17:28:56 -0400
Subject: [PATCH 248/297] Explicitly reject const C-variadic functions

Trying to use C-variadics in a const function would previously fail with
an error like "destructor of `VaListImpl<'_>` cannot be evaluated at
compile-time".

Add an explicit check for const C-variadics to provide a clearer error:
"functions cannot be both `const` and C-variadic".
---
 compiler/rustc_ast_passes/messages.ftl        |  4 +
 .../rustc_ast_passes/src/ast_validation.rs    | 19 +++-
 compiler/rustc_ast_passes/src/errors.rs       | 11 +++
 .../variadic-ffi-semantic-restrictions.rs     | 15 +++
 .../variadic-ffi-semantic-restrictions.stderr | 93 +++++++++++++++----
 5 files changed, 120 insertions(+), 22 deletions(-)

diff --git a/compiler/rustc_ast_passes/messages.ftl b/compiler/rustc_ast_passes/messages.ftl
index 43020a930788..d22bae816ef4 100644
--- a/compiler/rustc_ast_passes/messages.ftl
+++ b/compiler/rustc_ast_passes/messages.ftl
@@ -42,6 +42,10 @@ ast_passes_const_and_async = functions cannot be both `const` and `async`
     .async = `async` because of this
     .label = {""}
 
+ast_passes_const_and_c_variadic = functions cannot be both `const` and C-variadic
+    .const = `const` because of this
+    .variadic = C-variadic because of this
+
 ast_passes_const_without_body =
     free constant item without body
     .suggestion = provide a definition for the constant
diff --git a/compiler/rustc_ast_passes/src/ast_validation.rs b/compiler/rustc_ast_passes/src/ast_validation.rs
index f59f7d389cda..477d3732d464 100644
--- a/compiler/rustc_ast_passes/src/ast_validation.rs
+++ b/compiler/rustc_ast_passes/src/ast_validation.rs
@@ -482,8 +482,11 @@ impl<'a> AstValidator<'a> {
         }
     }
 
-    /// Reject C-variadic type unless the function is foreign,
-    /// or free and `unsafe extern "C"` semantically.
+    /// Reject invalid C-variadic types.
+    ///
+    /// C-variadics must be:
+    /// - Non-const
+    /// - Either foreign, or free and `unsafe extern "C"` semantically
     fn check_c_variadic_type(&self, fk: FnKind<'a>) {
         let variadic_spans: Vec<_> = fk
             .decl()
@@ -497,6 +500,18 @@ impl<'a> AstValidator<'a> {
             return;
         }
 
+        if let Some(header) = fk.header() {
+            if let Const::Yes(const_span) = header.constness {
+                let mut spans = variadic_spans.clone();
+                spans.push(const_span);
+                self.err_handler().emit_err(errors::ConstAndCVariadic {
+                    spans,
+                    const_span,
+                    variadic_spans: variadic_spans.clone(),
+                });
+            }
+        }
+
         match (fk.ctxt(), fk.header()) {
             (Some(FnCtxt::Foreign), _) => return,
             (Some(FnCtxt::Free), Some(header)) => match header.ext {
diff --git a/compiler/rustc_ast_passes/src/errors.rs b/compiler/rustc_ast_passes/src/errors.rs
index 579f528e20f9..d14b62d6bdc6 100644
--- a/compiler/rustc_ast_passes/src/errors.rs
+++ b/compiler/rustc_ast_passes/src/errors.rs
@@ -583,6 +583,17 @@ pub struct ConstAndAsync {
     pub span: Span,
 }
 
+#[derive(Diagnostic)]
+#[diag(ast_passes_const_and_c_variadic)]
+pub struct ConstAndCVariadic {
+    #[primary_span]
+    pub spans: Vec<Span>,
+    #[label(ast_passes_const)]
+    pub const_span: Span,
+    #[label(ast_passes_variadic)]
+    pub variadic_spans: Vec<Span>,
+}
+
 #[derive(Diagnostic)]
 #[diag(ast_passes_pattern_in_foreign, code = "E0130")]
 pub struct PatternInForeign {
diff --git a/tests/ui/parser/variadic-ffi-semantic-restrictions.rs b/tests/ui/parser/variadic-ffi-semantic-restrictions.rs
index 8e382314ca27..b173e23e7a18 100644
--- a/tests/ui/parser/variadic-ffi-semantic-restrictions.rs
+++ b/tests/ui/parser/variadic-ffi-semantic-restrictions.rs
@@ -32,6 +32,18 @@ extern "C" fn f3_3(..., x: isize) {}
 //~^ ERROR only foreign or `unsafe extern "C"` functions may be C-variadic
 //~| ERROR `...` must be the last argument of a C-variadic function
 
+const unsafe extern "C" fn f4_1(x: isize, ...) {}
+//~^ ERROR functions cannot be both `const` and C-variadic
+
+const extern "C" fn f4_2(x: isize, ...) {}
+//~^ ERROR functions cannot be both `const` and C-variadic
+//~| ERROR only foreign or `unsafe extern "C"` functions may be C-variadic
+
+const extern "C" fn f4_3(..., x: isize, ...) {}
+//~^ ERROR functions cannot be both `const` and C-variadic
+//~| ERROR only foreign or `unsafe extern "C"` functions may be C-variadic
+//~| ERROR `...` must be the last argument of a C-variadic function
+
 extern "C" {
     fn e_f1(...);
     //~^ ERROR C-variadic function must be declared with at least one named argument
@@ -53,6 +65,9 @@ impl X {
     fn i_f4(..., x: isize, ...) {}
     //~^ ERROR only foreign or `unsafe extern "C"` functions may be C-variadic
     //~| ERROR `...` must be the last argument of a C-variadic function
+    const fn i_f5(x: isize, ...) {}
+    //~^ ERROR only foreign or `unsafe extern "C"` functions may be C-variadic
+    //~| ERROR functions cannot be both `const` and C-variadic
 }
 
 trait T {
diff --git a/tests/ui/parser/variadic-ffi-semantic-restrictions.stderr b/tests/ui/parser/variadic-ffi-semantic-restrictions.stderr
index 95e381f06fc4..18526080e4cb 100644
--- a/tests/ui/parser/variadic-ffi-semantic-restrictions.stderr
+++ b/tests/ui/parser/variadic-ffi-semantic-restrictions.stderr
@@ -76,119 +76,172 @@ error: only foreign or `unsafe extern "C"` functions may be C-variadic
 LL | extern "C" fn f3_3(..., x: isize) {}
    |                    ^^^
 
+error: functions cannot be both `const` and C-variadic
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:35:1
+   |
+LL | const unsafe extern "C" fn f4_1(x: isize, ...) {}
+   | ^^^^^ `const` because of this             ^^^ C-variadic because of this
+
+error: functions cannot be both `const` and C-variadic
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:38:1
+   |
+LL | const extern "C" fn f4_2(x: isize, ...) {}
+   | ^^^^^ `const` because of this      ^^^ C-variadic because of this
+
+error: only foreign or `unsafe extern "C"` functions may be C-variadic
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:38:36
+   |
+LL | const extern "C" fn f4_2(x: isize, ...) {}
+   |                                    ^^^
+
+error: `...` must be the last argument of a C-variadic function
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:42:26
+   |
+LL | const extern "C" fn f4_3(..., x: isize, ...) {}
+   |                          ^^^
+
+error: functions cannot be both `const` and C-variadic
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:42:1
+   |
+LL | const extern "C" fn f4_3(..., x: isize, ...) {}
+   | ^^^^^                    ^^^            ^^^ C-variadic because of this
+   | |                        |
+   | |                        C-variadic because of this
+   | `const` because of this
+
+error: only foreign or `unsafe extern "C"` functions may be C-variadic
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:42:26
+   |
+LL | const extern "C" fn f4_3(..., x: isize, ...) {}
+   |                          ^^^            ^^^
+
 error: C-variadic function must be declared with at least one named argument
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:36:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:48:13
    |
 LL |     fn e_f1(...);
    |             ^^^
 
 error: `...` must be the last argument of a C-variadic function
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:38:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:50:13
    |
 LL |     fn e_f2(..., x: isize);
    |             ^^^
 
 error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:45:23
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:57:23
    |
 LL |     fn i_f1(x: isize, ...) {}
    |                       ^^^
 
 error: C-variadic function must be declared with at least one named argument
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:47:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:59:13
    |
 LL |     fn i_f2(...) {}
    |             ^^^
 
 error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:47:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:59:13
    |
 LL |     fn i_f2(...) {}
    |             ^^^
 
 error: `...` must be the last argument of a C-variadic function
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:50:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:62:13
    |
 LL |     fn i_f3(..., x: isize, ...) {}
    |             ^^^
 
 error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:50:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:62:13
    |
 LL |     fn i_f3(..., x: isize, ...) {}
    |             ^^^            ^^^
 
 error: `...` must be the last argument of a C-variadic function
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:53:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:65:13
    |
 LL |     fn i_f4(..., x: isize, ...) {}
    |             ^^^
 
 error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:53:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:65:13
    |
 LL |     fn i_f4(..., x: isize, ...) {}
    |             ^^^            ^^^
 
+error: functions cannot be both `const` and C-variadic
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:68:5
+   |
+LL |     const fn i_f5(x: isize, ...) {}
+   |     ^^^^^                   ^^^ C-variadic because of this
+   |     |
+   |     `const` because of this
+
 error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:59:23
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:68:29
+   |
+LL |     const fn i_f5(x: isize, ...) {}
+   |                             ^^^
+
+error: only foreign or `unsafe extern "C"` functions may be C-variadic
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:74:23
    |
 LL |     fn t_f1(x: isize, ...) {}
    |                       ^^^
 
 error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:61:23
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:76:23
    |
 LL |     fn t_f2(x: isize, ...);
    |                       ^^^
 
 error: C-variadic function must be declared with at least one named argument
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:63:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:78:13
    |
 LL |     fn t_f3(...) {}
    |             ^^^
 
 error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:63:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:78:13
    |
 LL |     fn t_f3(...) {}
    |             ^^^
 
 error: C-variadic function must be declared with at least one named argument
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:66:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:81:13
    |
 LL |     fn t_f4(...);
    |             ^^^
 
 error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:66:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:81:13
    |
 LL |     fn t_f4(...);
    |             ^^^
 
 error: `...` must be the last argument of a C-variadic function
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:69:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:84:13
    |
 LL |     fn t_f5(..., x: isize) {}
    |             ^^^
 
 error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:69:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:84:13
    |
 LL |     fn t_f5(..., x: isize) {}
    |             ^^^
 
 error: `...` must be the last argument of a C-variadic function
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:72:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:87:13
    |
 LL |     fn t_f6(..., x: isize);
    |             ^^^
 
 error: only foreign or `unsafe extern "C"` functions may be C-variadic
-  --> $DIR/variadic-ffi-semantic-restrictions.rs:72:13
+  --> $DIR/variadic-ffi-semantic-restrictions.rs:87:13
    |
 LL |     fn t_f6(..., x: isize);
    |             ^^^
 
-error: aborting due to 32 previous errors
+error: aborting due to 40 previous errors
 

From 147c4a50e0373dc29b8f3bd8d18a21407d9c8fa9 Mon Sep 17 00:00:00 2001
From: yukang <moorekang@gmail.com>
Date: Mon, 30 Oct 2023 23:33:48 +0800
Subject: [PATCH 249/297] Refactor: move suggestion functions from demand to
 suggestions

---
 compiler/rustc_hir_typeck/src/demand.rs       | 1276 +----------------
 .../src/fn_ctxt/suggestions.rs                | 1269 ++++++++++++++++
 2 files changed, 1272 insertions(+), 1273 deletions(-)

diff --git a/compiler/rustc_hir_typeck/src/demand.rs b/compiler/rustc_hir_typeck/src/demand.rs
index abb832280301..acaa3e02f092 100644
--- a/compiler/rustc_hir_typeck/src/demand.rs
+++ b/compiler/rustc_hir_typeck/src/demand.rs
@@ -1,30 +1,21 @@
 use crate::FnCtxt;
-use rustc_ast::util::parser::PREC_POSTFIX;
 use rustc_errors::MultiSpan;
 use rustc_errors::{Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed};
 use rustc_hir as hir;
-use rustc_hir::def::{CtorKind, Res};
+use rustc_hir::def::Res;
 use rustc_hir::intravisit::Visitor;
-use rustc_hir::lang_items::LangItem;
-use rustc_hir::{is_range_literal, Node};
 use rustc_infer::infer::{DefineOpaqueTypes, InferOk};
-use rustc_middle::lint::in_external_macro;
-use rustc_middle::middle::stability::EvalResult;
 use rustc_middle::ty::adjustment::AllowTwoPhase;
 use rustc_middle::ty::error::{ExpectedFound, TypeError};
 use rustc_middle::ty::fold::BottomUpFolder;
 use rustc_middle::ty::print::with_no_trimmed_paths;
-use rustc_middle::ty::{self, Article, AssocItem, Ty, TypeAndMut, TypeFoldable, TypeVisitableExt};
+use rustc_middle::ty::{self, AssocItem, Ty, TypeFoldable, TypeVisitableExt};
 use rustc_span::symbol::sym;
-use rustc_span::{BytePos, Span, DUMMY_SP};
-use rustc_trait_selection::infer::InferCtxtExt as _;
+use rustc_span::{Span, DUMMY_SP};
 use rustc_trait_selection::traits::ObligationCause;
 
 use super::method::probe;
 
-use std::cmp::min;
-use std::iter;
-
 impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
     pub fn emit_type_mismatch_suggestions(
         &self,
@@ -955,301 +946,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         );
     }
 
-    pub(crate) fn suggest_coercing_result_via_try_operator(
-        &self,
-        err: &mut Diagnostic,
-        expr: &hir::Expr<'tcx>,
-        expected: Ty<'tcx>,
-        found: Ty<'tcx>,
-    ) -> bool {
-        let map = self.tcx.hir();
-        let returned = matches!(
-            map.find_parent(expr.hir_id),
-            Some(hir::Node::Expr(hir::Expr { kind: hir::ExprKind::Ret(_), .. }))
-        ) || map.get_return_block(expr.hir_id).is_some();
-        if returned
-            && let ty::Adt(e, args_e) = expected.kind()
-            && let ty::Adt(f, args_f) = found.kind()
-            && e.did() == f.did()
-            && Some(e.did()) == self.tcx.get_diagnostic_item(sym::Result)
-            && let e_ok = args_e.type_at(0)
-            && let f_ok = args_f.type_at(0)
-            && self.infcx.can_eq(self.param_env, f_ok, e_ok)
-            && let e_err = args_e.type_at(1)
-            && let f_err = args_f.type_at(1)
-            && self
-                .infcx
-                .type_implements_trait(
-                    self.tcx.get_diagnostic_item(sym::Into).unwrap(),
-                    [f_err, e_err],
-                    self.param_env,
-                )
-                .must_apply_modulo_regions()
-        {
-            err.multipart_suggestion(
-                "use `?` to coerce and return an appropriate `Err`, and wrap the resulting value \
-                 in `Ok` so the expression remains of type `Result`",
-                vec![
-                    (expr.span.shrink_to_lo(), "Ok(".to_string()),
-                    (expr.span.shrink_to_hi(), "?)".to_string()),
-                ],
-                Applicability::MaybeIncorrect,
-            );
-            return true;
-        }
-        false
-    }
-
-    /// If the expected type is an enum (Issue #55250) with any variants whose
-    /// sole field is of the found type, suggest such variants. (Issue #42764)
-    fn suggest_compatible_variants(
-        &self,
-        err: &mut Diagnostic,
-        expr: &hir::Expr<'_>,
-        expected: Ty<'tcx>,
-        expr_ty: Ty<'tcx>,
-    ) -> bool {
-        if in_external_macro(self.tcx.sess, expr.span) {
-            return false;
-        }
-        if let ty::Adt(expected_adt, args) = expected.kind() {
-            if let hir::ExprKind::Field(base, ident) = expr.kind {
-                let base_ty = self.typeck_results.borrow().expr_ty(base);
-                if self.can_eq(self.param_env, base_ty, expected)
-                    && let Some(base_span) = base.span.find_ancestor_inside(expr.span)
-                {
-                    err.span_suggestion_verbose(
-                        expr.span.with_lo(base_span.hi()),
-                        format!("consider removing the tuple struct field `{ident}`"),
-                        "",
-                        Applicability::MaybeIncorrect,
-                    );
-                    return true;
-                }
-            }
-
-            // If the expression is of type () and it's the return expression of a block,
-            // we suggest adding a separate return expression instead.
-            // (To avoid things like suggesting `Ok(while .. { .. })`.)
-            if expr_ty.is_unit() {
-                let mut id = expr.hir_id;
-                let mut parent;
-
-                // Unroll desugaring, to make sure this works for `for` loops etc.
-                loop {
-                    parent = self.tcx.hir().parent_id(id);
-                    if let Some(parent_span) = self.tcx.hir().opt_span(parent) {
-                        if parent_span.find_ancestor_inside(expr.span).is_some() {
-                            // The parent node is part of the same span, so is the result of the
-                            // same expansion/desugaring and not the 'real' parent node.
-                            id = parent;
-                            continue;
-                        }
-                    }
-                    break;
-                }
-
-                if let Some(hir::Node::Block(&hir::Block {
-                    span: block_span, expr: Some(e), ..
-                })) = self.tcx.hir().find(parent)
-                {
-                    if e.hir_id == id {
-                        if let Some(span) = expr.span.find_ancestor_inside(block_span) {
-                            let return_suggestions = if self
-                                .tcx
-                                .is_diagnostic_item(sym::Result, expected_adt.did())
-                            {
-                                vec!["Ok(())"]
-                            } else if self.tcx.is_diagnostic_item(sym::Option, expected_adt.did()) {
-                                vec!["None", "Some(())"]
-                            } else {
-                                return false;
-                            };
-                            if let Some(indent) =
-                                self.tcx.sess.source_map().indentation_before(span.shrink_to_lo())
-                            {
-                                // Add a semicolon, except after `}`.
-                                let semicolon =
-                                    match self.tcx.sess.source_map().span_to_snippet(span) {
-                                        Ok(s) if s.ends_with('}') => "",
-                                        _ => ";",
-                                    };
-                                err.span_suggestions(
-                                    span.shrink_to_hi(),
-                                    "try adding an expression at the end of the block",
-                                    return_suggestions
-                                        .into_iter()
-                                        .map(|r| format!("{semicolon}\n{indent}{r}")),
-                                    Applicability::MaybeIncorrect,
-                                );
-                            }
-                            return true;
-                        }
-                    }
-                }
-            }
-
-            let compatible_variants: Vec<(String, _, _, Option<String>)> = expected_adt
-                .variants()
-                .iter()
-                .filter(|variant| {
-                    variant.fields.len() == 1
-                })
-                .filter_map(|variant| {
-                    let sole_field = &variant.single_field();
-
-                    let field_is_local = sole_field.did.is_local();
-                    let field_is_accessible =
-                        sole_field.vis.is_accessible_from(expr.hir_id.owner.def_id, self.tcx)
-                        // Skip suggestions for unstable public fields (for example `Pin::pointer`)
-                        && matches!(self.tcx.eval_stability(sole_field.did, None, expr.span, None), EvalResult::Allow | EvalResult::Unmarked);
-
-                    if !field_is_local && !field_is_accessible {
-                        return None;
-                    }
-
-                    let note_about_variant_field_privacy = (field_is_local && !field_is_accessible)
-                        .then(|| " (its field is private, but it's local to this crate and its privacy can be changed)".to_string());
-
-                    let sole_field_ty = sole_field.ty(self.tcx, args);
-                    if self.can_coerce(expr_ty, sole_field_ty) {
-                        let variant_path =
-                            with_no_trimmed_paths!(self.tcx.def_path_str(variant.def_id));
-                        // FIXME #56861: DRYer prelude filtering
-                        if let Some(path) = variant_path.strip_prefix("std::prelude::")
-                            && let Some((_, path)) = path.split_once("::")
-                        {
-                            return Some((path.to_string(), variant.ctor_kind(), sole_field.name, note_about_variant_field_privacy));
-                        }
-                        Some((variant_path, variant.ctor_kind(), sole_field.name, note_about_variant_field_privacy))
-                    } else {
-                        None
-                    }
-                })
-                .collect();
-
-            let suggestions_for = |variant: &_, ctor_kind, field_name| {
-                let prefix = match self.tcx.hir().maybe_get_struct_pattern_shorthand_field(expr) {
-                    Some(ident) => format!("{ident}: "),
-                    None => String::new(),
-                };
-
-                let (open, close) = match ctor_kind {
-                    Some(CtorKind::Fn) => ("(".to_owned(), ")"),
-                    None => (format!(" {{ {field_name}: "), " }"),
-
-                    // unit variants don't have fields
-                    Some(CtorKind::Const) => unreachable!(),
-                };
-
-                // Suggest constructor as deep into the block tree as possible.
-                // This fixes https://github.com/rust-lang/rust/issues/101065,
-                // and also just helps make the most minimal suggestions.
-                let mut expr = expr;
-                while let hir::ExprKind::Block(block, _) = &expr.kind
-                    && let Some(expr_) = &block.expr
-                {
-                    expr = expr_
-                }
-
-                vec![
-                    (expr.span.shrink_to_lo(), format!("{prefix}{variant}{open}")),
-                    (expr.span.shrink_to_hi(), close.to_owned()),
-                ]
-            };
-
-            match &compatible_variants[..] {
-                [] => { /* No variants to format */ }
-                [(variant, ctor_kind, field_name, note)] => {
-                    // Just a single matching variant.
-                    err.multipart_suggestion_verbose(
-                        format!(
-                            "try wrapping the expression in `{variant}`{note}",
-                            note = note.as_deref().unwrap_or("")
-                        ),
-                        suggestions_for(&**variant, *ctor_kind, *field_name),
-                        Applicability::MaybeIncorrect,
-                    );
-                    return true;
-                }
-                _ => {
-                    // More than one matching variant.
-                    err.multipart_suggestions(
-                        format!(
-                            "try wrapping the expression in a variant of `{}`",
-                            self.tcx.def_path_str(expected_adt.did())
-                        ),
-                        compatible_variants.into_iter().map(
-                            |(variant, ctor_kind, field_name, _)| {
-                                suggestions_for(&variant, ctor_kind, field_name)
-                            },
-                        ),
-                        Applicability::MaybeIncorrect,
-                    );
-                    return true;
-                }
-            }
-        }
-
-        false
-    }
-
-    fn suggest_non_zero_new_unwrap(
-        &self,
-        err: &mut Diagnostic,
-        expr: &hir::Expr<'_>,
-        expected: Ty<'tcx>,
-        expr_ty: Ty<'tcx>,
-    ) -> bool {
-        let tcx = self.tcx;
-        let (adt, unwrap) = match expected.kind() {
-            // In case Option<NonZero*> is wanted, but * is provided, suggest calling new
-            ty::Adt(adt, args) if tcx.is_diagnostic_item(sym::Option, adt.did()) => {
-                // Unwrap option
-                let ty::Adt(adt, _) = args.type_at(0).kind() else {
-                    return false;
-                };
-
-                (adt, "")
-            }
-            // In case NonZero* is wanted, but * is provided also add `.unwrap()` to satisfy types
-            ty::Adt(adt, _) => (adt, ".unwrap()"),
-            _ => return false,
-        };
-
-        let map = [
-            (sym::NonZeroU8, tcx.types.u8),
-            (sym::NonZeroU16, tcx.types.u16),
-            (sym::NonZeroU32, tcx.types.u32),
-            (sym::NonZeroU64, tcx.types.u64),
-            (sym::NonZeroU128, tcx.types.u128),
-            (sym::NonZeroI8, tcx.types.i8),
-            (sym::NonZeroI16, tcx.types.i16),
-            (sym::NonZeroI32, tcx.types.i32),
-            (sym::NonZeroI64, tcx.types.i64),
-            (sym::NonZeroI128, tcx.types.i128),
-        ];
-
-        let Some((s, _)) = map.iter().find(|&&(s, t)| {
-            self.tcx.is_diagnostic_item(s, adt.did()) && self.can_coerce(expr_ty, t)
-        }) else {
-            return false;
-        };
-
-        let path = self.tcx.def_path_str(adt.non_enum_variant().def_id);
-
-        err.multipart_suggestion(
-            format!("consider calling `{s}::new`"),
-            vec![
-                (expr.span.shrink_to_lo(), format!("{path}::new(")),
-                (expr.span.shrink_to_hi(), format!("){unwrap}")),
-            ],
-            Applicability::MaybeIncorrect,
-        );
-
-        true
-    }
-
     pub fn get_conversion_methods(
         &self,
         span: Span,
@@ -1296,82 +992,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
     }
 
-    /// Identify some cases where `as_ref()` would be appropriate and suggest it.
-    ///
-    /// Given the following code:
-    /// ```compile_fail,E0308
-    /// struct Foo;
-    /// fn takes_ref(_: &Foo) {}
-    /// let ref opt = Some(Foo);
-    ///
-    /// opt.map(|param| takes_ref(param));
-    /// ```
-    /// Suggest using `opt.as_ref().map(|param| takes_ref(param));` instead.
-    ///
-    /// It only checks for `Option` and `Result` and won't work with
-    /// ```ignore (illustrative)
-    /// opt.map(|param| { takes_ref(param) });
-    /// ```
-    fn can_use_as_ref(&self, expr: &hir::Expr<'_>) -> Option<(Vec<(Span, String)>, &'static str)> {
-        let hir::ExprKind::Path(hir::QPath::Resolved(_, ref path)) = expr.kind else {
-            return None;
-        };
-
-        let hir::def::Res::Local(local_id) = path.res else {
-            return None;
-        };
-
-        let local_parent = self.tcx.hir().parent_id(local_id);
-        let Some(Node::Param(hir::Param { hir_id: param_hir_id, .. })) =
-            self.tcx.hir().find(local_parent)
-        else {
-            return None;
-        };
-
-        let param_parent = self.tcx.hir().parent_id(*param_hir_id);
-        let Some(Node::Expr(hir::Expr {
-            hir_id: expr_hir_id,
-            kind: hir::ExprKind::Closure(hir::Closure { fn_decl: closure_fn_decl, .. }),
-            ..
-        })) = self.tcx.hir().find(param_parent)
-        else {
-            return None;
-        };
-
-        let expr_parent = self.tcx.hir().parent_id(*expr_hir_id);
-        let hir = self.tcx.hir().find(expr_parent);
-        let closure_params_len = closure_fn_decl.inputs.len();
-        let (
-            Some(Node::Expr(hir::Expr {
-                kind: hir::ExprKind::MethodCall(method_path, receiver, ..),
-                ..
-            })),
-            1,
-        ) = (hir, closure_params_len)
-        else {
-            return None;
-        };
-
-        let self_ty = self.typeck_results.borrow().expr_ty(receiver);
-        let name = method_path.ident.name;
-        let is_as_ref_able = match self_ty.peel_refs().kind() {
-            ty::Adt(def, _) => {
-                (self.tcx.is_diagnostic_item(sym::Option, def.did())
-                    || self.tcx.is_diagnostic_item(sym::Result, def.did()))
-                    && (name == sym::map || name == sym::and_then)
-            }
-            _ => false,
-        };
-        if is_as_ref_able {
-            Some((
-                vec![(method_path.ident.span.shrink_to_lo(), "as_ref().".to_string())],
-                "consider using `as_ref` instead",
-            ))
-        } else {
-            None
-        }
-    }
-
     /// If the given `HirId` corresponds to a block with a trailing expression, return that expression
     pub(crate) fn maybe_get_block_expr(
         &self,
@@ -1383,21 +1003,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         }
     }
 
-    /// Returns whether the given expression is an `else if`.
-    pub(crate) fn is_else_if_block(&self, expr: &hir::Expr<'_>) -> bool {
-        if let hir::ExprKind::If(..) = expr.kind {
-            let parent_id = self.tcx.hir().parent_id(expr.hir_id);
-            if let Some(Node::Expr(hir::Expr {
-                kind: hir::ExprKind::If(_, _, Some(else_expr)),
-                ..
-            })) = self.tcx.hir().find(parent_id)
-            {
-                return else_expr.hir_id == expr.hir_id;
-            }
-        }
-        false
-    }
-
     // Returns whether the given expression is a destruct assignment desugaring.
     // For example, `(a, b) = (1, &2);`
     // Here we try to find the pattern binding of the expression,
@@ -1422,881 +1027,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         return false;
     }
 
-    /// This function is used to determine potential "simple" improvements or users' errors and
-    /// provide them useful help. For example:
-    ///
-    /// ```compile_fail,E0308
-    /// fn some_fn(s: &str) {}
-    ///
-    /// let x = "hey!".to_owned();
-    /// some_fn(x); // error
-    /// ```
-    ///
-    /// No need to find every potential function which could make a coercion to transform a
-    /// `String` into a `&str` since a `&` would do the trick!
-    ///
-    /// In addition of this check, it also checks between references mutability state. If the
-    /// expected is mutable but the provided isn't, maybe we could just say "Hey, try with
-    /// `&mut`!".
-    pub fn suggest_deref_or_ref(
-        &self,
-        expr: &hir::Expr<'tcx>,
-        checked_ty: Ty<'tcx>,
-        expected: Ty<'tcx>,
-    ) -> Option<(
-        Vec<(Span, String)>,
-        String,
-        Applicability,
-        bool, /* verbose */
-        bool, /* suggest `&` or `&mut` type annotation */
-    )> {
-        let sess = self.sess();
-        let sp = expr.span;
-
-        // If the span is from an external macro, there's no suggestion we can make.
-        if in_external_macro(sess, sp) {
-            return None;
-        }
-
-        let sm = sess.source_map();
-
-        let replace_prefix = |s: &str, old: &str, new: &str| {
-            s.strip_prefix(old).map(|stripped| new.to_string() + stripped)
-        };
-
-        // `ExprKind::DropTemps` is semantically irrelevant for these suggestions.
-        let expr = expr.peel_drop_temps();
-
-        match (&expr.kind, expected.kind(), checked_ty.kind()) {
-            (_, &ty::Ref(_, exp, _), &ty::Ref(_, check, _)) => match (exp.kind(), check.kind()) {
-                (&ty::Str, &ty::Array(arr, _) | &ty::Slice(arr)) if arr == self.tcx.types.u8 => {
-                    if let hir::ExprKind::Lit(_) = expr.kind
-                        && let Ok(src) = sm.span_to_snippet(sp)
-                        && replace_prefix(&src, "b\"", "\"").is_some()
-                    {
-                        let pos = sp.lo() + BytePos(1);
-                        return Some((
-                            vec![(sp.with_hi(pos), String::new())],
-                            "consider removing the leading `b`".to_string(),
-                            Applicability::MachineApplicable,
-                            true,
-                            false,
-                        ));
-                    }
-                }
-                (&ty::Array(arr, _) | &ty::Slice(arr), &ty::Str) if arr == self.tcx.types.u8 => {
-                    if let hir::ExprKind::Lit(_) = expr.kind
-                        && let Ok(src) = sm.span_to_snippet(sp)
-                        && replace_prefix(&src, "\"", "b\"").is_some()
-                    {
-                        return Some((
-                            vec![(sp.shrink_to_lo(), "b".to_string())],
-                            "consider adding a leading `b`".to_string(),
-                            Applicability::MachineApplicable,
-                            true,
-                            false,
-                        ));
-                    }
-                }
-                _ => {}
-            },
-            (_, &ty::Ref(_, _, mutability), _) => {
-                // Check if it can work when put into a ref. For example:
-                //
-                // ```
-                // fn bar(x: &mut i32) {}
-                //
-                // let x = 0u32;
-                // bar(&x); // error, expected &mut
-                // ```
-                let ref_ty = match mutability {
-                    hir::Mutability::Mut => {
-                        Ty::new_mut_ref(self.tcx, self.tcx.lifetimes.re_static, checked_ty)
-                    }
-                    hir::Mutability::Not => {
-                        Ty::new_imm_ref(self.tcx, self.tcx.lifetimes.re_static, checked_ty)
-                    }
-                };
-                if self.can_coerce(ref_ty, expected) {
-                    let mut sugg_sp = sp;
-                    if let hir::ExprKind::MethodCall(ref segment, receiver, args, _) = expr.kind {
-                        let clone_trait =
-                            self.tcx.require_lang_item(LangItem::Clone, Some(segment.ident.span));
-                        if args.is_empty()
-                            && self.typeck_results.borrow().type_dependent_def_id(expr.hir_id).map(
-                                |did| {
-                                    let ai = self.tcx.associated_item(did);
-                                    ai.trait_container(self.tcx) == Some(clone_trait)
-                                },
-                            ) == Some(true)
-                            && segment.ident.name == sym::clone
-                        {
-                            // If this expression had a clone call when suggesting borrowing
-                            // we want to suggest removing it because it'd now be unnecessary.
-                            sugg_sp = receiver.span;
-                        }
-                    }
-
-                    if let hir::ExprKind::Unary(hir::UnOp::Deref, ref inner) = expr.kind
-                        && let Some(1) = self.deref_steps(expected, checked_ty)
-                    {
-                        // We have `*&T`, check if what was expected was `&T`.
-                        // If so, we may want to suggest removing a `*`.
-                        sugg_sp = sugg_sp.with_hi(inner.span.lo());
-                        return Some((
-                            vec![(sugg_sp, String::new())],
-                            "consider removing deref here".to_string(),
-                            Applicability::MachineApplicable,
-                            true,
-                            false,
-                        ));
-                    }
-
-                    let needs_parens = match expr.kind {
-                        // parenthesize if needed (Issue #46756)
-                        hir::ExprKind::Cast(_, _) | hir::ExprKind::Binary(_, _, _) => true,
-                        // parenthesize borrows of range literals (Issue #54505)
-                        _ if is_range_literal(expr) => true,
-                        _ => false,
-                    };
-
-                    if let Some((sugg, msg)) = self.can_use_as_ref(expr) {
-                        return Some((
-                            sugg,
-                            msg.to_string(),
-                            Applicability::MachineApplicable,
-                            true,
-                            false,
-                        ));
-                    }
-
-                    let prefix = match self.tcx.hir().maybe_get_struct_pattern_shorthand_field(expr)
-                    {
-                        Some(ident) => format!("{ident}: "),
-                        None => String::new(),
-                    };
-
-                    if let Some(hir::Node::Expr(hir::Expr {
-                        kind: hir::ExprKind::Assign(..),
-                        ..
-                    })) = self.tcx.hir().find_parent(expr.hir_id)
-                    {
-                        if mutability.is_mut() {
-                            // Suppressing this diagnostic, we'll properly print it in `check_expr_assign`
-                            return None;
-                        }
-                    }
-
-                    let sugg = mutability.ref_prefix_str();
-                    let (sugg, verbose) = if needs_parens {
-                        (
-                            vec![
-                                (sp.shrink_to_lo(), format!("{prefix}{sugg}(")),
-                                (sp.shrink_to_hi(), ")".to_string()),
-                            ],
-                            false,
-                        )
-                    } else {
-                        (vec![(sp.shrink_to_lo(), format!("{prefix}{sugg}"))], true)
-                    };
-                    return Some((
-                        sugg,
-                        format!("consider {}borrowing here", mutability.mutably_str()),
-                        Applicability::MachineApplicable,
-                        verbose,
-                        false,
-                    ));
-                }
-            }
-            (
-                hir::ExprKind::AddrOf(hir::BorrowKind::Ref, _, ref expr),
-                _,
-                &ty::Ref(_, checked, _),
-            ) if self.can_sub(self.param_env, checked, expected) => {
-                let make_sugg = |start: Span, end: BytePos| {
-                    // skip `(` for tuples such as `(c) = (&123)`.
-                    // make sure we won't suggest like `(c) = 123)` which is incorrect.
-                    let sp = sm
-                        .span_extend_while(start.shrink_to_lo(), |c| c == '(' || c.is_whitespace())
-                        .map_or(start, |s| s.shrink_to_hi());
-                    Some((
-                        vec![(sp.with_hi(end), String::new())],
-                        "consider removing the borrow".to_string(),
-                        Applicability::MachineApplicable,
-                        true,
-                        true,
-                    ))
-                };
-
-                // We have `&T`, check if what was expected was `T`. If so,
-                // we may want to suggest removing a `&`.
-                if sm.is_imported(expr.span) {
-                    // Go through the spans from which this span was expanded,
-                    // and find the one that's pointing inside `sp`.
-                    //
-                    // E.g. for `&format!("")`, where we want the span to the
-                    // `format!()` invocation instead of its expansion.
-                    if let Some(call_span) =
-                        iter::successors(Some(expr.span), |s| s.parent_callsite())
-                            .find(|&s| sp.contains(s))
-                        && sm.is_span_accessible(call_span)
-                    {
-                        return make_sugg(sp, call_span.lo());
-                    }
-                    return None;
-                }
-                if sp.contains(expr.span) && sm.is_span_accessible(expr.span) {
-                    return make_sugg(sp, expr.span.lo());
-                }
-            }
-            (
-                _,
-                &ty::RawPtr(TypeAndMut { ty: ty_b, mutbl: mutbl_b }),
-                &ty::Ref(_, ty_a, mutbl_a),
-            ) => {
-                if let Some(steps) = self.deref_steps(ty_a, ty_b)
-                    // Only suggest valid if dereferencing needed.
-                    && steps > 0
-                    // The pointer type implements `Copy` trait so the suggestion is always valid.
-                    && let Ok(src) = sm.span_to_snippet(sp)
-                {
-                    let derefs = "*".repeat(steps);
-                    let old_prefix = mutbl_a.ref_prefix_str();
-                    let new_prefix = mutbl_b.ref_prefix_str().to_owned() + &derefs;
-
-                    let suggestion = replace_prefix(&src, old_prefix, &new_prefix).map(|_| {
-                        // skip `&` or `&mut ` if both mutabilities are mutable
-                        let lo = sp.lo()
-                            + BytePos(min(old_prefix.len(), mutbl_b.ref_prefix_str().len()) as _);
-                        // skip `&` or `&mut `
-                        let hi = sp.lo() + BytePos(old_prefix.len() as _);
-                        let sp = sp.with_lo(lo).with_hi(hi);
-
-                        (
-                            sp,
-                            format!(
-                                "{}{derefs}",
-                                if mutbl_a != mutbl_b { mutbl_b.prefix_str() } else { "" }
-                            ),
-                            if mutbl_b <= mutbl_a {
-                                Applicability::MachineApplicable
-                            } else {
-                                Applicability::MaybeIncorrect
-                            },
-                        )
-                    });
-
-                    if let Some((span, src, applicability)) = suggestion {
-                        return Some((
-                            vec![(span, src)],
-                            "consider dereferencing".to_string(),
-                            applicability,
-                            true,
-                            false,
-                        ));
-                    }
-                }
-            }
-            _ if sp == expr.span => {
-                if let Some(mut steps) = self.deref_steps(checked_ty, expected) {
-                    let mut expr = expr.peel_blocks();
-                    let mut prefix_span = expr.span.shrink_to_lo();
-                    let mut remove = String::new();
-
-                    // Try peeling off any existing `&` and `&mut` to reach our target type
-                    while steps > 0 {
-                        if let hir::ExprKind::AddrOf(_, mutbl, inner) = expr.kind {
-                            // If the expression has `&`, removing it would fix the error
-                            prefix_span = prefix_span.with_hi(inner.span.lo());
-                            expr = inner;
-                            remove.push_str(mutbl.ref_prefix_str());
-                            steps -= 1;
-                        } else {
-                            break;
-                        }
-                    }
-                    // If we've reached our target type with just removing `&`, then just print now.
-                    if steps == 0 && !remove.trim().is_empty() {
-                        return Some((
-                            vec![(prefix_span, String::new())],
-                            format!("consider removing the `{}`", remove.trim()),
-                            // Do not remove `&&` to get to bool, because it might be something like
-                            // { a } && b, which we have a separate fixup suggestion that is more
-                            // likely correct...
-                            if remove.trim() == "&&" && expected == self.tcx.types.bool {
-                                Applicability::MaybeIncorrect
-                            } else {
-                                Applicability::MachineApplicable
-                            },
-                            true,
-                            false,
-                        ));
-                    }
-
-                    // For this suggestion to make sense, the type would need to be `Copy`,
-                    // or we have to be moving out of a `Box<T>`
-                    if self.type_is_copy_modulo_regions(self.param_env, expected)
-                        // FIXME(compiler-errors): We can actually do this if the checked_ty is
-                        // `steps` layers of boxes, not just one, but this is easier and most likely.
-                        || (checked_ty.is_box() && steps == 1)
-                        // We can always deref a binop that takes its arguments by ref.
-                        || matches!(
-                            self.tcx.hir().get_parent(expr.hir_id),
-                            hir::Node::Expr(hir::Expr { kind: hir::ExprKind::Binary(op, ..), .. })
-                                if !op.node.is_by_value()
-                        )
-                    {
-                        let deref_kind = if checked_ty.is_box() {
-                            "unboxing the value"
-                        } else if checked_ty.is_ref() {
-                            "dereferencing the borrow"
-                        } else {
-                            "dereferencing the type"
-                        };
-
-                        // Suggest removing `&` if we have removed any, otherwise suggest just
-                        // dereferencing the remaining number of steps.
-                        let message = if remove.is_empty() {
-                            format!("consider {deref_kind}")
-                        } else {
-                            format!(
-                                "consider removing the `{}` and {} instead",
-                                remove.trim(),
-                                deref_kind
-                            )
-                        };
-
-                        let prefix =
-                            match self.tcx.hir().maybe_get_struct_pattern_shorthand_field(expr) {
-                                Some(ident) => format!("{ident}: "),
-                                None => String::new(),
-                            };
-
-                        let (span, suggestion) = if self.is_else_if_block(expr) {
-                            // Don't suggest nonsense like `else *if`
-                            return None;
-                        } else if let Some(expr) = self.maybe_get_block_expr(expr) {
-                            // prefix should be empty here..
-                            (expr.span.shrink_to_lo(), "*".to_string())
-                        } else {
-                            (prefix_span, format!("{}{}", prefix, "*".repeat(steps)))
-                        };
-                        if suggestion.trim().is_empty() {
-                            return None;
-                        }
-
-                        return Some((
-                            vec![(span, suggestion)],
-                            message,
-                            Applicability::MachineApplicable,
-                            true,
-                            false,
-                        ));
-                    }
-                }
-            }
-            _ => {}
-        }
-        None
-    }
-
-    pub fn suggest_cast(
-        &self,
-        err: &mut Diagnostic,
-        expr: &hir::Expr<'_>,
-        checked_ty: Ty<'tcx>,
-        expected_ty: Ty<'tcx>,
-        expected_ty_expr: Option<&'tcx hir::Expr<'tcx>>,
-    ) -> bool {
-        if self.tcx.sess.source_map().is_imported(expr.span) {
-            // Ignore if span is from within a macro.
-            return false;
-        }
-
-        let Ok(src) = self.tcx.sess.source_map().span_to_snippet(expr.span) else {
-            return false;
-        };
-
-        // If casting this expression to a given numeric type would be appropriate in case of a type
-        // mismatch.
-        //
-        // We want to minimize the amount of casting operations that are suggested, as it can be a
-        // lossy operation with potentially bad side effects, so we only suggest when encountering
-        // an expression that indicates that the original type couldn't be directly changed.
-        //
-        // For now, don't suggest casting with `as`.
-        let can_cast = false;
-
-        let mut sugg = vec![];
-
-        if let Some(hir::Node::ExprField(field)) = self.tcx.hir().find_parent(expr.hir_id) {
-            // `expr` is a literal field for a struct, only suggest if appropriate
-            if field.is_shorthand {
-                // This is a field literal
-                sugg.push((field.ident.span.shrink_to_lo(), format!("{}: ", field.ident)));
-            } else {
-                // Likely a field was meant, but this field wasn't found. Do not suggest anything.
-                return false;
-            }
-        };
-
-        if let hir::ExprKind::Call(path, args) = &expr.kind
-            && let (hir::ExprKind::Path(hir::QPath::TypeRelative(base_ty, path_segment)), 1) =
-                (&path.kind, args.len())
-            // `expr` is a conversion like `u32::from(val)`, do not suggest anything (#63697).
-            && let (hir::TyKind::Path(hir::QPath::Resolved(None, base_ty_path)), sym::from) =
-                (&base_ty.kind, path_segment.ident.name)
-        {
-            if let Some(ident) = &base_ty_path.segments.iter().map(|s| s.ident).next() {
-                match ident.name {
-                    sym::i128
-                    | sym::i64
-                    | sym::i32
-                    | sym::i16
-                    | sym::i8
-                    | sym::u128
-                    | sym::u64
-                    | sym::u32
-                    | sym::u16
-                    | sym::u8
-                    | sym::isize
-                    | sym::usize
-                        if base_ty_path.segments.len() == 1 =>
-                    {
-                        return false;
-                    }
-                    _ => {}
-                }
-            }
-        }
-
-        let msg = format!(
-            "you can convert {} `{}` to {} `{}`",
-            checked_ty.kind().article(),
-            checked_ty,
-            expected_ty.kind().article(),
-            expected_ty,
-        );
-        let cast_msg = format!(
-            "you can cast {} `{}` to {} `{}`",
-            checked_ty.kind().article(),
-            checked_ty,
-            expected_ty.kind().article(),
-            expected_ty,
-        );
-        let lit_msg = format!(
-            "change the type of the numeric literal from `{checked_ty}` to `{expected_ty}`",
-        );
-
-        let close_paren = if expr.precedence().order() < PREC_POSTFIX {
-            sugg.push((expr.span.shrink_to_lo(), "(".to_string()));
-            ")"
-        } else {
-            ""
-        };
-
-        let mut cast_suggestion = sugg.clone();
-        cast_suggestion.push((expr.span.shrink_to_hi(), format!("{close_paren} as {expected_ty}")));
-        let mut into_suggestion = sugg.clone();
-        into_suggestion.push((expr.span.shrink_to_hi(), format!("{close_paren}.into()")));
-        let mut suffix_suggestion = sugg.clone();
-        suffix_suggestion.push((
-            if matches!(
-                (&expected_ty.kind(), &checked_ty.kind()),
-                (ty::Int(_) | ty::Uint(_), ty::Float(_))
-            ) {
-                // Remove fractional part from literal, for example `42.0f32` into `42`
-                let src = src.trim_end_matches(&checked_ty.to_string());
-                let len = src.split('.').next().unwrap().len();
-                expr.span.with_lo(expr.span.lo() + BytePos(len as u32))
-            } else {
-                let len = src.trim_end_matches(&checked_ty.to_string()).len();
-                expr.span.with_lo(expr.span.lo() + BytePos(len as u32))
-            },
-            if expr.precedence().order() < PREC_POSTFIX {
-                // Readd `)`
-                format!("{expected_ty})")
-            } else {
-                expected_ty.to_string()
-            },
-        ));
-        let literal_is_ty_suffixed = |expr: &hir::Expr<'_>| {
-            if let hir::ExprKind::Lit(lit) = &expr.kind { lit.node.is_suffixed() } else { false }
-        };
-        let is_negative_int =
-            |expr: &hir::Expr<'_>| matches!(expr.kind, hir::ExprKind::Unary(hir::UnOp::Neg, ..));
-        let is_uint = |ty: Ty<'_>| matches!(ty.kind(), ty::Uint(..));
-
-        let in_const_context = self.tcx.hir().is_inside_const_context(expr.hir_id);
-
-        let suggest_fallible_into_or_lhs_from =
-            |err: &mut Diagnostic, exp_to_found_is_fallible: bool| {
-                // If we know the expression the expected type is derived from, we might be able
-                // to suggest a widening conversion rather than a narrowing one (which may
-                // panic). For example, given x: u8 and y: u32, if we know the span of "x",
-                //   x > y
-                // can be given the suggestion "u32::from(x) > y" rather than
-                // "x > y.try_into().unwrap()".
-                let lhs_expr_and_src = expected_ty_expr.and_then(|expr| {
-                    self.tcx
-                        .sess
-                        .source_map()
-                        .span_to_snippet(expr.span)
-                        .ok()
-                        .map(|src| (expr, src))
-                });
-                let (msg, suggestion) = if let (Some((lhs_expr, lhs_src)), false) =
-                    (lhs_expr_and_src, exp_to_found_is_fallible)
-                {
-                    let msg = format!(
-                        "you can convert `{lhs_src}` from `{expected_ty}` to `{checked_ty}`, matching the type of `{src}`",
-                    );
-                    let suggestion = vec![
-                        (lhs_expr.span.shrink_to_lo(), format!("{checked_ty}::from(")),
-                        (lhs_expr.span.shrink_to_hi(), ")".to_string()),
-                    ];
-                    (msg, suggestion)
-                } else {
-                    let msg =
-                        format!("{} and panic if the converted value doesn't fit", msg.clone());
-                    let mut suggestion = sugg.clone();
-                    suggestion.push((
-                        expr.span.shrink_to_hi(),
-                        format!("{close_paren}.try_into().unwrap()"),
-                    ));
-                    (msg, suggestion)
-                };
-                err.multipart_suggestion_verbose(msg, suggestion, Applicability::MachineApplicable);
-            };
-
-        let suggest_to_change_suffix_or_into =
-            |err: &mut Diagnostic,
-             found_to_exp_is_fallible: bool,
-             exp_to_found_is_fallible: bool| {
-                let exp_is_lhs = expected_ty_expr.is_some_and(|e| self.tcx.hir().is_lhs(e.hir_id));
-
-                if exp_is_lhs {
-                    return;
-                }
-
-                let always_fallible = found_to_exp_is_fallible
-                    && (exp_to_found_is_fallible || expected_ty_expr.is_none());
-                let msg = if literal_is_ty_suffixed(expr) {
-                    lit_msg.clone()
-                } else if always_fallible && (is_negative_int(expr) && is_uint(expected_ty)) {
-                    // We now know that converting either the lhs or rhs is fallible. Before we
-                    // suggest a fallible conversion, check if the value can never fit in the
-                    // expected type.
-                    let msg = format!("`{src}` cannot fit into type `{expected_ty}`");
-                    err.note(msg);
-                    return;
-                } else if in_const_context {
-                    // Do not recommend `into` or `try_into` in const contexts.
-                    return;
-                } else if found_to_exp_is_fallible {
-                    return suggest_fallible_into_or_lhs_from(err, exp_to_found_is_fallible);
-                } else {
-                    msg.clone()
-                };
-                let suggestion = if literal_is_ty_suffixed(expr) {
-                    suffix_suggestion.clone()
-                } else {
-                    into_suggestion.clone()
-                };
-                err.multipart_suggestion_verbose(msg, suggestion, Applicability::MachineApplicable);
-            };
-
-        match (&expected_ty.kind(), &checked_ty.kind()) {
-            (ty::Int(exp), ty::Int(found)) => {
-                let (f2e_is_fallible, e2f_is_fallible) = match (exp.bit_width(), found.bit_width())
-                {
-                    (Some(exp), Some(found)) if exp < found => (true, false),
-                    (Some(exp), Some(found)) if exp > found => (false, true),
-                    (None, Some(8 | 16)) => (false, true),
-                    (Some(8 | 16), None) => (true, false),
-                    (None, _) | (_, None) => (true, true),
-                    _ => (false, false),
-                };
-                suggest_to_change_suffix_or_into(err, f2e_is_fallible, e2f_is_fallible);
-                true
-            }
-            (ty::Uint(exp), ty::Uint(found)) => {
-                let (f2e_is_fallible, e2f_is_fallible) = match (exp.bit_width(), found.bit_width())
-                {
-                    (Some(exp), Some(found)) if exp < found => (true, false),
-                    (Some(exp), Some(found)) if exp > found => (false, true),
-                    (None, Some(8 | 16)) => (false, true),
-                    (Some(8 | 16), None) => (true, false),
-                    (None, _) | (_, None) => (true, true),
-                    _ => (false, false),
-                };
-                suggest_to_change_suffix_or_into(err, f2e_is_fallible, e2f_is_fallible);
-                true
-            }
-            (&ty::Int(exp), &ty::Uint(found)) => {
-                let (f2e_is_fallible, e2f_is_fallible) = match (exp.bit_width(), found.bit_width())
-                {
-                    (Some(exp), Some(found)) if found < exp => (false, true),
-                    (None, Some(8)) => (false, true),
-                    _ => (true, true),
-                };
-                suggest_to_change_suffix_or_into(err, f2e_is_fallible, e2f_is_fallible);
-                true
-            }
-            (&ty::Uint(exp), &ty::Int(found)) => {
-                let (f2e_is_fallible, e2f_is_fallible) = match (exp.bit_width(), found.bit_width())
-                {
-                    (Some(exp), Some(found)) if found > exp => (true, false),
-                    (Some(8), None) => (true, false),
-                    _ => (true, true),
-                };
-                suggest_to_change_suffix_or_into(err, f2e_is_fallible, e2f_is_fallible);
-                true
-            }
-            (ty::Float(exp), ty::Float(found)) => {
-                if found.bit_width() < exp.bit_width() {
-                    suggest_to_change_suffix_or_into(err, false, true);
-                } else if literal_is_ty_suffixed(expr) {
-                    err.multipart_suggestion_verbose(
-                        lit_msg,
-                        suffix_suggestion,
-                        Applicability::MachineApplicable,
-                    );
-                } else if can_cast {
-                    // Missing try_into implementation for `f64` to `f32`
-                    err.multipart_suggestion_verbose(
-                        format!("{cast_msg}, producing the closest possible value"),
-                        cast_suggestion,
-                        Applicability::MaybeIncorrect, // lossy conversion
-                    );
-                }
-                true
-            }
-            (&ty::Uint(_) | &ty::Int(_), &ty::Float(_)) => {
-                if literal_is_ty_suffixed(expr) {
-                    err.multipart_suggestion_verbose(
-                        lit_msg,
-                        suffix_suggestion,
-                        Applicability::MachineApplicable,
-                    );
-                } else if can_cast {
-                    // Missing try_into implementation for `{float}` to `{integer}`
-                    err.multipart_suggestion_verbose(
-                        format!("{msg}, rounding the float towards zero"),
-                        cast_suggestion,
-                        Applicability::MaybeIncorrect, // lossy conversion
-                    );
-                }
-                true
-            }
-            (ty::Float(exp), ty::Uint(found)) => {
-                // if `found` is `None` (meaning found is `usize`), don't suggest `.into()`
-                if exp.bit_width() > found.bit_width().unwrap_or(256) {
-                    err.multipart_suggestion_verbose(
-                        format!(
-                            "{msg}, producing the floating point representation of the integer",
-                        ),
-                        into_suggestion,
-                        Applicability::MachineApplicable,
-                    );
-                } else if literal_is_ty_suffixed(expr) {
-                    err.multipart_suggestion_verbose(
-                        lit_msg,
-                        suffix_suggestion,
-                        Applicability::MachineApplicable,
-                    );
-                } else {
-                    // Missing try_into implementation for `{integer}` to `{float}`
-                    err.multipart_suggestion_verbose(
-                        format!(
-                            "{cast_msg}, producing the floating point representation of the integer, \
-                                 rounded if necessary",
-                        ),
-                        cast_suggestion,
-                        Applicability::MaybeIncorrect, // lossy conversion
-                    );
-                }
-                true
-            }
-            (ty::Float(exp), ty::Int(found)) => {
-                // if `found` is `None` (meaning found is `isize`), don't suggest `.into()`
-                if exp.bit_width() > found.bit_width().unwrap_or(256) {
-                    err.multipart_suggestion_verbose(
-                        format!(
-                            "{}, producing the floating point representation of the integer",
-                            msg.clone(),
-                        ),
-                        into_suggestion,
-                        Applicability::MachineApplicable,
-                    );
-                } else if literal_is_ty_suffixed(expr) {
-                    err.multipart_suggestion_verbose(
-                        lit_msg,
-                        suffix_suggestion,
-                        Applicability::MachineApplicable,
-                    );
-                } else {
-                    // Missing try_into implementation for `{integer}` to `{float}`
-                    err.multipart_suggestion_verbose(
-                        format!(
-                            "{}, producing the floating point representation of the integer, \
-                                rounded if necessary",
-                            &msg,
-                        ),
-                        cast_suggestion,
-                        Applicability::MaybeIncorrect, // lossy conversion
-                    );
-                }
-                true
-            }
-            (
-                &ty::Uint(ty::UintTy::U32 | ty::UintTy::U64 | ty::UintTy::U128)
-                | &ty::Int(ty::IntTy::I32 | ty::IntTy::I64 | ty::IntTy::I128),
-                &ty::Char,
-            ) => {
-                err.multipart_suggestion_verbose(
-                    format!("{cast_msg}, since a `char` always occupies 4 bytes"),
-                    cast_suggestion,
-                    Applicability::MachineApplicable,
-                );
-                true
-            }
-            _ => false,
-        }
-    }
-
-    /// Identify when the user has written `foo..bar()` instead of `foo.bar()`.
-    pub fn suggest_method_call_on_range_literal(
-        &self,
-        err: &mut Diagnostic,
-        expr: &hir::Expr<'tcx>,
-        checked_ty: Ty<'tcx>,
-        expected_ty: Ty<'tcx>,
-    ) {
-        if !hir::is_range_literal(expr) {
-            return;
-        }
-        let hir::ExprKind::Struct(hir::QPath::LangItem(LangItem::Range, ..), [start, end], _) =
-            expr.kind
-        else {
-            return;
-        };
-        let parent = self.tcx.hir().parent_id(expr.hir_id);
-        if let Some(hir::Node::ExprField(_)) = self.tcx.hir().find(parent) {
-            // Ignore `Foo { field: a..Default::default() }`
-            return;
-        }
-        let mut expr = end.expr;
-        let mut expectation = Some(expected_ty);
-        while let hir::ExprKind::MethodCall(_, rcvr, ..) = expr.kind {
-            // Getting to the root receiver and asserting it is a fn call let's us ignore cases in
-            // `tests/ui/methods/issues/issue-90315.stderr`.
-            expr = rcvr;
-            // If we have more than one layer of calls, then the expected ty
-            // cannot guide the method probe.
-            expectation = None;
-        }
-        let hir::ExprKind::Call(method_name, _) = expr.kind else {
-            return;
-        };
-        let ty::Adt(adt, _) = checked_ty.kind() else {
-            return;
-        };
-        if self.tcx.lang_items().range_struct() != Some(adt.did()) {
-            return;
-        }
-        if let ty::Adt(adt, _) = expected_ty.kind()
-            && self.tcx.lang_items().range_struct() == Some(adt.did())
-        {
-            return;
-        }
-        // Check if start has method named end.
-        let hir::ExprKind::Path(hir::QPath::Resolved(None, p)) = method_name.kind else {
-            return;
-        };
-        let [hir::PathSegment { ident, .. }] = p.segments else {
-            return;
-        };
-        let self_ty = self.typeck_results.borrow().expr_ty(start.expr);
-        let Ok(_pick) = self.lookup_probe_for_diagnostic(
-            *ident,
-            self_ty,
-            expr,
-            probe::ProbeScope::AllTraits,
-            expectation,
-        ) else {
-            return;
-        };
-        let mut sugg = ".";
-        let mut span = start.expr.span.between(end.expr.span);
-        if span.lo() + BytePos(2) == span.hi() {
-            // There's no space between the start, the range op and the end, suggest removal which
-            // will be more noticeable than the replacement of `..` with `.`.
-            span = span.with_lo(span.lo() + BytePos(1));
-            sugg = "";
-        }
-        err.span_suggestion_verbose(
-            span,
-            "you likely meant to write a method call instead of a range",
-            sugg,
-            Applicability::MachineApplicable,
-        );
-    }
-
-    /// Identify when the type error is because `()` is found in a binding that was assigned a
-    /// block without a tail expression.
-    fn suggest_return_binding_for_missing_tail_expr(
-        &self,
-        err: &mut Diagnostic,
-        expr: &hir::Expr<'_>,
-        checked_ty: Ty<'tcx>,
-        expected_ty: Ty<'tcx>,
-    ) {
-        if !checked_ty.is_unit() {
-            return;
-        }
-        let hir::ExprKind::Path(hir::QPath::Resolved(None, path)) = expr.kind else {
-            return;
-        };
-        let hir::def::Res::Local(hir_id) = path.res else {
-            return;
-        };
-        let Some(hir::Node::Pat(pat)) = self.tcx.hir().find(hir_id) else {
-            return;
-        };
-        let Some(hir::Node::Local(hir::Local { ty: None, init: Some(init), .. })) =
-            self.tcx.hir().find_parent(pat.hir_id)
-        else {
-            return;
-        };
-        let hir::ExprKind::Block(block, None) = init.kind else {
-            return;
-        };
-        if block.expr.is_some() {
-            return;
-        }
-        let [.., stmt] = block.stmts else {
-            err.span_label(block.span, "this empty block is missing a tail expression");
-            return;
-        };
-        let hir::StmtKind::Semi(tail_expr) = stmt.kind else {
-            return;
-        };
-        let Some(ty) = self.node_ty_opt(tail_expr.hir_id) else {
-            return;
-        };
-        if self.can_eq(self.param_env, expected_ty, ty) {
-            err.span_suggestion_short(
-                stmt.span.with_lo(tail_expr.span.hi()),
-                "remove this semicolon",
-                "",
-                Applicability::MachineApplicable,
-            );
-        } else {
-            err.span_label(block.span, "this block is missing a tail expression");
-        }
-    }
-
     fn note_wrong_return_ty_due_to_generic_arg(
         &self,
         err: &mut Diagnostic,
diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
index ca7dc298de64..c43d4932fb9f 100644
--- a/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
+++ b/compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs
@@ -2,7 +2,14 @@ use super::FnCtxt;
 
 use crate::errors;
 use crate::fluent_generated as fluent;
+use crate::fn_ctxt::rustc_span::BytePos;
+use crate::hir::is_range_literal;
+use crate::method::probe;
 use crate::method::probe::{IsSuggestion, Mode, ProbeScope};
+use crate::rustc_middle::ty::Article;
+use crate::ty::TypeAndMut;
+use core::cmp::min;
+use core::iter;
 use rustc_ast::util::parser::{ExprPrecedence, PREC_POSTFIX};
 use rustc_errors::{Applicability, Diagnostic, MultiSpan};
 use rustc_hir as hir;
@@ -16,6 +23,7 @@ use rustc_hir::{
 use rustc_hir_analysis::astconv::AstConv;
 use rustc_infer::traits::{self, StatementAsExpression};
 use rustc_middle::lint::in_external_macro;
+use rustc_middle::middle::stability::EvalResult;
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{
     self, suggest_constraining_type_params, Binder, IsSuggestable, ToPredicate, Ty,
@@ -1816,4 +1824,1265 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         );
         return true;
     }
+
+    pub(crate) fn suggest_coercing_result_via_try_operator(
+        &self,
+        err: &mut Diagnostic,
+        expr: &hir::Expr<'tcx>,
+        expected: Ty<'tcx>,
+        found: Ty<'tcx>,
+    ) -> bool {
+        let map = self.tcx.hir();
+        let returned = matches!(
+            map.find_parent(expr.hir_id),
+            Some(hir::Node::Expr(hir::Expr { kind: hir::ExprKind::Ret(_), .. }))
+        ) || map.get_return_block(expr.hir_id).is_some();
+        if returned
+            && let ty::Adt(e, args_e) = expected.kind()
+            && let ty::Adt(f, args_f) = found.kind()
+            && e.did() == f.did()
+            && Some(e.did()) == self.tcx.get_diagnostic_item(sym::Result)
+            && let e_ok = args_e.type_at(0)
+            && let f_ok = args_f.type_at(0)
+            && self.infcx.can_eq(self.param_env, f_ok, e_ok)
+            && let e_err = args_e.type_at(1)
+            && let f_err = args_f.type_at(1)
+            && self
+                .infcx
+                .type_implements_trait(
+                    self.tcx.get_diagnostic_item(sym::Into).unwrap(),
+                    [f_err, e_err],
+                    self.param_env,
+                )
+                .must_apply_modulo_regions()
+        {
+            err.multipart_suggestion(
+                "use `?` to coerce and return an appropriate `Err`, and wrap the resulting value \
+                 in `Ok` so the expression remains of type `Result`",
+                vec![
+                    (expr.span.shrink_to_lo(), "Ok(".to_string()),
+                    (expr.span.shrink_to_hi(), "?)".to_string()),
+                ],
+                Applicability::MaybeIncorrect,
+            );
+            return true;
+        }
+        false
+    }
+
+    /// If the expected type is an enum (Issue #55250) with any variants whose
+    /// sole field is of the found type, suggest such variants. (Issue #42764)
+    pub(crate) fn suggest_compatible_variants(
+        &self,
+        err: &mut Diagnostic,
+        expr: &hir::Expr<'_>,
+        expected: Ty<'tcx>,
+        expr_ty: Ty<'tcx>,
+    ) -> bool {
+        if in_external_macro(self.tcx.sess, expr.span) {
+            return false;
+        }
+        if let ty::Adt(expected_adt, args) = expected.kind() {
+            if let hir::ExprKind::Field(base, ident) = expr.kind {
+                let base_ty = self.typeck_results.borrow().expr_ty(base);
+                if self.can_eq(self.param_env, base_ty, expected)
+                    && let Some(base_span) = base.span.find_ancestor_inside(expr.span)
+                {
+                    err.span_suggestion_verbose(
+                        expr.span.with_lo(base_span.hi()),
+                        format!("consider removing the tuple struct field `{ident}`"),
+                        "",
+                        Applicability::MaybeIncorrect,
+                    );
+                    return true;
+                }
+            }
+
+            // If the expression is of type () and it's the return expression of a block,
+            // we suggest adding a separate return expression instead.
+            // (To avoid things like suggesting `Ok(while .. { .. })`.)
+            if expr_ty.is_unit() {
+                let mut id = expr.hir_id;
+                let mut parent;
+
+                // Unroll desugaring, to make sure this works for `for` loops etc.
+                loop {
+                    parent = self.tcx.hir().parent_id(id);
+                    if let Some(parent_span) = self.tcx.hir().opt_span(parent) {
+                        if parent_span.find_ancestor_inside(expr.span).is_some() {
+                            // The parent node is part of the same span, so is the result of the
+                            // same expansion/desugaring and not the 'real' parent node.
+                            id = parent;
+                            continue;
+                        }
+                    }
+                    break;
+                }
+
+                if let Some(hir::Node::Block(&hir::Block {
+                    span: block_span, expr: Some(e), ..
+                })) = self.tcx.hir().find(parent)
+                {
+                    if e.hir_id == id {
+                        if let Some(span) = expr.span.find_ancestor_inside(block_span) {
+                            let return_suggestions = if self
+                                .tcx
+                                .is_diagnostic_item(sym::Result, expected_adt.did())
+                            {
+                                vec!["Ok(())"]
+                            } else if self.tcx.is_diagnostic_item(sym::Option, expected_adt.did()) {
+                                vec!["None", "Some(())"]
+                            } else {
+                                return false;
+                            };
+                            if let Some(indent) =
+                                self.tcx.sess.source_map().indentation_before(span.shrink_to_lo())
+                            {
+                                // Add a semicolon, except after `}`.
+                                let semicolon =
+                                    match self.tcx.sess.source_map().span_to_snippet(span) {
+                                        Ok(s) if s.ends_with('}') => "",
+                                        _ => ";",
+                                    };
+                                err.span_suggestions(
+                                    span.shrink_to_hi(),
+                                    "try adding an expression at the end of the block",
+                                    return_suggestions
+                                        .into_iter()
+                                        .map(|r| format!("{semicolon}\n{indent}{r}")),
+                                    Applicability::MaybeIncorrect,
+                                );
+                            }
+                            return true;
+                        }
+                    }
+                }
+            }
+
+            let compatible_variants: Vec<(String, _, _, Option<String>)> = expected_adt
+                .variants()
+                .iter()
+                .filter(|variant| {
+                    variant.fields.len() == 1
+                })
+                .filter_map(|variant| {
+                    let sole_field = &variant.single_field();
+
+                    let field_is_local = sole_field.did.is_local();
+                    let field_is_accessible =
+                        sole_field.vis.is_accessible_from(expr.hir_id.owner.def_id, self.tcx)
+                        // Skip suggestions for unstable public fields (for example `Pin::pointer`)
+                        && matches!(self.tcx.eval_stability(sole_field.did, None, expr.span, None), EvalResult::Allow | EvalResult::Unmarked);
+
+                    if !field_is_local && !field_is_accessible {
+                        return None;
+                    }
+
+                    let note_about_variant_field_privacy = (field_is_local && !field_is_accessible)
+                        .then(|| " (its field is private, but it's local to this crate and its privacy can be changed)".to_string());
+
+                    let sole_field_ty = sole_field.ty(self.tcx, args);
+                    if self.can_coerce(expr_ty, sole_field_ty) {
+                        let variant_path =
+                            with_no_trimmed_paths!(self.tcx.def_path_str(variant.def_id));
+                        // FIXME #56861: DRYer prelude filtering
+                        if let Some(path) = variant_path.strip_prefix("std::prelude::")
+                            && let Some((_, path)) = path.split_once("::")
+                        {
+                            return Some((path.to_string(), variant.ctor_kind(), sole_field.name, note_about_variant_field_privacy));
+                        }
+                        Some((variant_path, variant.ctor_kind(), sole_field.name, note_about_variant_field_privacy))
+                    } else {
+                        None
+                    }
+                })
+                .collect();
+
+            let suggestions_for = |variant: &_, ctor_kind, field_name| {
+                let prefix = match self.tcx.hir().maybe_get_struct_pattern_shorthand_field(expr) {
+                    Some(ident) => format!("{ident}: "),
+                    None => String::new(),
+                };
+
+                let (open, close) = match ctor_kind {
+                    Some(CtorKind::Fn) => ("(".to_owned(), ")"),
+                    None => (format!(" {{ {field_name}: "), " }"),
+
+                    // unit variants don't have fields
+                    Some(CtorKind::Const) => unreachable!(),
+                };
+
+                // Suggest constructor as deep into the block tree as possible.
+                // This fixes https://github.com/rust-lang/rust/issues/101065,
+                // and also just helps make the most minimal suggestions.
+                let mut expr = expr;
+                while let hir::ExprKind::Block(block, _) = &expr.kind
+                    && let Some(expr_) = &block.expr
+                {
+                    expr = expr_
+                }
+
+                vec![
+                    (expr.span.shrink_to_lo(), format!("{prefix}{variant}{open}")),
+                    (expr.span.shrink_to_hi(), close.to_owned()),
+                ]
+            };
+
+            match &compatible_variants[..] {
+                [] => { /* No variants to format */ }
+                [(variant, ctor_kind, field_name, note)] => {
+                    // Just a single matching variant.
+                    err.multipart_suggestion_verbose(
+                        format!(
+                            "try wrapping the expression in `{variant}`{note}",
+                            note = note.as_deref().unwrap_or("")
+                        ),
+                        suggestions_for(&**variant, *ctor_kind, *field_name),
+                        Applicability::MaybeIncorrect,
+                    );
+                    return true;
+                }
+                _ => {
+                    // More than one matching variant.
+                    err.multipart_suggestions(
+                        format!(
+                            "try wrapping the expression in a variant of `{}`",
+                            self.tcx.def_path_str(expected_adt.did())
+                        ),
+                        compatible_variants.into_iter().map(
+                            |(variant, ctor_kind, field_name, _)| {
+                                suggestions_for(&variant, ctor_kind, field_name)
+                            },
+                        ),
+                        Applicability::MaybeIncorrect,
+                    );
+                    return true;
+                }
+            }
+        }
+
+        false
+    }
+
+    pub(crate) fn suggest_non_zero_new_unwrap(
+        &self,
+        err: &mut Diagnostic,
+        expr: &hir::Expr<'_>,
+        expected: Ty<'tcx>,
+        expr_ty: Ty<'tcx>,
+    ) -> bool {
+        let tcx = self.tcx;
+        let (adt, unwrap) = match expected.kind() {
+            // In case Option<NonZero*> is wanted, but * is provided, suggest calling new
+            ty::Adt(adt, args) if tcx.is_diagnostic_item(sym::Option, adt.did()) => {
+                // Unwrap option
+                let ty::Adt(adt, _) = args.type_at(0).kind() else {
+                    return false;
+                };
+
+                (adt, "")
+            }
+            // In case NonZero* is wanted, but * is provided also add `.unwrap()` to satisfy types
+            ty::Adt(adt, _) => (adt, ".unwrap()"),
+            _ => return false,
+        };
+
+        let map = [
+            (sym::NonZeroU8, tcx.types.u8),
+            (sym::NonZeroU16, tcx.types.u16),
+            (sym::NonZeroU32, tcx.types.u32),
+            (sym::NonZeroU64, tcx.types.u64),
+            (sym::NonZeroU128, tcx.types.u128),
+            (sym::NonZeroI8, tcx.types.i8),
+            (sym::NonZeroI16, tcx.types.i16),
+            (sym::NonZeroI32, tcx.types.i32),
+            (sym::NonZeroI64, tcx.types.i64),
+            (sym::NonZeroI128, tcx.types.i128),
+        ];
+
+        let Some((s, _)) = map.iter().find(|&&(s, t)| {
+            self.tcx.is_diagnostic_item(s, adt.did()) && self.can_coerce(expr_ty, t)
+        }) else {
+            return false;
+        };
+
+        let path = self.tcx.def_path_str(adt.non_enum_variant().def_id);
+
+        err.multipart_suggestion(
+            format!("consider calling `{s}::new`"),
+            vec![
+                (expr.span.shrink_to_lo(), format!("{path}::new(")),
+                (expr.span.shrink_to_hi(), format!("){unwrap}")),
+            ],
+            Applicability::MaybeIncorrect,
+        );
+
+        true
+    }
+
+    /// Identify some cases where `as_ref()` would be appropriate and suggest it.
+    ///
+    /// Given the following code:
+    /// ```compile_fail,E0308
+    /// struct Foo;
+    /// fn takes_ref(_: &Foo) {}
+    /// let ref opt = Some(Foo);
+    ///
+    /// opt.map(|param| takes_ref(param));
+    /// ```
+    /// Suggest using `opt.as_ref().map(|param| takes_ref(param));` instead.
+    ///
+    /// It only checks for `Option` and `Result` and won't work with
+    /// ```ignore (illustrative)
+    /// opt.map(|param| { takes_ref(param) });
+    /// ```
+    fn can_use_as_ref(&self, expr: &hir::Expr<'_>) -> Option<(Vec<(Span, String)>, &'static str)> {
+        let hir::ExprKind::Path(hir::QPath::Resolved(_, ref path)) = expr.kind else {
+            return None;
+        };
+
+        let hir::def::Res::Local(local_id) = path.res else {
+            return None;
+        };
+
+        let local_parent = self.tcx.hir().parent_id(local_id);
+        let Some(Node::Param(hir::Param { hir_id: param_hir_id, .. })) =
+            self.tcx.hir().find(local_parent)
+        else {
+            return None;
+        };
+
+        let param_parent = self.tcx.hir().parent_id(*param_hir_id);
+        let Some(Node::Expr(hir::Expr {
+            hir_id: expr_hir_id,
+            kind: hir::ExprKind::Closure(hir::Closure { fn_decl: closure_fn_decl, .. }),
+            ..
+        })) = self.tcx.hir().find(param_parent)
+        else {
+            return None;
+        };
+
+        let expr_parent = self.tcx.hir().parent_id(*expr_hir_id);
+        let hir = self.tcx.hir().find(expr_parent);
+        let closure_params_len = closure_fn_decl.inputs.len();
+        let (
+            Some(Node::Expr(hir::Expr {
+                kind: hir::ExprKind::MethodCall(method_path, receiver, ..),
+                ..
+            })),
+            1,
+        ) = (hir, closure_params_len)
+        else {
+            return None;
+        };
+
+        let self_ty = self.typeck_results.borrow().expr_ty(receiver);
+        let name = method_path.ident.name;
+        let is_as_ref_able = match self_ty.peel_refs().kind() {
+            ty::Adt(def, _) => {
+                (self.tcx.is_diagnostic_item(sym::Option, def.did())
+                    || self.tcx.is_diagnostic_item(sym::Result, def.did()))
+                    && (name == sym::map || name == sym::and_then)
+            }
+            _ => false,
+        };
+        if is_as_ref_able {
+            Some((
+                vec![(method_path.ident.span.shrink_to_lo(), "as_ref().".to_string())],
+                "consider using `as_ref` instead",
+            ))
+        } else {
+            None
+        }
+    }
+
+    /// This function is used to determine potential "simple" improvements or users' errors and
+    /// provide them useful help. For example:
+    ///
+    /// ```compile_fail,E0308
+    /// fn some_fn(s: &str) {}
+    ///
+    /// let x = "hey!".to_owned();
+    /// some_fn(x); // error
+    /// ```
+    ///
+    /// No need to find every potential function which could make a coercion to transform a
+    /// `String` into a `&str` since a `&` would do the trick!
+    ///
+    /// In addition of this check, it also checks between references mutability state. If the
+    /// expected is mutable but the provided isn't, maybe we could just say "Hey, try with
+    /// `&mut`!".
+    pub(crate) fn suggest_deref_or_ref(
+        &self,
+        expr: &hir::Expr<'tcx>,
+        checked_ty: Ty<'tcx>,
+        expected: Ty<'tcx>,
+    ) -> Option<(
+        Vec<(Span, String)>,
+        String,
+        Applicability,
+        bool, /* verbose */
+        bool, /* suggest `&` or `&mut` type annotation */
+    )> {
+        let sess = self.sess();
+        let sp = expr.span;
+
+        // If the span is from an external macro, there's no suggestion we can make.
+        if in_external_macro(sess, sp) {
+            return None;
+        }
+
+        let sm = sess.source_map();
+
+        let replace_prefix = |s: &str, old: &str, new: &str| {
+            s.strip_prefix(old).map(|stripped| new.to_string() + stripped)
+        };
+
+        // `ExprKind::DropTemps` is semantically irrelevant for these suggestions.
+        let expr = expr.peel_drop_temps();
+
+        match (&expr.kind, expected.kind(), checked_ty.kind()) {
+            (_, &ty::Ref(_, exp, _), &ty::Ref(_, check, _)) => match (exp.kind(), check.kind()) {
+                (&ty::Str, &ty::Array(arr, _) | &ty::Slice(arr)) if arr == self.tcx.types.u8 => {
+                    if let hir::ExprKind::Lit(_) = expr.kind
+                        && let Ok(src) = sm.span_to_snippet(sp)
+                        && replace_prefix(&src, "b\"", "\"").is_some()
+                    {
+                        let pos = sp.lo() + BytePos(1);
+                        return Some((
+                            vec![(sp.with_hi(pos), String::new())],
+                            "consider removing the leading `b`".to_string(),
+                            Applicability::MachineApplicable,
+                            true,
+                            false,
+                        ));
+                    }
+                }
+                (&ty::Array(arr, _) | &ty::Slice(arr), &ty::Str) if arr == self.tcx.types.u8 => {
+                    if let hir::ExprKind::Lit(_) = expr.kind
+                        && let Ok(src) = sm.span_to_snippet(sp)
+                        && replace_prefix(&src, "\"", "b\"").is_some()
+                    {
+                        return Some((
+                            vec![(sp.shrink_to_lo(), "b".to_string())],
+                            "consider adding a leading `b`".to_string(),
+                            Applicability::MachineApplicable,
+                            true,
+                            false,
+                        ));
+                    }
+                }
+                _ => {}
+            },
+            (_, &ty::Ref(_, _, mutability), _) => {
+                // Check if it can work when put into a ref. For example:
+                //
+                // ```
+                // fn bar(x: &mut i32) {}
+                //
+                // let x = 0u32;
+                // bar(&x); // error, expected &mut
+                // ```
+                let ref_ty = match mutability {
+                    hir::Mutability::Mut => {
+                        Ty::new_mut_ref(self.tcx, self.tcx.lifetimes.re_static, checked_ty)
+                    }
+                    hir::Mutability::Not => {
+                        Ty::new_imm_ref(self.tcx, self.tcx.lifetimes.re_static, checked_ty)
+                    }
+                };
+                if self.can_coerce(ref_ty, expected) {
+                    let mut sugg_sp = sp;
+                    if let hir::ExprKind::MethodCall(ref segment, receiver, args, _) = expr.kind {
+                        let clone_trait =
+                            self.tcx.require_lang_item(LangItem::Clone, Some(segment.ident.span));
+                        if args.is_empty()
+                            && self.typeck_results.borrow().type_dependent_def_id(expr.hir_id).map(
+                                |did| {
+                                    let ai = self.tcx.associated_item(did);
+                                    ai.trait_container(self.tcx) == Some(clone_trait)
+                                },
+                            ) == Some(true)
+                            && segment.ident.name == sym::clone
+                        {
+                            // If this expression had a clone call when suggesting borrowing
+                            // we want to suggest removing it because it'd now be unnecessary.
+                            sugg_sp = receiver.span;
+                        }
+                    }
+
+                    if let hir::ExprKind::Unary(hir::UnOp::Deref, ref inner) = expr.kind
+                        && let Some(1) = self.deref_steps(expected, checked_ty)
+                    {
+                        // We have `*&T`, check if what was expected was `&T`.
+                        // If so, we may want to suggest removing a `*`.
+                        sugg_sp = sugg_sp.with_hi(inner.span.lo());
+                        return Some((
+                            vec![(sugg_sp, String::new())],
+                            "consider removing deref here".to_string(),
+                            Applicability::MachineApplicable,
+                            true,
+                            false,
+                        ));
+                    }
+
+                    let needs_parens = match expr.kind {
+                        // parenthesize if needed (Issue #46756)
+                        hir::ExprKind::Cast(_, _) | hir::ExprKind::Binary(_, _, _) => true,
+                        // parenthesize borrows of range literals (Issue #54505)
+                        _ if is_range_literal(expr) => true,
+                        _ => false,
+                    };
+
+                    if let Some((sugg, msg)) = self.can_use_as_ref(expr) {
+                        return Some((
+                            sugg,
+                            msg.to_string(),
+                            Applicability::MachineApplicable,
+                            true,
+                            false,
+                        ));
+                    }
+
+                    let prefix = match self.tcx.hir().maybe_get_struct_pattern_shorthand_field(expr)
+                    {
+                        Some(ident) => format!("{ident}: "),
+                        None => String::new(),
+                    };
+
+                    if let Some(hir::Node::Expr(hir::Expr {
+                        kind: hir::ExprKind::Assign(..),
+                        ..
+                    })) = self.tcx.hir().find_parent(expr.hir_id)
+                    {
+                        if mutability.is_mut() {
+                            // Suppressing this diagnostic, we'll properly print it in `check_expr_assign`
+                            return None;
+                        }
+                    }
+
+                    let sugg = mutability.ref_prefix_str();
+                    let (sugg, verbose) = if needs_parens {
+                        (
+                            vec![
+                                (sp.shrink_to_lo(), format!("{prefix}{sugg}(")),
+                                (sp.shrink_to_hi(), ")".to_string()),
+                            ],
+                            false,
+                        )
+                    } else {
+                        (vec![(sp.shrink_to_lo(), format!("{prefix}{sugg}"))], true)
+                    };
+                    return Some((
+                        sugg,
+                        format!("consider {}borrowing here", mutability.mutably_str()),
+                        Applicability::MachineApplicable,
+                        verbose,
+                        false,
+                    ));
+                }
+            }
+            (
+                hir::ExprKind::AddrOf(hir::BorrowKind::Ref, _, ref expr),
+                _,
+                &ty::Ref(_, checked, _),
+            ) if self.can_sub(self.param_env, checked, expected) => {
+                let make_sugg = |start: Span, end: BytePos| {
+                    // skip `(` for tuples such as `(c) = (&123)`.
+                    // make sure we won't suggest like `(c) = 123)` which is incorrect.
+                    let sp = sm
+                        .span_extend_while(start.shrink_to_lo(), |c| c == '(' || c.is_whitespace())
+                        .map_or(start, |s| s.shrink_to_hi());
+                    Some((
+                        vec![(sp.with_hi(end), String::new())],
+                        "consider removing the borrow".to_string(),
+                        Applicability::MachineApplicable,
+                        true,
+                        true,
+                    ))
+                };
+
+                // We have `&T`, check if what was expected was `T`. If so,
+                // we may want to suggest removing a `&`.
+                if sm.is_imported(expr.span) {
+                    // Go through the spans from which this span was expanded,
+                    // and find the one that's pointing inside `sp`.
+                    //
+                    // E.g. for `&format!("")`, where we want the span to the
+                    // `format!()` invocation instead of its expansion.
+                    if let Some(call_span) =
+                        iter::successors(Some(expr.span), |s| s.parent_callsite())
+                            .find(|&s| sp.contains(s))
+                        && sm.is_span_accessible(call_span)
+                    {
+                        return make_sugg(sp, call_span.lo());
+                    }
+                    return None;
+                }
+                if sp.contains(expr.span) && sm.is_span_accessible(expr.span) {
+                    return make_sugg(sp, expr.span.lo());
+                }
+            }
+            (
+                _,
+                &ty::RawPtr(TypeAndMut { ty: ty_b, mutbl: mutbl_b }),
+                &ty::Ref(_, ty_a, mutbl_a),
+            ) => {
+                if let Some(steps) = self.deref_steps(ty_a, ty_b)
+                    // Only suggest valid if dereferencing needed.
+                    && steps > 0
+                    // The pointer type implements `Copy` trait so the suggestion is always valid.
+                    && let Ok(src) = sm.span_to_snippet(sp)
+                {
+                    let derefs = "*".repeat(steps);
+                    let old_prefix = mutbl_a.ref_prefix_str();
+                    let new_prefix = mutbl_b.ref_prefix_str().to_owned() + &derefs;
+
+                    let suggestion = replace_prefix(&src, old_prefix, &new_prefix).map(|_| {
+                        // skip `&` or `&mut ` if both mutabilities are mutable
+                        let lo = sp.lo()
+                            + BytePos(min(old_prefix.len(), mutbl_b.ref_prefix_str().len()) as _);
+                        // skip `&` or `&mut `
+                        let hi = sp.lo() + BytePos(old_prefix.len() as _);
+                        let sp = sp.with_lo(lo).with_hi(hi);
+
+                        (
+                            sp,
+                            format!(
+                                "{}{derefs}",
+                                if mutbl_a != mutbl_b { mutbl_b.prefix_str() } else { "" }
+                            ),
+                            if mutbl_b <= mutbl_a {
+                                Applicability::MachineApplicable
+                            } else {
+                                Applicability::MaybeIncorrect
+                            },
+                        )
+                    });
+
+                    if let Some((span, src, applicability)) = suggestion {
+                        return Some((
+                            vec![(span, src)],
+                            "consider dereferencing".to_string(),
+                            applicability,
+                            true,
+                            false,
+                        ));
+                    }
+                }
+            }
+            _ if sp == expr.span => {
+                if let Some(mut steps) = self.deref_steps(checked_ty, expected) {
+                    let mut expr = expr.peel_blocks();
+                    let mut prefix_span = expr.span.shrink_to_lo();
+                    let mut remove = String::new();
+
+                    // Try peeling off any existing `&` and `&mut` to reach our target type
+                    while steps > 0 {
+                        if let hir::ExprKind::AddrOf(_, mutbl, inner) = expr.kind {
+                            // If the expression has `&`, removing it would fix the error
+                            prefix_span = prefix_span.with_hi(inner.span.lo());
+                            expr = inner;
+                            remove.push_str(mutbl.ref_prefix_str());
+                            steps -= 1;
+                        } else {
+                            break;
+                        }
+                    }
+                    // If we've reached our target type with just removing `&`, then just print now.
+                    if steps == 0 && !remove.trim().is_empty() {
+                        return Some((
+                            vec![(prefix_span, String::new())],
+                            format!("consider removing the `{}`", remove.trim()),
+                            // Do not remove `&&` to get to bool, because it might be something like
+                            // { a } && b, which we have a separate fixup suggestion that is more
+                            // likely correct...
+                            if remove.trim() == "&&" && expected == self.tcx.types.bool {
+                                Applicability::MaybeIncorrect
+                            } else {
+                                Applicability::MachineApplicable
+                            },
+                            true,
+                            false,
+                        ));
+                    }
+
+                    // For this suggestion to make sense, the type would need to be `Copy`,
+                    // or we have to be moving out of a `Box<T>`
+                    if self.type_is_copy_modulo_regions(self.param_env, expected)
+                        // FIXME(compiler-errors): We can actually do this if the checked_ty is
+                        // `steps` layers of boxes, not just one, but this is easier and most likely.
+                        || (checked_ty.is_box() && steps == 1)
+                        // We can always deref a binop that takes its arguments by ref.
+                        || matches!(
+                            self.tcx.hir().get_parent(expr.hir_id),
+                            hir::Node::Expr(hir::Expr { kind: hir::ExprKind::Binary(op, ..), .. })
+                                if !op.node.is_by_value()
+                        )
+                    {
+                        let deref_kind = if checked_ty.is_box() {
+                            "unboxing the value"
+                        } else if checked_ty.is_ref() {
+                            "dereferencing the borrow"
+                        } else {
+                            "dereferencing the type"
+                        };
+
+                        // Suggest removing `&` if we have removed any, otherwise suggest just
+                        // dereferencing the remaining number of steps.
+                        let message = if remove.is_empty() {
+                            format!("consider {deref_kind}")
+                        } else {
+                            format!(
+                                "consider removing the `{}` and {} instead",
+                                remove.trim(),
+                                deref_kind
+                            )
+                        };
+
+                        let prefix =
+                            match self.tcx.hir().maybe_get_struct_pattern_shorthand_field(expr) {
+                                Some(ident) => format!("{ident}: "),
+                                None => String::new(),
+                            };
+
+                        let (span, suggestion) = if self.is_else_if_block(expr) {
+                            // Don't suggest nonsense like `else *if`
+                            return None;
+                        } else if let Some(expr) = self.maybe_get_block_expr(expr) {
+                            // prefix should be empty here..
+                            (expr.span.shrink_to_lo(), "*".to_string())
+                        } else {
+                            (prefix_span, format!("{}{}", prefix, "*".repeat(steps)))
+                        };
+                        if suggestion.trim().is_empty() {
+                            return None;
+                        }
+
+                        return Some((
+                            vec![(span, suggestion)],
+                            message,
+                            Applicability::MachineApplicable,
+                            true,
+                            false,
+                        ));
+                    }
+                }
+            }
+            _ => {}
+        }
+        None
+    }
+
+    /// Returns whether the given expression is an `else if`.
+    fn is_else_if_block(&self, expr: &hir::Expr<'_>) -> bool {
+        if let hir::ExprKind::If(..) = expr.kind {
+            let parent_id = self.tcx.hir().parent_id(expr.hir_id);
+            if let Some(Node::Expr(hir::Expr {
+                kind: hir::ExprKind::If(_, _, Some(else_expr)),
+                ..
+            })) = self.tcx.hir().find(parent_id)
+            {
+                return else_expr.hir_id == expr.hir_id;
+            }
+        }
+        false
+    }
+
+    pub(crate) fn suggest_cast(
+        &self,
+        err: &mut Diagnostic,
+        expr: &hir::Expr<'_>,
+        checked_ty: Ty<'tcx>,
+        expected_ty: Ty<'tcx>,
+        expected_ty_expr: Option<&'tcx hir::Expr<'tcx>>,
+    ) -> bool {
+        if self.tcx.sess.source_map().is_imported(expr.span) {
+            // Ignore if span is from within a macro.
+            return false;
+        }
+
+        let Ok(src) = self.tcx.sess.source_map().span_to_snippet(expr.span) else {
+            return false;
+        };
+
+        // If casting this expression to a given numeric type would be appropriate in case of a type
+        // mismatch.
+        //
+        // We want to minimize the amount of casting operations that are suggested, as it can be a
+        // lossy operation with potentially bad side effects, so we only suggest when encountering
+        // an expression that indicates that the original type couldn't be directly changed.
+        //
+        // For now, don't suggest casting with `as`.
+        let can_cast = false;
+
+        let mut sugg = vec![];
+
+        if let Some(hir::Node::ExprField(field)) = self.tcx.hir().find_parent(expr.hir_id) {
+            // `expr` is a literal field for a struct, only suggest if appropriate
+            if field.is_shorthand {
+                // This is a field literal
+                sugg.push((field.ident.span.shrink_to_lo(), format!("{}: ", field.ident)));
+            } else {
+                // Likely a field was meant, but this field wasn't found. Do not suggest anything.
+                return false;
+            }
+        };
+
+        if let hir::ExprKind::Call(path, args) = &expr.kind
+            && let (hir::ExprKind::Path(hir::QPath::TypeRelative(base_ty, path_segment)), 1) =
+                (&path.kind, args.len())
+            // `expr` is a conversion like `u32::from(val)`, do not suggest anything (#63697).
+            && let (hir::TyKind::Path(hir::QPath::Resolved(None, base_ty_path)), sym::from) =
+                (&base_ty.kind, path_segment.ident.name)
+        {
+            if let Some(ident) = &base_ty_path.segments.iter().map(|s| s.ident).next() {
+                match ident.name {
+                    sym::i128
+                    | sym::i64
+                    | sym::i32
+                    | sym::i16
+                    | sym::i8
+                    | sym::u128
+                    | sym::u64
+                    | sym::u32
+                    | sym::u16
+                    | sym::u8
+                    | sym::isize
+                    | sym::usize
+                        if base_ty_path.segments.len() == 1 =>
+                    {
+                        return false;
+                    }
+                    _ => {}
+                }
+            }
+        }
+
+        let msg = format!(
+            "you can convert {} `{}` to {} `{}`",
+            checked_ty.kind().article(),
+            checked_ty,
+            expected_ty.kind().article(),
+            expected_ty,
+        );
+        let cast_msg = format!(
+            "you can cast {} `{}` to {} `{}`",
+            checked_ty.kind().article(),
+            checked_ty,
+            expected_ty.kind().article(),
+            expected_ty,
+        );
+        let lit_msg = format!(
+            "change the type of the numeric literal from `{checked_ty}` to `{expected_ty}`",
+        );
+
+        let close_paren = if expr.precedence().order() < PREC_POSTFIX {
+            sugg.push((expr.span.shrink_to_lo(), "(".to_string()));
+            ")"
+        } else {
+            ""
+        };
+
+        let mut cast_suggestion = sugg.clone();
+        cast_suggestion.push((expr.span.shrink_to_hi(), format!("{close_paren} as {expected_ty}")));
+        let mut into_suggestion = sugg.clone();
+        into_suggestion.push((expr.span.shrink_to_hi(), format!("{close_paren}.into()")));
+        let mut suffix_suggestion = sugg.clone();
+        suffix_suggestion.push((
+            if matches!(
+                (&expected_ty.kind(), &checked_ty.kind()),
+                (ty::Int(_) | ty::Uint(_), ty::Float(_))
+            ) {
+                // Remove fractional part from literal, for example `42.0f32` into `42`
+                let src = src.trim_end_matches(&checked_ty.to_string());
+                let len = src.split('.').next().unwrap().len();
+                expr.span.with_lo(expr.span.lo() + BytePos(len as u32))
+            } else {
+                let len = src.trim_end_matches(&checked_ty.to_string()).len();
+                expr.span.with_lo(expr.span.lo() + BytePos(len as u32))
+            },
+            if expr.precedence().order() < PREC_POSTFIX {
+                // Readd `)`
+                format!("{expected_ty})")
+            } else {
+                expected_ty.to_string()
+            },
+        ));
+        let literal_is_ty_suffixed = |expr: &hir::Expr<'_>| {
+            if let hir::ExprKind::Lit(lit) = &expr.kind { lit.node.is_suffixed() } else { false }
+        };
+        let is_negative_int =
+            |expr: &hir::Expr<'_>| matches!(expr.kind, hir::ExprKind::Unary(hir::UnOp::Neg, ..));
+        let is_uint = |ty: Ty<'_>| matches!(ty.kind(), ty::Uint(..));
+
+        let in_const_context = self.tcx.hir().is_inside_const_context(expr.hir_id);
+
+        let suggest_fallible_into_or_lhs_from =
+            |err: &mut Diagnostic, exp_to_found_is_fallible: bool| {
+                // If we know the expression the expected type is derived from, we might be able
+                // to suggest a widening conversion rather than a narrowing one (which may
+                // panic). For example, given x: u8 and y: u32, if we know the span of "x",
+                //   x > y
+                // can be given the suggestion "u32::from(x) > y" rather than
+                // "x > y.try_into().unwrap()".
+                let lhs_expr_and_src = expected_ty_expr.and_then(|expr| {
+                    self.tcx
+                        .sess
+                        .source_map()
+                        .span_to_snippet(expr.span)
+                        .ok()
+                        .map(|src| (expr, src))
+                });
+                let (msg, suggestion) = if let (Some((lhs_expr, lhs_src)), false) =
+                    (lhs_expr_and_src, exp_to_found_is_fallible)
+                {
+                    let msg = format!(
+                        "you can convert `{lhs_src}` from `{expected_ty}` to `{checked_ty}`, matching the type of `{src}`",
+                    );
+                    let suggestion = vec![
+                        (lhs_expr.span.shrink_to_lo(), format!("{checked_ty}::from(")),
+                        (lhs_expr.span.shrink_to_hi(), ")".to_string()),
+                    ];
+                    (msg, suggestion)
+                } else {
+                    let msg =
+                        format!("{} and panic if the converted value doesn't fit", msg.clone());
+                    let mut suggestion = sugg.clone();
+                    suggestion.push((
+                        expr.span.shrink_to_hi(),
+                        format!("{close_paren}.try_into().unwrap()"),
+                    ));
+                    (msg, suggestion)
+                };
+                err.multipart_suggestion_verbose(msg, suggestion, Applicability::MachineApplicable);
+            };
+
+        let suggest_to_change_suffix_or_into =
+            |err: &mut Diagnostic,
+             found_to_exp_is_fallible: bool,
+             exp_to_found_is_fallible: bool| {
+                let exp_is_lhs = expected_ty_expr.is_some_and(|e| self.tcx.hir().is_lhs(e.hir_id));
+
+                if exp_is_lhs {
+                    return;
+                }
+
+                let always_fallible = found_to_exp_is_fallible
+                    && (exp_to_found_is_fallible || expected_ty_expr.is_none());
+                let msg = if literal_is_ty_suffixed(expr) {
+                    lit_msg.clone()
+                } else if always_fallible && (is_negative_int(expr) && is_uint(expected_ty)) {
+                    // We now know that converting either the lhs or rhs is fallible. Before we
+                    // suggest a fallible conversion, check if the value can never fit in the
+                    // expected type.
+                    let msg = format!("`{src}` cannot fit into type `{expected_ty}`");
+                    err.note(msg);
+                    return;
+                } else if in_const_context {
+                    // Do not recommend `into` or `try_into` in const contexts.
+                    return;
+                } else if found_to_exp_is_fallible {
+                    return suggest_fallible_into_or_lhs_from(err, exp_to_found_is_fallible);
+                } else {
+                    msg.clone()
+                };
+                let suggestion = if literal_is_ty_suffixed(expr) {
+                    suffix_suggestion.clone()
+                } else {
+                    into_suggestion.clone()
+                };
+                err.multipart_suggestion_verbose(msg, suggestion, Applicability::MachineApplicable);
+            };
+
+        match (&expected_ty.kind(), &checked_ty.kind()) {
+            (ty::Int(exp), ty::Int(found)) => {
+                let (f2e_is_fallible, e2f_is_fallible) = match (exp.bit_width(), found.bit_width())
+                {
+                    (Some(exp), Some(found)) if exp < found => (true, false),
+                    (Some(exp), Some(found)) if exp > found => (false, true),
+                    (None, Some(8 | 16)) => (false, true),
+                    (Some(8 | 16), None) => (true, false),
+                    (None, _) | (_, None) => (true, true),
+                    _ => (false, false),
+                };
+                suggest_to_change_suffix_or_into(err, f2e_is_fallible, e2f_is_fallible);
+                true
+            }
+            (ty::Uint(exp), ty::Uint(found)) => {
+                let (f2e_is_fallible, e2f_is_fallible) = match (exp.bit_width(), found.bit_width())
+                {
+                    (Some(exp), Some(found)) if exp < found => (true, false),
+                    (Some(exp), Some(found)) if exp > found => (false, true),
+                    (None, Some(8 | 16)) => (false, true),
+                    (Some(8 | 16), None) => (true, false),
+                    (None, _) | (_, None) => (true, true),
+                    _ => (false, false),
+                };
+                suggest_to_change_suffix_or_into(err, f2e_is_fallible, e2f_is_fallible);
+                true
+            }
+            (&ty::Int(exp), &ty::Uint(found)) => {
+                let (f2e_is_fallible, e2f_is_fallible) = match (exp.bit_width(), found.bit_width())
+                {
+                    (Some(exp), Some(found)) if found < exp => (false, true),
+                    (None, Some(8)) => (false, true),
+                    _ => (true, true),
+                };
+                suggest_to_change_suffix_or_into(err, f2e_is_fallible, e2f_is_fallible);
+                true
+            }
+            (&ty::Uint(exp), &ty::Int(found)) => {
+                let (f2e_is_fallible, e2f_is_fallible) = match (exp.bit_width(), found.bit_width())
+                {
+                    (Some(exp), Some(found)) if found > exp => (true, false),
+                    (Some(8), None) => (true, false),
+                    _ => (true, true),
+                };
+                suggest_to_change_suffix_or_into(err, f2e_is_fallible, e2f_is_fallible);
+                true
+            }
+            (ty::Float(exp), ty::Float(found)) => {
+                if found.bit_width() < exp.bit_width() {
+                    suggest_to_change_suffix_or_into(err, false, true);
+                } else if literal_is_ty_suffixed(expr) {
+                    err.multipart_suggestion_verbose(
+                        lit_msg,
+                        suffix_suggestion,
+                        Applicability::MachineApplicable,
+                    );
+                } else if can_cast {
+                    // Missing try_into implementation for `f64` to `f32`
+                    err.multipart_suggestion_verbose(
+                        format!("{cast_msg}, producing the closest possible value"),
+                        cast_suggestion,
+                        Applicability::MaybeIncorrect, // lossy conversion
+                    );
+                }
+                true
+            }
+            (&ty::Uint(_) | &ty::Int(_), &ty::Float(_)) => {
+                if literal_is_ty_suffixed(expr) {
+                    err.multipart_suggestion_verbose(
+                        lit_msg,
+                        suffix_suggestion,
+                        Applicability::MachineApplicable,
+                    );
+                } else if can_cast {
+                    // Missing try_into implementation for `{float}` to `{integer}`
+                    err.multipart_suggestion_verbose(
+                        format!("{msg}, rounding the float towards zero"),
+                        cast_suggestion,
+                        Applicability::MaybeIncorrect, // lossy conversion
+                    );
+                }
+                true
+            }
+            (ty::Float(exp), ty::Uint(found)) => {
+                // if `found` is `None` (meaning found is `usize`), don't suggest `.into()`
+                if exp.bit_width() > found.bit_width().unwrap_or(256) {
+                    err.multipart_suggestion_verbose(
+                        format!(
+                            "{msg}, producing the floating point representation of the integer",
+                        ),
+                        into_suggestion,
+                        Applicability::MachineApplicable,
+                    );
+                } else if literal_is_ty_suffixed(expr) {
+                    err.multipart_suggestion_verbose(
+                        lit_msg,
+                        suffix_suggestion,
+                        Applicability::MachineApplicable,
+                    );
+                } else {
+                    // Missing try_into implementation for `{integer}` to `{float}`
+                    err.multipart_suggestion_verbose(
+                        format!(
+                            "{cast_msg}, producing the floating point representation of the integer, \
+                                 rounded if necessary",
+                        ),
+                        cast_suggestion,
+                        Applicability::MaybeIncorrect, // lossy conversion
+                    );
+                }
+                true
+            }
+            (ty::Float(exp), ty::Int(found)) => {
+                // if `found` is `None` (meaning found is `isize`), don't suggest `.into()`
+                if exp.bit_width() > found.bit_width().unwrap_or(256) {
+                    err.multipart_suggestion_verbose(
+                        format!(
+                            "{}, producing the floating point representation of the integer",
+                            msg.clone(),
+                        ),
+                        into_suggestion,
+                        Applicability::MachineApplicable,
+                    );
+                } else if literal_is_ty_suffixed(expr) {
+                    err.multipart_suggestion_verbose(
+                        lit_msg,
+                        suffix_suggestion,
+                        Applicability::MachineApplicable,
+                    );
+                } else {
+                    // Missing try_into implementation for `{integer}` to `{float}`
+                    err.multipart_suggestion_verbose(
+                        format!(
+                            "{}, producing the floating point representation of the integer, \
+                                rounded if necessary",
+                            &msg,
+                        ),
+                        cast_suggestion,
+                        Applicability::MaybeIncorrect, // lossy conversion
+                    );
+                }
+                true
+            }
+            (
+                &ty::Uint(ty::UintTy::U32 | ty::UintTy::U64 | ty::UintTy::U128)
+                | &ty::Int(ty::IntTy::I32 | ty::IntTy::I64 | ty::IntTy::I128),
+                &ty::Char,
+            ) => {
+                err.multipart_suggestion_verbose(
+                    format!("{cast_msg}, since a `char` always occupies 4 bytes"),
+                    cast_suggestion,
+                    Applicability::MachineApplicable,
+                );
+                true
+            }
+            _ => false,
+        }
+    }
+
+    /// Identify when the user has written `foo..bar()` instead of `foo.bar()`.
+    pub(crate) fn suggest_method_call_on_range_literal(
+        &self,
+        err: &mut Diagnostic,
+        expr: &hir::Expr<'tcx>,
+        checked_ty: Ty<'tcx>,
+        expected_ty: Ty<'tcx>,
+    ) {
+        if !hir::is_range_literal(expr) {
+            return;
+        }
+        let hir::ExprKind::Struct(hir::QPath::LangItem(LangItem::Range, ..), [start, end], _) =
+            expr.kind
+        else {
+            return;
+        };
+        let parent = self.tcx.hir().parent_id(expr.hir_id);
+        if let Some(hir::Node::ExprField(_)) = self.tcx.hir().find(parent) {
+            // Ignore `Foo { field: a..Default::default() }`
+            return;
+        }
+        let mut expr = end.expr;
+        let mut expectation = Some(expected_ty);
+        while let hir::ExprKind::MethodCall(_, rcvr, ..) = expr.kind {
+            // Getting to the root receiver and asserting it is a fn call let's us ignore cases in
+            // `tests/ui/methods/issues/issue-90315.stderr`.
+            expr = rcvr;
+            // If we have more than one layer of calls, then the expected ty
+            // cannot guide the method probe.
+            expectation = None;
+        }
+        let hir::ExprKind::Call(method_name, _) = expr.kind else {
+            return;
+        };
+        let ty::Adt(adt, _) = checked_ty.kind() else {
+            return;
+        };
+        if self.tcx.lang_items().range_struct() != Some(adt.did()) {
+            return;
+        }
+        if let ty::Adt(adt, _) = expected_ty.kind()
+            && self.tcx.lang_items().range_struct() == Some(adt.did())
+        {
+            return;
+        }
+        // Check if start has method named end.
+        let hir::ExprKind::Path(hir::QPath::Resolved(None, p)) = method_name.kind else {
+            return;
+        };
+        let [hir::PathSegment { ident, .. }] = p.segments else {
+            return;
+        };
+        let self_ty = self.typeck_results.borrow().expr_ty(start.expr);
+        let Ok(_pick) = self.lookup_probe_for_diagnostic(
+            *ident,
+            self_ty,
+            expr,
+            probe::ProbeScope::AllTraits,
+            expectation,
+        ) else {
+            return;
+        };
+        let mut sugg = ".";
+        let mut span = start.expr.span.between(end.expr.span);
+        if span.lo() + BytePos(2) == span.hi() {
+            // There's no space between the start, the range op and the end, suggest removal which
+            // will be more noticeable than the replacement of `..` with `.`.
+            span = span.with_lo(span.lo() + BytePos(1));
+            sugg = "";
+        }
+        err.span_suggestion_verbose(
+            span,
+            "you likely meant to write a method call instead of a range",
+            sugg,
+            Applicability::MachineApplicable,
+        );
+    }
+
+    /// Identify when the type error is because `()` is found in a binding that was assigned a
+    /// block without a tail expression.
+    pub(crate) fn suggest_return_binding_for_missing_tail_expr(
+        &self,
+        err: &mut Diagnostic,
+        expr: &hir::Expr<'_>,
+        checked_ty: Ty<'tcx>,
+        expected_ty: Ty<'tcx>,
+    ) {
+        if !checked_ty.is_unit() {
+            return;
+        }
+        let hir::ExprKind::Path(hir::QPath::Resolved(None, path)) = expr.kind else {
+            return;
+        };
+        let hir::def::Res::Local(hir_id) = path.res else {
+            return;
+        };
+        let Some(hir::Node::Pat(pat)) = self.tcx.hir().find(hir_id) else {
+            return;
+        };
+        let Some(hir::Node::Local(hir::Local { ty: None, init: Some(init), .. })) =
+            self.tcx.hir().find_parent(pat.hir_id)
+        else {
+            return;
+        };
+        let hir::ExprKind::Block(block, None) = init.kind else {
+            return;
+        };
+        if block.expr.is_some() {
+            return;
+        }
+        let [.., stmt] = block.stmts else {
+            err.span_label(block.span, "this empty block is missing a tail expression");
+            return;
+        };
+        let hir::StmtKind::Semi(tail_expr) = stmt.kind else {
+            return;
+        };
+        let Some(ty) = self.node_ty_opt(tail_expr.hir_id) else {
+            return;
+        };
+        if self.can_eq(self.param_env, expected_ty, ty) {
+            err.span_suggestion_short(
+                stmt.span.with_lo(tail_expr.span.hi()),
+                "remove this semicolon",
+                "",
+                Applicability::MachineApplicable,
+            );
+        } else {
+            err.span_label(block.span, "this block is missing a tail expression");
+        }
+    }
 }

From 58a80c85b9323e59e1b7e744d6782997a3dcb5ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Le=C3=B3n=20Orell=20Valerian=20Liehr?= <me@fmease.dev>
Date: Fri, 9 Jun 2023 15:30:08 +0200
Subject: [PATCH 250/297] rustdoc: elide cross-crate default generic arguments

---
 src/librustdoc/clean/mod.rs                   |  12 +-
 src/librustdoc/clean/utils.rs                 | 130 ++++++++++++++----
 tests/rustdoc/const-generics/add-impl.rs      |   2 +-
 .../auxiliary/default-generic-args.rs         |  45 ++++++
 .../inline_cross/default-generic-args.rs      | 104 ++++++++++++++
 tests/rustdoc/inline_cross/dyn_trait.rs       |   8 +-
 tests/rustdoc/inline_cross/impl_trait.rs      |   4 +-
 tests/rustdoc/normalize-assoc-item.rs         |   4 +-
 tests/rustdoc/where-clause-order.rs           |   2 +-
 9 files changed, 271 insertions(+), 40 deletions(-)
 create mode 100644 tests/rustdoc/inline_cross/auxiliary/default-generic-args.rs
 create mode 100644 tests/rustdoc/inline_cross/default-generic-args.rs

diff --git a/src/librustdoc/clean/mod.rs b/src/librustdoc/clean/mod.rs
index f2447b877ca8..445f86a0b706 100644
--- a/src/librustdoc/clean/mod.rs
+++ b/src/librustdoc/clean/mod.rs
@@ -475,8 +475,9 @@ fn projection_to_path_segment<'tcx>(
     ty: ty::Binder<'tcx, ty::AliasTy<'tcx>>,
     cx: &mut DocContext<'tcx>,
 ) -> PathSegment {
-    let item = cx.tcx.associated_item(ty.skip_binder().def_id);
-    let generics = cx.tcx.generics_of(ty.skip_binder().def_id);
+    let def_id = ty.skip_binder().def_id;
+    let item = cx.tcx.associated_item(def_id);
+    let generics = cx.tcx.generics_of(def_id);
     PathSegment {
         name: item.name,
         args: GenericArgs::AngleBracketed {
@@ -484,7 +485,7 @@ fn projection_to_path_segment<'tcx>(
                 cx,
                 ty.map_bound(|ty| &ty.args[generics.parent_count..]),
                 false,
-                None,
+                def_id,
             )
             .into(),
             bindings: Default::default(),
@@ -2200,18 +2201,19 @@ pub(crate) fn clean_middle_ty<'tcx>(
         }
 
         ty::Alias(ty::Inherent, alias_ty) => {
+            let def_id = alias_ty.def_id;
             let alias_ty = bound_ty.rebind(alias_ty);
             let self_type = clean_middle_ty(alias_ty.map_bound(|ty| ty.self_ty()), cx, None, None);
 
             Type::QPath(Box::new(QPathData {
                 assoc: PathSegment {
-                    name: cx.tcx.associated_item(alias_ty.skip_binder().def_id).name,
+                    name: cx.tcx.associated_item(def_id).name,
                     args: GenericArgs::AngleBracketed {
                         args: ty_args_to_args(
                             cx,
                             alias_ty.map_bound(|ty| ty.args.as_slice()),
                             true,
-                            None,
+                            def_id,
                         )
                         .into(),
                         bindings: Default::default(),
diff --git a/src/librustdoc/clean/utils.rs b/src/librustdoc/clean/utils.rs
index c5302570489d..dea7bfaf7e22 100644
--- a/src/librustdoc/clean/utils.rs
+++ b/src/librustdoc/clean/utils.rs
@@ -17,6 +17,7 @@ use rustc_hir::def_id::{DefId, LocalDefId, LOCAL_CRATE};
 use rustc_metadata::rendered_const;
 use rustc_middle::mir;
 use rustc_middle::ty::{self, GenericArgKind, GenericArgsRef, TyCtxt};
+use rustc_middle::ty::{TypeVisitable, TypeVisitableExt};
 use rustc_span::symbol::{kw, sym, Symbol};
 use std::fmt::Write as _;
 use std::mem;
@@ -76,44 +77,123 @@ pub(crate) fn krate(cx: &mut DocContext<'_>) -> Crate {
 
 pub(crate) fn ty_args_to_args<'tcx>(
     cx: &mut DocContext<'tcx>,
-    args: ty::Binder<'tcx, &'tcx [ty::GenericArg<'tcx>]>,
+    ty_args: ty::Binder<'tcx, &'tcx [ty::GenericArg<'tcx>]>,
     has_self: bool,
-    container: Option<DefId>,
+    owner: DefId,
 ) -> Vec<GenericArg> {
-    let mut skip_first = has_self;
-    let mut ret_val =
-        Vec::with_capacity(args.skip_binder().len().saturating_sub(if skip_first { 1 } else { 0 }));
+    if ty_args.skip_binder().is_empty() {
+        // Fast path which avoids executing the query `generics_of`.
+        return Vec::new();
+    }
 
-    ret_val.extend(args.iter().enumerate().filter_map(|(index, kind)| {
-        match kind.skip_binder().unpack() {
-            GenericArgKind::Lifetime(lt) => {
-                Some(GenericArg::Lifetime(clean_middle_region(lt).unwrap_or(Lifetime::elided())))
+    let params = &cx.tcx.generics_of(owner).params;
+    let mut elision_has_failed_once_before = false;
+
+    let offset = if has_self { 1 } else { 0 };
+    let mut args = Vec::with_capacity(ty_args.skip_binder().len().saturating_sub(offset));
+
+    let ty_arg_to_arg = |(index, arg): (usize, &ty::GenericArg<'tcx>)| match arg.unpack() {
+        GenericArgKind::Lifetime(lt) => {
+            Some(GenericArg::Lifetime(clean_middle_region(lt).unwrap_or(Lifetime::elided())))
+        }
+        GenericArgKind::Type(_) if has_self && index == 0 => None,
+        GenericArgKind::Type(ty) => {
+            if !elision_has_failed_once_before
+                && let Some(default) = params[index].default_value(cx.tcx)
+            {
+                let default =
+                    ty_args.map_bound(|args| default.instantiate(cx.tcx, args).expect_ty());
+
+                if can_elide_generic_arg(ty_args.rebind(ty), default) {
+                    return None;
+                }
+
+                elision_has_failed_once_before = true;
             }
-            GenericArgKind::Type(_) if skip_first => {
-                skip_first = false;
-                None
-            }
-            GenericArgKind::Type(ty) => Some(GenericArg::Type(clean_middle_ty(
-                kind.rebind(ty),
+
+            Some(GenericArg::Type(clean_middle_ty(
+                ty_args.rebind(ty),
                 cx,
                 None,
-                container.map(|container| crate::clean::ContainerTy::Regular {
-                    ty: container,
-                    args,
+                Some(crate::clean::ContainerTy::Regular {
+                    ty: owner,
+                    args: ty_args,
                     has_self,
                     arg: index,
                 }),
-            ))),
+            )))
+        }
+        GenericArgKind::Const(ct) => {
             // FIXME(effects): this relies on the host effect being called `host`, which users could also name
             // their const generics.
             // FIXME(effects): this causes `host = true` and `host = false` generics to also be emitted.
-            GenericArgKind::Const(ct) if let ty::ConstKind::Param(p) = ct.kind() && p.name == sym::host => None,
-            GenericArgKind::Const(ct) => {
-                Some(GenericArg::Const(Box::new(clean_middle_const(kind.rebind(ct), cx))))
+            if let ty::ConstKind::Param(p) = ct.kind()
+                && p.name == sym::host
+            {
+                return None;
             }
+
+            if !elision_has_failed_once_before
+                && let Some(default) = params[index].default_value(cx.tcx)
+            {
+                let default =
+                    ty_args.map_bound(|args| default.instantiate(cx.tcx, args).expect_const());
+
+                if can_elide_generic_arg(ty_args.rebind(ct), default) {
+                    return None;
+                }
+
+                elision_has_failed_once_before = true;
+            }
+
+            Some(GenericArg::Const(Box::new(clean_middle_const(ty_args.rebind(ct), cx))))
         }
-    }));
-    ret_val
+    };
+
+    args.extend(ty_args.skip_binder().iter().enumerate().rev().filter_map(ty_arg_to_arg));
+    args.reverse();
+    args
+}
+
+/// Check if the generic argument `actual` coincides with the `default` and can therefore be elided.
+///
+/// This uses a very conservative approach for performance and correctness reasons, meaning for
+/// several classes of terms it claims that they cannot be elided even if they theoretically could.
+/// This is absolutely fine since it mostly concerns edge cases.
+fn can_elide_generic_arg<'tcx, Term>(
+    actual: ty::Binder<'tcx, Term>,
+    default: ty::Binder<'tcx, Term>,
+) -> bool
+where
+    Term: Eq + TypeVisitable<TyCtxt<'tcx>>,
+{
+    // In practice, we shouldn't have any inference variables at this point.
+    // However to be safe, we bail out if we do happen to stumble upon them.
+    if actual.has_infer() || default.has_infer() {
+        return false;
+    }
+
+    // Since we don't properly keep track of bound variables in rustdoc (yet), we don't attempt to
+    // make any sense out of escaping bound variables. We simply don't have enough context and it
+    // would be incorrect to try to do so anyway.
+    if actual.has_escaping_bound_vars() || default.has_escaping_bound_vars() {
+        return false;
+    }
+
+    // Theoretically we could now check if either term contains (non-escaping) late-bound regions or
+    // projections, relate the two using an `InferCtxt` and check if the resulting obligations hold.
+    // Having projections means that the terms can potentially be further normalized thereby possibly
+    // revealing that they are equal after all. Regarding late-bound regions, they could to be
+    // liberated allowing us to consider more types to be equal by ignoring the names of binders
+    // (e.g., `for<'a> TYPE<'a>` and `for<'b> TYPE<'b>`).
+    //
+    // However, we are mostly interested in “reeliding” generic args, i.e., eliding generic args that
+    // were originally elided by the user and later filled in by the compiler contrary to eliding
+    // arbitrary generic arguments if they happen to semantically coincide with the default (of course,
+    // we cannot possibly distinguish these two cases). Therefore and for performance reasons, it
+    // suffices to only perform a syntactic / structural check by comparing the memory addresses of
+    // the interned arguments.
+    actual.skip_binder() == default.skip_binder()
 }
 
 fn external_generic_args<'tcx>(
@@ -123,7 +203,7 @@ fn external_generic_args<'tcx>(
     bindings: ThinVec<TypeBinding>,
     ty_args: ty::Binder<'tcx, GenericArgsRef<'tcx>>,
 ) -> GenericArgs {
-    let args = ty_args_to_args(cx, ty_args.map_bound(|args| &args[..]), has_self, Some(did));
+    let args = ty_args_to_args(cx, ty_args.map_bound(|args| &args[..]), has_self, did);
 
     if cx.tcx.fn_trait_kind_from_def_id(did).is_some() {
         let ty = ty_args
diff --git a/tests/rustdoc/const-generics/add-impl.rs b/tests/rustdoc/const-generics/add-impl.rs
index 195e47bc8ba8..df490d2c636e 100644
--- a/tests/rustdoc/const-generics/add-impl.rs
+++ b/tests/rustdoc/const-generics/add-impl.rs
@@ -7,7 +7,7 @@ pub struct Simd<T, const WIDTH: usize> {
     inner: T,
 }
 
-// @has foo/struct.Simd.html '//div[@id="trait-implementations-list"]//h3[@class="code-header"]' 'impl Add<Simd<u8, 16>> for Simd<u8, 16>'
+// @has foo/struct.Simd.html '//div[@id="trait-implementations-list"]//h3[@class="code-header"]' 'impl Add for Simd<u8, 16>'
 impl Add for Simd<u8, 16> {
     type Output = Self;
 
diff --git a/tests/rustdoc/inline_cross/auxiliary/default-generic-args.rs b/tests/rustdoc/inline_cross/auxiliary/default-generic-args.rs
new file mode 100644
index 000000000000..1e31f18927e9
--- /dev/null
+++ b/tests/rustdoc/inline_cross/auxiliary/default-generic-args.rs
@@ -0,0 +1,45 @@
+pub type BoxedStr = Box<str>;
+pub type IntMap = std::collections::HashMap<i64, u64>;
+
+pub struct TyPair<T, U = T>(T, U);
+
+pub type T0 = TyPair<i32>;
+pub type T1 = TyPair<i32, u32>;
+pub type T2<K> = TyPair<i32, K>;
+pub type T3<Q> = TyPair<Q, Q>;
+
+pub struct CtPair<const C: u32, const D: u32 = C>;
+
+pub type C0 = CtPair<43, 43>;
+pub type C1 = CtPair<0, 1>;
+pub type C2 = CtPair<{1 + 2}, 3>;
+
+pub struct Re<'a, U = &'a ()>(&'a (), U);
+
+pub type R0<'q> = Re<'q>;
+pub type R1<'q> = Re<'q, &'q ()>;
+pub type R2<'q> = Re<'q, &'static ()>;
+pub type H0 = fn(for<'a> fn(Re<'a>));
+pub type H1 = for<'b> fn(for<'a> fn(Re<'a, &'b ()>));
+pub type H2 = for<'a> fn(for<'b> fn(Re<'a, &'b ()>));
+
+pub struct Proj<T: Basis, U = <T as Basis>::Assoc>(T, U);
+pub trait Basis { type Assoc; }
+impl Basis for () { type Assoc = bool; }
+
+pub type P0 = Proj<()>;
+pub type P1 = Proj<(), bool>;
+pub type P2 = Proj<(), ()>;
+
+pub struct Alpha<T = for<'any> fn(&'any ())>(T);
+
+pub type A0 = Alpha;
+pub type A1 = Alpha<for<'arbitrary> fn(&'arbitrary ())>;
+
+pub struct Multi<A = u64, B = u64>(A, B);
+
+pub type M0 = Multi<u64, ()>;
+
+pub trait Trait<'a, T = &'a ()> {}
+
+pub type F = dyn for<'a> Trait<'a>;
diff --git a/tests/rustdoc/inline_cross/default-generic-args.rs b/tests/rustdoc/inline_cross/default-generic-args.rs
new file mode 100644
index 000000000000..c9a87a199018
--- /dev/null
+++ b/tests/rustdoc/inline_cross/default-generic-args.rs
@@ -0,0 +1,104 @@
+#![crate_name = "user"]
+// aux-crate:default_generic_args=default-generic-args.rs
+// edition:2021
+
+// @has user/type.BoxedStr.html
+// @has - '//*[@class="rust item-decl"]//code' "Box<str>"
+pub use default_generic_args::BoxedStr;
+
+// @has user/type.IntMap.html
+// @has - '//*[@class="rust item-decl"]//code' "HashMap<i64, u64>"
+pub use default_generic_args::IntMap;
+
+// @has user/type.T0.html
+// @has - '//*[@class="rust item-decl"]//code' "TyPair<i32>"
+pub use default_generic_args::T0;
+
+// @has user/type.T1.html
+// @has - '//*[@class="rust item-decl"]//code' "TyPair<i32, u32>"
+pub use default_generic_args::T1;
+
+// @has user/type.T2.html
+// @has - '//*[@class="rust item-decl"]//code' "TyPair<i32, K>"
+pub use default_generic_args::T2;
+
+// @has user/type.T3.html
+// @has - '//*[@class="rust item-decl"]//code' "TyPair<Q>"
+pub use default_generic_args::T3;
+
+// @has user/type.C0.html
+// @has - '//*[@class="rust item-decl"]//code' "CtPair<43>"
+pub use default_generic_args::C0;
+
+// @has user/type.C1.html
+// @has - '//*[@class="rust item-decl"]//code' "CtPair<0, 1>"
+pub use default_generic_args::C1;
+
+// @has user/type.C2.html
+// @has - '//*[@class="rust item-decl"]//code' "CtPair<default_generic_args::::C2::{constant#0}, 3>"
+pub use default_generic_args::C2;
+
+// @has user/type.R0.html
+// @has - '//*[@class="rust item-decl"]//code' "Re<'q>"
+pub use default_generic_args::R0;
+
+// @has user/type.R1.html
+// @has - '//*[@class="rust item-decl"]//code' "Re<'q>"
+pub use default_generic_args::R1;
+
+// @has user/type.R2.html
+// Check that we consider regions:
+// @has - '//*[@class="rust item-decl"]//code' "Re<'q, &'static ()>"
+pub use default_generic_args::R2;
+
+// @has user/type.H0.html
+// Check that we handle higher-ranked regions correctly:
+// @has - '//*[@class="rust item-decl"]//code' "fn(_: for<'a> fn(_: Re<'a>))"
+pub use default_generic_args::H0;
+
+// @has user/type.H1.html
+// Check that we don't conflate distinct universially quantified regions (#1):
+// @has - '//*[@class="rust item-decl"]//code' "for<'b> fn(_: for<'a> fn(_: Re<'a, &'b ()>))"
+pub use default_generic_args::H1;
+
+// @has user/type.H2.html
+// Check that we don't conflate distinct universially quantified regions (#2):
+// @has - '//*[@class="rust item-decl"]//code' "for<'a> fn(_: for<'b> fn(_: Re<'a, &'b ()>))"
+pub use default_generic_args::H2;
+
+// @has user/type.P0.html
+// @has - '//*[@class="rust item-decl"]//code' "Proj<()>"
+pub use default_generic_args::P0;
+
+// @has user/type.P1.html
+// @has - '//*[@class="rust item-decl"]//code' "Proj<(), bool>"
+pub use default_generic_args::P1;
+
+// @has user/type.P2.html
+// @has - '//*[@class="rust item-decl"]//code' "Proj<(), ()>"
+pub use default_generic_args::P2;
+
+// @has user/type.A0.html
+// Ensure that we elide generic arguments that are alpha-equivalent to their respective
+// generic parameter (modulo substs) (#1):
+// @has - '//*[@class="rust item-decl"]//code' "Alpha"
+pub use default_generic_args::A0;
+
+// @has user/type.A1.html
+// Ensure that we elide generic arguments that are alpha-equivalent to their respective
+// generic parameter (modulo substs) (#1):
+// @has - '//*[@class="rust item-decl"]//code' "Alpha"
+pub use default_generic_args::A1;
+
+// @has user/type.M0.html
+// Test that we don't elide `u64` even if it coincides with `A`'s default precisely because
+// `()` is not the default of `B`. Mindlessly eliding `u64` would lead to `M<()>` which is a
+// different type (`M<(), u64>` versus `M<u64, ()>`).
+// @has - '//*[@class="rust item-decl"]//code' "Multi<u64, ()>"
+pub use default_generic_args::M0;
+
+// @has user/type.F.html
+// FIXME: Ideally, we would elide `&'a ()` but `'a` is an escaping bound var which we can't reason
+//        about at the moment since we don't keep track of bound vars.
+// @has - '//*[@class="rust item-decl"]//code' "dyn for<'a> Trait<'a, &'a ()>"
+pub use default_generic_args::F;
diff --git a/tests/rustdoc/inline_cross/dyn_trait.rs b/tests/rustdoc/inline_cross/dyn_trait.rs
index 679972f035ad..9871be79ca36 100644
--- a/tests/rustdoc/inline_cross/dyn_trait.rs
+++ b/tests/rustdoc/inline_cross/dyn_trait.rs
@@ -75,16 +75,16 @@ pub use dyn_trait::AmbiguousBoundWrappedEarly1;
 pub use dyn_trait::AmbiguousBoundWrappedStatic;
 
 // @has user/type.NoBoundsWrappedDefaulted.html
-// @has - '//*[@class="rust item-decl"]//code' "Box<dyn Trait, Global>;"
+// @has - '//*[@class="rust item-decl"]//code' "Box<dyn Trait>;"
 pub use dyn_trait::NoBoundsWrappedDefaulted;
 // @has user/type.NoBoundsWrappedEarly.html
-// @has - '//*[@class="rust item-decl"]//code' "Box<dyn Trait + 'e, Global>;"
+// @has - '//*[@class="rust item-decl"]//code' "Box<dyn Trait + 'e>;"
 pub use dyn_trait::NoBoundsWrappedEarly;
 // @has user/fn.nbwl.html
-// @has - '//pre[@class="rust item-decl"]' "nbwl<'l>(_: Box<dyn Trait + 'l, Global>)"
+// @has - '//pre[@class="rust item-decl"]' "nbwl<'l>(_: Box<dyn Trait + 'l>)"
 pub use dyn_trait::no_bounds_wrapped_late as nbwl;
 // @has user/fn.nbwel.html
-// @has - '//pre[@class="rust item-decl"]' "nbwel(_: Box<dyn Trait + '_, Global>)"
+// @has - '//pre[@class="rust item-decl"]' "nbwel(_: Box<dyn Trait + '_>)"
 // NB: It might seem counterintuitive to display the explicitly elided lifetime `'_` here instead of
 // eliding it but this behavior is correct: The default is `'static` here which != `'_`.
 pub use dyn_trait::no_bounds_wrapped_elided as nbwel;
diff --git a/tests/rustdoc/inline_cross/impl_trait.rs b/tests/rustdoc/inline_cross/impl_trait.rs
index 9b22026e4906..3a2f5d160045 100644
--- a/tests/rustdoc/inline_cross/impl_trait.rs
+++ b/tests/rustdoc/inline_cross/impl_trait.rs
@@ -4,7 +4,7 @@
 extern crate impl_trait_aux;
 
 // @has impl_trait/fn.func.html
-// @has - '//pre[@class="rust item-decl"]' "pub fn func<'a>(_x: impl Clone + Into<Vec<u8, Global>> + 'a)"
+// @has - '//pre[@class="rust item-decl"]' "pub fn func<'a>(_x: impl Clone + Into<Vec<u8>> + 'a)"
 // @!has - '//pre[@class="rust item-decl"]' 'where'
 pub use impl_trait_aux::func;
 
@@ -34,6 +34,6 @@ pub use impl_trait_aux::func4;
 pub use impl_trait_aux::func5;
 
 // @has impl_trait/struct.Foo.html
-// @has - '//*[@id="method.method"]//h4[@class="code-header"]' "pub fn method<'a>(_x: impl Clone + Into<Vec<u8, Global>> + 'a)"
+// @has - '//*[@id="method.method"]//h4[@class="code-header"]' "pub fn method<'a>(_x: impl Clone + Into<Vec<u8>> + 'a)"
 // @!has - '//*[@id="method.method"]//h4[@class="code-header"]' 'where'
 pub use impl_trait_aux::Foo;
diff --git a/tests/rustdoc/normalize-assoc-item.rs b/tests/rustdoc/normalize-assoc-item.rs
index c6fd5e1101ef..d39e1b15a4cb 100644
--- a/tests/rustdoc/normalize-assoc-item.rs
+++ b/tests/rustdoc/normalize-assoc-item.rs
@@ -30,7 +30,7 @@ pub fn f2() -> <isize as Trait>::X {
 }
 
 pub struct S {
-    // @has 'normalize_assoc_item/struct.S.html' '//span[@id="structfield.box_me_up"]' 'box_me_up: Box<S, Global>'
+    // @has 'normalize_assoc_item/struct.S.html' '//span[@id="structfield.box_me_up"]' 'box_me_up: Box<S>'
     pub box_me_up: <S as Trait>::X,
     // @has 'normalize_assoc_item/struct.S.html' '//span[@id="structfield.generic"]' 'generic: (usize, isize)'
     pub generic: <Generic<usize> as Trait>::X,
@@ -76,7 +76,7 @@ extern crate inner;
 // @has 'normalize_assoc_item/fn.foo.html' '//pre[@class="rust item-decl"]' "pub fn foo() -> i32"
 pub use inner::foo;
 
-// @has 'normalize_assoc_item/fn.h.html' '//pre[@class="rust item-decl"]' "pub fn h<T>() -> IntoIter<T, Global>"
+// @has 'normalize_assoc_item/fn.h.html' '//pre[@class="rust item-decl"]' "pub fn h<T>() -> IntoIter<T>"
 pub fn h<T>() -> <Vec<T> as IntoIterator>::IntoIter {
     vec![].into_iter()
 }
diff --git a/tests/rustdoc/where-clause-order.rs b/tests/rustdoc/where-clause-order.rs
index 7261dfa7dd91..e3184b646bf5 100644
--- a/tests/rustdoc/where-clause-order.rs
+++ b/tests/rustdoc/where-clause-order.rs
@@ -7,7 +7,7 @@ where
 }
 
 // @has 'foo/trait.SomeTrait.html'
-// @has - "//*[@id='impl-SomeTrait-for-(A,+B,+C,+D,+E)']/h3" "impl<A, B, C, D, E> SomeTrait<(A, B, C, D, E)> for (A, B, C, D, E)where A: PartialOrd<A> + PartialEq<A>, B: PartialOrd<B> + PartialEq<B>, C: PartialOrd<C> + PartialEq<C>, D: PartialOrd<D> + PartialEq<D>, E: PartialOrd<E> + PartialEq<E> + ?Sized, "
+// @has - "//*[@id='impl-SomeTrait-for-(A,+B,+C,+D,+E)']/h3" "impl<A, B, C, D, E> SomeTrait for (A, B, C, D, E)where A: PartialOrd<A> + PartialEq<A>, B: PartialOrd<B> + PartialEq<B>, C: PartialOrd<C> + PartialEq<C>, D: PartialOrd<D> + PartialEq<D>, E: PartialOrd<E> + PartialEq<E> + ?Sized, "
 impl<A, B, C, D, E> SomeTrait<(A, B, C, D, E)> for (A, B, C, D, E)
 where
     A: PartialOrd<A> + PartialEq<A>,

From 1e10fe9eb67fabb97729c3faf4852121f1f608da Mon Sep 17 00:00:00 2001
From: David Tolnay <dtolnay@gmail.com>
Date: Mon, 30 Oct 2023 09:00:13 -0700
Subject: [PATCH 251/297] Move deprecation_in_effect to inherent method on
 Deprecation

---
 compiler/rustc_attr/src/builtin.rs            | 16 ++++++++++++++++
 compiler/rustc_middle/src/middle/stability.rs | 19 +++----------------
 src/librustdoc/html/render/mod.rs             |  3 +--
 src/librustdoc/html/render/print_item.rs      |  7 +------
 4 files changed, 21 insertions(+), 24 deletions(-)

diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs
index 62709ef4db95..49c6b84a4a5c 100644
--- a/compiler/rustc_attr/src/builtin.rs
+++ b/compiler/rustc_attr/src/builtin.rs
@@ -744,6 +744,22 @@ pub enum DeprecatedSince {
     Symbol(Symbol),
 }
 
+impl Deprecation {
+    /// Whether an item marked with #[deprecated(since = "X")] is currently
+    /// deprecated (i.e., whether X is not greater than the current rustc
+    /// version).
+    pub fn is_in_effect(&self) -> bool {
+        match self.since {
+            Some(DeprecatedSince::RustcVersion(since)) => since <= RustcVersion::CURRENT,
+            Some(DeprecatedSince::Future) => false,
+            // The `since` field doesn't have semantic purpose without `#![staged_api]`.
+            Some(DeprecatedSince::Symbol(_)) => true,
+            // Assume deprecation is in effect if "since" field is missing.
+            None => true,
+        }
+    }
+}
+
 impl Display for DeprecatedSince {
     fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
diff --git a/compiler/rustc_middle/src/middle/stability.rs b/compiler/rustc_middle/src/middle/stability.rs
index b750352e13a6..27c2010fe297 100644
--- a/compiler/rustc_middle/src/middle/stability.rs
+++ b/compiler/rustc_middle/src/middle/stability.rs
@@ -18,7 +18,7 @@ use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_session::lint::builtin::{DEPRECATED, DEPRECATED_IN_FUTURE, SOFT_UNSTABLE};
 use rustc_session::lint::{BuiltinLintDiagnostics, Level, Lint, LintBuffer};
 use rustc_session::parse::feature_err_issue;
-use rustc_session::{RustcVersion, Session};
+use rustc_session::Session;
 use rustc_span::symbol::{sym, Symbol};
 use rustc_span::Span;
 use std::num::NonZeroU32;
@@ -125,19 +125,6 @@ pub fn report_unstable(
     }
 }
 
-/// Checks whether an item marked with `deprecated(since="X")` is currently
-/// deprecated (i.e., whether X is not greater than the current rustc version).
-pub fn deprecation_in_effect(depr: &Deprecation) -> bool {
-    match depr.since {
-        Some(DeprecatedSince::RustcVersion(since)) => since <= RustcVersion::CURRENT,
-        Some(DeprecatedSince::Future) => false,
-        // The `since` field doesn't have semantic purpose without `#![staged_api]`.
-        Some(DeprecatedSince::Symbol(_)) => true,
-        // Assume deprecation is in effect if "since" field is missing.
-        None => true,
-    }
-}
-
 pub fn deprecation_suggestion(
     diag: &mut Diagnostic,
     kind: &str,
@@ -191,7 +178,7 @@ pub fn deprecation_message_and_lint(
     kind: &str,
     path: &str,
 ) -> (String, &'static Lint) {
-    let is_in_effect = deprecation_in_effect(depr);
+    let is_in_effect = depr.is_in_effect();
     (
         deprecation_message(is_in_effect, depr.since, depr.note, kind, path),
         deprecation_lint(is_in_effect),
@@ -363,7 +350,7 @@ impl<'tcx> TyCtxt<'tcx> {
                     // Calculating message for lint involves calling `self.def_path_str`.
                     // Which by default to calculate visible path will invoke expensive `visible_parent_map` query.
                     // So we skip message calculation altogether, if lint is allowed.
-                    let is_in_effect = deprecation_in_effect(depr_attr);
+                    let is_in_effect = depr_attr.is_in_effect();
                     let lint = deprecation_lint(is_in_effect);
                     if self.lint_level_at_node(lint, id).0 != Level::Allow {
                         let def_path = with_no_trimmed_paths!(self.def_path_str(def_id));
diff --git a/src/librustdoc/html/render/mod.rs b/src/librustdoc/html/render/mod.rs
index e852d02b6acb..cf54dc3d6b7e 100644
--- a/src/librustdoc/html/render/mod.rs
+++ b/src/librustdoc/html/render/mod.rs
@@ -53,7 +53,6 @@ use rustc_data_structures::captures::Captures;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_hir::def_id::{DefId, DefIdSet};
 use rustc_hir::Mutability;
-use rustc_middle::middle::stability;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_session::RustcVersion;
 use rustc_span::{
@@ -621,7 +620,7 @@ fn short_item_info(
         // We display deprecation messages for #[deprecated], but only display
         // the future-deprecation messages for rustc versions.
         let mut message = if let Some(since) = since {
-            if !stability::deprecation_in_effect(&depr) {
+            if !depr.is_in_effect() {
                 if let DeprecatedSince::Future = since {
                     String::from("Deprecating in a future Rust version")
                 } else {
diff --git a/src/librustdoc/html/render/print_item.rs b/src/librustdoc/html/render/print_item.rs
index fdf455690617..1c5a1dc99add 100644
--- a/src/librustdoc/html/render/print_item.rs
+++ b/src/librustdoc/html/render/print_item.rs
@@ -6,7 +6,6 @@ use rustc_hir as hir;
 use rustc_hir::def::CtorKind;
 use rustc_hir::def_id::DefId;
 use rustc_index::IndexVec;
-use rustc_middle::middle::stability;
 use rustc_middle::query::Key;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_span::hygiene::MacroKind;
@@ -591,11 +590,7 @@ fn extra_info_tags<'a, 'tcx: 'a>(
 
         // The trailing space after each tag is to space it properly against the rest of the docs.
         if let Some(depr) = &item.deprecation(tcx) {
-            let message = if stability::deprecation_in_effect(depr) {
-                "Deprecated"
-            } else {
-                "Deprecation planned"
-            };
+            let message = if depr.is_in_effect() { "Deprecated" } else { "Deprecation planned" };
             write!(f, "{}", tag_html("deprecated", "", message))?;
         }
 

From ff3a818554a26c56c1ad0381307eed938b5bbbc5 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Mon, 30 Oct 2023 15:54:08 +0000
Subject: [PATCH 252/297] Poison check_well_formed if method receivers are
 invalid to prevent typeck from running on it

---
 .../rustc_hir_analysis/src/check/wfcheck.rs   | 35 ++++++++++++-------
 ...ary-self-from-method-substs.default.stderr | 13 +++++++
 ...ry-self-from-method-substs.feature.stderr} |  2 +-
 .../self/arbitrary-self-from-method-substs.rs |  8 +++--
 4 files changed, 41 insertions(+), 17 deletions(-)
 create mode 100644 tests/ui/self/arbitrary-self-from-method-substs.default.stderr
 rename tests/ui/self/{arbitrary-self-from-method-substs.stderr => arbitrary-self-from-method-substs.feature.stderr} (80%)

diff --git a/compiler/rustc_hir_analysis/src/check/wfcheck.rs b/compiler/rustc_hir_analysis/src/check/wfcheck.rs
index 3f31ce7aa58b..046983e90f7b 100644
--- a/compiler/rustc_hir_analysis/src/check/wfcheck.rs
+++ b/compiler/rustc_hir_analysis/src/check/wfcheck.rs
@@ -94,7 +94,7 @@ pub(super) fn enter_wf_checking_ctxt<'tcx, F>(
     f: F,
 ) -> Result<(), ErrorGuaranteed>
 where
-    F: for<'a> FnOnce(&WfCheckingCtxt<'a, 'tcx>),
+    F: for<'a> FnOnce(&WfCheckingCtxt<'a, 'tcx>) -> Result<(), ErrorGuaranteed>,
 {
     let param_env = tcx.param_env(body_def_id);
     let infcx = &tcx.infer_ctxt().build();
@@ -105,7 +105,7 @@ where
     if !tcx.features().trivial_bounds {
         wfcx.check_false_global_bounds()
     }
-    f(&mut wfcx);
+    f(&mut wfcx)?;
 
     let assumed_wf_types = wfcx.ocx.assumed_wf_types_and_report_errors(param_env, body_def_id)?;
 
@@ -875,6 +875,7 @@ fn check_param_wf(tcx: TyCtxt<'_>, param: &hir::GenericParam<'_>) -> Result<(),
                         ty,
                         trait_def_id,
                     );
+                    Ok(())
                 })
             } else {
                 let mut diag = match ty.kind() {
@@ -961,6 +962,7 @@ fn check_associated_item(
                 let ty = tcx.type_of(item.def_id).instantiate_identity();
                 let ty = wfcx.normalize(span, Some(WellFormedLoc::Ty(item_id)), ty);
                 wfcx.register_wf_obligation(span, loc, ty.into());
+                Ok(())
             }
             ty::AssocKind::Fn => {
                 let sig = tcx.fn_sig(item.def_id).instantiate_identity();
@@ -972,7 +974,7 @@ fn check_associated_item(
                     hir_sig.decl,
                     item.def_id.expect_local(),
                 );
-                check_method_receiver(wfcx, hir_sig, item, self_ty);
+                check_method_receiver(wfcx, hir_sig, item, self_ty)
             }
             ty::AssocKind::Type => {
                 if let ty::AssocItemContainer::TraitContainer = item.container {
@@ -983,6 +985,7 @@ fn check_associated_item(
                     let ty = wfcx.normalize(span, Some(WellFormedLoc::Ty(item_id)), ty);
                     wfcx.register_wf_obligation(span, loc, ty.into());
                 }
+                Ok(())
             }
         }
     })
@@ -1097,6 +1100,7 @@ fn check_type_defn<'tcx>(
         }
 
         check_where_clauses(wfcx, item.span, item.owner_id.def_id);
+        Ok(())
     })
 }
 
@@ -1121,7 +1125,8 @@ fn check_trait(tcx: TyCtxt<'_>, item: &hir::Item<'_>) -> Result<(), ErrorGuarant
     }
 
     let res = enter_wf_checking_ctxt(tcx, item.span, def_id, |wfcx| {
-        check_where_clauses(wfcx, item.span, def_id)
+        check_where_clauses(wfcx, item.span, def_id);
+        Ok(())
     });
 
     // Only check traits, don't check trait aliases
@@ -1164,6 +1169,7 @@ fn check_item_fn(
     enter_wf_checking_ctxt(tcx, span, def_id, |wfcx| {
         let sig = tcx.fn_sig(def_id).instantiate_identity();
         check_fn_or_method(wfcx, ident.span, sig, decl, def_id);
+        Ok(())
     })
 }
 
@@ -1218,6 +1224,7 @@ fn check_item_type(
                 tcx.require_lang_item(LangItem::Sync, Some(ty_span)),
             );
         }
+        Ok(())
     })
 }
 
@@ -1276,6 +1283,7 @@ fn check_impl<'tcx>(
         }
 
         check_where_clauses(wfcx, item.span, item.owner_id.def_id);
+        Ok(())
     })
 }
 
@@ -1548,11 +1556,11 @@ fn check_method_receiver<'tcx>(
     fn_sig: &hir::FnSig<'_>,
     method: ty::AssocItem,
     self_ty: Ty<'tcx>,
-) {
+) -> Result<(), ErrorGuaranteed> {
     let tcx = wfcx.tcx();
 
     if !method.fn_has_self_parameter {
-        return;
+        return Ok(());
     }
 
     let span = fn_sig.decl.inputs[0].span;
@@ -1571,11 +1579,11 @@ fn check_method_receiver<'tcx>(
     if tcx.features().arbitrary_self_types {
         if !receiver_is_valid(wfcx, span, receiver_ty, self_ty, true) {
             // Report error; `arbitrary_self_types` was enabled.
-            e0307(tcx, span, receiver_ty);
+            return Err(e0307(tcx, span, receiver_ty));
         }
     } else {
         if !receiver_is_valid(wfcx, span, receiver_ty, self_ty, false) {
-            if receiver_is_valid(wfcx, span, receiver_ty, self_ty, true) {
+            return Err(if receiver_is_valid(wfcx, span, receiver_ty, self_ty, true) {
                 // Report error; would have worked with `arbitrary_self_types`.
                 feature_err(
                     &tcx.sess.parse_sess,
@@ -1587,16 +1595,17 @@ fn check_method_receiver<'tcx>(
                     ),
                 )
                 .help(HELP_FOR_SELF_TYPE)
-                .emit();
+                .emit()
             } else {
                 // Report error; would not have worked with `arbitrary_self_types`.
-                e0307(tcx, span, receiver_ty);
-            }
+                e0307(tcx, span, receiver_ty)
+            });
         }
     }
+    Ok(())
 }
 
-fn e0307(tcx: TyCtxt<'_>, span: Span, receiver_ty: Ty<'_>) {
+fn e0307(tcx: TyCtxt<'_>, span: Span, receiver_ty: Ty<'_>) -> ErrorGuaranteed {
     struct_span_err!(
         tcx.sess.diagnostic(),
         span,
@@ -1605,7 +1614,7 @@ fn e0307(tcx: TyCtxt<'_>, span: Span, receiver_ty: Ty<'_>) {
     )
     .note("type of `self` must be `Self` or a type that dereferences to it")
     .help(HELP_FOR_SELF_TYPE)
-    .emit();
+    .emit()
 }
 
 /// Returns whether `receiver_ty` would be considered a valid receiver type for `self_ty`. If
diff --git a/tests/ui/self/arbitrary-self-from-method-substs.default.stderr b/tests/ui/self/arbitrary-self-from-method-substs.default.stderr
new file mode 100644
index 000000000000..cbf5e6c541a3
--- /dev/null
+++ b/tests/ui/self/arbitrary-self-from-method-substs.default.stderr
@@ -0,0 +1,13 @@
+error[E0658]: `R` cannot be used as the type of `self` without the `arbitrary_self_types` feature
+  --> $DIR/arbitrary-self-from-method-substs.rs:8:43
+   |
+LL |     fn get<R: Deref<Target = Self>>(self: R) -> u32 {
+   |                                           ^
+   |
+   = note: see issue #44874 <https://github.com/rust-lang/rust/issues/44874> for more information
+   = help: add `#![feature(arbitrary_self_types)]` to the crate attributes to enable
+   = help: consider changing to `self`, `&self`, `&mut self`, `self: Box<Self>`, `self: Rc<Self>`, `self: Arc<Self>`, or `self: Pin<P>` (where P is one of the previous types except `Self`)
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0658`.
diff --git a/tests/ui/self/arbitrary-self-from-method-substs.stderr b/tests/ui/self/arbitrary-self-from-method-substs.feature.stderr
similarity index 80%
rename from tests/ui/self/arbitrary-self-from-method-substs.stderr
rename to tests/ui/self/arbitrary-self-from-method-substs.feature.stderr
index 6c252fadf465..7378d53c3738 100644
--- a/tests/ui/self/arbitrary-self-from-method-substs.stderr
+++ b/tests/ui/self/arbitrary-self-from-method-substs.feature.stderr
@@ -1,5 +1,5 @@
 error[E0308]: mismatched types
-  --> $DIR/arbitrary-self-from-method-substs.rs:14:5
+  --> $DIR/arbitrary-self-from-method-substs.rs:16:5
    |
 LL |     foo.get::<&Foo>();
    |     ^^^ expected `&Foo`, found `Foo`
diff --git a/tests/ui/self/arbitrary-self-from-method-substs.rs b/tests/ui/self/arbitrary-self-from-method-substs.rs
index 0f911a20842b..004445dc3272 100644
--- a/tests/ui/self/arbitrary-self-from-method-substs.rs
+++ b/tests/ui/self/arbitrary-self-from-method-substs.rs
@@ -1,10 +1,12 @@
-#![feature(arbitrary_self_types)]
+// revisions: default feature
+#![cfg_attr(feature, feature(arbitrary_self_types))]
 
 use std::ops::Deref;
 
 struct Foo(u32);
 impl Foo {
-    fn get<R: Deref<Target=Self>>(self: R) -> u32 {
+    fn get<R: Deref<Target = Self>>(self: R) -> u32 {
+        //[default]~^ ERROR: `R` cannot be used as the type of `self`
         self.0
     }
 }
@@ -12,5 +14,5 @@ impl Foo {
 fn main() {
     let mut foo = Foo(1);
     foo.get::<&Foo>();
-    //~^ ERROR mismatched types
+    //[feature]~^ ERROR mismatched types
 }

From 224ddf8fd9ee1d30053ae55a5d9fd49711b53360 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Mon, 30 Oct 2023 16:05:25 +0000
Subject: [PATCH 253/297] Only run panic tests on targets that can unwind

---
 tests/ui/coroutine/gen_block_panic.rs     | 1 +
 tests/ui/coroutine/gen_block_panic.stderr | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/ui/coroutine/gen_block_panic.rs b/tests/ui/coroutine/gen_block_panic.rs
index 33e2b4b265a1..2da0eb512cc0 100644
--- a/tests/ui/coroutine/gen_block_panic.rs
+++ b/tests/ui/coroutine/gen_block_panic.rs
@@ -1,5 +1,6 @@
 //compile-flags: --edition 2024 -Zunstable-options
 // run-pass
+// needs-unwind
 #![feature(gen_blocks)]
 
 fn main() {
diff --git a/tests/ui/coroutine/gen_block_panic.stderr b/tests/ui/coroutine/gen_block_panic.stderr
index 33d2a95cab8b..a0a6d1063c45 100644
--- a/tests/ui/coroutine/gen_block_panic.stderr
+++ b/tests/ui/coroutine/gen_block_panic.stderr
@@ -1,5 +1,5 @@
 warning: unreachable statement
-  --> $DIR/gen_block_panic.rs:9:9
+  --> $DIR/gen_block_panic.rs:10:9
    |
 LL |         panic!("foo");
    |         ------------- any code following this expression is unreachable

From 162443b32e49bfc30ef57e2e9f9c3ede0edea5aa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Esteban=20K=C3=BCber?= <esteban@kuber.com.ar>
Date: Tue, 17 Oct 2023 22:06:58 +0000
Subject: [PATCH 254/297] Detect when trait is implemented for type and suggest
 importing it

Fix #57457.
---
 .../rustc_hir_typeck/src/method/suggest.rs    | 56 +++++++++++++------
 tests/ui/traits/item-privacy.stderr           | 21 +++----
 2 files changed, 49 insertions(+), 28 deletions(-)

diff --git a/compiler/rustc_hir_typeck/src/method/suggest.rs b/compiler/rustc_hir_typeck/src/method/suggest.rs
index 3ed22e095e89..12cc5ed2f1a5 100644
--- a/compiler/rustc_hir_typeck/src/method/suggest.rs
+++ b/compiler/rustc_hir_typeck/src/method/suggest.rs
@@ -35,6 +35,7 @@ use rustc_span::def_id::DefIdSet;
 use rustc_span::symbol::{kw, sym, Ident};
 use rustc_span::Symbol;
 use rustc_span::{edit_distance, source_map, ExpnKind, FileName, MacroKind, Span};
+use rustc_trait_selection::infer::InferCtxtExt;
 use rustc_trait_selection::traits::error_reporting::on_unimplemented::OnUnimplementedNote;
 use rustc_trait_selection::traits::error_reporting::on_unimplemented::TypeErrCtxtExt as _;
 use rustc_trait_selection::traits::query::evaluate_obligation::InferCtxtExt as _;
@@ -192,7 +193,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     .span_if_local(def_id)
                     .unwrap_or_else(|| self.tcx.def_span(def_id));
                 err.span_label(sp, format!("private {kind} defined here"));
-                self.suggest_valid_traits(&mut err, out_of_scope_traits);
+                self.suggest_valid_traits(&mut err, out_of_scope_traits, true);
                 err.emit();
             }
 
@@ -2464,6 +2465,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         &self,
         err: &mut Diagnostic,
         valid_out_of_scope_traits: Vec<DefId>,
+        explain: bool,
     ) -> bool {
         if !valid_out_of_scope_traits.is_empty() {
             let mut candidates = valid_out_of_scope_traits;
@@ -2476,7 +2478,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 .find(|did| self.tcx.is_diagnostic_item(sym::TryInto, **did))
                 .copied();
 
-            err.help("items from traits can only be used if the trait is in scope");
+            if explain {
+                err.help("items from traits can only be used if the trait is in scope");
+            }
             let msg = format!(
                 "the following {traits_are} implemented but not in scope; \
                  perhaps add a `use` for {one_of_them}:",
@@ -2693,7 +2697,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 }
             }
         }
-        if self.suggest_valid_traits(err, valid_out_of_scope_traits) {
+        if self.suggest_valid_traits(err, valid_out_of_scope_traits, true) {
             return;
         }
 
@@ -2970,22 +2974,39 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                 (candidates, Vec::new())
             };
 
+            let impls_trait = |def_id: DefId| {
+                let args = ty::GenericArgs::for_item(self.tcx, def_id, |param, _| {
+                    if param.index == 0 {
+                        rcvr_ty.into()
+                    } else {
+                        self.infcx.var_for_def(span, param)
+                    }
+                });
+                self.infcx
+                    .type_implements_trait(def_id, args, self.param_env)
+                    .must_apply_modulo_regions()
+                    && param_type.is_none()
+            };
             match &potential_candidates[..] {
                 [] => {}
                 [trait_info] if trait_info.def_id.is_local() => {
-                    err.subdiagnostic(CandidateTraitNote {
-                        span: self.tcx.def_span(trait_info.def_id),
-                        trait_name: self.tcx.def_path_str(trait_info.def_id),
-                        item_name,
-                        action_or_ty: if trait_missing_method {
-                            "NONE".to_string()
-                        } else {
-                            param_type.map_or_else(
-                                || "implement".to_string(), // FIXME: it might only need to be imported into scope, not implemented.
-                                ToString::to_string,
-                            )
-                        },
-                    });
+                    if impls_trait(trait_info.def_id) {
+                        self.suggest_valid_traits(err, vec![trait_info.def_id], false);
+                    } else {
+                        err.subdiagnostic(CandidateTraitNote {
+                            span: self.tcx.def_span(trait_info.def_id),
+                            trait_name: self.tcx.def_path_str(trait_info.def_id),
+                            item_name,
+                            action_or_ty: if trait_missing_method {
+                                "NONE".to_string()
+                            } else {
+                                param_type.map_or_else(
+                                    || "implement".to_string(), // FIXME: it might only need to be imported into scope, not implemented.
+                                    ToString::to_string,
+                                )
+                            },
+                        });
+                    }
                 }
                 trait_infos => {
                     let mut msg = message(param_type.map_or_else(
@@ -2993,6 +3014,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         |param| format!("restrict type parameter `{param}` with"),
                     ));
                     for (i, trait_info) in trait_infos.iter().enumerate() {
+                        if impls_trait(trait_info.def_id) {
+                            self.suggest_valid_traits(err, vec![trait_info.def_id], false);
+                        }
                         msg.push_str(&format!(
                             "\ncandidate #{}: `{}`",
                             i + 1,
diff --git a/tests/ui/traits/item-privacy.stderr b/tests/ui/traits/item-privacy.stderr
index af2e07632121..9382673afe04 100644
--- a/tests/ui/traits/item-privacy.stderr
+++ b/tests/ui/traits/item-privacy.stderr
@@ -8,11 +8,10 @@ LL |     S.a();
    |       ^ method not found in `S`
    |
    = help: items from traits can only be used if the trait is implemented and in scope
-note: `method::A` defines an item `a`, perhaps you need to implement it
-  --> $DIR/item-privacy.rs:6:5
+help: the following trait is implemented but not in scope; perhaps add a `use` for it:
+   |
+LL + use method::A;
    |
-LL |     trait A {
-   |     ^^^^^^^
 
 error[E0599]: no method named `b` found for struct `S` in the current scope
   --> $DIR/item-privacy.rs:68:7
@@ -51,11 +50,10 @@ LL |     S::a(&S);
    |        ^ function or associated item not found in `S`
    |
    = help: items from traits can only be used if the trait is implemented and in scope
-note: `method::A` defines an item `a`, perhaps you need to implement it
-  --> $DIR/item-privacy.rs:6:5
+help: the following trait is implemented but not in scope; perhaps add a `use` for it:
+   |
+LL + use method::A;
    |
-LL |     trait A {
-   |     ^^^^^^^
 
 error[E0599]: no function or associated item named `b` found for struct `S` in the current scope
   --> $DIR/item-privacy.rs:80:8
@@ -91,11 +89,10 @@ LL |     S::A;
    |        ^ associated item not found in `S`
    |
    = help: items from traits can only be used if the trait is implemented and in scope
-note: `assoc_const::A` defines an item `A`, perhaps you need to implement it
-  --> $DIR/item-privacy.rs:24:5
+help: the following trait is implemented but not in scope; perhaps add a `use` for it:
+   |
+LL + use assoc_const::A;
    |
-LL |     trait A {
-   |     ^^^^^^^
 
 error[E0599]: no associated item named `B` found for struct `S` in the current scope
   --> $DIR/item-privacy.rs:98:8

From 455cf5a4f6531598d72a7590c6f390d9296cc548 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Mon, 30 Oct 2023 17:47:07 +0000
Subject: [PATCH 255/297] Improve some diagnostics around `?Trait` bounds

---
 .../rustc_hir_analysis/src/astconv/bounds.rs  | 61 ++++++++++---------
 compiler/rustc_hir_analysis/src/errors.rs     |  2 +-
 tests/ui/issues/issue-37534.rs                |  6 +-
 tests/ui/issues/issue-37534.stderr            | 12 ++--
 tests/ui/issues/issue-87199.rs                |  6 +-
 tests/ui/issues/issue-87199.stderr            | 18 +++---
 tests/ui/unsized/maybe-bounds-where.rs        |  4 +-
 tests/ui/unsized/maybe-bounds-where.stderr    | 16 ++---
 8 files changed, 65 insertions(+), 60 deletions(-)

diff --git a/compiler/rustc_hir_analysis/src/astconv/bounds.rs b/compiler/rustc_hir_analysis/src/astconv/bounds.rs
index 711da6db5acc..3e700f2da869 100644
--- a/compiler/rustc_hir_analysis/src/astconv/bounds.rs
+++ b/compiler/rustc_hir_analysis/src/astconv/bounds.rs
@@ -8,6 +8,7 @@ use rustc_middle::ty::{self as ty, Ty, TypeVisitableExt};
 use rustc_span::symbol::Ident;
 use rustc_span::{ErrorGuaranteed, Span};
 use rustc_trait_selection::traits;
+use smallvec::SmallVec;
 
 use crate::astconv::{
     AstConv, ConvertedBinding, ConvertedBindingKind, OnlySelfBounds, PredicateFilter,
@@ -28,15 +29,11 @@ impl<'tcx> dyn AstConv<'tcx> + '_ {
         let tcx = self.tcx();
 
         // Try to find an unbound in bounds.
-        let mut unbound = None;
+        let mut unbounds: SmallVec<[_; 1]> = SmallVec::new();
         let mut search_bounds = |ast_bounds: &'tcx [hir::GenericBound<'tcx>]| {
             for ab in ast_bounds {
                 if let hir::GenericBound::Trait(ptr, hir::TraitBoundModifier::Maybe) = ab {
-                    if unbound.is_none() {
-                        unbound = Some(&ptr.trait_ref);
-                    } else {
-                        tcx.sess.emit_err(errors::MultipleRelaxedDefaultBounds { span });
-                    }
+                    unbounds.push(ptr)
                 }
             }
         };
@@ -51,33 +48,41 @@ impl<'tcx> dyn AstConv<'tcx> + '_ {
             }
         }
 
-        let sized_def_id = tcx.lang_items().sized_trait();
-        match (&sized_def_id, unbound) {
-            (Some(sized_def_id), Some(tpb))
-                if tpb.path.res == Res::Def(DefKind::Trait, *sized_def_id) =>
-            {
-                // There was in fact a `?Sized` bound, return without doing anything
-                return;
-            }
-            (_, Some(_)) => {
-                // There was a `?Trait` bound, but it was not `?Sized`; warn.
-                tcx.sess.span_warn(
-                    span,
-                    "default bound relaxed for a type parameter, but \
-                        this does nothing because the given bound is not \
-                        a default; only `?Sized` is supported",
-                );
-                // Otherwise, add implicitly sized if `Sized` is available.
-            }
-            _ => {
-                // There was no `?Sized` bound; add implicitly sized if `Sized` is available.
-            }
+        if unbounds.len() > 1 {
+            tcx.sess.emit_err(errors::MultipleRelaxedDefaultBounds {
+                spans: unbounds.iter().map(|ptr| ptr.span).collect(),
+            });
         }
+
+        let sized_def_id = tcx.lang_items().sized_trait();
+
+        let mut seen_sized_unbound = false;
+        for unbound in unbounds {
+            if let Some(sized_def_id) = sized_def_id {
+                if unbound.trait_ref.path.res == Res::Def(DefKind::Trait, sized_def_id) {
+                    seen_sized_unbound = true;
+                    continue;
+                }
+            }
+            // There was a `?Trait` bound, but it was not `?Sized`; warn.
+            tcx.sess.span_warn(
+                unbound.span,
+                "relaxing a default bound only does something for `?Sized`; \
+                all other traits are not bound by default",
+            );
+        }
+
+        // If the above loop finished there was no `?Sized` bound; add implicitly sized if `Sized` is available.
         if sized_def_id.is_none() {
             // No lang item for `Sized`, so we can't add it as a bound.
             return;
         }
-        bounds.push_sized(tcx, self_ty, span);
+        if seen_sized_unbound {
+            // There was in fact a `?Sized` bound, return without doing anything
+        } else {
+            // There was no `?Sized` bound; add implicitly sized if `Sized` is available.
+            bounds.push_sized(tcx, self_ty, span);
+        }
     }
 
     /// This helper takes a *converted* parameter type (`param_ty`)
diff --git a/compiler/rustc_hir_analysis/src/errors.rs b/compiler/rustc_hir_analysis/src/errors.rs
index 6a2db1d06276..dd83b5b6f2c0 100644
--- a/compiler/rustc_hir_analysis/src/errors.rs
+++ b/compiler/rustc_hir_analysis/src/errors.rs
@@ -96,7 +96,7 @@ pub struct CopyImplOnTypeWithDtor {
 #[diag(hir_analysis_multiple_relaxed_default_bounds, code = "E0203")]
 pub struct MultipleRelaxedDefaultBounds {
     #[primary_span]
-    pub span: Span,
+    pub spans: Vec<Span>,
 }
 
 #[derive(Diagnostic)]
diff --git a/tests/ui/issues/issue-37534.rs b/tests/ui/issues/issue-37534.rs
index 1e67e9a81583..40f7186db098 100644
--- a/tests/ui/issues/issue-37534.rs
+++ b/tests/ui/issues/issue-37534.rs
@@ -1,6 +1,6 @@
-struct Foo<T: ?Hash> { }
+struct Foo<T: ?Hash> {}
 //~^ ERROR expected trait, found derive macro `Hash`
 //~^^ ERROR parameter `T` is never used
-//~^^^ WARN default bound relaxed for a type parameter, but this does nothing
+//~^^^ WARN relaxing a default bound only does something for `?Sized`
 
-fn main() { }
+fn main() {}
diff --git a/tests/ui/issues/issue-37534.stderr b/tests/ui/issues/issue-37534.stderr
index 7d3dd8800bd3..03fea2c16486 100644
--- a/tests/ui/issues/issue-37534.stderr
+++ b/tests/ui/issues/issue-37534.stderr
@@ -1,7 +1,7 @@
 error[E0404]: expected trait, found derive macro `Hash`
   --> $DIR/issue-37534.rs:1:16
    |
-LL | struct Foo<T: ?Hash> { }
+LL | struct Foo<T: ?Hash> {}
    |                ^^^^ not a trait
    |
 help: consider importing this trait instead
@@ -9,16 +9,16 @@ help: consider importing this trait instead
 LL + use std::hash::Hash;
    |
 
-warning: default bound relaxed for a type parameter, but this does nothing because the given bound is not a default; only `?Sized` is supported
-  --> $DIR/issue-37534.rs:1:12
+warning: relaxing a default bound only does something for `?Sized`; all other traits are not bound by default
+  --> $DIR/issue-37534.rs:1:15
    |
-LL | struct Foo<T: ?Hash> { }
-   |            ^
+LL | struct Foo<T: ?Hash> {}
+   |               ^^^^^
 
 error[E0392]: parameter `T` is never used
   --> $DIR/issue-37534.rs:1:12
    |
-LL | struct Foo<T: ?Hash> { }
+LL | struct Foo<T: ?Hash> {}
    |            ^ unused parameter
    |
    = help: consider removing `T`, referring to it in a field, or using a marker such as `PhantomData`
diff --git a/tests/ui/issues/issue-87199.rs b/tests/ui/issues/issue-87199.rs
index a80a64a2f87e..d16d40676730 100644
--- a/tests/ui/issues/issue-87199.rs
+++ b/tests/ui/issues/issue-87199.rs
@@ -6,11 +6,11 @@
 
 // Check that these function definitions only emit warnings, not errors
 fn arg<T: ?Send>(_: T) {}
-//~^ warning: default bound relaxed for a type parameter, but this does nothing
+//~^ warning: relaxing a default bound only does something for `?Sized`
 fn ref_arg<T: ?Send>(_: &T) {}
-//~^ warning: default bound relaxed for a type parameter, but this does nothing
+//~^ warning: relaxing a default bound only does something for `?Sized`
 fn ret() -> impl Iterator<Item = ()> + ?Send { std::iter::empty() }
-//~^ warning: default bound relaxed for a type parameter, but this does nothing
+//~^ warning: relaxing a default bound only does something for `?Sized`
 
 // Check that there's no `?Sized` relaxation!
 fn main() {
diff --git a/tests/ui/issues/issue-87199.stderr b/tests/ui/issues/issue-87199.stderr
index 67949b37d409..e02cd7fcfa9e 100644
--- a/tests/ui/issues/issue-87199.stderr
+++ b/tests/ui/issues/issue-87199.stderr
@@ -1,20 +1,20 @@
-warning: default bound relaxed for a type parameter, but this does nothing because the given bound is not a default; only `?Sized` is supported
-  --> $DIR/issue-87199.rs:8:8
+warning: relaxing a default bound only does something for `?Sized`; all other traits are not bound by default
+  --> $DIR/issue-87199.rs:8:11
    |
 LL | fn arg<T: ?Send>(_: T) {}
-   |        ^
+   |           ^^^^^
 
-warning: default bound relaxed for a type parameter, but this does nothing because the given bound is not a default; only `?Sized` is supported
-  --> $DIR/issue-87199.rs:10:12
+warning: relaxing a default bound only does something for `?Sized`; all other traits are not bound by default
+  --> $DIR/issue-87199.rs:10:15
    |
 LL | fn ref_arg<T: ?Send>(_: &T) {}
-   |            ^
+   |               ^^^^^
 
-warning: default bound relaxed for a type parameter, but this does nothing because the given bound is not a default; only `?Sized` is supported
-  --> $DIR/issue-87199.rs:12:13
+warning: relaxing a default bound only does something for `?Sized`; all other traits are not bound by default
+  --> $DIR/issue-87199.rs:12:40
    |
 LL | fn ret() -> impl Iterator<Item = ()> + ?Send { std::iter::empty() }
-   |             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+   |                                        ^^^^^
 
 error[E0277]: the size for values of type `[i32]` cannot be known at compilation time
   --> $DIR/issue-87199.rs:18:15
diff --git a/tests/ui/unsized/maybe-bounds-where.rs b/tests/ui/unsized/maybe-bounds-where.rs
index d7af0c424805..7e82a3eb449c 100644
--- a/tests/ui/unsized/maybe-bounds-where.rs
+++ b/tests/ui/unsized/maybe-bounds-where.rs
@@ -11,11 +11,11 @@ trait Trait<'a> {}
 
 struct S4<T>(T) where for<'a> T: ?Trait<'a>;
 //~^ ERROR `?Trait` bounds are only permitted at the point where a type parameter is declared
-//~| WARN default bound relaxed for a type parameter
+//~| WARN relaxing a default bound only does something for `?Sized`
 
 struct S5<T>(*const T) where T: ?Trait<'static> + ?Sized;
 //~^ ERROR type parameter has more than one relaxed default bound
-//~| WARN default bound relaxed for a type parameter
+//~| WARN relaxing a default bound only does something for `?Sized`
 
 impl<T> S1<T> {
     fn f() where T: ?Sized {}
diff --git a/tests/ui/unsized/maybe-bounds-where.stderr b/tests/ui/unsized/maybe-bounds-where.stderr
index 39bc1b88e56d..683bd387bb29 100644
--- a/tests/ui/unsized/maybe-bounds-where.stderr
+++ b/tests/ui/unsized/maybe-bounds-where.stderr
@@ -28,23 +28,23 @@ error: `?Trait` bounds are only permitted at the point where a type parameter is
 LL |     fn f() where T: ?Sized {}
    |                     ^^^^^^
 
-warning: default bound relaxed for a type parameter, but this does nothing because the given bound is not a default; only `?Sized` is supported
-  --> $DIR/maybe-bounds-where.rs:12:11
+warning: relaxing a default bound only does something for `?Sized`; all other traits are not bound by default
+  --> $DIR/maybe-bounds-where.rs:12:34
    |
 LL | struct S4<T>(T) where for<'a> T: ?Trait<'a>;
-   |           ^
+   |                                  ^^^^^^^^^^
 
 error[E0203]: type parameter has more than one relaxed default bound, only one is supported
-  --> $DIR/maybe-bounds-where.rs:16:11
+  --> $DIR/maybe-bounds-where.rs:16:33
    |
 LL | struct S5<T>(*const T) where T: ?Trait<'static> + ?Sized;
-   |           ^
+   |                                 ^^^^^^^^^^^^^^^   ^^^^^^
 
-warning: default bound relaxed for a type parameter, but this does nothing because the given bound is not a default; only `?Sized` is supported
-  --> $DIR/maybe-bounds-where.rs:16:11
+warning: relaxing a default bound only does something for `?Sized`; all other traits are not bound by default
+  --> $DIR/maybe-bounds-where.rs:16:33
    |
 LL | struct S5<T>(*const T) where T: ?Trait<'static> + ?Sized;
-   |           ^
+   |                                 ^^^^^^^^^^^^^^^
 
 error: aborting due to 6 previous errors; 2 warnings emitted
 

From c91f60e22fccbd1ee7701b2a1b88457985f9df0b Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Mon, 30 Oct 2023 19:24:08 +0000
Subject: [PATCH 256/297] Don't super-fold types when we hit the recursion
 limit

---
 .../src/traits/query/normalize.rs                     | 11 ++++-------
 .../infinite-cycle-involving-weak.rs                  |  8 ++++++++
 .../infinite-cycle-involving-weak.stderr              |  9 +++++++++
 3 files changed, 21 insertions(+), 7 deletions(-)
 create mode 100644 tests/ui/type-alias-impl-trait/infinite-cycle-involving-weak.rs
 create mode 100644 tests/ui/type-alias-impl-trait/infinite-cycle-involving-weak.stderr

diff --git a/compiler/rustc_trait_selection/src/traits/query/normalize.rs b/compiler/rustc_trait_selection/src/traits/query/normalize.rs
index c761d0d103ea..2e31b560b38f 100644
--- a/compiler/rustc_trait_selection/src/traits/query/normalize.rs
+++ b/compiler/rustc_trait_selection/src/traits/query/normalize.rs
@@ -230,17 +230,14 @@ impl<'cx, 'tcx> FallibleTypeFolder<TyCtxt<'tcx>> for QueryNormalizer<'cx, 'tcx>
                     Reveal::All => {
                         let args = data.args.try_fold_with(self)?;
                         let recursion_limit = self.interner().recursion_limit();
+
                         if !recursion_limit.value_within_limit(self.anon_depth) {
-                            // A closure or coroutine may have itself as in its upvars.
-                            // This should be checked handled by the recursion check for opaque
-                            // types, but we may end up here before that check can happen.
-                            // In that case, we delay a bug to mark the trip, and continue without
-                            // revealing the opaque.
-                            self.infcx
+                            let guar = self
+                                .infcx
                                 .err_ctxt()
                                 .build_overflow_error(&ty, self.cause.span, true)
                                 .delay_as_bug();
-                            return ty.try_super_fold_with(self);
+                            return Ok(Ty::new_error(self.interner(), guar));
                         }
 
                         let generic_ty = self.interner().type_of(data.def_id);
diff --git a/tests/ui/type-alias-impl-trait/infinite-cycle-involving-weak.rs b/tests/ui/type-alias-impl-trait/infinite-cycle-involving-weak.rs
new file mode 100644
index 000000000000..6609d4eb5a27
--- /dev/null
+++ b/tests/ui/type-alias-impl-trait/infinite-cycle-involving-weak.rs
@@ -0,0 +1,8 @@
+#![feature(type_alias_impl_trait)]
+
+type T = impl Copy;
+//~^ ERROR cannot resolve opaque type
+
+static STATIC: T = None::<&'static T>;
+
+fn main() {}
diff --git a/tests/ui/type-alias-impl-trait/infinite-cycle-involving-weak.stderr b/tests/ui/type-alias-impl-trait/infinite-cycle-involving-weak.stderr
new file mode 100644
index 000000000000..50ae6f386411
--- /dev/null
+++ b/tests/ui/type-alias-impl-trait/infinite-cycle-involving-weak.stderr
@@ -0,0 +1,9 @@
+error[E0720]: cannot resolve opaque type
+  --> $DIR/infinite-cycle-involving-weak.rs:3:10
+   |
+LL | type T = impl Copy;
+   |          ^^^^^^^^^ cannot resolve opaque type
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0720`.

From af7472ecbc40693680669c7667c81811a3eb949a Mon Sep 17 00:00:00 2001
From: "Celina G. Val" <celinval@amazon.com>
Date: Mon, 30 Oct 2023 13:07:02 -0700
Subject: [PATCH 257/297] Add a stable MIR visitor

Add a few utility functions as well and extend most `mir` and `ty`
ADTs to implement `PartialEq` and `Eq`.
---
 compiler/rustc_smir/src/rustc_smir/mod.rs    |   6 +
 compiler/stable_mir/src/lib.rs               |   7 +-
 compiler/stable_mir/src/mir.rs               |   2 +
 compiler/stable_mir/src/mir/body.rs          |  74 ++--
 compiler/stable_mir/src/mir/mono.rs          |   4 +
 compiler/stable_mir/src/mir/visit.rs         | 414 +++++++++++++++++++
 compiler/stable_mir/src/ty.rs                |  92 ++---
 tests/ui-fulldeps/stable-mir/smir_visitor.rs | 148 +++++++
 8 files changed, 662 insertions(+), 85 deletions(-)
 create mode 100644 compiler/stable_mir/src/mir/visit.rs
 create mode 100644 tests/ui-fulldeps/stable-mir/smir_visitor.rs

diff --git a/compiler/rustc_smir/src/rustc_smir/mod.rs b/compiler/rustc_smir/src/rustc_smir/mod.rs
index 5ab5a048ffaf..b619ce6e35f9 100644
--- a/compiler/rustc_smir/src/rustc_smir/mod.rs
+++ b/compiler/rustc_smir/src/rustc_smir/mod.rs
@@ -216,6 +216,12 @@ impl<'tcx> Context for TablesWrapper<'tcx> {
         tables.create_def_id(def_id)
     }
 
+    fn instance_mangled_name(&self, def: InstanceDef) -> String {
+        let tables = self.0.borrow_mut();
+        let instance = tables.instances[def];
+        tables.tcx.symbol_name(instance).name.to_string()
+    }
+
     fn mono_instance(&self, item: stable_mir::CrateItem) -> stable_mir::mir::mono::Instance {
         let mut tables = self.0.borrow_mut();
         let def_id = tables[item.0];
diff --git a/compiler/stable_mir/src/lib.rs b/compiler/stable_mir/src/lib.rs
index 38915afaa0c8..f316671b278e 100644
--- a/compiler/stable_mir/src/lib.rs
+++ b/compiler/stable_mir/src/lib.rs
@@ -103,8 +103,6 @@ pub type DefKind = Opaque;
 pub type Filename = Opaque;
 
 /// Holds information about an item in the crate.
-/// For now, it only stores the item DefId. Use functions inside `rustc_internal` module to
-/// use this item.
 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
 pub struct CrateItem(pub DefId);
 
@@ -224,6 +222,9 @@ pub trait Context {
     /// Get the instance.
     fn instance_def_id(&self, instance: InstanceDef) -> DefId;
 
+    /// Get the instance mangled name.
+    fn instance_mangled_name(&self, instance: InstanceDef) -> String;
+
     /// Convert a non-generic crate item into an instance.
     /// This function will panic if the item is generic.
     fn mono_instance(&self, item: CrateItem) -> Instance;
@@ -259,7 +260,7 @@ pub fn with<R>(f: impl FnOnce(&dyn Context) -> R) -> R {
 }
 
 /// A type that provides internal information but that can still be used for debug purpose.
-#[derive(Clone)]
+#[derive(Clone, Eq, PartialEq)]
 pub struct Opaque(String);
 
 impl std::fmt::Display for Opaque {
diff --git a/compiler/stable_mir/src/mir.rs b/compiler/stable_mir/src/mir.rs
index 3138bb1ec832..2e1714b49c18 100644
--- a/compiler/stable_mir/src/mir.rs
+++ b/compiler/stable_mir/src/mir.rs
@@ -1,4 +1,6 @@
 mod body;
 pub mod mono;
+pub mod visit;
 
 pub use body::*;
+pub use visit::MirVisitor;
diff --git a/compiler/stable_mir/src/mir/body.rs b/compiler/stable_mir/src/mir/body.rs
index 9f69e61d6fe7..804f5e3d9d0c 100644
--- a/compiler/stable_mir/src/mir/body.rs
+++ b/compiler/stable_mir/src/mir/body.rs
@@ -1,6 +1,6 @@
-use crate::ty::{AdtDef, ClosureDef, Const, CoroutineDef, GenericArgs, Movability, Region};
+use crate::ty::{AdtDef, ClosureDef, Const, CoroutineDef, GenericArgs, Movability, Region, Ty};
 use crate::Opaque;
-use crate::{ty::Ty, Span};
+use crate::Span;
 
 /// The SMIR representation of a single function.
 #[derive(Clone, Debug)]
@@ -12,10 +12,10 @@ pub struct Body {
     // The first local is the return value pointer, followed by `arg_count`
     // locals for the function arguments, followed by any user-declared
     // variables and temporaries.
-    locals: LocalDecls,
+    pub(super) locals: LocalDecls,
 
     // The number of arguments this function takes.
-    arg_count: usize,
+    pub(super) arg_count: usize,
 }
 
 impl Body {
@@ -35,7 +35,7 @@ impl Body {
 
     /// Return local that holds this function's return value.
     pub fn ret_local(&self) -> &LocalDecl {
-        &self.locals[0]
+        &self.locals[RETURN_LOCAL]
     }
 
     /// Locals in `self` that correspond to this function's arguments.
@@ -60,7 +60,7 @@ impl Body {
 
 type LocalDecls = Vec<LocalDecl>;
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct LocalDecl {
     pub ty: Ty,
     pub span: Span,
@@ -72,13 +72,13 @@ pub struct BasicBlock {
     pub terminator: Terminator,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct Terminator {
     pub kind: TerminatorKind,
     pub span: Span,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum TerminatorKind {
     Goto {
         target: usize,
@@ -122,7 +122,7 @@ pub enum TerminatorKind {
     },
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct InlineAsmOperand {
     pub in_value: Option<Operand>,
     pub out_place: Option<Place>,
@@ -131,7 +131,7 @@ pub struct InlineAsmOperand {
     pub raw_rpr: String,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum UnwindAction {
     Continue,
     Unreachable,
@@ -139,7 +139,7 @@ pub enum UnwindAction {
     Cleanup(usize),
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum AssertMessage {
     BoundsCheck { len: Operand, index: Operand },
     Overflow(BinOp, Operand, Operand),
@@ -151,7 +151,7 @@ pub enum AssertMessage {
     MisalignedPointerDereference { required: Operand, found: Operand },
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum BinOp {
     Add,
     AddUnchecked,
@@ -177,20 +177,20 @@ pub enum BinOp {
     Offset,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum UnOp {
     Not,
     Neg,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum CoroutineKind {
     Async(CoroutineSource),
     Coroutine,
     Gen(CoroutineSource),
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum CoroutineSource {
     Block,
     Closure,
@@ -204,7 +204,7 @@ pub(crate) type LocalDefId = Opaque;
 pub(crate) type Coverage = Opaque;
 
 /// The FakeReadCause describes the type of pattern why a FakeRead statement exists.
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum FakeReadCause {
     ForMatchGuard,
     ForMatchedPlace(LocalDefId),
@@ -214,7 +214,7 @@ pub enum FakeReadCause {
 }
 
 /// Describes what kind of retag is to be performed
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq, Hash)]
 pub enum RetagKind {
     FnEntry,
     TwoPhase,
@@ -222,7 +222,7 @@ pub enum RetagKind {
     Default,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq, Hash)]
 pub enum Variance {
     Covariant,
     Invariant,
@@ -230,26 +230,26 @@ pub enum Variance {
     Bivariant,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct CopyNonOverlapping {
     pub src: Operand,
     pub dst: Operand,
     pub count: Operand,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum NonDivergingIntrinsic {
     Assume(Operand),
     CopyNonOverlapping(CopyNonOverlapping),
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct Statement {
     pub kind: StatementKind,
     pub span: Span,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum StatementKind {
     Assign(Place, Rvalue),
     FakeRead(FakeReadCause, Place),
@@ -266,7 +266,7 @@ pub enum StatementKind {
     Nop,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum Rvalue {
     /// Creates a pointer with the indicated mutability to the place.
     ///
@@ -378,7 +378,7 @@ pub enum Rvalue {
     Use(Operand),
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum AggregateKind {
     Array(Ty),
     Tuple,
@@ -387,21 +387,21 @@ pub enum AggregateKind {
     Coroutine(CoroutineDef, GenericArgs, Movability),
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum Operand {
     Copy(Place),
     Move(Place),
     Constant(Constant),
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct Place {
     pub local: Local,
     /// projection out of a place (access a field, deref a pointer, etc)
     pub projection: String,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct UserTypeProjection {
     pub base: UserTypeAnnotationIndex,
     pub projection: String,
@@ -409,6 +409,8 @@ pub struct UserTypeProjection {
 
 pub type Local = usize;
 
+pub const RETURN_LOCAL: Local = 0;
+
 type FieldIdx = usize;
 
 /// The source-order index of a variant in a type.
@@ -416,20 +418,20 @@ pub type VariantIdx = usize;
 
 type UserTypeAnnotationIndex = usize;
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct Constant {
     pub span: Span,
     pub user_ty: Option<UserTypeAnnotationIndex>,
     pub literal: Const,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct SwitchTarget {
     pub value: u128,
     pub target: usize,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum BorrowKind {
     /// Data must be immutable and is aliasable.
     Shared,
@@ -446,26 +448,26 @@ pub enum BorrowKind {
     },
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum MutBorrowKind {
     Default,
     TwoPhaseBorrow,
     ClosureCapture,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, PartialEq, Eq)]
 pub enum Mutability {
     Not,
     Mut,
 }
 
-#[derive(Copy, Clone, Debug)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
 pub enum Safety {
     Unsafe,
     Normal,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum PointerCoercion {
     /// Go from a fn-item type to a fn-pointer type.
     ReifyFnPointer,
@@ -492,7 +494,7 @@ pub enum PointerCoercion {
     Unsize,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum CastKind {
     PointerExposeAddress,
     PointerFromExposedAddress,
@@ -507,7 +509,7 @@ pub enum CastKind {
     Transmute,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum NullOp {
     /// Returns the size of a value of that type.
     SizeOf,
diff --git a/compiler/stable_mir/src/mir/mono.rs b/compiler/stable_mir/src/mir/mono.rs
index 997576fc7cbf..8f533349848e 100644
--- a/compiler/stable_mir/src/mir/mono.rs
+++ b/compiler/stable_mir/src/mir/mono.rs
@@ -42,6 +42,10 @@ impl Instance {
         with(|context| context.instance_ty(self.def))
     }
 
+    pub fn mangled_name(&self) -> String {
+        with(|context| context.instance_mangled_name(self.def))
+    }
+
     /// Resolve an instance starting from a function definition and generic arguments.
     pub fn resolve(def: FnDef, args: &GenericArgs) -> Result<Instance, crate::Error> {
         with(|context| {
diff --git a/compiler/stable_mir/src/mir/visit.rs b/compiler/stable_mir/src/mir/visit.rs
new file mode 100644
index 000000000000..806dced71ff3
--- /dev/null
+++ b/compiler/stable_mir/src/mir/visit.rs
@@ -0,0 +1,414 @@
+//! # The Stable MIR Visitor
+//!
+//! ## Overview
+//!
+//! We currently only support an immutable visitor.
+//! The structure of this visitor is similar to the ones internal to `rustc`,
+//! and it follows the following conventions:
+//!
+//! For every mir item, the trait has a `visit_<item>` and a `super_<item>` method.
+//! - `visit_<item>`, by default, calls `super_<item>`
+//! - `super_<item>`, by default, destructures the `<item>` and calls `visit_<sub_item>` for
+//!   all sub-items that compose the original item.
+//!
+//! In order to implement a visitor, override the `visit_*` methods for the types you are
+//! interested in analyzing, and invoke (within that method call)
+//! `self.super_*` to continue to the traverse.
+//! Avoid calling `super` methods in other circumstances.
+//!
+//! For the most part, we do not destructure things external to the
+//! MIR, e.g., types, spans, etc, but simply visit them and stop.
+//! This avoids duplication with other visitors like `TypeFoldable`.
+//!
+//! ## Updating
+//!
+//! The code is written in a very deliberate style intended to minimize
+//! the chance of things being overlooked.
+//!
+//! Use pattern matching to reference fields and ensure that all
+//! matches are exhaustive.
+//!
+//! For this to work, ALL MATCHES MUST BE EXHAUSTIVE IN FIELDS AND VARIANTS.
+//! That means you never write `..` to skip over fields, nor do you write `_`
+//! to skip over variants in a `match`.
+//!
+//! The only place that `_` is acceptable is to match a field (or
+//! variant argument) that does not require visiting.
+
+use crate::mir::*;
+use crate::ty::{Const, GenericArgs, Region, Ty};
+use crate::{Opaque, Span};
+
+pub trait MirVisitor {
+    fn visit_body(&mut self, body: &Body) {
+        self.super_body(body)
+    }
+
+    fn visit_basic_block(&mut self, bb: &BasicBlock) {
+        self.super_basic_block(bb)
+    }
+
+    fn visit_ret_decl(&mut self, local: Local, decl: &LocalDecl) {
+        self.super_ret_decl(local, decl)
+    }
+
+    fn visit_arg_decl(&mut self, local: Local, decl: &LocalDecl) {
+        self.super_arg_decl(local, decl)
+    }
+
+    fn visit_local_decl(&mut self, local: Local, decl: &LocalDecl) {
+        self.super_local_decl(local, decl)
+    }
+
+    fn visit_statement(&mut self, stmt: &Statement, location: Location) {
+        self.super_statement(stmt, location)
+    }
+
+    fn visit_terminator(&mut self, term: &Terminator, location: Location) {
+        self.super_terminator(term, location)
+    }
+
+    fn visit_span(&mut self, span: &Span) {
+        self.super_span(span)
+    }
+
+    fn visit_place(&mut self, place: &Place, ptx: PlaceContext, location: Location) {
+        self.super_place(place, ptx, location)
+    }
+
+    fn visit_local(&mut self, local: &Local, ptx: PlaceContext, location: Location) {
+        let _ = (local, ptx, location);
+    }
+
+    fn visit_rvalue(&mut self, rvalue: &Rvalue, location: Location) {
+        self.super_rvalue(rvalue, location)
+    }
+
+    fn visit_operand(&mut self, operand: &Operand, location: Location) {
+        self.super_operand(operand, location)
+    }
+
+    fn visit_user_type_projection(&mut self, projection: &UserTypeProjection) {
+        self.super_user_type_projection(projection)
+    }
+
+    fn visit_ty(&mut self, ty: &Ty, location: Location) {
+        let _ = location;
+        self.super_ty(ty)
+    }
+
+    fn visit_constant(&mut self, constant: &Constant, location: Location) {
+        self.super_constant(constant, location)
+    }
+
+    fn visit_const(&mut self, constant: &Const, location: Location) {
+        self.super_const(constant, location)
+    }
+
+    fn visit_region(&mut self, region: &Region, location: Location) {
+        let _ = location;
+        self.super_region(region)
+    }
+
+    fn visit_args(&mut self, args: &GenericArgs, location: Location) {
+        let _ = location;
+        self.super_args(args)
+    }
+
+    fn visit_assert_msg(&mut self, msg: &AssertMessage, location: Location) {
+        self.super_assert_msg(msg, location)
+    }
+
+    fn super_body(&mut self, body: &Body) {
+        let Body { blocks, locals: _, arg_count } = body;
+
+        for bb in blocks {
+            self.visit_basic_block(bb);
+        }
+
+        self.visit_ret_decl(RETURN_LOCAL, body.ret_local());
+
+        for (idx, arg) in body.arg_locals().iter().enumerate() {
+            self.visit_arg_decl(idx + 1, arg)
+        }
+
+        let local_start = arg_count + 1;
+        for (idx, arg) in body.arg_locals().iter().enumerate() {
+            self.visit_local_decl(idx + local_start, arg)
+        }
+    }
+
+    fn super_basic_block(&mut self, bb: &BasicBlock) {
+        let BasicBlock { statements, terminator } = bb;
+        for stmt in statements {
+            self.visit_statement(stmt, Location(stmt.span));
+        }
+        self.visit_terminator(terminator, Location(terminator.span));
+    }
+
+    fn super_local_decl(&mut self, local: Local, decl: &LocalDecl) {
+        let _ = local;
+        let LocalDecl { ty, span } = decl;
+        self.visit_ty(ty, Location(*span));
+    }
+
+    fn super_ret_decl(&mut self, local: Local, decl: &LocalDecl) {
+        self.super_local_decl(local, decl)
+    }
+
+    fn super_arg_decl(&mut self, local: Local, decl: &LocalDecl) {
+        self.super_local_decl(local, decl)
+    }
+
+    fn super_statement(&mut self, stmt: &Statement, location: Location) {
+        let Statement { kind, span } = stmt;
+        self.visit_span(span);
+        match kind {
+            StatementKind::Assign(place, rvalue) => {
+                self.visit_place(place, PlaceContext::MUTATING, location);
+                self.visit_rvalue(rvalue, location);
+            }
+            StatementKind::FakeRead(_, place) => {
+                self.visit_place(place, PlaceContext::NON_MUTATING, location);
+            }
+            StatementKind::SetDiscriminant { place, .. } => {
+                self.visit_place(place, PlaceContext::MUTATING, location);
+            }
+            StatementKind::Deinit(place) => {
+                self.visit_place(place, PlaceContext::MUTATING, location);
+            }
+            StatementKind::StorageLive(local) => {
+                self.visit_local(local, PlaceContext::NON_USE, location);
+            }
+            StatementKind::StorageDead(local) => {
+                self.visit_local(local, PlaceContext::NON_USE, location);
+            }
+            StatementKind::Retag(_, place) => {
+                self.visit_place(place, PlaceContext::MUTATING, location);
+            }
+            StatementKind::PlaceMention(place) => {
+                self.visit_place(place, PlaceContext::NON_MUTATING, location);
+            }
+            StatementKind::AscribeUserType { place, projections, variance: _ } => {
+                self.visit_place(place, PlaceContext::NON_USE, location);
+                self.visit_user_type_projection(projections);
+            }
+            StatementKind::Coverage(coverage) => visit_opaque(coverage),
+            StatementKind::Intrinsic(intrisic) => match intrisic {
+                NonDivergingIntrinsic::Assume(operand) => {
+                    self.visit_operand(operand, location);
+                }
+                NonDivergingIntrinsic::CopyNonOverlapping(CopyNonOverlapping {
+                    src,
+                    dst,
+                    count,
+                }) => {
+                    self.visit_operand(src, location);
+                    self.visit_operand(dst, location);
+                    self.visit_operand(count, location);
+                }
+            },
+            StatementKind::ConstEvalCounter => {}
+            StatementKind::Nop => {}
+        }
+    }
+
+    fn super_terminator(&mut self, term: &Terminator, location: Location) {
+        let Terminator { kind, span } = term;
+        self.visit_span(&span);
+        match kind {
+            TerminatorKind::Goto { .. }
+            | TerminatorKind::Resume
+            | TerminatorKind::Abort
+            | TerminatorKind::Unreachable
+            | TerminatorKind::CoroutineDrop => {}
+            TerminatorKind::Assert { cond, expected: _, msg, target: _, unwind: _ } => {
+                self.visit_operand(cond, location);
+                self.visit_assert_msg(msg, location);
+            }
+            TerminatorKind::Drop { place, target: _, unwind: _ } => {
+                self.visit_place(place, PlaceContext::MUTATING, location);
+            }
+            TerminatorKind::Call { func, args, destination, target: _, unwind: _ } => {
+                self.visit_operand(func, location);
+                for arg in args {
+                    self.visit_operand(arg, location);
+                }
+                self.visit_place(destination, PlaceContext::MUTATING, location);
+            }
+            TerminatorKind::InlineAsm { operands, .. } => {
+                for op in operands {
+                    let InlineAsmOperand { in_value, out_place, raw_rpr: _ } = op;
+                    if let Some(input) = in_value {
+                        self.visit_operand(input, location);
+                    }
+                    if let Some(output) = out_place {
+                        self.visit_place(output, PlaceContext::MUTATING, location);
+                    }
+                }
+            }
+            TerminatorKind::Return => {
+                let local = RETURN_LOCAL;
+                self.visit_local(&local, PlaceContext::NON_MUTATING, location);
+            }
+            TerminatorKind::SwitchInt { discr, targets: _, otherwise: _ } => {
+                self.visit_operand(discr, location);
+            }
+        }
+    }
+
+    fn super_span(&mut self, span: &Span) {
+        let _ = span;
+    }
+
+    fn super_place(&mut self, place: &Place, ptx: PlaceContext, location: Location) {
+        let _ = location;
+        let _ = ptx;
+        visit_opaque(&Opaque(place.projection.clone()));
+    }
+
+    fn super_rvalue(&mut self, rvalue: &Rvalue, location: Location) {
+        match rvalue {
+            Rvalue::AddressOf(mutability, place) => {
+                let pcx = PlaceContext { is_mut: *mutability == Mutability::Mut };
+                self.visit_place(place, pcx, location);
+            }
+            Rvalue::Aggregate(_, operands) => {
+                for op in operands {
+                    self.visit_operand(op, location);
+                }
+            }
+            Rvalue::BinaryOp(_, lhs, rhs) | Rvalue::CheckedBinaryOp(_, lhs, rhs) => {
+                self.visit_operand(lhs, location);
+                self.visit_operand(rhs, location);
+            }
+            Rvalue::Cast(_, op, ty) => {
+                self.visit_operand(op, location);
+                self.visit_ty(ty, location);
+            }
+            Rvalue::CopyForDeref(place) | Rvalue::Discriminant(place) | Rvalue::Len(place) => {
+                self.visit_place(place, PlaceContext::NON_MUTATING, location);
+            }
+            Rvalue::Ref(region, kind, place) => {
+                self.visit_region(region, location);
+                let pcx = PlaceContext { is_mut: matches!(kind, BorrowKind::Mut { .. }) };
+                self.visit_place(place, pcx, location);
+            }
+            Rvalue::Repeat(op, constant) => {
+                self.visit_operand(op, location);
+                self.visit_const(constant, location);
+            }
+            Rvalue::ShallowInitBox(op, ty) => {
+                self.visit_ty(ty, location);
+                self.visit_operand(op, location)
+            }
+            Rvalue::ThreadLocalRef(_) => {}
+            Rvalue::NullaryOp(_, ty) => {
+                self.visit_ty(ty, location);
+            }
+            Rvalue::UnaryOp(_, op) | Rvalue::Use(op) => {
+                self.visit_operand(op, location);
+            }
+        }
+    }
+
+    fn super_operand(&mut self, operand: &Operand, location: Location) {
+        match operand {
+            Operand::Copy(place) | Operand::Move(place) => {
+                self.visit_place(place, PlaceContext::NON_MUTATING, location)
+            }
+            Operand::Constant(constant) => {
+                self.visit_constant(constant, location);
+            }
+        }
+    }
+
+    fn super_user_type_projection(&mut self, projection: &UserTypeProjection) {
+        // This is a no-op on mir::Visitor.
+        let _ = projection;
+    }
+
+    fn super_ty(&mut self, ty: &Ty) {
+        let _ = ty;
+    }
+
+    fn super_constant(&mut self, constant: &Constant, location: Location) {
+        let Constant { span, user_ty: _, literal } = constant;
+        self.visit_span(span);
+        self.visit_const(literal, location);
+    }
+
+    fn super_const(&mut self, constant: &Const, location: Location) {
+        let Const { kind: _, ty, id: _ } = constant;
+        self.visit_ty(ty, location);
+    }
+
+    fn super_region(&mut self, region: &Region) {
+        let _ = region;
+    }
+
+    fn super_args(&mut self, args: &GenericArgs) {
+        let _ = args;
+    }
+
+    fn super_assert_msg(&mut self, msg: &AssertMessage, location: Location) {
+        match msg {
+            AssertMessage::BoundsCheck { len, index } => {
+                self.visit_operand(len, location);
+                self.visit_operand(index, location);
+            }
+            AssertMessage::Overflow(_, left, right) => {
+                self.visit_operand(left, location);
+                self.visit_operand(right, location);
+            }
+            AssertMessage::OverflowNeg(op)
+            | AssertMessage::DivisionByZero(op)
+            | AssertMessage::RemainderByZero(op) => {
+                self.visit_operand(op, location);
+            }
+            AssertMessage::ResumedAfterReturn(_) | AssertMessage::ResumedAfterPanic(_) => { //nothing to visit
+            }
+            AssertMessage::MisalignedPointerDereference { required, found } => {
+                self.visit_operand(required, location);
+                self.visit_operand(found, location);
+            }
+        }
+    }
+}
+
+/// This function is a no-op that gets used to ensure this visitor is kept up-to-date.
+///
+/// The idea is that whenever we replace an Opaque type by a real type, the compiler will fail
+/// when trying to invoke `visit_opaque`.
+///
+/// If you are here because your compilation is broken, replace the failing call to `visit_opaque()`
+/// by a `visit_<CONSTRUCT>` for your construct.
+fn visit_opaque(_: &Opaque) {}
+
+/// The location of a statement / terminator in the code and the CFG.
+#[derive(Clone, Copy, PartialEq, Eq)]
+pub struct Location(Span);
+
+impl Location {
+    pub fn span(&self) -> Span {
+        self.0
+    }
+}
+
+/// Information about a place's usage.
+#[derive(Copy, Clone, PartialEq, Eq, Hash)]
+pub struct PlaceContext {
+    /// Whether the access is mutable or not. Keep this private so we can increment the type in a
+    /// backward compatible manner.
+    is_mut: bool,
+}
+
+impl PlaceContext {
+    const MUTATING: Self = PlaceContext { is_mut: true };
+    const NON_MUTATING: Self = PlaceContext { is_mut: false };
+    const NON_USE: Self = PlaceContext { is_mut: false };
+
+    pub fn is_mutating(&self) -> bool {
+        self.is_mut
+    }
+}
diff --git a/compiler/stable_mir/src/ty.rs b/compiler/stable_mir/src/ty.rs
index e7440cc439b1..5dfaa0fd8915 100644
--- a/compiler/stable_mir/src/ty.rs
+++ b/compiler/stable_mir/src/ty.rs
@@ -22,12 +22,12 @@ impl Ty {
 }
 
 /// Represents a constant in MIR or from the Type system.
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct Const {
     /// The constant kind.
-    kind: ConstantKind,
+    pub(crate) kind: ConstantKind,
     /// The constant type.
-    ty: Ty,
+    pub(crate) ty: Ty,
     /// Used for internal tracking of the internal constant.
     pub id: ConstId,
 }
@@ -54,12 +54,12 @@ pub struct ConstId(pub usize);
 
 type Ident = Opaque;
 
-#[derive(Debug, Clone)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct Region {
     pub kind: RegionKind,
 }
 
-#[derive(Debug, Clone)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum RegionKind {
     ReEarlyBound(EarlyBoundRegion),
     ReLateBound(DebruijnIndex, BoundRegion),
@@ -70,7 +70,7 @@ pub enum RegionKind {
 
 pub(crate) type DebruijnIndex = u32;
 
-#[derive(Debug, Clone)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct EarlyBoundRegion {
     pub def_id: RegionDef,
     pub index: u32,
@@ -79,7 +79,7 @@ pub struct EarlyBoundRegion {
 
 pub(crate) type BoundVar = u32;
 
-#[derive(Debug, Clone)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct BoundRegion {
     pub var: BoundVar,
     pub kind: BoundRegionKind,
@@ -87,7 +87,7 @@ pub struct BoundRegion {
 
 pub(crate) type UniverseIndex = u32;
 
-#[derive(Debug, Clone)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct Placeholder<T> {
     pub universe: UniverseIndex,
     pub bound: T,
@@ -127,7 +127,7 @@ pub struct LineInfo {
     pub end_col: usize,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum TyKind {
     RigidTy(RigidTy),
     Alias(AliasKind, AliasTy),
@@ -135,7 +135,7 @@ pub enum TyKind {
     Bound(usize, BoundTy),
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum RigidTy {
     Bool,
     Char,
@@ -236,7 +236,7 @@ pub struct ImplDef(pub DefId);
 pub struct RegionDef(pub DefId);
 
 /// A list of generic arguments.
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct GenericArgs(pub Vec<GenericArgKind>);
 
 impl std::ops::Index<ParamTy> for GenericArgs {
@@ -255,7 +255,7 @@ impl std::ops::Index<ParamConst> for GenericArgs {
     }
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum GenericArgKind {
     Lifetime(Region),
     Type(Ty),
@@ -284,13 +284,13 @@ impl GenericArgKind {
     }
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum TermKind {
     Type(Ty),
     Const(Const),
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum AliasKind {
     Projection,
     Inherent,
@@ -298,7 +298,7 @@ pub enum AliasKind {
     Weak,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct AliasTy {
     pub def_id: AliasDef,
     pub args: GenericArgs,
@@ -306,7 +306,7 @@ pub struct AliasTy {
 
 pub type PolyFnSig = Binder<FnSig>;
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct FnSig {
     pub inputs_and_output: Vec<Ty>,
     pub c_variadic: bool,
@@ -345,18 +345,18 @@ pub enum Abi {
     RiscvInterruptS,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct Binder<T> {
     pub value: T,
     pub bound_vars: Vec<BoundVariableKind>,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct EarlyBinder<T> {
     pub value: T,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum BoundVariableKind {
     Ty(BoundTyKind),
     Region(BoundRegionKind),
@@ -369,46 +369,46 @@ pub enum BoundTyKind {
     Param(ParamDef, String),
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum BoundRegionKind {
     BrAnon,
     BrNamed(BrNamedDef, String),
     BrEnv,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum DynKind {
     Dyn,
     DynStar,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum ExistentialPredicate {
     Trait(ExistentialTraitRef),
     Projection(ExistentialProjection),
     AutoTrait(TraitDef),
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct ExistentialTraitRef {
     pub def_id: TraitDef,
     pub generic_args: GenericArgs,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct ExistentialProjection {
     pub def_id: TraitDef,
     pub generic_args: GenericArgs,
     pub term: TermKind,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct ParamTy {
     pub index: u32,
     pub name: String,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct BoundTy {
     pub var: usize,
     pub kind: BoundTyKind,
@@ -424,14 +424,14 @@ pub type Promoted = u32;
 pub type InitMaskMaterialized = Vec<u64>;
 
 /// Stores the provenance information of pointers stored in memory.
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct ProvenanceMap {
     /// Provenance in this map applies from the given offset for an entire pointer-size worth of
     /// bytes. Two entries in this map are always at least a pointer size apart.
     pub ptrs: Vec<(Size, Prov)>,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct Allocation {
     pub bytes: Bytes,
     pub provenance: ProvenanceMap,
@@ -439,7 +439,7 @@ pub struct Allocation {
     pub mutability: Mutability,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum ConstantKind {
     Allocated(Allocation),
     Unevaluated(UnevaluatedConst),
@@ -449,13 +449,13 @@ pub enum ConstantKind {
     ZeroSized,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct ParamConst {
     pub index: u32,
     pub name: String,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct UnevaluatedConst {
     pub def: ConstDef,
     pub args: GenericArgs,
@@ -469,7 +469,7 @@ pub enum TraitSpecializationKind {
     AlwaysApplicable,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct TraitDecl {
     pub def_id: TraitDef,
     pub unsafety: Safety,
@@ -500,13 +500,13 @@ impl TraitDecl {
 
 pub type ImplTrait = EarlyBinder<TraitRef>;
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct TraitRef {
     pub def_id: TraitDef,
     pub args: GenericArgs,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct Generics {
     pub parent: Option<GenericDef>,
     pub parent_count: usize,
@@ -517,14 +517,14 @@ pub struct Generics {
     pub host_effect_index: Option<usize>,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum GenericParamDefKind {
     Lifetime,
     Type { has_default: bool, synthetic: bool },
     Const { has_default: bool },
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct GenericParamDef {
     pub name: super::Symbol,
     pub def_id: GenericDef,
@@ -538,7 +538,7 @@ pub struct GenericPredicates {
     pub predicates: Vec<(PredicateKind, Span)>,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum PredicateKind {
     Clause(ClauseKind),
     ObjectSafe(TraitDef),
@@ -550,7 +550,7 @@ pub enum PredicateKind {
     AliasRelate(TermKind, TermKind, AliasRelationDirection),
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum ClauseKind {
     Trait(TraitPredicate),
     RegionOutlives(RegionOutlivesPredicate),
@@ -561,50 +561,50 @@ pub enum ClauseKind {
     ConstEvaluatable(Const),
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum ClosureKind {
     Fn,
     FnMut,
     FnOnce,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct SubtypePredicate {
     pub a: Ty,
     pub b: Ty,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct CoercePredicate {
     pub a: Ty,
     pub b: Ty,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum AliasRelationDirection {
     Equate,
     Subtype,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct TraitPredicate {
     pub trait_ref: TraitRef,
     pub polarity: ImplPolarity,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct OutlivesPredicate<A, B>(pub A, pub B);
 
 pub type RegionOutlivesPredicate = OutlivesPredicate<Region, Region>;
 pub type TypeOutlivesPredicate = OutlivesPredicate<Ty, Region>;
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub struct ProjectionPredicate {
     pub projection_ty: AliasTy,
     pub term: TermKind,
 }
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Eq, PartialEq)]
 pub enum ImplPolarity {
     Positive,
     Negative,
diff --git a/tests/ui-fulldeps/stable-mir/smir_visitor.rs b/tests/ui-fulldeps/stable-mir/smir_visitor.rs
new file mode 100644
index 000000000000..de5148bb5f42
--- /dev/null
+++ b/tests/ui-fulldeps/stable-mir/smir_visitor.rs
@@ -0,0 +1,148 @@
+// run-pass
+//! Sanity check Stable MIR Visitor
+
+// ignore-stage1
+// ignore-cross-compile
+// ignore-remote
+// ignore-windows-gnu mingw has troubles with linking https://github.com/rust-lang/rust/pull/116837
+// edition: 2021
+
+#![feature(rustc_private)]
+#![feature(assert_matches)]
+#![feature(control_flow_enum)]
+
+extern crate rustc_middle;
+#[macro_use]
+extern crate rustc_smir;
+extern crate rustc_driver;
+extern crate rustc_interface;
+extern crate stable_mir;
+
+use std::collections::HashSet;
+use rustc_middle::ty::TyCtxt;
+use rustc_smir::rustc_internal;
+use stable_mir::*;
+use stable_mir::mir::MirVisitor;
+use std::io::Write;
+use std::ops::ControlFlow;
+
+const CRATE_NAME: &str = "input";
+
+fn test_visitor(_tcx: TyCtxt<'_>) -> ControlFlow<()> {
+    let main_fn = stable_mir::entry_fn();
+    let main_body = main_fn.unwrap().body();
+    let main_visitor = TestVisitor::collect(&main_body);
+    assert!(main_visitor.ret_val.is_some());
+    assert!(main_visitor.args.is_empty());
+    assert!(main_visitor.tys.contains(&main_visitor.ret_val.unwrap().ty));
+    assert!(!main_visitor.calls.is_empty());
+
+    let exit_fn = main_visitor.calls.last().unwrap();
+    assert!(exit_fn.mangled_name().contains("exit_fn"), "Unexpected last function: {exit_fn:?}");
+
+    let exit_body = exit_fn.body();
+    let exit_visitor = TestVisitor::collect(&exit_body);
+    assert!(exit_visitor.ret_val.is_some());
+    assert_eq!(exit_visitor.args.len(), 1);
+    assert!(exit_visitor.tys.contains(&exit_visitor.ret_val.unwrap().ty));
+    assert!(exit_visitor.tys.contains(&exit_visitor.args[0].ty));
+    ControlFlow::Continue(())
+}
+
+struct TestVisitor<'a> {
+    pub body: &'a mir::Body,
+    pub tys: HashSet<ty::Ty>,
+    pub ret_val: Option<mir::LocalDecl>,
+    pub args: Vec<mir::LocalDecl>,
+    pub calls: Vec<mir::mono::Instance>
+}
+
+impl<'a> TestVisitor<'a> {
+    fn collect(body: &'a mir::Body) -> TestVisitor<'a> {
+        let mut visitor = TestVisitor {
+            body: &body,
+            tys: Default::default(),
+            ret_val: None,
+            args: vec![],
+            calls: vec![],
+        };
+        visitor.visit_body(&body);
+        visitor
+    }
+}
+
+impl<'a> mir::MirVisitor for TestVisitor<'a> {
+    fn visit_ty(&mut self, ty: &ty::Ty, _location: mir::visit::Location) {
+        self.tys.insert(*ty);
+        self.super_ty(ty)
+    }
+
+    fn visit_ret_decl(&mut self, local: mir::Local, decl: &mir::LocalDecl) {
+        assert!(local == mir::RETURN_LOCAL);
+        assert!(self.ret_val.is_none());
+        self.ret_val = Some(decl.clone());
+        self.super_ret_decl(local, decl);
+    }
+
+    fn visit_arg_decl(&mut self, local: mir::Local, decl: &mir::LocalDecl) {
+        self.args.push(decl.clone());
+        assert_eq!(local, self.args.len());
+        self.super_arg_decl(local, decl);
+    }
+
+    fn visit_terminator(&mut self, term: &mir::Terminator, location: mir::visit::Location) {
+        if let mir::TerminatorKind::Call { func, .. } = &term.kind {
+            let ty::TyKind::RigidTy(ty) = func.ty(self.body.locals()).kind() else { unreachable!
+            () };
+            let ty::RigidTy::FnDef(def, args) = ty else { unreachable!() };
+            self.calls.push(mir::mono::Instance::resolve(def, &args).unwrap());
+        }
+        self.super_terminator(term, location);
+    }
+}
+
+/// This test will generate and analyze a dummy crate using the stable mir.
+/// For that, it will first write the dummy crate into a file.
+/// Then it will create a `StableMir` using custom arguments and then
+/// it will run the compiler.
+fn main() {
+    let path = "sim_visitor_input.rs";
+    generate_input(&path).unwrap();
+    let args = vec![
+        "rustc".to_string(),
+        "-Cpanic=abort".to_string(),
+        "--crate-name".to_string(),
+        CRATE_NAME.to_string(),
+        path.to_string(),
+    ];
+    run!(args, tcx, test_visitor(tcx)).unwrap();
+}
+
+fn generate_input(path: &str) -> std::io::Result<()> {
+    let mut file = std::fs::File::create(path)?;
+    write!(
+        file,
+        r#"
+    fn main() -> std::process::ExitCode {{
+        let inputs = Inputs::new();
+        let total = inputs.values.iter().sum();
+        exit_fn(total)
+    }}
+
+    fn exit_fn(code: u8) -> std::process::ExitCode {{
+        std::process::ExitCode::from(code)
+    }}
+
+    struct Inputs {{
+        values: [u8; 3],
+    }}
+
+    impl Inputs {{
+        fn new() -> Inputs {{
+            Inputs {{ values: [0, 1, 2] }}
+        }}
+    }}
+    "#
+    )?;
+    Ok(())
+}

From 48491c182be06aad6db5edfd60bdcab0917cb79d Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Mon, 30 Oct 2023 20:15:25 +0000
Subject: [PATCH 258/297] Also consider TAIT to be uncomputable if the MIR body
 is tainted

---
 .../src/collect/type_of/opaque.rs               | 14 +++++++++++---
 .../unconstrained-due-to-bad-pattern.rs         | 14 ++++++++++++++
 .../unconstrained-due-to-bad-pattern.stderr     | 17 +++++++++++++++++
 3 files changed, 42 insertions(+), 3 deletions(-)
 create mode 100644 tests/ui/type-alias-impl-trait/unconstrained-due-to-bad-pattern.rs
 create mode 100644 tests/ui/type-alias-impl-trait/unconstrained-due-to-bad-pattern.stderr

diff --git a/compiler/rustc_hir_analysis/src/collect/type_of/opaque.rs b/compiler/rustc_hir_analysis/src/collect/type_of/opaque.rs
index e8d5264c2b8f..e8ab2651d728 100644
--- a/compiler/rustc_hir_analysis/src/collect/type_of/opaque.rs
+++ b/compiler/rustc_hir_analysis/src/collect/type_of/opaque.rs
@@ -183,9 +183,17 @@ impl TaitConstraintLocator<'_> {
         };
 
         // Use borrowck to get the type with unerased regions.
-        let concrete_opaque_types = &self.tcx.mir_borrowck(item_def_id).concrete_opaque_types;
-        debug!(?concrete_opaque_types);
-        if let Some(&concrete_type) = concrete_opaque_types.get(&self.def_id) {
+        let borrowck_results = &self.tcx.mir_borrowck(item_def_id);
+
+        // If the body was tainted, then assume the opaque may have been constrained and just set it to error.
+        if let Some(guar) = borrowck_results.tainted_by_errors {
+            self.found =
+                Some(ty::OpaqueHiddenType { span: DUMMY_SP, ty: Ty::new_error(self.tcx, guar) });
+            return;
+        }
+
+        debug!(?borrowck_results.concrete_opaque_types);
+        if let Some(&concrete_type) = borrowck_results.concrete_opaque_types.get(&self.def_id) {
             debug!(?concrete_type, "found constraint");
             if let Some(prev) = &mut self.found {
                 if concrete_type.ty != prev.ty && !(concrete_type, prev.ty).references_error() {
diff --git a/tests/ui/type-alias-impl-trait/unconstrained-due-to-bad-pattern.rs b/tests/ui/type-alias-impl-trait/unconstrained-due-to-bad-pattern.rs
new file mode 100644
index 000000000000..ae3d317ab464
--- /dev/null
+++ b/tests/ui/type-alias-impl-trait/unconstrained-due-to-bad-pattern.rs
@@ -0,0 +1,14 @@
+#![feature(type_alias_impl_trait)]
+
+type Tait = impl Copy;
+// Make sure that this TAIT isn't considered unconstrained...
+
+fn empty_opaque() -> Tait {
+    if false {
+        match empty_opaque() {}
+        //~^ ERROR non-empty
+    }
+    0u8
+}
+
+fn main() {}
diff --git a/tests/ui/type-alias-impl-trait/unconstrained-due-to-bad-pattern.stderr b/tests/ui/type-alias-impl-trait/unconstrained-due-to-bad-pattern.stderr
new file mode 100644
index 000000000000..6cc5b7a8a0a8
--- /dev/null
+++ b/tests/ui/type-alias-impl-trait/unconstrained-due-to-bad-pattern.stderr
@@ -0,0 +1,17 @@
+error[E0004]: non-exhaustive patterns: type `Tait` is non-empty
+  --> $DIR/unconstrained-due-to-bad-pattern.rs:8:15
+   |
+LL |         match empty_opaque() {}
+   |               ^^^^^^^^^^^^^^
+   |
+   = note: the matched value is of type `Tait`
+help: ensure that all possible cases are being handled by adding a match arm with a wildcard pattern as shown
+   |
+LL ~         match empty_opaque() {
+LL +             _ => todo!(),
+LL +         }
+   |
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0004`.

From 17a6ae2df3bc3344c87aacdaea1af84bb6e48582 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Esteban=20K=C3=BCber?= <esteban@kuber.com.ar>
Date: Wed, 4 Oct 2023 01:10:51 +0000
Subject: [PATCH 259/297] Detect object safety errors when assoc type is
 missing

When an associated type with GATs isn't specified in a `dyn Trait`, emit
an object safety error instead of only complaining about the missing
associated type, as it will lead the user down a path of three different
errors before letting them know that what they were trying to do is
impossible to begin with.

Fix #103155.
---
 compiler/rustc_hir_analysis/messages.ftl      |   2 +-
 .../rustc_hir_analysis/src/astconv/errors.rs  |  47 +++++--
 .../src/astconv/object_safety.rs              |   2 +-
 .../rustc_trait_selection/src/traits/mod.rs   |   2 +-
 .../src/traits/object_safety.rs               |   2 +-
 tests/ui/associated-type-bounds/duplicate.rs  | 132 +++++++++---------
 .../associated-type-bounds/duplicate.stderr   | 132 +++++++++---------
 .../overlaping-bound-suggestion.stderr        |   2 +-
 ...rojection-from-multiple-supertraits.stderr |   4 +-
 .../associated-types-incomplete-object.rs     |   6 +-
 .../associated-types-incomplete-object.stderr |   6 +-
 tests/ui/associated-types/issue-22560.stderr  |   2 +-
 .../ui/associated-types/issue-23595-1.stderr  |   2 +-
 .../missing-associated-types.stderr           |  10 +-
 tests/ui/error-codes/E0191.stderr             |   2 +-
 tests/ui/error-codes/E0220.stderr             |   2 +-
 tests/ui/error-codes/E0719.stderr             |   6 +-
 tests/ui/issues/issue-19482.rs                |   2 +-
 tests/ui/issues/issue-19482.stderr            |   2 +-
 tests/ui/issues/issue-21950.stderr            |   2 +-
 tests/ui/issues/issue-22434.rs                |   2 +-
 tests/ui/issues/issue-22434.stderr            |   2 +-
 tests/ui/issues/issue-23024.rs                |   2 +-
 tests/ui/issues/issue-23024.stderr            |   2 +-
 tests/ui/issues/issue-28344.stderr            |   4 +-
 tests/ui/object-safety/assoc_type_bounds.rs   |   4 +-
 .../ui/object-safety/assoc_type_bounds.stderr |   4 +-
 tests/ui/object-safety/assoc_type_bounds2.rs  |   4 +-
 .../object-safety/assoc_type_bounds2.stderr   |   4 +-
 .../assoc_type_bounds_sized_others.rs         |   4 +-
 .../assoc_type_bounds_sized_others.stderr     |   4 +-
 .../ui/suggestions/trait-hidden-method.stderr |   2 +-
 ...type-argument-instead-of-assoc-type.stderr |   2 +-
 tests/ui/traits/alias/object-fail.stderr      |   2 +-
 .../with-self-in-projection-output-bad.rs     |   4 +-
 .../with-self-in-projection-output-bad.stderr |   4 +-
 36 files changed, 223 insertions(+), 194 deletions(-)

diff --git a/compiler/rustc_hir_analysis/messages.ftl b/compiler/rustc_hir_analysis/messages.ftl
index e8d9918be22d..8ab91bebcf67 100644
--- a/compiler/rustc_hir_analysis/messages.ftl
+++ b/compiler/rustc_hir_analysis/messages.ftl
@@ -404,7 +404,7 @@ hir_analysis_unused_associated_type_bounds =
     .suggestion = remove this bound
 
 hir_analysis_value_of_associated_struct_already_specified =
-    the value of the associated type `{$item_name}` (from trait `{$def_path}`) is already specified
+    the value of the associated type `{$item_name}` in trait `{$def_path}` is already specified
     .label = re-bound here
     .previous_bound_label = `{$item_name}` bound here first
 
diff --git a/compiler/rustc_hir_analysis/src/astconv/errors.rs b/compiler/rustc_hir_analysis/src/astconv/errors.rs
index 889bc2ea4329..6e9929edc421 100644
--- a/compiler/rustc_hir_analysis/src/astconv/errors.rs
+++ b/compiler/rustc_hir_analysis/src/astconv/errors.rs
@@ -3,6 +3,7 @@ use crate::errors::{
     AssocTypeBindingNotAllowed, ManualImplementation, MissingTypeParams,
     ParenthesizedFnTraitExpansion,
 };
+use crate::traits::error_reporting::report_object_safety_error;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_errors::{pluralize, struct_span_err, Applicability, Diagnostic, ErrorGuaranteed};
 use rustc_hir as hir;
@@ -13,6 +14,7 @@ use rustc_session::parse::feature_err;
 use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::symbol::{sym, Ident};
 use rustc_span::{Span, Symbol, DUMMY_SP};
+use rustc_trait_selection::traits::object_safety_violations_for_assoc_item;
 
 use std::collections::BTreeSet;
 
@@ -520,24 +522,33 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 (span, def_ids.into_iter().map(|did| tcx.associated_item(did)).collect())
             })
             .collect();
-        let mut names = vec![];
+        let mut names: FxHashMap<String, Vec<Symbol>> = Default::default();
+        let mut names_len = 0;
 
         // Account for things like `dyn Foo + 'a`, like in tests `issue-22434.rs` and
         // `issue-22560.rs`.
         let mut trait_bound_spans: Vec<Span> = vec![];
+        let mut object_safety_violations = false;
         for (span, items) in &associated_types {
             if !items.is_empty() {
                 trait_bound_spans.push(*span);
             }
             for assoc_item in items {
                 let trait_def_id = assoc_item.container_id(tcx);
-                names.push(format!(
-                    "`{}` (from trait `{}`)",
-                    assoc_item.name,
-                    tcx.def_path_str(trait_def_id),
-                ));
+                names.entry(tcx.def_path_str(trait_def_id)).or_default().push(assoc_item.name);
+                names_len += 1;
+
+                let violations =
+                    object_safety_violations_for_assoc_item(tcx, trait_def_id, *assoc_item);
+                if !violations.is_empty() {
+                    report_object_safety_error(tcx, *span, trait_def_id, &violations).emit();
+                    object_safety_violations = true;
+                }
             }
         }
+        if object_safety_violations {
+            return;
+        }
         if let ([], [bound]) = (&potential_assoc_types[..], &trait_bounds) {
             match bound.trait_ref.path.segments {
                 // FIXME: `trait_ref.path.span` can point to a full path with multiple
@@ -573,15 +584,33 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                 _ => {}
             }
         }
-        names.sort();
+
+        let names = names
+            .into_iter()
+            .map(|(trait_, assocs)| {
+                format!(
+                    "{} in `{trait_}`",
+                    match &assocs[..] {
+                        [] => String::new(),
+                        [only] => format!("`{only}`"),
+                        [assocs @ .., last] => format!(
+                            "{} and `{last}`",
+                            assocs.iter().map(|a| format!("`{a}`")).collect::<Vec<_>>().join(", ")
+                        ),
+                    }
+                )
+            })
+            .collect::<Vec<String>>()
+            .join(", ");
+
         trait_bound_spans.sort();
         let mut err = struct_span_err!(
             tcx.sess,
             trait_bound_spans,
             E0191,
             "the value of the associated type{} {} must be specified",
-            pluralize!(names.len()),
-            names.join(", "),
+            pluralize!(names_len),
+            names,
         );
         let mut suggestions = vec![];
         let mut types_count = 0;
diff --git a/compiler/rustc_hir_analysis/src/astconv/object_safety.rs b/compiler/rustc_hir_analysis/src/astconv/object_safety.rs
index 30c2ab8f5458..00ff3f836adf 100644
--- a/compiler/rustc_hir_analysis/src/astconv/object_safety.rs
+++ b/compiler/rustc_hir_analysis/src/astconv/object_safety.rs
@@ -380,7 +380,7 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                             span,
                             E0228,
                             "the lifetime bound for this object type cannot be deduced \
-                         from context; please supply an explicit bound"
+                             from context; please supply an explicit bound"
                         );
                         let e = if borrowed {
                             // We will have already emitted an error E0106 complaining about a
diff --git a/compiler/rustc_trait_selection/src/traits/mod.rs b/compiler/rustc_trait_selection/src/traits/mod.rs
index 71007e1f0e0a..fff5510bbfbb 100644
--- a/compiler/rustc_trait_selection/src/traits/mod.rs
+++ b/compiler/rustc_trait_selection/src/traits/mod.rs
@@ -47,7 +47,7 @@ pub use self::engine::{ObligationCtxt, TraitEngineExt};
 pub use self::fulfill::{FulfillmentContext, PendingPredicateObligation};
 pub use self::object_safety::astconv_object_safety_violations;
 pub use self::object_safety::is_vtable_safe_method;
-pub use self::object_safety::MethodViolationCode;
+pub use self::object_safety::object_safety_violations_for_assoc_item;
 pub use self::object_safety::ObjectSafetyViolation;
 pub use self::project::NormalizeExt;
 pub use self::project::{normalize_inherent_projection, normalize_projection_type};
diff --git a/compiler/rustc_trait_selection/src/traits/object_safety.rs b/compiler/rustc_trait_selection/src/traits/object_safety.rs
index dfc4bdf14846..17c7f94ee883 100644
--- a/compiler/rustc_trait_selection/src/traits/object_safety.rs
+++ b/compiler/rustc_trait_selection/src/traits/object_safety.rs
@@ -360,7 +360,7 @@ fn generics_require_sized_self(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
 
 /// Returns `Some(_)` if this item makes the containing trait not object safe.
 #[instrument(level = "debug", skip(tcx), ret)]
-fn object_safety_violations_for_assoc_item(
+pub fn object_safety_violations_for_assoc_item(
     tcx: TyCtxt<'_>,
     trait_def_id: DefId,
     item: ty::AssocItem,
diff --git a/tests/ui/associated-type-bounds/duplicate.rs b/tests/ui/associated-type-bounds/duplicate.rs
index 4b8bf52c3749..5019804d4949 100644
--- a/tests/ui/associated-type-bounds/duplicate.rs
+++ b/tests/ui/associated-type-bounds/duplicate.rs
@@ -5,258 +5,258 @@ use std::iter;
 use std::mem::ManuallyDrop;
 
 struct SI1<T: Iterator<Item: Copy, Item: Send>> {
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
     f: T,
 }
 struct SI2<T: Iterator<Item: Copy, Item: Copy>> {
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
     f: T,
 }
 struct SI3<T: Iterator<Item: 'static, Item: 'static>> {
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
     f: T,
 }
 struct SW1<T>
 where
     T: Iterator<Item: Copy, Item: Send>,
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 {
     f: T,
 }
 struct SW2<T>
 where
     T: Iterator<Item: Copy, Item: Copy>,
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 {
     f: T,
 }
 struct SW3<T>
 where
     T: Iterator<Item: 'static, Item: 'static>,
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 {
     f: T,
 }
 
 enum EI1<T: Iterator<Item: Copy, Item: Send>> {
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
     V(T),
 }
 enum EI2<T: Iterator<Item: Copy, Item: Copy>> {
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
     V(T),
 }
 enum EI3<T: Iterator<Item: 'static, Item: 'static>> {
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
     V(T),
 }
 enum EW1<T>
 where
     T: Iterator<Item: Copy, Item: Send>,
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 {
     V(T),
 }
 enum EW2<T>
 where
     T: Iterator<Item: Copy, Item: Copy>,
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 {
     V(T),
 }
 enum EW3<T>
 where
     T: Iterator<Item: 'static, Item: 'static>,
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 {
     V(T),
 }
 
 union UI1<T: Iterator<Item: Copy, Item: Send>> {
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
     f: ManuallyDrop<T>,
 }
 union UI2<T: Iterator<Item: Copy, Item: Copy>> {
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
     f: ManuallyDrop<T>,
 }
 union UI3<T: Iterator<Item: 'static, Item: 'static>> {
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
     f: ManuallyDrop<T>,
 }
 union UW1<T>
 where
     T: Iterator<Item: Copy, Item: Send>,
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 {
     f: ManuallyDrop<T>,
 }
 union UW2<T>
 where
     T: Iterator<Item: Copy, Item: Copy>,
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 {
     f: ManuallyDrop<T>,
 }
 union UW3<T>
 where
     T: Iterator<Item: 'static, Item: 'static>,
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 {
     f: ManuallyDrop<T>,
 }
 
 fn FI1<T: Iterator<Item: Copy, Item: Send>>() {}
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 fn FI2<T: Iterator<Item: Copy, Item: Copy>>() {}
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 fn FI3<T: Iterator<Item: 'static, Item: 'static>>() {}
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 fn FW1<T>()
 where
     T: Iterator<Item: Copy, Item: Send>,
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 {
 }
 fn FW2<T>()
 where
     T: Iterator<Item: Copy, Item: Copy>,
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 {
 }
 fn FW3<T>()
 where
     T: Iterator<Item: 'static, Item: 'static>,
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 {
 }
 
 fn FRPIT1() -> impl Iterator<Item: Copy, Item: Send> {
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
     iter::empty()
 }
 fn FRPIT2() -> impl Iterator<Item: Copy, Item: Copy> {
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
     iter::empty()
 }
 fn FRPIT3() -> impl Iterator<Item: 'static, Item: 'static> {
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
     iter::empty()
 }
 fn FAPIT1(_: impl Iterator<Item: Copy, Item: Send>) {}
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 fn FAPIT2(_: impl Iterator<Item: Copy, Item: Copy>) {}
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 fn FAPIT3(_: impl Iterator<Item: 'static, Item: 'static>) {}
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 
 type TAI1<T: Iterator<Item: Copy, Item: Send>> = T;
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 type TAI2<T: Iterator<Item: Copy, Item: Copy>> = T;
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 type TAI3<T: Iterator<Item: 'static, Item: 'static>> = T;
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 type TAW1<T>
 where
     T: Iterator<Item: Copy, Item: Send>,
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 = T;
 type TAW2<T>
 where
     T: Iterator<Item: Copy, Item: Copy>,
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 = T;
 type TAW3<T>
 where
     T: Iterator<Item: 'static, Item: 'static>,
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 = T;
 
 type ETAI1<T: Iterator<Item: Copy, Item: Send>> = impl Copy;
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 type ETAI2<T: Iterator<Item: Copy, Item: Copy>> = impl Copy;
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 type ETAI3<T: Iterator<Item: 'static, Item: 'static>> = impl Copy;
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 type ETAI4 = impl Iterator<Item: Copy, Item: Send>;
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 type ETAI5 = impl Iterator<Item: Copy, Item: Copy>;
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 type ETAI6 = impl Iterator<Item: 'static, Item: 'static>;
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 
 trait TRI1<T: Iterator<Item: Copy, Item: Send>> {}
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 trait TRI2<T: Iterator<Item: Copy, Item: Copy>> {}
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 trait TRI3<T: Iterator<Item: 'static, Item: 'static>> {}
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 trait TRS1: Iterator<Item: Copy, Item: Send> {}
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
-//~| ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
+//~| ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 trait TRS2: Iterator<Item: Copy, Item: Copy> {}
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
-//~| ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
+//~| ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 trait TRS3: Iterator<Item: 'static, Item: 'static> {}
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
-//~| ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
+//~| ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 trait TRW1<T>
 where
     T: Iterator<Item: Copy, Item: Send>,
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 {
 }
 trait TRW2<T>
 where
     T: Iterator<Item: Copy, Item: Copy>,
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 {
 }
 trait TRW3<T>
 where
     T: Iterator<Item: 'static, Item: 'static>,
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 {
 }
 trait TRSW1
 where
     Self: Iterator<Item: Copy, Item: Send>,
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
-    //~| ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
+    //~| ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 {
 }
 trait TRSW2
 where
     Self: Iterator<Item: Copy, Item: Copy>,
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
-    //~| ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
+    //~| ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 {
 }
 trait TRSW3
 where
     Self: Iterator<Item: 'static, Item: 'static>,
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
-    //~| ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
+    //~| ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 {
 }
 trait TRA1 {
     type A: Iterator<Item: Copy, Item: Send>;
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 }
 trait TRA2 {
     type A: Iterator<Item: Copy, Item: Copy>;
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 }
 trait TRA3 {
     type A: Iterator<Item: 'static, Item: 'static>;
-    //~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+    //~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 }
 
 type TADyn1 = dyn Iterator<Item: Copy, Item: Send>;
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 type TADyn2 = Box<dyn Iterator<Item: Copy, Item: Copy>>;
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 type TADyn3 = dyn Iterator<Item: 'static, Item: 'static>;
-//~^ ERROR the value of the associated type `Item` (from trait `Iterator`) is already specified [E0719]
+//~^ ERROR the value of the associated type `Item` in trait `Iterator` is already specified [E0719]
 
 fn main() {}
diff --git a/tests/ui/associated-type-bounds/duplicate.stderr b/tests/ui/associated-type-bounds/duplicate.stderr
index 3629aa4fceac..3888e62230f1 100644
--- a/tests/ui/associated-type-bounds/duplicate.stderr
+++ b/tests/ui/associated-type-bounds/duplicate.stderr
@@ -1,4 +1,4 @@
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:7:36
    |
 LL | struct SI1<T: Iterator<Item: Copy, Item: Send>> {
@@ -6,7 +6,7 @@ LL | struct SI1<T: Iterator<Item: Copy, Item: Send>> {
    |                        |
    |                        `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:11:36
    |
 LL | struct SI2<T: Iterator<Item: Copy, Item: Copy>> {
@@ -14,7 +14,7 @@ LL | struct SI2<T: Iterator<Item: Copy, Item: Copy>> {
    |                        |
    |                        `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:15:39
    |
 LL | struct SI3<T: Iterator<Item: 'static, Item: 'static>> {
@@ -22,7 +22,7 @@ LL | struct SI3<T: Iterator<Item: 'static, Item: 'static>> {
    |                        |
    |                        `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:21:29
    |
 LL |     T: Iterator<Item: Copy, Item: Send>,
@@ -30,7 +30,7 @@ LL |     T: Iterator<Item: Copy, Item: Send>,
    |                 |
    |                 `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:28:29
    |
 LL |     T: Iterator<Item: Copy, Item: Copy>,
@@ -38,7 +38,7 @@ LL |     T: Iterator<Item: Copy, Item: Copy>,
    |                 |
    |                 `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:35:32
    |
 LL |     T: Iterator<Item: 'static, Item: 'static>,
@@ -46,7 +46,7 @@ LL |     T: Iterator<Item: 'static, Item: 'static>,
    |                 |
    |                 `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:41:34
    |
 LL | enum EI1<T: Iterator<Item: Copy, Item: Send>> {
@@ -54,7 +54,7 @@ LL | enum EI1<T: Iterator<Item: Copy, Item: Send>> {
    |                      |
    |                      `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:45:34
    |
 LL | enum EI2<T: Iterator<Item: Copy, Item: Copy>> {
@@ -62,7 +62,7 @@ LL | enum EI2<T: Iterator<Item: Copy, Item: Copy>> {
    |                      |
    |                      `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:49:37
    |
 LL | enum EI3<T: Iterator<Item: 'static, Item: 'static>> {
@@ -70,7 +70,7 @@ LL | enum EI3<T: Iterator<Item: 'static, Item: 'static>> {
    |                      |
    |                      `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:55:29
    |
 LL |     T: Iterator<Item: Copy, Item: Send>,
@@ -78,7 +78,7 @@ LL |     T: Iterator<Item: Copy, Item: Send>,
    |                 |
    |                 `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:62:29
    |
 LL |     T: Iterator<Item: Copy, Item: Copy>,
@@ -86,7 +86,7 @@ LL |     T: Iterator<Item: Copy, Item: Copy>,
    |                 |
    |                 `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:69:32
    |
 LL |     T: Iterator<Item: 'static, Item: 'static>,
@@ -94,7 +94,7 @@ LL |     T: Iterator<Item: 'static, Item: 'static>,
    |                 |
    |                 `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:75:35
    |
 LL | union UI1<T: Iterator<Item: Copy, Item: Send>> {
@@ -102,7 +102,7 @@ LL | union UI1<T: Iterator<Item: Copy, Item: Send>> {
    |                       |
    |                       `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:79:35
    |
 LL | union UI2<T: Iterator<Item: Copy, Item: Copy>> {
@@ -110,7 +110,7 @@ LL | union UI2<T: Iterator<Item: Copy, Item: Copy>> {
    |                       |
    |                       `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:83:38
    |
 LL | union UI3<T: Iterator<Item: 'static, Item: 'static>> {
@@ -118,7 +118,7 @@ LL | union UI3<T: Iterator<Item: 'static, Item: 'static>> {
    |                       |
    |                       `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:89:29
    |
 LL |     T: Iterator<Item: Copy, Item: Send>,
@@ -126,7 +126,7 @@ LL |     T: Iterator<Item: Copy, Item: Send>,
    |                 |
    |                 `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:96:29
    |
 LL |     T: Iterator<Item: Copy, Item: Copy>,
@@ -134,7 +134,7 @@ LL |     T: Iterator<Item: Copy, Item: Copy>,
    |                 |
    |                 `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:103:32
    |
 LL |     T: Iterator<Item: 'static, Item: 'static>,
@@ -142,7 +142,7 @@ LL |     T: Iterator<Item: 'static, Item: 'static>,
    |                 |
    |                 `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:109:32
    |
 LL | fn FI1<T: Iterator<Item: Copy, Item: Send>>() {}
@@ -150,7 +150,7 @@ LL | fn FI1<T: Iterator<Item: Copy, Item: Send>>() {}
    |                    |
    |                    `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:111:32
    |
 LL | fn FI2<T: Iterator<Item: Copy, Item: Copy>>() {}
@@ -158,7 +158,7 @@ LL | fn FI2<T: Iterator<Item: Copy, Item: Copy>>() {}
    |                    |
    |                    `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:113:35
    |
 LL | fn FI3<T: Iterator<Item: 'static, Item: 'static>>() {}
@@ -166,7 +166,7 @@ LL | fn FI3<T: Iterator<Item: 'static, Item: 'static>>() {}
    |                    |
    |                    `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:117:29
    |
 LL |     T: Iterator<Item: Copy, Item: Send>,
@@ -174,7 +174,7 @@ LL |     T: Iterator<Item: Copy, Item: Send>,
    |                 |
    |                 `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:123:29
    |
 LL |     T: Iterator<Item: Copy, Item: Copy>,
@@ -182,7 +182,7 @@ LL |     T: Iterator<Item: Copy, Item: Copy>,
    |                 |
    |                 `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:129:32
    |
 LL |     T: Iterator<Item: 'static, Item: 'static>,
@@ -190,7 +190,7 @@ LL |     T: Iterator<Item: 'static, Item: 'static>,
    |                 |
    |                 `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:134:42
    |
 LL | fn FRPIT1() -> impl Iterator<Item: Copy, Item: Send> {
@@ -198,7 +198,7 @@ LL | fn FRPIT1() -> impl Iterator<Item: Copy, Item: Send> {
    |                              |
    |                              `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:138:42
    |
 LL | fn FRPIT2() -> impl Iterator<Item: Copy, Item: Copy> {
@@ -206,7 +206,7 @@ LL | fn FRPIT2() -> impl Iterator<Item: Copy, Item: Copy> {
    |                              |
    |                              `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:142:45
    |
 LL | fn FRPIT3() -> impl Iterator<Item: 'static, Item: 'static> {
@@ -214,7 +214,7 @@ LL | fn FRPIT3() -> impl Iterator<Item: 'static, Item: 'static> {
    |                              |
    |                              `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:146:40
    |
 LL | fn FAPIT1(_: impl Iterator<Item: Copy, Item: Send>) {}
@@ -222,7 +222,7 @@ LL | fn FAPIT1(_: impl Iterator<Item: Copy, Item: Send>) {}
    |                            |
    |                            `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:148:40
    |
 LL | fn FAPIT2(_: impl Iterator<Item: Copy, Item: Copy>) {}
@@ -230,7 +230,7 @@ LL | fn FAPIT2(_: impl Iterator<Item: Copy, Item: Copy>) {}
    |                            |
    |                            `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:150:43
    |
 LL | fn FAPIT3(_: impl Iterator<Item: 'static, Item: 'static>) {}
@@ -238,7 +238,7 @@ LL | fn FAPIT3(_: impl Iterator<Item: 'static, Item: 'static>) {}
    |                            |
    |                            `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:153:35
    |
 LL | type TAI1<T: Iterator<Item: Copy, Item: Send>> = T;
@@ -246,7 +246,7 @@ LL | type TAI1<T: Iterator<Item: Copy, Item: Send>> = T;
    |                       |
    |                       `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:155:35
    |
 LL | type TAI2<T: Iterator<Item: Copy, Item: Copy>> = T;
@@ -254,7 +254,7 @@ LL | type TAI2<T: Iterator<Item: Copy, Item: Copy>> = T;
    |                       |
    |                       `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:157:38
    |
 LL | type TAI3<T: Iterator<Item: 'static, Item: 'static>> = T;
@@ -262,7 +262,7 @@ LL | type TAI3<T: Iterator<Item: 'static, Item: 'static>> = T;
    |                       |
    |                       `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:161:29
    |
 LL |     T: Iterator<Item: Copy, Item: Send>,
@@ -270,7 +270,7 @@ LL |     T: Iterator<Item: Copy, Item: Send>,
    |                 |
    |                 `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:166:29
    |
 LL |     T: Iterator<Item: Copy, Item: Copy>,
@@ -278,7 +278,7 @@ LL |     T: Iterator<Item: Copy, Item: Copy>,
    |                 |
    |                 `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:171:32
    |
 LL |     T: Iterator<Item: 'static, Item: 'static>,
@@ -286,7 +286,7 @@ LL |     T: Iterator<Item: 'static, Item: 'static>,
    |                 |
    |                 `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:175:36
    |
 LL | type ETAI1<T: Iterator<Item: Copy, Item: Send>> = impl Copy;
@@ -294,7 +294,7 @@ LL | type ETAI1<T: Iterator<Item: Copy, Item: Send>> = impl Copy;
    |                        |
    |                        `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:177:36
    |
 LL | type ETAI2<T: Iterator<Item: Copy, Item: Copy>> = impl Copy;
@@ -302,7 +302,7 @@ LL | type ETAI2<T: Iterator<Item: Copy, Item: Copy>> = impl Copy;
    |                        |
    |                        `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:179:39
    |
 LL | type ETAI3<T: Iterator<Item: 'static, Item: 'static>> = impl Copy;
@@ -310,7 +310,7 @@ LL | type ETAI3<T: Iterator<Item: 'static, Item: 'static>> = impl Copy;
    |                        |
    |                        `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:181:40
    |
 LL | type ETAI4 = impl Iterator<Item: Copy, Item: Send>;
@@ -318,7 +318,7 @@ LL | type ETAI4 = impl Iterator<Item: Copy, Item: Send>;
    |                            |
    |                            `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:183:40
    |
 LL | type ETAI5 = impl Iterator<Item: Copy, Item: Copy>;
@@ -326,7 +326,7 @@ LL | type ETAI5 = impl Iterator<Item: Copy, Item: Copy>;
    |                            |
    |                            `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:185:43
    |
 LL | type ETAI6 = impl Iterator<Item: 'static, Item: 'static>;
@@ -334,7 +334,7 @@ LL | type ETAI6 = impl Iterator<Item: 'static, Item: 'static>;
    |                            |
    |                            `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:188:36
    |
 LL | trait TRI1<T: Iterator<Item: Copy, Item: Send>> {}
@@ -342,7 +342,7 @@ LL | trait TRI1<T: Iterator<Item: Copy, Item: Send>> {}
    |                        |
    |                        `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:190:36
    |
 LL | trait TRI2<T: Iterator<Item: Copy, Item: Copy>> {}
@@ -350,7 +350,7 @@ LL | trait TRI2<T: Iterator<Item: Copy, Item: Copy>> {}
    |                        |
    |                        `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:192:39
    |
 LL | trait TRI3<T: Iterator<Item: 'static, Item: 'static>> {}
@@ -358,7 +358,7 @@ LL | trait TRI3<T: Iterator<Item: 'static, Item: 'static>> {}
    |                        |
    |                        `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:194:34
    |
 LL | trait TRS1: Iterator<Item: Copy, Item: Send> {}
@@ -366,7 +366,7 @@ LL | trait TRS1: Iterator<Item: Copy, Item: Send> {}
    |                      |
    |                      `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:194:34
    |
 LL | trait TRS1: Iterator<Item: Copy, Item: Send> {}
@@ -376,7 +376,7 @@ LL | trait TRS1: Iterator<Item: Copy, Item: Send> {}
    |
    = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:197:34
    |
 LL | trait TRS2: Iterator<Item: Copy, Item: Copy> {}
@@ -384,7 +384,7 @@ LL | trait TRS2: Iterator<Item: Copy, Item: Copy> {}
    |                      |
    |                      `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:197:34
    |
 LL | trait TRS2: Iterator<Item: Copy, Item: Copy> {}
@@ -394,7 +394,7 @@ LL | trait TRS2: Iterator<Item: Copy, Item: Copy> {}
    |
    = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:200:37
    |
 LL | trait TRS3: Iterator<Item: 'static, Item: 'static> {}
@@ -402,7 +402,7 @@ LL | trait TRS3: Iterator<Item: 'static, Item: 'static> {}
    |                      |
    |                      `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:200:37
    |
 LL | trait TRS3: Iterator<Item: 'static, Item: 'static> {}
@@ -412,7 +412,7 @@ LL | trait TRS3: Iterator<Item: 'static, Item: 'static> {}
    |
    = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:205:29
    |
 LL |     T: Iterator<Item: Copy, Item: Send>,
@@ -420,7 +420,7 @@ LL |     T: Iterator<Item: Copy, Item: Send>,
    |                 |
    |                 `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:211:29
    |
 LL |     T: Iterator<Item: Copy, Item: Copy>,
@@ -428,7 +428,7 @@ LL |     T: Iterator<Item: Copy, Item: Copy>,
    |                 |
    |                 `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:217:32
    |
 LL |     T: Iterator<Item: 'static, Item: 'static>,
@@ -436,7 +436,7 @@ LL |     T: Iterator<Item: 'static, Item: 'static>,
    |                 |
    |                 `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:223:32
    |
 LL |     Self: Iterator<Item: Copy, Item: Send>,
@@ -444,7 +444,7 @@ LL |     Self: Iterator<Item: Copy, Item: Send>,
    |                    |
    |                    `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:223:32
    |
 LL |     Self: Iterator<Item: Copy, Item: Send>,
@@ -454,7 +454,7 @@ LL |     Self: Iterator<Item: Copy, Item: Send>,
    |
    = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:230:32
    |
 LL |     Self: Iterator<Item: Copy, Item: Copy>,
@@ -462,7 +462,7 @@ LL |     Self: Iterator<Item: Copy, Item: Copy>,
    |                    |
    |                    `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:230:32
    |
 LL |     Self: Iterator<Item: Copy, Item: Copy>,
@@ -472,7 +472,7 @@ LL |     Self: Iterator<Item: Copy, Item: Copy>,
    |
    = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:237:35
    |
 LL |     Self: Iterator<Item: 'static, Item: 'static>,
@@ -480,7 +480,7 @@ LL |     Self: Iterator<Item: 'static, Item: 'static>,
    |                    |
    |                    `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:237:35
    |
 LL |     Self: Iterator<Item: 'static, Item: 'static>,
@@ -490,7 +490,7 @@ LL |     Self: Iterator<Item: 'static, Item: 'static>,
    |
    = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:255:40
    |
 LL | type TADyn1 = dyn Iterator<Item: Copy, Item: Send>;
@@ -498,7 +498,7 @@ LL | type TADyn1 = dyn Iterator<Item: Copy, Item: Send>;
    |                            |
    |                            `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:257:44
    |
 LL | type TADyn2 = Box<dyn Iterator<Item: Copy, Item: Copy>>;
@@ -506,7 +506,7 @@ LL | type TADyn2 = Box<dyn Iterator<Item: Copy, Item: Copy>>;
    |                                |
    |                                `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:259:43
    |
 LL | type TADyn3 = dyn Iterator<Item: 'static, Item: 'static>;
@@ -514,7 +514,7 @@ LL | type TADyn3 = dyn Iterator<Item: 'static, Item: 'static>;
    |                            |
    |                            `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:243:34
    |
 LL |     type A: Iterator<Item: Copy, Item: Send>;
@@ -522,7 +522,7 @@ LL |     type A: Iterator<Item: Copy, Item: Send>;
    |                      |
    |                      `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:247:34
    |
 LL |     type A: Iterator<Item: Copy, Item: Copy>;
@@ -530,7 +530,7 @@ LL |     type A: Iterator<Item: Copy, Item: Copy>;
    |                      |
    |                      `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/duplicate.rs:251:37
    |
 LL |     type A: Iterator<Item: 'static, Item: 'static>;
diff --git a/tests/ui/associated-type-bounds/overlaping-bound-suggestion.stderr b/tests/ui/associated-type-bounds/overlaping-bound-suggestion.stderr
index 61299550e98a..3f170baefb7d 100644
--- a/tests/ui/associated-type-bounds/overlaping-bound-suggestion.stderr
+++ b/tests/ui/associated-type-bounds/overlaping-bound-suggestion.stderr
@@ -1,4 +1,4 @@
-error[E0191]: the value of the associated types `IntoIter` (from trait `IntoIterator`), `IntoIter` (from trait `IntoIterator`), `Item` (from trait `IntoIterator`), `Item` (from trait `IntoIterator`) must be specified
+error[E0191]: the value of the associated types `Item`, `IntoIter`, `Item` and `IntoIter` in `IntoIterator` must be specified
   --> $DIR/overlaping-bound-suggestion.rs:7:13
    |
 LL |     inner: <IntoIterator<Item: IntoIterator<Item: >>::IntoIterator as Item>::Core,
diff --git a/tests/ui/associated-types/associated-type-projection-from-multiple-supertraits.stderr b/tests/ui/associated-types/associated-type-projection-from-multiple-supertraits.stderr
index 4f8954b80bc7..38906f274861 100644
--- a/tests/ui/associated-types/associated-type-projection-from-multiple-supertraits.stderr
+++ b/tests/ui/associated-types/associated-type-projection-from-multiple-supertraits.stderr
@@ -45,7 +45,7 @@ LL | fn dent_object<COLOR>(c: dyn BoxCar<Color=COLOR>) {
                    T: Vehicle::Color = COLOR,
                    T: Box::Color = COLOR
 
-error[E0191]: the value of the associated types `Color` (from trait `Box`), `Color` (from trait `Vehicle`) must be specified
+error[E0191]: the value of the associated types `Color` in `Vehicle`, `Color` in `Box` must be specified
   --> $DIR/associated-type-projection-from-multiple-supertraits.rs:23:30
    |
 LL |     type Color;
@@ -80,7 +80,7 @@ help: use fully-qualified syntax to disambiguate
 LL | fn paint<C:BoxCar>(c: C, d: <C as Box>::Color) {
    |                             ~~~~~~~~~~~~
 
-error[E0191]: the value of the associated types `Color` (from trait `Box`), `Color` (from trait `Vehicle`) must be specified
+error[E0191]: the value of the associated types `Color` in `Vehicle`, `Color` in `Box` must be specified
   --> $DIR/associated-type-projection-from-multiple-supertraits.rs:32:32
    |
 LL |     type Color;
diff --git a/tests/ui/associated-types/associated-types-incomplete-object.rs b/tests/ui/associated-types/associated-types-incomplete-object.rs
index 4627dfd2b780..f6e4bdfbf376 100644
--- a/tests/ui/associated-types/associated-types-incomplete-object.rs
+++ b/tests/ui/associated-types/associated-types-incomplete-object.rs
@@ -21,11 +21,11 @@ pub fn main() {
     let a = &42isize as &dyn Foo<A=usize, B=char>;
 
     let b = &42isize as &dyn Foo<A=usize>;
-    //~^ ERROR the value of the associated type `B` (from trait `Foo`) must be specified
+    //~^ ERROR the value of the associated type `B` in `Foo` must be specified
 
     let c = &42isize as &dyn Foo<B=char>;
-    //~^ ERROR the value of the associated type `A` (from trait `Foo`) must be specified
+    //~^ ERROR the value of the associated type `A` in `Foo` must be specified
 
     let d = &42isize as &dyn Foo;
-    //~^ ERROR the value of the associated types `A` (from trait `Foo`), `B` (from trait
+    //~^ ERROR the value of the associated types `A` and `B` in `Foo`
 }
diff --git a/tests/ui/associated-types/associated-types-incomplete-object.stderr b/tests/ui/associated-types/associated-types-incomplete-object.stderr
index 32866c714680..0c9761afeb51 100644
--- a/tests/ui/associated-types/associated-types-incomplete-object.stderr
+++ b/tests/ui/associated-types/associated-types-incomplete-object.stderr
@@ -1,4 +1,4 @@
-error[E0191]: the value of the associated type `B` (from trait `Foo`) must be specified
+error[E0191]: the value of the associated type `B` in `Foo` must be specified
   --> $DIR/associated-types-incomplete-object.rs:23:30
    |
 LL |     type B;
@@ -7,7 +7,7 @@ LL |     type B;
 LL |     let b = &42isize as &dyn Foo<A=usize>;
    |                              ^^^^^^^^^^^^ help: specify the associated type: `Foo<A=usize, B = Type>`
 
-error[E0191]: the value of the associated type `A` (from trait `Foo`) must be specified
+error[E0191]: the value of the associated type `A` in `Foo` must be specified
   --> $DIR/associated-types-incomplete-object.rs:26:30
    |
 LL |     type A;
@@ -16,7 +16,7 @@ LL |     type A;
 LL |     let c = &42isize as &dyn Foo<B=char>;
    |                              ^^^^^^^^^^^ help: specify the associated type: `Foo<B=char, A = Type>`
 
-error[E0191]: the value of the associated types `A` (from trait `Foo`), `B` (from trait `Foo`) must be specified
+error[E0191]: the value of the associated types `A` and `B` in `Foo` must be specified
   --> $DIR/associated-types-incomplete-object.rs:29:30
    |
 LL |     type A;
diff --git a/tests/ui/associated-types/issue-22560.stderr b/tests/ui/associated-types/issue-22560.stderr
index 2b88cf0b4411..4cc4ee2b4a3b 100644
--- a/tests/ui/associated-types/issue-22560.stderr
+++ b/tests/ui/associated-types/issue-22560.stderr
@@ -9,7 +9,7 @@ LL | type Test = dyn Add + Sub;
    = help: consider creating a new trait with all of these as supertraits and using that trait here instead: `trait NewTrait: Add + Sub {}`
    = note: auto-traits like `Send` and `Sync` are traits that have special properties; for more information on them, visit <https://doc.rust-lang.org/reference/special-types-and-traits.html#auto-traits>
 
-error[E0191]: the value of the associated types `Output` (from trait `Add`), `Output` (from trait `Sub`) must be specified
+error[E0191]: the value of the associated types `Output` in `Sub`, `Output` in `Add` must be specified
   --> $DIR/issue-22560.rs:9:17
    |
 LL |     type Output;
diff --git a/tests/ui/associated-types/issue-23595-1.stderr b/tests/ui/associated-types/issue-23595-1.stderr
index 4307477a56af..3443c4925f4e 100644
--- a/tests/ui/associated-types/issue-23595-1.stderr
+++ b/tests/ui/associated-types/issue-23595-1.stderr
@@ -1,4 +1,4 @@
-error[E0191]: the value of the associated types `ChildKey` (from trait `Hierarchy`), `Children` (from trait `Hierarchy`), `Value` (from trait `Hierarchy`) must be specified
+error[E0191]: the value of the associated types `Value`, `ChildKey` and `Children` in `Hierarchy` must be specified
   --> $DIR/issue-23595-1.rs:8:58
    |
 LL |     type Value;
diff --git a/tests/ui/associated-types/missing-associated-types.stderr b/tests/ui/associated-types/missing-associated-types.stderr
index f617df984ae3..e44c5f1a8d1f 100644
--- a/tests/ui/associated-types/missing-associated-types.stderr
+++ b/tests/ui/associated-types/missing-associated-types.stderr
@@ -9,7 +9,7 @@ LL | type Foo<Rhs> = dyn Add<Rhs> + Sub<Rhs> + X<Rhs> + Y<Rhs>;
    = help: consider creating a new trait with all of these as supertraits and using that trait here instead: `trait NewTrait: Add<Rhs> + Sub<Rhs> + X<Rhs> + Y<Rhs> {}`
    = note: auto-traits like `Send` and `Sync` are traits that have special properties; for more information on them, visit <https://doc.rust-lang.org/reference/special-types-and-traits.html#auto-traits>
 
-error[E0191]: the value of the associated types `A` (from trait `Y`), `Output` (from trait `Add`), `Output` (from trait `Mul`), `Output` (from trait `Sub`) must be specified
+error[E0191]: the value of the associated types `A` in `Y`, `Output` in `Sub`, `Output` in `Mul`, `Output` in `Add` must be specified
   --> $DIR/missing-associated-types.rs:12:21
    |
 LL |     type A;
@@ -38,7 +38,7 @@ LL | type Bar<Rhs> = dyn Add<Rhs> + Sub<Rhs> + X<Rhs> + Z<Rhs>;
    = help: consider creating a new trait with all of these as supertraits and using that trait here instead: `trait NewTrait: Add<Rhs> + Sub<Rhs> + X<Rhs> + Z<Rhs> {}`
    = note: auto-traits like `Send` and `Sync` are traits that have special properties; for more information on them, visit <https://doc.rust-lang.org/reference/special-types-and-traits.html#auto-traits>
 
-error[E0191]: the value of the associated types `A` (from trait `Z`), `B` (from trait `Z`), `Output` (from trait `Add`), `Output` (from trait `Div`), `Output` (from trait `Div`), `Output` (from trait `Mul`), `Output` (from trait `Sub`) must be specified
+error[E0191]: the value of the associated types `Output` and `Output` in `Div`, `Output` in `Add`, `Output` in `Sub`, `Output` in `Mul`, `A` and `B` in `Z` must be specified
   --> $DIR/missing-associated-types.rs:15:21
    |
 LL |     type A;
@@ -74,7 +74,7 @@ LL | type Baz<Rhs> = dyn Add<Rhs> + Sub<Rhs> + Y<Rhs>;
    = help: consider creating a new trait with all of these as supertraits and using that trait here instead: `trait NewTrait: Add<Rhs> + Sub<Rhs> + Y<Rhs> {}`
    = note: auto-traits like `Send` and `Sync` are traits that have special properties; for more information on them, visit <https://doc.rust-lang.org/reference/special-types-and-traits.html#auto-traits>
 
-error[E0191]: the value of the associated types `A` (from trait `Y`), `Output` (from trait `Add`), `Output` (from trait `Sub`) must be specified
+error[E0191]: the value of the associated types `Output` in `Sub`, `A` in `Y`, `Output` in `Add` must be specified
   --> $DIR/missing-associated-types.rs:18:21
    |
 LL |     type A;
@@ -102,7 +102,7 @@ LL | type Bat<Rhs> = dyn Add<Rhs> + Sub<Rhs> + Fine<Rhs>;
    = help: consider creating a new trait with all of these as supertraits and using that trait here instead: `trait NewTrait: Add<Rhs> + Sub<Rhs> + Fine<Rhs> {}`
    = note: auto-traits like `Send` and `Sync` are traits that have special properties; for more information on them, visit <https://doc.rust-lang.org/reference/special-types-and-traits.html#auto-traits>
 
-error[E0191]: the value of the associated types `Output` (from trait `Add`), `Output` (from trait `Sub`) must be specified
+error[E0191]: the value of the associated types `Output` in `Sub`, `Output` in `Add` must be specified
   --> $DIR/missing-associated-types.rs:21:21
    |
 LL | type Bat<Rhs> = dyn Add<Rhs> + Sub<Rhs> + Fine<Rhs>;
@@ -115,7 +115,7 @@ help: specify the associated types
 LL | type Bat<Rhs> = dyn Add<Rhs, Output = Type> + Sub<Rhs, Output = Type> + Fine<Rhs>;
    |                     ~~~~~~~~~~~~~~~~~~~~~~~   ~~~~~~~~~~~~~~~~~~~~~~~
 
-error[E0191]: the value of the associated types `Output` (from trait `Div`), `Output` (from trait `Mul`) must be specified
+error[E0191]: the value of the associated types `Output` in `Div`, `Output` in `Mul` must be specified
   --> $DIR/missing-associated-types.rs:24:21
    |
 LL | type Bal<Rhs> = dyn X<Rhs>;
diff --git a/tests/ui/error-codes/E0191.stderr b/tests/ui/error-codes/E0191.stderr
index cf80c9c46ca2..57eda4785a9f 100644
--- a/tests/ui/error-codes/E0191.stderr
+++ b/tests/ui/error-codes/E0191.stderr
@@ -1,4 +1,4 @@
-error[E0191]: the value of the associated type `Bar` (from trait `Trait`) must be specified
+error[E0191]: the value of the associated type `Bar` in `Trait` must be specified
   --> $DIR/E0191.rs:5:16
    |
 LL |     type Bar;
diff --git a/tests/ui/error-codes/E0220.stderr b/tests/ui/error-codes/E0220.stderr
index e03eadacae4f..0e0b5c7084cc 100644
--- a/tests/ui/error-codes/E0220.stderr
+++ b/tests/ui/error-codes/E0220.stderr
@@ -4,7 +4,7 @@ error[E0220]: associated type `F` not found for `Trait`
 LL | type Foo = dyn Trait<F=i32>;
    |                      ^ help: `Trait` has the following associated type: `Bar`
 
-error[E0191]: the value of the associated type `Bar` (from trait `Trait`) must be specified
+error[E0191]: the value of the associated type `Bar` in `Trait` must be specified
   --> $DIR/E0220.rs:5:16
    |
 LL |     type Bar;
diff --git a/tests/ui/error-codes/E0719.stderr b/tests/ui/error-codes/E0719.stderr
index 685bd7175e3f..00aea97139a7 100644
--- a/tests/ui/error-codes/E0719.stderr
+++ b/tests/ui/error-codes/E0719.stderr
@@ -1,4 +1,4 @@
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/E0719.rs:1:33
    |
 LL | trait Foo: Iterator<Item = i32, Item = i32> {}
@@ -6,7 +6,7 @@ LL | trait Foo: Iterator<Item = i32, Item = i32> {}
    |                     |
    |                     `Item` bound here first
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/E0719.rs:1:33
    |
 LL | trait Foo: Iterator<Item = i32, Item = i32> {}
@@ -16,7 +16,7 @@ LL | trait Foo: Iterator<Item = i32, Item = i32> {}
    |
    = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
 
-error[E0719]: the value of the associated type `Item` (from trait `Iterator`) is already specified
+error[E0719]: the value of the associated type `Item` in trait `Iterator` is already specified
   --> $DIR/E0719.rs:7:42
    |
 LL | fn test() -> Box<dyn Iterator<Item = (), Item = Unit>> {
diff --git a/tests/ui/issues/issue-19482.rs b/tests/ui/issues/issue-19482.rs
index 3f3c5de9b140..9d0c8d96d29a 100644
--- a/tests/ui/issues/issue-19482.rs
+++ b/tests/ui/issues/issue-19482.rs
@@ -8,6 +8,6 @@ trait Foo {
 }
 
 fn bar(x: &dyn Foo) {}
-//~^ ERROR the associated type `A` (from trait `Foo`) must be specified
+//~^ ERROR the associated type `A` in `Foo` must be specified
 
 pub fn main() {}
diff --git a/tests/ui/issues/issue-19482.stderr b/tests/ui/issues/issue-19482.stderr
index d51cc1f081e2..90e6f7995605 100644
--- a/tests/ui/issues/issue-19482.stderr
+++ b/tests/ui/issues/issue-19482.stderr
@@ -1,4 +1,4 @@
-error[E0191]: the value of the associated type `A` (from trait `Foo`) must be specified
+error[E0191]: the value of the associated type `A` in `Foo` must be specified
   --> $DIR/issue-19482.rs:10:16
    |
 LL |     type A;
diff --git a/tests/ui/issues/issue-21950.stderr b/tests/ui/issues/issue-21950.stderr
index 731615a6bd8d..e498565d4e65 100644
--- a/tests/ui/issues/issue-21950.stderr
+++ b/tests/ui/issues/issue-21950.stderr
@@ -1,4 +1,4 @@
-error[E0191]: the value of the associated type `Output` (from trait `Add`) must be specified
+error[E0191]: the value of the associated type `Output` in `Add` must be specified
   --> $DIR/issue-21950.rs:10:25
    |
 LL |     type Output;
diff --git a/tests/ui/issues/issue-22434.rs b/tests/ui/issues/issue-22434.rs
index 34057b46ecd4..d9f7b987c64f 100644
--- a/tests/ui/issues/issue-22434.rs
+++ b/tests/ui/issues/issue-22434.rs
@@ -3,6 +3,6 @@ pub trait Foo {
 }
 
 type I<'a> = &'a (dyn Foo + 'a);
-//~^ ERROR the value of the associated type `A` (from trait `Foo`) must be specified
+//~^ ERROR the value of the associated type `A` in `Foo` must be specified
 
 fn main() {}
diff --git a/tests/ui/issues/issue-22434.stderr b/tests/ui/issues/issue-22434.stderr
index b97fa2503b8a..dab62bcbb4d2 100644
--- a/tests/ui/issues/issue-22434.stderr
+++ b/tests/ui/issues/issue-22434.stderr
@@ -1,4 +1,4 @@
-error[E0191]: the value of the associated type `A` (from trait `Foo`) must be specified
+error[E0191]: the value of the associated type `A` in `Foo` must be specified
   --> $DIR/issue-22434.rs:5:23
    |
 LL |     type A;
diff --git a/tests/ui/issues/issue-23024.rs b/tests/ui/issues/issue-23024.rs
index 010281ee371c..25220dc3e611 100644
--- a/tests/ui/issues/issue-23024.rs
+++ b/tests/ui/issues/issue-23024.rs
@@ -8,5 +8,5 @@ fn main()
     println!("{:?}",(vfnfer[0] as dyn Fn)(3));
     //~^ ERROR the precise format of `Fn`-family traits'
     //~| ERROR missing generics for trait `Fn`
-    //~| ERROR the value of the associated type `Output` (from trait `FnOnce`)
+    //~| ERROR the value of the associated type `Output` in `FnOnce`
 }
diff --git a/tests/ui/issues/issue-23024.stderr b/tests/ui/issues/issue-23024.stderr
index 2c325ffcceea..7d187de1bc44 100644
--- a/tests/ui/issues/issue-23024.stderr
+++ b/tests/ui/issues/issue-23024.stderr
@@ -18,7 +18,7 @@ help: add missing generic argument
 LL |     println!("{:?}",(vfnfer[0] as dyn Fn<Args>)(3));
    |                                         ++++++
 
-error[E0191]: the value of the associated type `Output` (from trait `FnOnce`) must be specified
+error[E0191]: the value of the associated type `Output` in `FnOnce` must be specified
   --> $DIR/issue-23024.rs:8:39
    |
 LL |     println!("{:?}",(vfnfer[0] as dyn Fn)(3));
diff --git a/tests/ui/issues/issue-28344.stderr b/tests/ui/issues/issue-28344.stderr
index f398a5da3e93..71d642109ac8 100644
--- a/tests/ui/issues/issue-28344.stderr
+++ b/tests/ui/issues/issue-28344.stderr
@@ -12,7 +12,7 @@ help: use `dyn`
 LL |     let x: u8 = <dyn BitXor>::bitor(0 as u8, 0 as u8);
    |                 ++++       +
 
-error[E0191]: the value of the associated type `Output` (from trait `BitXor`) must be specified
+error[E0191]: the value of the associated type `Output` in `BitXor` must be specified
   --> $DIR/issue-28344.rs:4:17
    |
 LL |     let x: u8 = BitXor::bitor(0 as u8, 0 as u8);
@@ -40,7 +40,7 @@ help: use `dyn`
 LL |     let g = <dyn BitXor>::bitor;
    |             ++++       +
 
-error[E0191]: the value of the associated type `Output` (from trait `BitXor`) must be specified
+error[E0191]: the value of the associated type `Output` in `BitXor` must be specified
   --> $DIR/issue-28344.rs:10:13
    |
 LL |     let g = BitXor::bitor;
diff --git a/tests/ui/object-safety/assoc_type_bounds.rs b/tests/ui/object-safety/assoc_type_bounds.rs
index 9abf7939c430..8634ba626a18 100644
--- a/tests/ui/object-safety/assoc_type_bounds.rs
+++ b/tests/ui/object-safety/assoc_type_bounds.rs
@@ -7,7 +7,7 @@ trait Foo<T> {
 trait Cake {}
 impl Cake for () {}
 
-fn foo(_: &dyn Foo<()>) {} //~ ERROR: the value of the associated type `Bar` (from trait `Foo`) must be specified
-fn bar(_: &dyn Foo<i32>) {} //~ ERROR: the value of the associated type `Bar` (from trait `Foo`) must be specified
+fn foo(_: &dyn Foo<()>) {} //~ ERROR: the value of the associated type `Bar` in `Foo` must be specified
+fn bar(_: &dyn Foo<i32>) {} //~ ERROR: the value of the associated type `Bar` in `Foo` must be specified
 
 fn main() {}
diff --git a/tests/ui/object-safety/assoc_type_bounds.stderr b/tests/ui/object-safety/assoc_type_bounds.stderr
index a1396dc3ad40..3d5482625af3 100644
--- a/tests/ui/object-safety/assoc_type_bounds.stderr
+++ b/tests/ui/object-safety/assoc_type_bounds.stderr
@@ -1,4 +1,4 @@
-error[E0191]: the value of the associated type `Bar` (from trait `Foo`) must be specified
+error[E0191]: the value of the associated type `Bar` in `Foo` must be specified
   --> $DIR/assoc_type_bounds.rs:10:16
    |
 LL |     type Bar
@@ -7,7 +7,7 @@ LL |     type Bar
 LL | fn foo(_: &dyn Foo<()>) {}
    |                ^^^^^^^ help: specify the associated type: `Foo<(), Bar = Type>`
 
-error[E0191]: the value of the associated type `Bar` (from trait `Foo`) must be specified
+error[E0191]: the value of the associated type `Bar` in `Foo` must be specified
   --> $DIR/assoc_type_bounds.rs:11:16
    |
 LL |     type Bar
diff --git a/tests/ui/object-safety/assoc_type_bounds2.rs b/tests/ui/object-safety/assoc_type_bounds2.rs
index 0112123fd42c..f7dc2fb88390 100644
--- a/tests/ui/object-safety/assoc_type_bounds2.rs
+++ b/tests/ui/object-safety/assoc_type_bounds2.rs
@@ -7,7 +7,7 @@ trait Foo<T> {
 trait Cake {}
 impl Cake for () {}
 
-fn foo(_: &dyn Foo<()>) {} //~ ERROR: the value of the associated type `Bar` (from trait `Foo`) must be specified
-fn bar(_: &dyn Foo<i32>) {} //~ ERROR: the value of the associated type `Bar` (from trait `Foo`) must be specified
+fn foo(_: &dyn Foo<()>) {} //~ ERROR: the value of the associated type `Bar` in `Foo` must be specified
+fn bar(_: &dyn Foo<i32>) {} //~ ERROR: the value of the associated type `Bar` in `Foo` must be specified
 
 fn main() {}
diff --git a/tests/ui/object-safety/assoc_type_bounds2.stderr b/tests/ui/object-safety/assoc_type_bounds2.stderr
index 7a3c0e02d485..815747436bf0 100644
--- a/tests/ui/object-safety/assoc_type_bounds2.stderr
+++ b/tests/ui/object-safety/assoc_type_bounds2.stderr
@@ -1,4 +1,4 @@
-error[E0191]: the value of the associated type `Bar` (from trait `Foo`) must be specified
+error[E0191]: the value of the associated type `Bar` in `Foo` must be specified
   --> $DIR/assoc_type_bounds2.rs:10:16
    |
 LL |     type Bar
@@ -7,7 +7,7 @@ LL |     type Bar
 LL | fn foo(_: &dyn Foo<()>) {}
    |                ^^^^^^^ help: specify the associated type: `Foo<(), Bar = Type>`
 
-error[E0191]: the value of the associated type `Bar` (from trait `Foo`) must be specified
+error[E0191]: the value of the associated type `Bar` in `Foo` must be specified
   --> $DIR/assoc_type_bounds2.rs:11:16
    |
 LL |     type Bar
diff --git a/tests/ui/object-safety/assoc_type_bounds_sized_others.rs b/tests/ui/object-safety/assoc_type_bounds_sized_others.rs
index 647b72a759fa..5b07bc92f321 100644
--- a/tests/ui/object-safety/assoc_type_bounds_sized_others.rs
+++ b/tests/ui/object-safety/assoc_type_bounds_sized_others.rs
@@ -10,7 +10,7 @@ trait Foo {
 }
 
 fn foo(_: &dyn Foo) {}
-//~^ ERROR the value of the associated type `Bop` (from trait `Foo`) must be specified
+//~^ ERROR the value of the associated type `Bop` in `Foo` must be specified
 
 trait Bar {
     type Bop;
@@ -20,6 +20,6 @@ trait Bar {
 }
 
 fn bar(_: &dyn Bar) {}
-//~^ ERROR the value of the associated type `Bop` (from trait `Bar`) must be specified
+//~^ ERROR the value of the associated type `Bop` in `Bar` must be specified
 
 fn main() {}
diff --git a/tests/ui/object-safety/assoc_type_bounds_sized_others.stderr b/tests/ui/object-safety/assoc_type_bounds_sized_others.stderr
index e4c44334b346..5438faaaf054 100644
--- a/tests/ui/object-safety/assoc_type_bounds_sized_others.stderr
+++ b/tests/ui/object-safety/assoc_type_bounds_sized_others.stderr
@@ -1,4 +1,4 @@
-error[E0191]: the value of the associated type `Bop` (from trait `Foo`) must be specified
+error[E0191]: the value of the associated type `Bop` in `Foo` must be specified
   --> $DIR/assoc_type_bounds_sized_others.rs:12:16
    |
 LL |     type Bop;
@@ -7,7 +7,7 @@ LL |     type Bop;
 LL | fn foo(_: &dyn Foo) {}
    |                ^^^ help: specify the associated type: `Foo<Bop = Type>`
 
-error[E0191]: the value of the associated type `Bop` (from trait `Bar`) must be specified
+error[E0191]: the value of the associated type `Bop` in `Bar` must be specified
   --> $DIR/assoc_type_bounds_sized_others.rs:22:16
    |
 LL |     type Bop;
diff --git a/tests/ui/suggestions/trait-hidden-method.stderr b/tests/ui/suggestions/trait-hidden-method.stderr
index a5a65d193db0..5dec20718467 100644
--- a/tests/ui/suggestions/trait-hidden-method.stderr
+++ b/tests/ui/suggestions/trait-hidden-method.stderr
@@ -1,4 +1,4 @@
-error[E0191]: the value of the associated type `Item` (from trait `Iterator`) must be specified
+error[E0191]: the value of the associated type `Item` in `Iterator` must be specified
   --> $DIR/trait-hidden-method.rs:6:33
    |
 LL |     Box::new(1..=10) as Box<dyn Iterator>
diff --git a/tests/ui/suggestions/use-type-argument-instead-of-assoc-type.stderr b/tests/ui/suggestions/use-type-argument-instead-of-assoc-type.stderr
index 175a5fbba610..de828c7659dc 100644
--- a/tests/ui/suggestions/use-type-argument-instead-of-assoc-type.stderr
+++ b/tests/ui/suggestions/use-type-argument-instead-of-assoc-type.stderr
@@ -14,7 +14,7 @@ help: replace the generic bounds with the associated types
 LL |     i: Box<dyn T<usize, usize, A = usize, C = usize, B=usize>>,
    |                                +++        +++
 
-error[E0191]: the value of the associated types `A` (from trait `T`), `C` (from trait `T`) must be specified
+error[E0191]: the value of the associated types `A` and `C` in `T` must be specified
   --> $DIR/use-type-argument-instead-of-assoc-type.rs:7:16
    |
 LL |     type A;
diff --git a/tests/ui/traits/alias/object-fail.stderr b/tests/ui/traits/alias/object-fail.stderr
index 048a150df8c5..a27a3ea0ec03 100644
--- a/tests/ui/traits/alias/object-fail.stderr
+++ b/tests/ui/traits/alias/object-fail.stderr
@@ -9,7 +9,7 @@ note: for a trait to be "object safe" it needs to allow building a vtable to all
    |
    = note: the trait cannot be made into an object because it uses `Self` as a type parameter
 
-error[E0191]: the value of the associated type `Item` (from trait `Iterator`) must be specified
+error[E0191]: the value of the associated type `Item` in `Iterator` must be specified
   --> $DIR/object-fail.rs:9:17
    |
 LL |     let _: &dyn IteratorAlias = &vec![123].into_iter();
diff --git a/tests/ui/traits/object/with-self-in-projection-output-bad.rs b/tests/ui/traits/object/with-self-in-projection-output-bad.rs
index f34fa80a0cee..9515397fb8de 100644
--- a/tests/ui/traits/object/with-self-in-projection-output-bad.rs
+++ b/tests/ui/traits/object/with-self-in-projection-output-bad.rs
@@ -43,8 +43,8 @@ impl NormalizableHelper for u32
 
 fn main() {
     let _x: Box<dyn Helper<Target=i32>> = Box::new(2u32);
-    //~^ ERROR the value of the associated type `Output` (from trait `Base`) must be specified
+    //~^ ERROR the value of the associated type `Output` in `Base` must be specified
 
     let _y: Box<dyn NormalizableHelper<Target=i32>> = Box::new(2u32);
-    //~^ ERROR the value of the associated type `Output` (from trait `Base`) must be specified
+    //~^ ERROR the value of the associated type `Output` in `Base` must be specified
 }
diff --git a/tests/ui/traits/object/with-self-in-projection-output-bad.stderr b/tests/ui/traits/object/with-self-in-projection-output-bad.stderr
index 641bfe236660..c9b36e8d29de 100644
--- a/tests/ui/traits/object/with-self-in-projection-output-bad.stderr
+++ b/tests/ui/traits/object/with-self-in-projection-output-bad.stderr
@@ -1,4 +1,4 @@
-error[E0191]: the value of the associated type `Output` (from trait `Base`) must be specified
+error[E0191]: the value of the associated type `Output` in `Base` must be specified
   --> $DIR/with-self-in-projection-output-bad.rs:45:21
    |
 LL |     type Output;
@@ -7,7 +7,7 @@ LL |     type Output;
 LL |     let _x: Box<dyn Helper<Target=i32>> = Box::new(2u32);
    |                     ^^^^^^^^^^^^^^^^^^ help: specify the associated type: `Helper<Target=i32, Output = Type>`
 
-error[E0191]: the value of the associated type `Output` (from trait `Base`) must be specified
+error[E0191]: the value of the associated type `Output` in `Base` must be specified
   --> $DIR/with-self-in-projection-output-bad.rs:48:21
    |
 LL |     type Output;

From 8c5577283c05a4ee9989f7900bda3b1428877c91 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Esteban=20K=C3=BCber?= <esteban@kuber.com.ar>
Date: Wed, 18 Oct 2023 21:37:21 +0000
Subject: [PATCH 260/297] Add test

---
 .../object/object-unsafe-missing-assoc-type.rs |  7 +++++++
 .../object-unsafe-missing-assoc-type.stderr    | 18 ++++++++++++++++++
 2 files changed, 25 insertions(+)
 create mode 100644 tests/ui/traits/object/object-unsafe-missing-assoc-type.rs
 create mode 100644 tests/ui/traits/object/object-unsafe-missing-assoc-type.stderr

diff --git a/tests/ui/traits/object/object-unsafe-missing-assoc-type.rs b/tests/ui/traits/object/object-unsafe-missing-assoc-type.rs
new file mode 100644
index 000000000000..21f7fd92e80d
--- /dev/null
+++ b/tests/ui/traits/object/object-unsafe-missing-assoc-type.rs
@@ -0,0 +1,7 @@
+trait Foo {
+    type Bar<T>;
+}
+
+fn bar(x: &dyn Foo) {} //~ ERROR the trait `Foo` cannot be made into an object
+
+fn main() {}
diff --git a/tests/ui/traits/object/object-unsafe-missing-assoc-type.stderr b/tests/ui/traits/object/object-unsafe-missing-assoc-type.stderr
new file mode 100644
index 000000000000..fcaa583e2bd4
--- /dev/null
+++ b/tests/ui/traits/object/object-unsafe-missing-assoc-type.stderr
@@ -0,0 +1,18 @@
+error[E0038]: the trait `Foo` cannot be made into an object
+  --> $DIR/object-unsafe-missing-assoc-type.rs:5:16
+   |
+LL | fn bar(x: &dyn Foo) {}
+   |                ^^^ `Foo` cannot be made into an object
+   |
+note: for a trait to be "object safe" it needs to allow building a vtable to allow the call to be resolvable dynamically; for more information visit <https://doc.rust-lang.org/reference/items/traits.html#object-safety>
+  --> $DIR/object-unsafe-missing-assoc-type.rs:2:10
+   |
+LL | trait Foo {
+   |       --- this trait cannot be made into an object...
+LL |     type Bar<T>;
+   |          ^^^ ...because it contains the generic associated type `Bar`
+   = help: consider moving `Bar` to another trait
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0038`.

From b8a8ba9c919b76ef1e51ee42f1c7182fd1ac20cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Esteban=20K=C3=BCber?= <esteban@kuber.com.ar>
Date: Mon, 30 Oct 2023 22:19:17 +0000
Subject: [PATCH 261/297] Sort errors

---
 compiler/rustc_hir_analysis/src/astconv/errors.rs      | 10 ++++++----
 .../overlaping-bound-suggestion.stderr                 |  2 +-
 ...ed-type-projection-from-multiple-supertraits.stderr |  4 ++--
 tests/ui/associated-types/issue-22560.stderr           |  2 +-
 .../associated-types/missing-associated-types.stderr   |  8 ++++----
 .../use-type-argument-instead-of-assoc-type.stderr     |  2 +-
 6 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/compiler/rustc_hir_analysis/src/astconv/errors.rs b/compiler/rustc_hir_analysis/src/astconv/errors.rs
index 6e9929edc421..32be7e0837b6 100644
--- a/compiler/rustc_hir_analysis/src/astconv/errors.rs
+++ b/compiler/rustc_hir_analysis/src/astconv/errors.rs
@@ -585,9 +585,10 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
             }
         }
 
-        let names = names
+        let mut names = names
             .into_iter()
-            .map(|(trait_, assocs)| {
+            .map(|(trait_, mut assocs)| {
+                assocs.sort();
                 format!(
                     "{} in `{trait_}`",
                     match &assocs[..] {
@@ -600,8 +601,9 @@ impl<'o, 'tcx> dyn AstConv<'tcx> + 'o {
                     }
                 )
             })
-            .collect::<Vec<String>>()
-            .join(", ");
+            .collect::<Vec<String>>();
+        names.sort();
+        let names = names.join(", ");
 
         trait_bound_spans.sort();
         let mut err = struct_span_err!(
diff --git a/tests/ui/associated-type-bounds/overlaping-bound-suggestion.stderr b/tests/ui/associated-type-bounds/overlaping-bound-suggestion.stderr
index 3f170baefb7d..2a308f83731f 100644
--- a/tests/ui/associated-type-bounds/overlaping-bound-suggestion.stderr
+++ b/tests/ui/associated-type-bounds/overlaping-bound-suggestion.stderr
@@ -1,4 +1,4 @@
-error[E0191]: the value of the associated types `Item`, `IntoIter`, `Item` and `IntoIter` in `IntoIterator` must be specified
+error[E0191]: the value of the associated types `Item`, `Item`, `IntoIter` and `IntoIter` in `IntoIterator` must be specified
   --> $DIR/overlaping-bound-suggestion.rs:7:13
    |
 LL |     inner: <IntoIterator<Item: IntoIterator<Item: >>::IntoIterator as Item>::Core,
diff --git a/tests/ui/associated-types/associated-type-projection-from-multiple-supertraits.stderr b/tests/ui/associated-types/associated-type-projection-from-multiple-supertraits.stderr
index 38906f274861..66037054e064 100644
--- a/tests/ui/associated-types/associated-type-projection-from-multiple-supertraits.stderr
+++ b/tests/ui/associated-types/associated-type-projection-from-multiple-supertraits.stderr
@@ -45,7 +45,7 @@ LL | fn dent_object<COLOR>(c: dyn BoxCar<Color=COLOR>) {
                    T: Vehicle::Color = COLOR,
                    T: Box::Color = COLOR
 
-error[E0191]: the value of the associated types `Color` in `Vehicle`, `Color` in `Box` must be specified
+error[E0191]: the value of the associated types `Color` in `Box`, `Color` in `Vehicle` must be specified
   --> $DIR/associated-type-projection-from-multiple-supertraits.rs:23:30
    |
 LL |     type Color;
@@ -80,7 +80,7 @@ help: use fully-qualified syntax to disambiguate
 LL | fn paint<C:BoxCar>(c: C, d: <C as Box>::Color) {
    |                             ~~~~~~~~~~~~
 
-error[E0191]: the value of the associated types `Color` in `Vehicle`, `Color` in `Box` must be specified
+error[E0191]: the value of the associated types `Color` in `Box`, `Color` in `Vehicle` must be specified
   --> $DIR/associated-type-projection-from-multiple-supertraits.rs:32:32
    |
 LL |     type Color;
diff --git a/tests/ui/associated-types/issue-22560.stderr b/tests/ui/associated-types/issue-22560.stderr
index 4cc4ee2b4a3b..46e6e3951a5b 100644
--- a/tests/ui/associated-types/issue-22560.stderr
+++ b/tests/ui/associated-types/issue-22560.stderr
@@ -9,7 +9,7 @@ LL | type Test = dyn Add + Sub;
    = help: consider creating a new trait with all of these as supertraits and using that trait here instead: `trait NewTrait: Add + Sub {}`
    = note: auto-traits like `Send` and `Sync` are traits that have special properties; for more information on them, visit <https://doc.rust-lang.org/reference/special-types-and-traits.html#auto-traits>
 
-error[E0191]: the value of the associated types `Output` in `Sub`, `Output` in `Add` must be specified
+error[E0191]: the value of the associated types `Output` in `Add`, `Output` in `Sub` must be specified
   --> $DIR/issue-22560.rs:9:17
    |
 LL |     type Output;
diff --git a/tests/ui/associated-types/missing-associated-types.stderr b/tests/ui/associated-types/missing-associated-types.stderr
index e44c5f1a8d1f..e9669afe8c70 100644
--- a/tests/ui/associated-types/missing-associated-types.stderr
+++ b/tests/ui/associated-types/missing-associated-types.stderr
@@ -9,7 +9,7 @@ LL | type Foo<Rhs> = dyn Add<Rhs> + Sub<Rhs> + X<Rhs> + Y<Rhs>;
    = help: consider creating a new trait with all of these as supertraits and using that trait here instead: `trait NewTrait: Add<Rhs> + Sub<Rhs> + X<Rhs> + Y<Rhs> {}`
    = note: auto-traits like `Send` and `Sync` are traits that have special properties; for more information on them, visit <https://doc.rust-lang.org/reference/special-types-and-traits.html#auto-traits>
 
-error[E0191]: the value of the associated types `A` in `Y`, `Output` in `Sub`, `Output` in `Mul`, `Output` in `Add` must be specified
+error[E0191]: the value of the associated types `A` in `Y`, `Output` in `Add`, `Output` in `Mul`, `Output` in `Sub` must be specified
   --> $DIR/missing-associated-types.rs:12:21
    |
 LL |     type A;
@@ -38,7 +38,7 @@ LL | type Bar<Rhs> = dyn Add<Rhs> + Sub<Rhs> + X<Rhs> + Z<Rhs>;
    = help: consider creating a new trait with all of these as supertraits and using that trait here instead: `trait NewTrait: Add<Rhs> + Sub<Rhs> + X<Rhs> + Z<Rhs> {}`
    = note: auto-traits like `Send` and `Sync` are traits that have special properties; for more information on them, visit <https://doc.rust-lang.org/reference/special-types-and-traits.html#auto-traits>
 
-error[E0191]: the value of the associated types `Output` and `Output` in `Div`, `Output` in `Add`, `Output` in `Sub`, `Output` in `Mul`, `A` and `B` in `Z` must be specified
+error[E0191]: the value of the associated types `A` and `B` in `Z`, `Output` and `Output` in `Div`, `Output` in `Add`, `Output` in `Mul`, `Output` in `Sub` must be specified
   --> $DIR/missing-associated-types.rs:15:21
    |
 LL |     type A;
@@ -74,7 +74,7 @@ LL | type Baz<Rhs> = dyn Add<Rhs> + Sub<Rhs> + Y<Rhs>;
    = help: consider creating a new trait with all of these as supertraits and using that trait here instead: `trait NewTrait: Add<Rhs> + Sub<Rhs> + Y<Rhs> {}`
    = note: auto-traits like `Send` and `Sync` are traits that have special properties; for more information on them, visit <https://doc.rust-lang.org/reference/special-types-and-traits.html#auto-traits>
 
-error[E0191]: the value of the associated types `Output` in `Sub`, `A` in `Y`, `Output` in `Add` must be specified
+error[E0191]: the value of the associated types `A` in `Y`, `Output` in `Add`, `Output` in `Sub` must be specified
   --> $DIR/missing-associated-types.rs:18:21
    |
 LL |     type A;
@@ -102,7 +102,7 @@ LL | type Bat<Rhs> = dyn Add<Rhs> + Sub<Rhs> + Fine<Rhs>;
    = help: consider creating a new trait with all of these as supertraits and using that trait here instead: `trait NewTrait: Add<Rhs> + Sub<Rhs> + Fine<Rhs> {}`
    = note: auto-traits like `Send` and `Sync` are traits that have special properties; for more information on them, visit <https://doc.rust-lang.org/reference/special-types-and-traits.html#auto-traits>
 
-error[E0191]: the value of the associated types `Output` in `Sub`, `Output` in `Add` must be specified
+error[E0191]: the value of the associated types `Output` in `Add`, `Output` in `Sub` must be specified
   --> $DIR/missing-associated-types.rs:21:21
    |
 LL | type Bat<Rhs> = dyn Add<Rhs> + Sub<Rhs> + Fine<Rhs>;
diff --git a/tests/ui/suggestions/use-type-argument-instead-of-assoc-type.stderr b/tests/ui/suggestions/use-type-argument-instead-of-assoc-type.stderr
index de828c7659dc..7c84dd4b8ff3 100644
--- a/tests/ui/suggestions/use-type-argument-instead-of-assoc-type.stderr
+++ b/tests/ui/suggestions/use-type-argument-instead-of-assoc-type.stderr
@@ -14,7 +14,7 @@ help: replace the generic bounds with the associated types
 LL |     i: Box<dyn T<usize, usize, A = usize, C = usize, B=usize>>,
    |                                +++        +++
 
-error[E0191]: the value of the associated types `A` and `C` in `T` must be specified
+error[E0191]: the value of the associated types `C` and `A` in `T` must be specified
   --> $DIR/use-type-argument-instead-of-assoc-type.rs:7:16
    |
 LL |     type A;

From b106167673c271940e1277cb27f6b75a58f66473 Mon Sep 17 00:00:00 2001
From: David Tolnay <dtolnay@gmail.com>
Date: Mon, 30 Oct 2023 08:53:21 -0700
Subject: [PATCH 262/297] Add a DeprecatedSince::Err variant for versions that
 fail to parse

---
 compiler/rustc_attr/src/builtin.rs            | 27 ++++++-------------
 compiler/rustc_middle/src/middle/stability.rs | 19 ++++++-------
 src/librustdoc/html/render/mod.rs             | 19 ++++++-------
 src/librustdoc/json/conversions.rs            |  9 ++++++-
 4 files changed, 36 insertions(+), 38 deletions(-)

diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs
index 49c6b84a4a5c..8ffd322b93f3 100644
--- a/compiler/rustc_attr/src/builtin.rs
+++ b/compiler/rustc_attr/src/builtin.rs
@@ -13,7 +13,6 @@ use rustc_session::parse::{feature_err, ParseSess};
 use rustc_session::{RustcVersion, Session};
 use rustc_span::hygiene::Transparency;
 use rustc_span::{symbol::sym, symbol::Symbol, Span};
-use std::fmt::{self, Display};
 use std::num::NonZeroU32;
 
 use crate::session_diagnostics::{self, IncorrectReprFormatGenericCause};
@@ -736,12 +735,12 @@ pub enum DeprecatedSince {
     RustcVersion(RustcVersion),
     /// Deprecated in the future ("to be determined").
     Future,
-    /// `feature(staged_api)` is off, or it's on but the deprecation version
-    /// cannot be parsed as a RustcVersion. In the latter case, an error has
-    /// already been emitted. In the former case, deprecation versions outside
-    /// the standard library are allowed to be arbitrary strings, for better or
-    /// worse.
+    /// `feature(staged_api)` is off. Deprecation versions outside the standard
+    /// library are allowed to be arbitrary strings, for better or worse.
     Symbol(Symbol),
+    /// Failed to parse a deprecation version. An error has already been
+    /// emitted.
+    Err,
 }
 
 impl Deprecation {
@@ -754,18 +753,8 @@ impl Deprecation {
             Some(DeprecatedSince::Future) => false,
             // The `since` field doesn't have semantic purpose without `#![staged_api]`.
             Some(DeprecatedSince::Symbol(_)) => true,
-            // Assume deprecation is in effect if "since" field is missing.
-            None => true,
-        }
-    }
-}
-
-impl Display for DeprecatedSince {
-    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            DeprecatedSince::RustcVersion(since) => Display::fmt(since, formatter),
-            DeprecatedSince::Future => formatter.write_str("TBD"),
-            DeprecatedSince::Symbol(since) => Display::fmt(since, formatter),
+            // Assume deprecation is in effect if "since" field is absent or invalid.
+            None | Some(DeprecatedSince::Err) => true,
         }
     }
 }
@@ -885,7 +874,7 @@ pub fn find_deprecation(
                 Some(DeprecatedSince::RustcVersion(version))
             } else {
                 sess.emit_err(session_diagnostics::InvalidSince { span: attr.span });
-                Some(DeprecatedSince::Symbol(since))
+                Some(DeprecatedSince::Err)
             }
         } else if is_rustc {
             sess.emit_err(session_diagnostics::MissingSince { span: attr.span });
diff --git a/compiler/rustc_middle/src/middle/stability.rs b/compiler/rustc_middle/src/middle/stability.rs
index 27c2010fe297..4d27bcad911d 100644
--- a/compiler/rustc_middle/src/middle/stability.rs
+++ b/compiler/rustc_middle/src/middle/stability.rs
@@ -155,15 +155,16 @@ fn deprecation_message(
     let message = if is_in_effect {
         format!("use of deprecated {kind} `{path}`")
     } else {
-        if let Some(DeprecatedSince::Future) = since {
-            format!("use of {kind} `{path}` that will be deprecated in a future Rust version")
-        } else {
-            format!(
-                "use of {} `{}` that will be deprecated in future version {}",
-                kind,
-                path,
-                since.unwrap()
-            )
+        match since {
+            Some(DeprecatedSince::RustcVersion(version)) => format!(
+                "use of {kind} `{path}` that will be deprecated in future version {version}"
+            ),
+            Some(DeprecatedSince::Future) => {
+                format!("use of {kind} `{path}` that will be deprecated in a future Rust version")
+            }
+            Some(DeprecatedSince::Symbol(_)) | Some(DeprecatedSince::Err) | None => {
+                unreachable!("this deprecation is always in effect; {since:?}")
+            }
         }
     };
 
diff --git a/src/librustdoc/html/render/mod.rs b/src/librustdoc/html/render/mod.rs
index cf54dc3d6b7e..b64bfa2ae8a0 100644
--- a/src/librustdoc/html/render/mod.rs
+++ b/src/librustdoc/html/render/mod.rs
@@ -619,18 +619,19 @@ fn short_item_info(
     if let Some(depr @ Deprecation { note, since, suggestion: _ }) = item.deprecation(cx.tcx()) {
         // We display deprecation messages for #[deprecated], but only display
         // the future-deprecation messages for rustc versions.
-        let mut message = if let Some(since) = since {
-            if !depr.is_in_effect() {
-                if let DeprecatedSince::Future = since {
-                    String::from("Deprecating in a future Rust version")
+        let mut message = match since {
+            Some(DeprecatedSince::RustcVersion(version)) => {
+                if depr.is_in_effect() {
+                    format!("Deprecated since {version}")
                 } else {
-                    format!("Deprecating in {}", Escape(&since.to_string()))
+                    format!("Deprecating in {version}")
                 }
-            } else {
-                format!("Deprecated since {}", Escape(&since.to_string()))
             }
-        } else {
-            String::from("Deprecated")
+            Some(DeprecatedSince::Future) => String::from("Deprecating in a future Rust version"),
+            Some(DeprecatedSince::Symbol(since)) => {
+                format!("Deprecated since {}", Escape(since.as_str()))
+            }
+            Some(DeprecatedSince::Err) | None => String::from("Deprecated"),
         };
 
         if let Some(note) = note {
diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs
index ff2e6b1ebc9a..acc928360d2e 100644
--- a/src/librustdoc/json/conversions.rs
+++ b/src/librustdoc/json/conversions.rs
@@ -7,6 +7,7 @@
 use std::fmt;
 
 use rustc_ast::ast;
+use rustc_attr::DeprecatedSince;
 use rustc_hir::{def::CtorKind, def::DefKind, def_id::DefId};
 use rustc_metadata::rendered_const;
 use rustc_middle::ty::{self, TyCtxt};
@@ -139,7 +140,13 @@ where
 
 pub(crate) fn from_deprecation(deprecation: rustc_attr::Deprecation) -> Deprecation {
     let rustc_attr::Deprecation { since, note, suggestion: _ } = deprecation;
-    Deprecation { since: since.map(|s| s.to_string()), note: note.map(|s| s.to_string()) }
+    let since = match since {
+        Some(DeprecatedSince::RustcVersion(version)) => Some(version.to_string()),
+        Some(DeprecatedSince::Future) => Some("TBD".to_owned()),
+        Some(DeprecatedSince::Symbol(since)) => Some(since.to_string()),
+        Some(DeprecatedSince::Err) | None => None,
+    };
+    Deprecation { since, note: note.map(|s| s.to_string()) }
 }
 
 impl FromWithTcx<clean::GenericArgs> for GenericArgs {

From c52367276d01e15c5154ef4001017b73f0eb7cde Mon Sep 17 00:00:00 2001
From: David Tolnay <dtolnay@gmail.com>
Date: Mon, 30 Oct 2023 15:44:09 -0700
Subject: [PATCH 263/297] Preserve deprecation attribute even if 'since'
 version is missing

---
 compiler/rustc_attr/src/builtin.rs                  |  6 +++---
 .../stability-attribute-sanity-4.rs                 |  1 +
 .../stability-attribute-sanity-4.stderr             | 13 ++++++++++---
 3 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs
index 8ffd322b93f3..a9466d7df9d6 100644
--- a/compiler/rustc_attr/src/builtin.rs
+++ b/compiler/rustc_attr/src/builtin.rs
@@ -876,10 +876,10 @@ pub fn find_deprecation(
                 sess.emit_err(session_diagnostics::InvalidSince { span: attr.span });
                 Some(DeprecatedSince::Err)
             }
-        } else if is_rustc {
-            sess.emit_err(session_diagnostics::MissingSince { span: attr.span });
-            continue;
         } else {
+            if is_rustc {
+                sess.emit_err(session_diagnostics::MissingSince { span: attr.span });
+            }
             None
         };
 
diff --git a/tests/ui/stability-attribute/stability-attribute-sanity-4.rs b/tests/ui/stability-attribute/stability-attribute-sanity-4.rs
index 4fe8e45fd049..01a46d15c8db 100644
--- a/tests/ui/stability-attribute/stability-attribute-sanity-4.rs
+++ b/tests/ui/stability-attribute/stability-attribute-sanity-4.rs
@@ -19,6 +19,7 @@ mod bogus_attribute_types_2 {
 
     #[stable(feature = "a", since = "3.3.3")]
     #[deprecated] //~ ERROR missing 'since'
+    //~^ ERROR missing 'note'
     fn f5() { }
 
     #[stable(feature = "a", since = "3.3.3")]
diff --git a/tests/ui/stability-attribute/stability-attribute-sanity-4.stderr b/tests/ui/stability-attribute/stability-attribute-sanity-4.stderr
index a76f5be1e3d7..8ead943ffe3a 100644
--- a/tests/ui/stability-attribute/stability-attribute-sanity-4.stderr
+++ b/tests/ui/stability-attribute/stability-attribute-sanity-4.stderr
@@ -28,12 +28,19 @@ error[E0542]: missing 'since'
 LL |     #[deprecated]
    |     ^^^^^^^^^^^^^
 
+error[E0543]: missing 'note'
+  --> $DIR/stability-attribute-sanity-4.rs:21:5
+   |
+LL |     #[deprecated]
+   |     ^^^^^^^^^^^^^
+
 error[E0542]: missing 'since'
-  --> $DIR/stability-attribute-sanity-4.rs:25:5
+  --> $DIR/stability-attribute-sanity-4.rs:26:5
    |
 LL |     #[deprecated = "a"]
    |     ^^^^^^^^^^^^^^^^^^^
 
-error: aborting due to 6 previous errors
+error: aborting due to 7 previous errors
 
-For more information about this error, try `rustc --explain E0542`.
+Some errors have detailed explanations: E0542, E0543.
+For more information about an error, try `rustc --explain E0542`.

From e8868af75bfaf6c9c2890f0e97c38a6b1d94ee62 Mon Sep 17 00:00:00 2001
From: David Tolnay <dtolnay@gmail.com>
Date: Mon, 30 Oct 2023 15:51:26 -0700
Subject: [PATCH 264/297] Represent absence of 'since' attribute as a variant
 of DeprecatedSince

---
 compiler/rustc_attr/src/builtin.rs            | 32 ++++++++++---------
 compiler/rustc_middle/src/middle/stability.rs | 10 +++---
 compiler/rustc_passes/src/stability.rs        |  4 +--
 src/librustdoc/html/render/mod.rs             |  8 ++---
 src/librustdoc/json/conversions.rs            |  8 ++---
 5 files changed, 32 insertions(+), 30 deletions(-)

diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs
index a9466d7df9d6..1334868d05cc 100644
--- a/compiler/rustc_attr/src/builtin.rs
+++ b/compiler/rustc_attr/src/builtin.rs
@@ -720,7 +720,7 @@ pub fn eval_condition(
 
 #[derive(Copy, Debug, Encodable, Decodable, Clone, HashStable_Generic)]
 pub struct Deprecation {
-    pub since: Option<DeprecatedSince>,
+    pub since: DeprecatedSince,
     /// The note to issue a reason.
     pub note: Option<Symbol>,
     /// A text snippet used to completely replace any use of the deprecated item in an expression.
@@ -738,8 +738,10 @@ pub enum DeprecatedSince {
     /// `feature(staged_api)` is off. Deprecation versions outside the standard
     /// library are allowed to be arbitrary strings, for better or worse.
     Symbol(Symbol),
-    /// Failed to parse a deprecation version. An error has already been
-    /// emitted.
+    /// Deprecation version is unspecified but optional.
+    Unspecified,
+    /// Failed to parse a deprecation version, or the deprecation version is
+    /// unspecified and required. An error has already been emitted.
     Err,
 }
 
@@ -749,12 +751,12 @@ impl Deprecation {
     /// version).
     pub fn is_in_effect(&self) -> bool {
         match self.since {
-            Some(DeprecatedSince::RustcVersion(since)) => since <= RustcVersion::CURRENT,
-            Some(DeprecatedSince::Future) => false,
+            DeprecatedSince::RustcVersion(since) => since <= RustcVersion::CURRENT,
+            DeprecatedSince::Future => false,
             // The `since` field doesn't have semantic purpose without `#![staged_api]`.
-            Some(DeprecatedSince::Symbol(_)) => true,
+            DeprecatedSince::Symbol(_) => true,
             // Assume deprecation is in effect if "since" field is absent or invalid.
-            None | Some(DeprecatedSince::Err) => true,
+            DeprecatedSince::Unspecified | DeprecatedSince::Err => true,
         }
     }
 }
@@ -867,20 +869,20 @@ pub fn find_deprecation(
 
         let since = if let Some(since) = since {
             if since.as_str() == "TBD" {
-                Some(DeprecatedSince::Future)
+                DeprecatedSince::Future
             } else if !is_rustc {
-                Some(DeprecatedSince::Symbol(since))
+                DeprecatedSince::Symbol(since)
             } else if let Some(version) = parse_version(since) {
-                Some(DeprecatedSince::RustcVersion(version))
+                DeprecatedSince::RustcVersion(version)
             } else {
                 sess.emit_err(session_diagnostics::InvalidSince { span: attr.span });
-                Some(DeprecatedSince::Err)
+                DeprecatedSince::Err
             }
+        } else if is_rustc {
+            sess.emit_err(session_diagnostics::MissingSince { span: attr.span });
+            DeprecatedSince::Err
         } else {
-            if is_rustc {
-                sess.emit_err(session_diagnostics::MissingSince { span: attr.span });
-            }
-            None
+            DeprecatedSince::Unspecified
         };
 
         if is_rustc && note.is_none() {
diff --git a/compiler/rustc_middle/src/middle/stability.rs b/compiler/rustc_middle/src/middle/stability.rs
index 4d27bcad911d..c4439896faa0 100644
--- a/compiler/rustc_middle/src/middle/stability.rs
+++ b/compiler/rustc_middle/src/middle/stability.rs
@@ -147,7 +147,7 @@ fn deprecation_lint(is_in_effect: bool) -> &'static Lint {
 
 fn deprecation_message(
     is_in_effect: bool,
-    since: Option<DeprecatedSince>,
+    since: DeprecatedSince,
     note: Option<Symbol>,
     kind: &str,
     path: &str,
@@ -156,13 +156,13 @@ fn deprecation_message(
         format!("use of deprecated {kind} `{path}`")
     } else {
         match since {
-            Some(DeprecatedSince::RustcVersion(version)) => format!(
+            DeprecatedSince::RustcVersion(version) => format!(
                 "use of {kind} `{path}` that will be deprecated in future version {version}"
             ),
-            Some(DeprecatedSince::Future) => {
+            DeprecatedSince::Future => {
                 format!("use of {kind} `{path}` that will be deprecated in a future Rust version")
             }
-            Some(DeprecatedSince::Symbol(_)) | Some(DeprecatedSince::Err) | None => {
+            DeprecatedSince::Symbol(_) | DeprecatedSince::Unspecified | DeprecatedSince::Err => {
                 unreachable!("this deprecation is always in effect; {since:?}")
             }
         }
@@ -347,7 +347,7 @@ impl<'tcx> TyCtxt<'tcx> {
                 // With #![staged_api], we want to emit down the whole
                 // hierarchy.
                 let depr_attr = &depr_entry.attr;
-                if !skip || matches!(depr_attr.since, Some(DeprecatedSince::RustcVersion(_))) {
+                if !skip || matches!(depr_attr.since, DeprecatedSince::RustcVersion(_)) {
                     // Calculating message for lint involves calling `self.def_path_str`.
                     // Which by default to calculate visible path will invoke expensive `visible_parent_map` query.
                     // So we skip message calculation altogether, if lint is allowed.
diff --git a/compiler/rustc_passes/src/stability.rs b/compiler/rustc_passes/src/stability.rs
index 675fe6c5d42b..6ef160f3b393 100644
--- a/compiler/rustc_passes/src/stability.rs
+++ b/compiler/rustc_passes/src/stability.rs
@@ -197,7 +197,7 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
         }
 
         if let Some((
-            rustc_attr::Deprecation { since: Some(DeprecatedSince::RustcVersion(_)), .. },
+            rustc_attr::Deprecation { since: DeprecatedSince::RustcVersion(_), .. },
             span,
         )) = &depr
         {
@@ -228,7 +228,7 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
             if let (
                 &Some(DeprecatedSince::RustcVersion(dep_since)),
                 &attr::Stable { since: stab_since, .. },
-            ) = (&depr.as_ref().and_then(|(d, _)| d.since), &stab.level)
+            ) = (&depr.as_ref().map(|(d, _)| d.since), &stab.level)
             {
                 match stab_since {
                     StableSince::Current => {
diff --git a/src/librustdoc/html/render/mod.rs b/src/librustdoc/html/render/mod.rs
index b64bfa2ae8a0..ea697108ae24 100644
--- a/src/librustdoc/html/render/mod.rs
+++ b/src/librustdoc/html/render/mod.rs
@@ -620,18 +620,18 @@ fn short_item_info(
         // We display deprecation messages for #[deprecated], but only display
         // the future-deprecation messages for rustc versions.
         let mut message = match since {
-            Some(DeprecatedSince::RustcVersion(version)) => {
+            DeprecatedSince::RustcVersion(version) => {
                 if depr.is_in_effect() {
                     format!("Deprecated since {version}")
                 } else {
                     format!("Deprecating in {version}")
                 }
             }
-            Some(DeprecatedSince::Future) => String::from("Deprecating in a future Rust version"),
-            Some(DeprecatedSince::Symbol(since)) => {
+            DeprecatedSince::Future => String::from("Deprecating in a future Rust version"),
+            DeprecatedSince::Symbol(since) => {
                 format!("Deprecated since {}", Escape(since.as_str()))
             }
-            Some(DeprecatedSince::Err) | None => String::from("Deprecated"),
+            DeprecatedSince::Unspecified | DeprecatedSince::Err => String::from("Deprecated"),
         };
 
         if let Some(note) = note {
diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs
index acc928360d2e..0b05e304dfc2 100644
--- a/src/librustdoc/json/conversions.rs
+++ b/src/librustdoc/json/conversions.rs
@@ -141,10 +141,10 @@ where
 pub(crate) fn from_deprecation(deprecation: rustc_attr::Deprecation) -> Deprecation {
     let rustc_attr::Deprecation { since, note, suggestion: _ } = deprecation;
     let since = match since {
-        Some(DeprecatedSince::RustcVersion(version)) => Some(version.to_string()),
-        Some(DeprecatedSince::Future) => Some("TBD".to_owned()),
-        Some(DeprecatedSince::Symbol(since)) => Some(since.to_string()),
-        Some(DeprecatedSince::Err) | None => None,
+        DeprecatedSince::RustcVersion(version) => Some(version.to_string()),
+        DeprecatedSince::Future => Some("TBD".to_owned()),
+        DeprecatedSince::Symbol(since) => Some(since.to_string()),
+        DeprecatedSince::Unspecified | DeprecatedSince::Err => None,
     };
     Deprecation { since, note: note.map(|s| s.to_string()) }
 }

From add09e66f28c92eef41e730a63191cd2a824598a Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Mon, 30 Oct 2023 23:35:35 +0000
Subject: [PATCH 265/297] Some more coroutine renamings

---
 compiler/rustc_ast_lowering/src/expr.rs       |  8 ++--
 compiler/rustc_ast_lowering/src/item.rs       |  4 +-
 compiler/rustc_hir/src/lang_items.rs          |  4 +-
 compiler/rustc_hir_typeck/src/check.rs        |  8 ++--
 compiler/rustc_hir_typeck/src/closure.rs      |  4 +-
 compiler/rustc_middle/src/ty/print/pretty.rs  |  2 +-
 compiler/rustc_mir_build/src/build/mod.rs     |  8 ++--
 compiler/rustc_mir_build/src/thir/cx/mod.rs   | 13 +++--
 compiler/rustc_mir_transform/src/coroutine.rs | 47 ++++++++++---------
 compiler/rustc_mir_transform/src/shim.rs      | 14 +++---
 .../src/solve/assembly/mod.rs                 |  2 +-
 .../error_reporting/type_err_ctxt_ext.rs      |  5 +-
 .../src/traits/project.rs                     | 30 ++++++------
 .../src/traits/select/candidate_assembly.rs   |  2 +-
 .../src/traits/select/confirmation.rs         |  8 ++--
 compiler/rustc_ty_utils/src/instance.rs       |  2 +-
 16 files changed, 85 insertions(+), 76 deletions(-)

diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs
index d6fa74dc3e6f..93805710cb54 100644
--- a/compiler/rustc_ast_lowering/src/expr.rs
+++ b/compiler/rustc_ast_lowering/src/expr.rs
@@ -606,7 +606,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         closure_node_id: NodeId,
         ret_ty: Option<hir::FnRetTy<'hir>>,
         span: Span,
-        async_gen_kind: hir::CoroutineSource,
+        async_coroutine_source: hir::CoroutineSource,
         body: impl FnOnce(&mut Self) -> hir::Expr<'hir>,
     ) -> hir::ExprKind<'hir> {
         let output = ret_ty.unwrap_or_else(|| hir::FnRetTy::DefaultReturn(self.lower_span(span)));
@@ -645,7 +645,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         let params = arena_vec![self; param];
 
         let body = self.lower_body(move |this| {
-            this.coroutine_kind = Some(hir::CoroutineKind::Async(async_gen_kind));
+            this.coroutine_kind = Some(hir::CoroutineKind::Async(async_coroutine_source));
 
             let old_ctx = this.task_context;
             this.task_context = Some(task_context_hid);
@@ -684,7 +684,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         closure_node_id: NodeId,
         _yield_ty: Option<hir::FnRetTy<'hir>>,
         span: Span,
-        gen_kind: hir::CoroutineSource,
+        coroutine_source: hir::CoroutineSource,
         body: impl FnOnce(&mut Self) -> hir::Expr<'hir>,
     ) -> hir::ExprKind<'hir> {
         let output = hir::FnRetTy::DefaultReturn(self.lower_span(span));
@@ -699,7 +699,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         });
 
         let body = self.lower_body(move |this| {
-            this.coroutine_kind = Some(hir::CoroutineKind::Gen(gen_kind));
+            this.coroutine_kind = Some(hir::CoroutineKind::Gen(coroutine_source));
 
             let res = body(this);
             (&[], res)
diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs
index ca851bd4d35f..299ca91460d8 100644
--- a/compiler/rustc_ast_lowering/src/item.rs
+++ b/compiler/rustc_ast_lowering/src/item.rs
@@ -988,12 +988,12 @@ impl<'hir> LoweringContext<'_, 'hir> {
         &mut self,
         f: impl FnOnce(&mut Self) -> (&'hir [hir::Param<'hir>], hir::Expr<'hir>),
     ) -> hir::BodyId {
-        let prev_gen_kind = self.coroutine_kind.take();
+        let prev_coroutine_kind = self.coroutine_kind.take();
         let task_context = self.task_context.take();
         let (parameters, result) = f(self);
         let body_id = self.record_body(parameters, result);
         self.task_context = task_context;
-        self.coroutine_kind = prev_gen_kind;
+        self.coroutine_kind = prev_coroutine_kind;
         body_id
     }
 
diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs
index c8ac95e29949..1d1a1ee88627 100644
--- a/compiler/rustc_hir/src/lang_items.rs
+++ b/compiler/rustc_hir/src/lang_items.rs
@@ -212,8 +212,8 @@ language_item_table! {
 
     Iterator,                sym::iterator,            iterator_trait,             Target::Trait,          GenericRequirement::Exact(0);
     Future,                  sym::future_trait,        future_trait,               Target::Trait,          GenericRequirement::Exact(0);
-    CoroutineState,          sym::coroutine_state,     gen_state,                  Target::Enum,           GenericRequirement::None;
-    Coroutine,               sym::coroutine,           gen_trait,                  Target::Trait,          GenericRequirement::Minimum(1);
+    CoroutineState,          sym::coroutine_state,     coroutine_state,            Target::Enum,           GenericRequirement::None;
+    Coroutine,               sym::coroutine,           coroutine_trait,            Target::Trait,          GenericRequirement::Minimum(1);
     Unpin,                   sym::unpin,               unpin_trait,                Target::Trait,          GenericRequirement::None;
     Pin,                     sym::pin,                 pin_type,                   Target::Struct,         GenericRequirement::None;
 
diff --git a/compiler/rustc_hir_typeck/src/check.rs b/compiler/rustc_hir_typeck/src/check.rs
index 14cd3881a068..b8a265d4971d 100644
--- a/compiler/rustc_hir_typeck/src/check.rs
+++ b/compiler/rustc_hir_typeck/src/check.rs
@@ -128,14 +128,16 @@ pub(super) fn check_fn<'a, 'tcx>(
     // We insert the deferred_coroutine_interiors entry after visiting the body.
     // This ensures that all nested coroutines appear before the entry of this coroutine.
     // resolve_coroutine_interiors relies on this property.
-    let gen_ty = if let (Some(_), Some(gen_kind)) = (can_be_coroutine, body.coroutine_kind) {
+    let coroutine_ty = if let (Some(_), Some(coroutine_kind)) =
+        (can_be_coroutine, body.coroutine_kind)
+    {
         let interior = fcx
             .next_ty_var(TypeVariableOrigin { kind: TypeVariableOriginKind::MiscVariable, span });
         fcx.deferred_coroutine_interiors.borrow_mut().push((
             fn_def_id,
             body.id(),
             interior,
-            gen_kind,
+            coroutine_kind,
         ));
 
         let (resume_ty, yield_ty) = fcx.resume_yield_tys.unwrap();
@@ -184,7 +186,7 @@ pub(super) fn check_fn<'a, 'tcx>(
         check_lang_start_fn(tcx, fn_sig, fn_def_id);
     }
 
-    gen_ty
+    coroutine_ty
 }
 
 fn check_panic_info_fn(tcx: TyCtxt<'_>, fn_id: LocalDefId, fn_sig: ty::FnSig<'_>) {
diff --git a/compiler/rustc_hir_typeck/src/closure.rs b/compiler/rustc_hir_typeck/src/closure.rs
index b102ddd8d866..d6073dbc128a 100644
--- a/compiler/rustc_hir_typeck/src/closure.rs
+++ b/compiler/rustc_hir_typeck/src/closure.rs
@@ -302,8 +302,8 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
         let is_fn = tcx.is_fn_trait(trait_def_id);
 
-        let gen_trait = tcx.lang_items().gen_trait();
-        let is_gen = gen_trait == Some(trait_def_id);
+        let coroutine_trait = tcx.lang_items().coroutine_trait();
+        let is_gen = coroutine_trait == Some(trait_def_id);
 
         if !is_fn && !is_gen {
             debug!("not fn or coroutine");
diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs
index 799ba2c35ec8..baf160bcc99d 100644
--- a/compiler/rustc_middle/src/ty/print/pretty.rs
+++ b/compiler/rustc_middle/src/ty/print/pretty.rs
@@ -1053,7 +1053,7 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write {
                         let term = if let Some(ty) = term.skip_binder().ty()
                             && let ty::Alias(ty::Projection, proj) = ty.kind()
                             && let Some(assoc) = tcx.opt_associated_item(proj.def_id)
-                            && assoc.trait_container(tcx) == tcx.lang_items().gen_trait()
+                            && assoc.trait_container(tcx) == tcx.lang_items().coroutine_trait()
                             && assoc.name == rustc_span::sym::Return
                         {
                             if let ty::Coroutine(_, args, _) = args.type_at(0).kind() {
diff --git a/compiler/rustc_mir_build/src/build/mod.rs b/compiler/rustc_mir_build/src/build/mod.rs
index 1f817633a2a1..58d6be50b905 100644
--- a/compiler/rustc_mir_build/src/build/mod.rs
+++ b/compiler/rustc_mir_build/src/build/mod.rs
@@ -483,14 +483,14 @@ fn construct_fn<'tcx>(
     let arguments = &thir.params;
 
     let (yield_ty, return_ty) = if coroutine_kind.is_some() {
-        let gen_ty = arguments[thir::UPVAR_ENV_PARAM].ty;
-        let gen_sig = match gen_ty.kind() {
+        let coroutine_ty = arguments[thir::UPVAR_ENV_PARAM].ty;
+        let coroutine_sig = match coroutine_ty.kind() {
             ty::Coroutine(_, gen_args, ..) => gen_args.as_coroutine().sig(),
             _ => {
-                span_bug!(span, "coroutine w/o coroutine type: {:?}", gen_ty)
+                span_bug!(span, "coroutine w/o coroutine type: {:?}", coroutine_ty)
             }
         };
-        (Some(gen_sig.yield_ty), gen_sig.return_ty)
+        (Some(coroutine_sig.yield_ty), coroutine_sig.return_ty)
     } else {
         (None, fn_sig.output())
     };
diff --git a/compiler/rustc_mir_build/src/thir/cx/mod.rs b/compiler/rustc_mir_build/src/thir/cx/mod.rs
index 8bc4cbb95329..b6adb383fa6b 100644
--- a/compiler/rustc_mir_build/src/thir/cx/mod.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/mod.rs
@@ -149,10 +149,15 @@ impl<'tcx> Cx<'tcx> {
                 Some(env_param)
             }
             DefKind::Coroutine => {
-                let gen_ty = self.typeck_results.node_type(owner_id);
-                let gen_param =
-                    Param { ty: gen_ty, pat: None, ty_span: None, self_kind: None, hir_id: None };
-                Some(gen_param)
+                let coroutine_ty = self.typeck_results.node_type(owner_id);
+                let coroutine_param = Param {
+                    ty: coroutine_ty,
+                    pat: None,
+                    ty_span: None,
+                    self_kind: None,
+                    hir_id: None,
+                };
+                Some(coroutine_param)
             }
             _ => None,
         }
diff --git a/compiler/rustc_mir_transform/src/coroutine.rs b/compiler/rustc_mir_transform/src/coroutine.rs
index 50d244d2831d..fc30a718cbb5 100644
--- a/compiler/rustc_mir_transform/src/coroutine.rs
+++ b/compiler/rustc_mir_transform/src/coroutine.rs
@@ -147,7 +147,7 @@ impl<'tcx> MutVisitor<'tcx> for DerefArgVisitor<'tcx> {
 }
 
 struct PinArgVisitor<'tcx> {
-    ref_gen_ty: Ty<'tcx>,
+    ref_coroutine_ty: Ty<'tcx>,
     tcx: TyCtxt<'tcx>,
 }
 
@@ -168,7 +168,7 @@ impl<'tcx> MutVisitor<'tcx> for PinArgVisitor<'tcx> {
                     local: SELF_ARG,
                     projection: self.tcx().mk_place_elems(&[ProjectionElem::Field(
                         FieldIdx::new(0),
-                        self.ref_gen_ty,
+                        self.ref_coroutine_ty,
                     )]),
                 },
                 self.tcx,
@@ -468,34 +468,34 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> {
 }
 
 fn make_coroutine_state_argument_indirect<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-    let gen_ty = body.local_decls.raw[1].ty;
+    let coroutine_ty = body.local_decls.raw[1].ty;
 
-    let ref_gen_ty = Ty::new_ref(
+    let ref_coroutine_ty = Ty::new_ref(
         tcx,
         tcx.lifetimes.re_erased,
-        ty::TypeAndMut { ty: gen_ty, mutbl: Mutability::Mut },
+        ty::TypeAndMut { ty: coroutine_ty, mutbl: Mutability::Mut },
     );
 
     // Replace the by value coroutine argument
-    body.local_decls.raw[1].ty = ref_gen_ty;
+    body.local_decls.raw[1].ty = ref_coroutine_ty;
 
     // Add a deref to accesses of the coroutine state
     DerefArgVisitor { tcx }.visit_body(body);
 }
 
 fn make_coroutine_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-    let ref_gen_ty = body.local_decls.raw[1].ty;
+    let ref_coroutine_ty = body.local_decls.raw[1].ty;
 
     let pin_did = tcx.require_lang_item(LangItem::Pin, Some(body.span));
     let pin_adt_ref = tcx.adt_def(pin_did);
-    let args = tcx.mk_args(&[ref_gen_ty.into()]);
-    let pin_ref_gen_ty = Ty::new_adt(tcx, pin_adt_ref, args);
+    let args = tcx.mk_args(&[ref_coroutine_ty.into()]);
+    let pin_ref_coroutine_ty = Ty::new_adt(tcx, pin_adt_ref, args);
 
     // Replace the by ref coroutine argument
-    body.local_decls.raw[1].ty = pin_ref_gen_ty;
+    body.local_decls.raw[1].ty = pin_ref_coroutine_ty;
 
     // Add the Pin field access to accesses of the coroutine state
-    PinArgVisitor { ref_gen_ty, tcx }.visit_body(body);
+    PinArgVisitor { ref_coroutine_ty, tcx }.visit_body(body);
 }
 
 /// Allocates a new local and replaces all references of `local` with it. Returns the new local.
@@ -1104,7 +1104,7 @@ fn elaborate_coroutine_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
 fn create_coroutine_drop_shim<'tcx>(
     tcx: TyCtxt<'tcx>,
     transform: &TransformVisitor<'tcx>,
-    gen_ty: Ty<'tcx>,
+    coroutine_ty: Ty<'tcx>,
     body: &mut Body<'tcx>,
     drop_clean: BasicBlock,
 ) -> Body<'tcx> {
@@ -1136,7 +1136,7 @@ fn create_coroutine_drop_shim<'tcx>(
 
     // Change the coroutine argument from &mut to *mut
     body.local_decls[SELF_ARG] = LocalDecl::with_source_info(
-        Ty::new_ptr(tcx, ty::TypeAndMut { ty: gen_ty, mutbl: hir::Mutability::Mut }),
+        Ty::new_ptr(tcx, ty::TypeAndMut { ty: coroutine_ty, mutbl: hir::Mutability::Mut }),
         source_info,
     );
 
@@ -1146,9 +1146,9 @@ fn create_coroutine_drop_shim<'tcx>(
 
     // Update the body's def to become the drop glue.
     // This needs to be updated before the AbortUnwindingCalls pass.
-    let gen_instance = body.source.instance;
+    let coroutine_instance = body.source.instance;
     let drop_in_place = tcx.require_lang_item(LangItem::DropInPlace, None);
-    let drop_instance = InstanceDef::DropGlue(drop_in_place, Some(gen_ty));
+    let drop_instance = InstanceDef::DropGlue(drop_in_place, Some(coroutine_ty));
     body.source.instance = drop_instance;
 
     pm::run_passes_no_validate(
@@ -1160,7 +1160,7 @@ fn create_coroutine_drop_shim<'tcx>(
 
     // Temporary change MirSource to coroutine's instance so that dump_mir produces more sensible
     // filename.
-    body.source.instance = gen_instance;
+    body.source.instance = coroutine_instance;
     dump_mir(tcx, false, "coroutine_drop", &0, &body, |_, _| Ok(()));
     body.source.instance = drop_instance;
 
@@ -1447,13 +1447,13 @@ pub(crate) fn mir_coroutine_witnesses<'tcx>(
     let body = &*body;
 
     // The first argument is the coroutine type passed by value
-    let gen_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty;
+    let coroutine_ty = body.local_decls[ty::CAPTURE_STRUCT_LOCAL].ty;
 
     // Get the interior types and args which typeck computed
-    let movable = match *gen_ty.kind() {
+    let movable = match *coroutine_ty.kind() {
         ty::Coroutine(_, _, movability) => movability == hir::Movability::Movable,
         ty::Error(_) => return None,
-        _ => span_bug!(body.span, "unexpected coroutine type {}", gen_ty),
+        _ => span_bug!(body.span, "unexpected coroutine type {}", coroutine_ty),
     };
 
     // When first entering the coroutine, move the resume argument into its new local.
@@ -1481,16 +1481,17 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
         assert!(body.coroutine_drop().is_none());
 
         // The first argument is the coroutine type passed by value
-        let gen_ty = body.local_decls.raw[1].ty;
+        let coroutine_ty = body.local_decls.raw[1].ty;
 
         // Get the discriminant type and args which typeck computed
-        let (discr_ty, movable) = match *gen_ty.kind() {
+        let (discr_ty, movable) = match *coroutine_ty.kind() {
             ty::Coroutine(_, args, movability) => {
                 let args = args.as_coroutine();
                 (args.discr_ty(tcx), movability == hir::Movability::Movable)
             }
             _ => {
-                tcx.sess.delay_span_bug(body.span, format!("unexpected coroutine type {gen_ty}"));
+                tcx.sess
+                    .delay_span_bug(body.span, format!("unexpected coroutine type {coroutine_ty}"));
                 return;
             }
         };
@@ -1626,7 +1627,7 @@ impl<'tcx> MirPass<'tcx> for StateTransform {
         dump_mir(tcx, false, "coroutine_post-transform", &0, body, |_, _| Ok(()));
 
         // Create a copy of our MIR and use it to create the drop shim for the coroutine
-        let drop_shim = create_coroutine_drop_shim(tcx, &transform, gen_ty, body, drop_clean);
+        let drop_shim = create_coroutine_drop_shim(tcx, &transform, coroutine_ty, body, drop_clean);
 
         body.coroutine.as_mut().unwrap().coroutine_drop = Some(drop_shim);
 
diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs
index 2400cfa21fba..4ae5ea4c8d68 100644
--- a/compiler/rustc_mir_transform/src/shim.rs
+++ b/compiler/rustc_mir_transform/src/shim.rs
@@ -69,8 +69,8 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<'
         ty::InstanceDef::DropGlue(def_id, ty) => {
             // FIXME(#91576): Drop shims for coroutines aren't subject to the MIR passes at the end
             // of this function. Is this intentional?
-            if let Some(ty::Coroutine(gen_def_id, args, _)) = ty.map(Ty::kind) {
-                let body = tcx.optimized_mir(*gen_def_id).coroutine_drop().unwrap();
+            if let Some(ty::Coroutine(coroutine_def_id, args, _)) = ty.map(Ty::kind) {
+                let body = tcx.optimized_mir(*coroutine_def_id).coroutine_drop().unwrap();
                 let mut body = EarlyBinder::bind(body.clone()).instantiate(tcx, args);
                 debug!("make_shim({:?}) = {:?}", instance, body);
 
@@ -392,8 +392,8 @@ fn build_clone_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -
         _ if is_copy => builder.copy_shim(),
         ty::Closure(_, args) => builder.tuple_like_shim(dest, src, args.as_closure().upvar_tys()),
         ty::Tuple(..) => builder.tuple_like_shim(dest, src, self_ty.tuple_fields()),
-        ty::Coroutine(gen_def_id, args, hir::Movability::Movable) => {
-            builder.coroutine_shim(dest, src, *gen_def_id, args.as_coroutine())
+        ty::Coroutine(coroutine_def_id, args, hir::Movability::Movable) => {
+            builder.coroutine_shim(dest, src, *coroutine_def_id, args.as_coroutine())
         }
         _ => bug!("clone shim for `{:?}` which is not `Copy` and is not an aggregate", self_ty),
     };
@@ -597,7 +597,7 @@ impl<'tcx> CloneShimBuilder<'tcx> {
         &mut self,
         dest: Place<'tcx>,
         src: Place<'tcx>,
-        gen_def_id: DefId,
+        coroutine_def_id: DefId,
         args: CoroutineArgs<'tcx>,
     ) {
         self.block(vec![], TerminatorKind::Goto { target: self.block_index_offset(3) }, false);
@@ -607,8 +607,8 @@ impl<'tcx> CloneShimBuilder<'tcx> {
         let unwind = self.clone_fields(dest, src, switch, unwind, args.upvar_tys());
         let target = self.block(vec![], TerminatorKind::Return, false);
         let unreachable = self.block(vec![], TerminatorKind::Unreachable, false);
-        let mut cases = Vec::with_capacity(args.state_tys(gen_def_id, self.tcx).count());
-        for (index, state_tys) in args.state_tys(gen_def_id, self.tcx).enumerate() {
+        let mut cases = Vec::with_capacity(args.state_tys(coroutine_def_id, self.tcx).count());
+        for (index, state_tys) in args.state_tys(coroutine_def_id, self.tcx).enumerate() {
             let variant_index = VariantIdx::new(index);
             let dest = self.tcx.mk_place_downcast_unnamed(dest, variant_index);
             let src = self.tcx.mk_place_downcast_unnamed(src, variant_index);
diff --git a/compiler/rustc_trait_selection/src/solve/assembly/mod.rs b/compiler/rustc_trait_selection/src/solve/assembly/mod.rs
index 1ab2bb40af79..27d2bdead838 100644
--- a/compiler/rustc_trait_selection/src/solve/assembly/mod.rs
+++ b/compiler/rustc_trait_selection/src/solve/assembly/mod.rs
@@ -564,7 +564,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             G::consider_builtin_future_candidate(self, goal)
         } else if lang_items.iterator_trait() == Some(trait_def_id) {
             G::consider_builtin_iterator_candidate(self, goal)
-        } else if lang_items.gen_trait() == Some(trait_def_id) {
+        } else if lang_items.coroutine_trait() == Some(trait_def_id) {
             G::consider_builtin_coroutine_candidate(self, goal)
         } else if lang_items.discriminant_kind_trait() == Some(trait_def_id) {
             G::consider_builtin_discriminant_kind_candidate(self, goal)
diff --git a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
index f9ce81ce1655..ba2e3d1ae282 100644
--- a/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
+++ b/compiler/rustc_trait_selection/src/traits/error_reporting/type_err_ctxt_ext.rs
@@ -1648,7 +1648,7 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
     }
 
     fn describe_coroutine(&self, body_id: hir::BodyId) -> Option<&'static str> {
-        self.tcx.hir().body(body_id).coroutine_kind.map(|gen_kind| match gen_kind {
+        self.tcx.hir().body(body_id).coroutine_kind.map(|coroutine_source| match coroutine_source {
             hir::CoroutineKind::Coroutine => "a coroutine",
             hir::CoroutineKind::Async(hir::CoroutineSource::Block) => "an async block",
             hir::CoroutineKind::Async(hir::CoroutineSource::Fn) => "an async function",
@@ -3187,7 +3187,8 @@ impl<'tcx> InferCtxtPrivExt<'tcx> for TypeErrCtxt<'_, 'tcx> {
         // traits manually, but don't make it more confusing when it does
         // happen.
         Some(
-            if Some(expected_trait_ref.def_id()) != self.tcx.lang_items().gen_trait() && not_tupled
+            if Some(expected_trait_ref.def_id()) != self.tcx.lang_items().coroutine_trait()
+                && not_tupled
             {
                 self.report_and_explain_type_error(
                     TypeTrace::poly_trait_refs(
diff --git a/compiler/rustc_trait_selection/src/traits/project.rs b/compiler/rustc_trait_selection/src/traits/project.rs
index 20aa3cec8733..e4f7592c409e 100644
--- a/compiler/rustc_trait_selection/src/traits/project.rs
+++ b/compiler/rustc_trait_selection/src/traits/project.rs
@@ -1798,7 +1798,7 @@ fn assemble_candidates_from_impls<'cx, 'tcx>(
                 let self_ty = selcx.infcx.shallow_resolve(obligation.predicate.self_ty());
 
                 let lang_items = selcx.tcx().lang_items();
-                if [lang_items.gen_trait(), lang_items.future_trait(), lang_items.iterator_trait()].contains(&Some(trait_ref.def_id))
+                if [lang_items.coroutine_trait(), lang_items.future_trait(), lang_items.iterator_trait()].contains(&Some(trait_ref.def_id))
                     || selcx.tcx().fn_trait_kind_from_def_id(trait_ref.def_id).is_some()
                 {
                     true
@@ -2011,7 +2011,7 @@ fn confirm_select_candidate<'cx, 'tcx>(
         ImplSource::Builtin(BuiltinImplSource::Misc, data) => {
             let trait_def_id = obligation.predicate.trait_def_id(selcx.tcx());
             let lang_items = selcx.tcx().lang_items();
-            if lang_items.gen_trait() == Some(trait_def_id) {
+            if lang_items.coroutine_trait() == Some(trait_def_id) {
                 confirm_coroutine_candidate(selcx, obligation, data)
             } else if lang_items.future_trait() == Some(trait_def_id) {
                 confirm_future_candidate(selcx, obligation, data)
@@ -2051,26 +2051,26 @@ fn confirm_coroutine_candidate<'cx, 'tcx>(
     else {
         unreachable!()
     };
-    let gen_sig = args.as_coroutine().poly_sig();
-    let Normalized { value: gen_sig, obligations } = normalize_with_depth(
+    let coroutine_sig = args.as_coroutine().poly_sig();
+    let Normalized { value: coroutine_sig, obligations } = normalize_with_depth(
         selcx,
         obligation.param_env,
         obligation.cause.clone(),
         obligation.recursion_depth + 1,
-        gen_sig,
+        coroutine_sig,
     );
 
-    debug!(?obligation, ?gen_sig, ?obligations, "confirm_coroutine_candidate");
+    debug!(?obligation, ?coroutine_sig, ?obligations, "confirm_coroutine_candidate");
 
     let tcx = selcx.tcx();
 
-    let gen_def_id = tcx.require_lang_item(LangItem::Coroutine, None);
+    let coroutine_def_id = tcx.require_lang_item(LangItem::Coroutine, None);
 
     let predicate = super::util::coroutine_trait_ref_and_outputs(
         tcx,
-        gen_def_id,
+        coroutine_def_id,
         obligation.predicate.self_ty(),
-        gen_sig,
+        coroutine_sig,
     )
     .map_bound(|(trait_ref, yield_ty, return_ty)| {
         let name = tcx.associated_item(obligation.predicate.def_id).name;
@@ -2103,16 +2103,16 @@ fn confirm_future_candidate<'cx, 'tcx>(
     else {
         unreachable!()
     };
-    let gen_sig = args.as_coroutine().poly_sig();
-    let Normalized { value: gen_sig, obligations } = normalize_with_depth(
+    let coroutine_sig = args.as_coroutine().poly_sig();
+    let Normalized { value: coroutine_sig, obligations } = normalize_with_depth(
         selcx,
         obligation.param_env,
         obligation.cause.clone(),
         obligation.recursion_depth + 1,
-        gen_sig,
+        coroutine_sig,
     );
 
-    debug!(?obligation, ?gen_sig, ?obligations, "confirm_future_candidate");
+    debug!(?obligation, ?coroutine_sig, ?obligations, "confirm_future_candidate");
 
     let tcx = selcx.tcx();
     let fut_def_id = tcx.require_lang_item(LangItem::Future, None);
@@ -2121,7 +2121,7 @@ fn confirm_future_candidate<'cx, 'tcx>(
         tcx,
         fut_def_id,
         obligation.predicate.self_ty(),
-        gen_sig,
+        coroutine_sig,
     )
     .map_bound(|(trait_ref, return_ty)| {
         debug_assert_eq!(tcx.associated_item(obligation.predicate.def_id).name, sym::Output);
@@ -2156,7 +2156,7 @@ fn confirm_iterator_candidate<'cx, 'tcx>(
         gen_sig,
     );
 
-    debug!(?obligation, ?gen_sig, ?obligations, "confirm_future_candidate");
+    debug!(?obligation, ?gen_sig, ?obligations, "confirm_iterator_candidate");
 
     let tcx = selcx.tcx();
     let iter_def_id = tcx.require_lang_item(LangItem::Iterator, None);
diff --git a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
index ee8e1bf7675d..5c67188dd240 100644
--- a/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/candidate_assembly.rs
@@ -109,7 +109,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
                     self.assemble_builtin_bound_candidates(clone_conditions, &mut candidates);
                 }
 
-                if lang_items.gen_trait() == Some(def_id) {
+                if lang_items.coroutine_trait() == Some(def_id) {
                     self.assemble_coroutine_candidates(obligation, &mut candidates);
                 } else if lang_items.future_trait() == Some(def_id) {
                     self.assemble_future_candidates(obligation, &mut candidates);
diff --git a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
index 952184175f4d..4bfa341e3332 100644
--- a/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
+++ b/compiler/rustc_trait_selection/src/traits/select/confirmation.rs
@@ -730,7 +730,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
 
         debug!(?obligation, ?coroutine_def_id, ?args, "confirm_coroutine_candidate");
 
-        let gen_sig = args.as_coroutine().poly_sig();
+        let coroutine_sig = args.as_coroutine().poly_sig();
 
         // NOTE: The self-type is a coroutine type and hence is
         // in fact unparameterized (or at least does not reference any
@@ -745,7 +745,7 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
             self.tcx(),
             obligation.predicate.def_id(),
             self_ty,
-            gen_sig,
+            coroutine_sig,
         )
         .map_bound(|(trait_ref, ..)| trait_ref);
 
@@ -769,13 +769,13 @@ impl<'cx, 'tcx> SelectionContext<'cx, 'tcx> {
 
         debug!(?obligation, ?coroutine_def_id, ?args, "confirm_future_candidate");
 
-        let gen_sig = args.as_coroutine().poly_sig();
+        let coroutine_sig = args.as_coroutine().poly_sig();
 
         let trait_ref = super::util::future_trait_ref_and_outputs(
             self.tcx(),
             obligation.predicate.def_id(),
             obligation.predicate.no_bound_vars().expect("future has no bound vars").self_ty(),
-            gen_sig,
+            coroutine_sig,
         )
         .map_bound(|(trait_ref, ..)| trait_ref);
 
diff --git a/compiler/rustc_ty_utils/src/instance.rs b/compiler/rustc_ty_utils/src/instance.rs
index c0fe13ac9965..1487f40fd994 100644
--- a/compiler/rustc_ty_utils/src/instance.rs
+++ b/compiler/rustc_ty_utils/src/instance.rs
@@ -271,7 +271,7 @@ fn resolve_associated_item<'tcx>(
                     debug_assert!(tcx.defaultness(trait_item_id).has_value());
                     Some(Instance::new(trait_item_id, rcvr_args))
                 }
-            } else if Some(trait_ref.def_id) == lang_items.gen_trait() {
+            } else if Some(trait_ref.def_id) == lang_items.coroutine_trait() {
                 let ty::Coroutine(coroutine_def_id, args, _) = *rcvr_args.type_at(0).kind() else {
                     bug!()
                 };

From dccf10e98969c31f4a395a3555f781b8dd17b25d Mon Sep 17 00:00:00 2001
From: David Tolnay <dtolnay@gmail.com>
Date: Mon, 30 Oct 2023 16:48:28 -0700
Subject: [PATCH 266/297] Descriptive variant name deprecation versions outside
 the standard library

---
 compiler/rustc_attr/src/builtin.rs            | 6 +++---
 compiler/rustc_middle/src/middle/stability.rs | 4 +++-
 src/librustdoc/html/render/mod.rs             | 2 +-
 src/librustdoc/json/conversions.rs            | 2 +-
 4 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs
index 1334868d05cc..33e9421eeb4d 100644
--- a/compiler/rustc_attr/src/builtin.rs
+++ b/compiler/rustc_attr/src/builtin.rs
@@ -737,7 +737,7 @@ pub enum DeprecatedSince {
     Future,
     /// `feature(staged_api)` is off. Deprecation versions outside the standard
     /// library are allowed to be arbitrary strings, for better or worse.
-    Symbol(Symbol),
+    NonStandard(Symbol),
     /// Deprecation version is unspecified but optional.
     Unspecified,
     /// Failed to parse a deprecation version, or the deprecation version is
@@ -754,7 +754,7 @@ impl Deprecation {
             DeprecatedSince::RustcVersion(since) => since <= RustcVersion::CURRENT,
             DeprecatedSince::Future => false,
             // The `since` field doesn't have semantic purpose without `#![staged_api]`.
-            DeprecatedSince::Symbol(_) => true,
+            DeprecatedSince::NonStandard(_) => true,
             // Assume deprecation is in effect if "since" field is absent or invalid.
             DeprecatedSince::Unspecified | DeprecatedSince::Err => true,
         }
@@ -871,7 +871,7 @@ pub fn find_deprecation(
             if since.as_str() == "TBD" {
                 DeprecatedSince::Future
             } else if !is_rustc {
-                DeprecatedSince::Symbol(since)
+                DeprecatedSince::NonStandard(since)
             } else if let Some(version) = parse_version(since) {
                 DeprecatedSince::RustcVersion(version)
             } else {
diff --git a/compiler/rustc_middle/src/middle/stability.rs b/compiler/rustc_middle/src/middle/stability.rs
index c4439896faa0..afa624850ac8 100644
--- a/compiler/rustc_middle/src/middle/stability.rs
+++ b/compiler/rustc_middle/src/middle/stability.rs
@@ -162,7 +162,9 @@ fn deprecation_message(
             DeprecatedSince::Future => {
                 format!("use of {kind} `{path}` that will be deprecated in a future Rust version")
             }
-            DeprecatedSince::Symbol(_) | DeprecatedSince::Unspecified | DeprecatedSince::Err => {
+            DeprecatedSince::NonStandard(_)
+            | DeprecatedSince::Unspecified
+            | DeprecatedSince::Err => {
                 unreachable!("this deprecation is always in effect; {since:?}")
             }
         }
diff --git a/src/librustdoc/html/render/mod.rs b/src/librustdoc/html/render/mod.rs
index ea697108ae24..c52fa01bdc41 100644
--- a/src/librustdoc/html/render/mod.rs
+++ b/src/librustdoc/html/render/mod.rs
@@ -628,7 +628,7 @@ fn short_item_info(
                 }
             }
             DeprecatedSince::Future => String::from("Deprecating in a future Rust version"),
-            DeprecatedSince::Symbol(since) => {
+            DeprecatedSince::NonStandard(since) => {
                 format!("Deprecated since {}", Escape(since.as_str()))
             }
             DeprecatedSince::Unspecified | DeprecatedSince::Err => String::from("Deprecated"),
diff --git a/src/librustdoc/json/conversions.rs b/src/librustdoc/json/conversions.rs
index 0b05e304dfc2..285923251f73 100644
--- a/src/librustdoc/json/conversions.rs
+++ b/src/librustdoc/json/conversions.rs
@@ -143,7 +143,7 @@ pub(crate) fn from_deprecation(deprecation: rustc_attr::Deprecation) -> Deprecat
     let since = match since {
         DeprecatedSince::RustcVersion(version) => Some(version.to_string()),
         DeprecatedSince::Future => Some("TBD".to_owned()),
-        DeprecatedSince::Symbol(since) => Some(since.to_string()),
+        DeprecatedSince::NonStandard(since) => Some(since.to_string()),
         DeprecatedSince::Unspecified | DeprecatedSince::Err => None,
     };
     Deprecation { since, note: note.map(|s| s.to_string()) }

From 8b8906b2641ab2c7b852ac956b7388cb614e5391 Mon Sep 17 00:00:00 2001
From: David Tolnay <dtolnay@gmail.com>
Date: Mon, 30 Oct 2023 16:50:40 -0700
Subject: [PATCH 267/297] Add method for checking if deprecation is a rustc
 version

---
 compiler/rustc_attr/src/builtin.rs            |  4 ++++
 compiler/rustc_middle/src/middle/stability.rs |  2 +-
 compiler/rustc_passes/src/stability.rs        | 10 ++--------
 3 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/compiler/rustc_attr/src/builtin.rs b/compiler/rustc_attr/src/builtin.rs
index 33e9421eeb4d..ad92d5855100 100644
--- a/compiler/rustc_attr/src/builtin.rs
+++ b/compiler/rustc_attr/src/builtin.rs
@@ -759,6 +759,10 @@ impl Deprecation {
             DeprecatedSince::Unspecified | DeprecatedSince::Err => true,
         }
     }
+
+    pub fn is_since_rustc_version(&self) -> bool {
+        matches!(self.since, DeprecatedSince::RustcVersion(_))
+    }
 }
 
 /// Finds the deprecation attribute. `None` if none exists.
diff --git a/compiler/rustc_middle/src/middle/stability.rs b/compiler/rustc_middle/src/middle/stability.rs
index afa624850ac8..f7a55fa95b69 100644
--- a/compiler/rustc_middle/src/middle/stability.rs
+++ b/compiler/rustc_middle/src/middle/stability.rs
@@ -349,7 +349,7 @@ impl<'tcx> TyCtxt<'tcx> {
                 // With #![staged_api], we want to emit down the whole
                 // hierarchy.
                 let depr_attr = &depr_entry.attr;
-                if !skip || matches!(depr_attr.since, DeprecatedSince::RustcVersion(_)) {
+                if !skip || depr_attr.is_since_rustc_version() {
                     // Calculating message for lint involves calling `self.def_path_str`.
                     // Which by default to calculate visible path will invoke expensive `visible_parent_map` query.
                     // So we skip message calculation altogether, if lint is allowed.
diff --git a/compiler/rustc_passes/src/stability.rs b/compiler/rustc_passes/src/stability.rs
index 6ef160f3b393..6a2498f3f99e 100644
--- a/compiler/rustc_passes/src/stability.rs
+++ b/compiler/rustc_passes/src/stability.rs
@@ -196,14 +196,8 @@ impl<'a, 'tcx> Annotator<'a, 'tcx> {
             }
         }
 
-        if let Some((
-            rustc_attr::Deprecation { since: DeprecatedSince::RustcVersion(_), .. },
-            span,
-        )) = &depr
-        {
-            if stab.is_none() {
-                self.tcx.sess.emit_err(errors::DeprecatedAttribute { span: *span });
-            }
+        if let Some((depr, span)) = &depr && depr.is_since_rustc_version() && stab.is_none() {
+            self.tcx.sess.emit_err(errors::DeprecatedAttribute { span: *span });
         }
 
         if let Some((body_stab, _span)) = body_stab {

From bcfc48db762ca4ce8758b5c8c4c52ee6e5ae5764 Mon Sep 17 00:00:00 2001
From: Josh Triplett <josh@joshtriplett.org>
Date: Tue, 31 Oct 2023 10:26:03 +0800
Subject: [PATCH 268/297] Stabilize `file_set_times`

Approved via FCP in https://github.com/rust-lang/rust/issues/98245 .
---
 library/std/src/fs.rs            | 16 +++++++---------
 library/std/src/os/ios/fs.rs     |  6 +++---
 library/std/src/os/macos/fs.rs   |  6 +++---
 library/std/src/os/watchos/fs.rs |  6 +++---
 library/std/src/os/windows/fs.rs |  6 +++---
 5 files changed, 19 insertions(+), 21 deletions(-)

diff --git a/library/std/src/fs.rs b/library/std/src/fs.rs
index c44223a2f32c..61c391336171 100644
--- a/library/std/src/fs.rs
+++ b/library/std/src/fs.rs
@@ -189,7 +189,7 @@ pub struct OpenOptions(fs_imp::OpenOptions);
 
 /// Representation of the various timestamps on a file.
 #[derive(Copy, Clone, Debug, Default)]
-#[unstable(feature = "file_set_times", issue = "98245")]
+#[stable(feature = "file_set_times", since = "CURRENT_RUSTC_VERSION")]
 pub struct FileTimes(fs_imp::FileTimes);
 
 /// Representation of the various permissions on a file.
@@ -676,8 +676,6 @@ impl File {
     /// # Examples
     ///
     /// ```no_run
-    /// #![feature(file_set_times)]
-    ///
     /// fn main() -> std::io::Result<()> {
     ///     use std::fs::{self, File, FileTimes};
     ///
@@ -690,7 +688,7 @@ impl File {
     ///     Ok(())
     /// }
     /// ```
-    #[unstable(feature = "file_set_times", issue = "98245")]
+    #[stable(feature = "file_set_times", since = "CURRENT_RUSTC_VERSION")]
     #[doc(alias = "futimens")]
     #[doc(alias = "futimes")]
     #[doc(alias = "SetFileTime")]
@@ -701,7 +699,7 @@ impl File {
     /// Changes the modification time of the underlying file.
     ///
     /// This is an alias for `set_times(FileTimes::new().set_modified(time))`.
-    #[unstable(feature = "file_set_times", issue = "98245")]
+    #[stable(feature = "file_set_times", since = "CURRENT_RUSTC_VERSION")]
     #[inline]
     pub fn set_modified(&self, time: SystemTime) -> io::Result<()> {
         self.set_times(FileTimes::new().set_modified(time))
@@ -1415,20 +1413,20 @@ impl FileTimes {
     /// Create a new `FileTimes` with no times set.
     ///
     /// Using the resulting `FileTimes` in [`File::set_times`] will not modify any timestamps.
-    #[unstable(feature = "file_set_times", issue = "98245")]
+    #[stable(feature = "file_set_times", since = "CURRENT_RUSTC_VERSION")]
     pub fn new() -> Self {
         Self::default()
     }
 
     /// Set the last access time of a file.
-    #[unstable(feature = "file_set_times", issue = "98245")]
+    #[stable(feature = "file_set_times", since = "CURRENT_RUSTC_VERSION")]
     pub fn set_accessed(mut self, t: SystemTime) -> Self {
         self.0.set_accessed(t.into_inner());
         self
     }
 
     /// Set the last modified time of a file.
-    #[unstable(feature = "file_set_times", issue = "98245")]
+    #[stable(feature = "file_set_times", since = "CURRENT_RUSTC_VERSION")]
     pub fn set_modified(mut self, t: SystemTime) -> Self {
         self.0.set_modified(t.into_inner());
         self
@@ -1442,7 +1440,7 @@ impl AsInnerMut<fs_imp::FileTimes> for FileTimes {
 }
 
 // For implementing OS extension traits in `std::os`
-#[unstable(feature = "file_set_times", issue = "98245")]
+#[stable(feature = "file_set_times", since = "CURRENT_RUSTC_VERSION")]
 impl Sealed for FileTimes {}
 
 impl Permissions {
diff --git a/library/std/src/os/ios/fs.rs b/library/std/src/os/ios/fs.rs
index b319527a541f..0d2a7189032b 100644
--- a/library/std/src/os/ios/fs.rs
+++ b/library/std/src/os/ios/fs.rs
@@ -144,14 +144,14 @@ impl MetadataExt for Metadata {
 }
 
 /// OS-specific extensions to [`fs::FileTimes`].
-#[unstable(feature = "file_set_times", issue = "98245")]
+#[stable(feature = "file_set_times", since = "CURRENT_RUSTC_VERSION")]
 pub trait FileTimesExt: Sealed {
     /// Set the creation time of a file.
-    #[unstable(feature = "file_set_times", issue = "98245")]
+    #[stable(feature = "file_set_times", since = "CURRENT_RUSTC_VERSION")]
     fn set_created(self, t: SystemTime) -> Self;
 }
 
-#[unstable(feature = "file_set_times", issue = "98245")]
+#[stable(feature = "file_set_times", since = "CURRENT_RUSTC_VERSION")]
 impl FileTimesExt for fs::FileTimes {
     fn set_created(mut self, t: SystemTime) -> Self {
         self.as_inner_mut().set_created(t.into_inner());
diff --git a/library/std/src/os/macos/fs.rs b/library/std/src/os/macos/fs.rs
index fe82d03d8698..098b07337234 100644
--- a/library/std/src/os/macos/fs.rs
+++ b/library/std/src/os/macos/fs.rs
@@ -150,14 +150,14 @@ impl MetadataExt for Metadata {
 }
 
 /// OS-specific extensions to [`fs::FileTimes`].
-#[unstable(feature = "file_set_times", issue = "98245")]
+#[stable(feature = "file_set_times", since = "CURRENT_RUSTC_VERSION")]
 pub trait FileTimesExt: Sealed {
     /// Set the creation time of a file.
-    #[unstable(feature = "file_set_times", issue = "98245")]
+    #[stable(feature = "file_set_times", since = "CURRENT_RUSTC_VERSION")]
     fn set_created(self, t: SystemTime) -> Self;
 }
 
-#[unstable(feature = "file_set_times", issue = "98245")]
+#[stable(feature = "file_set_times", since = "CURRENT_RUSTC_VERSION")]
 impl FileTimesExt for fs::FileTimes {
     fn set_created(mut self, t: SystemTime) -> Self {
         self.as_inner_mut().set_created(t.into_inner());
diff --git a/library/std/src/os/watchos/fs.rs b/library/std/src/os/watchos/fs.rs
index 2ecc4c68a962..2838501817c2 100644
--- a/library/std/src/os/watchos/fs.rs
+++ b/library/std/src/os/watchos/fs.rs
@@ -144,14 +144,14 @@ impl MetadataExt for Metadata {
 }
 
 /// OS-specific extensions to [`fs::FileTimes`].
-#[unstable(feature = "file_set_times", issue = "98245")]
+#[stable(feature = "file_set_times", since = "CURRENT_RUSTC_VERSION")]
 pub trait FileTimesExt: Sealed {
     /// Set the creation time of a file.
-    #[unstable(feature = "file_set_times", issue = "98245")]
+    #[stable(feature = "file_set_times", since = "CURRENT_RUSTC_VERSION")]
     fn set_created(self, t: SystemTime) -> Self;
 }
 
-#[unstable(feature = "file_set_times", issue = "98245")]
+#[stable(feature = "file_set_times", since = "CURRENT_RUSTC_VERSION")]
 impl FileTimesExt for fs::FileTimes {
     fn set_created(mut self, t: SystemTime) -> Self {
         self.as_inner_mut().set_created(t.into_inner());
diff --git a/library/std/src/os/windows/fs.rs b/library/std/src/os/windows/fs.rs
index 94509e547964..3b591a35dd72 100644
--- a/library/std/src/os/windows/fs.rs
+++ b/library/std/src/os/windows/fs.rs
@@ -528,14 +528,14 @@ impl FileTypeExt for fs::FileType {
 }
 
 /// Windows-specific extensions to [`fs::FileTimes`].
-#[unstable(feature = "file_set_times", issue = "98245")]
+#[stable(feature = "file_set_times", since = "CURRENT_RUSTC_VERSION")]
 pub trait FileTimesExt: Sealed {
     /// Set the creation time of a file.
-    #[unstable(feature = "file_set_times", issue = "98245")]
+    #[stable(feature = "file_set_times", since = "CURRENT_RUSTC_VERSION")]
     fn set_created(self, t: SystemTime) -> Self;
 }
 
-#[unstable(feature = "file_set_times", issue = "98245")]
+#[stable(feature = "file_set_times", since = "CURRENT_RUSTC_VERSION")]
 impl FileTimesExt for fs::FileTimes {
     fn set_created(mut self, t: SystemTime) -> Self {
         self.as_inner_mut().set_created(t.into_inner());

From 9df857f658ad3acfdf45268952911d51ad58123a Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Tue, 31 Oct 2023 09:30:13 +0100
Subject: [PATCH 269/297] Update to LLVM 17.0.4

---
 src/llvm-project | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/llvm-project b/src/llvm-project
index febc39711a7c..fef3d7b14ede 160000
--- a/src/llvm-project
+++ b/src/llvm-project
@@ -1 +1 @@
-Subproject commit febc39711a7c91560eb0f0980916ae23c343b99d
+Subproject commit fef3d7b14ede45d051dc688aae0bb8c8b02a0566

From 5b7cc9d7041f738700f5bd70a641a632a072abe7 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Tue, 31 Oct 2023 10:31:10 +0000
Subject: [PATCH 270/297] Do not ICE on constant evaluation failure in GVN.

---
 compiler/rustc_middle/src/mir/consts.rs       |  6 ++++--
 .../const-eval/issue-50814-2.mir-opt.stderr   | 21 +++++++++++++++++++
 ...4-2.stderr => issue-50814-2.normal.stderr} |  6 +++---
 tests/ui/consts/const-eval/issue-50814-2.rs   |  2 ++
 4 files changed, 30 insertions(+), 5 deletions(-)
 create mode 100644 tests/ui/consts/const-eval/issue-50814-2.mir-opt.stderr
 rename tests/ui/consts/const-eval/{issue-50814-2.stderr => issue-50814-2.normal.stderr} (85%)

diff --git a/compiler/rustc_middle/src/mir/consts.rs b/compiler/rustc_middle/src/mir/consts.rs
index c3300753fd16..92218f6948f4 100644
--- a/compiler/rustc_middle/src/mir/consts.rs
+++ b/compiler/rustc_middle/src/mir/consts.rs
@@ -520,11 +520,13 @@ impl<'tcx> Const<'tcx> {
                 // types are fine though.
                 ty::ConstKind::Value(_) => c.ty().is_primitive(),
                 ty::ConstKind::Unevaluated(..) | ty::ConstKind::Expr(..) => false,
+                // This can happen if evaluation of a constant failed. The result does not matter
+                // much since compilation is doomed.
+                ty::ConstKind::Error(..) => false,
                 // Should not appear in runtime MIR.
                 ty::ConstKind::Infer(..)
                 | ty::ConstKind::Bound(..)
-                | ty::ConstKind::Placeholder(..)
-                | ty::ConstKind::Error(..) => bug!(),
+                | ty::ConstKind::Placeholder(..) => bug!(),
             },
             Const::Unevaluated(..) => false,
             // If the same slice appears twice in the MIR, we cannot guarantee that we will
diff --git a/tests/ui/consts/const-eval/issue-50814-2.mir-opt.stderr b/tests/ui/consts/const-eval/issue-50814-2.mir-opt.stderr
new file mode 100644
index 000000000000..6454ce3d1ae1
--- /dev/null
+++ b/tests/ui/consts/const-eval/issue-50814-2.mir-opt.stderr
@@ -0,0 +1,21 @@
+error[E0080]: evaluation of `<A<()> as Foo<()>>::BAR` failed
+  --> $DIR/issue-50814-2.rs:16:24
+   |
+LL |     const BAR: usize = [5, 6, 7][T::BOO];
+   |                        ^^^^^^^^^^^^^^^^^ index out of bounds: the length is 3 but the index is 42
+
+note: erroneous constant encountered
+  --> $DIR/issue-50814-2.rs:20:6
+   |
+LL |     &<A<T> as Foo<T>>::BAR
+   |      ^^^^^^^^^^^^^^^^^^^^^
+
+note: erroneous constant encountered
+  --> $DIR/issue-50814-2.rs:20:5
+   |
+LL |     &<A<T> as Foo<T>>::BAR
+   |     ^^^^^^^^^^^^^^^^^^^^^^
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0080`.
diff --git a/tests/ui/consts/const-eval/issue-50814-2.stderr b/tests/ui/consts/const-eval/issue-50814-2.normal.stderr
similarity index 85%
rename from tests/ui/consts/const-eval/issue-50814-2.stderr
rename to tests/ui/consts/const-eval/issue-50814-2.normal.stderr
index 450fb0023730..c6b1df6c8f4d 100644
--- a/tests/ui/consts/const-eval/issue-50814-2.stderr
+++ b/tests/ui/consts/const-eval/issue-50814-2.normal.stderr
@@ -1,17 +1,17 @@
 error[E0080]: evaluation of `<A<()> as Foo<()>>::BAR` failed
-  --> $DIR/issue-50814-2.rs:14:24
+  --> $DIR/issue-50814-2.rs:16:24
    |
 LL |     const BAR: usize = [5, 6, 7][T::BOO];
    |                        ^^^^^^^^^^^^^^^^^ index out of bounds: the length is 3 but the index is 42
 
 note: erroneous constant encountered
-  --> $DIR/issue-50814-2.rs:18:6
+  --> $DIR/issue-50814-2.rs:20:6
    |
 LL |     &<A<T> as Foo<T>>::BAR
    |      ^^^^^^^^^^^^^^^^^^^^^
 
 note: the above error was encountered while instantiating `fn foo::<()>`
-  --> $DIR/issue-50814-2.rs:30:22
+  --> $DIR/issue-50814-2.rs:32:22
    |
 LL |     println!("{:x}", foo::<()>() as *const usize as usize);
    |                      ^^^^^^^^^^^
diff --git a/tests/ui/consts/const-eval/issue-50814-2.rs b/tests/ui/consts/const-eval/issue-50814-2.rs
index 53eb7b149f93..2eab93beb201 100644
--- a/tests/ui/consts/const-eval/issue-50814-2.rs
+++ b/tests/ui/consts/const-eval/issue-50814-2.rs
@@ -1,4 +1,6 @@
 // build-fail
+// revisions: normal mir-opt
+// [mir-opt]compile-flags: -Zmir-opt-level=4
 
 trait C {
     const BOO: usize;

From 078144e3e893ef1e52eb6bfa909777ece444dedf Mon Sep 17 00:00:00 2001
From: lcnr <rust@lcnr.de>
Date: Tue, 31 Oct 2023 11:50:30 +0100
Subject: [PATCH 271/297] prepopulate opaque ty storage before using it :>

---
 compiler/rustc_borrowck/src/type_check/mod.rs | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs
index 8477686d4f57..9f30d9d8ba12 100644
--- a/compiler/rustc_borrowck/src/type_check/mod.rs
+++ b/compiler/rustc_borrowck/src/type_check/mod.rs
@@ -188,11 +188,7 @@ pub(crate) fn type_check<'mir, 'tcx>(
         &mut borrowck_context,
     );
 
-    // FIXME(-Ztrait-solver=next): A bit dubious that we're only registering
-    // predefined opaques in the typeck root.
-    if infcx.next_trait_solver() && !infcx.tcx.is_typeck_child(body.source.def_id()) {
-        checker.register_predefined_opaques_in_new_solver();
-    }
+    checker.check_user_type_annotations();
 
     let mut verifier = TypeVerifier::new(&mut checker, promoted);
     verifier.visit_body(&body);
@@ -1021,7 +1017,13 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
             borrowck_context,
             reported_errors: Default::default(),
         };
-        checker.check_user_type_annotations();
+
+        // FIXME(-Ztrait-solver=next): A bit dubious that we're only registering
+        // predefined opaques in the typeck root.
+        if infcx.next_trait_solver() && !infcx.tcx.is_typeck_child(body.source.def_id()) {
+            checker.register_predefined_opaques_in_new_solver();
+        }
+
         checker
     }
 

From 8ef67d0f01f2271c2e6d9eb11e6437f529697927 Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Sun, 29 Oct 2023 23:20:04 +1100
Subject: [PATCH 272/297] coverage: Promote some debug-only checks to always
 run

These checks should be cheap, so there's little reason for them to be
debug-only.
---
 .../src/coverage/counters.rs                  | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs
index dcf603a4f77c..e6534b6f2e52 100644
--- a/compiler/rustc_mir_transform/src/coverage/counters.rs
+++ b/compiler/rustc_mir_transform/src/coverage/counters.rs
@@ -53,7 +53,7 @@ pub(super) struct CoverageCounters {
     /// edge between two BCBs.
     bcb_edge_counters: FxHashMap<(BasicCoverageBlock, BasicCoverageBlock), BcbCounter>,
     /// Tracks which BCBs have a counter associated with some incoming edge.
-    /// Only used by debug assertions, to verify that BCBs with incoming edge
+    /// Only used by assertions, to verify that BCBs with incoming edge
     /// counters do not have their own physical counters (expressions are allowed).
     bcb_has_incoming_edge_counters: BitSet<BasicCoverageBlock>,
     /// Table of expression data, associating each expression ID with its
@@ -116,13 +116,14 @@ impl CoverageCounters {
         bcb: BasicCoverageBlock,
         counter_kind: BcbCounter,
     ) -> Result<CovTerm, Error> {
-        debug_assert!(
+        assert!(
             // If the BCB has an edge counter (to be injected into a new `BasicBlock`), it can also
             // have an expression (to be injected into an existing `BasicBlock` represented by this
             // `BasicCoverageBlock`).
             counter_kind.is_expression() || !self.bcb_has_incoming_edge_counters.contains(bcb),
             "attempt to add a `Counter` to a BCB target with existing incoming edge counters"
         );
+
         let term = counter_kind.as_term();
         if let Some(replaced) = self.bcb_counters[bcb].replace(counter_kind) {
             Error::from_string(format!(
@@ -140,17 +141,16 @@ impl CoverageCounters {
         to_bcb: BasicCoverageBlock,
         counter_kind: BcbCounter,
     ) -> Result<CovTerm, Error> {
-        if level_enabled!(tracing::Level::DEBUG) {
-            // If the BCB has an edge counter (to be injected into a new `BasicBlock`), it can also
-            // have an expression (to be injected into an existing `BasicBlock` represented by this
-            // `BasicCoverageBlock`).
-            if self.bcb_counter(to_bcb).is_some_and(|c| !c.is_expression()) {
-                return Error::from_string(format!(
-                    "attempt to add an incoming edge counter from {from_bcb:?} when the target BCB already \
-                    has a `Counter`"
-                ));
-            }
+        // If the BCB has an edge counter (to be injected into a new `BasicBlock`), it can also
+        // have an expression (to be injected into an existing `BasicBlock` represented by this
+        // `BasicCoverageBlock`).
+        if let Some(node_counter) = self.bcb_counter(to_bcb) && !node_counter.is_expression() {
+            return Error::from_string(format!(
+                "attempt to add an incoming edge counter from {from_bcb:?} \
+                when the target BCB already has {node_counter:?}"
+            ));
         }
+
         self.bcb_has_incoming_edge_counters.insert(to_bcb);
         let term = counter_kind.as_term();
         if let Some(replaced) = self.bcb_edge_counters.insert((from_bcb, to_bcb), counter_kind) {

From 6d956a228b9808d0eb78300e988c260887a1f88e Mon Sep 17 00:00:00 2001
From: Zalathar <Zalathar@users.noreply.github.com>
Date: Sun, 29 Oct 2023 22:33:29 +1100
Subject: [PATCH 273/297] coverage: Replace impossible `coverage::Error` with
 assertions

Historically, these errors existed so that the coverage debug code could dump
additional information before reporting a compiler bug. That debug code was
removed by #115962, so we can now simplify these methods by making them panic
when they detect a bug.
---
 .../src/coverage/counters.rs                  | 77 ++++++++-----------
 .../rustc_mir_transform/src/coverage/mod.rs   | 17 +---
 .../rustc_mir_transform/src/coverage/tests.rs |  6 +-
 3 files changed, 33 insertions(+), 67 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/coverage/counters.rs b/compiler/rustc_mir_transform/src/coverage/counters.rs
index e6534b6f2e52..b34ec95b4e8d 100644
--- a/compiler/rustc_mir_transform/src/coverage/counters.rs
+++ b/compiler/rustc_mir_transform/src/coverage/counters.rs
@@ -1,5 +1,3 @@
-use super::Error;
-
 use super::graph;
 
 use graph::{BasicCoverageBlock, BcbBranch, CoverageGraph, TraverseCoverageGraphWithLoops};
@@ -81,7 +79,7 @@ impl CoverageCounters {
         &mut self,
         basic_coverage_blocks: &CoverageGraph,
         bcb_has_coverage_spans: impl Fn(BasicCoverageBlock) -> bool,
-    ) -> Result<(), Error> {
+    ) {
         MakeBcbCounters::new(self, basic_coverage_blocks).make_bcb_counters(bcb_has_coverage_spans)
     }
 
@@ -111,11 +109,7 @@ impl CoverageCounters {
         self.expressions.len()
     }
 
-    fn set_bcb_counter(
-        &mut self,
-        bcb: BasicCoverageBlock,
-        counter_kind: BcbCounter,
-    ) -> Result<CovTerm, Error> {
+    fn set_bcb_counter(&mut self, bcb: BasicCoverageBlock, counter_kind: BcbCounter) -> CovTerm {
         assert!(
             // If the BCB has an edge counter (to be injected into a new `BasicBlock`), it can also
             // have an expression (to be injected into an existing `BasicBlock` represented by this
@@ -126,12 +120,12 @@ impl CoverageCounters {
 
         let term = counter_kind.as_term();
         if let Some(replaced) = self.bcb_counters[bcb].replace(counter_kind) {
-            Error::from_string(format!(
+            bug!(
                 "attempt to set a BasicCoverageBlock coverage counter more than once; \
                 {bcb:?} already had counter {replaced:?}",
-            ))
+            );
         } else {
-            Ok(term)
+            term
         }
     }
 
@@ -140,26 +134,26 @@ impl CoverageCounters {
         from_bcb: BasicCoverageBlock,
         to_bcb: BasicCoverageBlock,
         counter_kind: BcbCounter,
-    ) -> Result<CovTerm, Error> {
+    ) -> CovTerm {
         // If the BCB has an edge counter (to be injected into a new `BasicBlock`), it can also
         // have an expression (to be injected into an existing `BasicBlock` represented by this
         // `BasicCoverageBlock`).
         if let Some(node_counter) = self.bcb_counter(to_bcb) && !node_counter.is_expression() {
-            return Error::from_string(format!(
+            bug!(
                 "attempt to add an incoming edge counter from {from_bcb:?} \
                 when the target BCB already has {node_counter:?}"
-            ));
+            );
         }
 
         self.bcb_has_incoming_edge_counters.insert(to_bcb);
         let term = counter_kind.as_term();
         if let Some(replaced) = self.bcb_edge_counters.insert((from_bcb, to_bcb), counter_kind) {
-            Error::from_string(format!(
+            bug!(
                 "attempt to set an edge counter more than once; from_bcb: \
                 {from_bcb:?} already had counter {replaced:?}",
-            ))
+            );
         } else {
-            Ok(term)
+            term
         }
     }
 
@@ -213,14 +207,7 @@ impl<'a> MakeBcbCounters<'a> {
     /// One way to predict which branch executes the least is by considering loops. A loop is exited
     /// at a branch, so the branch that jumps to a `BasicCoverageBlock` outside the loop is almost
     /// always executed less than the branch that does not exit the loop.
-    ///
-    /// Returns any non-code-span expressions created to represent intermediate values (such as to
-    /// add two counters so the result can be subtracted from another counter), or an Error with
-    /// message for subsequent debugging.
-    fn make_bcb_counters(
-        &mut self,
-        bcb_has_coverage_spans: impl Fn(BasicCoverageBlock) -> bool,
-    ) -> Result<(), Error> {
+    fn make_bcb_counters(&mut self, bcb_has_coverage_spans: impl Fn(BasicCoverageBlock) -> bool) {
         debug!("make_bcb_counters(): adding a counter or expression to each BasicCoverageBlock");
 
         // Walk the `CoverageGraph`. For each `BasicCoverageBlock` node with an associated
@@ -237,10 +224,10 @@ impl<'a> MakeBcbCounters<'a> {
         while let Some(bcb) = traversal.next() {
             if bcb_has_coverage_spans(bcb) {
                 debug!("{:?} has at least one coverage span. Get or make its counter", bcb);
-                let branching_counter_operand = self.get_or_make_counter_operand(bcb)?;
+                let branching_counter_operand = self.get_or_make_counter_operand(bcb);
 
                 if self.bcb_needs_branch_counters(bcb) {
-                    self.make_branch_counters(&traversal, bcb, branching_counter_operand)?;
+                    self.make_branch_counters(&traversal, bcb, branching_counter_operand);
                 }
             } else {
                 debug!(
@@ -251,14 +238,11 @@ impl<'a> MakeBcbCounters<'a> {
             }
         }
 
-        if traversal.is_complete() {
-            Ok(())
-        } else {
-            Error::from_string(format!(
-                "`TraverseCoverageGraphWithLoops` missed some `BasicCoverageBlock`s: {:?}",
-                traversal.unvisited(),
-            ))
-        }
+        assert!(
+            traversal.is_complete(),
+            "`TraverseCoverageGraphWithLoops` missed some `BasicCoverageBlock`s: {:?}",
+            traversal.unvisited(),
+        );
     }
 
     fn make_branch_counters(
@@ -266,7 +250,7 @@ impl<'a> MakeBcbCounters<'a> {
         traversal: &TraverseCoverageGraphWithLoops<'_>,
         branching_bcb: BasicCoverageBlock,
         branching_counter_operand: CovTerm,
-    ) -> Result<(), Error> {
+    ) {
         let branches = self.bcb_branches(branching_bcb);
         debug!(
             "{:?} has some branch(es) without counters:\n  {}",
@@ -299,10 +283,10 @@ impl<'a> MakeBcbCounters<'a> {
                         counter",
                         branch, branching_bcb
                     );
-                    self.get_or_make_counter_operand(branch.target_bcb)?
+                    self.get_or_make_counter_operand(branch.target_bcb)
                 } else {
                     debug!("  {:?} has multiple incoming edges, so adding an edge counter", branch);
-                    self.get_or_make_edge_counter_operand(branching_bcb, branch.target_bcb)?
+                    self.get_or_make_edge_counter_operand(branching_bcb, branch.target_bcb)
                 };
                 if let Some(sumup_counter_operand) =
                     some_sumup_counter_operand.replace(branch_counter_operand)
@@ -337,19 +321,18 @@ impl<'a> MakeBcbCounters<'a> {
         debug!("{:?} gets an expression: {:?}", expression_branch, expression);
         let bcb = expression_branch.target_bcb;
         if expression_branch.is_only_path_to_target() {
-            self.coverage_counters.set_bcb_counter(bcb, expression)?;
+            self.coverage_counters.set_bcb_counter(bcb, expression);
         } else {
-            self.coverage_counters.set_bcb_edge_counter(branching_bcb, bcb, expression)?;
+            self.coverage_counters.set_bcb_edge_counter(branching_bcb, bcb, expression);
         }
-        Ok(())
     }
 
     #[instrument(level = "debug", skip(self))]
-    fn get_or_make_counter_operand(&mut self, bcb: BasicCoverageBlock) -> Result<CovTerm, Error> {
+    fn get_or_make_counter_operand(&mut self, bcb: BasicCoverageBlock) -> CovTerm {
         // If the BCB already has a counter, return it.
         if let Some(counter_kind) = &self.coverage_counters.bcb_counters[bcb] {
             debug!("{bcb:?} already has a counter: {counter_kind:?}");
-            return Ok(counter_kind.as_term());
+            return counter_kind.as_term();
         }
 
         // A BCB with only one incoming edge gets a simple `Counter` (via `make_counter()`).
@@ -378,10 +361,10 @@ impl<'a> MakeBcbCounters<'a> {
 
         let mut predecessors = self.bcb_predecessors(bcb).to_owned().into_iter();
         let first_edge_counter_operand =
-            self.get_or_make_edge_counter_operand(predecessors.next().unwrap(), bcb)?;
+            self.get_or_make_edge_counter_operand(predecessors.next().unwrap(), bcb);
         let mut some_sumup_edge_counter_operand = None;
         for predecessor in predecessors {
-            let edge_counter_operand = self.get_or_make_edge_counter_operand(predecessor, bcb)?;
+            let edge_counter_operand = self.get_or_make_edge_counter_operand(predecessor, bcb);
             if let Some(sumup_edge_counter_operand) =
                 some_sumup_edge_counter_operand.replace(edge_counter_operand)
             {
@@ -411,7 +394,7 @@ impl<'a> MakeBcbCounters<'a> {
         &mut self,
         from_bcb: BasicCoverageBlock,
         to_bcb: BasicCoverageBlock,
-    ) -> Result<CovTerm, Error> {
+    ) -> CovTerm {
         // If the source BCB has only one successor (assumed to be the given target), an edge
         // counter is unnecessary. Just get or make a counter for the source BCB.
         let successors = self.bcb_successors(from_bcb).iter();
@@ -424,7 +407,7 @@ impl<'a> MakeBcbCounters<'a> {
             self.coverage_counters.bcb_edge_counters.get(&(from_bcb, to_bcb))
         {
             debug!("Edge {from_bcb:?}->{to_bcb:?} already has a counter: {counter_kind:?}");
-            return Ok(counter_kind.as_term());
+            return counter_kind.as_term();
         }
 
         // Make a new counter to count this edge.
diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs
index c9b36ba25ac3..97e4468a0e8b 100644
--- a/compiler/rustc_mir_transform/src/coverage/mod.rs
+++ b/compiler/rustc_mir_transform/src/coverage/mod.rs
@@ -26,18 +26,6 @@ use rustc_span::def_id::DefId;
 use rustc_span::source_map::SourceMap;
 use rustc_span::{ExpnKind, SourceFile, Span, Symbol};
 
-/// A simple error message wrapper for `coverage::Error`s.
-#[derive(Debug)]
-struct Error {
-    message: String,
-}
-
-impl Error {
-    pub fn from_string<T>(message: String) -> Result<T, Error> {
-        Err(Self { message })
-    }
-}
-
 /// Inserts `StatementKind::Coverage` statements that either instrument the binary with injected
 /// counters, via intrinsic `llvm.instrprof.increment`, and/or inject metadata used during codegen
 /// to construct the coverage map.
@@ -167,10 +155,7 @@ impl<'a, 'tcx> Instrumentor<'a, 'tcx> {
         // `BasicCoverageBlock`s not already associated with a coverage span.
         let bcb_has_coverage_spans = |bcb| coverage_spans.bcb_has_coverage_spans(bcb);
         self.coverage_counters
-            .make_bcb_counters(&mut self.basic_coverage_blocks, bcb_has_coverage_spans)
-            .unwrap_or_else(|e| {
-                bug!("Error processing: {:?}: {:?}", self.mir_body.source.def_id(), e.message)
-            });
+            .make_bcb_counters(&self.basic_coverage_blocks, bcb_has_coverage_spans);
 
         let mappings = self.create_mappings_and_inject_coverage_statements(&coverage_spans);
 
diff --git a/compiler/rustc_mir_transform/src/coverage/tests.rs b/compiler/rustc_mir_transform/src/coverage/tests.rs
index 795cbce963d1..702fe5f563e5 100644
--- a/compiler/rustc_mir_transform/src/coverage/tests.rs
+++ b/compiler/rustc_mir_transform/src/coverage/tests.rs
@@ -647,15 +647,13 @@ fn test_traverse_coverage_with_loops() {
 fn test_make_bcb_counters() {
     rustc_span::create_default_session_globals_then(|| {
         let mir_body = goto_switchint();
-        let mut basic_coverage_blocks = graph::CoverageGraph::from_mir(&mir_body);
+        let basic_coverage_blocks = graph::CoverageGraph::from_mir(&mir_body);
         // Historically this test would use `spans` internals to set up fake
         // coverage spans for BCBs 1 and 2. Now we skip that step and just tell
         // BCB counter construction that those BCBs have spans.
         let bcb_has_coverage_spans = |bcb: BasicCoverageBlock| (1..=2).contains(&bcb.as_usize());
         let mut coverage_counters = counters::CoverageCounters::new(&basic_coverage_blocks);
-        coverage_counters
-            .make_bcb_counters(&mut basic_coverage_blocks, bcb_has_coverage_spans)
-            .expect("should be Ok");
+        coverage_counters.make_bcb_counters(&basic_coverage_blocks, bcb_has_coverage_spans);
         assert_eq!(coverage_counters.num_expressions(), 0);
 
         let_bcb!(1);

From 7c673db195cd01873be2115318ce2adcc8cfe0de Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Fri, 29 Sep 2023 14:39:37 +0000
Subject: [PATCH 274/297] don't use the moral equivalent of `assert!(false,
 "foo")`

---
 compiler/rustc_codegen_llvm/src/intrinsic.rs | 55 ++++++++------------
 1 file changed, 22 insertions(+), 33 deletions(-)

diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index a97b803fc645..81f70434ace1 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -1470,15 +1470,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         match element_ty2.kind() {
             ty::Int(_) => (),
             _ => {
-                require!(
-                    false,
-                    InvalidMonomorphization::ThirdArgElementType {
-                        span,
-                        name,
-                        expected_element: element_ty2,
-                        third_arg: arg_tys[2]
-                    }
-                );
+                return_error!(InvalidMonomorphization::ThirdArgElementType {
+                    span,
+                    name,
+                    expected_element: element_ty2,
+                    third_arg: arg_tys[2]
+                });
             }
         }
 
@@ -1590,15 +1587,12 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         match element_ty2.kind() {
             ty::Int(_) => (),
             _ => {
-                require!(
-                    false,
-                    InvalidMonomorphization::ThirdArgElementType {
-                        span,
-                        name,
-                        expected_element: element_ty2,
-                        third_arg: arg_tys[2]
-                    }
-                );
+                return_error!(InvalidMonomorphization::ThirdArgElementType {
+                    span,
+                    name,
+                    expected_element: element_ty2,
+                    third_arg: arg_tys[2]
+                });
             }
         }
 
@@ -1989,17 +1983,14 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             }
             _ => { /* Unsupported. Fallthrough. */ }
         }
-        require!(
-            false,
-            InvalidMonomorphization::UnsupportedCast {
-                span,
-                name,
-                in_ty,
-                in_elem,
-                ret_ty,
-                out_elem
-            }
-        );
+        return_error!(InvalidMonomorphization::UnsupportedCast {
+            span,
+            name,
+            in_ty,
+            in_elem,
+            ret_ty,
+            out_elem
+        });
     }
     macro_rules! arith_binary {
         ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => {
@@ -2010,8 +2001,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
                     })*
                     _ => {},
                 }
-                require!(
-                    false,
+                return_error!(
                     InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
                 );
             })*
@@ -2041,8 +2031,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
                     })*
                     _ => {},
                 }
-                require!(
-                    false,
+                return_error!(
                     InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
                 );
             })*

From 9a49ef38c7b4e8f7f62271eb92776da0b9b4cf18 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Fri, 29 Sep 2023 14:54:38 +0000
Subject: [PATCH 275/297] Simplify all `require_simd` invocations by moving all
 of the shared invocation arguments into the macro

---
 compiler/rustc_codegen_llvm/src/intrinsic.rs | 56 +++++++-------------
 1 file changed, 19 insertions(+), 37 deletions(-)

diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index 81f70434ace1..026e84befcde 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -935,8 +935,8 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     }
 
     macro_rules! require_simd {
-        ($ty: expr, $diag: expr) => {
-            require!($ty.is_simd(), $diag)
+        ($ty: expr, $variant:ident) => {
+            require!($ty.is_simd(), InvalidMonomorphization::$variant { span, name, ty: $ty })
         };
     }
 
@@ -946,10 +946,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     let arg_tys = sig.inputs();
 
     if name == sym::simd_select_bitmask {
-        require_simd!(
-            arg_tys[1],
-            InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] }
-        );
+        require_simd!(arg_tys[1], SimdArgument);
 
         let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
 
@@ -988,7 +985,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     }
 
     // every intrinsic below takes a SIMD vector as its first argument
-    require_simd!(arg_tys[0], InvalidMonomorphization::SimdInput { span, name, ty: arg_tys[0] });
+    require_simd!(arg_tys[0], SimdInput);
     let in_ty = arg_tys[0];
 
     let comparison = match name {
@@ -1003,7 +1000,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
 
     let (in_len, in_elem) = arg_tys[0].simd_size_and_type(bx.tcx());
     if let Some(cmp_op) = comparison {
-        require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
+        require_simd!(ret_ty, SimdReturn);
 
         let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx());
 
@@ -1041,7 +1038,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             .unwrap_branch();
         let n = idx.len() as u64;
 
-        require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
+        require_simd!(ret_ty, SimdReturn);
         let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx());
         require!(
             out_len == n,
@@ -1099,7 +1096,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             }),
         };
 
-        require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
+        require_simd!(ret_ty, SimdReturn);
         let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx());
         require!(
             out_len == n,
@@ -1179,10 +1176,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     if name == sym::simd_select {
         let m_elem_ty = in_elem;
         let m_len = in_len;
-        require_simd!(
-            arg_tys[1],
-            InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] }
-        );
+        require_simd!(arg_tys[1], SimdArgument);
         let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
         require!(
             m_len == v_len,
@@ -1401,16 +1395,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         // * M: any integer width is supported, will be truncated to i1
 
         // All types must be simd vector types
-        require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty });
-        require_simd!(
-            arg_tys[1],
-            InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] }
-        );
-        require_simd!(
-            arg_tys[2],
-            InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] }
-        );
-        require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
+        require_simd!(in_ty, SimdFirst);
+        require_simd!(arg_tys[1], SimdSecond);
+        require_simd!(arg_tys[2], SimdThird);
+        require_simd!(ret_ty, SimdReturn);
 
         // Of the same length:
         let (out_len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
@@ -1524,15 +1512,9 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         // * M: any integer width is supported, will be truncated to i1
 
         // All types must be simd vector types
-        require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty });
-        require_simd!(
-            arg_tys[1],
-            InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] }
-        );
-        require_simd!(
-            arg_tys[2],
-            InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] }
-        );
+        require_simd!(in_ty, SimdFirst);
+        require_simd!(arg_tys[1], SimdSecond);
+        require_simd!(arg_tys[2], SimdThird);
 
         // Of the same length:
         let (element_len1, _) = arg_tys[1].simd_size_and_type(bx.tcx());
@@ -1788,7 +1770,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     bitwise_red!(simd_reduce_any: vector_reduce_or, true);
 
     if name == sym::simd_cast_ptr {
-        require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
+        require_simd!(ret_ty, SimdReturn);
         let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
         require!(
             in_len == out_len,
@@ -1837,7 +1819,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     }
 
     if name == sym::simd_expose_addr {
-        require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
+        require_simd!(ret_ty, SimdReturn);
         let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
         require!(
             in_len == out_len,
@@ -1866,7 +1848,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     }
 
     if name == sym::simd_from_exposed_addr {
-        require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
+        require_simd!(ret_ty, SimdReturn);
         let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
         require!(
             in_len == out_len,
@@ -1895,7 +1877,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     }
 
     if name == sym::simd_cast || name == sym::simd_as {
-        require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty });
+        require_simd!(ret_ty, SimdReturn);
         let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
         require!(
             in_len == out_len,

From f8372df63148db076ff00c36e4f4523c216c7d86 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Fri, 29 Sep 2023 15:14:53 +0000
Subject: [PATCH 276/297] Merge simd size and type extraction into checking
 whether a type is simd, as these always go together.

---
 compiler/rustc_codegen_llvm/src/intrinsic.rs | 74 +++++++-------------
 1 file changed, 26 insertions(+), 48 deletions(-)

diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs
index 026e84befcde..cc7e78b9c62b 100644
--- a/compiler/rustc_codegen_llvm/src/intrinsic.rs
+++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs
@@ -935,9 +935,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     }
 
     macro_rules! require_simd {
-        ($ty: expr, $variant:ident) => {
-            require!($ty.is_simd(), InvalidMonomorphization::$variant { span, name, ty: $ty })
-        };
+        ($ty: expr, $variant:ident) => {{
+            require!($ty.is_simd(), InvalidMonomorphization::$variant { span, name, ty: $ty });
+            $ty.simd_size_and_type(bx.tcx())
+        }};
     }
 
     let tcx = bx.tcx();
@@ -946,9 +947,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     let arg_tys = sig.inputs();
 
     if name == sym::simd_select_bitmask {
-        require_simd!(arg_tys[1], SimdArgument);
-
-        let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
+        let (len, _) = require_simd!(arg_tys[1], SimdArgument);
 
         let expected_int_bits = (len.max(8) - 1).next_power_of_two();
         let expected_bytes = len / 8 + ((len % 8 > 0) as u64);
@@ -985,7 +984,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     }
 
     // every intrinsic below takes a SIMD vector as its first argument
-    require_simd!(arg_tys[0], SimdInput);
+    let (in_len, in_elem) = require_simd!(arg_tys[0], SimdInput);
     let in_ty = arg_tys[0];
 
     let comparison = match name {
@@ -998,11 +997,8 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         _ => None,
     };
 
-    let (in_len, in_elem) = arg_tys[0].simd_size_and_type(bx.tcx());
     if let Some(cmp_op) = comparison {
-        require_simd!(ret_ty, SimdReturn);
-
-        let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx());
+        let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
 
         require!(
             in_len == out_len,
@@ -1038,8 +1034,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             .unwrap_branch();
         let n = idx.len() as u64;
 
-        require_simd!(ret_ty, SimdReturn);
-        let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx());
+        let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
         require!(
             out_len == n,
             InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
@@ -1096,8 +1091,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             }),
         };
 
-        require_simd!(ret_ty, SimdReturn);
-        let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx());
+        let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn);
         require!(
             out_len == n,
             InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len }
@@ -1176,8 +1170,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     if name == sym::simd_select {
         let m_elem_ty = in_elem;
         let m_len = in_len;
-        require_simd!(arg_tys[1], SimdArgument);
-        let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
+        let (v_len, _) = require_simd!(arg_tys[1], SimdArgument);
         require!(
             m_len == v_len,
             InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len }
@@ -1395,14 +1388,16 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         // * M: any integer width is supported, will be truncated to i1
 
         // All types must be simd vector types
-        require_simd!(in_ty, SimdFirst);
-        require_simd!(arg_tys[1], SimdSecond);
-        require_simd!(arg_tys[2], SimdThird);
+
+        // The second argument must be a simd vector with an element type that's a pointer
+        // to the element type of the first argument
+        let (_, element_ty0) = require_simd!(in_ty, SimdFirst);
+        let (out_len, element_ty1) = require_simd!(arg_tys[1], SimdSecond);
+        // The element type of the third argument must be a signed integer type of any width:
+        let (out_len2, element_ty2) = require_simd!(arg_tys[2], SimdThird);
         require_simd!(ret_ty, SimdReturn);
 
         // Of the same length:
-        let (out_len, _) = arg_tys[1].simd_size_and_type(bx.tcx());
-        let (out_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx());
         require!(
             in_len == out_len,
             InvalidMonomorphization::SecondArgumentLength {
@@ -1432,11 +1427,6 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty }
         );
 
-        // The second argument must be a simd vector with an element type that's a pointer
-        // to the element type of the first argument
-        let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx());
-        let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx());
-
         require!(
             matches!(
                 element_ty1.kind(),
@@ -1453,8 +1443,6 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             }
         );
 
-        // The element type of the third argument must be a signed integer type of any width:
-        let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx());
         match element_ty2.kind() {
             ty::Int(_) => (),
             _ => {
@@ -1512,13 +1500,13 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
         // * M: any integer width is supported, will be truncated to i1
 
         // All types must be simd vector types
-        require_simd!(in_ty, SimdFirst);
-        require_simd!(arg_tys[1], SimdSecond);
-        require_simd!(arg_tys[2], SimdThird);
+        // The second argument must be a simd vector with an element type that's a pointer
+        // to the element type of the first argument
+        let (_, element_ty0) = require_simd!(in_ty, SimdFirst);
+        let (element_len1, element_ty1) = require_simd!(arg_tys[1], SimdSecond);
+        let (element_len2, element_ty2) = require_simd!(arg_tys[2], SimdThird);
 
         // Of the same length:
-        let (element_len1, _) = arg_tys[1].simd_size_and_type(bx.tcx());
-        let (element_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx());
         require!(
             in_len == element_len1,
             InvalidMonomorphization::SecondArgumentLength {
@@ -1542,12 +1530,6 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             }
         );
 
-        // The second argument must be a simd vector with an element type that's a pointer
-        // to the element type of the first argument
-        let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx());
-        let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx());
-        let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx());
-
         require!(
             matches!(
                 element_ty1.kind(),
@@ -1770,8 +1752,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     bitwise_red!(simd_reduce_any: vector_reduce_or, true);
 
     if name == sym::simd_cast_ptr {
-        require_simd!(ret_ty, SimdReturn);
-        let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
+        let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
         require!(
             in_len == out_len,
             InvalidMonomorphization::ReturnLengthInputType {
@@ -1819,8 +1800,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     }
 
     if name == sym::simd_expose_addr {
-        require_simd!(ret_ty, SimdReturn);
-        let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
+        let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
         require!(
             in_len == out_len,
             InvalidMonomorphization::ReturnLengthInputType {
@@ -1848,8 +1828,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     }
 
     if name == sym::simd_from_exposed_addr {
-        require_simd!(ret_ty, SimdReturn);
-        let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
+        let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
         require!(
             in_len == out_len,
             InvalidMonomorphization::ReturnLengthInputType {
@@ -1877,8 +1856,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     }
 
     if name == sym::simd_cast || name == sym::simd_as {
-        require_simd!(ret_ty, SimdReturn);
-        let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx());
+        let (out_len, out_elem) = require_simd!(ret_ty, SimdReturn);
         require!(
             in_len == out_len,
             InvalidMonomorphization::ReturnLengthInputType {

From ed27cb0f493f79234cde209fa2d1c44597b89be7 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 22 Jul 2023 15:34:54 +0000
Subject: [PATCH 277/297] Reorder passes.

---
 compiler/rustc_mir_transform/src/lib.rs       |  7 ++---
 .../src/unreachable_prop.rs                   |  7 +----
 ...t_switch.identity.SeparateConstSwitch.diff |  6 +----
 ...in.UnreachablePropagation.panic-abort.diff | 27 ++++++++++---------
 ...n.UnreachablePropagation.panic-unwind.diff | 27 ++++++++++---------
 tests/mir-opt/unreachable.rs                  | 18 ++++++++++++-
 tests/mir-opt/unreachable_diverging.rs        | 18 ++++++++++++-
 7 files changed, 68 insertions(+), 42 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs
index dc35381fe22b..bf5f0ca7cbd2 100644
--- a/compiler/rustc_mir_transform/src/lib.rs
+++ b/compiler/rustc_mir_transform/src/lib.rs
@@ -568,10 +568,11 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         &[
             &check_alignment::CheckAlignment,
             &lower_slice_len::LowerSliceLenCalls, // has to be done before inlining, otherwise actual call will be almost always inlined. Also simple, so can just do first
-            &unreachable_prop::UnreachablePropagation,
-            &uninhabited_enum_branching::UninhabitedEnumBranching,
-            &o1(simplify::SimplifyCfg::AfterUninhabitedEnumBranching),
             &inline::Inline,
+            // Substitutions during inlining may introduce switch on enums with uninhabited branches.
+            &uninhabited_enum_branching::UninhabitedEnumBranching,
+            &unreachable_prop::UnreachablePropagation,
+            &o1(simplify::SimplifyCfg::AfterUninhabitedEnumBranching),
             &remove_storage_markers::RemoveStorageMarkers,
             &remove_zsts::RemoveZsts,
             &normalize_array_len::NormalizeArrayLen, // has to run after `slice::len` lowering
diff --git a/compiler/rustc_mir_transform/src/unreachable_prop.rs b/compiler/rustc_mir_transform/src/unreachable_prop.rs
index ea7aafd866b1..c7e22dfe842d 100644
--- a/compiler/rustc_mir_transform/src/unreachable_prop.rs
+++ b/compiler/rustc_mir_transform/src/unreachable_prop.rs
@@ -2,7 +2,6 @@
 //! when all of their successors are unreachable. This is achieved through a
 //! post-order traversal of the blocks.
 
-use crate::simplify;
 use crate::MirPass;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_middle::mir::*;
@@ -52,8 +51,6 @@ impl MirPass<'_> for UnreachablePropagation {
             body.basic_blocks_mut()[bb].statements.clear();
         }
 
-        let replaced = !replacements.is_empty();
-
         for (bb, terminator_kind) in replacements {
             if !tcx.consider_optimizing(|| {
                 format!("UnreachablePropagation {:?} ", body.source.def_id())
@@ -64,9 +61,7 @@ impl MirPass<'_> for UnreachablePropagation {
             body.basic_blocks_mut()[bb].terminator_mut().kind = terminator_kind;
         }
 
-        if replaced {
-            simplify::remove_dead_blocks(body);
-        }
+        // Do not remove dead blocks, let `SimplifyCfg` do it.
     }
 }
 
diff --git a/tests/mir-opt/separate_const_switch.identity.SeparateConstSwitch.diff b/tests/mir-opt/separate_const_switch.identity.SeparateConstSwitch.diff
index 491db551a7d2..fe4b33001fc4 100644
--- a/tests/mir-opt/separate_const_switch.identity.SeparateConstSwitch.diff
+++ b/tests/mir-opt/separate_const_switch.identity.SeparateConstSwitch.diff
@@ -52,7 +52,7 @@
           StorageLive(_10);
           StorageLive(_11);
           _9 = discriminant(_1);
-          switchInt(move _9) -> [0: bb7, 1: bb5, otherwise: bb6];
+          switchInt(move _9) -> [0: bb6, 1: bb5, otherwise: bb3];
       }
   
       bb1: {
@@ -92,10 +92,6 @@
       }
   
       bb6: {
-          unreachable;
-      }
-  
-      bb7: {
           _10 = ((_1 as Ok).0: i32);
           _3 = ControlFlow::<Result<Infallible, i32>, i32>::Continue(move _10);
           goto -> bb1;
diff --git a/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-abort.diff b/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-abort.diff
index eb5a0c39b0b2..aec22e0328cb 100644
--- a/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-abort.diff
+++ b/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-abort.diff
@@ -24,8 +24,7 @@
   
       bb1: {
           _2 = discriminant(_1);
--         switchInt(move _2) -> [1: bb2, otherwise: bb6];
-+         switchInt(move _2) -> [1: bb2, otherwise: bb3];
+          switchInt(move _2) -> [1: bb2, otherwise: bb6];
       }
   
       bb2: {
@@ -36,29 +35,31 @@
 -         StorageLive(_6);
 -         _6 = const true;
 -         switchInt(move _6) -> [0: bb4, otherwise: bb3];
--     }
-- 
--     bb3: {
++         unreachable;
+      }
+  
+      bb3: {
 -         _4 = const 21_i32;
 -         _5 = const ();
 -         goto -> bb5;
--     }
-- 
--     bb4: {
++         unreachable;
+      }
+  
+      bb4: {
 -         _4 = const 42_i32;
 -         _5 = const ();
 -         goto -> bb5;
--     }
-- 
--     bb5: {
++         unreachable;
+      }
+  
+      bb5: {
 -         StorageDead(_6);
 -         StorageDead(_5);
 -         StorageLive(_7);
           unreachable;
       }
   
--     bb6: {
-+     bb3: {
+      bb6: {
           _0 = const ();
           StorageDead(_1);
           return;
diff --git a/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-unwind.diff b/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-unwind.diff
index 906dce9819fd..b2e8ab95d946 100644
--- a/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-unwind.diff
+++ b/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-unwind.diff
@@ -24,8 +24,7 @@
   
       bb1: {
           _2 = discriminant(_1);
--         switchInt(move _2) -> [1: bb2, otherwise: bb6];
-+         switchInt(move _2) -> [1: bb2, otherwise: bb3];
+          switchInt(move _2) -> [1: bb2, otherwise: bb6];
       }
   
       bb2: {
@@ -36,29 +35,31 @@
 -         StorageLive(_6);
 -         _6 = const true;
 -         switchInt(move _6) -> [0: bb4, otherwise: bb3];
--     }
-- 
--     bb3: {
++         unreachable;
+      }
+  
+      bb3: {
 -         _4 = const 21_i32;
 -         _5 = const ();
 -         goto -> bb5;
--     }
-- 
--     bb4: {
++         unreachable;
+      }
+  
+      bb4: {
 -         _4 = const 42_i32;
 -         _5 = const ();
 -         goto -> bb5;
--     }
-- 
--     bb5: {
++         unreachable;
+      }
+  
+      bb5: {
 -         StorageDead(_6);
 -         StorageDead(_5);
 -         StorageLive(_7);
           unreachable;
       }
   
--     bb6: {
-+     bb3: {
+      bb6: {
           _0 = const ();
           StorageDead(_1);
           return;
diff --git a/tests/mir-opt/unreachable.rs b/tests/mir-opt/unreachable.rs
index 5c0df09b752e..9e8b0600b19e 100644
--- a/tests/mir-opt/unreachable.rs
+++ b/tests/mir-opt/unreachable.rs
@@ -1,5 +1,6 @@
-// skip-filecheck
+// unit-test: UnreachablePropagation
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+
 enum Empty {}
 
 fn empty() -> Option<Empty> {
@@ -8,6 +9,21 @@ fn empty() -> Option<Empty> {
 
 // EMIT_MIR unreachable.main.UnreachablePropagation.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: bb0: {
+    // CHECK: {{_.*}} = empty()
+    // CHECK: bb1: {
+    // CHECK: switchInt({{.*}}) -> [1: bb2, otherwise: bb6];
+    // CHECK: bb2: {
+    // CHECK: unreachable;
+    // CHECK: bb3: {
+    // CHECK: unreachable;
+    // CHECK: bb4: {
+    // CHECK: unreachable;
+    // CHECK: bb5: {
+    // CHECK: unreachable;
+    // CHECK: bb6: {
+    // CHECK: return;
     if let Some(_x) = empty() {
         let mut _y;
 
diff --git a/tests/mir-opt/unreachable_diverging.rs b/tests/mir-opt/unreachable_diverging.rs
index 3713bcaea169..95ea6cb00f9e 100644
--- a/tests/mir-opt/unreachable_diverging.rs
+++ b/tests/mir-opt/unreachable_diverging.rs
@@ -1,5 +1,6 @@
-// skip-filecheck
+// unit-test: UnreachablePropagation
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+
 pub enum Empty {}
 
 fn empty() -> Option<Empty> {
@@ -12,6 +13,21 @@ fn loop_forever() {
 
 // EMIT_MIR unreachable_diverging.main.UnreachablePropagation.diff
 fn main() {
+    // CHECK-LABEL: fn main(
+    // CHECK: bb0: {
+    // CHECK: {{_.*}} = empty()
+    // CHECK: bb1: {
+    // CHECK: switchInt({{.*}}) -> [1: bb2, otherwise: bb6];
+    // CHECK: bb2: {
+    // CHECK: switchInt({{.*}}) -> [0: bb4, otherwise: bb3];
+    // CHECK: bb3: {
+    // CHECK: {{_.*}} = loop_forever()
+    // CHECK: bb4: {
+    // CHECK: unreachable;
+    // CHECK: bb5: {
+    // CHECK: unreachable;
+    // CHECK: bb6: {
+    // CHECK: return;
     let x = true;
     if let Some(bomb) = empty() {
         if x {

From c748ac1f11c2e711cf205bdf262d3b4c1b9de405 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 22 Jul 2023 15:34:42 +0000
Subject: [PATCH 278/297] Replace SwitchInt to unreachable by an assumption.

---
 .../src/unreachable_prop.rs                   | 195 +++++++++++-------
 ...d.unwrap_unchecked.Inline.panic-abort.diff |  12 +-
 ....unwrap_unchecked.Inline.panic-unwind.diff |  12 +-
 ...unchecked.PreCodegen.after.panic-abort.mir |  22 +-
 ...nchecked.PreCodegen.after.panic-unwind.mir |  22 +-
 ...witch_targets.ub_if_b.PreCodegen.after.mir |   8 +-
 ...after-uninhabited-enum-branching.after.mir |  25 ++-
 ...in.UnreachablePropagation.panic-abort.diff |   7 +-
 ...n.UnreachablePropagation.panic-unwind.diff |   7 +-
 tests/mir-opt/unreachable.rs                  |  12 +-
 ...in.UnreachablePropagation.panic-abort.diff |   6 +-
 ...n.UnreachablePropagation.panic-unwind.diff |   6 +-
 tests/mir-opt/unreachable_diverging.rs        |   4 +-
 13 files changed, 214 insertions(+), 124 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/unreachable_prop.rs b/compiler/rustc_mir_transform/src/unreachable_prop.rs
index c7e22dfe842d..d25c6d471cd2 100644
--- a/compiler/rustc_mir_transform/src/unreachable_prop.rs
+++ b/compiler/rustc_mir_transform/src/unreachable_prop.rs
@@ -3,9 +3,12 @@
 //! post-order traversal of the blocks.
 
 use crate::MirPass;
-use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_data_structures::fx::FxHashSet;
+use rustc_middle::mir::interpret::Scalar;
+use rustc_middle::mir::patch::MirPatch;
 use rustc_middle::mir::*;
-use rustc_middle::ty::TyCtxt;
+use rustc_middle::ty::{self, TyCtxt};
+use rustc_target::abi::Size;
 
 pub struct UnreachablePropagation;
 
@@ -20,102 +23,134 @@ impl MirPass<'_> for UnreachablePropagation {
     }
 
     fn run_pass<'tcx>(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+        let mut patch = MirPatch::new(body);
         let mut unreachable_blocks = FxHashSet::default();
-        let mut replacements = FxHashMap::default();
 
         for (bb, bb_data) in traversal::postorder(body) {
             let terminator = bb_data.terminator();
-            if terminator.kind == TerminatorKind::Unreachable {
-                unreachable_blocks.insert(bb);
-            } else {
-                let is_unreachable = |succ: BasicBlock| unreachable_blocks.contains(&succ);
-                let terminator_kind_opt = remove_successors(&terminator.kind, is_unreachable);
-
-                if let Some(terminator_kind) = terminator_kind_opt {
-                    if terminator_kind == TerminatorKind::Unreachable {
-                        unreachable_blocks.insert(bb);
-                    }
-                    replacements.insert(bb, terminator_kind);
+            let is_unreachable = match &terminator.kind {
+                TerminatorKind::Unreachable => true,
+                // This will unconditionally run into an unreachable and is therefore unreachable as well.
+                TerminatorKind::Goto { target } if unreachable_blocks.contains(target) => {
+                    patch.patch_terminator(bb, TerminatorKind::Unreachable);
+                    true
                 }
+                // Try to remove unreachable targets from the switch.
+                TerminatorKind::SwitchInt { .. } => {
+                    remove_successors_from_switch(tcx, bb, &unreachable_blocks, body, &mut patch)
+                }
+                _ => false,
+            };
+            if is_unreachable {
+                unreachable_blocks.insert(bb);
             }
         }
 
+        if !tcx
+            .consider_optimizing(|| format!("UnreachablePropagation {:?} ", body.source.def_id()))
+        {
+            return;
+        }
+
+        patch.apply(body);
+
         // We do want do keep some unreachable blocks, but make them empty.
         for bb in unreachable_blocks {
-            if !tcx.consider_optimizing(|| {
-                format!("UnreachablePropagation {:?} ", body.source.def_id())
-            }) {
-                break;
-            }
-
             body.basic_blocks_mut()[bb].statements.clear();
         }
-
-        for (bb, terminator_kind) in replacements {
-            if !tcx.consider_optimizing(|| {
-                format!("UnreachablePropagation {:?} ", body.source.def_id())
-            }) {
-                break;
-            }
-
-            body.basic_blocks_mut()[bb].terminator_mut().kind = terminator_kind;
-        }
-
-        // Do not remove dead blocks, let `SimplifyCfg` do it.
     }
 }
 
-fn remove_successors<'tcx, F>(
-    terminator_kind: &TerminatorKind<'tcx>,
-    is_unreachable: F,
-) -> Option<TerminatorKind<'tcx>>
-where
-    F: Fn(BasicBlock) -> bool,
-{
-    let terminator = match terminator_kind {
-        // This will unconditionally run into an unreachable and is therefore unreachable as well.
-        TerminatorKind::Goto { target } if is_unreachable(*target) => TerminatorKind::Unreachable,
-        TerminatorKind::SwitchInt { targets, discr } => {
-            let otherwise = targets.otherwise();
+/// Return whether the current terminator is fully unreachable.
+fn remove_successors_from_switch<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    bb: BasicBlock,
+    unreachable_blocks: &FxHashSet<BasicBlock>,
+    body: &Body<'tcx>,
+    patch: &mut MirPatch<'tcx>,
+) -> bool {
+    let terminator = body.basic_blocks[bb].terminator();
+    let TerminatorKind::SwitchInt { discr, targets } = &terminator.kind else { bug!() };
+    let source_info = terminator.source_info;
+    let location = body.terminator_loc(bb);
 
-            // If all targets are unreachable, we can be unreachable as well.
-            if targets.all_targets().iter().all(|bb| is_unreachable(*bb)) {
-                TerminatorKind::Unreachable
-            } else if is_unreachable(otherwise) {
-                // If there are multiple targets, don't delete unreachable branches (like an unreachable otherwise)
-                // unless otherwise is unreachable, in which case deleting a normal branch causes it to be merged with
-                // the otherwise, keeping its unreachable.
-                // This looses information about reachability causing worse codegen.
-                // For example (see tests/codegen/match-optimizes-away.rs)
-                //
-                // pub enum Two { A, B }
-                // pub fn identity(x: Two) -> Two {
-                //     match x {
-                //         Two::A => Two::A,
-                //         Two::B => Two::B,
-                //     }
-                // }
-                //
-                // This generates a `switchInt() -> [0: 0, 1: 1, otherwise: unreachable]`, which allows us or LLVM to
-                // turn it into just `x` later. Without the unreachable, such a transformation would be illegal.
-                // If the otherwise branch is unreachable, we can delete all other unreachable targets, as they will
-                // still point to the unreachable and therefore not lose reachability information.
-                let reachable_iter = targets.iter().filter(|(_, bb)| !is_unreachable(*bb));
+    let is_unreachable = |bb| unreachable_blocks.contains(&bb);
 
-                let new_targets = SwitchTargets::new(reachable_iter, otherwise);
+    // If there are multiple targets, we want to keep information about reachability for codegen.
+    // For example (see tests/codegen/match-optimizes-away.rs)
+    //
+    // pub enum Two { A, B }
+    // pub fn identity(x: Two) -> Two {
+    //     match x {
+    //         Two::A => Two::A,
+    //         Two::B => Two::B,
+    //     }
+    // }
+    //
+    // This generates a `switchInt() -> [0: 0, 1: 1, otherwise: unreachable]`, which allows us or LLVM to
+    // turn it into just `x` later. Without the unreachable, such a transformation would be illegal.
+    //
+    // In order to preserve this information, we record reachable and unreachable targets as
+    // `Assume` statements in MIR.
 
-                // No unreachable branches were removed.
-                if new_targets.all_targets().len() == targets.all_targets().len() {
-                    return None;
-                }
+    let discr_ty = discr.ty(body, tcx);
+    let discr_size = Size::from_bits(match discr_ty.kind() {
+        ty::Uint(uint) => uint.normalize(tcx.sess.target.pointer_width).bit_width().unwrap(),
+        ty::Int(int) => int.normalize(tcx.sess.target.pointer_width).bit_width().unwrap(),
+        ty::Char => 32,
+        ty::Bool => 1,
+        other => bug!("unhandled type: {:?}", other),
+    });
 
-                TerminatorKind::SwitchInt { discr: discr.clone(), targets: new_targets }
-            } else {
-                // If the otherwise branch is reachable, we don't want to delete any unreachable branches.
-                return None;
-            }
-        }
-        _ => return None,
+    let mut add_assumption = |binop, value| {
+        let local = patch.new_temp(tcx.types.bool, source_info.span);
+        let value = Operand::Constant(Box::new(ConstOperand {
+            span: source_info.span,
+            user_ty: None,
+            const_: Const::from_scalar(tcx, Scalar::from_uint(value, discr_size), discr_ty),
+        }));
+        let cmp = Rvalue::BinaryOp(binop, Box::new((discr.to_copy(), value)));
+        patch.add_assign(location, local.into(), cmp);
+
+        let assume = NonDivergingIntrinsic::Assume(Operand::Move(local.into()));
+        patch.add_statement(location, StatementKind::Intrinsic(Box::new(assume)));
     };
-    Some(terminator)
+
+    let reachable_iter = targets.iter().filter(|&(value, bb)| {
+        let is_unreachable = is_unreachable(bb);
+        if is_unreachable {
+            // We remove this target from the switch, so record the inequality using `Assume`.
+            add_assumption(BinOp::Ne, value);
+            false
+        } else {
+            true
+        }
+    });
+
+    let otherwise = targets.otherwise();
+    let new_targets = SwitchTargets::new(reachable_iter, otherwise);
+
+    let num_targets = new_targets.all_targets().len();
+    let otherwise_unreachable = is_unreachable(otherwise);
+    let fully_unreachable = num_targets == 1 && otherwise_unreachable;
+
+    let terminator = match (num_targets, otherwise_unreachable) {
+        // If all targets are unreachable, we can be unreachable as well.
+        (1, true) => TerminatorKind::Unreachable,
+        (1, false) => TerminatorKind::Goto { target: otherwise },
+        (2, true) => {
+            // All targets are unreachable except one. Record the equality, and make it a goto.
+            let (value, target) = new_targets.iter().next().unwrap();
+            add_assumption(BinOp::Eq, value);
+            TerminatorKind::Goto { target }
+        }
+        _ if num_targets == targets.all_targets().len() => {
+            // Nothing has changed.
+            return false;
+        }
+        _ => TerminatorKind::SwitchInt { discr: discr.clone(), targets: new_targets },
+    };
+
+    patch.patch_terminator(bb, terminator);
+    fully_unreachable
 }
diff --git a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-abort.diff b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-abort.diff
index 018b6c1ee958..14a8b22657f6 100644
--- a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-abort.diff
+++ b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-abort.diff
@@ -16,6 +16,7 @@
 +             scope 5 (inlined unreachable_unchecked) {
 +                 scope 6 {
 +                     scope 7 (inlined unreachable_unchecked::runtime) {
++                         let _5: !;
 +                     }
 +                 }
 +             }
@@ -31,16 +32,23 @@
 -         _0 = Option::<T>::unwrap_unchecked(move _2) -> [return: bb1, unwind unreachable];
 +         StorageLive(_3);
 +         StorageLive(_4);
++         StorageLive(_5);
 +         _4 = discriminant(_2);
-+         switchInt(move _4) -> [1: bb2, otherwise: bb1];
++         switchInt(move _4) -> [0: bb1, 1: bb3, otherwise: bb2];
       }
   
       bb1: {
-+         unreachable;
++         assume(const false);
++         _5 = core::panicking::panic_nounwind(const "unsafe precondition(s) violated: hint::unreachable_unchecked must never be reached") -> unwind unreachable;
 +     }
 + 
 +     bb2: {
++         unreachable;
++     }
++ 
++     bb3: {
 +         _0 = move ((_2 as Some).0: T);
++         StorageDead(_5);
 +         StorageDead(_4);
 +         StorageDead(_3);
           StorageDead(_2);
diff --git a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-unwind.diff b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-unwind.diff
index 47845758a3f6..a6901ca08926 100644
--- a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-unwind.diff
+++ b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-unwind.diff
@@ -16,6 +16,7 @@
 +             scope 5 (inlined unreachable_unchecked) {
 +                 scope 6 {
 +                     scope 7 (inlined unreachable_unchecked::runtime) {
++                         let _5: !;
 +                     }
 +                 }
 +             }
@@ -31,20 +32,27 @@
 -         _0 = Option::<T>::unwrap_unchecked(move _2) -> [return: bb1, unwind: bb2];
 +         StorageLive(_3);
 +         StorageLive(_4);
++         StorageLive(_5);
 +         _4 = discriminant(_2);
-+         switchInt(move _4) -> [1: bb2, otherwise: bb1];
++         switchInt(move _4) -> [0: bb1, 1: bb3, otherwise: bb2];
       }
   
       bb1: {
 -         StorageDead(_2);
 -         return;
-+         unreachable;
++         assume(const false);
++         _5 = core::panicking::panic_nounwind(const "unsafe precondition(s) violated: hint::unreachable_unchecked must never be reached") -> unwind unreachable;
       }
   
 -     bb2 (cleanup): {
 -         resume;
 +     bb2: {
++         unreachable;
++     }
++ 
++     bb3: {
 +         _0 = move ((_2 as Some).0: T);
++         StorageDead(_5);
 +         StorageDead(_4);
 +         StorageDead(_3);
 +         StorageDead(_2);
diff --git a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-abort.mir b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-abort.mir
index 392f085bd4d1..37a2d28e0c48 100644
--- a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-abort.mir
+++ b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-abort.mir
@@ -6,7 +6,7 @@ fn unwrap_unchecked(_1: Option<T>) -> T {
     scope 1 (inlined #[track_caller] Option::<T>::unwrap_unchecked) {
         debug self => _1;
         let mut _2: isize;
-        let mut _3: &std::option::Option<T>;
+        let mut _4: &std::option::Option<T>;
         scope 2 {
             debug val => _0;
         }
@@ -14,30 +14,36 @@ fn unwrap_unchecked(_1: Option<T>) -> T {
             scope 5 (inlined unreachable_unchecked) {
                 scope 6 {
                     scope 7 (inlined unreachable_unchecked::runtime) {
+                        let _3: !;
                     }
                 }
             }
         }
         scope 4 (inlined Option::<T>::is_some) {
-            debug self => _3;
+            debug self => _4;
         }
     }
 
     bb0: {
-        StorageLive(_3);
+        StorageLive(_4);
         StorageLive(_2);
         _2 = discriminant(_1);
-        switchInt(move _2) -> [1: bb1, otherwise: bb2];
+        switchInt(move _2) -> [0: bb1, 1: bb2, otherwise: bb3];
     }
 
     bb1: {
-        _0 = move ((_1 as Some).0: T);
-        StorageDead(_2);
-        StorageDead(_3);
-        return;
+        assume(const false);
+        _3 = core::panicking::panic_nounwind(const "unsafe precondition(s) violated: hint::unreachable_unchecked must never be reached") -> unwind unreachable;
     }
 
     bb2: {
+        _0 = move ((_1 as Some).0: T);
+        StorageDead(_2);
+        StorageDead(_4);
+        return;
+    }
+
+    bb3: {
         unreachable;
     }
 }
diff --git a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-unwind.mir
index 392f085bd4d1..37a2d28e0c48 100644
--- a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-unwind.mir
+++ b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-unwind.mir
@@ -6,7 +6,7 @@ fn unwrap_unchecked(_1: Option<T>) -> T {
     scope 1 (inlined #[track_caller] Option::<T>::unwrap_unchecked) {
         debug self => _1;
         let mut _2: isize;
-        let mut _3: &std::option::Option<T>;
+        let mut _4: &std::option::Option<T>;
         scope 2 {
             debug val => _0;
         }
@@ -14,30 +14,36 @@ fn unwrap_unchecked(_1: Option<T>) -> T {
             scope 5 (inlined unreachable_unchecked) {
                 scope 6 {
                     scope 7 (inlined unreachable_unchecked::runtime) {
+                        let _3: !;
                     }
                 }
             }
         }
         scope 4 (inlined Option::<T>::is_some) {
-            debug self => _3;
+            debug self => _4;
         }
     }
 
     bb0: {
-        StorageLive(_3);
+        StorageLive(_4);
         StorageLive(_2);
         _2 = discriminant(_1);
-        switchInt(move _2) -> [1: bb1, otherwise: bb2];
+        switchInt(move _2) -> [0: bb1, 1: bb2, otherwise: bb3];
     }
 
     bb1: {
-        _0 = move ((_1 as Some).0: T);
-        StorageDead(_2);
-        StorageDead(_3);
-        return;
+        assume(const false);
+        _3 = core::panicking::panic_nounwind(const "unsafe precondition(s) violated: hint::unreachable_unchecked must never be reached") -> unwind unreachable;
     }
 
     bb2: {
+        _0 = move ((_1 as Some).0: T);
+        StorageDead(_2);
+        StorageDead(_4);
+        return;
+    }
+
+    bb3: {
         unreachable;
     }
 }
diff --git a/tests/mir-opt/pre-codegen/duplicate_switch_targets.ub_if_b.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/duplicate_switch_targets.ub_if_b.PreCodegen.after.mir
index 65d71199aaeb..d5d5253a8b78 100644
--- a/tests/mir-opt/pre-codegen/duplicate_switch_targets.ub_if_b.PreCodegen.after.mir
+++ b/tests/mir-opt/pre-codegen/duplicate_switch_targets.ub_if_b.PreCodegen.after.mir
@@ -7,13 +7,14 @@ fn ub_if_b(_1: Thing) -> Thing {
     scope 1 (inlined unreachable_unchecked) {
         scope 2 {
             scope 3 (inlined unreachable_unchecked::runtime) {
+                let _3: !;
             }
         }
     }
 
     bb0: {
         _2 = discriminant(_1);
-        switchInt(move _2) -> [0: bb1, otherwise: bb2];
+        switchInt(move _2) -> [0: bb1, 1: bb2, otherwise: bb3];
     }
 
     bb1: {
@@ -22,6 +23,11 @@ fn ub_if_b(_1: Thing) -> Thing {
     }
 
     bb2: {
+        assume(const false);
+        _3 = core::panicking::panic_nounwind(const "unsafe precondition(s) violated: hint::unreachable_unchecked must never be reached") -> unwind unreachable;
+    }
+
+    bb3: {
         unreachable;
     }
 }
diff --git a/tests/mir-opt/uninhabited_enum_branching.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir b/tests/mir-opt/uninhabited_enum_branching.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir
index 474f43104bb4..1ee44e48c8ec 100644
--- a/tests/mir-opt/uninhabited_enum_branching.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir
+++ b/tests/mir-opt/uninhabited_enum_branching.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir
@@ -11,16 +11,15 @@ fn main() -> () {
     let mut _7: Test2;
     let mut _8: isize;
     let _9: &str;
+    let mut _10: bool;
 
     bb0: {
         StorageLive(_1);
         StorageLive(_2);
         _2 = Test1::C;
         _3 = discriminant(_2);
-        switchInt(move _3) -> [2: bb1, otherwise: bb2];
-    }
-
-    bb1: {
+        _10 = Eq(_3, const 2_isize);
+        assume(move _10);
         StorageLive(_5);
         _5 = const "C";
         _1 = &(*_5);
@@ -31,27 +30,27 @@ fn main() -> () {
         StorageLive(_7);
         _7 = Test2::D;
         _8 = discriminant(_7);
-        switchInt(move _8) -> [4: bb4, 5: bb3, otherwise: bb2];
+        switchInt(move _8) -> [4: bb3, 5: bb2, otherwise: bb1];
     }
 
-    bb2: {
+    bb1: {
         unreachable;
     }
 
-    bb3: {
+    bb2: {
         StorageLive(_9);
         _9 = const "E";
         _6 = &(*_9);
         StorageDead(_9);
-        goto -> bb5;
+        goto -> bb4;
+    }
+
+    bb3: {
+        _6 = const "D";
+        goto -> bb4;
     }
 
     bb4: {
-        _6 = const "D";
-        goto -> bb5;
-    }
-
-    bb5: {
         StorageDead(_7);
         StorageDead(_6);
         _0 = const ();
diff --git a/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-abort.diff b/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-abort.diff
index aec22e0328cb..6f4faefce145 100644
--- a/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-abort.diff
+++ b/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-abort.diff
@@ -8,6 +8,8 @@
       let _5: ();
       let mut _6: bool;
       let mut _7: !;
++     let mut _8: bool;
++     let mut _9: bool;
       scope 1 {
           debug _x => _3;
           let _3: Empty;
@@ -24,7 +26,10 @@
   
       bb1: {
           _2 = discriminant(_1);
-          switchInt(move _2) -> [1: bb2, otherwise: bb6];
+-         switchInt(move _2) -> [1: bb2, otherwise: bb6];
++         _9 = Ne(_2, const 1_isize);
++         assume(move _9);
++         goto -> bb6;
       }
   
       bb2: {
diff --git a/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-unwind.diff b/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-unwind.diff
index b2e8ab95d946..5bacb42bed3f 100644
--- a/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-unwind.diff
+++ b/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-unwind.diff
@@ -8,6 +8,8 @@
       let _5: ();
       let mut _6: bool;
       let mut _7: !;
++     let mut _8: bool;
++     let mut _9: bool;
       scope 1 {
           debug _x => _3;
           let _3: Empty;
@@ -24,7 +26,10 @@
   
       bb1: {
           _2 = discriminant(_1);
-          switchInt(move _2) -> [1: bb2, otherwise: bb6];
+-         switchInt(move _2) -> [1: bb2, otherwise: bb6];
++         _9 = Ne(_2, const 1_isize);
++         assume(move _9);
++         goto -> bb6;
       }
   
       bb2: {
diff --git a/tests/mir-opt/unreachable.rs b/tests/mir-opt/unreachable.rs
index 9e8b0600b19e..3d934ace261e 100644
--- a/tests/mir-opt/unreachable.rs
+++ b/tests/mir-opt/unreachable.rs
@@ -13,15 +13,17 @@ fn main() {
     // CHECK: bb0: {
     // CHECK: {{_.*}} = empty()
     // CHECK: bb1: {
-    // CHECK: switchInt({{.*}}) -> [1: bb2, otherwise: bb6];
+    // CHECK: [[ne:_.*]] = Ne({{.*}}, const 1_isize);
+    // CHECK-NEXT: assume(move [[ne]]);
+    // CHECK-NEXT: goto -> bb6;
     // CHECK: bb2: {
-    // CHECK: unreachable;
+    // CHECK-NEXT: unreachable;
     // CHECK: bb3: {
-    // CHECK: unreachable;
+    // CHECK-NEXT: unreachable;
     // CHECK: bb4: {
-    // CHECK: unreachable;
+    // CHECK-NEXT: unreachable;
     // CHECK: bb5: {
-    // CHECK: unreachable;
+    // CHECK-NEXT: unreachable;
     // CHECK: bb6: {
     // CHECK: return;
     if let Some(_x) = empty() {
diff --git a/tests/mir-opt/unreachable_diverging.main.UnreachablePropagation.panic-abort.diff b/tests/mir-opt/unreachable_diverging.main.UnreachablePropagation.panic-abort.diff
index 713757ce6e08..11d7924e7360 100644
--- a/tests/mir-opt/unreachable_diverging.main.UnreachablePropagation.panic-abort.diff
+++ b/tests/mir-opt/unreachable_diverging.main.UnreachablePropagation.panic-abort.diff
@@ -9,6 +9,7 @@
       let _5: ();
       let mut _6: bool;
       let mut _7: !;
++     let mut _8: bool;
       scope 1 {
           debug x => _1;
           scope 2 {
@@ -35,7 +36,10 @@
           StorageLive(_5);
           StorageLive(_6);
           _6 = _1;
-          switchInt(move _6) -> [0: bb4, otherwise: bb3];
+-         switchInt(move _6) -> [0: bb4, otherwise: bb3];
++         _8 = Ne(_6, const false);
++         assume(move _8);
++         goto -> bb3;
       }
   
       bb3: {
diff --git a/tests/mir-opt/unreachable_diverging.main.UnreachablePropagation.panic-unwind.diff b/tests/mir-opt/unreachable_diverging.main.UnreachablePropagation.panic-unwind.diff
index a0479fb91304..df6f5609fbfa 100644
--- a/tests/mir-opt/unreachable_diverging.main.UnreachablePropagation.panic-unwind.diff
+++ b/tests/mir-opt/unreachable_diverging.main.UnreachablePropagation.panic-unwind.diff
@@ -9,6 +9,7 @@
       let _5: ();
       let mut _6: bool;
       let mut _7: !;
++     let mut _8: bool;
       scope 1 {
           debug x => _1;
           scope 2 {
@@ -35,7 +36,10 @@
           StorageLive(_5);
           StorageLive(_6);
           _6 = _1;
-          switchInt(move _6) -> [0: bb4, otherwise: bb3];
+-         switchInt(move _6) -> [0: bb4, otherwise: bb3];
++         _8 = Ne(_6, const false);
++         assume(move _8);
++         goto -> bb3;
       }
   
       bb3: {
diff --git a/tests/mir-opt/unreachable_diverging.rs b/tests/mir-opt/unreachable_diverging.rs
index 95ea6cb00f9e..b1df6f85262d 100644
--- a/tests/mir-opt/unreachable_diverging.rs
+++ b/tests/mir-opt/unreachable_diverging.rs
@@ -19,7 +19,9 @@ fn main() {
     // CHECK: bb1: {
     // CHECK: switchInt({{.*}}) -> [1: bb2, otherwise: bb6];
     // CHECK: bb2: {
-    // CHECK: switchInt({{.*}}) -> [0: bb4, otherwise: bb3];
+    // CHECK: [[ne:_.*]] = Ne({{.*}}, const false);
+    // CHECK: assume(move [[ne]]);
+    // CHECK: goto -> bb3;
     // CHECK: bb3: {
     // CHECK: {{_.*}} = loop_forever()
     // CHECK: bb4: {

From 0b13e636f54e1762016ffc8ffbfd43285a99b8ab Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 22 Jul 2023 15:34:22 +0000
Subject: [PATCH 279/297] Simplify assume of a constant.

---
 .../src/simplify_branches.rs                  | 19 +++++++++-
 ...d.unwrap_unchecked.Inline.panic-abort.diff | 25 ++++++-------
 ....unwrap_unchecked.Inline.panic-unwind.diff | 37 +++++++++----------
 ...unchecked.PreCodegen.after.panic-abort.mir | 32 +++++++---------
 ...nchecked.PreCodegen.after.panic-unwind.mir | 32 +++++++---------
 ...witch_targets.ub_if_b.PreCodegen.after.mir | 20 +++-------
 6 files changed, 80 insertions(+), 85 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/simplify_branches.rs b/compiler/rustc_mir_transform/src/simplify_branches.rs
index b508cd1c9cc5..1f0e605c3b85 100644
--- a/compiler/rustc_mir_transform/src/simplify_branches.rs
+++ b/compiler/rustc_mir_transform/src/simplify_branches.rs
@@ -16,8 +16,25 @@ impl<'tcx> MirPass<'tcx> for SimplifyConstCondition {
     }
 
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+        trace!("Running SimplifyConstCondition on {:?}", body.source);
         let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id());
-        for block in body.basic_blocks_mut() {
+        'blocks: for block in body.basic_blocks_mut() {
+            for stmt in block.statements.iter_mut() {
+                if let StatementKind::Intrinsic(box ref intrinsic) = stmt.kind
+                    && let NonDivergingIntrinsic::Assume(discr) = intrinsic
+                    && let Operand::Constant(ref c) = discr
+                    && let Some(constant) = c.const_.try_eval_bool(tcx, param_env)
+                {
+                    if constant {
+                        stmt.make_nop();
+                    } else {
+                        block.statements.clear();
+                        block.terminator_mut().kind = TerminatorKind::Unreachable;
+                        continue 'blocks;
+                    }
+                }
+            }
+
             let terminator = block.terminator_mut();
             terminator.kind = match terminator.kind {
                 TerminatorKind::SwitchInt {
diff --git a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-abort.diff b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-abort.diff
index 14a8b22657f6..4ca7a2c4294f 100644
--- a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-abort.diff
+++ b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-abort.diff
@@ -9,6 +9,8 @@
 +         debug self => _2;
 +         let mut _3: &std::option::Option<T>;
 +         let mut _4: isize;
++         let mut _5: bool;
++         let mut _6: bool;
 +         scope 2 {
 +             debug val => _0;
 +         }
@@ -16,7 +18,6 @@
 +             scope 5 (inlined unreachable_unchecked) {
 +                 scope 6 {
 +                     scope 7 (inlined unreachable_unchecked::runtime) {
-+                         let _5: !;
 +                     }
 +                 }
 +             }
@@ -30,24 +31,20 @@
           StorageLive(_2);
           _2 = move _1;
 -         _0 = Option::<T>::unwrap_unchecked(move _2) -> [return: bb1, unwind unreachable];
+-     }
+- 
+-     bb1: {
 +         StorageLive(_3);
 +         StorageLive(_4);
 +         StorageLive(_5);
++         StorageLive(_6);
 +         _4 = discriminant(_2);
-+         switchInt(move _4) -> [0: bb1, 1: bb3, otherwise: bb2];
-      }
-  
-      bb1: {
-+         assume(const false);
-+         _5 = core::panicking::panic_nounwind(const "unsafe precondition(s) violated: hint::unreachable_unchecked must never be reached") -> unwind unreachable;
-+     }
-+ 
-+     bb2: {
-+         unreachable;
-+     }
-+ 
-+     bb3: {
++         _5 = Ne(_4, const 0_isize);
++         assume(move _5);
++         _6 = Eq(_4, const 1_isize);
++         assume(move _6);
 +         _0 = move ((_2 as Some).0: T);
++         StorageDead(_6);
 +         StorageDead(_5);
 +         StorageDead(_4);
 +         StorageDead(_3);
diff --git a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-unwind.diff b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-unwind.diff
index a6901ca08926..caabb7ea4635 100644
--- a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-unwind.diff
+++ b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-unwind.diff
@@ -9,6 +9,8 @@
 +         debug self => _2;
 +         let mut _3: &std::option::Option<T>;
 +         let mut _4: isize;
++         let mut _5: bool;
++         let mut _6: bool;
 +         scope 2 {
 +             debug val => _0;
 +         }
@@ -16,7 +18,6 @@
 +             scope 5 (inlined unreachable_unchecked) {
 +                 scope 6 {
 +                     scope 7 (inlined unreachable_unchecked::runtime) {
-+                         let _5: !;
 +                     }
 +                 }
 +             }
@@ -30,33 +31,29 @@
           StorageLive(_2);
           _2 = move _1;
 -         _0 = Option::<T>::unwrap_unchecked(move _2) -> [return: bb1, unwind: bb2];
+-     }
+- 
+-     bb1: {
 +         StorageLive(_3);
 +         StorageLive(_4);
 +         StorageLive(_5);
++         StorageLive(_6);
 +         _4 = discriminant(_2);
-+         switchInt(move _4) -> [0: bb1, 1: bb3, otherwise: bb2];
-      }
-  
-      bb1: {
--         StorageDead(_2);
--         return;
-+         assume(const false);
-+         _5 = core::panicking::panic_nounwind(const "unsafe precondition(s) violated: hint::unreachable_unchecked must never be reached") -> unwind unreachable;
-      }
-  
--     bb2 (cleanup): {
--         resume;
-+     bb2: {
-+         unreachable;
-+     }
-+ 
-+     bb3: {
++         _5 = Ne(_4, const 0_isize);
++         assume(move _5);
++         _6 = Eq(_4, const 1_isize);
++         assume(move _6);
 +         _0 = move ((_2 as Some).0: T);
++         StorageDead(_6);
 +         StorageDead(_5);
 +         StorageDead(_4);
 +         StorageDead(_3);
-+         StorageDead(_2);
-+         return;
+          StorageDead(_2);
+          return;
+-     }
+- 
+-     bb2 (cleanup): {
+-         resume;
       }
   }
   
diff --git a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-abort.mir b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-abort.mir
index 37a2d28e0c48..521266925e0d 100644
--- a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-abort.mir
+++ b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-abort.mir
@@ -6,7 +6,9 @@ fn unwrap_unchecked(_1: Option<T>) -> T {
     scope 1 (inlined #[track_caller] Option::<T>::unwrap_unchecked) {
         debug self => _1;
         let mut _2: isize;
-        let mut _4: &std::option::Option<T>;
+        let mut _3: bool;
+        let mut _4: bool;
+        let mut _5: &std::option::Option<T>;
         scope 2 {
             debug val => _0;
         }
@@ -14,36 +16,30 @@ fn unwrap_unchecked(_1: Option<T>) -> T {
             scope 5 (inlined unreachable_unchecked) {
                 scope 6 {
                     scope 7 (inlined unreachable_unchecked::runtime) {
-                        let _3: !;
                     }
                 }
             }
         }
         scope 4 (inlined Option::<T>::is_some) {
-            debug self => _4;
+            debug self => _5;
         }
     }
 
     bb0: {
-        StorageLive(_4);
+        StorageLive(_5);
         StorageLive(_2);
+        StorageLive(_3);
+        StorageLive(_4);
         _2 = discriminant(_1);
-        switchInt(move _2) -> [0: bb1, 1: bb2, otherwise: bb3];
-    }
-
-    bb1: {
-        assume(const false);
-        _3 = core::panicking::panic_nounwind(const "unsafe precondition(s) violated: hint::unreachable_unchecked must never be reached") -> unwind unreachable;
-    }
-
-    bb2: {
+        _3 = Ne(_2, const 0_isize);
+        assume(move _3);
+        _4 = Eq(_2, const 1_isize);
+        assume(move _4);
         _0 = move ((_1 as Some).0: T);
-        StorageDead(_2);
         StorageDead(_4);
+        StorageDead(_3);
+        StorageDead(_2);
+        StorageDead(_5);
         return;
     }
-
-    bb3: {
-        unreachable;
-    }
 }
diff --git a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-unwind.mir
index 37a2d28e0c48..521266925e0d 100644
--- a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-unwind.mir
+++ b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-unwind.mir
@@ -6,7 +6,9 @@ fn unwrap_unchecked(_1: Option<T>) -> T {
     scope 1 (inlined #[track_caller] Option::<T>::unwrap_unchecked) {
         debug self => _1;
         let mut _2: isize;
-        let mut _4: &std::option::Option<T>;
+        let mut _3: bool;
+        let mut _4: bool;
+        let mut _5: &std::option::Option<T>;
         scope 2 {
             debug val => _0;
         }
@@ -14,36 +16,30 @@ fn unwrap_unchecked(_1: Option<T>) -> T {
             scope 5 (inlined unreachable_unchecked) {
                 scope 6 {
                     scope 7 (inlined unreachable_unchecked::runtime) {
-                        let _3: !;
                     }
                 }
             }
         }
         scope 4 (inlined Option::<T>::is_some) {
-            debug self => _4;
+            debug self => _5;
         }
     }
 
     bb0: {
-        StorageLive(_4);
+        StorageLive(_5);
         StorageLive(_2);
+        StorageLive(_3);
+        StorageLive(_4);
         _2 = discriminant(_1);
-        switchInt(move _2) -> [0: bb1, 1: bb2, otherwise: bb3];
-    }
-
-    bb1: {
-        assume(const false);
-        _3 = core::panicking::panic_nounwind(const "unsafe precondition(s) violated: hint::unreachable_unchecked must never be reached") -> unwind unreachable;
-    }
-
-    bb2: {
+        _3 = Ne(_2, const 0_isize);
+        assume(move _3);
+        _4 = Eq(_2, const 1_isize);
+        assume(move _4);
         _0 = move ((_1 as Some).0: T);
-        StorageDead(_2);
         StorageDead(_4);
+        StorageDead(_3);
+        StorageDead(_2);
+        StorageDead(_5);
         return;
     }
-
-    bb3: {
-        unreachable;
-    }
 }
diff --git a/tests/mir-opt/pre-codegen/duplicate_switch_targets.ub_if_b.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/duplicate_switch_targets.ub_if_b.PreCodegen.after.mir
index d5d5253a8b78..fe7beadc8188 100644
--- a/tests/mir-opt/pre-codegen/duplicate_switch_targets.ub_if_b.PreCodegen.after.mir
+++ b/tests/mir-opt/pre-codegen/duplicate_switch_targets.ub_if_b.PreCodegen.after.mir
@@ -4,30 +4,22 @@ fn ub_if_b(_1: Thing) -> Thing {
     debug t => _1;
     let mut _0: Thing;
     let mut _2: isize;
+    let mut _3: bool;
+    let mut _4: bool;
     scope 1 (inlined unreachable_unchecked) {
         scope 2 {
             scope 3 (inlined unreachable_unchecked::runtime) {
-                let _3: !;
             }
         }
     }
 
     bb0: {
         _2 = discriminant(_1);
-        switchInt(move _2) -> [0: bb1, 1: bb2, otherwise: bb3];
-    }
-
-    bb1: {
+        _3 = Ne(_2, const 1_isize);
+        assume(move _3);
+        _4 = Eq(_2, const 0_isize);
+        assume(move _4);
         _0 = move _1;
         return;
     }
-
-    bb2: {
-        assume(const false);
-        _3 = core::panicking::panic_nounwind(const "unsafe precondition(s) violated: hint::unreachable_unchecked must never be reached") -> unwind unreachable;
-    }
-
-    bb3: {
-        unreachable;
-    }
 }

From 096196d5b0f8035d461b9611e32c18d9696fe7e1 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 22 Jul 2023 16:59:13 +0000
Subject: [PATCH 280/297] Refactor UninhabitedEnumBranching to mark targets
 unreachable.

---
 .../src/uninhabited_enum_branching.rs         | 104 ++++++++----------
 ...after-uninhabited-enum-branching.after.mir |   8 +-
 ...anching.main.UninhabitedEnumBranching.diff |   9 +-
 ...after-uninhabited-enum-branching.after.mir |  12 ++
 ...nching2.main.UninhabitedEnumBranching.diff |   8 +-
 ..._fallthrough.UninhabitedEnumBranching.diff |   6 +-
 6 files changed, 84 insertions(+), 63 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs b/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs
index cb028a92d49b..98f67e18a8d0 100644
--- a/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs
+++ b/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs
@@ -3,8 +3,7 @@
 use crate::MirPass;
 use rustc_data_structures::fx::FxHashSet;
 use rustc_middle::mir::{
-    BasicBlockData, Body, Local, Operand, Rvalue, StatementKind, SwitchTargets, Terminator,
-    TerminatorKind,
+    BasicBlockData, Body, Local, Operand, Rvalue, StatementKind, Terminator, TerminatorKind,
 };
 use rustc_middle::ty::layout::TyAndLayout;
 use rustc_middle::ty::{Ty, TyCtxt};
@@ -30,17 +29,20 @@ fn get_switched_on_type<'tcx>(
     let terminator = block_data.terminator();
 
     // Only bother checking blocks which terminate by switching on a local.
-    if let Some(local) = get_discriminant_local(&terminator.kind)
-        && let [.., stmt_before_term] = &block_data.statements[..]
-        && let StatementKind::Assign(box (l, Rvalue::Discriminant(place))) = stmt_before_term.kind
+    let local = get_discriminant_local(&terminator.kind)?;
+
+    let stmt_before_term = block_data.statements.last()?;
+
+    if let StatementKind::Assign(box (l, Rvalue::Discriminant(place))) = stmt_before_term.kind
         && l.as_local() == Some(local)
-        && let ty = place.ty(body, tcx).ty
-        && ty.is_enum()
     {
-        Some(ty)
-    } else {
-        None
+        let ty = place.ty(body, tcx).ty;
+        if ty.is_enum() {
+            return Some(ty);
+        }
     }
+
+    None
 }
 
 fn variant_discriminants<'tcx>(
@@ -67,28 +69,6 @@ fn variant_discriminants<'tcx>(
     }
 }
 
-/// Ensures that the `otherwise` branch leads to an unreachable bb, returning `None` if so and a new
-/// bb to use as the new target if not.
-fn ensure_otherwise_unreachable<'tcx>(
-    body: &Body<'tcx>,
-    targets: &SwitchTargets,
-) -> Option<BasicBlockData<'tcx>> {
-    let otherwise = targets.otherwise();
-    let bb = &body.basic_blocks[otherwise];
-    if bb.terminator().kind == TerminatorKind::Unreachable
-        && bb.statements.iter().all(|s| matches!(&s.kind, StatementKind::StorageDead(_)))
-    {
-        return None;
-    }
-
-    let mut new_block = BasicBlockData::new(Some(Terminator {
-        source_info: bb.terminator().source_info,
-        kind: TerminatorKind::Unreachable,
-    }));
-    new_block.is_cleanup = bb.is_cleanup;
-    Some(new_block)
-}
-
 impl<'tcx> MirPass<'tcx> for UninhabitedEnumBranching {
     fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
         sess.mir_opt_level() > 0
@@ -97,13 +77,16 @@ impl<'tcx> MirPass<'tcx> for UninhabitedEnumBranching {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         trace!("UninhabitedEnumBranching starting for {:?}", body.source);
 
-        for bb in body.basic_blocks.indices() {
+        let mut removable_switchs = Vec::new();
+
+        for (bb, bb_data) in body.basic_blocks.iter_enumerated() {
             trace!("processing block {:?}", bb);
 
-            let Some(discriminant_ty) = get_switched_on_type(&body.basic_blocks[bb], tcx, body)
-            else {
+            if bb_data.is_cleanup {
                 continue;
-            };
+            }
+
+            let Some(discriminant_ty) = get_switched_on_type(&bb_data, tcx, body) else { continue };
 
             let layout = tcx.layout_of(
                 tcx.param_env_reveal_all_normalized(body.source.def_id()).and(discriminant_ty),
@@ -117,31 +100,38 @@ impl<'tcx> MirPass<'tcx> for UninhabitedEnumBranching {
 
             trace!("allowed_variants = {:?}", allowed_variants);
 
-            if let TerminatorKind::SwitchInt { targets, .. } =
-                &mut body.basic_blocks_mut()[bb].terminator_mut().kind
-            {
-                let mut new_targets = SwitchTargets::new(
-                    targets.iter().filter(|(val, _)| allowed_variants.contains(val)),
-                    targets.otherwise(),
-                );
+            let terminator = bb_data.terminator();
+            let TerminatorKind::SwitchInt { targets, .. } = &terminator.kind else { bug!() };
 
-                if new_targets.iter().count() == allowed_variants.len() {
-                    if let Some(updated) = ensure_otherwise_unreachable(body, &new_targets) {
-                        let new_otherwise = body.basic_blocks_mut().push(updated);
-                        *new_targets.all_targets_mut().last_mut().unwrap() = new_otherwise;
-                    }
-                }
-
-                if let TerminatorKind::SwitchInt { targets, .. } =
-                    &mut body.basic_blocks_mut()[bb].terminator_mut().kind
-                {
-                    *targets = new_targets;
+            let mut reachable_count = 0;
+            for (index, (val, _)) in targets.iter().enumerate() {
+                if allowed_variants.contains(&val) {
+                    reachable_count += 1;
                 } else {
-                    unreachable!()
+                    removable_switchs.push((bb, index));
                 }
-            } else {
-                unreachable!()
             }
+
+            if reachable_count == allowed_variants.len() {
+                removable_switchs.push((bb, targets.iter().count()));
+            }
+        }
+
+        if removable_switchs.is_empty() {
+            return;
+        }
+
+        let new_block = BasicBlockData::new(Some(Terminator {
+            source_info: body.basic_blocks[removable_switchs[0].0].terminator().source_info,
+            kind: TerminatorKind::Unreachable,
+        }));
+        let unreachable_block = body.basic_blocks.as_mut().push(new_block);
+
+        for (bb, index) in removable_switchs {
+            let bb = &mut body.basic_blocks.as_mut()[bb];
+            let terminator = bb.terminator_mut();
+            let TerminatorKind::SwitchInt { targets, .. } = &mut terminator.kind else { bug!() };
+            targets.all_targets_mut()[index] = unreachable_block;
         }
     }
 }
diff --git a/tests/mir-opt/uninhabited_enum_branching.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir b/tests/mir-opt/uninhabited_enum_branching.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir
index 1ee44e48c8ec..1df3b74b3e7d 100644
--- a/tests/mir-opt/uninhabited_enum_branching.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir
+++ b/tests/mir-opt/uninhabited_enum_branching.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir
@@ -12,14 +12,20 @@ fn main() -> () {
     let mut _8: isize;
     let _9: &str;
     let mut _10: bool;
+    let mut _11: bool;
+    let mut _12: bool;
 
     bb0: {
         StorageLive(_1);
         StorageLive(_2);
         _2 = Test1::C;
         _3 = discriminant(_2);
-        _10 = Eq(_3, const 2_isize);
+        _10 = Ne(_3, const 0_isize);
         assume(move _10);
+        _11 = Ne(_3, const 1_isize);
+        assume(move _11);
+        _12 = Eq(_3, const 2_isize);
+        assume(move _12);
         StorageLive(_5);
         _5 = const "C";
         _1 = &(*_5);
diff --git a/tests/mir-opt/uninhabited_enum_branching.main.UninhabitedEnumBranching.diff b/tests/mir-opt/uninhabited_enum_branching.main.UninhabitedEnumBranching.diff
index 9db95abec34c..5b107f80ce80 100644
--- a/tests/mir-opt/uninhabited_enum_branching.main.UninhabitedEnumBranching.diff
+++ b/tests/mir-opt/uninhabited_enum_branching.main.UninhabitedEnumBranching.diff
@@ -19,7 +19,7 @@
           _2 = Test1::C;
           _3 = discriminant(_2);
 -         switchInt(move _3) -> [0: bb3, 1: bb4, 2: bb1, otherwise: bb2];
-+         switchInt(move _3) -> [2: bb1, otherwise: bb2];
++         switchInt(move _3) -> [0: bb9, 1: bb9, 2: bb1, otherwise: bb9];
       }
   
       bb1: {
@@ -54,7 +54,8 @@
           StorageLive(_7);
           _7 = Test2::D;
           _8 = discriminant(_7);
-          switchInt(move _8) -> [4: bb7, 5: bb6, otherwise: bb2];
+-         switchInt(move _8) -> [4: bb7, 5: bb6, otherwise: bb2];
++         switchInt(move _8) -> [4: bb7, 5: bb6, otherwise: bb9];
       }
   
       bb6: {
@@ -75,6 +76,10 @@
           StorageDead(_6);
           _0 = const ();
           return;
++     }
++ 
++     bb9: {
++         unreachable;
       }
   }
   
diff --git a/tests/mir-opt/uninhabited_enum_branching2.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir b/tests/mir-opt/uninhabited_enum_branching2.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir
index 9c0c5d18917a..06375b3ffaeb 100644
--- a/tests/mir-opt/uninhabited_enum_branching2.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir
+++ b/tests/mir-opt/uninhabited_enum_branching2.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir
@@ -15,6 +15,10 @@ fn main() -> () {
     let _11: &str;
     let _12: &str;
     let _13: &str;
+    let mut _14: bool;
+    let mut _15: bool;
+    let mut _16: bool;
+    let mut _17: bool;
     scope 1 {
         debug plop => _1;
     }
@@ -29,6 +33,10 @@ fn main() -> () {
         StorageLive(_4);
         _4 = &(_1.1: Test1);
         _5 = discriminant((*_4));
+        _16 = Ne(_5, const 0_isize);
+        assume(move _16);
+        _17 = Ne(_5, const 1_isize);
+        assume(move _17);
         switchInt(move _5) -> [2: bb3, 3: bb1, otherwise: bb2];
     }
 
@@ -57,6 +65,10 @@ fn main() -> () {
         StorageDead(_3);
         StorageLive(_9);
         _10 = discriminant((_1.1: Test1));
+        _14 = Ne(_10, const 0_isize);
+        assume(move _14);
+        _15 = Ne(_10, const 1_isize);
+        assume(move _15);
         switchInt(move _10) -> [2: bb6, 3: bb5, otherwise: bb2];
     }
 
diff --git a/tests/mir-opt/uninhabited_enum_branching2.main.UninhabitedEnumBranching.diff b/tests/mir-opt/uninhabited_enum_branching2.main.UninhabitedEnumBranching.diff
index 12ce6505af94..165421acd691 100644
--- a/tests/mir-opt/uninhabited_enum_branching2.main.UninhabitedEnumBranching.diff
+++ b/tests/mir-opt/uninhabited_enum_branching2.main.UninhabitedEnumBranching.diff
@@ -31,7 +31,7 @@
           _4 = &(_1.1: Test1);
           _5 = discriminant((*_4));
 -         switchInt(move _5) -> [0: bb3, 1: bb4, 2: bb5, 3: bb1, otherwise: bb2];
-+         switchInt(move _5) -> [2: bb5, 3: bb1, otherwise: bb2];
++         switchInt(move _5) -> [0: bb12, 1: bb12, 2: bb5, 3: bb1, otherwise: bb12];
       }
   
       bb1: {
@@ -73,7 +73,7 @@
           StorageLive(_9);
           _10 = discriminant((_1.1: Test1));
 -         switchInt(move _10) -> [0: bb8, 1: bb9, 2: bb10, 3: bb7, otherwise: bb2];
-+         switchInt(move _10) -> [2: bb10, 3: bb7, otherwise: bb2];
++         switchInt(move _10) -> [0: bb12, 1: bb12, 2: bb10, 3: bb7, otherwise: bb12];
       }
   
       bb7: {
@@ -110,6 +110,10 @@
           _0 = const ();
           StorageDead(_1);
           return;
++     }
++ 
++     bb12: {
++         unreachable;
       }
   }
   
diff --git a/tests/mir-opt/uninhabited_fallthrough_elimination.keep_fallthrough.UninhabitedEnumBranching.diff b/tests/mir-opt/uninhabited_fallthrough_elimination.keep_fallthrough.UninhabitedEnumBranching.diff
index 498e1e20f8a7..79948139f888 100644
--- a/tests/mir-opt/uninhabited_fallthrough_elimination.keep_fallthrough.UninhabitedEnumBranching.diff
+++ b/tests/mir-opt/uninhabited_fallthrough_elimination.keep_fallthrough.UninhabitedEnumBranching.diff
@@ -9,7 +9,7 @@
       bb0: {
           _2 = discriminant(_1);
 -         switchInt(move _2) -> [0: bb2, 1: bb3, otherwise: bb1];
-+         switchInt(move _2) -> [1: bb3, otherwise: bb1];
++         switchInt(move _2) -> [0: bb5, 1: bb3, otherwise: bb1];
       }
   
       bb1: {
@@ -29,6 +29,10 @@
   
       bb4: {
           return;
++     }
++ 
++     bb5: {
++         unreachable;
       }
   }
   

From cb918904fe693e93c679d4c3417cfe7776f8ebe2 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sun, 24 Sep 2023 08:27:42 +0000
Subject: [PATCH 281/297] Only emit `!=` assumptions if the otherwise target is
 reachable.

---
 .../src/unreachable_prop.rs                   | 13 +++---
 ...d.unwrap_unchecked.Inline.panic-abort.diff |  7 +--
 ....unwrap_unchecked.Inline.panic-unwind.diff |  7 +--
 ...unchecked.PreCodegen.after.panic-abort.mir | 15 +++----
 ...nchecked.PreCodegen.after.panic-unwind.mir | 15 +++----
 ...witch_targets.ub_if_b.PreCodegen.after.mir |  5 +--
 ...ch.UnreachablePropagation.panic-abort.diff | 45 +++++++++++++++++++
 ...h.UnreachablePropagation.panic-unwind.diff | 45 +++++++++++++++++++
 ...t.UnreachablePropagation.panic-abort.diff} | 11 +++--
 ....UnreachablePropagation.panic-unwind.diff} | 11 +++--
 tests/mir-opt/unreachable.rs                  | 32 +++++++++++--
 11 files changed, 148 insertions(+), 58 deletions(-)
 create mode 100644 tests/mir-opt/unreachable.as_match.UnreachablePropagation.panic-abort.diff
 create mode 100644 tests/mir-opt/unreachable.as_match.UnreachablePropagation.panic-unwind.diff
 rename tests/mir-opt/{unreachable.main.UnreachablePropagation.panic-abort.diff => unreachable.if_let.UnreachablePropagation.panic-abort.diff} (86%)
 rename tests/mir-opt/{unreachable.main.UnreachablePropagation.panic-unwind.diff => unreachable.if_let.UnreachablePropagation.panic-unwind.diff} (86%)

diff --git a/compiler/rustc_mir_transform/src/unreachable_prop.rs b/compiler/rustc_mir_transform/src/unreachable_prop.rs
index d25c6d471cd2..919e8d6a2340 100644
--- a/compiler/rustc_mir_transform/src/unreachable_prop.rs
+++ b/compiler/rustc_mir_transform/src/unreachable_prop.rs
@@ -116,22 +116,21 @@ fn remove_successors_from_switch<'tcx>(
         patch.add_statement(location, StatementKind::Intrinsic(Box::new(assume)));
     };
 
+    let otherwise = targets.otherwise();
+    let otherwise_unreachable = is_unreachable(otherwise);
+
     let reachable_iter = targets.iter().filter(|&(value, bb)| {
         let is_unreachable = is_unreachable(bb);
-        if is_unreachable {
-            // We remove this target from the switch, so record the inequality using `Assume`.
+        // We remove this target from the switch, so record the inequality using `Assume`.
+        if is_unreachable && !otherwise_unreachable {
             add_assumption(BinOp::Ne, value);
-            false
-        } else {
-            true
         }
+        !is_unreachable
     });
 
-    let otherwise = targets.otherwise();
     let new_targets = SwitchTargets::new(reachable_iter, otherwise);
 
     let num_targets = new_targets.all_targets().len();
-    let otherwise_unreachable = is_unreachable(otherwise);
     let fully_unreachable = num_targets == 1 && otherwise_unreachable;
 
     let terminator = match (num_targets, otherwise_unreachable) {
diff --git a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-abort.diff b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-abort.diff
index 4ca7a2c4294f..2a36ad9230e4 100644
--- a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-abort.diff
+++ b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-abort.diff
@@ -10,7 +10,6 @@
 +         let mut _3: &std::option::Option<T>;
 +         let mut _4: isize;
 +         let mut _5: bool;
-+         let mut _6: bool;
 +         scope 2 {
 +             debug val => _0;
 +         }
@@ -37,14 +36,10 @@
 +         StorageLive(_3);
 +         StorageLive(_4);
 +         StorageLive(_5);
-+         StorageLive(_6);
 +         _4 = discriminant(_2);
-+         _5 = Ne(_4, const 0_isize);
++         _5 = Eq(_4, const 1_isize);
 +         assume(move _5);
-+         _6 = Eq(_4, const 1_isize);
-+         assume(move _6);
 +         _0 = move ((_2 as Some).0: T);
-+         StorageDead(_6);
 +         StorageDead(_5);
 +         StorageDead(_4);
 +         StorageDead(_3);
diff --git a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-unwind.diff b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-unwind.diff
index caabb7ea4635..14c8c671d3fe 100644
--- a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-unwind.diff
+++ b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.Inline.panic-unwind.diff
@@ -10,7 +10,6 @@
 +         let mut _3: &std::option::Option<T>;
 +         let mut _4: isize;
 +         let mut _5: bool;
-+         let mut _6: bool;
 +         scope 2 {
 +             debug val => _0;
 +         }
@@ -37,14 +36,10 @@
 +         StorageLive(_3);
 +         StorageLive(_4);
 +         StorageLive(_5);
-+         StorageLive(_6);
 +         _4 = discriminant(_2);
-+         _5 = Ne(_4, const 0_isize);
++         _5 = Eq(_4, const 1_isize);
 +         assume(move _5);
-+         _6 = Eq(_4, const 1_isize);
-+         assume(move _6);
 +         _0 = move ((_2 as Some).0: T);
-+         StorageDead(_6);
 +         StorageDead(_5);
 +         StorageDead(_4);
 +         StorageDead(_3);
diff --git a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-abort.mir b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-abort.mir
index 521266925e0d..aeb93bd334fe 100644
--- a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-abort.mir
+++ b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-abort.mir
@@ -7,8 +7,7 @@ fn unwrap_unchecked(_1: Option<T>) -> T {
         debug self => _1;
         let mut _2: isize;
         let mut _3: bool;
-        let mut _4: bool;
-        let mut _5: &std::option::Option<T>;
+        let mut _4: &std::option::Option<T>;
         scope 2 {
             debug val => _0;
         }
@@ -21,25 +20,21 @@ fn unwrap_unchecked(_1: Option<T>) -> T {
             }
         }
         scope 4 (inlined Option::<T>::is_some) {
-            debug self => _5;
+            debug self => _4;
         }
     }
 
     bb0: {
-        StorageLive(_5);
+        StorageLive(_4);
         StorageLive(_2);
         StorageLive(_3);
-        StorageLive(_4);
         _2 = discriminant(_1);
-        _3 = Ne(_2, const 0_isize);
+        _3 = Eq(_2, const 1_isize);
         assume(move _3);
-        _4 = Eq(_2, const 1_isize);
-        assume(move _4);
         _0 = move ((_1 as Some).0: T);
-        StorageDead(_4);
         StorageDead(_3);
         StorageDead(_2);
-        StorageDead(_5);
+        StorageDead(_4);
         return;
     }
 }
diff --git a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-unwind.mir
index 521266925e0d..aeb93bd334fe 100644
--- a/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-unwind.mir
+++ b/tests/mir-opt/inline/unwrap_unchecked.unwrap_unchecked.PreCodegen.after.panic-unwind.mir
@@ -7,8 +7,7 @@ fn unwrap_unchecked(_1: Option<T>) -> T {
         debug self => _1;
         let mut _2: isize;
         let mut _3: bool;
-        let mut _4: bool;
-        let mut _5: &std::option::Option<T>;
+        let mut _4: &std::option::Option<T>;
         scope 2 {
             debug val => _0;
         }
@@ -21,25 +20,21 @@ fn unwrap_unchecked(_1: Option<T>) -> T {
             }
         }
         scope 4 (inlined Option::<T>::is_some) {
-            debug self => _5;
+            debug self => _4;
         }
     }
 
     bb0: {
-        StorageLive(_5);
+        StorageLive(_4);
         StorageLive(_2);
         StorageLive(_3);
-        StorageLive(_4);
         _2 = discriminant(_1);
-        _3 = Ne(_2, const 0_isize);
+        _3 = Eq(_2, const 1_isize);
         assume(move _3);
-        _4 = Eq(_2, const 1_isize);
-        assume(move _4);
         _0 = move ((_1 as Some).0: T);
-        StorageDead(_4);
         StorageDead(_3);
         StorageDead(_2);
-        StorageDead(_5);
+        StorageDead(_4);
         return;
     }
 }
diff --git a/tests/mir-opt/pre-codegen/duplicate_switch_targets.ub_if_b.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/duplicate_switch_targets.ub_if_b.PreCodegen.after.mir
index fe7beadc8188..0114309dbb58 100644
--- a/tests/mir-opt/pre-codegen/duplicate_switch_targets.ub_if_b.PreCodegen.after.mir
+++ b/tests/mir-opt/pre-codegen/duplicate_switch_targets.ub_if_b.PreCodegen.after.mir
@@ -5,7 +5,6 @@ fn ub_if_b(_1: Thing) -> Thing {
     let mut _0: Thing;
     let mut _2: isize;
     let mut _3: bool;
-    let mut _4: bool;
     scope 1 (inlined unreachable_unchecked) {
         scope 2 {
             scope 3 (inlined unreachable_unchecked::runtime) {
@@ -15,10 +14,8 @@ fn ub_if_b(_1: Thing) -> Thing {
 
     bb0: {
         _2 = discriminant(_1);
-        _3 = Ne(_2, const 1_isize);
+        _3 = Eq(_2, const 0_isize);
         assume(move _3);
-        _4 = Eq(_2, const 0_isize);
-        assume(move _4);
         _0 = move _1;
         return;
     }
diff --git a/tests/mir-opt/unreachable.as_match.UnreachablePropagation.panic-abort.diff b/tests/mir-opt/unreachable.as_match.UnreachablePropagation.panic-abort.diff
new file mode 100644
index 000000000000..f6e594ffac76
--- /dev/null
+++ b/tests/mir-opt/unreachable.as_match.UnreachablePropagation.panic-abort.diff
@@ -0,0 +1,45 @@
+- // MIR for `as_match` before UnreachablePropagation
++ // MIR for `as_match` after UnreachablePropagation
+  
+  fn as_match() -> () {
+      let mut _0: ();
+      let mut _1: std::option::Option<Empty>;
+      let mut _2: isize;
+      let _3: Empty;
+      let mut _4: !;
++     let mut _5: bool;
+      scope 1 {
+          debug _x => _3;
+      }
+  
+      bb0: {
+          StorageLive(_1);
+          _1 = empty() -> [return: bb1, unwind unreachable];
+      }
+  
+      bb1: {
+          _2 = discriminant(_1);
+-         switchInt(move _2) -> [0: bb4, 1: bb2, otherwise: bb3];
++         _5 = Eq(_2, const 0_isize);
++         assume(move _5);
++         goto -> bb4;
+      }
+  
+      bb2: {
+-         StorageLive(_3);
+-         _3 = move ((_1 as Some).0: Empty);
+-         StorageLive(_4);
+          unreachable;
+      }
+  
+      bb3: {
+          unreachable;
+      }
+  
+      bb4: {
+          _0 = const ();
+          StorageDead(_1);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/unreachable.as_match.UnreachablePropagation.panic-unwind.diff b/tests/mir-opt/unreachable.as_match.UnreachablePropagation.panic-unwind.diff
new file mode 100644
index 000000000000..2813d64672e6
--- /dev/null
+++ b/tests/mir-opt/unreachable.as_match.UnreachablePropagation.panic-unwind.diff
@@ -0,0 +1,45 @@
+- // MIR for `as_match` before UnreachablePropagation
++ // MIR for `as_match` after UnreachablePropagation
+  
+  fn as_match() -> () {
+      let mut _0: ();
+      let mut _1: std::option::Option<Empty>;
+      let mut _2: isize;
+      let _3: Empty;
+      let mut _4: !;
++     let mut _5: bool;
+      scope 1 {
+          debug _x => _3;
+      }
+  
+      bb0: {
+          StorageLive(_1);
+          _1 = empty() -> [return: bb1, unwind continue];
+      }
+  
+      bb1: {
+          _2 = discriminant(_1);
+-         switchInt(move _2) -> [0: bb4, 1: bb2, otherwise: bb3];
++         _5 = Eq(_2, const 0_isize);
++         assume(move _5);
++         goto -> bb4;
+      }
+  
+      bb2: {
+-         StorageLive(_3);
+-         _3 = move ((_1 as Some).0: Empty);
+-         StorageLive(_4);
+          unreachable;
+      }
+  
+      bb3: {
+          unreachable;
+      }
+  
+      bb4: {
+          _0 = const ();
+          StorageDead(_1);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-abort.diff b/tests/mir-opt/unreachable.if_let.UnreachablePropagation.panic-abort.diff
similarity index 86%
rename from tests/mir-opt/unreachable.main.UnreachablePropagation.panic-abort.diff
rename to tests/mir-opt/unreachable.if_let.UnreachablePropagation.panic-abort.diff
index 6f4faefce145..61959732720e 100644
--- a/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-abort.diff
+++ b/tests/mir-opt/unreachable.if_let.UnreachablePropagation.panic-abort.diff
@@ -1,7 +1,7 @@
-- // MIR for `main` before UnreachablePropagation
-+ // MIR for `main` after UnreachablePropagation
+- // MIR for `if_let` before UnreachablePropagation
++ // MIR for `if_let` after UnreachablePropagation
   
-  fn main() -> () {
+  fn if_let() -> () {
       let mut _0: ();
       let mut _1: std::option::Option<Empty>;
       let mut _2: isize;
@@ -9,7 +9,6 @@
       let mut _6: bool;
       let mut _7: !;
 +     let mut _8: bool;
-+     let mut _9: bool;
       scope 1 {
           debug _x => _3;
           let _3: Empty;
@@ -27,8 +26,8 @@
       bb1: {
           _2 = discriminant(_1);
 -         switchInt(move _2) -> [1: bb2, otherwise: bb6];
-+         _9 = Ne(_2, const 1_isize);
-+         assume(move _9);
++         _8 = Ne(_2, const 1_isize);
++         assume(move _8);
 +         goto -> bb6;
       }
   
diff --git a/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-unwind.diff b/tests/mir-opt/unreachable.if_let.UnreachablePropagation.panic-unwind.diff
similarity index 86%
rename from tests/mir-opt/unreachable.main.UnreachablePropagation.panic-unwind.diff
rename to tests/mir-opt/unreachable.if_let.UnreachablePropagation.panic-unwind.diff
index 5bacb42bed3f..476e2f559944 100644
--- a/tests/mir-opt/unreachable.main.UnreachablePropagation.panic-unwind.diff
+++ b/tests/mir-opt/unreachable.if_let.UnreachablePropagation.panic-unwind.diff
@@ -1,7 +1,7 @@
-- // MIR for `main` before UnreachablePropagation
-+ // MIR for `main` after UnreachablePropagation
+- // MIR for `if_let` before UnreachablePropagation
++ // MIR for `if_let` after UnreachablePropagation
   
-  fn main() -> () {
+  fn if_let() -> () {
       let mut _0: ();
       let mut _1: std::option::Option<Empty>;
       let mut _2: isize;
@@ -9,7 +9,6 @@
       let mut _6: bool;
       let mut _7: !;
 +     let mut _8: bool;
-+     let mut _9: bool;
       scope 1 {
           debug _x => _3;
           let _3: Empty;
@@ -27,8 +26,8 @@
       bb1: {
           _2 = discriminant(_1);
 -         switchInt(move _2) -> [1: bb2, otherwise: bb6];
-+         _9 = Ne(_2, const 1_isize);
-+         assume(move _9);
++         _8 = Ne(_2, const 1_isize);
++         assume(move _8);
 +         goto -> bb6;
       }
   
diff --git a/tests/mir-opt/unreachable.rs b/tests/mir-opt/unreachable.rs
index 3d934ace261e..5b96681d9df5 100644
--- a/tests/mir-opt/unreachable.rs
+++ b/tests/mir-opt/unreachable.rs
@@ -7,9 +7,9 @@ fn empty() -> Option<Empty> {
     None
 }
 
-// EMIT_MIR unreachable.main.UnreachablePropagation.diff
-fn main() {
-    // CHECK-LABEL: fn main(
+// EMIT_MIR unreachable.if_let.UnreachablePropagation.diff
+fn if_let() {
+    // CHECK-LABEL: fn if_let(
     // CHECK: bb0: {
     // CHECK: {{_.*}} = empty()
     // CHECK: bb1: {
@@ -38,3 +38,29 @@ fn main() {
         match _x { }
     }
 }
+
+// EMIT_MIR unreachable.as_match.UnreachablePropagation.diff
+fn as_match() {
+    // CHECK-LABEL: fn as_match(
+    // CHECK: bb0: {
+    // CHECK: {{_.*}} = empty()
+    // CHECK: bb1: {
+    // CHECK: [[eq:_.*]] = Eq({{.*}}, const 0_isize);
+    // CHECK-NEXT: assume(move [[eq]]);
+    // CHECK-NEXT: goto -> bb4;
+    // CHECK: bb2: {
+    // CHECK-NEXT: unreachable;
+    // CHECK: bb3: {
+    // CHECK-NEXT: unreachable;
+    // CHECK: bb4: {
+    // CHECK: return;
+    match empty() {
+        None => {}
+        Some(_x) => match _x {}
+    }
+}
+
+fn main() {
+    if_let();
+    as_match();
+}

From ae2e21114becdd27b426baaa385a90aca0cfa412 Mon Sep 17 00:00:00 2001
From: Camille GILLOT <gillot.camille@gmail.com>
Date: Sat, 21 Oct 2023 13:45:25 +0000
Subject: [PATCH 282/297] FileCheck uninhabited_enum_branching.

---
 ...ching.byref.UninhabitedEnumBranching.diff} | 18 ++--
 ...discriminant.UninhabitedEnumBranching.diff | 48 +++++++++
 ...after-uninhabited-enum-branching.after.mir | 65 -------------
 tests/mir-opt/uninhabited_enum_branching.rs   | 70 +++++++++++--
 ...hing.simple.UninhabitedEnumBranching.diff} | 38 +-------
 ...after-uninhabited-enum-branching.after.mir | 97 -------------------
 tests/mir-opt/uninhabited_enum_branching2.rs  | 35 -------
 7 files changed, 126 insertions(+), 245 deletions(-)
 rename tests/mir-opt/{uninhabited_enum_branching2.main.UninhabitedEnumBranching.diff => uninhabited_enum_branching.byref.UninhabitedEnumBranching.diff} (87%)
 create mode 100644 tests/mir-opt/uninhabited_enum_branching.custom_discriminant.UninhabitedEnumBranching.diff
 delete mode 100644 tests/mir-opt/uninhabited_enum_branching.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir
 rename tests/mir-opt/{uninhabited_enum_branching.main.UninhabitedEnumBranching.diff => uninhabited_enum_branching.simple.UninhabitedEnumBranching.diff} (52%)
 delete mode 100644 tests/mir-opt/uninhabited_enum_branching2.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir
 delete mode 100644 tests/mir-opt/uninhabited_enum_branching2.rs

diff --git a/tests/mir-opt/uninhabited_enum_branching2.main.UninhabitedEnumBranching.diff b/tests/mir-opt/uninhabited_enum_branching.byref.UninhabitedEnumBranching.diff
similarity index 87%
rename from tests/mir-opt/uninhabited_enum_branching2.main.UninhabitedEnumBranching.diff
rename to tests/mir-opt/uninhabited_enum_branching.byref.UninhabitedEnumBranching.diff
index 165421acd691..7919450cdc52 100644
--- a/tests/mir-opt/uninhabited_enum_branching2.main.UninhabitedEnumBranching.diff
+++ b/tests/mir-opt/uninhabited_enum_branching.byref.UninhabitedEnumBranching.diff
@@ -1,12 +1,12 @@
-- // MIR for `main` before UninhabitedEnumBranching
-+ // MIR for `main` after UninhabitedEnumBranching
+- // MIR for `byref` before UninhabitedEnumBranching
++ // MIR for `byref` after UninhabitedEnumBranching
   
-  fn main() -> () {
+  fn byref() -> () {
       let mut _0: ();
       let _1: Plop;
-      let mut _2: Test1;
+      let mut _2: Test3;
       let _3: &str;
-      let mut _4: &Test1;
+      let mut _4: &Test3;
       let mut _5: isize;
       let _6: &str;
       let _7: &str;
@@ -23,12 +23,12 @@
       bb0: {
           StorageLive(_1);
           StorageLive(_2);
-          _2 = Test1::C;
-          _1 = Plop { xx: const 51_u32, test1: move _2 };
+          _2 = Test3::C;
+          _1 = Plop { xx: const 51_u32, test3: move _2 };
           StorageDead(_2);
           StorageLive(_3);
           StorageLive(_4);
-          _4 = &(_1.1: Test1);
+          _4 = &(_1.1: Test3);
           _5 = discriminant((*_4));
 -         switchInt(move _5) -> [0: bb3, 1: bb4, 2: bb5, 3: bb1, otherwise: bb2];
 +         switchInt(move _5) -> [0: bb12, 1: bb12, 2: bb5, 3: bb1, otherwise: bb12];
@@ -71,7 +71,7 @@
           StorageDead(_4);
           StorageDead(_3);
           StorageLive(_9);
-          _10 = discriminant((_1.1: Test1));
+          _10 = discriminant((_1.1: Test3));
 -         switchInt(move _10) -> [0: bb8, 1: bb9, 2: bb10, 3: bb7, otherwise: bb2];
 +         switchInt(move _10) -> [0: bb12, 1: bb12, 2: bb10, 3: bb7, otherwise: bb12];
       }
diff --git a/tests/mir-opt/uninhabited_enum_branching.custom_discriminant.UninhabitedEnumBranching.diff b/tests/mir-opt/uninhabited_enum_branching.custom_discriminant.UninhabitedEnumBranching.diff
new file mode 100644
index 000000000000..5e15298a78c1
--- /dev/null
+++ b/tests/mir-opt/uninhabited_enum_branching.custom_discriminant.UninhabitedEnumBranching.diff
@@ -0,0 +1,48 @@
+- // MIR for `custom_discriminant` before UninhabitedEnumBranching
++ // MIR for `custom_discriminant` after UninhabitedEnumBranching
+  
+  fn custom_discriminant() -> () {
+      let mut _0: ();
+      let _1: &str;
+      let mut _2: Test2;
+      let mut _3: isize;
+      let _4: &str;
+  
+      bb0: {
+          StorageLive(_1);
+          StorageLive(_2);
+          _2 = Test2::D;
+          _3 = discriminant(_2);
+-         switchInt(move _3) -> [4: bb3, 5: bb1, otherwise: bb2];
++         switchInt(move _3) -> [4: bb3, 5: bb1, otherwise: bb5];
+      }
+  
+      bb1: {
+          StorageLive(_4);
+          _4 = const "E";
+          _1 = &(*_4);
+          StorageDead(_4);
+          goto -> bb4;
+      }
+  
+      bb2: {
+          unreachable;
+      }
+  
+      bb3: {
+          _1 = const "D";
+          goto -> bb4;
+      }
+  
+      bb4: {
+          StorageDead(_2);
+          StorageDead(_1);
+          _0 = const ();
+          return;
++     }
++ 
++     bb5: {
++         unreachable;
+      }
+  }
+  
diff --git a/tests/mir-opt/uninhabited_enum_branching.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir b/tests/mir-opt/uninhabited_enum_branching.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir
deleted file mode 100644
index 1df3b74b3e7d..000000000000
--- a/tests/mir-opt/uninhabited_enum_branching.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir
+++ /dev/null
@@ -1,65 +0,0 @@
-// MIR for `main` after SimplifyCfg-after-uninhabited-enum-branching
-
-fn main() -> () {
-    let mut _0: ();
-    let _1: &str;
-    let mut _2: Test1;
-    let mut _3: isize;
-    let _4: &str;
-    let _5: &str;
-    let _6: &str;
-    let mut _7: Test2;
-    let mut _8: isize;
-    let _9: &str;
-    let mut _10: bool;
-    let mut _11: bool;
-    let mut _12: bool;
-
-    bb0: {
-        StorageLive(_1);
-        StorageLive(_2);
-        _2 = Test1::C;
-        _3 = discriminant(_2);
-        _10 = Ne(_3, const 0_isize);
-        assume(move _10);
-        _11 = Ne(_3, const 1_isize);
-        assume(move _11);
-        _12 = Eq(_3, const 2_isize);
-        assume(move _12);
-        StorageLive(_5);
-        _5 = const "C";
-        _1 = &(*_5);
-        StorageDead(_5);
-        StorageDead(_2);
-        StorageDead(_1);
-        StorageLive(_6);
-        StorageLive(_7);
-        _7 = Test2::D;
-        _8 = discriminant(_7);
-        switchInt(move _8) -> [4: bb3, 5: bb2, otherwise: bb1];
-    }
-
-    bb1: {
-        unreachable;
-    }
-
-    bb2: {
-        StorageLive(_9);
-        _9 = const "E";
-        _6 = &(*_9);
-        StorageDead(_9);
-        goto -> bb4;
-    }
-
-    bb3: {
-        _6 = const "D";
-        goto -> bb4;
-    }
-
-    bb4: {
-        StorageDead(_7);
-        StorageDead(_6);
-        _0 = const ();
-        return;
-    }
-}
diff --git a/tests/mir-opt/uninhabited_enum_branching.rs b/tests/mir-opt/uninhabited_enum_branching.rs
index 96ae84eca509..60389117b161 100644
--- a/tests/mir-opt/uninhabited_enum_branching.rs
+++ b/tests/mir-opt/uninhabited_enum_branching.rs
@@ -1,11 +1,11 @@
-// skip-filecheck
-enum Empty { }
+// unit-test: UninhabitedEnumBranching
+enum Empty {}
 
 // test matching an enum with uninhabited variants
 enum Test1 {
     A(Empty),
     B(Empty),
-    C
+    C,
 }
 
 // test an enum where the discriminants don't match the variant indexes
@@ -15,17 +15,75 @@ enum Test2 {
     E = 5,
 }
 
-// EMIT_MIR uninhabited_enum_branching.main.UninhabitedEnumBranching.diff
-// EMIT_MIR uninhabited_enum_branching.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir
-fn main() {
+// test matching an enum with uninhabited variants and multiple inhabited
+enum Test3 {
+    A(Empty),
+    B(Empty),
+    C,
+    D,
+}
+
+struct Plop {
+    xx: u32,
+    test3: Test3,
+}
+
+// EMIT_MIR uninhabited_enum_branching.simple.UninhabitedEnumBranching.diff
+fn simple() {
+    // CHECK-LABEL: fn simple(
+    // CHECK: [[discr:_.*]] = discriminant(
+    // CHECK: switchInt(move [[discr]]) -> [0: [[unreachable:bb.*]], 1: [[unreachable]], 2: bb1, otherwise: [[unreachable]]];
+    // CHECK: [[unreachable]]: {
+    // CHECK-NEXT: unreachable;
     match Test1::C {
         Test1::A(_) => "A(Empty)",
         Test1::B(_) => "B(Empty)",
         Test1::C => "C",
     };
+}
 
+// EMIT_MIR uninhabited_enum_branching.custom_discriminant.UninhabitedEnumBranching.diff
+fn custom_discriminant() {
+    // CHECK-LABEL: fn custom_discriminant(
+    // CHECK: [[discr:_.*]] = discriminant(
+    // CHECK: switchInt(move [[discr]]) -> [4: bb3, 5: bb1, otherwise: bb5];
+    // CHECK: bb5: {
+    // CHECK-NEXT: unreachable;
     match Test2::D {
         Test2::D => "D",
         Test2::E => "E",
     };
 }
+
+// EMIT_MIR uninhabited_enum_branching.byref.UninhabitedEnumBranching.diff
+fn byref() {
+    // CHECK-LABEL: fn byref(
+    let plop = Plop { xx: 51, test3: Test3::C };
+
+    // CHECK: [[ref_discr:_.*]] = discriminant((*
+    // CHECK: switchInt(move [[ref_discr]]) -> [0: [[unreachable:bb.*]], 1: [[unreachable]], 2: bb5, 3: bb1, otherwise: [[unreachable]]];
+    match &plop.test3 {
+        Test3::A(_) => "A(Empty)",
+        Test3::B(_) => "B(Empty)",
+        Test3::C => "C",
+        Test3::D => "D",
+    };
+
+    // CHECK: [[discr:_.*]] = discriminant(
+    // CHECK: switchInt(move [[discr]]) -> [0: [[unreachable]], 1: [[unreachable]], 2: bb10, 3: bb7, otherwise: [[unreachable]]];
+    match plop.test3 {
+        Test3::A(_) => "A(Empty)",
+        Test3::B(_) => "B(Empty)",
+        Test3::C => "C",
+        Test3::D => "D",
+    };
+
+    // CHECK: [[unreachable]]: {
+    // CHECK-NEXT: unreachable;
+}
+
+fn main() {
+    simple();
+    custom_discriminant();
+    byref();
+}
diff --git a/tests/mir-opt/uninhabited_enum_branching.main.UninhabitedEnumBranching.diff b/tests/mir-opt/uninhabited_enum_branching.simple.UninhabitedEnumBranching.diff
similarity index 52%
rename from tests/mir-opt/uninhabited_enum_branching.main.UninhabitedEnumBranching.diff
rename to tests/mir-opt/uninhabited_enum_branching.simple.UninhabitedEnumBranching.diff
index 5b107f80ce80..410db79802ed 100644
--- a/tests/mir-opt/uninhabited_enum_branching.main.UninhabitedEnumBranching.diff
+++ b/tests/mir-opt/uninhabited_enum_branching.simple.UninhabitedEnumBranching.diff
@@ -1,17 +1,13 @@
-- // MIR for `main` before UninhabitedEnumBranching
-+ // MIR for `main` after UninhabitedEnumBranching
+- // MIR for `simple` before UninhabitedEnumBranching
++ // MIR for `simple` after UninhabitedEnumBranching
   
-  fn main() -> () {
+  fn simple() -> () {
       let mut _0: ();
       let _1: &str;
       let mut _2: Test1;
       let mut _3: isize;
       let _4: &str;
       let _5: &str;
-      let _6: &str;
-      let mut _7: Test2;
-      let mut _8: isize;
-      let _9: &str;
   
       bb0: {
           StorageLive(_1);
@@ -19,7 +15,7 @@
           _2 = Test1::C;
           _3 = discriminant(_2);
 -         switchInt(move _3) -> [0: bb3, 1: bb4, 2: bb1, otherwise: bb2];
-+         switchInt(move _3) -> [0: bb9, 1: bb9, 2: bb1, otherwise: bb9];
++         switchInt(move _3) -> [0: bb6, 1: bb6, 2: bb1, otherwise: bb6];
       }
   
       bb1: {
@@ -50,35 +46,11 @@
       bb5: {
           StorageDead(_2);
           StorageDead(_1);
-          StorageLive(_6);
-          StorageLive(_7);
-          _7 = Test2::D;
-          _8 = discriminant(_7);
--         switchInt(move _8) -> [4: bb7, 5: bb6, otherwise: bb2];
-+         switchInt(move _8) -> [4: bb7, 5: bb6, otherwise: bb9];
-      }
-  
-      bb6: {
-          StorageLive(_9);
-          _9 = const "E";
-          _6 = &(*_9);
-          StorageDead(_9);
-          goto -> bb8;
-      }
-  
-      bb7: {
-          _6 = const "D";
-          goto -> bb8;
-      }
-  
-      bb8: {
-          StorageDead(_7);
-          StorageDead(_6);
           _0 = const ();
           return;
 +     }
 + 
-+     bb9: {
++     bb6: {
 +         unreachable;
       }
   }
diff --git a/tests/mir-opt/uninhabited_enum_branching2.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir b/tests/mir-opt/uninhabited_enum_branching2.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir
deleted file mode 100644
index 06375b3ffaeb..000000000000
--- a/tests/mir-opt/uninhabited_enum_branching2.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir
+++ /dev/null
@@ -1,97 +0,0 @@
-// MIR for `main` after SimplifyCfg-after-uninhabited-enum-branching
-
-fn main() -> () {
-    let mut _0: ();
-    let _1: Plop;
-    let mut _2: Test1;
-    let _3: &str;
-    let mut _4: &Test1;
-    let mut _5: isize;
-    let _6: &str;
-    let _7: &str;
-    let _8: &str;
-    let _9: &str;
-    let mut _10: isize;
-    let _11: &str;
-    let _12: &str;
-    let _13: &str;
-    let mut _14: bool;
-    let mut _15: bool;
-    let mut _16: bool;
-    let mut _17: bool;
-    scope 1 {
-        debug plop => _1;
-    }
-
-    bb0: {
-        StorageLive(_1);
-        StorageLive(_2);
-        _2 = Test1::C;
-        _1 = Plop { xx: const 51_u32, test1: move _2 };
-        StorageDead(_2);
-        StorageLive(_3);
-        StorageLive(_4);
-        _4 = &(_1.1: Test1);
-        _5 = discriminant((*_4));
-        _16 = Ne(_5, const 0_isize);
-        assume(move _16);
-        _17 = Ne(_5, const 1_isize);
-        assume(move _17);
-        switchInt(move _5) -> [2: bb3, 3: bb1, otherwise: bb2];
-    }
-
-    bb1: {
-        StorageLive(_8);
-        _8 = const "D";
-        _3 = &(*_8);
-        StorageDead(_8);
-        goto -> bb4;
-    }
-
-    bb2: {
-        unreachable;
-    }
-
-    bb3: {
-        StorageLive(_7);
-        _7 = const "C";
-        _3 = &(*_7);
-        StorageDead(_7);
-        goto -> bb4;
-    }
-
-    bb4: {
-        StorageDead(_4);
-        StorageDead(_3);
-        StorageLive(_9);
-        _10 = discriminant((_1.1: Test1));
-        _14 = Ne(_10, const 0_isize);
-        assume(move _14);
-        _15 = Ne(_10, const 1_isize);
-        assume(move _15);
-        switchInt(move _10) -> [2: bb6, 3: bb5, otherwise: bb2];
-    }
-
-    bb5: {
-        StorageLive(_13);
-        _13 = const "D";
-        _9 = &(*_13);
-        StorageDead(_13);
-        goto -> bb7;
-    }
-
-    bb6: {
-        StorageLive(_12);
-        _12 = const "C";
-        _9 = &(*_12);
-        StorageDead(_12);
-        goto -> bb7;
-    }
-
-    bb7: {
-        StorageDead(_9);
-        _0 = const ();
-        StorageDead(_1);
-        return;
-    }
-}
diff --git a/tests/mir-opt/uninhabited_enum_branching2.rs b/tests/mir-opt/uninhabited_enum_branching2.rs
deleted file mode 100644
index 751f2ae01f8b..000000000000
--- a/tests/mir-opt/uninhabited_enum_branching2.rs
+++ /dev/null
@@ -1,35 +0,0 @@
-// skip-filecheck
-enum Empty { }
-
-// test matching an enum with uninhabited variants
-enum Test1 {
-    A(Empty),
-    B(Empty),
-    C,
-    D,
-}
-
-struct Plop {
-    xx: u32,
-    test1: Test1,
-}
-
-// EMIT_MIR uninhabited_enum_branching2.main.UninhabitedEnumBranching.diff
-// EMIT_MIR uninhabited_enum_branching2.main.SimplifyCfg-after-uninhabited-enum-branching.after.mir
-fn main() {
-    let plop = Plop { xx: 51, test1: Test1::C };
-
-    match &plop.test1 {
-        Test1::A(_) => "A(Empty)",
-        Test1::B(_) => "B(Empty)",
-        Test1::C => "C",
-        Test1::D => "D",
-    };
-
-    match plop.test1 {
-        Test1::A(_) => "A(Empty)",
-        Test1::B(_) => "B(Empty)",
-        Test1::C => "C",
-        Test1::D => "D",
-    };
-}

From de83057ac4b050702042c01177dd6e4e9aa7b2af Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Mon, 30 Oct 2023 17:09:17 +0000
Subject: [PATCH 283/297] Use derivative for Clone

---
 Cargo.lock                                   | 12 +++++++
 compiler/rustc_type_ir/Cargo.toml            |  1 +
 compiler/rustc_type_ir/src/canonical.rs      | 12 ++-----
 compiler/rustc_type_ir/src/const_kind.rs     | 18 ++--------
 compiler/rustc_type_ir/src/predicate_kind.rs | 37 +++-----------------
 compiler/rustc_type_ir/src/region_kind.rs    | 18 ++--------
 compiler/rustc_type_ir/src/ty_kind.rs        | 37 ++------------------
 src/tools/tidy/src/deps.rs                   |  1 +
 8 files changed, 27 insertions(+), 109 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 2b1e918e2f5e..a9090201710a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1011,6 +1011,17 @@ dependencies = [
  "syn 2.0.29",
 ]
 
+[[package]]
+name = "derivative"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
 [[package]]
 name = "derive_builder"
 version = "0.12.0"
@@ -4667,6 +4678,7 @@ name = "rustc_type_ir"
 version = "0.0.0"
 dependencies = [
  "bitflags 1.3.2",
+ "derivative",
  "rustc_data_structures",
  "rustc_index",
  "rustc_macros",
diff --git a/compiler/rustc_type_ir/Cargo.toml b/compiler/rustc_type_ir/Cargo.toml
index 484757e2fc1a..b39ba305913e 100644
--- a/compiler/rustc_type_ir/Cargo.toml
+++ b/compiler/rustc_type_ir/Cargo.toml
@@ -6,6 +6,7 @@ edition = "2021"
 [dependencies]
 # tidy-alphabetical-start
 bitflags = "1.2.1"
+derivative = "2.2.0"
 rustc_data_structures = { path = "../rustc_data_structures" }
 rustc_index = { path = "../rustc_index" }
 rustc_macros = { path = "../rustc_macros" }
diff --git a/compiler/rustc_type_ir/src/canonical.rs b/compiler/rustc_type_ir/src/canonical.rs
index c0b6aed98ef2..1333a29a141c 100644
--- a/compiler/rustc_type_ir/src/canonical.rs
+++ b/compiler/rustc_type_ir/src/canonical.rs
@@ -13,6 +13,8 @@ use crate::{HashStableContext, Interner, TyEncoder, UniverseIndex};
 /// A "canonicalized" type `V` is one where all free inference
 /// variables have been rewritten to "canonical vars". These are
 /// numbered starting from 0 in order of first appearance.
+#[derive(derivative::Derivative)]
+#[derivative(Clone(bound = "V: Clone"))]
 pub struct Canonical<I: Interner, V> {
     pub value: V,
     pub max_universe: UniverseIndex,
@@ -108,16 +110,6 @@ impl<I: Interner, V: fmt::Debug> fmt::Debug for Canonical<I, V> {
     }
 }
 
-impl<I: Interner, V: Clone> Clone for Canonical<I, V> {
-    fn clone(&self) -> Self {
-        Canonical {
-            value: self.value.clone(),
-            max_universe: self.max_universe.clone(),
-            variables: self.variables.clone(),
-        }
-    }
-}
-
 impl<I: Interner, V: Copy> Copy for Canonical<I, V> where I::CanonicalVars: Copy {}
 
 impl<I: Interner, V: TypeFoldable<I>> TypeFoldable<I> for Canonical<I, V>
diff --git a/compiler/rustc_type_ir/src/const_kind.rs b/compiler/rustc_type_ir/src/const_kind.rs
index a40c41583af5..3e0162328eef 100644
--- a/compiler/rustc_type_ir/src/const_kind.rs
+++ b/compiler/rustc_type_ir/src/const_kind.rs
@@ -13,7 +13,8 @@ use crate::{
 use self::ConstKind::*;
 
 /// Represents a constant in Rust.
-// #[derive(derive_more::From)]
+#[derive(derivative::Derivative)]
+#[derivative(Clone(bound = ""))]
 pub enum ConstKind<I: Interner> {
     /// A const generic parameter.
     Param(I::ParamConst),
@@ -211,21 +212,6 @@ impl<I: Interner> PartialEq for ConstKind<I> {
 
 impl<I: Interner> Eq for ConstKind<I> {}
 
-impl<I: Interner> Clone for ConstKind<I> {
-    fn clone(&self) -> Self {
-        match self {
-            Param(arg0) => Param(arg0.clone()),
-            Infer(arg0) => Infer(arg0.clone()),
-            Bound(arg0, arg1) => Bound(arg0.clone(), arg1.clone()),
-            Placeholder(arg0) => Placeholder(arg0.clone()),
-            Unevaluated(arg0) => Unevaluated(arg0.clone()),
-            Value(arg0) => Value(arg0.clone()),
-            Error(arg0) => Error(arg0.clone()),
-            Expr(arg0) => Expr(arg0.clone()),
-        }
-    }
-}
-
 impl<I: Interner> fmt::Debug for ConstKind<I> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         WithInfcx::with_no_infcx(self).fmt(f)
diff --git a/compiler/rustc_type_ir/src/predicate_kind.rs b/compiler/rustc_type_ir/src/predicate_kind.rs
index f6fabe691c6a..124d9d0eb959 100644
--- a/compiler/rustc_type_ir/src/predicate_kind.rs
+++ b/compiler/rustc_type_ir/src/predicate_kind.rs
@@ -12,6 +12,8 @@ use crate::{TyDecoder, TyEncoder};
 
 /// A clause is something that can appear in where bounds or be inferred
 /// by implied bounds.
+#[derive(derivative::Derivative)]
+#[derivative(Clone(bound = ""))]
 pub enum ClauseKind<I: Interner> {
     /// Corresponds to `where Foo: Bar<A, B, C>`. `Foo` here would be
     /// the `Self` type of the trait reference and `A`, `B`, and `C`
@@ -39,20 +41,6 @@ pub enum ClauseKind<I: Interner> {
     ConstEvaluatable(I::Const),
 }
 
-impl<I: Interner> Clone for ClauseKind<I> {
-    fn clone(&self) -> Self {
-        match self {
-            Self::Trait(arg0) => Self::Trait(arg0.clone()),
-            Self::RegionOutlives(arg0) => Self::RegionOutlives(arg0.clone()),
-            Self::TypeOutlives(arg0) => Self::TypeOutlives(arg0.clone()),
-            Self::Projection(arg0) => Self::Projection(arg0.clone()),
-            Self::ConstArgHasType(arg0, arg1) => Self::ConstArgHasType(arg0.clone(), arg1.clone()),
-            Self::WellFormed(arg0) => Self::WellFormed(arg0.clone()),
-            Self::ConstEvaluatable(arg0) => Self::ConstEvaluatable(arg0.clone()),
-        }
-    }
-}
-
 impl<I: Interner> Copy for ClauseKind<I>
 where
     I::Ty: Copy,
@@ -249,6 +237,8 @@ where
     }
 }
 
+#[derive(derivative::Derivative)]
+#[derivative(Clone(bound = ""))]
 pub enum PredicateKind<I: Interner> {
     /// Prove a clause
     Clause(ClauseKind<I>),
@@ -305,25 +295,6 @@ where
 {
 }
 
-impl<I: Interner> Clone for PredicateKind<I> {
-    fn clone(&self) -> Self {
-        match self {
-            Self::Clause(arg0) => Self::Clause(arg0.clone()),
-            Self::ObjectSafe(arg0) => Self::ObjectSafe(arg0.clone()),
-            Self::ClosureKind(arg0, arg1, arg2) => {
-                Self::ClosureKind(arg0.clone(), arg1.clone(), arg2.clone())
-            }
-            Self::Subtype(arg0) => Self::Subtype(arg0.clone()),
-            Self::Coerce(arg0) => Self::Coerce(arg0.clone()),
-            Self::ConstEquate(arg0, arg1) => Self::ConstEquate(arg0.clone(), arg1.clone()),
-            Self::Ambiguous => Self::Ambiguous,
-            Self::AliasRelate(arg0, arg1, arg2) => {
-                Self::AliasRelate(arg0.clone(), arg1.clone(), arg2.clone())
-            }
-        }
-    }
-}
-
 impl<I: Interner> PartialEq for PredicateKind<I> {
     fn eq(&self, other: &Self) -> bool {
         match (self, other) {
diff --git a/compiler/rustc_type_ir/src/region_kind.rs b/compiler/rustc_type_ir/src/region_kind.rs
index 19576ea58f1f..66115adefeaf 100644
--- a/compiler/rustc_type_ir/src/region_kind.rs
+++ b/compiler/rustc_type_ir/src/region_kind.rs
@@ -118,6 +118,8 @@ use self::RegionKind::*;
 /// [1]: https://smallcultfollowing.com/babysteps/blog/2013/10/29/intermingled-parameter-lists/
 /// [2]: https://smallcultfollowing.com/babysteps/blog/2013/11/04/intermingled-parameter-lists/
 /// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/traits/hrtb.html
+#[derive(derivative::Derivative)]
+#[derivative(Clone(bound = ""))]
 pub enum RegionKind<I: Interner> {
     /// Region bound in a type or fn declaration which will be
     /// substituted 'early' -- that is, at the same time when type
@@ -178,22 +180,6 @@ where
 {
 }
 
-// This is manually implemented because a derive would require `I: Clone`
-impl<I: Interner> Clone for RegionKind<I> {
-    fn clone(&self) -> Self {
-        match self {
-            ReEarlyBound(r) => ReEarlyBound(r.clone()),
-            ReLateBound(d, r) => ReLateBound(*d, r.clone()),
-            ReFree(r) => ReFree(r.clone()),
-            ReStatic => ReStatic,
-            ReVar(r) => ReVar(r.clone()),
-            RePlaceholder(r) => RePlaceholder(r.clone()),
-            ReErased => ReErased,
-            ReError(r) => ReError(r.clone()),
-        }
-    }
-}
-
 // This is manually implemented because a derive would require `I: PartialEq`
 impl<I: Interner> PartialEq for RegionKind<I> {
     #[inline]
diff --git a/compiler/rustc_type_ir/src/ty_kind.rs b/compiler/rustc_type_ir/src/ty_kind.rs
index b542547589ac..39116d33c363 100644
--- a/compiler/rustc_type_ir/src/ty_kind.rs
+++ b/compiler/rustc_type_ir/src/ty_kind.rs
@@ -114,6 +114,8 @@ pub enum AliasKind {
 /// Types written by the user start out as `hir::TyKind` and get
 /// converted to this representation using `AstConv::ast_ty_to_ty`.
 #[rustc_diagnostic_item = "IrTyKind"]
+#[derive(derivative::Derivative)]
+#[derivative(Clone(bound = ""))]
 pub enum TyKind<I: Interner> {
     /// The primitive boolean type. Written as `bool`.
     Bool,
@@ -324,40 +326,6 @@ const fn tykind_discriminant<I: Interner>(value: &TyKind<I>) -> usize {
     }
 }
 
-// This is manually implemented because a derive would require `I: Clone`
-impl<I: Interner> Clone for TyKind<I> {
-    fn clone(&self) -> Self {
-        match self {
-            Bool => Bool,
-            Char => Char,
-            Int(i) => Int(*i),
-            Uint(u) => Uint(*u),
-            Float(f) => Float(*f),
-            Adt(d, s) => Adt(d.clone(), s.clone()),
-            Foreign(d) => Foreign(d.clone()),
-            Str => Str,
-            Array(t, c) => Array(t.clone(), c.clone()),
-            Slice(t) => Slice(t.clone()),
-            RawPtr(p) => RawPtr(p.clone()),
-            Ref(r, t, m) => Ref(r.clone(), t.clone(), m.clone()),
-            FnDef(d, s) => FnDef(d.clone(), s.clone()),
-            FnPtr(s) => FnPtr(s.clone()),
-            Dynamic(p, r, repr) => Dynamic(p.clone(), r.clone(), *repr),
-            Closure(d, s) => Closure(d.clone(), s.clone()),
-            Coroutine(d, s, m) => Coroutine(d.clone(), s.clone(), m.clone()),
-            CoroutineWitness(d, s) => CoroutineWitness(d.clone(), s.clone()),
-            Never => Never,
-            Tuple(t) => Tuple(t.clone()),
-            Alias(k, p) => Alias(*k, p.clone()),
-            Param(p) => Param(p.clone()),
-            Bound(d, b) => Bound(*d, b.clone()),
-            Placeholder(p) => Placeholder(p.clone()),
-            Infer(t) => Infer(t.clone()),
-            Error(e) => Error(e.clone()),
-        }
-    }
-}
-
 // This is manually implemented because a derive would require `I: PartialEq`
 impl<I: Interner> PartialEq for TyKind<I> {
     #[inline]
@@ -614,6 +582,7 @@ impl<I: Interner> DebugWithInfcx<I> for TyKind<I> {
         }
     }
 }
+
 // This is manually implemented because a derive would require `I: Debug`
 impl<I: Interner> fmt::Debug for TyKind<I> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs
index f89faacc2d17..4b12e9172af7 100644
--- a/src/tools/tidy/src/deps.rs
+++ b/src/tools/tidy/src/deps.rs
@@ -142,6 +142,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "darling_core",
     "darling_macro",
     "datafrog",
+    "derivative",
     "derive_more",
     "derive_setters",
     "digest",

From 8eb932dcf07753f532c12ae880359f8f392728b0 Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Mon, 30 Oct 2023 17:22:55 +0000
Subject: [PATCH 284/297] Use derivative for PartialOrd/ord

---
 compiler/rustc_type_ir/src/const_kind.rs  | 36 +++---------
 compiler/rustc_type_ir/src/region_kind.rs | 41 +++-----------
 compiler/rustc_type_ir/src/ty_kind.rs     | 67 +++--------------------
 3 files changed, 21 insertions(+), 123 deletions(-)

diff --git a/compiler/rustc_type_ir/src/const_kind.rs b/compiler/rustc_type_ir/src/const_kind.rs
index 3e0162328eef..3c986f64d38d 100644
--- a/compiler/rustc_type_ir/src/const_kind.rs
+++ b/compiler/rustc_type_ir/src/const_kind.rs
@@ -1,7 +1,6 @@
 use rustc_data_structures::stable_hasher::HashStable;
 use rustc_data_structures::stable_hasher::StableHasher;
 use rustc_serialize::{Decodable, Decoder, Encodable};
-use std::cmp::Ordering;
 use std::fmt;
 use std::hash;
 
@@ -14,7 +13,13 @@ use self::ConstKind::*;
 
 /// Represents a constant in Rust.
 #[derive(derivative::Derivative)]
-#[derivative(Clone(bound = ""))]
+#[derivative(
+    Clone(bound = ""),
+    PartialOrd(bound = ""),
+    PartialOrd = "feature_allow_slow_enum",
+    Ord(bound = ""),
+    Ord = "feature_allow_slow_enum"
+)]
 pub enum ConstKind<I: Interner> {
     /// A const generic parameter.
     Param(I::ParamConst),
@@ -167,33 +172,6 @@ where
     }
 }
 
-impl<I: Interner> PartialOrd for ConstKind<I> {
-    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl<I: Interner> Ord for ConstKind<I> {
-    fn cmp(&self, other: &Self) -> Ordering {
-        const_kind_discriminant(self)
-            .cmp(&const_kind_discriminant(other))
-            .then_with(|| match (self, other) {
-                (Param(p1), Param(p2)) => p1.cmp(p2),
-                (Infer(i1), Infer(i2)) => i1.cmp(i2),
-                (Bound(d1, b1), Bound(d2, b2)) => d1.cmp(d2).then_with(|| b1.cmp(b2)),
-                (Placeholder(p1), Placeholder(p2)) => p1.cmp(p2),
-                (Unevaluated(u1), Unevaluated(u2)) => u1.cmp(u2),
-                (Value(v1), Value(v2)) => v1.cmp(v2),
-                (Error(e1), Error(e2)) => e1.cmp(e2),
-                (Expr(e1), Expr(e2)) => e1.cmp(e2),
-                _ => {
-                    debug_assert!(false, "This branch must be unreachable, maybe the match is missing an arm? self = {self:?}, other = {other:?}");
-                    Ordering::Equal
-                }
-            })
-    }
-}
-
 impl<I: Interner> PartialEq for ConstKind<I> {
     fn eq(&self, other: &Self) -> bool {
         match (self, other) {
diff --git a/compiler/rustc_type_ir/src/region_kind.rs b/compiler/rustc_type_ir/src/region_kind.rs
index 66115adefeaf..8dc1cc080058 100644
--- a/compiler/rustc_type_ir/src/region_kind.rs
+++ b/compiler/rustc_type_ir/src/region_kind.rs
@@ -1,7 +1,6 @@
 use rustc_data_structures::stable_hasher::HashStable;
 use rustc_data_structures::stable_hasher::StableHasher;
 use rustc_serialize::{Decodable, Decoder, Encodable};
-use std::cmp::Ordering;
 use std::fmt;
 use std::hash;
 
@@ -119,7 +118,13 @@ use self::RegionKind::*;
 /// [2]: https://smallcultfollowing.com/babysteps/blog/2013/11/04/intermingled-parameter-lists/
 /// [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/traits/hrtb.html
 #[derive(derivative::Derivative)]
-#[derivative(Clone(bound = ""))]
+#[derivative(
+    Clone(bound = ""),
+    PartialOrd(bound = ""),
+    PartialOrd = "feature_allow_slow_enum",
+    Ord(bound = ""),
+    Ord = "feature_allow_slow_enum"
+)]
 pub enum RegionKind<I: Interner> {
     /// Region bound in a type or fn declaration which will be
     /// substituted 'early' -- that is, at the same time when type
@@ -208,38 +213,6 @@ impl<I: Interner> PartialEq for RegionKind<I> {
 // This is manually implemented because a derive would require `I: Eq`
 impl<I: Interner> Eq for RegionKind<I> {}
 
-// This is manually implemented because a derive would require `I: PartialOrd`
-impl<I: Interner> PartialOrd for RegionKind<I> {
-    #[inline]
-    fn partial_cmp(&self, other: &RegionKind<I>) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-// This is manually implemented because a derive would require `I: Ord`
-impl<I: Interner> Ord for RegionKind<I> {
-    #[inline]
-    fn cmp(&self, other: &RegionKind<I>) -> Ordering {
-        regionkind_discriminant(self).cmp(&regionkind_discriminant(other)).then_with(|| {
-            match (self, other) {
-                (ReEarlyBound(a_r), ReEarlyBound(b_r)) => a_r.cmp(b_r),
-                (ReLateBound(a_d, a_r), ReLateBound(b_d, b_r)) => {
-                    a_d.cmp(b_d).then_with(|| a_r.cmp(b_r))
-                }
-                (ReFree(a_r), ReFree(b_r)) => a_r.cmp(b_r),
-                (ReStatic, ReStatic) => Ordering::Equal,
-                (ReVar(a_r), ReVar(b_r)) => a_r.cmp(b_r),
-                (RePlaceholder(a_r), RePlaceholder(b_r)) => a_r.cmp(b_r),
-                (ReErased, ReErased) => Ordering::Equal,
-                _ => {
-                    debug_assert!(false, "This branch must be unreachable, maybe the match is missing an arm? self = self = {self:?}, other = {other:?}");
-                    Ordering::Equal
-                }
-            }
-        })
-    }
-}
-
 // This is manually implemented because a derive would require `I: Hash`
 impl<I: Interner> hash::Hash for RegionKind<I> {
     fn hash<H: hash::Hasher>(&self, state: &mut H) -> () {
diff --git a/compiler/rustc_type_ir/src/ty_kind.rs b/compiler/rustc_type_ir/src/ty_kind.rs
index 39116d33c363..f180ceb7b071 100644
--- a/compiler/rustc_type_ir/src/ty_kind.rs
+++ b/compiler/rustc_type_ir/src/ty_kind.rs
@@ -3,7 +3,6 @@
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::unify::{EqUnifyValue, UnifyKey};
 use rustc_serialize::{Decodable, Decoder, Encodable};
-use std::cmp::Ordering;
 use std::mem::discriminant;
 use std::{fmt, hash};
 
@@ -115,7 +114,13 @@ pub enum AliasKind {
 /// converted to this representation using `AstConv::ast_ty_to_ty`.
 #[rustc_diagnostic_item = "IrTyKind"]
 #[derive(derivative::Derivative)]
-#[derivative(Clone(bound = ""))]
+#[derivative(
+    Clone(bound = ""),
+    PartialOrd(bound = ""),
+    PartialOrd = "feature_allow_slow_enum",
+    Ord(bound = ""),
+    Ord = "feature_allow_slow_enum"
+)]
 pub enum TyKind<I: Interner> {
     /// The primitive boolean type. Written as `bool`.
     Bool,
@@ -378,64 +383,6 @@ impl<I: Interner> PartialEq for TyKind<I> {
 // This is manually implemented because a derive would require `I: Eq`
 impl<I: Interner> Eq for TyKind<I> {}
 
-// This is manually implemented because a derive would require `I: PartialOrd`
-impl<I: Interner> PartialOrd for TyKind<I> {
-    #[inline]
-    fn partial_cmp(&self, other: &TyKind<I>) -> Option<Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-// This is manually implemented because a derive would require `I: Ord`
-impl<I: Interner> Ord for TyKind<I> {
-    #[inline]
-    fn cmp(&self, other: &TyKind<I>) -> Ordering {
-        tykind_discriminant(self).cmp(&tykind_discriminant(other)).then_with(|| {
-            match (self, other) {
-                (Int(a_i), Int(b_i)) => a_i.cmp(b_i),
-                (Uint(a_u), Uint(b_u)) => a_u.cmp(b_u),
-                (Float(a_f), Float(b_f)) => a_f.cmp(b_f),
-                (Adt(a_d, a_s), Adt(b_d, b_s)) => a_d.cmp(b_d).then_with(|| a_s.cmp(b_s)),
-                (Foreign(a_d), Foreign(b_d)) => a_d.cmp(b_d),
-                (Array(a_t, a_c), Array(b_t, b_c)) => a_t.cmp(b_t).then_with(|| a_c.cmp(b_c)),
-                (Slice(a_t), Slice(b_t)) => a_t.cmp(b_t),
-                (RawPtr(a_t), RawPtr(b_t)) => a_t.cmp(b_t),
-                (Ref(a_r, a_t, a_m), Ref(b_r, b_t, b_m)) => {
-                    a_r.cmp(b_r).then_with(|| a_t.cmp(b_t).then_with(|| a_m.cmp(b_m)))
-                }
-                (FnDef(a_d, a_s), FnDef(b_d, b_s)) => a_d.cmp(b_d).then_with(|| a_s.cmp(b_s)),
-                (FnPtr(a_s), FnPtr(b_s)) => a_s.cmp(b_s),
-                (Dynamic(a_p, a_r, a_repr), Dynamic(b_p, b_r, b_repr)) => {
-                    a_p.cmp(b_p).then_with(|| a_r.cmp(b_r).then_with(|| a_repr.cmp(b_repr)))
-                }
-                (Closure(a_p, a_s), Closure(b_p, b_s)) => a_p.cmp(b_p).then_with(|| a_s.cmp(b_s)),
-                (Coroutine(a_d, a_s, a_m), Coroutine(b_d, b_s, b_m)) => {
-                    a_d.cmp(b_d).then_with(|| a_s.cmp(b_s).then_with(|| a_m.cmp(b_m)))
-                }
-                (
-                    CoroutineWitness(a_d, a_s),
-                    CoroutineWitness(b_d, b_s),
-                ) => match Ord::cmp(a_d, b_d) {
-                    Ordering::Equal => Ord::cmp(a_s, b_s),
-                    cmp => cmp,
-                },
-                (Tuple(a_t), Tuple(b_t)) => a_t.cmp(b_t),
-                (Alias(a_i, a_p), Alias(b_i, b_p)) => a_i.cmp(b_i).then_with(|| a_p.cmp(b_p)),
-                (Param(a_p), Param(b_p)) => a_p.cmp(b_p),
-                (Bound(a_d, a_b), Bound(b_d, b_b)) => a_d.cmp(b_d).then_with(|| a_b.cmp(b_b)),
-                (Placeholder(a_p), Placeholder(b_p)) => a_p.cmp(b_p),
-                (Infer(a_t), Infer(b_t)) => a_t.cmp(b_t),
-                (Error(a_e), Error(b_e)) => a_e.cmp(b_e),
-                (Bool, Bool) | (Char, Char) | (Str, Str) | (Never, Never) => Ordering::Equal,
-                _ => {
-                    debug_assert!(false, "This branch must be unreachable, maybe the match is missing an arm? self = {self:?}, other = {other:?}");
-                    Ordering::Equal
-                }
-            }
-        })
-    }
-}
-
 // This is manually implemented because a derive would require `I: Hash`
 impl<I: Interner> hash::Hash for TyKind<I> {
     fn hash<__H: hash::Hasher>(&self, state: &mut __H) -> () {

From 8b4fa0f8b507c7c98a31d18c0969e73284383542 Mon Sep 17 00:00:00 2001
From: Michael Goulet <michael@errs.io>
Date: Mon, 30 Oct 2023 17:26:29 +0000
Subject: [PATCH 285/297] Use derivative for Hash

---
 compiler/rustc_type_ir/src/canonical.rs      | 12 +---
 compiler/rustc_type_ir/src/const_kind.rs     | 23 +------
 compiler/rustc_type_ir/src/predicate_kind.rs | 50 +-------------
 compiler/rustc_type_ir/src/region_kind.rs    | 24 +------
 compiler/rustc_type_ir/src/ty_kind.rs        | 70 +-------------------
 5 files changed, 11 insertions(+), 168 deletions(-)

diff --git a/compiler/rustc_type_ir/src/canonical.rs b/compiler/rustc_type_ir/src/canonical.rs
index 1333a29a141c..c8e730b585ad 100644
--- a/compiler/rustc_type_ir/src/canonical.rs
+++ b/compiler/rustc_type_ir/src/canonical.rs
@@ -1,5 +1,5 @@
 use std::fmt;
-use std::hash;
+use std::hash::Hash;
 use std::ops::ControlFlow;
 
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
@@ -14,7 +14,7 @@ use crate::{HashStableContext, Interner, TyEncoder, UniverseIndex};
 /// variables have been rewritten to "canonical vars". These are
 /// numbered starting from 0 in order of first appearance.
 #[derive(derivative::Derivative)]
-#[derivative(Clone(bound = "V: Clone"))]
+#[derivative(Clone(bound = "V: Clone"), Hash(bound = "V: Hash"))]
 pub struct Canonical<I: Interner, V> {
     pub value: V,
     pub max_universe: UniverseIndex,
@@ -61,14 +61,6 @@ impl<I: Interner, V> Canonical<I, V> {
     }
 }
 
-impl<I: Interner, V: hash::Hash> hash::Hash for Canonical<I, V> {
-    fn hash<H: hash::Hasher>(&self, state: &mut H) {
-        self.value.hash(state);
-        self.max_universe.hash(state);
-        self.variables.hash(state);
-    }
-}
-
 impl<CTX: HashStableContext, I: Interner, V: HashStable<CTX>> HashStable<CTX> for Canonical<I, V>
 where
     I::CanonicalVars: HashStable<CTX>,
diff --git a/compiler/rustc_type_ir/src/const_kind.rs b/compiler/rustc_type_ir/src/const_kind.rs
index 3c986f64d38d..cf67ba0b21a1 100644
--- a/compiler/rustc_type_ir/src/const_kind.rs
+++ b/compiler/rustc_type_ir/src/const_kind.rs
@@ -2,7 +2,6 @@ use rustc_data_structures::stable_hasher::HashStable;
 use rustc_data_structures::stable_hasher::StableHasher;
 use rustc_serialize::{Decodable, Decoder, Encodable};
 use std::fmt;
-use std::hash;
 
 use crate::{
     DebruijnIndex, DebugWithInfcx, HashStableContext, InferCtxtLike, Interner, TyDecoder,
@@ -18,7 +17,8 @@ use self::ConstKind::*;
     PartialOrd(bound = ""),
     PartialOrd = "feature_allow_slow_enum",
     Ord(bound = ""),
-    Ord = "feature_allow_slow_enum"
+    Ord = "feature_allow_slow_enum",
+    Hash(bound = "")
 )]
 pub enum ConstKind<I: Interner> {
     /// A const generic parameter.
@@ -63,25 +63,6 @@ const fn const_kind_discriminant<I: Interner>(value: &ConstKind<I>) -> usize {
     }
 }
 
-impl<I: Interner> hash::Hash for ConstKind<I> {
-    fn hash<H: hash::Hasher>(&self, state: &mut H) {
-        const_kind_discriminant(self).hash(state);
-        match self {
-            Param(p) => p.hash(state),
-            Infer(i) => i.hash(state),
-            Bound(d, b) => {
-                d.hash(state);
-                b.hash(state);
-            }
-            Placeholder(p) => p.hash(state),
-            Unevaluated(u) => u.hash(state),
-            Value(v) => v.hash(state),
-            Error(e) => e.hash(state),
-            Expr(e) => e.hash(state),
-        }
-    }
-}
-
 impl<CTX: HashStableContext, I: Interner> HashStable<CTX> for ConstKind<I>
 where
     I::ParamConst: HashStable<CTX>,
diff --git a/compiler/rustc_type_ir/src/predicate_kind.rs b/compiler/rustc_type_ir/src/predicate_kind.rs
index 124d9d0eb959..23117fdd5310 100644
--- a/compiler/rustc_type_ir/src/predicate_kind.rs
+++ b/compiler/rustc_type_ir/src/predicate_kind.rs
@@ -2,7 +2,6 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_serialize::Decoder;
 use rustc_serialize::{Decodable, Encodable};
 use std::fmt;
-use std::hash;
 use std::ops::ControlFlow;
 
 use crate::fold::{FallibleTypeFolder, TypeFoldable};
@@ -13,7 +12,7 @@ use crate::{TyDecoder, TyEncoder};
 /// A clause is something that can appear in where bounds or be inferred
 /// by implied bounds.
 #[derive(derivative::Derivative)]
-#[derivative(Clone(bound = ""))]
+#[derivative(Clone(bound = ""), Hash(bound = ""))]
 pub enum ClauseKind<I: Interner> {
     /// Corresponds to `where Foo: Bar<A, B, C>`. `Foo` here would be
     /// the `Self` type of the trait reference and `A`, `B`, and `C`
@@ -82,24 +81,6 @@ fn clause_kind_discriminant<I: Interner>(value: &ClauseKind<I>) -> usize {
     }
 }
 
-impl<I: Interner> hash::Hash for ClauseKind<I> {
-    fn hash<H: hash::Hasher>(&self, state: &mut H) {
-        clause_kind_discriminant(self).hash(state);
-        match self {
-            ClauseKind::Trait(p) => p.hash(state),
-            ClauseKind::RegionOutlives(p) => p.hash(state),
-            ClauseKind::TypeOutlives(p) => p.hash(state),
-            ClauseKind::Projection(p) => p.hash(state),
-            ClauseKind::ConstArgHasType(c, t) => {
-                c.hash(state);
-                t.hash(state);
-            }
-            ClauseKind::WellFormed(t) => t.hash(state),
-            ClauseKind::ConstEvaluatable(c) => c.hash(state),
-        }
-    }
-}
-
 impl<CTX: HashStableContext, I: Interner> HashStable<CTX> for ClauseKind<I>
 where
     I::Ty: HashStable<CTX>,
@@ -238,7 +219,7 @@ where
 }
 
 #[derive(derivative::Derivative)]
-#[derivative(Clone(bound = ""))]
+#[derivative(Clone(bound = ""), Hash(bound = ""))]
 pub enum PredicateKind<I: Interner> {
     /// Prove a clause
     Clause(ClauseKind<I>),
@@ -329,33 +310,6 @@ fn predicate_kind_discriminant<I: Interner>(value: &PredicateKind<I>) -> usize {
     }
 }
 
-impl<I: Interner> hash::Hash for PredicateKind<I> {
-    fn hash<H: hash::Hasher>(&self, state: &mut H) {
-        predicate_kind_discriminant(self).hash(state);
-        match self {
-            PredicateKind::Clause(p) => p.hash(state),
-            PredicateKind::ObjectSafe(d) => d.hash(state),
-            PredicateKind::ClosureKind(d, g, k) => {
-                d.hash(state);
-                g.hash(state);
-                k.hash(state);
-            }
-            PredicateKind::Subtype(p) => p.hash(state),
-            PredicateKind::Coerce(p) => p.hash(state),
-            PredicateKind::ConstEquate(c1, c2) => {
-                c1.hash(state);
-                c2.hash(state);
-            }
-            PredicateKind::Ambiguous => {}
-            PredicateKind::AliasRelate(t1, t2, r) => {
-                t1.hash(state);
-                t2.hash(state);
-                r.hash(state);
-            }
-        }
-    }
-}
-
 impl<CTX: HashStableContext, I: Interner> HashStable<CTX> for PredicateKind<I>
 where
     I::DefId: HashStable<CTX>,
diff --git a/compiler/rustc_type_ir/src/region_kind.rs b/compiler/rustc_type_ir/src/region_kind.rs
index 8dc1cc080058..72f86fc06929 100644
--- a/compiler/rustc_type_ir/src/region_kind.rs
+++ b/compiler/rustc_type_ir/src/region_kind.rs
@@ -2,7 +2,6 @@ use rustc_data_structures::stable_hasher::HashStable;
 use rustc_data_structures::stable_hasher::StableHasher;
 use rustc_serialize::{Decodable, Decoder, Encodable};
 use std::fmt;
-use std::hash;
 
 use crate::{
     DebruijnIndex, DebugWithInfcx, HashStableContext, InferCtxtLike, Interner, TyDecoder,
@@ -123,7 +122,8 @@ use self::RegionKind::*;
     PartialOrd(bound = ""),
     PartialOrd = "feature_allow_slow_enum",
     Ord(bound = ""),
-    Ord = "feature_allow_slow_enum"
+    Ord = "feature_allow_slow_enum",
+    Hash(bound = "")
 )]
 pub enum RegionKind<I: Interner> {
     /// Region bound in a type or fn declaration which will be
@@ -213,26 +213,6 @@ impl<I: Interner> PartialEq for RegionKind<I> {
 // This is manually implemented because a derive would require `I: Eq`
 impl<I: Interner> Eq for RegionKind<I> {}
 
-// This is manually implemented because a derive would require `I: Hash`
-impl<I: Interner> hash::Hash for RegionKind<I> {
-    fn hash<H: hash::Hasher>(&self, state: &mut H) -> () {
-        regionkind_discriminant(self).hash(state);
-        match self {
-            ReEarlyBound(r) => r.hash(state),
-            ReLateBound(d, r) => {
-                d.hash(state);
-                r.hash(state)
-            }
-            ReFree(r) => r.hash(state),
-            ReStatic => (),
-            ReVar(r) => r.hash(state),
-            RePlaceholder(r) => r.hash(state),
-            ReErased => (),
-            ReError(_) => (),
-        }
-    }
-}
-
 impl<I: Interner> DebugWithInfcx<I> for RegionKind<I> {
     fn fmt<Infcx: InferCtxtLike<Interner = I>>(
         this: WithInfcx<'_, Infcx, &Self>,
diff --git a/compiler/rustc_type_ir/src/ty_kind.rs b/compiler/rustc_type_ir/src/ty_kind.rs
index f180ceb7b071..dceb5ba0560b 100644
--- a/compiler/rustc_type_ir/src/ty_kind.rs
+++ b/compiler/rustc_type_ir/src/ty_kind.rs
@@ -3,8 +3,8 @@
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_data_structures::unify::{EqUnifyValue, UnifyKey};
 use rustc_serialize::{Decodable, Decoder, Encodable};
+use std::fmt;
 use std::mem::discriminant;
-use std::{fmt, hash};
 
 use crate::HashStableContext;
 use crate::Interner;
@@ -119,7 +119,8 @@ pub enum AliasKind {
     PartialOrd(bound = ""),
     PartialOrd = "feature_allow_slow_enum",
     Ord(bound = ""),
-    Ord = "feature_allow_slow_enum"
+    Ord = "feature_allow_slow_enum",
+    Hash(bound = "")
 )]
 pub enum TyKind<I: Interner> {
     /// The primitive boolean type. Written as `bool`.
@@ -383,71 +384,6 @@ impl<I: Interner> PartialEq for TyKind<I> {
 // This is manually implemented because a derive would require `I: Eq`
 impl<I: Interner> Eq for TyKind<I> {}
 
-// This is manually implemented because a derive would require `I: Hash`
-impl<I: Interner> hash::Hash for TyKind<I> {
-    fn hash<__H: hash::Hasher>(&self, state: &mut __H) -> () {
-        tykind_discriminant(self).hash(state);
-        match self {
-            Int(i) => i.hash(state),
-            Uint(u) => u.hash(state),
-            Float(f) => f.hash(state),
-            Adt(d, s) => {
-                d.hash(state);
-                s.hash(state)
-            }
-            Foreign(d) => d.hash(state),
-            Array(t, c) => {
-                t.hash(state);
-                c.hash(state)
-            }
-            Slice(t) => t.hash(state),
-            RawPtr(t) => t.hash(state),
-            Ref(r, t, m) => {
-                r.hash(state);
-                t.hash(state);
-                m.hash(state)
-            }
-            FnDef(d, s) => {
-                d.hash(state);
-                s.hash(state)
-            }
-            FnPtr(s) => s.hash(state),
-            Dynamic(p, r, repr) => {
-                p.hash(state);
-                r.hash(state);
-                repr.hash(state)
-            }
-            Closure(d, s) => {
-                d.hash(state);
-                s.hash(state)
-            }
-            Coroutine(d, s, m) => {
-                d.hash(state);
-                s.hash(state);
-                m.hash(state)
-            }
-            CoroutineWitness(d, s) => {
-                d.hash(state);
-                s.hash(state);
-            }
-            Tuple(t) => t.hash(state),
-            Alias(i, p) => {
-                i.hash(state);
-                p.hash(state);
-            }
-            Param(p) => p.hash(state),
-            Bound(d, b) => {
-                d.hash(state);
-                b.hash(state)
-            }
-            Placeholder(p) => p.hash(state),
-            Infer(t) => t.hash(state),
-            Error(e) => e.hash(state),
-            Bool | Char | Str | Never => (),
-        }
-    }
-}
-
 impl<I: Interner> DebugWithInfcx<I> for TyKind<I> {
     fn fmt<Infcx: InferCtxtLike<Interner = I>>(
         this: WithInfcx<'_, Infcx, &Self>,

From 4512f211ae77306b0cd0686a7871d56edbf36d30 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Tue, 31 Oct 2023 13:58:03 +0000
Subject: [PATCH 286/297] Accept less invalid Rust in rustdoc

---
 library/core/src/primitive_docs.rs            |  20 --
 src/librustdoc/core.rs                        |   6 +-
 tests/rustdoc-gui/src/lib2/lib.rs             |   9 +-
 tests/rustdoc-json/enums/field_hidden.rs      |   5 +-
 tests/rustdoc-json/enums/kind.rs              |   5 +-
 .../rustdoc-json/enums/tuple_fields_hidden.rs |   5 +-
 .../generic-associated-types/gats.rs          |   5 +-
 tests/rustdoc-json/impls/auto.rs              |   9 +-
 .../type/inherent_associated_type_bound.rs    |   2 +-
 .../infinite-recursive-type-2.rs              |   3 +-
 .../infinite-recursive-type-2.stderr          |  17 ++
 .../infinite-recursive-type.rs                |   3 +-
 .../infinite-recursive-type.stderr            |  17 ++
 .../invalid_const_in_lifetime_position.rs     |   5 +
 .../invalid_const_in_lifetime_position.stderr |  84 ++++++-
 tests/rustdoc-ui/issues/issue-105742.rs       |   8 +
 tests/rustdoc-ui/issues/issue-105742.stderr   | 234 ++++++++++++++----
 .../not-wf-ambiguous-normalization.rs         |   1 +
 .../not-wf-ambiguous-normalization.stderr     |   9 +
 tests/rustdoc-ui/recursive-deref-ice.rs       |   5 +-
 tests/rustdoc-ui/unable-fulfill-trait.rs      |   3 +-
 tests/rustdoc-ui/unable-fulfill-trait.stderr  |  23 +-
 tests/rustdoc/const-generics/const-impl.rs    |   4 +-
 .../decl-trailing-whitespace.declaration.html |  14 +-
 tests/rustdoc/decl-trailing-whitespace.rs     |   8 +-
 ...ide-complex-unevaluated-const-arguments.rs |  11 +-
 .../intra-doc/prim-associated-traits.rs       |   2 +-
 ...oc-notable_trait_box_is_not_an_iterator.rs |  11 +-
 .../whitespace-after-where-clause.enum.html   |   2 +-
 .../whitespace-after-where-clause.enum2.html  |   2 +-
 .../rustdoc/whitespace-after-where-clause.rs  |  15 +-
 .../whitespace-after-where-clause.struct.html |   2 +-
 ...whitespace-after-where-clause.struct2.html |   2 +-
 .../whitespace-after-where-clause.union.html  |   2 +-
 .../whitespace-after-where-clause.union2.html |   2 +-
 35 files changed, 428 insertions(+), 127 deletions(-)
 create mode 100644 tests/rustdoc-ui/error-in-impl-trait/infinite-recursive-type-2.stderr
 create mode 100644 tests/rustdoc-ui/error-in-impl-trait/infinite-recursive-type.stderr
 rename tests/{rustdoc => rustdoc-ui}/not-wf-ambiguous-normalization.rs (92%)
 create mode 100644 tests/rustdoc-ui/not-wf-ambiguous-normalization.stderr

diff --git a/library/core/src/primitive_docs.rs b/library/core/src/primitive_docs.rs
index 380a21b376bd..f3695d16d7a6 100644
--- a/library/core/src/primitive_docs.rs
+++ b/library/core/src/primitive_docs.rs
@@ -1077,26 +1077,6 @@ mod prim_tuple {}
 #[doc(hidden)]
 impl<T> (T,) {}
 
-// Fake impl that's only really used for docs.
-#[cfg(doc)]
-#[stable(feature = "rust1", since = "1.0.0")]
-#[doc(fake_variadic)]
-/// This trait is implemented on arbitrary-length tuples.
-impl<T: Clone> Clone for (T,) {
-    fn clone(&self) -> Self {
-        loop {}
-    }
-}
-
-// Fake impl that's only really used for docs.
-#[cfg(doc)]
-#[stable(feature = "rust1", since = "1.0.0")]
-#[doc(fake_variadic)]
-/// This trait is implemented on arbitrary-length tuples.
-impl<T: Copy> Copy for (T,) {
-    // empty
-}
-
 #[rustc_doc_primitive = "f32"]
 /// A 32-bit floating point type (specifically, the "binary32" type defined in IEEE 754-2008).
 ///
diff --git a/src/librustdoc/core.rs b/src/librustdoc/core.rs
index 17b117de5d47..6d9f8b820c48 100644
--- a/src/librustdoc/core.rs
+++ b/src/librustdoc/core.rs
@@ -319,10 +319,14 @@ pub(crate) fn run_global_ctxt(
         tcx.hir().for_each_module(|module| tcx.ensure().collect_mod_item_types(module))
     });
 
-    // NOTE: This is copy/pasted from typeck/lib.rs and should be kept in sync with those changes.
+    // NOTE: These are copy/pasted from typeck/lib.rs and should be kept in sync with those changes.
+    let _ = tcx.sess.time("wf_checking", || {
+        tcx.hir().try_par_for_each_module(|module| tcx.ensure().check_mod_type_wf(module))
+    });
     tcx.sess.time("item_types_checking", || {
         tcx.hir().for_each_module(|module| tcx.ensure().check_mod_item_types(module))
     });
+
     tcx.sess.abort_if_errors();
     tcx.sess.time("missing_docs", || rustc_lint::check_crate(tcx));
     tcx.sess.time("check_mod_attrs", || {
diff --git a/tests/rustdoc-gui/src/lib2/lib.rs b/tests/rustdoc-gui/src/lib2/lib.rs
index 34e67d9d2543..a2a3c31878ba 100644
--- a/tests/rustdoc-gui/src/lib2/lib.rs
+++ b/tests/rustdoc-gui/src/lib2/lib.rs
@@ -147,13 +147,13 @@ pub struct LongItemInfo2;
 #[doc(cfg(any(target_os = "android", target_os = "linux", target_os = "emscripten", target_os = "dragonfly", target_os = "freebsd", target_os = "netbsd", target_os = "openbsd")))]
 impl SimpleTrait for LongItemInfo2 {}
 
-pub struct WhereWhitespace<T>;
+pub struct WhereWhitespace<T>(T);
 
 impl<T> WhereWhitespace<T> {
     pub fn new<F>(f: F) -> Self
     where
         F: FnMut() -> i32,
-    {}
+    {todo!()}
 }
 
 impl<K, T> Whitespace<&K> for WhereWhitespace<T>
@@ -187,6 +187,11 @@ impl ItemInfoAlignmentTest {
 pub mod scroll_traits {
     use std::iter::*;
 
+    struct Intersperse<T>(T);
+    struct IntersperseWith<T, U>(T, U);
+    struct Flatten<T>(T);
+    struct Peekable<T>(T);
+
     /// Shamelessly (partially) copied from `std::iter::Iterator`.
     /// It allows us to check that the scroll is working as expected on "hidden" items.
     pub trait Iterator {
diff --git a/tests/rustdoc-json/enums/field_hidden.rs b/tests/rustdoc-json/enums/field_hidden.rs
index 5c0d0ffd3dfb..f24098580579 100644
--- a/tests/rustdoc-json/enums/field_hidden.rs
+++ b/tests/rustdoc-json/enums/field_hidden.rs
@@ -1,7 +1,10 @@
 // Regression test for <https://github.com/rust-lang/rust/issues/100529>.
 
 #![no_core]
-#![feature(no_core)]
+#![feature(no_core, lang_items)]
+
+#[lang = "sized"]
+trait Sized {}
 
 // @has "$.index[*][?(@.name=='ParseError')]"
 // @has "$.index[*][?(@.name=='UnexpectedEndTag')]"
diff --git a/tests/rustdoc-json/enums/kind.rs b/tests/rustdoc-json/enums/kind.rs
index e283c0740065..777161c4e4ba 100644
--- a/tests/rustdoc-json/enums/kind.rs
+++ b/tests/rustdoc-json/enums/kind.rs
@@ -1,8 +1,11 @@
 // ignore-tidy-linelength
 
-#![feature(no_core)]
+#![feature(no_core, lang_items)]
 #![no_core]
 
+#[lang = "sized"]
+trait Sized {}
+
 pub enum Foo {
     // @set Unit = "$.index[*][?(@.name=='Unit')].id"
     // @is "$.index[*][?(@.name=='Unit')].inner.variant.kind" '"plain"'
diff --git a/tests/rustdoc-json/enums/tuple_fields_hidden.rs b/tests/rustdoc-json/enums/tuple_fields_hidden.rs
index 3aeb0356420a..34a4f4aec538 100644
--- a/tests/rustdoc-json/enums/tuple_fields_hidden.rs
+++ b/tests/rustdoc-json/enums/tuple_fields_hidden.rs
@@ -1,6 +1,9 @@
-#![feature(no_core)]
+#![feature(no_core, lang_items)]
 #![no_core]
 
+#[lang = "sized"]
+trait Sized {}
+
 // @set 1.1.0 = "$.index[*][?(@.docs=='1.1.0')].id"
 // @set 2.1.0 = "$.index[*][?(@.docs=='2.1.0')].id"
 // @set 2.1.1 = "$.index[*][?(@.docs=='2.1.1')].id"
diff --git a/tests/rustdoc-json/generic-associated-types/gats.rs b/tests/rustdoc-json/generic-associated-types/gats.rs
index 99c57ff65409..9cfe649243f8 100644
--- a/tests/rustdoc-json/generic-associated-types/gats.rs
+++ b/tests/rustdoc-json/generic-associated-types/gats.rs
@@ -1,11 +1,14 @@
 // ignore-tidy-linelength
 
 #![no_core]
-#![feature(lang_items, no_core)]
+#![feature(lang_items, no_core, arbitrary_self_types)]
 
 #[lang = "sized"]
 pub trait Sized {}
 
+#[lang = "receiver"]
+pub trait Receiver {}
+
 pub trait Display {}
 
 pub trait LendingIterator {
diff --git a/tests/rustdoc-json/impls/auto.rs b/tests/rustdoc-json/impls/auto.rs
index ace37e5b3dfe..96c3ab08b996 100644
--- a/tests/rustdoc-json/impls/auto.rs
+++ b/tests/rustdoc-json/impls/auto.rs
@@ -1,9 +1,12 @@
-#![feature(no_core, auto_traits, lang_items)]
+#![feature(no_core, auto_traits, lang_items, arbitrary_self_types)]
 #![no_core]
 
 #[lang = "sized"]
 trait Sized {}
 
+#[lang = "receiver"]
+pub trait Receiver {}
+
 pub auto trait Bar {}
 
 /// has span
@@ -12,8 +15,8 @@ impl Foo {
 }
 
 // Testing spans, so all tests below code
-// @is "$.index[*][?(@.docs=='has span')].span.begin" "[10, 0]"
-// @is "$.index[*][?(@.docs=='has span')].span.end" "[12, 1]"
+// @is "$.index[*][?(@.docs=='has span')].span.begin" "[13, 0]"
+// @is "$.index[*][?(@.docs=='has span')].span.end" "[15, 1]"
 // FIXME: this doesn't work due to https://github.com/freestrings/jsonpath/issues/91
 // is "$.index[*][?(@.inner.impl.synthetic==true)].span" null
 pub struct Foo;
diff --git a/tests/rustdoc-json/type/inherent_associated_type_bound.rs b/tests/rustdoc-json/type/inherent_associated_type_bound.rs
index 8e39f471824c..6da23c8fb4e7 100644
--- a/tests/rustdoc-json/type/inherent_associated_type_bound.rs
+++ b/tests/rustdoc-json/type/inherent_associated_type_bound.rs
@@ -16,5 +16,5 @@ pub struct Carrier<'a>(&'a ());
 pub fn user(_: for<'b> fn(Carrier<'b>::Focus<i32>)) {}
 
 impl<'a> Carrier<'a> {
-    pub type Focus<T> = &'a mut T;
+    pub type Focus<T> = &'a mut T where T: 'a;
 }
diff --git a/tests/rustdoc-ui/error-in-impl-trait/infinite-recursive-type-2.rs b/tests/rustdoc-ui/error-in-impl-trait/infinite-recursive-type-2.rs
index e49fe0798137..7151ebd599f6 100644
--- a/tests/rustdoc-ui/error-in-impl-trait/infinite-recursive-type-2.rs
+++ b/tests/rustdoc-ui/error-in-impl-trait/infinite-recursive-type-2.rs
@@ -1,7 +1,6 @@
-// check-pass
-
 pub fn f() -> impl Sized {
     pub enum E {
+        //~^ ERROR: recursive type
         V(E),
     }
 
diff --git a/tests/rustdoc-ui/error-in-impl-trait/infinite-recursive-type-2.stderr b/tests/rustdoc-ui/error-in-impl-trait/infinite-recursive-type-2.stderr
new file mode 100644
index 000000000000..edb5dfd4d55e
--- /dev/null
+++ b/tests/rustdoc-ui/error-in-impl-trait/infinite-recursive-type-2.stderr
@@ -0,0 +1,17 @@
+error[E0072]: recursive type `f::E` has infinite size
+  --> $DIR/infinite-recursive-type-2.rs:2:5
+   |
+LL |     pub enum E {
+   |     ^^^^^^^^^^
+LL |
+LL |         V(E),
+   |           - recursive without indirection
+   |
+help: insert some indirection (e.g., a `Box`, `Rc`, or `&`) to break the cycle
+   |
+LL |         V(Box<E>),
+   |           ++++ +
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0072`.
diff --git a/tests/rustdoc-ui/error-in-impl-trait/infinite-recursive-type.rs b/tests/rustdoc-ui/error-in-impl-trait/infinite-recursive-type.rs
index 096130d77682..1f8550517293 100644
--- a/tests/rustdoc-ui/error-in-impl-trait/infinite-recursive-type.rs
+++ b/tests/rustdoc-ui/error-in-impl-trait/infinite-recursive-type.rs
@@ -1,7 +1,6 @@
-// check-pass
-
 fn f() -> impl Sized {
     enum E {
+        //~^ ERROR: recursive type
         V(E),
     }
 
diff --git a/tests/rustdoc-ui/error-in-impl-trait/infinite-recursive-type.stderr b/tests/rustdoc-ui/error-in-impl-trait/infinite-recursive-type.stderr
new file mode 100644
index 000000000000..349a569414cc
--- /dev/null
+++ b/tests/rustdoc-ui/error-in-impl-trait/infinite-recursive-type.stderr
@@ -0,0 +1,17 @@
+error[E0072]: recursive type `f::E` has infinite size
+  --> $DIR/infinite-recursive-type.rs:2:5
+   |
+LL |     enum E {
+   |     ^^^^^^
+LL |
+LL |         V(E),
+   |           - recursive without indirection
+   |
+help: insert some indirection (e.g., a `Box`, `Rc`, or `&`) to break the cycle
+   |
+LL |         V(Box<E>),
+   |           ++++ +
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0072`.
diff --git a/tests/rustdoc-ui/invalid_const_in_lifetime_position.rs b/tests/rustdoc-ui/invalid_const_in_lifetime_position.rs
index c3f4fd63bac7..07fc239a8f86 100644
--- a/tests/rustdoc-ui/invalid_const_in_lifetime_position.rs
+++ b/tests/rustdoc-ui/invalid_const_in_lifetime_position.rs
@@ -4,3 +4,8 @@ trait X {
 fn f<'a>(arg : Box<dyn X<Y<1> = &'a ()>>) {}
 //~^ ERROR associated type takes 1 lifetime argument but 0 lifetime arguments
 //~| ERROR associated type takes 0 generic arguments but 1 generic argument
+//~| ERROR associated type takes 1 lifetime argument but 0 lifetime arguments
+//~| ERROR associated type takes 0 generic arguments but 1 generic argument
+//~| ERROR associated type takes 1 lifetime argument but 0 lifetime arguments
+//~| ERROR associated type takes 0 generic arguments but 1 generic argument
+//~| ERROR trait `X` cannot be made into an object
diff --git a/tests/rustdoc-ui/invalid_const_in_lifetime_position.stderr b/tests/rustdoc-ui/invalid_const_in_lifetime_position.stderr
index 527729a82286..50d55284754e 100644
--- a/tests/rustdoc-ui/invalid_const_in_lifetime_position.stderr
+++ b/tests/rustdoc-ui/invalid_const_in_lifetime_position.stderr
@@ -28,6 +28,86 @@ note: associated type defined here, with 0 generic parameters
 LL |     type Y<'a>;
    |          ^
 
-error: aborting due to 2 previous errors
+error[E0107]: associated type takes 1 lifetime argument but 0 lifetime arguments were supplied
+  --> $DIR/invalid_const_in_lifetime_position.rs:4:26
+   |
+LL | fn f<'a>(arg : Box<dyn X<Y<1> = &'a ()>>) {}
+   |                          ^ expected 1 lifetime argument
+   |
+note: associated type defined here, with 1 lifetime parameter: `'a`
+  --> $DIR/invalid_const_in_lifetime_position.rs:2:10
+   |
+LL |     type Y<'a>;
+   |          ^ --
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+help: add missing lifetime argument
+   |
+LL | fn f<'a>(arg : Box<dyn X<Y<'_, 1> = &'a ()>>) {}
+   |                            +++
 
-For more information about this error, try `rustc --explain E0107`.
+error[E0107]: associated type takes 0 generic arguments but 1 generic argument was supplied
+  --> $DIR/invalid_const_in_lifetime_position.rs:4:26
+   |
+LL | fn f<'a>(arg : Box<dyn X<Y<1> = &'a ()>>) {}
+   |                          ^--- help: remove these generics
+   |                          |
+   |                          expected 0 generic arguments
+   |
+note: associated type defined here, with 0 generic parameters
+  --> $DIR/invalid_const_in_lifetime_position.rs:2:10
+   |
+LL |     type Y<'a>;
+   |          ^
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+
+error[E0107]: associated type takes 1 lifetime argument but 0 lifetime arguments were supplied
+  --> $DIR/invalid_const_in_lifetime_position.rs:4:26
+   |
+LL | fn f<'a>(arg : Box<dyn X<Y<1> = &'a ()>>) {}
+   |                          ^ expected 1 lifetime argument
+   |
+note: associated type defined here, with 1 lifetime parameter: `'a`
+  --> $DIR/invalid_const_in_lifetime_position.rs:2:10
+   |
+LL |     type Y<'a>;
+   |          ^ --
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+help: add missing lifetime argument
+   |
+LL | fn f<'a>(arg : Box<dyn X<Y<'_, 1> = &'a ()>>) {}
+   |                            +++
+
+error[E0107]: associated type takes 0 generic arguments but 1 generic argument was supplied
+  --> $DIR/invalid_const_in_lifetime_position.rs:4:26
+   |
+LL | fn f<'a>(arg : Box<dyn X<Y<1> = &'a ()>>) {}
+   |                          ^--- help: remove these generics
+   |                          |
+   |                          expected 0 generic arguments
+   |
+note: associated type defined here, with 0 generic parameters
+  --> $DIR/invalid_const_in_lifetime_position.rs:2:10
+   |
+LL |     type Y<'a>;
+   |          ^
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+
+error[E0038]: the trait `X` cannot be made into an object
+  --> $DIR/invalid_const_in_lifetime_position.rs:4:20
+   |
+LL | fn f<'a>(arg : Box<dyn X<Y<1> = &'a ()>>) {}
+   |                    ^^^^^^^^^^^^^^^^^^^^ `X` cannot be made into an object
+   |
+note: for a trait to be "object safe" it needs to allow building a vtable to allow the call to be resolvable dynamically; for more information visit <https://doc.rust-lang.org/reference/items/traits.html#object-safety>
+  --> $DIR/invalid_const_in_lifetime_position.rs:2:10
+   |
+LL | trait X {
+   |       - this trait cannot be made into an object...
+LL |     type Y<'a>;
+   |          ^ ...because it contains the generic associated type `Y`
+   = help: consider moving `Y` to another trait
+
+error: aborting due to 7 previous errors
+
+Some errors have detailed explanations: E0038, E0107.
+For more information about an error, try `rustc --explain E0038`.
diff --git a/tests/rustdoc-ui/issues/issue-105742.rs b/tests/rustdoc-ui/issues/issue-105742.rs
index 1fbb70c7808e..5e493515cad5 100644
--- a/tests/rustdoc-ui/issues/issue-105742.rs
+++ b/tests/rustdoc-ui/issues/issue-105742.rs
@@ -21,6 +21,8 @@ pub trait SVec: Index<
     //~| missing generics for associated type `SVec::Item`
     //~| missing generics for associated type `SVec::Item`
     //~| missing generics for associated type `SVec::Item`
+    //~| missing generics for associated type `SVec::Item`
+    //~| missing generics for associated type `SVec::Item`
     Output = <Index<<Self as SVec>::Item,
     //~^ expected 1 lifetime argument
     //~| expected 1 generic argument
@@ -30,6 +32,8 @@ pub trait SVec: Index<
     //~| missing generics for associated type `SVec::Item`
     //~| missing generics for associated type `SVec::Item`
     //~| missing generics for associated type `SVec::Item`
+    //~| missing generics for associated type `SVec::Item`
+    //~| missing generics for associated type `SVec::Item`
     Output = <Self as SVec>::Item> as SVec>::Item,
     //~^ expected 1 lifetime argument
     //~| expected 1 generic argument
@@ -47,6 +51,10 @@ pub trait SVec: Index<
     //~| missing generics for associated type `SVec::Item`
     //~| missing generics for associated type `SVec::Item`
     //~| missing generics for associated type `SVec::Item`
+    //~| missing generics for associated type `SVec::Item`
+    //~| missing generics for associated type `SVec::Item`
+    //~| missing generics for associated type `SVec::Item`
+    //~| missing generics for associated type `SVec::Item`
 > {
     type Item<'a, T>;
 
diff --git a/tests/rustdoc-ui/issues/issue-105742.stderr b/tests/rustdoc-ui/issues/issue-105742.stderr
index 4d3f9f5bced7..ad1020a1f081 100644
--- a/tests/rustdoc-ui/issues/issue-105742.stderr
+++ b/tests/rustdoc-ui/issues/issue-105742.stderr
@@ -5,7 +5,7 @@ LL |     <Self as SVec>::Item,
    |                     ^^^^ expected 1 lifetime argument
    |
 note: associated type defined here, with 1 lifetime parameter: `'a`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^ --
@@ -21,7 +21,7 @@ LL |     <Self as SVec>::Item,
    |                     ^^^^ expected 1 generic argument
    |
 note: associated type defined here, with 1 generic parameter: `T`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^     -
@@ -31,13 +31,13 @@ LL |     <Self as SVec>::Item<T>,
    |                         +++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:24:37
+  --> $DIR/issue-105742.rs:26:37
    |
 LL |     Output = <Index<<Self as SVec>::Item,
    |                                     ^^^^ expected 1 lifetime argument
    |
 note: associated type defined here, with 1 lifetime parameter: `'a`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^ --
@@ -47,13 +47,13 @@ LL |     Output = <Index<<Self as SVec>::Item<'a>,
    |                                         ++++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:24:37
+  --> $DIR/issue-105742.rs:26:37
    |
 LL |     Output = <Index<<Self as SVec>::Item,
    |                                     ^^^^ expected 1 generic argument
    |
 note: associated type defined here, with 1 generic parameter: `T`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^     -
@@ -63,13 +63,13 @@ LL |     Output = <Index<<Self as SVec>::Item<T>,
    |                                         +++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:33:30
+  --> $DIR/issue-105742.rs:37:30
    |
 LL |     Output = <Self as SVec>::Item> as SVec>::Item,
    |                              ^^^^ expected 1 lifetime argument
    |
 note: associated type defined here, with 1 lifetime parameter: `'a`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^ --
@@ -79,13 +79,13 @@ LL |     Output = <Self as SVec>::Item<'a>> as SVec>::Item,
    |                                  ++++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:33:30
+  --> $DIR/issue-105742.rs:37:30
    |
 LL |     Output = <Self as SVec>::Item> as SVec>::Item,
    |                              ^^^^ expected 1 generic argument
    |
 note: associated type defined here, with 1 generic parameter: `T`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^     -
@@ -95,13 +95,13 @@ LL |     Output = <Self as SVec>::Item<T>> as SVec>::Item,
    |                                  +++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:33:46
+  --> $DIR/issue-105742.rs:37:46
    |
 LL |     Output = <Self as SVec>::Item> as SVec>::Item,
    |                                              ^^^^ expected 1 lifetime argument
    |
 note: associated type defined here, with 1 lifetime parameter: `'a`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^ --
@@ -111,13 +111,13 @@ LL |     Output = <Self as SVec>::Item> as SVec>::Item<'a>,
    |                                                  ++++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:33:46
+  --> $DIR/issue-105742.rs:37:46
    |
 LL |     Output = <Self as SVec>::Item> as SVec>::Item,
    |                                              ^^^^ expected 1 generic argument
    |
 note: associated type defined here, with 1 generic parameter: `T`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^     -
@@ -133,7 +133,7 @@ LL | pub fn next<'a, T>(s: &'a mut dyn SVec<Item = T, Output = T>) {
    |                                        ^^^^ expected 1 lifetime argument
    |
 note: associated type defined here, with 1 lifetime parameter: `'a`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^ --
@@ -149,7 +149,7 @@ LL | pub fn next<'a, T>(s: &'a mut dyn SVec<Item = T, Output = T>) {
    |                                        ^^^^ expected 1 generic argument
    |
 note: associated type defined here, with 1 generic parameter: `T`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^     -
@@ -165,7 +165,7 @@ LL |     <Self as SVec>::Item,
    |                     ^^^^ expected 1 lifetime argument
    |
 note: associated type defined here, with 1 lifetime parameter: `'a`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^ --
@@ -182,7 +182,7 @@ LL |     <Self as SVec>::Item,
    |                     ^^^^ expected 1 generic argument
    |
 note: associated type defined here, with 1 generic parameter: `T`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^     -
@@ -193,13 +193,13 @@ LL |     <Self as SVec>::Item<T>,
    |                         +++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:24:37
+  --> $DIR/issue-105742.rs:26:37
    |
 LL |     Output = <Index<<Self as SVec>::Item,
    |                                     ^^^^ expected 1 lifetime argument
    |
 note: associated type defined here, with 1 lifetime parameter: `'a`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^ --
@@ -210,13 +210,13 @@ LL |     Output = <Index<<Self as SVec>::Item<'a>,
    |                                         ++++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:24:37
+  --> $DIR/issue-105742.rs:26:37
    |
 LL |     Output = <Index<<Self as SVec>::Item,
    |                                     ^^^^ expected 1 generic argument
    |
 note: associated type defined here, with 1 generic parameter: `T`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^     -
@@ -227,13 +227,13 @@ LL |     Output = <Index<<Self as SVec>::Item<T>,
    |                                         +++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:33:30
+  --> $DIR/issue-105742.rs:37:30
    |
 LL |     Output = <Self as SVec>::Item> as SVec>::Item,
    |                              ^^^^ expected 1 lifetime argument
    |
 note: associated type defined here, with 1 lifetime parameter: `'a`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^ --
@@ -244,13 +244,13 @@ LL |     Output = <Self as SVec>::Item<'a>> as SVec>::Item,
    |                                  ++++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:33:30
+  --> $DIR/issue-105742.rs:37:30
    |
 LL |     Output = <Self as SVec>::Item> as SVec>::Item,
    |                              ^^^^ expected 1 generic argument
    |
 note: associated type defined here, with 1 generic parameter: `T`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^     -
@@ -261,13 +261,13 @@ LL |     Output = <Self as SVec>::Item<T>> as SVec>::Item,
    |                                  +++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:33:46
+  --> $DIR/issue-105742.rs:37:46
    |
 LL |     Output = <Self as SVec>::Item> as SVec>::Item,
    |                                              ^^^^ expected 1 lifetime argument
    |
 note: associated type defined here, with 1 lifetime parameter: `'a`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^ --
@@ -278,13 +278,13 @@ LL |     Output = <Self as SVec>::Item> as SVec>::Item<'a>,
    |                                                  ++++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:33:46
+  --> $DIR/issue-105742.rs:37:46
    |
 LL |     Output = <Self as SVec>::Item> as SVec>::Item,
    |                                              ^^^^ expected 1 generic argument
    |
 note: associated type defined here, with 1 generic parameter: `T`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^     -
@@ -331,7 +331,7 @@ LL |     <Self as SVec>::Item,
    |                     ^^^^ expected 1 lifetime argument
    |
 note: associated type defined here, with 1 lifetime parameter: `'a`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^ --
@@ -348,7 +348,7 @@ LL |     <Self as SVec>::Item,
    |                     ^^^^ expected 1 generic argument
    |
 note: associated type defined here, with 1 generic parameter: `T`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^     -
@@ -359,13 +359,13 @@ LL |     <Self as SVec>::Item<T>,
    |                         +++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:24:37
+  --> $DIR/issue-105742.rs:26:37
    |
 LL |     Output = <Index<<Self as SVec>::Item,
    |                                     ^^^^ expected 1 lifetime argument
    |
 note: associated type defined here, with 1 lifetime parameter: `'a`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^ --
@@ -376,13 +376,13 @@ LL |     Output = <Index<<Self as SVec>::Item<'a>,
    |                                         ++++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:24:37
+  --> $DIR/issue-105742.rs:26:37
    |
 LL |     Output = <Index<<Self as SVec>::Item,
    |                                     ^^^^ expected 1 generic argument
    |
 note: associated type defined here, with 1 generic parameter: `T`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^     -
@@ -393,13 +393,13 @@ LL |     Output = <Index<<Self as SVec>::Item<T>,
    |                                         +++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:33:30
+  --> $DIR/issue-105742.rs:37:30
    |
 LL |     Output = <Self as SVec>::Item> as SVec>::Item,
    |                              ^^^^ expected 1 lifetime argument
    |
 note: associated type defined here, with 1 lifetime parameter: `'a`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^ --
@@ -410,13 +410,13 @@ LL |     Output = <Self as SVec>::Item<'a>> as SVec>::Item,
    |                                  ++++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:33:30
+  --> $DIR/issue-105742.rs:37:30
    |
 LL |     Output = <Self as SVec>::Item> as SVec>::Item,
    |                              ^^^^ expected 1 generic argument
    |
 note: associated type defined here, with 1 generic parameter: `T`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^     -
@@ -427,13 +427,13 @@ LL |     Output = <Self as SVec>::Item<T>> as SVec>::Item,
    |                                  +++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:33:46
+  --> $DIR/issue-105742.rs:37:46
    |
 LL |     Output = <Self as SVec>::Item> as SVec>::Item,
    |                                              ^^^^ expected 1 lifetime argument
    |
 note: associated type defined here, with 1 lifetime parameter: `'a`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^ --
@@ -444,13 +444,13 @@ LL |     Output = <Self as SVec>::Item> as SVec>::Item<'a>,
    |                                                  ++++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:33:46
+  --> $DIR/issue-105742.rs:37:46
    |
 LL |     Output = <Self as SVec>::Item> as SVec>::Item,
    |                                              ^^^^ expected 1 generic argument
    |
 note: associated type defined here, with 1 generic parameter: `T`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^     -
@@ -461,13 +461,13 @@ LL |     Output = <Self as SVec>::Item> as SVec>::Item<T>,
    |                                                  +++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:53:38
+  --> $DIR/issue-105742.rs:61:38
    |
 LL |     fn len(&self) -> <Self as SVec>::Item;
    |                                      ^^^^ expected 1 lifetime argument
    |
 note: associated type defined here, with 1 lifetime parameter: `'a`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^ --
@@ -477,13 +477,13 @@ LL |     fn len(&self) -> <Self as SVec>::Item<'_>;
    |                                          ++++
 
 error[E0107]: missing generics for associated type `SVec::Item`
-  --> $DIR/issue-105742.rs:53:38
+  --> $DIR/issue-105742.rs:61:38
    |
 LL |     fn len(&self) -> <Self as SVec>::Item;
    |                                      ^^^^ expected 1 generic argument
    |
 note: associated type defined here, with 1 generic parameter: `T`
-  --> $DIR/issue-105742.rs:51:10
+  --> $DIR/issue-105742.rs:59:10
    |
 LL |     type Item<'a, T>;
    |          ^^^^     -
@@ -492,7 +492,143 @@ help: add missing generic argument
 LL |     fn len(&self) -> <Self as SVec>::Item<T>;
    |                                          +++
 
-error: aborting due to 29 previous errors
+error[E0107]: missing generics for associated type `SVec::Item`
+  --> $DIR/issue-105742.rs:15:21
+   |
+LL |     <Self as SVec>::Item,
+   |                     ^^^^ expected 1 lifetime argument
+   |
+note: associated type defined here, with 1 lifetime parameter: `'a`
+  --> $DIR/issue-105742.rs:59:10
+   |
+LL |     type Item<'a, T>;
+   |          ^^^^ --
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+help: add missing lifetime argument
+   |
+LL |     <Self as SVec>::Item<'a>,
+   |                         ++++
+
+error[E0107]: missing generics for associated type `SVec::Item`
+  --> $DIR/issue-105742.rs:15:21
+   |
+LL |     <Self as SVec>::Item,
+   |                     ^^^^ expected 1 generic argument
+   |
+note: associated type defined here, with 1 generic parameter: `T`
+  --> $DIR/issue-105742.rs:59:10
+   |
+LL |     type Item<'a, T>;
+   |          ^^^^     -
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+help: add missing generic argument
+   |
+LL |     <Self as SVec>::Item<T>,
+   |                         +++
+
+error[E0107]: missing generics for associated type `SVec::Item`
+  --> $DIR/issue-105742.rs:26:37
+   |
+LL |     Output = <Index<<Self as SVec>::Item,
+   |                                     ^^^^ expected 1 lifetime argument
+   |
+note: associated type defined here, with 1 lifetime parameter: `'a`
+  --> $DIR/issue-105742.rs:59:10
+   |
+LL |     type Item<'a, T>;
+   |          ^^^^ --
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+help: add missing lifetime argument
+   |
+LL |     Output = <Index<<Self as SVec>::Item<'a>,
+   |                                         ++++
+
+error[E0107]: missing generics for associated type `SVec::Item`
+  --> $DIR/issue-105742.rs:26:37
+   |
+LL |     Output = <Index<<Self as SVec>::Item,
+   |                                     ^^^^ expected 1 generic argument
+   |
+note: associated type defined here, with 1 generic parameter: `T`
+  --> $DIR/issue-105742.rs:59:10
+   |
+LL |     type Item<'a, T>;
+   |          ^^^^     -
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+help: add missing generic argument
+   |
+LL |     Output = <Index<<Self as SVec>::Item<T>,
+   |                                         +++
+
+error[E0107]: missing generics for associated type `SVec::Item`
+  --> $DIR/issue-105742.rs:37:30
+   |
+LL |     Output = <Self as SVec>::Item> as SVec>::Item,
+   |                              ^^^^ expected 1 lifetime argument
+   |
+note: associated type defined here, with 1 lifetime parameter: `'a`
+  --> $DIR/issue-105742.rs:59:10
+   |
+LL |     type Item<'a, T>;
+   |          ^^^^ --
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+help: add missing lifetime argument
+   |
+LL |     Output = <Self as SVec>::Item<'a>> as SVec>::Item,
+   |                                  ++++
+
+error[E0107]: missing generics for associated type `SVec::Item`
+  --> $DIR/issue-105742.rs:37:30
+   |
+LL |     Output = <Self as SVec>::Item> as SVec>::Item,
+   |                              ^^^^ expected 1 generic argument
+   |
+note: associated type defined here, with 1 generic parameter: `T`
+  --> $DIR/issue-105742.rs:59:10
+   |
+LL |     type Item<'a, T>;
+   |          ^^^^     -
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+help: add missing generic argument
+   |
+LL |     Output = <Self as SVec>::Item<T>> as SVec>::Item,
+   |                                  +++
+
+error[E0107]: missing generics for associated type `SVec::Item`
+  --> $DIR/issue-105742.rs:37:46
+   |
+LL |     Output = <Self as SVec>::Item> as SVec>::Item,
+   |                                              ^^^^ expected 1 lifetime argument
+   |
+note: associated type defined here, with 1 lifetime parameter: `'a`
+  --> $DIR/issue-105742.rs:59:10
+   |
+LL |     type Item<'a, T>;
+   |          ^^^^ --
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+help: add missing lifetime argument
+   |
+LL |     Output = <Self as SVec>::Item> as SVec>::Item<'a>,
+   |                                                  ++++
+
+error[E0107]: missing generics for associated type `SVec::Item`
+  --> $DIR/issue-105742.rs:37:46
+   |
+LL |     Output = <Self as SVec>::Item> as SVec>::Item,
+   |                                              ^^^^ expected 1 generic argument
+   |
+note: associated type defined here, with 1 generic parameter: `T`
+  --> $DIR/issue-105742.rs:59:10
+   |
+LL |     type Item<'a, T>;
+   |          ^^^^     -
+   = note: duplicate diagnostic emitted due to `-Z deduplicate-diagnostics=no`
+help: add missing generic argument
+   |
+LL |     Output = <Self as SVec>::Item> as SVec>::Item<T>,
+   |                                                  +++
+
+error: aborting due to 37 previous errors
 
 Some errors have detailed explanations: E0038, E0107.
 For more information about an error, try `rustc --explain E0038`.
diff --git a/tests/rustdoc/not-wf-ambiguous-normalization.rs b/tests/rustdoc-ui/not-wf-ambiguous-normalization.rs
similarity index 92%
rename from tests/rustdoc/not-wf-ambiguous-normalization.rs
rename to tests/rustdoc-ui/not-wf-ambiguous-normalization.rs
index 1e9f925f8458..3e4825d83b1f 100644
--- a/tests/rustdoc/not-wf-ambiguous-normalization.rs
+++ b/tests/rustdoc-ui/not-wf-ambiguous-normalization.rs
@@ -12,6 +12,7 @@ struct DefaultAllocator;
 // `<DefaultAllocator as Allocator>::Buffer` to be ambiguous,
 // which caused an ICE with `-Znormalize-docs`.
 impl<T> Allocator for DefaultAllocator {
+    //~^ ERROR: type annotations needed
     type Buffer = ();
 }
 
diff --git a/tests/rustdoc-ui/not-wf-ambiguous-normalization.stderr b/tests/rustdoc-ui/not-wf-ambiguous-normalization.stderr
new file mode 100644
index 000000000000..34b20a0b32ce
--- /dev/null
+++ b/tests/rustdoc-ui/not-wf-ambiguous-normalization.stderr
@@ -0,0 +1,9 @@
+error[E0282]: type annotations needed
+  --> $DIR/not-wf-ambiguous-normalization.rs:14:23
+   |
+LL | impl<T> Allocator for DefaultAllocator {
+   |                       ^^^^^^^^^^^^^^^^ cannot infer type for type parameter `T`
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0282`.
diff --git a/tests/rustdoc-ui/recursive-deref-ice.rs b/tests/rustdoc-ui/recursive-deref-ice.rs
index c44fd27f4030..9e62841f99be 100644
--- a/tests/rustdoc-ui/recursive-deref-ice.rs
+++ b/tests/rustdoc-ui/recursive-deref-ice.rs
@@ -4,7 +4,10 @@
 
 pub struct Attribute;
 
-pub struct Map<'hir> {}
+pub struct Map<'hir> {
+    lt: &'hir (),
+}
+
 impl<'hir> Map<'hir> {
     pub fn attrs(&self) -> &'hir [Attribute] { &[] }
 }
diff --git a/tests/rustdoc-ui/unable-fulfill-trait.rs b/tests/rustdoc-ui/unable-fulfill-trait.rs
index 703570822480..10887ab19030 100644
--- a/tests/rustdoc-ui/unable-fulfill-trait.rs
+++ b/tests/rustdoc-ui/unable-fulfill-trait.rs
@@ -3,7 +3,8 @@
 pub struct Foo<'a, 'b, T> {
     field1: dyn Bar<'a, 'b,>,
     //~^ ERROR
-    //~^^ ERROR
+    //~| ERROR
+    //~| ERROR
 }
 
 pub trait Bar<'x, 's, U>
diff --git a/tests/rustdoc-ui/unable-fulfill-trait.stderr b/tests/rustdoc-ui/unable-fulfill-trait.stderr
index 72f35cb92244..d7735a4fd112 100644
--- a/tests/rustdoc-ui/unable-fulfill-trait.stderr
+++ b/tests/rustdoc-ui/unable-fulfill-trait.stderr
@@ -5,7 +5,7 @@ LL |     field1: dyn Bar<'a, 'b,>,
    |                 ^^^ expected 1 generic argument
    |
 note: trait defined here, with 1 generic parameter: `U`
-  --> $DIR/unable-fulfill-trait.rs:9:11
+  --> $DIR/unable-fulfill-trait.rs:10:11
    |
 LL | pub trait Bar<'x, 's, U>
    |           ^^^         -
@@ -20,7 +20,24 @@ error[E0227]: ambiguous lifetime bound, explicit lifetime bound required
 LL |     field1: dyn Bar<'a, 'b,>,
    |             ^^^^^^^^^^^^^^^^
 
-error: aborting due to 2 previous errors
+error[E0478]: lifetime bound not satisfied
+  --> $DIR/unable-fulfill-trait.rs:4:13
+   |
+LL |     field1: dyn Bar<'a, 'b,>,
+   |             ^^^^^^^^^^^^^^^^
+   |
+note: lifetime parameter instantiated with the lifetime `'b` as defined here
+  --> $DIR/unable-fulfill-trait.rs:3:20
+   |
+LL | pub struct Foo<'a, 'b, T> {
+   |                    ^^
+note: but lifetime parameter must outlive the lifetime `'a` as defined here
+  --> $DIR/unable-fulfill-trait.rs:3:16
+   |
+LL | pub struct Foo<'a, 'b, T> {
+   |                ^^
 
-Some errors have detailed explanations: E0107, E0227.
+error: aborting due to 3 previous errors
+
+Some errors have detailed explanations: E0107, E0227, E0478.
 For more information about an error, try `rustc --explain E0107`.
diff --git a/tests/rustdoc/const-generics/const-impl.rs b/tests/rustdoc/const-generics/const-impl.rs
index b424ea4b33ce..f4eefcc1c33b 100644
--- a/tests/rustdoc/const-generics/const-impl.rs
+++ b/tests/rustdoc/const-generics/const-impl.rs
@@ -2,7 +2,9 @@
 #![feature(adt_const_params)]
 #![crate_name = "foo"]
 
-#[derive(PartialEq, Eq)]
+use std::marker::ConstParamTy;
+
+#[derive(PartialEq, Eq, ConstParamTy)]
 pub enum Order {
     Sorted,
     Unsorted,
diff --git a/tests/rustdoc/decl-trailing-whitespace.declaration.html b/tests/rustdoc/decl-trailing-whitespace.declaration.html
index d73393633f3b..59c318c16f3b 100644
--- a/tests/rustdoc/decl-trailing-whitespace.declaration.html
+++ b/tests/rustdoc/decl-trailing-whitespace.declaration.html
@@ -1,22 +1,16 @@
 <code>pub trait Write {
     // Required methods
     fn <a href="#tymethod.poll_write" class="fn">poll_write</a>(
-        self: <a class="enum" href="{{channel}}/core/option/enum.Option.html" title="enum core::option::Option">Option</a>&lt;<a class="struct" href="{{channel}}/alloc/string/struct.String.html" title="struct alloc::string::String">String</a>&gt;,
+        self,
         cx: &amp;mut <a class="enum" href="{{channel}}/core/option/enum.Option.html" title="enum core::option::Option">Option</a>&lt;<a class="struct" href="{{channel}}/alloc/string/struct.String.html" title="struct alloc::string::String">String</a>&gt;,
         buf: &amp;mut [<a class="primitive" href="{{channel}}/std/primitive.usize.html">usize</a>]
     ) -&gt; <a class="enum" href="{{channel}}/core/option/enum.Option.html" title="enum core::option::Option">Option</a>&lt;<a class="enum" href="{{channel}}/core/result/enum.Result.html" title="enum core::result::Result">Result</a>&lt;<a class="primitive" href="{{channel}}/std/primitive.usize.html">usize</a>, <a class="struct" href="struct.Error.html" title="struct foo::Error">Error</a>&gt;&gt;;
-<span class="item-spacer" />    fn <a href="#tymethod.poll_flush" class="fn">poll_flush</a>(
-        self: <a class="enum" href="{{channel}}/core/option/enum.Option.html" title="enum core::option::Option">Option</a>&lt;<a class="struct" href="{{channel}}/alloc/string/struct.String.html" title="struct alloc::string::String">String</a>&gt;,
-        cx: &amp;mut <a class="enum" href="{{channel}}/core/option/enum.Option.html" title="enum core::option::Option">Option</a>&lt;<a class="struct" href="{{channel}}/alloc/string/struct.String.html" title="struct alloc::string::String">String</a>&gt;
-    ) -&gt; <a class="enum" href="{{channel}}/core/option/enum.Option.html" title="enum core::option::Option">Option</a>&lt;<a class="enum" href="{{channel}}/core/result/enum.Result.html" title="enum core::result::Result">Result</a>&lt;<a class="primitive" href="{{channel}}/std/primitive.unit.html">()</a>, <a class="struct" href="struct.Error.html" title="struct foo::Error">Error</a>&gt;&gt;;
-<span class="item-spacer" />    fn <a href="#tymethod.poll_close" class="fn">poll_close</a>(
-        self: <a class="enum" href="{{channel}}/core/option/enum.Option.html" title="enum core::option::Option">Option</a>&lt;<a class="struct" href="{{channel}}/alloc/string/struct.String.html" title="struct alloc::string::String">String</a>&gt;,
-        cx: &amp;mut <a class="enum" href="{{channel}}/core/option/enum.Option.html" title="enum core::option::Option">Option</a>&lt;<a class="struct" href="{{channel}}/alloc/string/struct.String.html" title="struct alloc::string::String">String</a>&gt;
-    ) -&gt; <a class="enum" href="{{channel}}/core/option/enum.Option.html" title="enum core::option::Option">Option</a>&lt;<a class="enum" href="{{channel}}/core/result/enum.Result.html" title="enum core::result::Result">Result</a>&lt;<a class="primitive" href="{{channel}}/std/primitive.unit.html">()</a>, <a class="struct" href="struct.Error.html" title="struct foo::Error">Error</a>&gt;&gt;;
+<span class="item-spacer" />    fn <a href="#tymethod.poll_flush" class="fn">poll_flush</a>(self, cx: &amp;mut <a class="enum" href="{{channel}}/core/option/enum.Option.html" title="enum core::option::Option">Option</a>&lt;<a class="struct" href="{{channel}}/alloc/string/struct.String.html" title="struct alloc::string::String">String</a>&gt;) -&gt; <a class="enum" href="{{channel}}/core/option/enum.Option.html" title="enum core::option::Option">Option</a>&lt;<a class="enum" href="{{channel}}/core/result/enum.Result.html" title="enum core::result::Result">Result</a>&lt;<a class="primitive" href="{{channel}}/std/primitive.unit.html">()</a>, <a class="struct" href="struct.Error.html" title="struct foo::Error">Error</a>&gt;&gt;;
+<span class="item-spacer" />    fn <a href="#tymethod.poll_close" class="fn">poll_close</a>(self, cx: &amp;mut <a class="enum" href="{{channel}}/core/option/enum.Option.html" title="enum core::option::Option">Option</a>&lt;<a class="struct" href="{{channel}}/alloc/string/struct.String.html" title="struct alloc::string::String">String</a>&gt;) -&gt; <a class="enum" href="{{channel}}/core/option/enum.Option.html" title="enum core::option::Option">Option</a>&lt;<a class="enum" href="{{channel}}/core/result/enum.Result.html" title="enum core::result::Result">Result</a>&lt;<a class="primitive" href="{{channel}}/std/primitive.unit.html">()</a>, <a class="struct" href="struct.Error.html" title="struct foo::Error">Error</a>&gt;&gt;;
 
     // Provided method
     fn <a href="#method.poll_write_vectored" class="fn">poll_write_vectored</a>(
-        self: <a class="enum" href="{{channel}}/core/option/enum.Option.html" title="enum core::option::Option">Option</a>&lt;<a class="struct" href="{{channel}}/alloc/string/struct.String.html" title="struct alloc::string::String">String</a>&gt;,
+        self,
         cx: &amp;mut <a class="enum" href="{{channel}}/core/option/enum.Option.html" title="enum core::option::Option">Option</a>&lt;<a class="struct" href="{{channel}}/alloc/string/struct.String.html" title="struct alloc::string::String">String</a>&gt;,
         bufs: &amp;[<a class="primitive" href="{{channel}}/std/primitive.usize.html">usize</a>]
     ) -&gt; <a class="enum" href="{{channel}}/core/option/enum.Option.html" title="enum core::option::Option">Option</a>&lt;<a class="enum" href="{{channel}}/core/result/enum.Result.html" title="enum core::result::Result">Result</a>&lt;<a class="primitive" href="{{channel}}/std/primitive.usize.html">usize</a>, <a class="struct" href="struct.Error.html" title="struct foo::Error">Error</a>&gt;&gt; { ... }
diff --git a/tests/rustdoc/decl-trailing-whitespace.rs b/tests/rustdoc/decl-trailing-whitespace.rs
index d2a12435d8f2..667837743088 100644
--- a/tests/rustdoc/decl-trailing-whitespace.rs
+++ b/tests/rustdoc/decl-trailing-whitespace.rs
@@ -9,21 +9,21 @@ pub struct Error;
 pub trait Write {
     // @snapshot 'declaration' - '//*[@class="rust item-decl"]//code'
     fn poll_write(
-        self: Option<String>,
+        self,
         cx: &mut Option<String>,
         buf: &mut [usize]
     ) -> Option<Result<usize, Error>>;
     fn poll_flush(
-        self: Option<String>,
+        self,
         cx: &mut Option<String>
     ) -> Option<Result<(), Error>>;
     fn poll_close(
-        self: Option<String>,
+        self,
         cx: &mut Option<String>,
     ) -> Option<Result<(), Error>>;
 
     fn poll_write_vectored(
-        self: Option<String>,
+        self,
         cx: &mut Option<String>,
         bufs: &[usize]
     ) -> Option<Result<usize, Error>> {}
diff --git a/tests/rustdoc/hide-complex-unevaluated-const-arguments.rs b/tests/rustdoc/hide-complex-unevaluated-const-arguments.rs
index 6006354eba4e..d728f772a69c 100644
--- a/tests/rustdoc/hide-complex-unevaluated-const-arguments.rs
+++ b/tests/rustdoc/hide-complex-unevaluated-const-arguments.rs
@@ -4,9 +4,11 @@
 //
 // Read the documentation of `rustdoc::clean::utils::print_const_expr`
 // for further details.
-#![feature(const_trait_impl, generic_const_exprs)]
+#![feature(const_trait_impl, generic_const_exprs, adt_const_params, generic_const_items)]
 #![allow(incomplete_features)]
 
+use std::marker::ConstParamTy;
+
 // @has hide_complex_unevaluated_const_arguments/trait.Stage.html
 pub trait Stage {
     // A helper constant that prevents const expressions containing it
@@ -29,11 +31,13 @@ pub trait Stage {
     //
     // @has - '//*[@id="associatedconstant.ARRAY1"]' \
     //        'const ARRAY1: [u8; { _ }]'
-    const ARRAY1: [u8; Struct::new(/* ... */).do_something(Self::ABSTRACT * 1_000)];
+    const ARRAY1: [u8; Struct::new(/* ... */).do_something(Self::ABSTRACT * 1_000)]
+        where [(); Struct::new(/* ... */).do_something(Self::ABSTRACT * 1_000)]:;
 
     // @has - '//*[@id="associatedconstant.VERBOSE"]' \
     //        'const VERBOSE: [u16; { _ }]'
-    const VERBOSE: [u16; compute("thing", 9 + 9) * Self::ABSTRACT];
+    const VERBOSE: [u16; compute("thing", 9 + 9) * Self::ABSTRACT]
+        where [(); compute("thing", 9 + 9) * Self::ABSTRACT]:;
 
     // Check that we do not leak the private struct field contained within
     // the path. The output could definitely be improved upon
@@ -69,6 +73,7 @@ pub trait Sub: Sup<{ 90 * 20 * 4 }, { Struct { private: () } }> {}
 
 pub trait Sup<const N: usize, const S: Struct> {}
 
+#[derive(ConstParamTy, PartialEq, Eq)]
 pub struct Struct { private: () }
 
 impl Struct {
diff --git a/tests/rustdoc/intra-doc/prim-associated-traits.rs b/tests/rustdoc/intra-doc/prim-associated-traits.rs
index 8639a24f7f38..71d7d2189e6f 100644
--- a/tests/rustdoc/intra-doc/prim-associated-traits.rs
+++ b/tests/rustdoc/intra-doc/prim-associated-traits.rs
@@ -41,6 +41,6 @@ pub struct Number {
     pub u_128: u128,
     pub ch: char,
     pub boolean: bool,
-    pub string: str,
+    pub string: &'static str,
     pub n: !,
 }
diff --git a/tests/rustdoc/notable-trait/doc-notable_trait_box_is_not_an_iterator.rs b/tests/rustdoc/notable-trait/doc-notable_trait_box_is_not_an_iterator.rs
index 3fb00c7db841..6b94d7994839 100644
--- a/tests/rustdoc/notable-trait/doc-notable_trait_box_is_not_an_iterator.rs
+++ b/tests/rustdoc/notable-trait/doc-notable_trait_box_is_not_an_iterator.rs
@@ -3,25 +3,28 @@
 #![feature(no_core)]
 #![no_core]
 #[lang = "owned_box"]
-pub struct Box<T>;
+pub struct Box<T>(*const T);
 
 impl<T> Box<T> {
     pub fn new(x: T) -> Box<T> {
-        Box
+        Box(std::ptr::null())
     }
 }
 
+#[lang = "sized"]
+trait Sized {}
+
 #[doc(notable_trait)]
 pub trait FakeIterator {}
 
 impl<I: FakeIterator> FakeIterator for Box<I> {}
 
 #[lang = "pin"]
-pub struct Pin<T>;
+pub struct Pin<T>(T);
 
 impl<T> Pin<T> {
     pub fn new(x: T) -> Pin<T> {
-        Pin
+        Pin(x)
     }
 }
 
diff --git a/tests/rustdoc/whitespace-after-where-clause.enum.html b/tests/rustdoc/whitespace-after-where-clause.enum.html
index 20b60b68e88c..ff4971f33cd6 100644
--- a/tests/rustdoc/whitespace-after-where-clause.enum.html
+++ b/tests/rustdoc/whitespace-after-where-clause.enum.html
@@ -1,5 +1,5 @@
 <pre class="rust item-decl"><code>pub enum Cow&lt;'a, B&gt;<span class="where fmt-newline">where
-    B: <a class="trait" href="trait.ToOwned.html" title="trait foo::ToOwned">ToOwned</a>&lt;dyn <a class="trait" href="{{channel}}/core/clone/trait.Clone.html" title="trait core::clone::Clone">Clone</a>&gt; + ?<a class="trait" href="{{channel}}/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a> + 'a,</span>{
+    B: <a class="trait" href="trait.ToOwned.html" title="trait foo::ToOwned">ToOwned</a>&lt;<a class="primitive" href="{{channel}}/std/primitive.unit.html">()</a>&gt; + ?<a class="trait" href="{{channel}}/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a> + 'a,</span>{
     Borrowed(<a class="primitive" href="{{channel}}/std/primitive.reference.html">&amp;'a B</a>),
     Whatever(<a class="primitive" href="{{channel}}/std/primitive.u32.html">u32</a>),
 }</code></pre>
\ No newline at end of file
diff --git a/tests/rustdoc/whitespace-after-where-clause.enum2.html b/tests/rustdoc/whitespace-after-where-clause.enum2.html
index 065ce757de1a..bfc50f8adcdc 100644
--- a/tests/rustdoc/whitespace-after-where-clause.enum2.html
+++ b/tests/rustdoc/whitespace-after-where-clause.enum2.html
@@ -1,4 +1,4 @@
-<pre class="rust item-decl"><code>pub enum Cow2&lt;'a, B: ?<a class="trait" href="{{channel}}/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a> + <a class="trait" href="trait.ToOwned.html" title="trait foo::ToOwned">ToOwned</a>&lt;dyn <a class="trait" href="{{channel}}/core/clone/trait.Clone.html" title="trait core::clone::Clone">Clone</a>&gt; + 'a&gt; {
+<pre class="rust item-decl"><code>pub enum Cow2&lt;'a, B: ?<a class="trait" href="{{channel}}/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a> + <a class="trait" href="trait.ToOwned.html" title="trait foo::ToOwned">ToOwned</a>&lt;<a class="primitive" href="{{channel}}/std/primitive.unit.html">()</a>&gt; + 'a&gt; {
     Borrowed(<a class="primitive" href="{{channel}}/std/primitive.reference.html">&amp;'a B</a>),
     Whatever(<a class="primitive" href="{{channel}}/std/primitive.u32.html">u32</a>),
 }</code></pre>
\ No newline at end of file
diff --git a/tests/rustdoc/whitespace-after-where-clause.rs b/tests/rustdoc/whitespace-after-where-clause.rs
index b540c7c97c1f..832d3728e759 100644
--- a/tests/rustdoc/whitespace-after-where-clause.rs
+++ b/tests/rustdoc/whitespace-after-where-clause.rs
@@ -6,7 +6,8 @@
 // @has 'foo/trait.ToOwned.html'
 // @snapshot trait - '//*[@class="rust item-decl"]'
 pub trait ToOwned<T>
-where T: Clone
+where
+    T: Clone,
 {
     type Owned;
     fn to_owned(&self) -> Self::Owned;
@@ -26,7 +27,7 @@ pub trait ToOwned2<T: Clone> {
 // @snapshot enum - '//*[@class="rust item-decl"]'
 pub enum Cow<'a, B: ?Sized + 'a>
 where
-    B: ToOwned<Clone>,
+    B: ToOwned<()>,
 {
     Borrowed(&'a B),
     Whatever(u32),
@@ -35,7 +36,7 @@ where
 // @has 'foo/enum.Cow2.html'
 // @snapshot enum2 - '//*[@class="rust item-decl"]'
 // There should be a whitespace before `{` in this case!
-pub enum Cow2<'a, B: ?Sized + ToOwned<Clone> + 'a> {
+pub enum Cow2<'a, B: ?Sized + ToOwned<()> + 'a> {
     Borrowed(&'a B),
     Whatever(u32),
 }
@@ -44,7 +45,7 @@ pub enum Cow2<'a, B: ?Sized + ToOwned<Clone> + 'a> {
 // @snapshot struct - '//*[@class="rust item-decl"]'
 pub struct Struct<'a, B: ?Sized + 'a>
 where
-    B: ToOwned<Clone>,
+    B: ToOwned<()>,
 {
     pub a: &'a B,
     pub b: u32,
@@ -53,7 +54,7 @@ where
 // @has 'foo/struct.Struct2.html'
 // @snapshot struct2 - '//*[@class="rust item-decl"]'
 // There should be a whitespace before `{` in this case!
-pub struct Struct2<'a, B: ?Sized + ToOwned<Clone> + 'a> {
+pub struct Struct2<'a, B: ?Sized + ToOwned<()> + 'a> {
     pub a: &'a B,
     pub b: u32,
 }
@@ -62,7 +63,7 @@ pub struct Struct2<'a, B: ?Sized + ToOwned<Clone> + 'a> {
 // @snapshot union - '//*[@class="rust item-decl"]'
 pub union Union<'a, B: ?Sized + 'a>
 where
-    B: ToOwned<Clone>,
+    B: ToOwned<()>,
 {
     a: &'a B,
     b: u32,
@@ -71,7 +72,7 @@ where
 // @has 'foo/union.Union2.html'
 // @snapshot union2 - '//*[@class="rust item-decl"]'
 // There should be a whitespace before `{` in this case!
-pub union Union2<'a, B: ?Sized + ToOwned<Clone> + 'a> {
+pub union Union2<'a, B: ?Sized + ToOwned<()> + 'a> {
     a: &'a B,
     b: u32,
 }
diff --git a/tests/rustdoc/whitespace-after-where-clause.struct.html b/tests/rustdoc/whitespace-after-where-clause.struct.html
index 948ddc499da8..ca6853586339 100644
--- a/tests/rustdoc/whitespace-after-where-clause.struct.html
+++ b/tests/rustdoc/whitespace-after-where-clause.struct.html
@@ -1,5 +1,5 @@
 <pre class="rust item-decl"><code>pub struct Struct&lt;'a, B&gt;<span class="where fmt-newline">where
-    B: <a class="trait" href="trait.ToOwned.html" title="trait foo::ToOwned">ToOwned</a>&lt;dyn <a class="trait" href="{{channel}}/core/clone/trait.Clone.html" title="trait core::clone::Clone">Clone</a>&gt; + ?<a class="trait" href="{{channel}}/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a> + 'a,</span>{
+    B: <a class="trait" href="trait.ToOwned.html" title="trait foo::ToOwned">ToOwned</a>&lt;<a class="primitive" href="{{channel}}/std/primitive.unit.html">()</a>&gt; + ?<a class="trait" href="{{channel}}/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a> + 'a,</span>{
     pub a: <a class="primitive" href="{{channel}}/std/primitive.reference.html">&amp;'a B</a>,
     pub b: <a class="primitive" href="{{channel}}/std/primitive.u32.html">u32</a>,
 }</code></pre>
\ No newline at end of file
diff --git a/tests/rustdoc/whitespace-after-where-clause.struct2.html b/tests/rustdoc/whitespace-after-where-clause.struct2.html
index c647e8d71218..5aa8110c18f7 100644
--- a/tests/rustdoc/whitespace-after-where-clause.struct2.html
+++ b/tests/rustdoc/whitespace-after-where-clause.struct2.html
@@ -1,4 +1,4 @@
-<pre class="rust item-decl"><code>pub struct Struct2&lt;'a, B: ?<a class="trait" href="{{channel}}/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a> + <a class="trait" href="trait.ToOwned.html" title="trait foo::ToOwned">ToOwned</a>&lt;dyn <a class="trait" href="{{channel}}/core/clone/trait.Clone.html" title="trait core::clone::Clone">Clone</a>&gt; + 'a&gt; {
+<pre class="rust item-decl"><code>pub struct Struct2&lt;'a, B: ?<a class="trait" href="{{channel}}/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a> + <a class="trait" href="trait.ToOwned.html" title="trait foo::ToOwned">ToOwned</a>&lt;<a class="primitive" href="{{channel}}/std/primitive.unit.html">()</a>&gt; + 'a&gt; {
     pub a: <a class="primitive" href="{{channel}}/std/primitive.reference.html">&amp;'a B</a>,
     pub b: <a class="primitive" href="{{channel}}/std/primitive.u32.html">u32</a>,
 }</code></pre>
\ No newline at end of file
diff --git a/tests/rustdoc/whitespace-after-where-clause.union.html b/tests/rustdoc/whitespace-after-where-clause.union.html
index 38b6cb8b5c61..40b0c6712842 100644
--- a/tests/rustdoc/whitespace-after-where-clause.union.html
+++ b/tests/rustdoc/whitespace-after-where-clause.union.html
@@ -1,4 +1,4 @@
 <pre class="rust item-decl"><code>pub union Union&lt;'a, B&gt;<span class="where fmt-newline">where
-    B: <a class="trait" href="trait.ToOwned.html" title="trait foo::ToOwned">ToOwned</a>&lt;dyn <a class="trait" href="{{channel}}/core/clone/trait.Clone.html" title="trait core::clone::Clone">Clone</a>&gt; + ?<a class="trait" href="{{channel}}/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a> + 'a,</span>{
+    B: <a class="trait" href="trait.ToOwned.html" title="trait foo::ToOwned">ToOwned</a>&lt;<a class="primitive" href="{{channel}}/std/primitive.unit.html">()</a>&gt; + ?<a class="trait" href="{{channel}}/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a> + 'a,</span>{
     /* private fields */
 }</code></pre>
\ No newline at end of file
diff --git a/tests/rustdoc/whitespace-after-where-clause.union2.html b/tests/rustdoc/whitespace-after-where-clause.union2.html
index 66ad30c92001..177a161b83a6 100644
--- a/tests/rustdoc/whitespace-after-where-clause.union2.html
+++ b/tests/rustdoc/whitespace-after-where-clause.union2.html
@@ -1,3 +1,3 @@
-<pre class="rust item-decl"><code>pub union Union2&lt;'a, B: ?<a class="trait" href="{{channel}}/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a> + <a class="trait" href="trait.ToOwned.html" title="trait foo::ToOwned">ToOwned</a>&lt;dyn <a class="trait" href="{{channel}}/core/clone/trait.Clone.html" title="trait core::clone::Clone">Clone</a>&gt; + 'a&gt; {
+<pre class="rust item-decl"><code>pub union Union2&lt;'a, B: ?<a class="trait" href="{{channel}}/core/marker/trait.Sized.html" title="trait core::marker::Sized">Sized</a> + <a class="trait" href="trait.ToOwned.html" title="trait foo::ToOwned">ToOwned</a>&lt;<a class="primitive" href="{{channel}}/std/primitive.unit.html">()</a>&gt; + 'a&gt; {
     /* private fields */
 }</code></pre>
\ No newline at end of file

From a8ece1190bf6b340175bc5b688e52bd29924f483 Mon Sep 17 00:00:00 2001
From: Sebastian Thiel <sebastian.thiel@icloud.com>
Date: Tue, 31 Oct 2023 15:05:06 +0100
Subject: [PATCH 287/297] Add support for pre-unix-epoch file dates on Apple
 platforms (#108277)

Time in UNIX system calls counts from the epoch, 1970-01-01. The timespec
struct used in various system calls represents this as a number of seconds and
a number of nanoseconds. Nanoseconds are required to be between 0 and
999_999_999, because the portion outside that range should be represented in
the seconds field; if nanoseconds were larger than 999_999_999, the seconds
field should go up instead.

Suppose you ask for the time 1969-12-31, what time is that? On UNIX systems
that support times before the epoch, that's seconds=-86400, one day before the
epoch. But now, suppose you ask for the time 1969-12-31 23:59:00.1. In other
words, a tenth of a second after one minute before the epoch.  On most UNIX
systems, that's represented as seconds=-60, nanoseconds=100_000_000. The macOS
bug is that it returns seconds=-59, nanoseconds=-900_000_000.

While that's in some sense an accurate description of the time (59.9 seconds
before the epoch), that violates the invariant of the timespec data structure:
nanoseconds must be between 0 and 999999999. This causes this assertion in the
Rust standard library.

So, on macOS, if we get a Timespec value with seconds less than or equal to
zero, and nanoseconds between -999_999_999 and -1 (inclusive), we can add
1_000_000_000 to the nanoseconds and subtract 1 from the seconds, and then
convert.  The resulting timespec value is still accepted by macOS, and when fed
back into the OS, produces the same results. (If you set a file's mtime with
that timestamp, then read it back, you get back the one with negative
nanoseconds again.)

Co-authored-by: Josh Triplett <josh@joshtriplett.org>
---
 library/std/src/fs/tests.rs      | 42 ++++++++++++++++++++++++++++++++
 library/std/src/sys/unix/time.rs | 24 ++++++++++++++++++
 2 files changed, 66 insertions(+)

diff --git a/library/std/src/fs/tests.rs b/library/std/src/fs/tests.rs
index 736b495343e9..547a7b7052f4 100644
--- a/library/std/src/fs/tests.rs
+++ b/library/std/src/fs/tests.rs
@@ -1708,6 +1708,48 @@ fn test_file_times() {
     }
 }
 
+#[test]
+#[cfg(any(target_os = "macos", target_os = "ios", target_os = "tvos", target_os = "watchos"))]
+fn test_file_times_pre_epoch_with_nanos() {
+    #[cfg(target_os = "ios")]
+    use crate::os::ios::fs::FileTimesExt;
+    #[cfg(target_os = "macos")]
+    use crate::os::macos::fs::FileTimesExt;
+    #[cfg(target_os = "tvos")]
+    use crate::os::tvos::fs::FileTimesExt;
+    #[cfg(target_os = "watchos")]
+    use crate::os::watchos::fs::FileTimesExt;
+
+    let tmp = tmpdir();
+    let file = File::create(tmp.join("foo")).unwrap();
+
+    for (accessed, modified, created) in [
+        // The first round is to set filetimes to something we know works, but this time
+        // it's validated with nanoseconds as well which probe the numeric boundary.
+        (
+            SystemTime::UNIX_EPOCH + Duration::new(12345, 1),
+            SystemTime::UNIX_EPOCH + Duration::new(54321, 100_000_000),
+            SystemTime::UNIX_EPOCH + Duration::new(32123, 999_999_999),
+        ),
+        // The second rounds uses pre-epoch dates along with nanoseconds that probe
+        // the numeric boundary.
+        (
+            SystemTime::UNIX_EPOCH - Duration::new(1, 1),
+            SystemTime::UNIX_EPOCH - Duration::new(60, 100_000_000),
+            SystemTime::UNIX_EPOCH - Duration::new(3600, 999_999_999),
+        ),
+    ] {
+        let mut times = FileTimes::new();
+        times = times.set_accessed(accessed).set_modified(modified).set_created(created);
+        file.set_times(times).unwrap();
+
+        let metadata = file.metadata().unwrap();
+        assert_eq!(metadata.accessed().unwrap(), accessed);
+        assert_eq!(metadata.modified().unwrap(), modified);
+        assert_eq!(metadata.created().unwrap(), created);
+    }
+}
+
 #[test]
 #[cfg(windows)]
 fn windows_unix_socket_exists() {
diff --git a/library/std/src/sys/unix/time.rs b/library/std/src/sys/unix/time.rs
index e4540b99413b..f2e86a4fb2b1 100644
--- a/library/std/src/sys/unix/time.rs
+++ b/library/std/src/sys/unix/time.rs
@@ -76,6 +76,30 @@ impl Timespec {
     }
 
     const fn new(tv_sec: i64, tv_nsec: i64) -> Timespec {
+        // On Apple OS, dates before epoch are represented differently than on other
+        // Unix platforms: e.g. 1/10th of a second before epoch is represented as `seconds=-1`
+        // and `nanoseconds=100_000_000` on other platforms, but is `seconds=0` and
+        // `nanoseconds=-900_000_000` on Apple OS.
+        //
+        // To compensate, we first detect this special case by checking if both
+        // seconds and nanoseconds are in range, and then correct the value for seconds
+        // and nanoseconds to match the common unix representation.
+        //
+        // Please note that Apple OS nonetheless accepts the standard unix format when
+        // setting file times, which makes this compensation round-trippable and generally
+        // transparent.
+        #[cfg(any(
+            target_os = "macos",
+            target_os = "ios",
+            target_os = "tvos",
+            target_os = "watchos"
+        ))]
+        let (tv_sec, tv_nsec) =
+            if (tv_sec <= 0 && tv_sec > i64::MIN) && (tv_nsec < 0 && tv_nsec > -1_000_000_000) {
+                (tv_sec - 1, tv_nsec + 1_000_000_000)
+            } else {
+                (tv_sec, tv_nsec)
+            };
         assert!(tv_nsec >= 0 && tv_nsec < NSEC_PER_SEC as i64);
         // SAFETY: The assert above checks tv_nsec is within the valid range
         Timespec { tv_sec, tv_nsec: unsafe { Nanoseconds(tv_nsec as u32) } }

From 77174d3f29a44d2473a924a350862bd2824d06d6 Mon Sep 17 00:00:00 2001
From: Oli Scherer <git-spam-no-reply9815368754983@oli-obk.de>
Date: Mon, 30 Oct 2023 09:58:54 +0000
Subject: [PATCH 288/297] Turn const_caller_location from a query to a hook

---
 compiler/rustc_const_eval/src/lib.rs                |  2 +-
 .../rustc_const_eval/src/util/caller_location.rs    | 13 ++++++++-----
 compiler/rustc_middle/src/hooks/mod.rs              |  3 +++
 compiler/rustc_middle/src/mir/consts.rs             |  4 ++--
 compiler/rustc_middle/src/query/mod.rs              |  4 ----
 5 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/compiler/rustc_const_eval/src/lib.rs b/compiler/rustc_const_eval/src/lib.rs
index 7fc87ed88810..7d36e2eaefed 100644
--- a/compiler/rustc_const_eval/src/lib.rs
+++ b/compiler/rustc_const_eval/src/lib.rs
@@ -49,7 +49,7 @@ pub fn provide(providers: &mut Providers) {
     const_eval::provide(providers);
     providers.eval_to_const_value_raw = const_eval::eval_to_const_value_raw_provider;
     providers.eval_to_allocation_raw = const_eval::eval_to_allocation_raw_provider;
-    providers.const_caller_location = util::caller_location::const_caller_location_provider;
+    providers.hooks.const_caller_location = util::caller_location::const_caller_location_provider;
     providers.eval_to_valtree = |tcx, param_env_and_value| {
         let (param_env, raw) = param_env_and_value.into_parts();
         const_eval::eval_to_valtree(tcx, param_env, raw)
diff --git a/compiler/rustc_const_eval/src/util/caller_location.rs b/compiler/rustc_const_eval/src/util/caller_location.rs
index d67863d75d66..4a3cfd50b441 100644
--- a/compiler/rustc_const_eval/src/util/caller_location.rs
+++ b/compiler/rustc_const_eval/src/util/caller_location.rs
@@ -1,8 +1,9 @@
 use rustc_hir::LangItem;
 use rustc_middle::mir;
+use rustc_middle::query::TyCtxtAt;
+use rustc_middle::ty;
 use rustc_middle::ty::layout::LayoutOf;
-use rustc_middle::ty::{self, TyCtxt};
-use rustc_span::{source_map::DUMMY_SP, symbol::Symbol};
+use rustc_span::symbol::Symbol;
 use rustc_type_ir::Mutability;
 
 use crate::const_eval::{mk_eval_cx, CanAccessStatics, CompileTimeEvalContext};
@@ -49,11 +50,13 @@ fn alloc_caller_location<'mir, 'tcx>(
 }
 
 pub(crate) fn const_caller_location_provider(
-    tcx: TyCtxt<'_>,
-    (file, line, col): (Symbol, u32, u32),
+    tcx: TyCtxtAt<'_>,
+    file: Symbol,
+    line: u32,
+    col: u32,
 ) -> mir::ConstValue<'_> {
     trace!("const_caller_location: {}:{}:{}", file, line, col);
-    let mut ecx = mk_eval_cx(tcx, DUMMY_SP, ty::ParamEnv::reveal_all(), CanAccessStatics::No);
+    let mut ecx = mk_eval_cx(tcx.tcx, tcx.span, ty::ParamEnv::reveal_all(), CanAccessStatics::No);
 
     let loc_place = alloc_caller_location(&mut ecx, file, line, col);
     if intern_const_alloc_recursive(&mut ecx, InternKind::Constant, &loc_place).is_err() {
diff --git a/compiler/rustc_middle/src/hooks/mod.rs b/compiler/rustc_middle/src/hooks/mod.rs
index 572751d95118..34838ca4302f 100644
--- a/compiler/rustc_middle/src/hooks/mod.rs
+++ b/compiler/rustc_middle/src/hooks/mod.rs
@@ -67,4 +67,7 @@ declare_hooks! {
     /// Tries to destructure an `mir::Const` ADT or array into its variant index
     /// and its field values. This should only be used for pretty printing.
     hook try_destructure_mir_constant_for_diagnostics(val: mir::ConstValue<'tcx>, ty: Ty<'tcx>) -> Option<mir::DestructuredConstant<'tcx>>;
+
+    /// Getting a &core::panic::Location referring to a span.
+    hook const_caller_location(file: rustc_span::Symbol, line: u32, col: u32) -> mir::ConstValue<'tcx>;
 }
diff --git a/compiler/rustc_middle/src/mir/consts.rs b/compiler/rustc_middle/src/mir/consts.rs
index 92218f6948f4..a9d09709e842 100644
--- a/compiler/rustc_middle/src/mir/consts.rs
+++ b/compiler/rustc_middle/src/mir/consts.rs
@@ -590,12 +590,12 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn span_as_caller_location(self, span: Span) -> ConstValue<'tcx> {
         let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
         let caller = self.sess.source_map().lookup_char_pos(topmost.lo());
-        self.const_caller_location((
+        self.const_caller_location(
             rustc_span::symbol::Symbol::intern(
                 &caller.file.name.for_codegen(&self.sess).to_string_lossy(),
             ),
             caller.line as u32,
             caller.col_display as u32 + 1,
-        ))
+        )
     }
 }
diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs
index 062b03e71fdc..f9ec368361c5 100644
--- a/compiler/rustc_middle/src/query/mod.rs
+++ b/compiler/rustc_middle/src/query/mod.rs
@@ -1101,10 +1101,6 @@ rustc_queries! {
         desc { "destructuring type level constant"}
     }
 
-    query const_caller_location(key: (rustc_span::Symbol, u32, u32)) -> mir::ConstValue<'tcx> {
-        desc { "getting a &core::panic::Location referring to a span" }
-    }
-
     // FIXME get rid of this with valtrees
     query lit_to_const(
         key: LitToConstInput<'tcx>

From 2fcb4d92b0cfe13e9f30444739545a53a569b579 Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Tue, 31 Oct 2023 11:58:20 -0700
Subject: [PATCH 289/297] change inline_retag to after.mir

---
 .../inline/inline_retag.bar.Inline.after.mir  | 59 +++++++++++++++++
 .../inline/inline_retag.bar.Inline.diff       | 65 -------------------
 tests/mir-opt/inline/inline_retag.rs          |  2 +-
 3 files changed, 60 insertions(+), 66 deletions(-)
 create mode 100644 tests/mir-opt/inline/inline_retag.bar.Inline.after.mir
 delete mode 100644 tests/mir-opt/inline/inline_retag.bar.Inline.diff

diff --git a/tests/mir-opt/inline/inline_retag.bar.Inline.after.mir b/tests/mir-opt/inline/inline_retag.bar.Inline.after.mir
new file mode 100644
index 000000000000..8c3f3a4589e6
--- /dev/null
+++ b/tests/mir-opt/inline/inline_retag.bar.Inline.after.mir
@@ -0,0 +1,59 @@
+// MIR for `bar` after Inline
+
+fn bar() -> bool {
+    let mut _0: bool;
+    let _1: for<'a, 'b> fn(&'a i32, &'b i32) -> bool {foo};
+    let mut _2: for<'a, 'b> fn(&'a i32, &'b i32) -> bool {foo};
+    let mut _3: &i32;
+    let _4: &i32;
+    let _5: i32;
+    let mut _6: &i32;
+    let _7: &i32;
+    let _8: i32;
+    scope 1 {
+        debug f => _1;
+        let mut _9: &i32;
+        let mut _10: &i32;
+        scope 2 (inlined foo) {
+            debug x => _3;
+            debug y => _6;
+            let mut _11: i32;
+            let mut _12: i32;
+        }
+    }
+
+    bb0: {
+        StorageLive(_1);
+        _1 = foo;
+        StorageLive(_2);
+        _2 = _1;
+        StorageLive(_3);
+        StorageLive(_4);
+        _10 = const _;
+        Retag(_10);
+        _4 = &(*_10);
+        _3 = &(*_4);
+        StorageLive(_6);
+        StorageLive(_7);
+        _9 = const _;
+        Retag(_9);
+        _7 = &(*_9);
+        _6 = &(*_7);
+        Retag(_3);
+        Retag(_6);
+        StorageLive(_11);
+        _11 = (*_3);
+        StorageLive(_12);
+        _12 = (*_6);
+        _0 = Eq(move _11, move _12);
+        StorageDead(_12);
+        StorageDead(_11);
+        StorageDead(_6);
+        StorageDead(_3);
+        StorageDead(_2);
+        StorageDead(_1);
+        StorageDead(_7);
+        StorageDead(_4);
+        return;
+    }
+}
diff --git a/tests/mir-opt/inline/inline_retag.bar.Inline.diff b/tests/mir-opt/inline/inline_retag.bar.Inline.diff
deleted file mode 100644
index 8f53f6342ec6..000000000000
--- a/tests/mir-opt/inline/inline_retag.bar.Inline.diff
+++ /dev/null
@@ -1,65 +0,0 @@
-- // MIR for `bar` before Inline
-+ // MIR for `bar` after Inline
-  
-  fn bar() -> bool {
-      let mut _0: bool;
-      let _1: for<'a, 'b> fn(&'a i32, &'b i32) -> bool {foo};
-      let mut _2: for<'a, 'b> fn(&'a i32, &'b i32) -> bool {foo};
-      let mut _3: &i32;
-      let _4: &i32;
-      let _5: i32;
-      let mut _6: &i32;
-      let _7: &i32;
-      let _8: i32;
-      scope 1 {
-          debug f => _1;
-          let mut _9: &i32;
-          let mut _10: &i32;
-+         scope 2 (inlined foo) {
-+             debug x => _3;
-+             debug y => _6;
-+             let mut _11: i32;
-+             let mut _12: i32;
-+         }
-      }
-  
-      bb0: {
-          StorageLive(_1);
-          _1 = foo;
-          StorageLive(_2);
-          _2 = _1;
-          StorageLive(_3);
-          StorageLive(_4);
-          _10 = const _;
-          Retag(_10);
-          _4 = &(*_10);
-          _3 = &(*_4);
-          StorageLive(_6);
-          StorageLive(_7);
-          _9 = const _;
-          Retag(_9);
-          _7 = &(*_9);
-          _6 = &(*_7);
--         _0 = move _2(move _3, move _6) -> [return: bb1, unwind continue];
--     }
-- 
--     bb1: {
-+         Retag(_3);
-+         Retag(_6);
-+         StorageLive(_11);
-+         _11 = (*_3);
-+         StorageLive(_12);
-+         _12 = (*_6);
-+         _0 = Eq(move _11, move _12);
-+         StorageDead(_12);
-+         StorageDead(_11);
-          StorageDead(_6);
-          StorageDead(_3);
-          StorageDead(_2);
-          StorageDead(_1);
-          StorageDead(_7);
-          StorageDead(_4);
-          return;
-      }
-  }
-  
diff --git a/tests/mir-opt/inline/inline_retag.rs b/tests/mir-opt/inline/inline_retag.rs
index b9058905892b..9fb6f709223c 100644
--- a/tests/mir-opt/inline/inline_retag.rs
+++ b/tests/mir-opt/inline/inline_retag.rs
@@ -6,7 +6,7 @@ fn main() {
     println!("{}", bar());
 }
 
-// EMIT_MIR inline_retag.bar.Inline.diff
+// EMIT_MIR inline_retag.bar.Inline.after.mir
 fn bar() -> bool {
     // CHECK-LABEL: fn bar(
     // CHECK: (inlined foo)

From 7ea697764970f3f00fa86442e2a69f6eaa2621f8 Mon Sep 17 00:00:00 2001
From: Weihang Lo <me@weihanglo.tw>
Date: Tue, 31 Oct 2023 15:56:37 -0400
Subject: [PATCH 290/297] Update cargo

---
 src/tools/cargo | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tools/cargo b/src/tools/cargo
index 708383d620e1..b4d18d4bd3db 160000
--- a/src/tools/cargo
+++ b/src/tools/cargo
@@ -1 +1 @@
-Subproject commit 708383d620e183a9ece69b8fe930c411d83dee27
+Subproject commit b4d18d4bd3db6d872892f6c87c51a02999b80802

From e936416a8d3cfb501504f00d7250d5b595fed244 Mon Sep 17 00:00:00 2001
From: George Bateman <george.bateman16@gmail.com>
Date: Tue, 15 Aug 2023 20:10:45 +0100
Subject: [PATCH 291/297] Support enum variants in offset_of!

---
 compiler/rustc_abi/src/lib.rs                 | 33 ++++++++++
 compiler/rustc_codegen_cranelift/src/base.rs  |  2 +-
 compiler/rustc_codegen_ssa/src/mir/rvalue.rs  |  2 +-
 .../rustc_const_eval/src/interpret/step.rs    |  2 +-
 .../src/transform/validate.rs                 | 33 ++++++----
 compiler/rustc_error_codes/src/error_codes.rs |  1 +
 .../src/error_codes/E0795.md                  | 28 +++++++++
 compiler/rustc_hir_typeck/src/expr.rs         | 60 +++++++++++++++++--
 compiler/rustc_middle/src/mir/syntax.rs       |  4 +-
 compiler/rustc_middle/src/thir.rs             |  5 +-
 compiler/rustc_middle/src/ty/codec.rs         | 12 +++-
 compiler/rustc_middle/src/ty/context.rs       | 13 +++-
 .../rustc_middle/src/ty/typeck_results.rs     | 10 ++--
 compiler/rustc_mir_build/src/thir/cx/expr.rs  |  2 +-
 .../src/dataflow_const_prop.rs                |  2 +-
 compiler/rustc_passes/src/dead.rs             | 29 ++++++---
 compiler/rustc_smir/src/rustc_smir/mod.rs     | 12 +++-
 compiler/rustc_target/src/abi/mod.rs          | 24 +++++---
 ...set_of.concrete.ConstProp.panic-abort.diff | 56 +++++++++++++++--
 ...et_of.concrete.ConstProp.panic-unwind.diff | 56 +++++++++++++++--
 ...fset_of.generic.ConstProp.panic-abort.diff | 50 ++++++++++++++--
 ...set_of.generic.ConstProp.panic-unwind.diff | 50 ++++++++++++++--
 tests/mir-opt/const_prop/offset_of.rs         | 17 ++++++
 tests/ui/offset-of/offset-of-enum.rs          |  3 +-
 tests/ui/offset-of/offset-of-enum.stderr      |  8 +--
 tests/ui/offset-of/offset-of-private.rs       | 11 ++++
 tests/ui/offset-of/offset-of-private.stderr   | 22 ++++---
 27 files changed, 472 insertions(+), 75 deletions(-)
 create mode 100644 compiler/rustc_error_codes/src/error_codes/E0795.md

diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs
index 8e7aa59ee341..0654bfd11c5e 100644
--- a/compiler/rustc_abi/src/lib.rs
+++ b/compiler/rustc_abi/src/lib.rs
@@ -1107,6 +1107,39 @@ impl Scalar {
 }
 
 // NOTE: This struct is generic over the FieldIdx for rust-analyzer usage.
+rustc_index::newtype_index! {
+    /// The *source-order* index of a field in a variant.
+    ///
+    /// This is how most code after type checking refers to fields, rather than
+    /// using names (as names have hygiene complications and more complex lookup).
+    ///
+    /// Particularly for `repr(Rust)` types, this may not be the same as *layout* order.
+    /// (It is for `repr(C)` `struct`s, however.)
+    ///
+    /// For example, in the following types,
+    /// ```rust
+    /// # enum Never {}
+    /// # #[repr(u16)]
+    /// enum Demo1 {
+    ///    Variant0 { a: Never, b: i32 } = 100,
+    ///    Variant1 { c: u8, d: u64 } = 10,
+    /// }
+    /// struct Demo2 { e: u8, f: u16, g: u8 }
+    /// ```
+    /// `b` is `FieldIdx(1)` in `VariantIdx(0)`,
+    /// `d` is `FieldIdx(1)` in `VariantIdx(1)`, and
+    /// `f` is `FieldIdx(1)` in `VariantIdx(0)`.
+    #[derive(HashStable_Generic)]
+    pub struct FieldIdx {}
+}
+
+/// `offset_of` can traverse fields and enum variants and should keep track of which is which.
+#[derive(Copy, Clone, Debug, Eq, Hash, HashStable_Generic, PartialEq, Encodable, Decodable)]
+pub enum OffsetOfIdx {
+    Field(FieldIdx),
+    Variant(VariantIdx),
+}
+
 /// Describes how the fields of a type are located in memory.
 #[derive(PartialEq, Eq, Hash, Clone, Debug)]
 #[cfg_attr(feature = "nightly", derive(HashStable_Generic))]
diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs
index 80e7c5bd9edb..91b1547cb6ea 100644
--- a/compiler/rustc_codegen_cranelift/src/base.rs
+++ b/compiler/rustc_codegen_cranelift/src/base.rs
@@ -766,7 +766,7 @@ fn codegen_stmt<'tcx>(
                         NullOp::SizeOf => layout.size.bytes(),
                         NullOp::AlignOf => layout.align.abi.bytes(),
                         NullOp::OffsetOf(fields) => {
-                            layout.offset_of_subfield(fx, fields.iter().map(|f| f.index())).bytes()
+                            layout.offset_of_subfield(fx, fields.iter()).bytes()
                         }
                     };
                     let val = CValue::by_val(
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
index f591afaaaf42..b0f757898e32 100644
--- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
@@ -680,7 +680,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         layout.align.abi.bytes()
                     }
                     mir::NullOp::OffsetOf(fields) => {
-                        layout.offset_of_subfield(bx.cx(), fields.iter().map(|f| f.index())).bytes()
+                        layout.offset_of_subfield(bx.cx(), fields.iter()).bytes()
                     }
                 };
                 let val = bx.cx().const_usize(val);
diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs
index 8c34d05042b1..b6993d939c58 100644
--- a/compiler/rustc_const_eval/src/interpret/step.rs
+++ b/compiler/rustc_const_eval/src/interpret/step.rs
@@ -275,7 +275,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
                     mir::NullOp::SizeOf => layout.size.bytes(),
                     mir::NullOp::AlignOf => layout.align.abi.bytes(),
                     mir::NullOp::OffsetOf(fields) => {
-                        layout.offset_of_subfield(self, fields.iter().map(|f| f.index())).bytes()
+                        layout.offset_of_subfield(self, fields.iter()).bytes()
                     }
                 };
                 self.write_scalar(Scalar::from_target_usize(val, self), &dest)?;
diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs
index b70a342aa15e..bdccd9b0edb5 100644
--- a/compiler/rustc_const_eval/src/transform/validate.rs
+++ b/compiler/rustc_const_eval/src/transform/validate.rs
@@ -1056,16 +1056,19 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                     }
                 }
             }
-            Rvalue::NullaryOp(NullOp::OffsetOf(fields), container) => {
+            Rvalue::NullaryOp(NullOp::OffsetOf(indices), container) => {
                 let fail_out_of_bounds = |this: &mut Self, location, field, ty| {
                     this.fail(location, format!("Out of bounds field {field:?} for {ty:?}"));
                 };
 
                 let mut current_ty = *container;
+                let mut indices = indices.into_iter();
 
-                for field in fields.iter() {
-                    match current_ty.kind() {
-                        ty::Tuple(fields) => {
+                use rustc_target::abi::OffsetOfIdx::*;
+
+                while let Some(index) = indices.next() {
+                    match (current_ty.kind(), index) {
+                        (ty::Tuple(fields), Field(field)) => {
                             let Some(&f_ty) = fields.get(field.as_usize()) else {
                                 fail_out_of_bounds(self, location, field, current_ty);
                                 return;
@@ -1073,16 +1076,24 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
 
                             current_ty = self.tcx.normalize_erasing_regions(self.param_env, f_ty);
                         }
-                        ty::Adt(adt_def, args) => {
-                            if adt_def.is_enum() {
+                        (ty::Adt(adt_def, args), Field(field)) if !adt_def.is_enum() => {
+                            let Some(field) = adt_def.non_enum_variant().fields.get(field) else {
+                                fail_out_of_bounds(self, location, field, current_ty);
+                                return;
+                            };
+
+                            let f_ty = field.ty(self.tcx, args);
+                            current_ty = self.tcx.normalize_erasing_regions(self.param_env, f_ty);
+                        }
+                        (ty::Adt(adt_def, args), Variant(variant)) if adt_def.is_enum() => {
+                            let Some(Field(field)) = indices.next() else {
                                 self.fail(
                                     location,
-                                    format!("Cannot get field offset from enum {current_ty:?}"),
+                                    format!("enum variant must be followed by field index in offset_of; in {current_ty:?}"),
                                 );
                                 return;
-                            }
-
-                            let Some(field) = adt_def.non_enum_variant().fields.get(field) else {
+                            };
+                            let Some(field) = adt_def.variant(variant).fields.get(field) else {
                                 fail_out_of_bounds(self, location, field, current_ty);
                                 return;
                             };
@@ -1093,7 +1104,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                         _ => {
                             self.fail(
                                 location,
-                                format!("Cannot get field offset from non-adt type {current_ty:?}"),
+                                format!("Cannot get offset {index:?} from type {current_ty:?}"),
                             );
                             return;
                         }
diff --git a/compiler/rustc_error_codes/src/error_codes.rs b/compiler/rustc_error_codes/src/error_codes.rs
index 89c44c6ec8a9..6680e8875c3e 100644
--- a/compiler/rustc_error_codes/src/error_codes.rs
+++ b/compiler/rustc_error_codes/src/error_codes.rs
@@ -514,6 +514,7 @@ E0791: include_str!("./error_codes/E0791.md"),
 E0792: include_str!("./error_codes/E0792.md"),
 E0793: include_str!("./error_codes/E0793.md"),
 E0794: include_str!("./error_codes/E0794.md"),
+E0795: include_str!("./error_codes/E0795.md"),
 }
 
 // Undocumented removed error codes. Note that many removed error codes are kept in the list above
diff --git a/compiler/rustc_error_codes/src/error_codes/E0795.md b/compiler/rustc_error_codes/src/error_codes/E0795.md
new file mode 100644
index 000000000000..8b4b2dc87fd7
--- /dev/null
+++ b/compiler/rustc_error_codes/src/error_codes/E0795.md
@@ -0,0 +1,28 @@
+Invalid argument for the `offset_of!` macro.
+
+Erroneous code example:
+
+```compile_fail,E0795
+#![feature(offset_of)]
+
+let x = std::mem::offset_of!(Option<u8>, Some);
+```
+
+The `offset_of!` macro gives the offset of a field within a type. It can
+navigate through enum variants, but the final component of its second argument
+must be a field and not a variant.
+
+The offset of the contained `u8` in the `Option<u8>` can be found by specifying
+the field name `0`:
+
+```
+#![feature(offset_of)]
+
+let x: usize = std::mem::offset_of!(Option<u8>, Some.0);
+```
+
+The discriminant of an enumeration may be read with `core::mem::discriminant`,
+but this is not always a value physically present within the enum.
+
+Further information about enum layout may be found at
+https://rust-lang.github.io/unsafe-code-guidelines/layout/enums.html.
diff --git a/compiler/rustc_hir_typeck/src/expr.rs b/compiler/rustc_hir_typeck/src/expr.rs
index b1d6e4f0523b..394a9033765a 100644
--- a/compiler/rustc_hir_typeck/src/expr.rs
+++ b/compiler/rustc_hir_typeck/src/expr.rs
@@ -3103,16 +3103,68 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         fields: &[Ident],
         expr: &'tcx hir::Expr<'tcx>,
     ) -> Ty<'tcx> {
+        use rustc_target::abi::OffsetOfIdx::*;
+
         let container = self.to_ty(container).normalized;
 
         let mut field_indices = Vec::with_capacity(fields.len());
         let mut current_container = container;
+        let mut fields = fields.into_iter();
 
-        for &field in fields {
+        while let Some(&field) = fields.next() {
             let container = self.structurally_resolve_type(expr.span, current_container);
 
             match container.kind() {
-                ty::Adt(container_def, args) if !container_def.is_enum() => {
+                ty::Adt(container_def, args) if container_def.is_enum() => {
+                    let block = self.tcx.hir().local_def_id_to_hir_id(self.body_id);
+                    let (ident, _def_scope) =
+                        self.tcx.adjust_ident_and_get_scope(field, container_def.did(), block);
+
+                    if let Some((index, variant)) = container_def.variants()
+                        .iter_enumerated()
+                        .find(|(_, v)| v.ident(self.tcx).normalize_to_macros_2_0() == ident)
+                    {
+                        let Some(&subfield) = fields.next() else {
+                            let mut err = type_error_struct!(
+                                self.tcx().sess,
+                                ident.span,
+                                container,
+                                E0795,
+                                "`{ident}` is an enum variant; expected field at end of `offset_of`",
+                                );
+                            err.span_label(field.span, "enum variant");
+                            err.emit();
+                            break;
+                        };
+                        let (subident, sub_def_scope) =
+                            self.tcx.adjust_ident_and_get_scope(subfield, variant.def_id, block);
+
+                        if let Some((subindex, field)) = variant.fields
+                            .iter_enumerated()
+                            .find(|(_, f)| f.ident(self.tcx).normalize_to_macros_2_0() == subident)
+                        {
+                            let field_ty = self.field_ty(expr.span, field, args);
+
+                            // FIXME: DSTs with static alignment should be allowed
+                            self.require_type_is_sized(field_ty, expr.span, traits::MiscObligation);
+
+                            if field.vis.is_accessible_from(sub_def_scope, self.tcx) {
+                                self.tcx.check_stability(field.did, Some(expr.hir_id), expr.span, None);
+                            } else {
+                                self.private_field_err(ident, container_def.did()).emit();
+                            }
+
+                            // Save the index of all fields regardless of their visibility in case
+                            // of error recovery.
+                            field_indices.push(Variant(index));
+                            field_indices.push(Field(subindex));
+                            current_container = field_ty;
+
+                            continue;
+                        }
+                    }
+                }
+                ty::Adt(container_def, args) => {
                     let block = self.tcx.hir().local_def_id_to_hir_id(self.body_id);
                     let (ident, def_scope) =
                         self.tcx.adjust_ident_and_get_scope(field, container_def.did(), block);
@@ -3135,7 +3187,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
                         // Save the index of all fields regardless of their visibility in case
                         // of error recovery.
-                        field_indices.push(index);
+                        field_indices.push(Field(index));
                         current_container = field_ty;
 
                         continue;
@@ -3149,7 +3201,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             self.require_type_is_sized(ty, expr.span, traits::MiscObligation);
                         }
                         if let Some(&field_ty) = tys.get(index) {
-                            field_indices.push(index.into());
+                            field_indices.push(Field(index.into()));
                             current_container = field_ty;
 
                             continue;
diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs
index b89c7bc512e1..d0094a81634d 100644
--- a/compiler/rustc_middle/src/mir/syntax.rs
+++ b/compiler/rustc_middle/src/mir/syntax.rs
@@ -17,7 +17,7 @@ use rustc_hir::def_id::DefId;
 use rustc_hir::{self as hir};
 use rustc_hir::{self, CoroutineKind};
 use rustc_index::IndexVec;
-use rustc_target::abi::{FieldIdx, VariantIdx};
+use rustc_target::abi::{FieldIdx, OffsetOfIdx, VariantIdx};
 
 use rustc_ast::Mutability;
 use rustc_span::def_id::LocalDefId;
@@ -1354,7 +1354,7 @@ pub enum NullOp<'tcx> {
     /// Returns the minimum alignment of a type
     AlignOf,
     /// Returns the offset of a field
-    OffsetOf(&'tcx List<FieldIdx>),
+    OffsetOf(&'tcx List<OffsetOfIdx>),
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs
index f1747356139d..2ee726c76bf0 100644
--- a/compiler/rustc_middle/src/thir.rs
+++ b/compiler/rustc_middle/src/thir.rs
@@ -25,7 +25,8 @@ use rustc_middle::ty::{
 };
 use rustc_span::def_id::LocalDefId;
 use rustc_span::{sym, ErrorGuaranteed, Span, Symbol, DUMMY_SP};
-use rustc_target::abi::{FieldIdx, VariantIdx};
+use rustc_span::{sym, Span, Symbol, DUMMY_SP};
+use rustc_target::abi::{FieldIdx, OffsetOfIdx, VariantIdx};
 use rustc_target::asm::InlineAsmRegOrRegClass;
 use std::fmt;
 use std::ops::Index;
@@ -490,7 +491,7 @@ pub enum ExprKind<'tcx> {
     /// Field offset (`offset_of!`)
     OffsetOf {
         container: Ty<'tcx>,
-        fields: &'tcx List<FieldIdx>,
+        fields: &'tcx List<OffsetOfIdx>,
     },
     /// An expression taking a reference to a thread local.
     ThreadLocalRef(DefId),
diff --git a/compiler/rustc_middle/src/ty/codec.rs b/compiler/rustc_middle/src/ty/codec.rs
index d52a717b6b09..ce5198ee3baf 100644
--- a/compiler/rustc_middle/src/ty/codec.rs
+++ b/compiler/rustc_middle/src/ty/codec.rs
@@ -19,7 +19,7 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::ty::TyCtxt;
 use rustc_serialize::{Decodable, Encodable};
 use rustc_span::Span;
-use rustc_target::abi::FieldIdx;
+use rustc_target::abi::{FieldIdx, OffsetOfIdx};
 pub use rustc_type_ir::{TyDecoder, TyEncoder};
 use std::hash::Hash;
 use std::intrinsics;
@@ -414,6 +414,15 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for ty::List<Fi
     }
 }
 
+impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for ty::List<OffsetOfIdx> {
+    fn decode(decoder: &mut D) -> &'tcx Self {
+        let len = decoder.read_usize();
+        decoder
+            .interner()
+            .mk_offset_of_from_iter((0..len).map::<OffsetOfIdx, _>(|_| Decodable::decode(decoder)))
+    }
+}
+
 impl_decodable_via_ref! {
     &'tcx ty::TypeckResults<'tcx>,
     &'tcx ty::List<Ty<'tcx>>,
@@ -426,6 +435,7 @@ impl_decodable_via_ref! {
     &'tcx ty::List<ty::BoundVariableKind>,
     &'tcx ty::List<ty::Clause<'tcx>>,
     &'tcx ty::List<FieldIdx>,
+    &'tcx ty::List<OffsetOfIdx>,
 }
 
 #[macro_export]
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index 0b3b6700cec1..c89ae1e661ed 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -63,7 +63,7 @@ use rustc_session::{Limit, MetadataKind, Session};
 use rustc_span::def_id::{DefPathHash, StableCrateId};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
-use rustc_target::abi::{FieldIdx, Layout, LayoutS, TargetDataLayout, VariantIdx};
+use rustc_target::abi::{FieldIdx, Layout, LayoutS, OffsetOfIdx, TargetDataLayout, VariantIdx};
 use rustc_target::spec::abi;
 use rustc_type_ir::TyKind::*;
 use rustc_type_ir::WithCachedTypeInfo;
@@ -163,6 +163,7 @@ pub struct CtxtInterners<'tcx> {
     predefined_opaques_in_body: InternedSet<'tcx, PredefinedOpaquesData<'tcx>>,
     fields: InternedSet<'tcx, List<FieldIdx>>,
     local_def_ids: InternedSet<'tcx, List<LocalDefId>>,
+    offset_of: InternedSet<'tcx, List<OffsetOfIdx>>,
 }
 
 impl<'tcx> CtxtInterners<'tcx> {
@@ -189,6 +190,7 @@ impl<'tcx> CtxtInterners<'tcx> {
             predefined_opaques_in_body: Default::default(),
             fields: Default::default(),
             local_def_ids: Default::default(),
+            offset_of: Default::default(),
         }
     }
 
@@ -1587,6 +1589,7 @@ slice_interners!(
     bound_variable_kinds: pub mk_bound_variable_kinds(ty::BoundVariableKind),
     fields: pub mk_fields(FieldIdx),
     local_def_ids: intern_local_def_ids(LocalDefId),
+    offset_of: pub mk_offset_of(OffsetOfIdx),
 );
 
 impl<'tcx> TyCtxt<'tcx> {
@@ -1914,6 +1917,14 @@ impl<'tcx> TyCtxt<'tcx> {
         T::collect_and_apply(iter, |xs| self.mk_fields(xs))
     }
 
+    pub fn mk_offset_of_from_iter<I, T>(self, iter: I) -> T::Output
+    where
+        I: Iterator<Item = T>,
+        T: CollectAndApply<OffsetOfIdx, &'tcx List<OffsetOfIdx>>,
+    {
+        T::collect_and_apply(iter, |xs| self.mk_offset_of(xs))
+    }
+
     pub fn mk_args_trait(
         self,
         self_ty: Ty<'tcx>,
diff --git a/compiler/rustc_middle/src/ty/typeck_results.rs b/compiler/rustc_middle/src/ty/typeck_results.rs
index 58ad1eb900fd..623d8d61d2d5 100644
--- a/compiler/rustc_middle/src/ty/typeck_results.rs
+++ b/compiler/rustc_middle/src/ty/typeck_results.rs
@@ -24,7 +24,7 @@ use rustc_macros::HashStable;
 use rustc_middle::mir::FakeReadCause;
 use rustc_session::Session;
 use rustc_span::Span;
-use rustc_target::abi::FieldIdx;
+use rustc_target::abi::{FieldIdx, OffsetOfIdx};
 use std::{collections::hash_map::Entry, hash::Hash, iter};
 
 use super::RvalueScopes;
@@ -205,7 +205,7 @@ pub struct TypeckResults<'tcx> {
     pub closure_size_eval: LocalDefIdMap<ClosureSizeProfileData<'tcx>>,
 
     /// Container types and field indices of `offset_of!` expressions
-    offset_of_data: ItemLocalMap<(Ty<'tcx>, Vec<FieldIdx>)>,
+    offset_of_data: ItemLocalMap<(Ty<'tcx>, Vec<OffsetOfIdx>)>,
 }
 
 impl<'tcx> TypeckResults<'tcx> {
@@ -464,11 +464,13 @@ impl<'tcx> TypeckResults<'tcx> {
         &self.coercion_casts
     }
 
-    pub fn offset_of_data(&self) -> LocalTableInContext<'_, (Ty<'tcx>, Vec<FieldIdx>)> {
+    pub fn offset_of_data(&self) -> LocalTableInContext<'_, (Ty<'tcx>, Vec<OffsetOfIdx>)> {
         LocalTableInContext { hir_owner: self.hir_owner, data: &self.offset_of_data }
     }
 
-    pub fn offset_of_data_mut(&mut self) -> LocalTableInContextMut<'_, (Ty<'tcx>, Vec<FieldIdx>)> {
+    pub fn offset_of_data_mut(
+        &mut self,
+    ) -> LocalTableInContextMut<'_, (Ty<'tcx>, Vec<OffsetOfIdx>)> {
         LocalTableInContextMut { hir_owner: self.hir_owner, data: &mut self.offset_of_data }
     }
 }
diff --git a/compiler/rustc_mir_build/src/thir/cx/expr.rs b/compiler/rustc_mir_build/src/thir/cx/expr.rs
index 6c3564a20f6b..dfd39b512e24 100644
--- a/compiler/rustc_mir_build/src/thir/cx/expr.rs
+++ b/compiler/rustc_mir_build/src/thir/cx/expr.rs
@@ -670,7 +670,7 @@ impl<'tcx> Cx<'tcx> {
             hir::ExprKind::OffsetOf(_, _) => {
                 let data = self.typeck_results.offset_of_data();
                 let &(container, ref indices) = data.get(expr.hir_id).unwrap();
-                let fields = tcx.mk_fields_from_iter(indices.iter().copied());
+                let fields = tcx.mk_offset_of_from_iter(indices.iter().copied());
 
                 ExprKind::OffsetOf { container, fields }
             }
diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
index fd067cb234bd..ca10dc48eb80 100644
--- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
+++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
@@ -287,7 +287,7 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> {
                     NullOp::SizeOf if layout.is_sized() => layout.size.bytes(),
                     NullOp::AlignOf if layout.is_sized() => layout.align.abi.bytes(),
                     NullOp::OffsetOf(fields) => layout
-                        .offset_of_subfield(&self.ecx, fields.iter().map(|f| f.index()))
+                        .offset_of_subfield(&self.ecx, fields.iter())
                         .bytes(),
                     _ => return ValueOrPlace::Value(FlatSet::Top),
                 };
diff --git a/compiler/rustc_passes/src/dead.rs b/compiler/rustc_passes/src/dead.rs
index 87d4850b475b..3b4f6a618c8b 100644
--- a/compiler/rustc_passes/src/dead.rs
+++ b/compiler/rustc_passes/src/dead.rs
@@ -248,6 +248,8 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
     }
 
     fn handle_offset_of(&mut self, expr: &'tcx hir::Expr<'tcx>) {
+        use rustc_target::abi::OffsetOfIdx::*;
+
         let data = self.typeck_results().offset_of_data();
         let &(container, ref indices) =
             data.get(expr.hir_id).expect("no offset_of_data for offset_of");
@@ -256,11 +258,23 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
         let param_env = self.tcx.param_env(body_did);
 
         let mut current_ty = container;
+        let mut indices = indices.into_iter();
 
-        for &index in indices {
-            match current_ty.kind() {
-                ty::Adt(def, subst) => {
-                    let field = &def.non_enum_variant().fields[index];
+        while let Some(&index) = indices.next() {
+            match (current_ty.kind(), index) {
+                (ty::Adt(def, subst), Field(field)) if !def.is_enum() => {
+                    let field = &def.non_enum_variant().fields[field];
+
+                    self.insert_def_id(field.did);
+                    let field_ty = field.ty(self.tcx, subst);
+
+                    current_ty = self.tcx.normalize_erasing_regions(param_env, field_ty);
+                }
+                (ty::Adt(def, subst), Variant(variant)) if def.is_enum() => {
+                    let Some(&Field(field)) = indices.next() else {
+                        span_bug!(expr.span, "variant must be followed by field in offset_of")
+                    };
+                    let field = &def.variant(variant).fields[field];
 
                     self.insert_def_id(field.did);
                     let field_ty = field.ty(self.tcx, subst);
@@ -269,11 +283,12 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
                 }
                 // we don't need to mark tuple fields as live,
                 // but we may need to mark subfields
-                ty::Tuple(tys) => {
+                (ty::Tuple(tys), Field(field)) => {
                     current_ty =
-                        self.tcx.normalize_erasing_regions(param_env, tys[index.as_usize()]);
+                        self.tcx.normalize_erasing_regions(param_env, tys[field.as_usize()]);
                 }
-                _ => span_bug!(expr.span, "named field access on non-ADT"),
+                (_, Field(_)) => span_bug!(expr.span, "named field access on non-ADT"),
+                (_, Variant(_)) => span_bug!(expr.span, "enum variant access on non-enum"),
             }
         }
     }
diff --git a/compiler/rustc_smir/src/rustc_smir/mod.rs b/compiler/rustc_smir/src/rustc_smir/mod.rs
index 5ab5a048ffaf..e64afa5f8d93 100644
--- a/compiler/rustc_smir/src/rustc_smir/mod.rs
+++ b/compiler/rustc_smir/src/rustc_smir/mod.rs
@@ -16,7 +16,7 @@ use rustc_middle::mir::interpret::{alloc_range, AllocId};
 use rustc_middle::mir::mono::MonoItem;
 use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TyCtxt, Variance};
 use rustc_span::def_id::{CrateNum, DefId, LOCAL_CRATE};
-use rustc_target::abi::FieldIdx;
+use rustc_target::abi::{FieldIdx, OffsetOfIdx};
 use stable_mir::mir::mono::InstanceDef;
 use stable_mir::mir::{Body, CopyNonOverlapping, Statement, UserTypeProjection, VariantIdx};
 use stable_mir::ty::{
@@ -643,6 +643,16 @@ impl<'tcx> Stable<'tcx> for FieldIdx {
     }
 }
 
+impl<'tcx> Stable<'tcx> for OffsetOfIdx {
+    type T = usize;
+    fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
+        match self {
+            OffsetOfIdx::Field(f) => f.as_usize(),
+            OffsetOfIdx::Variant(v) => v.as_usize(),
+        }
+    }
+}
+
 impl<'tcx> Stable<'tcx> for mir::Operand<'tcx> {
     type T = stable_mir::mir::Operand;
     fn stable(&self, tables: &mut Tables<'tcx>) -> Self::T {
diff --git a/compiler/rustc_target/src/abi/mod.rs b/compiler/rustc_target/src/abi/mod.rs
index f6f8b53d130b..a37a5bb26893 100644
--- a/compiler/rustc_target/src/abi/mod.rs
+++ b/compiler/rustc_target/src/abi/mod.rs
@@ -250,7 +250,7 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
         Ty::is_transparent(self)
     }
 
-    pub fn offset_of_subfield<C>(self, cx: &C, indices: impl Iterator<Item = usize>) -> Size
+    pub fn offset_of_subfield<C>(self, cx: &C, indices: impl Iterator<Item = OffsetOfIdx>) -> Size
     where
         Ty: TyAbiInterface<'a, C>,
     {
@@ -258,13 +258,21 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
         let mut offset = Size::ZERO;
 
         for index in indices {
-            offset += layout.fields.offset(index);
-            layout = layout.field(cx, index);
-            assert!(
-                layout.is_sized(),
-                "offset of unsized field (type {:?}) cannot be computed statically",
-                layout.ty
-            );
+            match index {
+                OffsetOfIdx::Field(field) => {
+                    let index = field.index();
+                    offset += layout.fields.offset(index);
+                    layout = layout.field(cx, index);
+                    assert!(
+                        layout.is_sized(),
+                        "offset of unsized field (type {:?}) cannot be computed statically",
+                        layout.ty
+                    );
+                }
+                OffsetOfIdx::Variant(variant) => {
+                    layout = layout.for_variant(cx, variant);
+                }
+            }
         }
 
         offset
diff --git a/tests/mir-opt/const_prop/offset_of.concrete.ConstProp.panic-abort.diff b/tests/mir-opt/const_prop/offset_of.concrete.ConstProp.panic-abort.diff
index c73d217aeec4..e460c08f9bb8 100644
--- a/tests/mir-opt/const_prop/offset_of.concrete.ConstProp.panic-abort.diff
+++ b/tests/mir-opt/const_prop/offset_of.concrete.ConstProp.panic-abort.diff
@@ -8,6 +8,9 @@
       let mut _4: usize;
       let mut _6: usize;
       let mut _8: usize;
+      let mut _10: usize;
+      let mut _12: usize;
+      let mut _14: usize;
       scope 1 {
           debug x => _1;
           let _3: usize;
@@ -19,6 +22,18 @@
                   let _7: usize;
                   scope 4 {
                       debug z1 => _7;
+                      let _9: usize;
+                      scope 5 {
+                          debug eA0 => _9;
+                          let _11: usize;
+                          scope 6 {
+                              debug eA1 => _11;
+                              let _13: usize;
+                              scope 7 {
+                                  debug eC => _13;
+                              }
+                          }
+                      }
                   }
               }
           }
@@ -27,7 +42,7 @@
       bb0: {
           StorageLive(_1);
           StorageLive(_2);
--         _2 = OffsetOf(Alpha, [0]);
+-         _2 = OffsetOf(Alpha, [Field(0)]);
 -         _1 = must_use::<usize>(move _2) -> [return: bb1, unwind unreachable];
 +         _2 = const 4_usize;
 +         _1 = must_use::<usize>(const 4_usize) -> [return: bb1, unwind unreachable];
@@ -37,7 +52,7 @@
           StorageDead(_2);
           StorageLive(_3);
           StorageLive(_4);
--         _4 = OffsetOf(Alpha, [1]);
+-         _4 = OffsetOf(Alpha, [Field(1)]);
 -         _3 = must_use::<usize>(move _4) -> [return: bb2, unwind unreachable];
 +         _4 = const 0_usize;
 +         _3 = must_use::<usize>(const 0_usize) -> [return: bb2, unwind unreachable];
@@ -47,7 +62,7 @@
           StorageDead(_4);
           StorageLive(_5);
           StorageLive(_6);
--         _6 = OffsetOf(Alpha, [2, 0]);
+-         _6 = OffsetOf(Alpha, [Field(2), Field(0)]);
 -         _5 = must_use::<usize>(move _6) -> [return: bb3, unwind unreachable];
 +         _6 = const 2_usize;
 +         _5 = must_use::<usize>(const 2_usize) -> [return: bb3, unwind unreachable];
@@ -57,7 +72,7 @@
           StorageDead(_6);
           StorageLive(_7);
           StorageLive(_8);
--         _8 = OffsetOf(Alpha, [2, 1]);
+-         _8 = OffsetOf(Alpha, [Field(2), Field(1)]);
 -         _7 = must_use::<usize>(move _8) -> [return: bb4, unwind unreachable];
 +         _8 = const 3_usize;
 +         _7 = must_use::<usize>(const 3_usize) -> [return: bb4, unwind unreachable];
@@ -65,7 +80,40 @@
   
       bb4: {
           StorageDead(_8);
+          StorageLive(_9);
+          StorageLive(_10);
+-         _10 = OffsetOf(Epsilon, [Variant(0), Field(0)]);
+-         _9 = must_use::<usize>(move _10) -> [return: bb5, unwind unreachable];
++         _10 = const 1_usize;
++         _9 = must_use::<usize>(const 1_usize) -> [return: bb5, unwind unreachable];
+      }
+  
+      bb5: {
+          StorageDead(_10);
+          StorageLive(_11);
+          StorageLive(_12);
+-         _12 = OffsetOf(Epsilon, [Variant(0), Field(1)]);
+-         _11 = must_use::<usize>(move _12) -> [return: bb6, unwind unreachable];
++         _12 = const 2_usize;
++         _11 = must_use::<usize>(const 2_usize) -> [return: bb6, unwind unreachable];
+      }
+  
+      bb6: {
+          StorageDead(_12);
+          StorageLive(_13);
+          StorageLive(_14);
+-         _14 = OffsetOf(Epsilon, [Variant(2), Field(0)]);
+-         _13 = must_use::<usize>(move _14) -> [return: bb7, unwind unreachable];
++         _14 = const 4_usize;
++         _13 = must_use::<usize>(const 4_usize) -> [return: bb7, unwind unreachable];
+      }
+  
+      bb7: {
+          StorageDead(_14);
           _0 = const ();
+          StorageDead(_13);
+          StorageDead(_11);
+          StorageDead(_9);
           StorageDead(_7);
           StorageDead(_5);
           StorageDead(_3);
diff --git a/tests/mir-opt/const_prop/offset_of.concrete.ConstProp.panic-unwind.diff b/tests/mir-opt/const_prop/offset_of.concrete.ConstProp.panic-unwind.diff
index 913ffca4ae90..3b8d0d75101f 100644
--- a/tests/mir-opt/const_prop/offset_of.concrete.ConstProp.panic-unwind.diff
+++ b/tests/mir-opt/const_prop/offset_of.concrete.ConstProp.panic-unwind.diff
@@ -8,6 +8,9 @@
       let mut _4: usize;
       let mut _6: usize;
       let mut _8: usize;
+      let mut _10: usize;
+      let mut _12: usize;
+      let mut _14: usize;
       scope 1 {
           debug x => _1;
           let _3: usize;
@@ -19,6 +22,18 @@
                   let _7: usize;
                   scope 4 {
                       debug z1 => _7;
+                      let _9: usize;
+                      scope 5 {
+                          debug eA0 => _9;
+                          let _11: usize;
+                          scope 6 {
+                              debug eA1 => _11;
+                              let _13: usize;
+                              scope 7 {
+                                  debug eC => _13;
+                              }
+                          }
+                      }
                   }
               }
           }
@@ -27,7 +42,7 @@
       bb0: {
           StorageLive(_1);
           StorageLive(_2);
--         _2 = OffsetOf(Alpha, [0]);
+-         _2 = OffsetOf(Alpha, [Field(0)]);
 -         _1 = must_use::<usize>(move _2) -> [return: bb1, unwind continue];
 +         _2 = const 4_usize;
 +         _1 = must_use::<usize>(const 4_usize) -> [return: bb1, unwind continue];
@@ -37,7 +52,7 @@
           StorageDead(_2);
           StorageLive(_3);
           StorageLive(_4);
--         _4 = OffsetOf(Alpha, [1]);
+-         _4 = OffsetOf(Alpha, [Field(1)]);
 -         _3 = must_use::<usize>(move _4) -> [return: bb2, unwind continue];
 +         _4 = const 0_usize;
 +         _3 = must_use::<usize>(const 0_usize) -> [return: bb2, unwind continue];
@@ -47,7 +62,7 @@
           StorageDead(_4);
           StorageLive(_5);
           StorageLive(_6);
--         _6 = OffsetOf(Alpha, [2, 0]);
+-         _6 = OffsetOf(Alpha, [Field(2), Field(0)]);
 -         _5 = must_use::<usize>(move _6) -> [return: bb3, unwind continue];
 +         _6 = const 2_usize;
 +         _5 = must_use::<usize>(const 2_usize) -> [return: bb3, unwind continue];
@@ -57,7 +72,7 @@
           StorageDead(_6);
           StorageLive(_7);
           StorageLive(_8);
--         _8 = OffsetOf(Alpha, [2, 1]);
+-         _8 = OffsetOf(Alpha, [Field(2), Field(1)]);
 -         _7 = must_use::<usize>(move _8) -> [return: bb4, unwind continue];
 +         _8 = const 3_usize;
 +         _7 = must_use::<usize>(const 3_usize) -> [return: bb4, unwind continue];
@@ -65,7 +80,40 @@
   
       bb4: {
           StorageDead(_8);
+          StorageLive(_9);
+          StorageLive(_10);
+-         _10 = OffsetOf(Epsilon, [Variant(0), Field(0)]);
+-         _9 = must_use::<usize>(move _10) -> [return: bb5, unwind continue];
++         _10 = const 1_usize;
++         _9 = must_use::<usize>(const 1_usize) -> [return: bb5, unwind continue];
+      }
+  
+      bb5: {
+          StorageDead(_10);
+          StorageLive(_11);
+          StorageLive(_12);
+-         _12 = OffsetOf(Epsilon, [Variant(0), Field(1)]);
+-         _11 = must_use::<usize>(move _12) -> [return: bb6, unwind continue];
++         _12 = const 2_usize;
++         _11 = must_use::<usize>(const 2_usize) -> [return: bb6, unwind continue];
+      }
+  
+      bb6: {
+          StorageDead(_12);
+          StorageLive(_13);
+          StorageLive(_14);
+-         _14 = OffsetOf(Epsilon, [Variant(2), Field(0)]);
+-         _13 = must_use::<usize>(move _14) -> [return: bb7, unwind continue];
++         _14 = const 4_usize;
++         _13 = must_use::<usize>(const 4_usize) -> [return: bb7, unwind continue];
+      }
+  
+      bb7: {
+          StorageDead(_14);
           _0 = const ();
+          StorageDead(_13);
+          StorageDead(_11);
+          StorageDead(_9);
           StorageDead(_7);
           StorageDead(_5);
           StorageDead(_3);
diff --git a/tests/mir-opt/const_prop/offset_of.generic.ConstProp.panic-abort.diff b/tests/mir-opt/const_prop/offset_of.generic.ConstProp.panic-abort.diff
index 7519331f6d7d..f26b104e065d 100644
--- a/tests/mir-opt/const_prop/offset_of.generic.ConstProp.panic-abort.diff
+++ b/tests/mir-opt/const_prop/offset_of.generic.ConstProp.panic-abort.diff
@@ -8,6 +8,9 @@
       let mut _4: usize;
       let mut _6: usize;
       let mut _8: usize;
+      let mut _10: usize;
+      let mut _12: usize;
+      let mut _14: usize;
       scope 1 {
           debug gx => _1;
           let _3: usize;
@@ -19,6 +22,18 @@
                   let _7: usize;
                   scope 4 {
                       debug dy => _7;
+                      let _9: usize;
+                      scope 5 {
+                          debug zA0 => _9;
+                          let _11: usize;
+                          scope 6 {
+                              debug zA1 => _11;
+                              let _13: usize;
+                              scope 7 {
+                                  debug zB => _13;
+                              }
+                          }
+                      }
                   }
               }
           }
@@ -27,7 +42,7 @@
       bb0: {
           StorageLive(_1);
           StorageLive(_2);
-          _2 = OffsetOf(Gamma<T>, [0]);
+          _2 = OffsetOf(Gamma<T>, [Field(0)]);
           _1 = must_use::<usize>(move _2) -> [return: bb1, unwind unreachable];
       }
   
@@ -35,7 +50,7 @@
           StorageDead(_2);
           StorageLive(_3);
           StorageLive(_4);
-          _4 = OffsetOf(Gamma<T>, [1]);
+          _4 = OffsetOf(Gamma<T>, [Field(1)]);
           _3 = must_use::<usize>(move _4) -> [return: bb2, unwind unreachable];
       }
   
@@ -43,7 +58,7 @@
           StorageDead(_4);
           StorageLive(_5);
           StorageLive(_6);
-          _6 = OffsetOf(Delta<T>, [1]);
+          _6 = OffsetOf(Delta<T>, [Field(1)]);
           _5 = must_use::<usize>(move _6) -> [return: bb3, unwind unreachable];
       }
   
@@ -51,13 +66,40 @@
           StorageDead(_6);
           StorageLive(_7);
           StorageLive(_8);
-          _8 = OffsetOf(Delta<T>, [2]);
+          _8 = OffsetOf(Delta<T>, [Field(2)]);
           _7 = must_use::<usize>(move _8) -> [return: bb4, unwind unreachable];
       }
   
       bb4: {
           StorageDead(_8);
+          StorageLive(_9);
+          StorageLive(_10);
+          _10 = OffsetOf(Zeta<T>, [Variant(0), Field(0)]);
+          _9 = must_use::<usize>(move _10) -> [return: bb5, unwind unreachable];
+      }
+  
+      bb5: {
+          StorageDead(_10);
+          StorageLive(_11);
+          StorageLive(_12);
+          _12 = OffsetOf(Zeta<T>, [Variant(0), Field(1)]);
+          _11 = must_use::<usize>(move _12) -> [return: bb6, unwind unreachable];
+      }
+  
+      bb6: {
+          StorageDead(_12);
+          StorageLive(_13);
+          StorageLive(_14);
+          _14 = OffsetOf(Zeta<T>, [Variant(1), Field(0)]);
+          _13 = must_use::<usize>(move _14) -> [return: bb7, unwind unreachable];
+      }
+  
+      bb7: {
+          StorageDead(_14);
           _0 = const ();
+          StorageDead(_13);
+          StorageDead(_11);
+          StorageDead(_9);
           StorageDead(_7);
           StorageDead(_5);
           StorageDead(_3);
diff --git a/tests/mir-opt/const_prop/offset_of.generic.ConstProp.panic-unwind.diff b/tests/mir-opt/const_prop/offset_of.generic.ConstProp.panic-unwind.diff
index fd5206e460c6..0f8750fce460 100644
--- a/tests/mir-opt/const_prop/offset_of.generic.ConstProp.panic-unwind.diff
+++ b/tests/mir-opt/const_prop/offset_of.generic.ConstProp.panic-unwind.diff
@@ -8,6 +8,9 @@
       let mut _4: usize;
       let mut _6: usize;
       let mut _8: usize;
+      let mut _10: usize;
+      let mut _12: usize;
+      let mut _14: usize;
       scope 1 {
           debug gx => _1;
           let _3: usize;
@@ -19,6 +22,18 @@
                   let _7: usize;
                   scope 4 {
                       debug dy => _7;
+                      let _9: usize;
+                      scope 5 {
+                          debug zA0 => _9;
+                          let _11: usize;
+                          scope 6 {
+                              debug zA1 => _11;
+                              let _13: usize;
+                              scope 7 {
+                                  debug zB => _13;
+                              }
+                          }
+                      }
                   }
               }
           }
@@ -27,7 +42,7 @@
       bb0: {
           StorageLive(_1);
           StorageLive(_2);
-          _2 = OffsetOf(Gamma<T>, [0]);
+          _2 = OffsetOf(Gamma<T>, [Field(0)]);
           _1 = must_use::<usize>(move _2) -> [return: bb1, unwind continue];
       }
   
@@ -35,7 +50,7 @@
           StorageDead(_2);
           StorageLive(_3);
           StorageLive(_4);
-          _4 = OffsetOf(Gamma<T>, [1]);
+          _4 = OffsetOf(Gamma<T>, [Field(1)]);
           _3 = must_use::<usize>(move _4) -> [return: bb2, unwind continue];
       }
   
@@ -43,7 +58,7 @@
           StorageDead(_4);
           StorageLive(_5);
           StorageLive(_6);
-          _6 = OffsetOf(Delta<T>, [1]);
+          _6 = OffsetOf(Delta<T>, [Field(1)]);
           _5 = must_use::<usize>(move _6) -> [return: bb3, unwind continue];
       }
   
@@ -51,13 +66,40 @@
           StorageDead(_6);
           StorageLive(_7);
           StorageLive(_8);
-          _8 = OffsetOf(Delta<T>, [2]);
+          _8 = OffsetOf(Delta<T>, [Field(2)]);
           _7 = must_use::<usize>(move _8) -> [return: bb4, unwind continue];
       }
   
       bb4: {
           StorageDead(_8);
+          StorageLive(_9);
+          StorageLive(_10);
+          _10 = OffsetOf(Zeta<T>, [Variant(0), Field(0)]);
+          _9 = must_use::<usize>(move _10) -> [return: bb5, unwind continue];
+      }
+  
+      bb5: {
+          StorageDead(_10);
+          StorageLive(_11);
+          StorageLive(_12);
+          _12 = OffsetOf(Zeta<T>, [Variant(0), Field(1)]);
+          _11 = must_use::<usize>(move _12) -> [return: bb6, unwind continue];
+      }
+  
+      bb6: {
+          StorageDead(_12);
+          StorageLive(_13);
+          StorageLive(_14);
+          _14 = OffsetOf(Zeta<T>, [Variant(1), Field(0)]);
+          _13 = must_use::<usize>(move _14) -> [return: bb7, unwind continue];
+      }
+  
+      bb7: {
+          StorageDead(_14);
           _0 = const ();
+          StorageDead(_13);
+          StorageDead(_11);
+          StorageDead(_9);
           StorageDead(_7);
           StorageDead(_5);
           StorageDead(_3);
diff --git a/tests/mir-opt/const_prop/offset_of.rs b/tests/mir-opt/const_prop/offset_of.rs
index 21911f8dbb5d..8a5289d58991 100644
--- a/tests/mir-opt/const_prop/offset_of.rs
+++ b/tests/mir-opt/const_prop/offset_of.rs
@@ -28,12 +28,26 @@ struct Delta<T> {
     y: u16,
 }
 
+enum Epsilon {
+    A(u8, u16),
+    B,
+    C { c: u32 }
+}
+
+enum Zeta<T> {
+    A(T, bool),
+    B(char),
+}
+
 // EMIT_MIR offset_of.concrete.ConstProp.diff
 fn concrete() {
     let x = offset_of!(Alpha, x);
     let y = offset_of!(Alpha, y);
     let z0 = offset_of!(Alpha, z.0);
     let z1 = offset_of!(Alpha, z.1);
+    let eA0 = offset_of!(Epsilon, A.0);
+    let eA1 = offset_of!(Epsilon, A.1);
+    let eC = offset_of!(Epsilon, C.c);
 }
 
 // EMIT_MIR offset_of.generic.ConstProp.diff
@@ -42,6 +56,9 @@ fn generic<T>() {
     let gy = offset_of!(Gamma<T>, y);
     let dx = offset_of!(Delta<T>, x);
     let dy = offset_of!(Delta<T>, y);
+    let zA0 = offset_of!(Zeta<T>, A.0);
+    let zA1 = offset_of!(Zeta<T>, A.1);
+    let zB = offset_of!(Zeta<T>, B.0);
 }
 
 fn main() {
diff --git a/tests/ui/offset-of/offset-of-enum.rs b/tests/ui/offset-of/offset-of-enum.rs
index d73505821ff5..92d34e67c71b 100644
--- a/tests/ui/offset-of/offset-of-enum.rs
+++ b/tests/ui/offset-of/offset-of-enum.rs
@@ -9,5 +9,6 @@ enum Alpha {
 
 fn main() {
     offset_of!(Alpha::One, 0); //~ ERROR expected type, found variant `Alpha::One`
-    offset_of!(Alpha, Two.0); //~ ERROR no field `Two` on type `Alpha`
+    offset_of!(Alpha, One); //~ ERROR `One` is an enum variant; expected field at end of `offset_of`
+    offset_of!(Alpha, Two.0);
 }
diff --git a/tests/ui/offset-of/offset-of-enum.stderr b/tests/ui/offset-of/offset-of-enum.stderr
index 6958d199fbdb..1d849e682e69 100644
--- a/tests/ui/offset-of/offset-of-enum.stderr
+++ b/tests/ui/offset-of/offset-of-enum.stderr
@@ -7,13 +7,13 @@ LL |     offset_of!(Alpha::One, 0);
    |                not a type
    |                help: try using the variant's enum: `Alpha`
 
-error[E0609]: no field `Two` on type `Alpha`
+error[E0795]: `One` is an enum variant; expected field at end of `offset_of`
   --> $DIR/offset-of-enum.rs:12:23
    |
-LL |     offset_of!(Alpha, Two.0);
-   |                       ^^^
+LL |     offset_of!(Alpha, One);
+   |                       ^^^ enum variant
 
 error: aborting due to 2 previous errors
 
-Some errors have detailed explanations: E0573, E0609.
+Some errors have detailed explanations: E0573, E0795.
 For more information about an error, try `rustc --explain E0573`.
diff --git a/tests/ui/offset-of/offset-of-private.rs b/tests/ui/offset-of/offset-of-private.rs
index 6b1a16ba62b6..6fa30d63fb86 100644
--- a/tests/ui/offset-of/offset-of-private.rs
+++ b/tests/ui/offset-of/offset-of-private.rs
@@ -8,13 +8,20 @@ mod m {
         pub public: u8,
         private: u8,
     }
+
     #[repr(C)]
     pub struct FooTuple(pub u8, u8);
+
     #[repr(C)]
     struct Bar {
         pub public: u8,
         private: u8,
     }
+
+    pub enum Baz {
+        Var1(Foo),
+        Var2(u64),
+    }
 }
 
 fn main() {
@@ -25,4 +32,8 @@ fn main() {
     offset_of!(m::Bar, public); //~ ERROR struct `Bar` is private
     offset_of!(m::Bar, private); //~ ERROR struct `Bar` is private
     //~| ERROR field `private` of struct `Bar` is private
+
+    offset_of!(m::Baz, Var1.0.public);
+    offset_of!(m::Baz, Var1.0.private); //~ ERROR field `private` of struct `Foo` is private
+    offset_of!(m::Baz, Var2.0);
 }
diff --git a/tests/ui/offset-of/offset-of-private.stderr b/tests/ui/offset-of/offset-of-private.stderr
index 0674b58f8608..930e30e6390f 100644
--- a/tests/ui/offset-of/offset-of-private.stderr
+++ b/tests/ui/offset-of/offset-of-private.stderr
@@ -1,46 +1,52 @@
 error[E0603]: struct `Bar` is private
-  --> $DIR/offset-of-private.rs:25:19
+  --> $DIR/offset-of-private.rs:32:19
    |
 LL |     offset_of!(m::Bar, public);
    |                   ^^^ private struct
    |
 note: the struct `Bar` is defined here
-  --> $DIR/offset-of-private.rs:14:5
+  --> $DIR/offset-of-private.rs:16:5
    |
 LL |     struct Bar {
    |     ^^^^^^^^^^
 
 error[E0603]: struct `Bar` is private
-  --> $DIR/offset-of-private.rs:26:19
+  --> $DIR/offset-of-private.rs:33:19
    |
 LL |     offset_of!(m::Bar, private);
    |                   ^^^ private struct
    |
 note: the struct `Bar` is defined here
-  --> $DIR/offset-of-private.rs:14:5
+  --> $DIR/offset-of-private.rs:16:5
    |
 LL |     struct Bar {
    |     ^^^^^^^^^^
 
 error[E0616]: field `private` of struct `Foo` is private
-  --> $DIR/offset-of-private.rs:22:24
+  --> $DIR/offset-of-private.rs:29:24
    |
 LL |     offset_of!(m::Foo, private);
    |                        ^^^^^^^ private field
 
 error[E0616]: field `1` of struct `FooTuple` is private
-  --> $DIR/offset-of-private.rs:24:29
+  --> $DIR/offset-of-private.rs:31:29
    |
 LL |     offset_of!(m::FooTuple, 1);
    |                             ^ private field
 
 error[E0616]: field `private` of struct `Bar` is private
-  --> $DIR/offset-of-private.rs:26:24
+  --> $DIR/offset-of-private.rs:33:24
    |
 LL |     offset_of!(m::Bar, private);
    |                        ^^^^^^^ private field
 
-error: aborting due to 5 previous errors
+error[E0616]: field `private` of struct `Foo` is private
+  --> $DIR/offset-of-private.rs:37:31
+   |
+LL |     offset_of!(m::Baz, Var1.0.private);
+   |                               ^^^^^^^ private field
+
+error: aborting due to 6 previous errors
 
 Some errors have detailed explanations: E0603, E0616.
 For more information about an error, try `rustc --explain E0603`.

From d995bd61e77986d8466c4379ca575683c594a9c6 Mon Sep 17 00:00:00 2001
From: George Bateman <george.bateman16@gmail.com>
Date: Tue, 15 Aug 2023 22:32:55 +0100
Subject: [PATCH 292/297] Enums in offset_of: update based on est31, scottmcm &
 llogiq review

---
 compiler/rustc_abi/src/lib.rs                 |   7 --
 .../src/transform/validate.rs                 |  36 +++----
 compiler/rustc_hir_typeck/src/expr.rs         | 101 ++++++++++--------
 compiler/rustc_middle/src/mir/syntax.rs       |   4 +-
 compiler/rustc_middle/src/thir.rs             |   4 +-
 compiler/rustc_middle/src/ty/codec.rs         |  14 +--
 compiler/rustc_middle/src/ty/context.rs       |   8 +-
 .../rustc_middle/src/ty/typeck_results.rs     |  10 +-
 compiler/rustc_passes/src/dead.rs             |  25 +----
 compiler/rustc_smir/src/rustc_smir/mod.rs     |  11 +-
 compiler/rustc_target/src/abi/mod.rs          |  29 +++--
 compiler/stable_mir/src/mir/body.rs           |   2 +-
 library/core/src/mem/mod.rs                   |  10 +-
 tests/ui/offset-of/offset-of-enum.rs          |   3 +
 tests/ui/offset-of/offset-of-enum.stderr      |  26 ++++-
 15 files changed, 150 insertions(+), 140 deletions(-)

diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs
index 0654bfd11c5e..9626eeea03bd 100644
--- a/compiler/rustc_abi/src/lib.rs
+++ b/compiler/rustc_abi/src/lib.rs
@@ -1133,13 +1133,6 @@ rustc_index::newtype_index! {
     pub struct FieldIdx {}
 }
 
-/// `offset_of` can traverse fields and enum variants and should keep track of which is which.
-#[derive(Copy, Clone, Debug, Eq, Hash, HashStable_Generic, PartialEq, Encodable, Decodable)]
-pub enum OffsetOfIdx {
-    Field(FieldIdx),
-    Variant(VariantIdx),
-}
-
 /// Describes how the fields of a type are located in memory.
 #[derive(PartialEq, Eq, Hash, Clone, Debug)]
 #[cfg_attr(feature = "nightly", derive(HashStable_Generic))]
diff --git a/compiler/rustc_const_eval/src/transform/validate.rs b/compiler/rustc_const_eval/src/transform/validate.rs
index bdccd9b0edb5..4d71f7636676 100644
--- a/compiler/rustc_const_eval/src/transform/validate.rs
+++ b/compiler/rustc_const_eval/src/transform/validate.rs
@@ -1062,13 +1062,17 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                 };
 
                 let mut current_ty = *container;
-                let mut indices = indices.into_iter();
 
-                use rustc_target::abi::OffsetOfIdx::*;
-
-                while let Some(index) = indices.next() {
-                    match (current_ty.kind(), index) {
-                        (ty::Tuple(fields), Field(field)) => {
+                for (variant, field) in indices.iter() {
+                    match current_ty.kind() {
+                        ty::Tuple(fields) => {
+                            if variant != FIRST_VARIANT {
+                                self.fail(
+                                    location,
+                                    format!("tried to get variant {variant:?} of tuple"),
+                                );
+                                return;
+                            }
                             let Some(&f_ty) = fields.get(field.as_usize()) else {
                                 fail_out_of_bounds(self, location, field, current_ty);
                                 return;
@@ -1076,23 +1080,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
 
                             current_ty = self.tcx.normalize_erasing_regions(self.param_env, f_ty);
                         }
-                        (ty::Adt(adt_def, args), Field(field)) if !adt_def.is_enum() => {
-                            let Some(field) = adt_def.non_enum_variant().fields.get(field) else {
-                                fail_out_of_bounds(self, location, field, current_ty);
-                                return;
-                            };
-
-                            let f_ty = field.ty(self.tcx, args);
-                            current_ty = self.tcx.normalize_erasing_regions(self.param_env, f_ty);
-                        }
-                        (ty::Adt(adt_def, args), Variant(variant)) if adt_def.is_enum() => {
-                            let Some(Field(field)) = indices.next() else {
-                                self.fail(
-                                    location,
-                                    format!("enum variant must be followed by field index in offset_of; in {current_ty:?}"),
-                                );
-                                return;
-                            };
+                        ty::Adt(adt_def, args) => {
                             let Some(field) = adt_def.variant(variant).fields.get(field) else {
                                 fail_out_of_bounds(self, location, field, current_ty);
                                 return;
@@ -1104,7 +1092,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> {
                         _ => {
                             self.fail(
                                 location,
-                                format!("Cannot get offset {index:?} from type {current_ty:?}"),
+                                format!("Cannot get offset ({variant:?}, {field:?}) from type {current_ty:?}"),
                             );
                             return;
                         }
diff --git a/compiler/rustc_hir_typeck/src/expr.rs b/compiler/rustc_hir_typeck/src/expr.rs
index 394a9033765a..bdac886bf9e7 100644
--- a/compiler/rustc_hir_typeck/src/expr.rs
+++ b/compiler/rustc_hir_typeck/src/expr.rs
@@ -54,7 +54,7 @@ use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::hygiene::DesugaringKind;
 use rustc_span::source_map::{Span, Spanned};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
-use rustc_target::abi::FieldIdx;
+use rustc_target::abi::{FieldIdx, FIRST_VARIANT};
 use rustc_target::spec::abi::Abi::RustIntrinsic;
 use rustc_trait_selection::infer::InferCtxtExt;
 use rustc_trait_selection::traits::error_reporting::TypeErrCtxtExt;
@@ -3103,8 +3103,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
         fields: &[Ident],
         expr: &'tcx hir::Expr<'tcx>,
     ) -> Ty<'tcx> {
-        use rustc_target::abi::OffsetOfIdx::*;
-
         let container = self.to_ty(container).normalized;
 
         let mut field_indices = Vec::with_capacity(fields.len());
@@ -3120,49 +3118,68 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                     let (ident, _def_scope) =
                         self.tcx.adjust_ident_and_get_scope(field, container_def.did(), block);
 
-                    if let Some((index, variant)) = container_def.variants()
+                    let Some((index, variant)) = container_def.variants()
                         .iter_enumerated()
-                        .find(|(_, v)| v.ident(self.tcx).normalize_to_macros_2_0() == ident)
-                    {
-                        let Some(&subfield) = fields.next() else {
-                            let mut err = type_error_struct!(
-                                self.tcx().sess,
-                                ident.span,
-                                container,
-                                E0795,
-                                "`{ident}` is an enum variant; expected field at end of `offset_of`",
-                                );
-                            err.span_label(field.span, "enum variant");
-                            err.emit();
-                            break;
-                        };
-                        let (subident, sub_def_scope) =
-                            self.tcx.adjust_ident_and_get_scope(subfield, variant.def_id, block);
+                        .find(|(_, v)| v.ident(self.tcx).normalize_to_macros_2_0() == ident) else {
+                        let mut err = type_error_struct!(
+                            self.tcx().sess,
+                            ident.span,
+                            container,
+                            E0599,
+                            "no variant named `{ident}` found for enum `{container}`",
+                            );
+                        err.span_label(field.span, "variant not found");
+                        err.emit();
+                        break;
+                    };
+                    let Some(&subfield) = fields.next() else {
+                        let mut err = type_error_struct!(
+                            self.tcx().sess,
+                            ident.span,
+                            container,
+                            E0795,
+                            "`{ident}` is an enum variant; expected field at end of `offset_of`",
+                            );
+                        err.span_label(field.span, "enum variant");
+                        err.emit();
+                        break;
+                    };
+                    let (subident, sub_def_scope) =
+                        self.tcx.adjust_ident_and_get_scope(subfield, variant.def_id, block);
 
-                        if let Some((subindex, field)) = variant.fields
-                            .iter_enumerated()
-                            .find(|(_, f)| f.ident(self.tcx).normalize_to_macros_2_0() == subident)
-                        {
-                            let field_ty = self.field_ty(expr.span, field, args);
+                    let Some((subindex, field)) = variant.fields
+                        .iter_enumerated()
+                        .find(|(_, f)| f.ident(self.tcx).normalize_to_macros_2_0() == subident) else {
+                        let mut err = type_error_struct!(
+                            self.tcx().sess,
+                            ident.span,
+                            container,
+                            E0609,
+                            "no field named `{subfield}` on enum variant `{container}::{ident}`",
+                            );
+                        err.span_label(field.span, "this enum variant...");
+                        err.span_label(subident.span, "...does not have this field");
+                        err.emit();
+                        break;
+                    };
 
-                            // FIXME: DSTs with static alignment should be allowed
-                            self.require_type_is_sized(field_ty, expr.span, traits::MiscObligation);
+                    let field_ty = self.field_ty(expr.span, field, args);
 
-                            if field.vis.is_accessible_from(sub_def_scope, self.tcx) {
-                                self.tcx.check_stability(field.did, Some(expr.hir_id), expr.span, None);
-                            } else {
-                                self.private_field_err(ident, container_def.did()).emit();
-                            }
+                    // FIXME: DSTs with static alignment should be allowed
+                    self.require_type_is_sized(field_ty, expr.span, traits::MiscObligation);
 
-                            // Save the index of all fields regardless of their visibility in case
-                            // of error recovery.
-                            field_indices.push(Variant(index));
-                            field_indices.push(Field(subindex));
-                            current_container = field_ty;
-
-                            continue;
-                        }
+                    if field.vis.is_accessible_from(sub_def_scope, self.tcx) {
+                        self.tcx.check_stability(field.did, Some(expr.hir_id), expr.span, None);
+                    } else {
+                        self.private_field_err(ident, container_def.did()).emit();
                     }
+
+                    // Save the index of all fields regardless of their visibility in case
+                    // of error recovery.
+                    field_indices.push((index, subindex));
+                    current_container = field_ty;
+
+                    continue;
                 }
                 ty::Adt(container_def, args) => {
                     let block = self.tcx.hir().local_def_id_to_hir_id(self.body_id);
@@ -3187,7 +3204,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
 
                         // Save the index of all fields regardless of their visibility in case
                         // of error recovery.
-                        field_indices.push(Field(index));
+                        field_indices.push((FIRST_VARIANT, index));
                         current_container = field_ty;
 
                         continue;
@@ -3201,7 +3218,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                             self.require_type_is_sized(ty, expr.span, traits::MiscObligation);
                         }
                         if let Some(&field_ty) = tys.get(index) {
-                            field_indices.push(Field(index.into()));
+                            field_indices.push((FIRST_VARIANT, index.into()));
                             current_container = field_ty;
 
                             continue;
diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs
index d0094a81634d..b6543affc6db 100644
--- a/compiler/rustc_middle/src/mir/syntax.rs
+++ b/compiler/rustc_middle/src/mir/syntax.rs
@@ -17,7 +17,7 @@ use rustc_hir::def_id::DefId;
 use rustc_hir::{self as hir};
 use rustc_hir::{self, CoroutineKind};
 use rustc_index::IndexVec;
-use rustc_target::abi::{FieldIdx, OffsetOfIdx, VariantIdx};
+use rustc_target::abi::{FieldIdx, VariantIdx};
 
 use rustc_ast::Mutability;
 use rustc_span::def_id::LocalDefId;
@@ -1354,7 +1354,7 @@ pub enum NullOp<'tcx> {
     /// Returns the minimum alignment of a type
     AlignOf,
     /// Returns the offset of a field
-    OffsetOf(&'tcx List<OffsetOfIdx>),
+    OffsetOf(&'tcx List<(VariantIdx, FieldIdx)>),
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs
index 2ee726c76bf0..3780eb4f9ac9 100644
--- a/compiler/rustc_middle/src/thir.rs
+++ b/compiler/rustc_middle/src/thir.rs
@@ -26,7 +26,7 @@ use rustc_middle::ty::{
 use rustc_span::def_id::LocalDefId;
 use rustc_span::{sym, ErrorGuaranteed, Span, Symbol, DUMMY_SP};
 use rustc_span::{sym, Span, Symbol, DUMMY_SP};
-use rustc_target::abi::{FieldIdx, OffsetOfIdx, VariantIdx};
+use rustc_target::abi::{FieldIdx, VariantIdx};
 use rustc_target::asm::InlineAsmRegOrRegClass;
 use std::fmt;
 use std::ops::Index;
@@ -491,7 +491,7 @@ pub enum ExprKind<'tcx> {
     /// Field offset (`offset_of!`)
     OffsetOf {
         container: Ty<'tcx>,
-        fields: &'tcx List<OffsetOfIdx>,
+        fields: &'tcx List<(VariantIdx, FieldIdx)>,
     },
     /// An expression taking a reference to a thread local.
     ThreadLocalRef(DefId),
diff --git a/compiler/rustc_middle/src/ty/codec.rs b/compiler/rustc_middle/src/ty/codec.rs
index ce5198ee3baf..8b67e39667b8 100644
--- a/compiler/rustc_middle/src/ty/codec.rs
+++ b/compiler/rustc_middle/src/ty/codec.rs
@@ -19,7 +19,7 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_middle::ty::TyCtxt;
 use rustc_serialize::{Decodable, Encodable};
 use rustc_span::Span;
-use rustc_target::abi::{FieldIdx, OffsetOfIdx};
+use rustc_target::abi::{FieldIdx, VariantIdx};
 pub use rustc_type_ir::{TyDecoder, TyEncoder};
 use std::hash::Hash;
 use std::intrinsics;
@@ -414,12 +414,14 @@ impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for ty::List<Fi
     }
 }
 
-impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D> for ty::List<OffsetOfIdx> {
+impl<'tcx, D: TyDecoder<I = TyCtxt<'tcx>>> RefDecodable<'tcx, D>
+    for ty::List<(VariantIdx, FieldIdx)>
+{
     fn decode(decoder: &mut D) -> &'tcx Self {
         let len = decoder.read_usize();
-        decoder
-            .interner()
-            .mk_offset_of_from_iter((0..len).map::<OffsetOfIdx, _>(|_| Decodable::decode(decoder)))
+        decoder.interner().mk_offset_of_from_iter(
+            (0..len).map::<(VariantIdx, FieldIdx), _>(|_| Decodable::decode(decoder)),
+        )
     }
 }
 
@@ -435,7 +437,7 @@ impl_decodable_via_ref! {
     &'tcx ty::List<ty::BoundVariableKind>,
     &'tcx ty::List<ty::Clause<'tcx>>,
     &'tcx ty::List<FieldIdx>,
-    &'tcx ty::List<OffsetOfIdx>,
+    &'tcx ty::List<(VariantIdx, FieldIdx)>,
 }
 
 #[macro_export]
diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs
index c89ae1e661ed..68812bba42fd 100644
--- a/compiler/rustc_middle/src/ty/context.rs
+++ b/compiler/rustc_middle/src/ty/context.rs
@@ -63,7 +63,7 @@ use rustc_session::{Limit, MetadataKind, Session};
 use rustc_span::def_id::{DefPathHash, StableCrateId};
 use rustc_span::symbol::{kw, sym, Ident, Symbol};
 use rustc_span::{Span, DUMMY_SP};
-use rustc_target::abi::{FieldIdx, Layout, LayoutS, OffsetOfIdx, TargetDataLayout, VariantIdx};
+use rustc_target::abi::{FieldIdx, Layout, LayoutS, TargetDataLayout, VariantIdx};
 use rustc_target::spec::abi;
 use rustc_type_ir::TyKind::*;
 use rustc_type_ir::WithCachedTypeInfo;
@@ -163,7 +163,7 @@ pub struct CtxtInterners<'tcx> {
     predefined_opaques_in_body: InternedSet<'tcx, PredefinedOpaquesData<'tcx>>,
     fields: InternedSet<'tcx, List<FieldIdx>>,
     local_def_ids: InternedSet<'tcx, List<LocalDefId>>,
-    offset_of: InternedSet<'tcx, List<OffsetOfIdx>>,
+    offset_of: InternedSet<'tcx, List<(VariantIdx, FieldIdx)>>,
 }
 
 impl<'tcx> CtxtInterners<'tcx> {
@@ -1589,7 +1589,7 @@ slice_interners!(
     bound_variable_kinds: pub mk_bound_variable_kinds(ty::BoundVariableKind),
     fields: pub mk_fields(FieldIdx),
     local_def_ids: intern_local_def_ids(LocalDefId),
-    offset_of: pub mk_offset_of(OffsetOfIdx),
+    offset_of: pub mk_offset_of((VariantIdx, FieldIdx)),
 );
 
 impl<'tcx> TyCtxt<'tcx> {
@@ -1920,7 +1920,7 @@ impl<'tcx> TyCtxt<'tcx> {
     pub fn mk_offset_of_from_iter<I, T>(self, iter: I) -> T::Output
     where
         I: Iterator<Item = T>,
-        T: CollectAndApply<OffsetOfIdx, &'tcx List<OffsetOfIdx>>,
+        T: CollectAndApply<(VariantIdx, FieldIdx), &'tcx List<(VariantIdx, FieldIdx)>>,
     {
         T::collect_and_apply(iter, |xs| self.mk_offset_of(xs))
     }
diff --git a/compiler/rustc_middle/src/ty/typeck_results.rs b/compiler/rustc_middle/src/ty/typeck_results.rs
index 623d8d61d2d5..e9240d1b268e 100644
--- a/compiler/rustc_middle/src/ty/typeck_results.rs
+++ b/compiler/rustc_middle/src/ty/typeck_results.rs
@@ -24,7 +24,7 @@ use rustc_macros::HashStable;
 use rustc_middle::mir::FakeReadCause;
 use rustc_session::Session;
 use rustc_span::Span;
-use rustc_target::abi::{FieldIdx, OffsetOfIdx};
+use rustc_target::abi::{FieldIdx, VariantIdx};
 use std::{collections::hash_map::Entry, hash::Hash, iter};
 
 use super::RvalueScopes;
@@ -205,7 +205,7 @@ pub struct TypeckResults<'tcx> {
     pub closure_size_eval: LocalDefIdMap<ClosureSizeProfileData<'tcx>>,
 
     /// Container types and field indices of `offset_of!` expressions
-    offset_of_data: ItemLocalMap<(Ty<'tcx>, Vec<OffsetOfIdx>)>,
+    offset_of_data: ItemLocalMap<(Ty<'tcx>, Vec<(VariantIdx, FieldIdx)>)>,
 }
 
 impl<'tcx> TypeckResults<'tcx> {
@@ -464,13 +464,15 @@ impl<'tcx> TypeckResults<'tcx> {
         &self.coercion_casts
     }
 
-    pub fn offset_of_data(&self) -> LocalTableInContext<'_, (Ty<'tcx>, Vec<OffsetOfIdx>)> {
+    pub fn offset_of_data(
+        &self,
+    ) -> LocalTableInContext<'_, (Ty<'tcx>, Vec<(VariantIdx, FieldIdx)>)> {
         LocalTableInContext { hir_owner: self.hir_owner, data: &self.offset_of_data }
     }
 
     pub fn offset_of_data_mut(
         &mut self,
-    ) -> LocalTableInContextMut<'_, (Ty<'tcx>, Vec<OffsetOfIdx>)> {
+    ) -> LocalTableInContextMut<'_, (Ty<'tcx>, Vec<(VariantIdx, FieldIdx)>)> {
         LocalTableInContextMut { hir_owner: self.hir_owner, data: &mut self.offset_of_data }
     }
 }
diff --git a/compiler/rustc_passes/src/dead.rs b/compiler/rustc_passes/src/dead.rs
index 3b4f6a618c8b..2e8c58b0241f 100644
--- a/compiler/rustc_passes/src/dead.rs
+++ b/compiler/rustc_passes/src/dead.rs
@@ -248,8 +248,6 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
     }
 
     fn handle_offset_of(&mut self, expr: &'tcx hir::Expr<'tcx>) {
-        use rustc_target::abi::OffsetOfIdx::*;
-
         let data = self.typeck_results().offset_of_data();
         let &(container, ref indices) =
             data.get(expr.hir_id).expect("no offset_of_data for offset_of");
@@ -258,22 +256,10 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
         let param_env = self.tcx.param_env(body_did);
 
         let mut current_ty = container;
-        let mut indices = indices.into_iter();
 
-        while let Some(&index) = indices.next() {
-            match (current_ty.kind(), index) {
-                (ty::Adt(def, subst), Field(field)) if !def.is_enum() => {
-                    let field = &def.non_enum_variant().fields[field];
-
-                    self.insert_def_id(field.did);
-                    let field_ty = field.ty(self.tcx, subst);
-
-                    current_ty = self.tcx.normalize_erasing_regions(param_env, field_ty);
-                }
-                (ty::Adt(def, subst), Variant(variant)) if def.is_enum() => {
-                    let Some(&Field(field)) = indices.next() else {
-                        span_bug!(expr.span, "variant must be followed by field in offset_of")
-                    };
+        for &(variant, field) in indices {
+            match current_ty.kind() {
+                ty::Adt(def, subst) => {
                     let field = &def.variant(variant).fields[field];
 
                     self.insert_def_id(field.did);
@@ -283,12 +269,11 @@ impl<'tcx> MarkSymbolVisitor<'tcx> {
                 }
                 // we don't need to mark tuple fields as live,
                 // but we may need to mark subfields
-                (ty::Tuple(tys), Field(field)) => {
+                ty::Tuple(tys) => {
                     current_ty =
                         self.tcx.normalize_erasing_regions(param_env, tys[field.as_usize()]);
                 }
-                (_, Field(_)) => span_bug!(expr.span, "named field access on non-ADT"),
-                (_, Variant(_)) => span_bug!(expr.span, "enum variant access on non-enum"),
+                _ => span_bug!(expr.span, "named field access on non-ADT"),
             }
         }
     }
diff --git a/compiler/rustc_smir/src/rustc_smir/mod.rs b/compiler/rustc_smir/src/rustc_smir/mod.rs
index e64afa5f8d93..2abad677bf04 100644
--- a/compiler/rustc_smir/src/rustc_smir/mod.rs
+++ b/compiler/rustc_smir/src/rustc_smir/mod.rs
@@ -16,7 +16,7 @@ use rustc_middle::mir::interpret::{alloc_range, AllocId};
 use rustc_middle::mir::mono::MonoItem;
 use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TyCtxt, Variance};
 use rustc_span::def_id::{CrateNum, DefId, LOCAL_CRATE};
-use rustc_target::abi::{FieldIdx, OffsetOfIdx};
+use rustc_target::abi::FieldIdx;
 use stable_mir::mir::mono::InstanceDef;
 use stable_mir::mir::{Body, CopyNonOverlapping, Statement, UserTypeProjection, VariantIdx};
 use stable_mir::ty::{
@@ -643,13 +643,10 @@ impl<'tcx> Stable<'tcx> for FieldIdx {
     }
 }
 
-impl<'tcx> Stable<'tcx> for OffsetOfIdx {
-    type T = usize;
+impl<'tcx> Stable<'tcx> for (rustc_target::abi::VariantIdx, FieldIdx) {
+    type T = (usize, usize);
     fn stable(&self, _: &mut Tables<'tcx>) -> Self::T {
-        match self {
-            OffsetOfIdx::Field(f) => f.as_usize(),
-            OffsetOfIdx::Variant(v) => v.as_usize(),
-        }
+        (self.0.as_usize(), self.1.as_usize())
     }
 }
 
diff --git a/compiler/rustc_target/src/abi/mod.rs b/compiler/rustc_target/src/abi/mod.rs
index a37a5bb26893..b00567e87c65 100644
--- a/compiler/rustc_target/src/abi/mod.rs
+++ b/compiler/rustc_target/src/abi/mod.rs
@@ -250,29 +250,24 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
         Ty::is_transparent(self)
     }
 
-    pub fn offset_of_subfield<C>(self, cx: &C, indices: impl Iterator<Item = OffsetOfIdx>) -> Size
+    pub fn offset_of_subfield<C, I>(self, cx: &C, indices: I) -> Size
     where
         Ty: TyAbiInterface<'a, C>,
+        I: Iterator<Item = (VariantIdx, FieldIdx)>,
     {
         let mut layout = self;
         let mut offset = Size::ZERO;
 
-        for index in indices {
-            match index {
-                OffsetOfIdx::Field(field) => {
-                    let index = field.index();
-                    offset += layout.fields.offset(index);
-                    layout = layout.field(cx, index);
-                    assert!(
-                        layout.is_sized(),
-                        "offset of unsized field (type {:?}) cannot be computed statically",
-                        layout.ty
-                    );
-                }
-                OffsetOfIdx::Variant(variant) => {
-                    layout = layout.for_variant(cx, variant);
-                }
-            }
+        for (variant, field) in indices {
+            layout = layout.for_variant(cx, variant);
+            let index = field.index();
+            offset += layout.fields.offset(index);
+            layout = layout.field(cx, index);
+            assert!(
+                layout.is_sized(),
+                "offset of unsized field (type {:?}) cannot be computed statically",
+                layout.ty
+            );
         }
 
         offset
diff --git a/compiler/stable_mir/src/mir/body.rs b/compiler/stable_mir/src/mir/body.rs
index 9f69e61d6fe7..9dc746f5c4d2 100644
--- a/compiler/stable_mir/src/mir/body.rs
+++ b/compiler/stable_mir/src/mir/body.rs
@@ -514,7 +514,7 @@ pub enum NullOp {
     /// Returns the minimum alignment of a type.
     AlignOf,
     /// Returns the offset of a field.
-    OffsetOf(Vec<FieldIdx>),
+    OffsetOf(Vec<(VariantIdx, FieldIdx)>),
 }
 
 impl Operand {
diff --git a/library/core/src/mem/mod.rs b/library/core/src/mem/mod.rs
index df79c3a338a1..862619f31f26 100644
--- a/library/core/src/mem/mod.rs
+++ b/library/core/src/mem/mod.rs
@@ -1292,11 +1292,15 @@ impl<T> SizedTypeProperties for T {}
 
 /// Expands to the offset in bytes of a field from the beginning of the given type.
 ///
-/// Only structs, unions and tuples are supported.
+/// Structs, enums, unions and tuples are supported.
 ///
 /// Nested field accesses may be used, but not array indexes like in `C`'s `offsetof`.
 ///
-/// Note that the output of this macro is not stable, except for `#[repr(C)]` types.
+/// Enum variants may be traversed as if they were fields. Variants themselves do
+/// not have an offset.
+///
+/// Note that type layout is, in general, [platform-specific, and subject to
+/// change](https://doc.rust-lang.org/reference/type-layout.html).
 ///
 /// # Examples
 ///
@@ -1324,6 +1328,8 @@ impl<T> SizedTypeProperties for T {}
 /// struct NestedB(u8);
 ///
 /// assert_eq!(mem::offset_of!(NestedA, b.0), 0);
+///
+/// assert_eq!(mem::offset_of!(Option<&u8>, Some.0), 0);
 /// ```
 #[unstable(feature = "offset_of", issue = "106655")]
 #[allow_internal_unstable(builtin_syntax, hint_must_use)]
diff --git a/tests/ui/offset-of/offset-of-enum.rs b/tests/ui/offset-of/offset-of-enum.rs
index 92d34e67c71b..cf516175085b 100644
--- a/tests/ui/offset-of/offset-of-enum.rs
+++ b/tests/ui/offset-of/offset-of-enum.rs
@@ -11,4 +11,7 @@ fn main() {
     offset_of!(Alpha::One, 0); //~ ERROR expected type, found variant `Alpha::One`
     offset_of!(Alpha, One); //~ ERROR `One` is an enum variant; expected field at end of `offset_of`
     offset_of!(Alpha, Two.0);
+    offset_of!(Alpha, Two.1); //~ ERROR no field named `1` on enum variant `Alpha::Two`
+    offset_of!(Alpha, Two.foo); //~ ERROR no field named `foo` on enum variant `Alpha::Two`
+    offset_of!(Alpha, NonExistent); //~ ERROR no variant named `NonExistent` found for enum `Alpha`
 }
diff --git a/tests/ui/offset-of/offset-of-enum.stderr b/tests/ui/offset-of/offset-of-enum.stderr
index 1d849e682e69..2867cc6befb5 100644
--- a/tests/ui/offset-of/offset-of-enum.stderr
+++ b/tests/ui/offset-of/offset-of-enum.stderr
@@ -13,7 +13,29 @@ error[E0795]: `One` is an enum variant; expected field at end of `offset_of`
 LL |     offset_of!(Alpha, One);
    |                       ^^^ enum variant
 
-error: aborting due to 2 previous errors
+error[E0609]: no field named `1` on enum variant `Alpha::Two`
+  --> $DIR/offset-of-enum.rs:14:23
+   |
+LL |     offset_of!(Alpha, Two.1);
+   |                       ^^^ - ...does not have this field
+   |                       |
+   |                       this enum variant...
 
-Some errors have detailed explanations: E0573, E0795.
+error[E0609]: no field named `foo` on enum variant `Alpha::Two`
+  --> $DIR/offset-of-enum.rs:15:23
+   |
+LL |     offset_of!(Alpha, Two.foo);
+   |                       ^^^ --- ...does not have this field
+   |                       |
+   |                       this enum variant...
+
+error[E0599]: no variant named `NonExistent` found for enum `Alpha`
+  --> $DIR/offset-of-enum.rs:16:23
+   |
+LL |     offset_of!(Alpha, NonExistent);
+   |                       ^^^^^^^^^^^ variant not found
+
+error: aborting due to 5 previous errors
+
+Some errors have detailed explanations: E0573, E0599, E0609, E0795.
 For more information about an error, try `rustc --explain E0573`.

From 9d6ce61376ef77bae3268b1b69ec6e7f3b815923 Mon Sep 17 00:00:00 2001
From: George Bateman <george.bateman16@gmail.com>
Date: Tue, 31 Oct 2023 21:25:45 +0000
Subject: [PATCH 293/297] Update MIR tests for offset_of

---
 compiler/rustc_abi/src/lib.rs                 | 26 -------------------
 compiler/rustc_middle/src/thir.rs             |  1 -
 .../src/dataflow_const_prop.rs                |  6 ++---
 library/core/src/mem/mod.rs                   |  1 +
 ...set_of.concrete.ConstProp.panic-abort.diff | 14 +++++-----
 ...et_of.concrete.ConstProp.panic-unwind.diff | 14 +++++-----
 ...fset_of.generic.ConstProp.panic-abort.diff | 14 +++++-----
 ...set_of.generic.ConstProp.panic-unwind.diff | 14 +++++-----
 ...oncrete.DataflowConstProp.panic-abort.diff |  8 +++---
 ...ncrete.DataflowConstProp.panic-unwind.diff |  8 +++---
 ...generic.DataflowConstProp.panic-abort.diff |  8 +++---
 ...eneric.DataflowConstProp.panic-unwind.diff |  8 +++---
 12 files changed, 48 insertions(+), 74 deletions(-)

diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs
index 9626eeea03bd..8e7aa59ee341 100644
--- a/compiler/rustc_abi/src/lib.rs
+++ b/compiler/rustc_abi/src/lib.rs
@@ -1107,32 +1107,6 @@ impl Scalar {
 }
 
 // NOTE: This struct is generic over the FieldIdx for rust-analyzer usage.
-rustc_index::newtype_index! {
-    /// The *source-order* index of a field in a variant.
-    ///
-    /// This is how most code after type checking refers to fields, rather than
-    /// using names (as names have hygiene complications and more complex lookup).
-    ///
-    /// Particularly for `repr(Rust)` types, this may not be the same as *layout* order.
-    /// (It is for `repr(C)` `struct`s, however.)
-    ///
-    /// For example, in the following types,
-    /// ```rust
-    /// # enum Never {}
-    /// # #[repr(u16)]
-    /// enum Demo1 {
-    ///    Variant0 { a: Never, b: i32 } = 100,
-    ///    Variant1 { c: u8, d: u64 } = 10,
-    /// }
-    /// struct Demo2 { e: u8, f: u16, g: u8 }
-    /// ```
-    /// `b` is `FieldIdx(1)` in `VariantIdx(0)`,
-    /// `d` is `FieldIdx(1)` in `VariantIdx(1)`, and
-    /// `f` is `FieldIdx(1)` in `VariantIdx(0)`.
-    #[derive(HashStable_Generic)]
-    pub struct FieldIdx {}
-}
-
 /// Describes how the fields of a type are located in memory.
 #[derive(PartialEq, Eq, Hash, Clone, Debug)]
 #[cfg_attr(feature = "nightly", derive(HashStable_Generic))]
diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs
index 3780eb4f9ac9..e83ebe651abf 100644
--- a/compiler/rustc_middle/src/thir.rs
+++ b/compiler/rustc_middle/src/thir.rs
@@ -25,7 +25,6 @@ use rustc_middle::ty::{
 };
 use rustc_span::def_id::LocalDefId;
 use rustc_span::{sym, ErrorGuaranteed, Span, Symbol, DUMMY_SP};
-use rustc_span::{sym, Span, Symbol, DUMMY_SP};
 use rustc_target::abi::{FieldIdx, VariantIdx};
 use rustc_target::asm::InlineAsmRegOrRegClass;
 use std::fmt;
diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
index ca10dc48eb80..81d2bba989a7 100644
--- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
+++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs
@@ -286,9 +286,9 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> {
                 let val = match null_op {
                     NullOp::SizeOf if layout.is_sized() => layout.size.bytes(),
                     NullOp::AlignOf if layout.is_sized() => layout.align.abi.bytes(),
-                    NullOp::OffsetOf(fields) => layout
-                        .offset_of_subfield(&self.ecx, fields.iter())
-                        .bytes(),
+                    NullOp::OffsetOf(fields) => {
+                        layout.offset_of_subfield(&self.ecx, fields.iter()).bytes()
+                    }
                     _ => return ValueOrPlace::Value(FlatSet::Top),
                 };
                 FlatSet::Elem(Scalar::from_target_usize(val, &self.tcx))
diff --git a/library/core/src/mem/mod.rs b/library/core/src/mem/mod.rs
index 862619f31f26..b5b0e5882dad 100644
--- a/library/core/src/mem/mod.rs
+++ b/library/core/src/mem/mod.rs
@@ -1329,6 +1329,7 @@ impl<T> SizedTypeProperties for T {}
 ///
 /// assert_eq!(mem::offset_of!(NestedA, b.0), 0);
 ///
+/// # #[cfg(not(bootstrap))]
 /// assert_eq!(mem::offset_of!(Option<&u8>, Some.0), 0);
 /// ```
 #[unstable(feature = "offset_of", issue = "106655")]
diff --git a/tests/mir-opt/const_prop/offset_of.concrete.ConstProp.panic-abort.diff b/tests/mir-opt/const_prop/offset_of.concrete.ConstProp.panic-abort.diff
index e460c08f9bb8..711db3d21dd7 100644
--- a/tests/mir-opt/const_prop/offset_of.concrete.ConstProp.panic-abort.diff
+++ b/tests/mir-opt/const_prop/offset_of.concrete.ConstProp.panic-abort.diff
@@ -42,7 +42,7 @@
       bb0: {
           StorageLive(_1);
           StorageLive(_2);
--         _2 = OffsetOf(Alpha, [Field(0)]);
+-         _2 = OffsetOf(Alpha, [(0, 0)]);
 -         _1 = must_use::<usize>(move _2) -> [return: bb1, unwind unreachable];
 +         _2 = const 4_usize;
 +         _1 = must_use::<usize>(const 4_usize) -> [return: bb1, unwind unreachable];
@@ -52,7 +52,7 @@
           StorageDead(_2);
           StorageLive(_3);
           StorageLive(_4);
--         _4 = OffsetOf(Alpha, [Field(1)]);
+-         _4 = OffsetOf(Alpha, [(0, 1)]);
 -         _3 = must_use::<usize>(move _4) -> [return: bb2, unwind unreachable];
 +         _4 = const 0_usize;
 +         _3 = must_use::<usize>(const 0_usize) -> [return: bb2, unwind unreachable];
@@ -62,7 +62,7 @@
           StorageDead(_4);
           StorageLive(_5);
           StorageLive(_6);
--         _6 = OffsetOf(Alpha, [Field(2), Field(0)]);
+-         _6 = OffsetOf(Alpha, [(0, 2), (0, 0)]);
 -         _5 = must_use::<usize>(move _6) -> [return: bb3, unwind unreachable];
 +         _6 = const 2_usize;
 +         _5 = must_use::<usize>(const 2_usize) -> [return: bb3, unwind unreachable];
@@ -72,7 +72,7 @@
           StorageDead(_6);
           StorageLive(_7);
           StorageLive(_8);
--         _8 = OffsetOf(Alpha, [Field(2), Field(1)]);
+-         _8 = OffsetOf(Alpha, [(0, 2), (0, 1)]);
 -         _7 = must_use::<usize>(move _8) -> [return: bb4, unwind unreachable];
 +         _8 = const 3_usize;
 +         _7 = must_use::<usize>(const 3_usize) -> [return: bb4, unwind unreachable];
@@ -82,7 +82,7 @@
           StorageDead(_8);
           StorageLive(_9);
           StorageLive(_10);
--         _10 = OffsetOf(Epsilon, [Variant(0), Field(0)]);
+-         _10 = OffsetOf(Epsilon, [(0, 0)]);
 -         _9 = must_use::<usize>(move _10) -> [return: bb5, unwind unreachable];
 +         _10 = const 1_usize;
 +         _9 = must_use::<usize>(const 1_usize) -> [return: bb5, unwind unreachable];
@@ -92,7 +92,7 @@
           StorageDead(_10);
           StorageLive(_11);
           StorageLive(_12);
--         _12 = OffsetOf(Epsilon, [Variant(0), Field(1)]);
+-         _12 = OffsetOf(Epsilon, [(0, 1)]);
 -         _11 = must_use::<usize>(move _12) -> [return: bb6, unwind unreachable];
 +         _12 = const 2_usize;
 +         _11 = must_use::<usize>(const 2_usize) -> [return: bb6, unwind unreachable];
@@ -102,7 +102,7 @@
           StorageDead(_12);
           StorageLive(_13);
           StorageLive(_14);
--         _14 = OffsetOf(Epsilon, [Variant(2), Field(0)]);
+-         _14 = OffsetOf(Epsilon, [(2, 0)]);
 -         _13 = must_use::<usize>(move _14) -> [return: bb7, unwind unreachable];
 +         _14 = const 4_usize;
 +         _13 = must_use::<usize>(const 4_usize) -> [return: bb7, unwind unreachable];
diff --git a/tests/mir-opt/const_prop/offset_of.concrete.ConstProp.panic-unwind.diff b/tests/mir-opt/const_prop/offset_of.concrete.ConstProp.panic-unwind.diff
index 3b8d0d75101f..49458145415c 100644
--- a/tests/mir-opt/const_prop/offset_of.concrete.ConstProp.panic-unwind.diff
+++ b/tests/mir-opt/const_prop/offset_of.concrete.ConstProp.panic-unwind.diff
@@ -42,7 +42,7 @@
       bb0: {
           StorageLive(_1);
           StorageLive(_2);
--         _2 = OffsetOf(Alpha, [Field(0)]);
+-         _2 = OffsetOf(Alpha, [(0, 0)]);
 -         _1 = must_use::<usize>(move _2) -> [return: bb1, unwind continue];
 +         _2 = const 4_usize;
 +         _1 = must_use::<usize>(const 4_usize) -> [return: bb1, unwind continue];
@@ -52,7 +52,7 @@
           StorageDead(_2);
           StorageLive(_3);
           StorageLive(_4);
--         _4 = OffsetOf(Alpha, [Field(1)]);
+-         _4 = OffsetOf(Alpha, [(0, 1)]);
 -         _3 = must_use::<usize>(move _4) -> [return: bb2, unwind continue];
 +         _4 = const 0_usize;
 +         _3 = must_use::<usize>(const 0_usize) -> [return: bb2, unwind continue];
@@ -62,7 +62,7 @@
           StorageDead(_4);
           StorageLive(_5);
           StorageLive(_6);
--         _6 = OffsetOf(Alpha, [Field(2), Field(0)]);
+-         _6 = OffsetOf(Alpha, [(0, 2), (0, 0)]);
 -         _5 = must_use::<usize>(move _6) -> [return: bb3, unwind continue];
 +         _6 = const 2_usize;
 +         _5 = must_use::<usize>(const 2_usize) -> [return: bb3, unwind continue];
@@ -72,7 +72,7 @@
           StorageDead(_6);
           StorageLive(_7);
           StorageLive(_8);
--         _8 = OffsetOf(Alpha, [Field(2), Field(1)]);
+-         _8 = OffsetOf(Alpha, [(0, 2), (0, 1)]);
 -         _7 = must_use::<usize>(move _8) -> [return: bb4, unwind continue];
 +         _8 = const 3_usize;
 +         _7 = must_use::<usize>(const 3_usize) -> [return: bb4, unwind continue];
@@ -82,7 +82,7 @@
           StorageDead(_8);
           StorageLive(_9);
           StorageLive(_10);
--         _10 = OffsetOf(Epsilon, [Variant(0), Field(0)]);
+-         _10 = OffsetOf(Epsilon, [(0, 0)]);
 -         _9 = must_use::<usize>(move _10) -> [return: bb5, unwind continue];
 +         _10 = const 1_usize;
 +         _9 = must_use::<usize>(const 1_usize) -> [return: bb5, unwind continue];
@@ -92,7 +92,7 @@
           StorageDead(_10);
           StorageLive(_11);
           StorageLive(_12);
--         _12 = OffsetOf(Epsilon, [Variant(0), Field(1)]);
+-         _12 = OffsetOf(Epsilon, [(0, 1)]);
 -         _11 = must_use::<usize>(move _12) -> [return: bb6, unwind continue];
 +         _12 = const 2_usize;
 +         _11 = must_use::<usize>(const 2_usize) -> [return: bb6, unwind continue];
@@ -102,7 +102,7 @@
           StorageDead(_12);
           StorageLive(_13);
           StorageLive(_14);
--         _14 = OffsetOf(Epsilon, [Variant(2), Field(0)]);
+-         _14 = OffsetOf(Epsilon, [(2, 0)]);
 -         _13 = must_use::<usize>(move _14) -> [return: bb7, unwind continue];
 +         _14 = const 4_usize;
 +         _13 = must_use::<usize>(const 4_usize) -> [return: bb7, unwind continue];
diff --git a/tests/mir-opt/const_prop/offset_of.generic.ConstProp.panic-abort.diff b/tests/mir-opt/const_prop/offset_of.generic.ConstProp.panic-abort.diff
index f26b104e065d..768970a7250f 100644
--- a/tests/mir-opt/const_prop/offset_of.generic.ConstProp.panic-abort.diff
+++ b/tests/mir-opt/const_prop/offset_of.generic.ConstProp.panic-abort.diff
@@ -42,7 +42,7 @@
       bb0: {
           StorageLive(_1);
           StorageLive(_2);
-          _2 = OffsetOf(Gamma<T>, [Field(0)]);
+          _2 = OffsetOf(Gamma<T>, [(0, 0)]);
           _1 = must_use::<usize>(move _2) -> [return: bb1, unwind unreachable];
       }
   
@@ -50,7 +50,7 @@
           StorageDead(_2);
           StorageLive(_3);
           StorageLive(_4);
-          _4 = OffsetOf(Gamma<T>, [Field(1)]);
+          _4 = OffsetOf(Gamma<T>, [(0, 1)]);
           _3 = must_use::<usize>(move _4) -> [return: bb2, unwind unreachable];
       }
   
@@ -58,7 +58,7 @@
           StorageDead(_4);
           StorageLive(_5);
           StorageLive(_6);
-          _6 = OffsetOf(Delta<T>, [Field(1)]);
+          _6 = OffsetOf(Delta<T>, [(0, 1)]);
           _5 = must_use::<usize>(move _6) -> [return: bb3, unwind unreachable];
       }
   
@@ -66,7 +66,7 @@
           StorageDead(_6);
           StorageLive(_7);
           StorageLive(_8);
-          _8 = OffsetOf(Delta<T>, [Field(2)]);
+          _8 = OffsetOf(Delta<T>, [(0, 2)]);
           _7 = must_use::<usize>(move _8) -> [return: bb4, unwind unreachable];
       }
   
@@ -74,7 +74,7 @@
           StorageDead(_8);
           StorageLive(_9);
           StorageLive(_10);
-          _10 = OffsetOf(Zeta<T>, [Variant(0), Field(0)]);
+          _10 = OffsetOf(Zeta<T>, [(0, 0)]);
           _9 = must_use::<usize>(move _10) -> [return: bb5, unwind unreachable];
       }
   
@@ -82,7 +82,7 @@
           StorageDead(_10);
           StorageLive(_11);
           StorageLive(_12);
-          _12 = OffsetOf(Zeta<T>, [Variant(0), Field(1)]);
+          _12 = OffsetOf(Zeta<T>, [(0, 1)]);
           _11 = must_use::<usize>(move _12) -> [return: bb6, unwind unreachable];
       }
   
@@ -90,7 +90,7 @@
           StorageDead(_12);
           StorageLive(_13);
           StorageLive(_14);
-          _14 = OffsetOf(Zeta<T>, [Variant(1), Field(0)]);
+          _14 = OffsetOf(Zeta<T>, [(1, 0)]);
           _13 = must_use::<usize>(move _14) -> [return: bb7, unwind unreachable];
       }
   
diff --git a/tests/mir-opt/const_prop/offset_of.generic.ConstProp.panic-unwind.diff b/tests/mir-opt/const_prop/offset_of.generic.ConstProp.panic-unwind.diff
index 0f8750fce460..04ccd2b36e0d 100644
--- a/tests/mir-opt/const_prop/offset_of.generic.ConstProp.panic-unwind.diff
+++ b/tests/mir-opt/const_prop/offset_of.generic.ConstProp.panic-unwind.diff
@@ -42,7 +42,7 @@
       bb0: {
           StorageLive(_1);
           StorageLive(_2);
-          _2 = OffsetOf(Gamma<T>, [Field(0)]);
+          _2 = OffsetOf(Gamma<T>, [(0, 0)]);
           _1 = must_use::<usize>(move _2) -> [return: bb1, unwind continue];
       }
   
@@ -50,7 +50,7 @@
           StorageDead(_2);
           StorageLive(_3);
           StorageLive(_4);
-          _4 = OffsetOf(Gamma<T>, [Field(1)]);
+          _4 = OffsetOf(Gamma<T>, [(0, 1)]);
           _3 = must_use::<usize>(move _4) -> [return: bb2, unwind continue];
       }
   
@@ -58,7 +58,7 @@
           StorageDead(_4);
           StorageLive(_5);
           StorageLive(_6);
-          _6 = OffsetOf(Delta<T>, [Field(1)]);
+          _6 = OffsetOf(Delta<T>, [(0, 1)]);
           _5 = must_use::<usize>(move _6) -> [return: bb3, unwind continue];
       }
   
@@ -66,7 +66,7 @@
           StorageDead(_6);
           StorageLive(_7);
           StorageLive(_8);
-          _8 = OffsetOf(Delta<T>, [Field(2)]);
+          _8 = OffsetOf(Delta<T>, [(0, 2)]);
           _7 = must_use::<usize>(move _8) -> [return: bb4, unwind continue];
       }
   
@@ -74,7 +74,7 @@
           StorageDead(_8);
           StorageLive(_9);
           StorageLive(_10);
-          _10 = OffsetOf(Zeta<T>, [Variant(0), Field(0)]);
+          _10 = OffsetOf(Zeta<T>, [(0, 0)]);
           _9 = must_use::<usize>(move _10) -> [return: bb5, unwind continue];
       }
   
@@ -82,7 +82,7 @@
           StorageDead(_10);
           StorageLive(_11);
           StorageLive(_12);
-          _12 = OffsetOf(Zeta<T>, [Variant(0), Field(1)]);
+          _12 = OffsetOf(Zeta<T>, [(0, 1)]);
           _11 = must_use::<usize>(move _12) -> [return: bb6, unwind continue];
       }
   
@@ -90,7 +90,7 @@
           StorageDead(_12);
           StorageLive(_13);
           StorageLive(_14);
-          _14 = OffsetOf(Zeta<T>, [Variant(1), Field(0)]);
+          _14 = OffsetOf(Zeta<T>, [(1, 0)]);
           _13 = must_use::<usize>(move _14) -> [return: bb7, unwind continue];
       }
   
diff --git a/tests/mir-opt/dataflow-const-prop/offset_of.concrete.DataflowConstProp.panic-abort.diff b/tests/mir-opt/dataflow-const-prop/offset_of.concrete.DataflowConstProp.panic-abort.diff
index c61414b6541a..f8f891750337 100644
--- a/tests/mir-opt/dataflow-const-prop/offset_of.concrete.DataflowConstProp.panic-abort.diff
+++ b/tests/mir-opt/dataflow-const-prop/offset_of.concrete.DataflowConstProp.panic-abort.diff
@@ -27,7 +27,7 @@
       bb0: {
           StorageLive(_1);
           StorageLive(_2);
--         _2 = OffsetOf(Alpha, [0]);
+-         _2 = OffsetOf(Alpha, [(0, 0)]);
 -         _1 = must_use::<usize>(move _2) -> [return: bb1, unwind unreachable];
 +         _2 = const 4_usize;
 +         _1 = must_use::<usize>(const 4_usize) -> [return: bb1, unwind unreachable];
@@ -37,7 +37,7 @@
           StorageDead(_2);
           StorageLive(_3);
           StorageLive(_4);
--         _4 = OffsetOf(Alpha, [1]);
+-         _4 = OffsetOf(Alpha, [(0, 1)]);
 -         _3 = must_use::<usize>(move _4) -> [return: bb2, unwind unreachable];
 +         _4 = const 0_usize;
 +         _3 = must_use::<usize>(const 0_usize) -> [return: bb2, unwind unreachable];
@@ -47,7 +47,7 @@
           StorageDead(_4);
           StorageLive(_5);
           StorageLive(_6);
--         _6 = OffsetOf(Alpha, [2, 0]);
+-         _6 = OffsetOf(Alpha, [(0, 2), (0, 0)]);
 -         _5 = must_use::<usize>(move _6) -> [return: bb3, unwind unreachable];
 +         _6 = const 2_usize;
 +         _5 = must_use::<usize>(const 2_usize) -> [return: bb3, unwind unreachable];
@@ -57,7 +57,7 @@
           StorageDead(_6);
           StorageLive(_7);
           StorageLive(_8);
--         _8 = OffsetOf(Alpha, [2, 1]);
+-         _8 = OffsetOf(Alpha, [(0, 2), (0, 1)]);
 -         _7 = must_use::<usize>(move _8) -> [return: bb4, unwind unreachable];
 +         _8 = const 3_usize;
 +         _7 = must_use::<usize>(const 3_usize) -> [return: bb4, unwind unreachable];
diff --git a/tests/mir-opt/dataflow-const-prop/offset_of.concrete.DataflowConstProp.panic-unwind.diff b/tests/mir-opt/dataflow-const-prop/offset_of.concrete.DataflowConstProp.panic-unwind.diff
index 0c3939a3456a..d4f8cb66704a 100644
--- a/tests/mir-opt/dataflow-const-prop/offset_of.concrete.DataflowConstProp.panic-unwind.diff
+++ b/tests/mir-opt/dataflow-const-prop/offset_of.concrete.DataflowConstProp.panic-unwind.diff
@@ -27,7 +27,7 @@
       bb0: {
           StorageLive(_1);
           StorageLive(_2);
--         _2 = OffsetOf(Alpha, [0]);
+-         _2 = OffsetOf(Alpha, [(0, 0)]);
 -         _1 = must_use::<usize>(move _2) -> [return: bb1, unwind continue];
 +         _2 = const 4_usize;
 +         _1 = must_use::<usize>(const 4_usize) -> [return: bb1, unwind continue];
@@ -37,7 +37,7 @@
           StorageDead(_2);
           StorageLive(_3);
           StorageLive(_4);
--         _4 = OffsetOf(Alpha, [1]);
+-         _4 = OffsetOf(Alpha, [(0, 1)]);
 -         _3 = must_use::<usize>(move _4) -> [return: bb2, unwind continue];
 +         _4 = const 0_usize;
 +         _3 = must_use::<usize>(const 0_usize) -> [return: bb2, unwind continue];
@@ -47,7 +47,7 @@
           StorageDead(_4);
           StorageLive(_5);
           StorageLive(_6);
--         _6 = OffsetOf(Alpha, [2, 0]);
+-         _6 = OffsetOf(Alpha, [(0, 2), (0, 0)]);
 -         _5 = must_use::<usize>(move _6) -> [return: bb3, unwind continue];
 +         _6 = const 2_usize;
 +         _5 = must_use::<usize>(const 2_usize) -> [return: bb3, unwind continue];
@@ -57,7 +57,7 @@
           StorageDead(_6);
           StorageLive(_7);
           StorageLive(_8);
--         _8 = OffsetOf(Alpha, [2, 1]);
+-         _8 = OffsetOf(Alpha, [(0, 2), (0, 1)]);
 -         _7 = must_use::<usize>(move _8) -> [return: bb4, unwind continue];
 +         _8 = const 3_usize;
 +         _7 = must_use::<usize>(const 3_usize) -> [return: bb4, unwind continue];
diff --git a/tests/mir-opt/dataflow-const-prop/offset_of.generic.DataflowConstProp.panic-abort.diff b/tests/mir-opt/dataflow-const-prop/offset_of.generic.DataflowConstProp.panic-abort.diff
index d54d46870609..7f166e4fa356 100644
--- a/tests/mir-opt/dataflow-const-prop/offset_of.generic.DataflowConstProp.panic-abort.diff
+++ b/tests/mir-opt/dataflow-const-prop/offset_of.generic.DataflowConstProp.panic-abort.diff
@@ -27,7 +27,7 @@
       bb0: {
           StorageLive(_1);
           StorageLive(_2);
-          _2 = OffsetOf(Gamma<T>, [0]);
+          _2 = OffsetOf(Gamma<T>, [(0, 0)]);
           _1 = must_use::<usize>(move _2) -> [return: bb1, unwind unreachable];
       }
   
@@ -35,7 +35,7 @@
           StorageDead(_2);
           StorageLive(_3);
           StorageLive(_4);
-          _4 = OffsetOf(Gamma<T>, [1]);
+          _4 = OffsetOf(Gamma<T>, [(0, 1)]);
           _3 = must_use::<usize>(move _4) -> [return: bb2, unwind unreachable];
       }
   
@@ -43,7 +43,7 @@
           StorageDead(_4);
           StorageLive(_5);
           StorageLive(_6);
--         _6 = OffsetOf(Delta<T>, [1]);
+-         _6 = OffsetOf(Delta<T>, [(0, 1)]);
 -         _5 = must_use::<usize>(move _6) -> [return: bb3, unwind unreachable];
 +         _6 = const 0_usize;
 +         _5 = must_use::<usize>(const 0_usize) -> [return: bb3, unwind unreachable];
@@ -53,7 +53,7 @@
           StorageDead(_6);
           StorageLive(_7);
           StorageLive(_8);
--         _8 = OffsetOf(Delta<T>, [2]);
+-         _8 = OffsetOf(Delta<T>, [(0, 2)]);
 -         _7 = must_use::<usize>(move _8) -> [return: bb4, unwind unreachable];
 +         _8 = const 2_usize;
 +         _7 = must_use::<usize>(const 2_usize) -> [return: bb4, unwind unreachable];
diff --git a/tests/mir-opt/dataflow-const-prop/offset_of.generic.DataflowConstProp.panic-unwind.diff b/tests/mir-opt/dataflow-const-prop/offset_of.generic.DataflowConstProp.panic-unwind.diff
index 6032a2274efe..38ad6f798016 100644
--- a/tests/mir-opt/dataflow-const-prop/offset_of.generic.DataflowConstProp.panic-unwind.diff
+++ b/tests/mir-opt/dataflow-const-prop/offset_of.generic.DataflowConstProp.panic-unwind.diff
@@ -27,7 +27,7 @@
       bb0: {
           StorageLive(_1);
           StorageLive(_2);
-          _2 = OffsetOf(Gamma<T>, [0]);
+          _2 = OffsetOf(Gamma<T>, [(0, 0)]);
           _1 = must_use::<usize>(move _2) -> [return: bb1, unwind continue];
       }
   
@@ -35,7 +35,7 @@
           StorageDead(_2);
           StorageLive(_3);
           StorageLive(_4);
-          _4 = OffsetOf(Gamma<T>, [1]);
+          _4 = OffsetOf(Gamma<T>, [(0, 1)]);
           _3 = must_use::<usize>(move _4) -> [return: bb2, unwind continue];
       }
   
@@ -43,7 +43,7 @@
           StorageDead(_4);
           StorageLive(_5);
           StorageLive(_6);
--         _6 = OffsetOf(Delta<T>, [1]);
+-         _6 = OffsetOf(Delta<T>, [(0, 1)]);
 -         _5 = must_use::<usize>(move _6) -> [return: bb3, unwind continue];
 +         _6 = const 0_usize;
 +         _5 = must_use::<usize>(const 0_usize) -> [return: bb3, unwind continue];
@@ -53,7 +53,7 @@
           StorageDead(_6);
           StorageLive(_7);
           StorageLive(_8);
--         _8 = OffsetOf(Delta<T>, [2]);
+-         _8 = OffsetOf(Delta<T>, [(0, 2)]);
 -         _7 = must_use::<usize>(move _8) -> [return: bb4, unwind continue];
 +         _8 = const 2_usize;
 +         _7 = must_use::<usize>(const 2_usize) -> [return: bb4, unwind continue];

From e742f809f68d571a28b7c9153366194e47714b65 Mon Sep 17 00:00:00 2001
From: George Bateman <george.bateman16@gmail.com>
Date: Tue, 31 Oct 2023 23:41:40 +0000
Subject: [PATCH 294/297] Update based on wesleywiser review

---
 compiler/rustc_mir_transform/src/gvn.rs  |  6 +++---
 library/core/src/mem/mod.rs              |  4 ++--
 tests/ui/offset-of/offset-of-enum.rs     |  1 +
 tests/ui/offset-of/offset-of-enum.stderr | 12 +++++++++---
 4 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs
index de0dc25808b3..dce298e92e16 100644
--- a/compiler/rustc_mir_transform/src/gvn.rs
+++ b/compiler/rustc_mir_transform/src/gvn.rs
@@ -467,9 +467,9 @@ impl<'body, 'tcx> VnState<'body, 'tcx> {
                 let val = match null_op {
                     NullOp::SizeOf => layout.size.bytes(),
                     NullOp::AlignOf => layout.align.abi.bytes(),
-                    NullOp::OffsetOf(fields) => layout
-                        .offset_of_subfield(&self.ecx, fields.iter().map(|f| f.index()))
-                        .bytes(),
+                    NullOp::OffsetOf(fields) => {
+                        layout.offset_of_subfield(&self.ecx, fields.iter()).bytes()
+                    }
                 };
                 let usize_layout = self.ecx.layout_of(self.tcx.types.usize).unwrap();
                 let imm = ImmTy::try_from_uint(val, usize_layout)?;
diff --git a/library/core/src/mem/mod.rs b/library/core/src/mem/mod.rs
index b5b0e5882dad..8792134cdc86 100644
--- a/library/core/src/mem/mod.rs
+++ b/library/core/src/mem/mod.rs
@@ -1299,8 +1299,8 @@ impl<T> SizedTypeProperties for T {}
 /// Enum variants may be traversed as if they were fields. Variants themselves do
 /// not have an offset.
 ///
-/// Note that type layout is, in general, [platform-specific, and subject to
-/// change](https://doc.rust-lang.org/reference/type-layout.html).
+/// Note that type layout is, in general, [subject to change and
+/// platform-specific](https://doc.rust-lang.org/reference/type-layout.html).
 ///
 /// # Examples
 ///
diff --git a/tests/ui/offset-of/offset-of-enum.rs b/tests/ui/offset-of/offset-of-enum.rs
index cf516175085b..e8b5a08377bd 100644
--- a/tests/ui/offset-of/offset-of-enum.rs
+++ b/tests/ui/offset-of/offset-of-enum.rs
@@ -14,4 +14,5 @@ fn main() {
     offset_of!(Alpha, Two.1); //~ ERROR no field named `1` on enum variant `Alpha::Two`
     offset_of!(Alpha, Two.foo); //~ ERROR no field named `foo` on enum variant `Alpha::Two`
     offset_of!(Alpha, NonExistent); //~ ERROR no variant named `NonExistent` found for enum `Alpha`
+    offset_of!(Beta, One); //~ ERROR cannot find type `Beta` in this scope
 }
diff --git a/tests/ui/offset-of/offset-of-enum.stderr b/tests/ui/offset-of/offset-of-enum.stderr
index 2867cc6befb5..7e7ad41f5b6a 100644
--- a/tests/ui/offset-of/offset-of-enum.stderr
+++ b/tests/ui/offset-of/offset-of-enum.stderr
@@ -7,6 +7,12 @@ LL |     offset_of!(Alpha::One, 0);
    |                not a type
    |                help: try using the variant's enum: `Alpha`
 
+error[E0412]: cannot find type `Beta` in this scope
+  --> $DIR/offset-of-enum.rs:17:16
+   |
+LL |     offset_of!(Beta, One);
+   |                ^^^^ not found in this scope
+
 error[E0795]: `One` is an enum variant; expected field at end of `offset_of`
   --> $DIR/offset-of-enum.rs:12:23
    |
@@ -35,7 +41,7 @@ error[E0599]: no variant named `NonExistent` found for enum `Alpha`
 LL |     offset_of!(Alpha, NonExistent);
    |                       ^^^^^^^^^^^ variant not found
 
-error: aborting due to 5 previous errors
+error: aborting due to 6 previous errors
 
-Some errors have detailed explanations: E0573, E0599, E0609, E0795.
-For more information about an error, try `rustc --explain E0573`.
+Some errors have detailed explanations: E0412, E0573, E0599, E0609, E0795.
+For more information about an error, try `rustc --explain E0412`.

From 587af910459fe408f03e004d264fdf218203849d Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Tue, 31 Oct 2023 09:32:10 +1100
Subject: [PATCH 295/297] Inline and remove `create_session`.

Currently the parts of session initialization that happen within
`rustc_interface` are split between `run_compiler` and `create_session`.
This split isn't necessary and obscures what's happening.

This commit merges the two functions. I think a single longer function
is much clearer than splitting this code across two functions in
different modules, especially when `create_session` has 13 parameters,
and is misnamed (it also creates the codegen backend). The net result is
43 fewer lines of code.
---
 compiler/rustc_interface/src/interface.rs | 64 +++++++++++++----
 compiler/rustc_interface/src/util.rs      | 87 +----------------------
 2 files changed, 54 insertions(+), 97 deletions(-)

diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs
index 26034dbaf997..ffdc05526f23 100644
--- a/compiler/rustc_interface/src/interface.rs
+++ b/compiler/rustc_interface/src/interface.rs
@@ -18,10 +18,9 @@ use rustc_query_system::query::print_query_stack;
 use rustc_session::config::{
     self, Cfg, CheckCfg, ExpectedValues, Input, OutFileName, OutputFilenames,
 };
+use rustc_session::filesearch::sysroot_candidates;
 use rustc_session::parse::ParseSess;
-use rustc_session::CompilerIO;
-use rustc_session::Session;
-use rustc_session::{lint, EarlyErrorHandler};
+use rustc_session::{lint, CompilerIO, EarlyErrorHandler, Session};
 use rustc_span::source_map::{FileLoader, FileName};
 use rustc_span::symbol::sym;
 use std::path::PathBuf;
@@ -398,32 +397,71 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
         || {
             crate::callbacks::setup_callbacks();
 
-            let registry = &config.registry;
-
             let handler = EarlyErrorHandler::new(config.opts.error_format);
 
+            let codegen_backend = if let Some(make_codegen_backend) = config.make_codegen_backend {
+                make_codegen_backend(&config.opts)
+            } else {
+                util::get_codegen_backend(
+                    &handler,
+                    &config.opts.maybe_sysroot,
+                    config.opts.unstable_opts.codegen_backend.as_deref(),
+                )
+            };
+
             let temps_dir = config.opts.unstable_opts.temps_dir.as_deref().map(PathBuf::from);
-            let (mut sess, codegen_backend) = util::create_session(
+
+            let bundle = match rustc_errors::fluent_bundle(
+                config.opts.maybe_sysroot.clone(),
+                sysroot_candidates().to_vec(),
+                config.opts.unstable_opts.translate_lang.clone(),
+                config.opts.unstable_opts.translate_additional_ftl.as_deref(),
+                config.opts.unstable_opts.translate_directionality_markers,
+            ) {
+                Ok(bundle) => bundle,
+                Err(e) => {
+                    handler.early_error(format!("failed to load fluent bundle: {e}"));
+                }
+            };
+
+            let mut locale_resources = Vec::from(config.locale_resources);
+            locale_resources.push(codegen_backend.locale_resource());
+
+            // target_override is documented to be called before init(), so this is okay
+            let target_override = codegen_backend.target_override(&config.opts);
+
+            let mut sess = rustc_session::build_session(
                 &handler,
                 config.opts,
-                parse_cfg(&handler, config.crate_cfg),
-                parse_check_cfg(&handler, config.crate_check_cfg),
-                config.locale_resources,
-                config.file_loader,
                 CompilerIO {
                     input: config.input,
                     output_dir: config.output_dir,
                     output_file: config.output_file,
                     temps_dir,
                 },
+                bundle,
+                config.registry.clone(),
+                locale_resources,
                 config.lint_caps,
-                config.make_codegen_backend,
-                registry.clone(),
+                config.file_loader,
+                target_override,
+                util::rustc_version_str().unwrap_or("unknown"),
                 config.ice_file,
                 config.using_internal_features,
                 config.expanded_args,
             );
 
+            codegen_backend.init(&sess);
+
+            let cfg = parse_cfg(&handler, config.crate_cfg);
+            let mut cfg = config::build_configuration(&sess, cfg);
+            util::add_configuration(&mut cfg, &mut sess, &*codegen_backend);
+            sess.parse_sess.config = cfg;
+
+            let mut check_cfg = parse_check_cfg(&handler, config.crate_check_cfg);
+            check_cfg.fill_well_known(&sess.target);
+            sess.parse_sess.check_config = check_cfg;
+
             if let Some(parse_sess_created) = config.parse_sess_created {
                 parse_sess_created(&mut sess.parse_sess);
             }
@@ -444,7 +482,7 @@ pub fn run_compiler<R: Send>(config: Config, f: impl FnOnce(&Compiler) -> R + Se
             rustc_span::set_source_map(compiler.sess.parse_sess.clone_source_map(), move || {
                 let r = {
                     let _sess_abort_error = defer(|| {
-                        compiler.sess.finish_diagnostics(registry);
+                        compiler.sess.finish_diagnostics(&config.registry);
                     });
 
                     f(&compiler)
diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs
index 5fde98c7c062..22d12793464a 100644
--- a/compiler/rustc_interface/src/util.rs
+++ b/compiler/rustc_interface/src/util.rs
@@ -3,29 +3,24 @@ use info;
 use libloading::Library;
 use rustc_ast as ast;
 use rustc_codegen_ssa::traits::CodegenBackend;
-use rustc_data_structures::fx::FxHashMap;
 #[cfg(parallel_compiler)]
 use rustc_data_structures::sync;
-use rustc_errors::registry::Registry;
 use rustc_parse::validate_attr;
 use rustc_session as session;
-use rustc_session::config::{
-    self, Cfg, CheckCfg, CrateType, OutFileName, OutputFilenames, OutputTypes,
-};
+use rustc_session::config::{self, Cfg, CrateType, OutFileName, OutputFilenames, OutputTypes};
 use rustc_session::filesearch::sysroot_candidates;
 use rustc_session::lint::{self, BuiltinLintDiagnostics, LintBuffer};
 use rustc_session::{filesearch, output, Session};
 use rustc_span::edit_distance::find_best_match_for_name;
 use rustc_span::edition::Edition;
-use rustc_span::source_map::FileLoader;
 use rustc_span::symbol::{sym, Symbol};
-use session::{CompilerIO, EarlyErrorHandler};
+use session::EarlyErrorHandler;
 use std::env;
 use std::env::consts::{DLL_PREFIX, DLL_SUFFIX};
 use std::mem;
 use std::path::{Path, PathBuf};
 use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, OnceLock};
+use std::sync::OnceLock;
 use std::thread;
 
 /// Function pointer type that constructs a new CodegenBackend.
@@ -52,82 +47,6 @@ pub fn add_configuration(cfg: &mut Cfg, sess: &mut Session, codegen_backend: &dy
     }
 }
 
-pub fn create_session(
-    handler: &EarlyErrorHandler,
-    sopts: config::Options,
-    cfg: Cfg,
-    mut check_cfg: CheckCfg,
-    locale_resources: &'static [&'static str],
-    file_loader: Option<Box<dyn FileLoader + Send + Sync + 'static>>,
-    io: CompilerIO,
-    lint_caps: FxHashMap<lint::LintId, lint::Level>,
-    make_codegen_backend: Option<
-        Box<dyn FnOnce(&config::Options) -> Box<dyn CodegenBackend> + Send>,
-    >,
-    descriptions: Registry,
-    ice_file: Option<PathBuf>,
-    using_internal_features: Arc<AtomicBool>,
-    expanded_args: Vec<String>,
-) -> (Session, Box<dyn CodegenBackend>) {
-    let codegen_backend = if let Some(make_codegen_backend) = make_codegen_backend {
-        make_codegen_backend(&sopts)
-    } else {
-        get_codegen_backend(
-            handler,
-            &sopts.maybe_sysroot,
-            sopts.unstable_opts.codegen_backend.as_deref(),
-        )
-    };
-
-    // target_override is documented to be called before init(), so this is okay
-    let target_override = codegen_backend.target_override(&sopts);
-
-    let bundle = match rustc_errors::fluent_bundle(
-        sopts.maybe_sysroot.clone(),
-        sysroot_candidates().to_vec(),
-        sopts.unstable_opts.translate_lang.clone(),
-        sopts.unstable_opts.translate_additional_ftl.as_deref(),
-        sopts.unstable_opts.translate_directionality_markers,
-    ) {
-        Ok(bundle) => bundle,
-        Err(e) => {
-            handler.early_error(format!("failed to load fluent bundle: {e}"));
-        }
-    };
-
-    let mut locale_resources = Vec::from(locale_resources);
-    locale_resources.push(codegen_backend.locale_resource());
-
-    let mut sess = session::build_session(
-        handler,
-        sopts,
-        io,
-        bundle,
-        descriptions,
-        locale_resources,
-        lint_caps,
-        file_loader,
-        target_override,
-        rustc_version_str().unwrap_or("unknown"),
-        ice_file,
-        using_internal_features,
-        expanded_args,
-    );
-
-    codegen_backend.init(&sess);
-
-    let mut cfg = config::build_configuration(&sess, cfg);
-    add_configuration(&mut cfg, &mut sess, &*codegen_backend);
-
-    check_cfg.fill_well_known(&sess.target);
-
-    // These configs use symbols, rather than strings.
-    sess.parse_sess.config = cfg;
-    sess.parse_sess.check_config = check_cfg;
-
-    (sess, codegen_backend)
-}
-
 const STACK_SIZE: usize = 8 * 1024 * 1024;
 
 fn get_stack_size() -> Option<usize> {

From 5f75326d74d131aae7ce656efa7ee40019be151d Mon Sep 17 00:00:00 2001
From: Ryan Mehri <ryan.mehri1@gmail.com>
Date: Wed, 1 Nov 2023 07:42:32 -0700
Subject: [PATCH 296/297] fix spans for inline_couroutine panic-abort

---
 ...ine_coroutine.main.Inline.panic-abort.diff | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/tests/mir-opt/inline/inline_coroutine.main.Inline.panic-abort.diff b/tests/mir-opt/inline/inline_coroutine.main.Inline.panic-abort.diff
index 357d95f7c22d..40eeda539087 100644
--- a/tests/mir-opt/inline/inline_coroutine.main.Inline.panic-abort.diff
+++ b/tests/mir-opt/inline/inline_coroutine.main.Inline.panic-abort.diff
@@ -4,26 +4,26 @@
   fn main() -> () {
       let mut _0: ();
       let _1: std::ops::CoroutineState<i32, bool>;
-      let mut _2: std::pin::Pin<&mut {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8}>;
-      let mut _3: &mut {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8};
-      let mut _4: {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8};
+      let mut _2: std::pin::Pin<&mut {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8}>;
+      let mut _3: &mut {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8};
+      let mut _4: {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8};
 +     let mut _5: bool;
       scope 1 {
           debug _r => _1;
       }
 +     scope 2 (inlined g) {
 +     }
-+     scope 3 (inlined Pin::<&mut {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8}>::new) {
++     scope 3 (inlined Pin::<&mut {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8}>::new) {
 +         debug pointer => _3;
 +         scope 4 {
-+             scope 5 (inlined Pin::<&mut {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8}>::new_unchecked) {
++             scope 5 (inlined Pin::<&mut {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8}>::new_unchecked) {
 +                 debug pointer => _3;
 +             }
 +         }
 +     }
 +     scope 6 (inlined g::{closure#0}) {
 +         debug a => _5;
-+         let mut _6: &mut {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8};
++         let mut _6: &mut {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8};
 +         let mut _7: u32;
 +         let mut _8: i32;
 +     }
@@ -34,22 +34,22 @@
           StorageLive(_3);
           StorageLive(_4);
 -         _4 = g() -> [return: bb1, unwind unreachable];
-+         _4 = {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8 (#0)};
++         _4 = {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8 (#0)};
 +         _3 = &mut _4;
-+         _2 = Pin::<&mut {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8}> { pointer: move _3 };
++         _2 = Pin::<&mut {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8}> { pointer: move _3 };
 +         StorageDead(_3);
 +         StorageLive(_5);
 +         _5 = const false;
 +         StorageLive(_6);
 +         StorageLive(_7);
-+         _6 = (_2.0: &mut {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8});
++         _6 = (_2.0: &mut {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8});
 +         _7 = discriminant((*_6));
 +         switchInt(move _7) -> [0: bb3, 1: bb7, 3: bb8, otherwise: bb9];
       }
   
       bb1: {
 -         _3 = &mut _4;
--         _2 = Pin::<&mut {coroutine@$DIR/inline_coroutine.rs:17:5: 17:8}>::new(move _3) -> [return: bb2, unwind unreachable];
+-         _2 = Pin::<&mut {coroutine@$DIR/inline_coroutine.rs:19:5: 19:8}>::new(move _3) -> [return: bb2, unwind unreachable];
 +         StorageDead(_7);
 +         StorageDead(_6);
 +         StorageDead(_5);
@@ -59,7 +59,7 @@
   
       bb2: {
 -         StorageDead(_3);
--         _1 = <{coroutine@$DIR/inline_coroutine.rs:17:5: 17:8} as Coroutine<bool>>::resume(move _2, const false) -> [return: bb3, unwind unreachable];
+-         _1 = <{coroutine@$DIR/inline_coroutine.rs:19:5: 19:8} as Coroutine<bool>>::resume(move _2, const false) -> [return: bb3, unwind unreachable];
 +         StorageDead(_4);
 +         _0 = const ();
 +         StorageDead(_1);

From 9e7345be1f5201634e421aa0963c56a976f36da9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Esteban=20K=C3=BCber?= <esteban@kuber.com.ar>
Date: Thu, 2 Nov 2023 02:20:04 +0000
Subject: [PATCH 297/297] Fix incorrect trait bound restriction suggestion

Suggest

```
error[E0308]: mismatched types
  --> $DIR/restrict-assoc-type-of-generic-bound.rs:9:12
   |
LL | pub fn foo<A: MyTrait, B>(a: A) -> B {
   |                        -           - expected `B` because of return type
   |                        |
   |                        expected this type parameter
LL |     return a.bar();
   |            ^^^^^^^ expected type parameter `B`, found associated type
   |
   = note: expected type parameter `B`
             found associated type `<A as MyTrait>::T`
help: consider further restricting this bound
   |
LL | pub fn foo<A: MyTrait<T = B>, B>(a: A) -> B {
   |                      +++++++
```

instead of

```
error[E0308]: mismatched types
  --> $DIR/restrict-assoc-type-of-generic-bound.rs:9:12
   |
LL | pub fn foo<A: MyTrait, B>(a: A) -> B {
   |                        -           - expected `B` because of return type
   |                        |
   |                        expected this type parameter
LL |     return a.bar();
   |            ^^^^^^^ expected type parameter `B`, found associated type
   |
   = note: expected type parameter `B`
             found associated type `<A as MyTrait>::T`
help: consider further restricting this bound
   |
LL | pub fn foo<A: MyTrait + <T = B>, B>(a: A) -> B {
   |                      +++++++++
```

Fix #117501.
---
 compiler/rustc_middle/src/ty/diagnostics.rs   |  2 ++
 ...restrict-assoc-type-of-generic-bound.fixed | 11 ++++++++++
 .../restrict-assoc-type-of-generic-bound.rs   | 11 ++++++++++
 ...estrict-assoc-type-of-generic-bound.stderr | 20 +++++++++++++++++++
 4 files changed, 44 insertions(+)
 create mode 100644 tests/ui/trait-bounds/restrict-assoc-type-of-generic-bound.fixed
 create mode 100644 tests/ui/trait-bounds/restrict-assoc-type-of-generic-bound.rs
 create mode 100644 tests/ui/trait-bounds/restrict-assoc-type-of-generic-bound.stderr

diff --git a/compiler/rustc_middle/src/ty/diagnostics.rs b/compiler/rustc_middle/src/ty/diagnostics.rs
index 7a782b2c2498..77a50fa9276d 100644
--- a/compiler/rustc_middle/src/ty/diagnostics.rs
+++ b/compiler/rustc_middle/src/ty/diagnostics.rs
@@ -274,6 +274,8 @@ pub fn suggest_constraining_type_params<'a>(
                 span,
                 if span_to_replace.is_some() {
                     constraint.clone()
+                } else if constraint.starts_with("<") {
+                    constraint.to_string()
                 } else if bound_list_non_empty {
                     format!(" + {constraint}")
                 } else {
diff --git a/tests/ui/trait-bounds/restrict-assoc-type-of-generic-bound.fixed b/tests/ui/trait-bounds/restrict-assoc-type-of-generic-bound.fixed
new file mode 100644
index 000000000000..b3f5ad52db59
--- /dev/null
+++ b/tests/ui/trait-bounds/restrict-assoc-type-of-generic-bound.fixed
@@ -0,0 +1,11 @@
+// run-rustfix
+pub trait MyTrait {
+    type T;
+
+    fn bar(self) -> Self::T;
+}
+
+pub fn foo<A: MyTrait<T = B>, B>(a: A) -> B {
+    return a.bar(); //~ ERROR mismatched types
+}
+fn main() {}
diff --git a/tests/ui/trait-bounds/restrict-assoc-type-of-generic-bound.rs b/tests/ui/trait-bounds/restrict-assoc-type-of-generic-bound.rs
new file mode 100644
index 000000000000..213abda77829
--- /dev/null
+++ b/tests/ui/trait-bounds/restrict-assoc-type-of-generic-bound.rs
@@ -0,0 +1,11 @@
+// run-rustfix
+pub trait MyTrait {
+    type T;
+
+    fn bar(self) -> Self::T;
+}
+
+pub fn foo<A: MyTrait, B>(a: A) -> B {
+    return a.bar(); //~ ERROR mismatched types
+}
+fn main() {}
diff --git a/tests/ui/trait-bounds/restrict-assoc-type-of-generic-bound.stderr b/tests/ui/trait-bounds/restrict-assoc-type-of-generic-bound.stderr
new file mode 100644
index 000000000000..61132efc414d
--- /dev/null
+++ b/tests/ui/trait-bounds/restrict-assoc-type-of-generic-bound.stderr
@@ -0,0 +1,20 @@
+error[E0308]: mismatched types
+  --> $DIR/restrict-assoc-type-of-generic-bound.rs:9:12
+   |
+LL | pub fn foo<A: MyTrait, B>(a: A) -> B {
+   |                        -           - expected `B` because of return type
+   |                        |
+   |                        expected this type parameter
+LL |     return a.bar();
+   |            ^^^^^^^ expected type parameter `B`, found associated type
+   |
+   = note: expected type parameter `B`
+             found associated type `<A as MyTrait>::T`
+help: consider further restricting this bound
+   |
+LL | pub fn foo<A: MyTrait<T = B>, B>(a: A) -> B {
+   |                      +++++++
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0308`.